diff options
464 files changed, 13718 insertions, 5103 deletions
diff --git a/Makefile.inc1 b/Makefile.inc1 index d5458e587813c..34e55d7448f57 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -628,7 +628,7 @@ XCFLAGS+= ${BFLAGS} .endif .if ${MK_LIB32} != "no" && (${TARGET_ARCH} == "amd64" || \ - ${TARGET_ARCH} == "powerpc64") || ${TARGET_ARCH:Mmips64*} != "" + ${TARGET_ARCH} == "powerpc64" || ${TARGET_ARCH:Mmips64*} != "") LIBCOMPAT= 32 .include "Makefile.libcompat" .elif ${MK_LIBSOFT} != "no" && ${TARGET_ARCH} == "armv6" diff --git a/bin/sh/TOUR b/bin/sh/TOUR index e9bbe9b121b56..14dc3458b65c5 100644 --- a/bin/sh/TOUR +++ b/bin/sh/TOUR @@ -24,7 +24,7 @@ programs is: program input files generates ------- ----------- --------- - mkbuiltins builtins builtins.h builtins.c + mkbuiltins builtins.def builtins.h builtins.c mknodes nodetypes nodes.h nodes.c mksyntax - syntax.h syntax.c mktokens - token.h @@ -108,10 +108,12 @@ The text field of a NARG structure points to the text of the word. The text consists of ordinary characters and a number of special codes defined in parser.h. The special codes are: - CTLVAR Variable substitution - CTLENDVAR End of variable substitution + CTLVAR Parameter expansion + CTLENDVAR End of parameter expansion CTLBACKQ Command substitution CTLBACKQ|CTLQUOTE Command substitution inside double quotes + CTLARI Arithmetic expansion + CTLENDARI End of arithmetic expansion CTLESC Escape next character A variable substitution contains the following elements: @@ -130,18 +132,31 @@ stitution. The possible types are: VSQUESTION|VSNUL ${var:?text} VSASSIGN ${var=text} VSASSIGN|VSNUL ${var:=text} + VSTRIMLEFT ${var#text} + VSTRIMLEFTMAX ${var##text} + VSTRIMRIGHT ${var%text} + VSTRIMRIGHTMAX ${var%%text} + VSLENGTH ${#var} + VSERROR delayed error In addition, the type field will have the VSQUOTE flag set if the -variable is enclosed in double quotes. The name of the variable -comes next, terminated by an equals sign. If the type is not -VSNORMAL, then the text field in the substitution follows, ter- -minated by a CTLENDVAR byte. +variable is enclosed in double quotes and the VSLINENO flag if +LINENO is being expanded (the parameter name is the decimal line +number). The parameter's name comes next, terminated by an equals +sign. If the type is not VSNORMAL (including when it is VSLENGTH), +then the text field in the substitution follows, terminated by a +CTLENDVAR byte. + +The type VSERROR is used to allow parsing bad substitutions like +${var[7]} and generate an error when they are expanded. Commands in back quotes are parsed and stored in a linked list. The locations of these commands in the string are indicated by CTLBACKQ and CTLBACKQ+CTLQUOTE characters, depending upon whether the back quotes were enclosed in double quotes. +Arithmetic expansion starts with CTLARI and ends with CTLENDARI. + The character CTLESC escapes the next character, so that in case any of the CTL characters mentioned above appear in the input, they can be passed through transparently. CTLESC is also used to @@ -153,11 +168,11 @@ right. In the case of here documents which are not subject to variable and command substitution, the parser doesn't insert any CTLESC characters to begin with (so the contents of the text field can be written without any processing). Other here docu- -ments, and words which are not subject to splitting and file name -generation, have the CTLESC characters removed during the vari- -able and command substitution phase. Words which are subject to -splitting and file name generation have the CTLESC characters re- -moved as part of the file name phase. +ments, and words which are not subject to file name generation, +have the CTLESC characters removed during the variable and command +substitution phase. Words which are subject to file name +generation have the CTLESC characters removed as part of the file +name phase. EXECUTION: Command execution is handled by the following files: eval.c The top level routines. @@ -199,10 +214,10 @@ later.) The routine shellexec is the interface to the exec system call. -EXPAND.C: Arguments are processed in three passes. The first -(performed by the routine argstr) performs variable and command -substitution. The second (ifsbreakup) performs word splitting -and the third (expandmeta) performs file name generation. +EXPAND.C: As the routine argstr generates words by parameter +expansion, command substitution and arithmetic expansion, it +performs word splitting on the result. As each word is output, +the routine expandmeta performs file name generation (if enabled). VAR.C: Variables are stored in a hash table. Probably we should switch to extensible hashing. The variable name is stored in the @@ -221,8 +236,8 @@ BUILTIN COMMANDS: The procedures for handling these are scat- tered throughout the code, depending on which location appears most appropriate. They can be recognized because their names al- ways end in "cmd". The mapping from names to procedures is -specified in the file builtins, which is processed by the mkbuilt- -ins command. +specified in the file builtins.def, which is processed by the +mkbuiltins command. A builtin command is invoked with argc and argv set up like a normal program. A builtin command is allowed to overwrite its @@ -230,22 +245,20 @@ arguments. Builtin routines can call nextopt to do option pars- ing. This is kind of like getopt, but you don't pass argc and argv to it. Builtin routines can also call error. This routine normally terminates the shell (or returns to the main command -loop if the shell is interactive), but when called from a builtin -command it causes the builtin command to terminate with an exit -status of 2. +loop if the shell is interactive), but when called from a non- +special builtin command it causes the builtin command to +terminate with an exit status of 2. The directory bltins contains commands which can be compiled in- dependently but can also be built into the shell for efficiency -reasons. The makefile in this directory compiles these programs -in the normal fashion (so that they can be run regardless of -whether the invoker is ash), but also creates a library named -bltinlib.a which can be linked with ash. The header file bltin.h -takes care of most of the differences between the ash and the -stand-alone environment. The user should call the main routine -"main", and #define main to be the name of the routine to use -when the program is linked into ash. This #define should appear -before bltin.h is included; bltin.h will #undef main if the pro- -gram is to be compiled stand-alone. +reasons. The header file bltin.h takes care of most of the +differences between the ash and the stand-alone environment. +The user should call the main routine "main", and #define main to +be the name of the routine to use when the program is linked into +ash. This #define should appear before bltin.h is included; +bltin.h will #undef main if the program is to be compiled +stand-alone. A similar approach is used for a few utilities from +bin and usr.bin. CD.C: This file defines the cd and pwd builtins. @@ -258,7 +271,7 @@ is called at appropriate points to actually handle the signal. When an interrupt is caught and no trap has been set for that signal, the routine "onint" in error.c is called. -OUTPUT: Ash uses it's own output routines. There are three out- +OUTPUT: Ash uses its own output routines. There are three out- put structures allocated. "Output" represents the standard out- put, "errout" the standard error, and "memout" contains output which is to be stored in memory. This last is used when a buil- diff --git a/bin/sh/eval.c b/bin/sh/eval.c index a327ba6f048e7..e09549295f66d 100644 --- a/bin/sh/eval.c +++ b/bin/sh/eval.c @@ -1222,7 +1222,7 @@ bltincmd(int argc, char **argv) return 127; } /* - * Preserve exitstatus of a previous possible redirection + * Preserve exitstatus of a previous possible command substitution * as POSIX mandates */ return exitstatus; diff --git a/bin/sh/exec.c b/bin/sh/exec.c index b8fb41c390cee..fb0868f5c8893 100644 --- a/bin/sh/exec.c +++ b/bin/sh/exec.c @@ -338,7 +338,7 @@ find_command(const char *name, struct cmdentry *entry, int act, cd = 0; - /* If name is in the table, and not invalidated by cd, we're done */ + /* If name is in the table, we're done */ if ((cmdp = cmdlookup(name, 0)) != NULL) { if (cmdp->cmdtype == CMDFUNCTION && act & DO_NOFUNC) cmdp = NULL; @@ -485,8 +485,7 @@ changepath(const char *newval __unused) /* - * Clear out command entries. The argument specifies the first entry in - * PATH which has changed. + * Clear out cached utility locations. */ void diff --git a/bin/sh/expand.c b/bin/sh/expand.c index 30f0ec50ec581..74bb63e1f67e6 100644 --- a/bin/sh/expand.c +++ b/bin/sh/expand.c @@ -222,9 +222,9 @@ stputs_split(const char *data, const char *syntax, int flag, char *p, * The result is left in the stack string. * When arglist is NULL, perform here document expansion. * - * Caution: this function uses global state and is not reentrant. - * However, a new invocation after an interrupted invocation is safe - * and will reset the global state for the new call. + * When doing something that may cause this to be re-entered, make sure + * the stack string is empty via grabstackstr() and do not assume expdest + * remains valid. */ void expandarg(union node *arg, struct arglist *arglist, int flag) @@ -476,7 +476,7 @@ expbackq(union node *cmd, int quoted, int flag, struct worddest *dst) ifs = ifsset() ? ifsval() : " \t\n"; else ifs = ""; - /* Don't copy trailing newlines */ + /* Remove trailing newlines */ for (;;) { if (--in.nleft < 0) { if (in.fd < 0) @@ -821,7 +821,7 @@ evalvar(const char *p, struct nodelist **restrict argbackq, int flag, /* - * Test whether a specialized variable is set. + * Test whether a special or positional parameter is set. */ static int @@ -918,7 +918,7 @@ reprocess(int startloc, int flag, int subtype, int quoted, } /* - * Add the value of a specialized variable to the stack string. + * Add the value of a special or positional parameter to the stack string. */ static void diff --git a/bin/sh/options.c b/bin/sh/options.c index 8b501f2d1b6a0..2ac1a4bb46b0f 100644 --- a/bin/sh/options.c +++ b/bin/sh/options.c @@ -141,6 +141,8 @@ optschanged(void) /* * Process shell options. The global variable argptr contains a pointer * to the argument list; we advance it past the options. + * If cmdline is true, process the shell's argv; otherwise, process arguments + * to the set special builtin. */ static void @@ -392,7 +394,7 @@ shiftcmd(int argc, char **argv) /* - * The set command builtin. + * The set builtin command. */ int @@ -558,7 +560,7 @@ out: /* * Standard option processing (a la getopt) for builtin routines. The * only argument that is passed to nextopt is the option string; the - * other arguments are unnecessary. It return the character, or '\0' on + * other arguments are unnecessary. It returns the option, or '\0' on * end of input. */ diff --git a/bin/sh/tests/builtins/Makefile b/bin/sh/tests/builtins/Makefile index b3e325507dd5d..864c8767d0344 100644 --- a/bin/sh/tests/builtins/Makefile +++ b/bin/sh/tests/builtins/Makefile @@ -120,6 +120,7 @@ ${PACKAGE}FILES+= local7.0 .if ${MK_NLS} != "no" ${PACKAGE}FILES+= locale1.0 .endif +${PACKAGE}FILES+= locale2.0 ${PACKAGE}FILES+= printf1.0 ${PACKAGE}FILES+= printf2.0 ${PACKAGE}FILES+= printf3.0 diff --git a/bin/sh/tests/builtins/locale2.0 b/bin/sh/tests/builtins/locale2.0 new file mode 100644 index 0000000000000..86dd237ff54ac --- /dev/null +++ b/bin/sh/tests/builtins/locale2.0 @@ -0,0 +1,5 @@ +# $FreeBSD$ + +$SH -c 'LC_ALL=C true; kill -INT $$; echo continued' +r=$? +[ "$r" -gt 128 ] && [ "$(kill -l "$r")" = INT ] diff --git a/bin/sh/var.c b/bin/sh/var.c index 491ebe952e6a9..f533969a9b0a3 100644 --- a/bin/sh/var.c +++ b/bin/sh/var.c @@ -513,7 +513,7 @@ bltinunsetlocale(void) if (localevar(cmdenviron->args[i])) { setlocale(LC_ALL, ""); updatecharset(); - return; + break; } } INTON; diff --git a/bin/stty/stty.1 b/bin/stty/stty.1 index f546c9602fe61..5cdc483b644d6 100644 --- a/bin/stty/stty.1 +++ b/bin/stty/stty.1 @@ -588,6 +588,7 @@ Same as the control character .Sh EXIT STATUS .Ex -std .Sh SEE ALSO +.Xr resizewin 1 , .Xr termios 4 .Sh STANDARDS The diff --git a/contrib/ipfilter/tools/ippool.c b/contrib/ipfilter/tools/ippool.c index 243932c8cc344..883f82a675b64 100644 --- a/contrib/ipfilter/tools/ippool.c +++ b/contrib/ipfilter/tools/ippool.c @@ -1047,7 +1047,9 @@ setnodeaddr(int type, int role, void *ptr, char *arg) if (type == IPLT_POOL) { ip_pool_node_t *node = ptr; +#ifdef USE_INET6 if (node->ipn_addr.adf_family == AF_INET) +#endif node->ipn_addr.adf_len = offsetof(addrfamily_t, adf_addr) + sizeof(struct in_addr); diff --git a/contrib/less/LICENSE b/contrib/less/LICENSE index 376b8c8bed74c..832ca1b370759 100644 --- a/contrib/less/LICENSE +++ b/contrib/less/LICENSE @@ -2,7 +2,7 @@ ------------ Less -Copyright (C) 1984-2015 Mark Nudelman +Copyright (C) 1984-2016 Mark Nudelman Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/contrib/less/NEWS b/contrib/less/NEWS index 43f1247377a29..e8bb6efc00191 100644 --- a/contrib/less/NEWS +++ b/contrib/less/NEWS @@ -11,6 +11,40 @@ ====================================================================== + Major changes between "less" versions 487 and 491 + +* Don't output terminal init sequence if using -F and file fits on one screen. + +* Use ANSI prototypes in funcs.h declarations. + +* Fix some const mismatches. + +* Remove "register" in variable declarations. + +* Fix some memory leaks. + +====================================================================== + + Major changes between "less" versions 481 and 487 + +* New commands ESC-{ and ESC-} to shift to start/end of displayed lines. + +* Make search highlights work correctly when changing caselessness with -i. + +* New option -Da in Windows version to enable SGR mode. + +* Fix "nothing to search" error when top or bottom line on screen is empty. + +* Fix bug when terminal has no "cm" termcap entry. + +* Fix incorrect display when entering double-width chars in search string. + +* Fix bug in Unicode handling that missed some double width characters. + +* Update Unicode database to 9.0.0. + +====================================================================== + Major changes between "less" versions 458 and 481 * Don't overwrite history file; just append to it. diff --git a/contrib/less/README b/contrib/less/README index 10884310fa267..a0f8a3ae94ccc 100644 --- a/contrib/less/README +++ b/contrib/less/README @@ -7,9 +7,9 @@ ************************************************************************** ************************************************************************** - Less, version 481 + Less, version 491 - This is the distribution of less, version 481, released 31 Aug 2015. + This is the distribution of less, version 491, released 07 Apr 2017. This program is part of the GNU project (http://www.gnu.org). This program is free software. You may redistribute it and/or diff --git a/contrib/less/brac.c b/contrib/less/brac.c index 298a313126eb5..44f240361cbd5 100644 --- a/contrib/less/brac.c +++ b/contrib/less/brac.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -24,14 +24,18 @@ * "close bracket" are given. */ public void -match_brac(int obrac, int cbrac, int forwdir, int n) +match_brac(obrac, cbrac, forwdir, n) + int obrac; + int cbrac; + int forwdir; + int n; { int c; int nest; POSITION pos; - int (*chget)(void); + int (*chget)(); - extern int ch_forw_get(void), ch_back_get(void); + extern int ch_forw_get(), ch_back_get(); /* * Seek to the line containing the open bracket. diff --git a/contrib/less/ch.c b/contrib/less/ch.c index bffd75ee8c842..0fc65aca28480 100644 --- a/contrib/less/ch.c +++ b/contrib/less/ch.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -144,7 +144,7 @@ static int ch_addbuf(); * Get the character pointed to by the read pointer. */ int -ch_get(void) +ch_get() { struct buf *bp; struct bufnode *bn; @@ -378,7 +378,8 @@ ch_get(void) * a single char onto an input file descriptor. */ public void -ch_ungetchar(int c) +ch_ungetchar(c) + int c; { if (c != -1 && ch_ungotchar != -1) error("ch_ungetchar overrun", NULL_PARG); @@ -391,7 +392,7 @@ ch_ungetchar(int c) * If we haven't read all of standard input into it, do that now. */ public void -end_logfile(void) +end_logfile() { static int tried = FALSE; @@ -416,7 +417,7 @@ end_logfile(void) * Write all the existing buffered data to the log file. */ public void -sync_logfile(void) +sync_logfile() { struct buf *bp; struct bufnode *bn; @@ -453,7 +454,8 @@ sync_logfile(void) * Determine if a specific block is currently in one of the buffers. */ static int -buffered(BLOCKNUM block) +buffered(block) + BLOCKNUM block; { struct buf *bp; struct bufnode *bn; @@ -474,7 +476,8 @@ buffered(BLOCKNUM block) * Return 0 if successful, non-zero if can't seek there. */ public int -ch_seek(POSITION pos) +ch_seek(pos) + POSITION pos; { BLOCKNUM new_block; POSITION len; @@ -512,7 +515,7 @@ ch_seek(POSITION pos) * Seek to the end of the file. */ public int -ch_end_seek(void) +ch_end_seek() { POSITION len; @@ -539,7 +542,7 @@ ch_end_seek(void) * Seek to the last position in the file that is currently buffered. */ public int -ch_end_buffer_seek(void) +ch_end_buffer_seek() { struct buf *bp; struct bufnode *bn; @@ -567,7 +570,7 @@ ch_end_buffer_seek(void) * beginning of the pipe is no longer buffered. */ public int -ch_beg_seek(void) +ch_beg_seek() { struct bufnode *bn; struct bufnode *firstbn; @@ -599,7 +602,7 @@ ch_beg_seek(void) * Return the length of the file, if known. */ public POSITION -ch_length(void) +ch_length() { if (thisfile == NULL) return (NULL_POSITION); @@ -616,7 +619,7 @@ ch_length(void) * Return the current position in the file. */ public POSITION -ch_tell(void) +ch_tell() { if (thisfile == NULL) return (NULL_POSITION); @@ -627,7 +630,7 @@ ch_tell(void) * Get the current char and post-increment the read pointer. */ public int -ch_forw_get(void) +ch_forw_get() { int c; @@ -650,7 +653,7 @@ ch_forw_get(void) * Pre-decrement the read pointer and get the new current char. */ public int -ch_back_get(void) +ch_back_get() { if (thisfile == NULL) return (EOI); @@ -673,7 +676,8 @@ ch_back_get(void) * bufspace is in units of 1024 bytes. -1 mean no limit. */ public void -ch_setbufspace(int bufspace) +ch_setbufspace(bufspace) + int bufspace; { if (bufspace < 0) maxbufs = -1; @@ -689,7 +693,7 @@ ch_setbufspace(int bufspace) * Flush (discard) any saved file state, including buffer contents. */ public void -ch_flush(void) +ch_flush() { struct bufnode *bn; @@ -756,7 +760,7 @@ ch_flush(void) * The buffer is added to the tail of the buffer chain. */ static int -ch_addbuf(void) +ch_addbuf() { struct buf *bp; struct bufnode *bn; @@ -781,7 +785,7 @@ ch_addbuf(void) * */ static void -init_hashtbl(void) +init_hashtbl() { int h; @@ -796,7 +800,7 @@ init_hashtbl(void) * Delete all buffers for this file. */ static void -ch_delbufs(void) +ch_delbufs() { struct bufnode *bn; @@ -814,7 +818,8 @@ ch_delbufs(void) * Is it possible to seek on a file descriptor? */ public int -seekable(int f) +seekable(f) + int f; { #if MSDOS_COMPILER extern int fd0; @@ -835,7 +840,7 @@ seekable(int f) * This is used after an ignore_eof read, during which the EOF may change. */ public void -ch_set_eof(void) +ch_set_eof() { ch_fsize = ch_fpos; } @@ -845,7 +850,9 @@ ch_set_eof(void) * Initialize file state for a new file. */ public void -ch_init(int f, int flags) +ch_init(f, flags) + int f; + int flags; { /* * See if we already have a filestate for this file. @@ -884,7 +891,7 @@ ch_init(int f, int flags) * Close a filestate. */ public void -ch_close(void) +ch_close() { int keepstate = FALSE; @@ -927,7 +934,7 @@ ch_close(void) * Return ch_flags for the current file. */ public int -ch_getflags(void) +ch_getflags() { if (thisfile == NULL) return (0); diff --git a/contrib/less/charset.c b/contrib/less/charset.c index db7e848f0c870..7aec0ea140e83 100644 --- a/contrib/less/charset.c +++ b/contrib/less/charset.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -64,6 +64,8 @@ struct cs_alias { char *oname; } cs_aliases[] = { { "UTF-8", "utf-8" }, + { "utf8", "utf-8" }, + { "UTF8", "utf-8" }, { "ANSI_X3.4-1968", "ascii" }, { "US-ASCII", "ascii" }, { "latin1", "iso8859" }, @@ -130,7 +132,8 @@ public int binattr = AT_STANDOUT; * c control character */ static void -ichardef(char *s) +ichardef(s) + char *s; { char *cp; int n; @@ -186,7 +189,9 @@ ichardef(char *s) * The valid charset names are listed in the "charsets" array. */ static int -icharset(char *name, int no_error) +icharset(name, no_error) + char *name; + int no_error; { struct charset *p; struct cs_alias *a; @@ -227,7 +232,7 @@ icharset(char *name, int no_error) * Define a charset, given a locale name. */ static void -ilocale(void) +ilocale() { int c; @@ -247,7 +252,10 @@ ilocale(void) * Define the printing format for control (or binary utf) chars. */ static void -setbinfmt(char *s, char **fmtvarptr, char *default_fmt) +setbinfmt(s, fmtvarptr, default_fmt) + char *s; + char **fmtvarptr; + char *default_fmt; { if (s && utf_mode) { @@ -293,7 +301,7 @@ setbinfmt(char *s, char **fmtvarptr, char *default_fmt) * */ static void -set_charset(void) +set_charset() { char *s; @@ -364,7 +372,7 @@ set_charset(void) * Initialize charset data structures. */ public void -init_charset(void) +init_charset() { char *s; @@ -385,7 +393,8 @@ init_charset(void) * Is a given character a "binary" character? */ public int -binary_char(LWCHAR c) +binary_char(c) + LWCHAR c; { if (utf_mode) return (is_ubin_char(c)); @@ -397,7 +406,8 @@ binary_char(LWCHAR c) * Is a given character a "control" character? */ public int -control_char(LWCHAR c) +control_char(c) + LWCHAR c; { c &= 0377; return (chardef[c] & IS_CONTROL_CHAR); @@ -408,7 +418,8 @@ control_char(LWCHAR c) * For example, in the "ascii" charset '\3' is printed as "^C". */ public char * -prchar(LWCHAR c) +prchar(c) + LWCHAR c; { /* {{ This buffer can be overrun if LESSBINFMT is a long string. }} */ static char buf[32]; @@ -443,7 +454,8 @@ prchar(LWCHAR c) * Return the printable form of a UTF-8 character. */ public char * -prutfchar(LWCHAR ch) +prutfchar(ch) + LWCHAR ch; { static char buf[32]; @@ -473,7 +485,8 @@ prutfchar(LWCHAR ch) * Get the length of a UTF-8 character in bytes. */ public int -utf_len(char ch) +utf_len(ch) + unsigned char ch; { if ((ch & 0x80) == 0) return 1; @@ -495,15 +508,18 @@ utf_len(char ch) * Does the parameter point to the lead byte of a well-formed UTF-8 character? */ public int -is_utf8_well_formed(unsigned char *s, int slen) +is_utf8_well_formed(ss, slen) + char *ss; + int slen; { int i; int len; + unsigned char *s = (unsigned char *) ss; if (IS_UTF8_INVALID(s[0])) return (0); - len = utf_len((char) s[0]); + len = utf_len(s[0]); if (len > slen) return (0); if (len == 1) @@ -530,14 +546,16 @@ is_utf8_well_formed(unsigned char *s, int slen) * Return number of invalid UTF-8 sequences found in a buffer. */ public int -utf_bin_count(unsigned char *data, int len) +utf_bin_count(data, len) + char *data; + int len; { int bin_count = 0; while (len > 0) { if (is_utf8_well_formed(data, len)) { - int clen = utf_len(*data); + int clen = utf_len(*data & 0377); data += clen; len -= clen; } else @@ -547,7 +565,7 @@ utf_bin_count(unsigned char *data, int len) do { ++data; --len; - } while (len > 0 && !IS_UTF8_LEAD(*data)); + } while (len > 0 && !IS_UTF8_LEAD(*data & 0377)); } } return (bin_count); @@ -557,7 +575,8 @@ utf_bin_count(unsigned char *data, int len) * Get the value of a UTF-8 character. */ public LWCHAR -get_wchar(constant char *p) +get_wchar(p) + constant char *p; { switch (utf_len(p[0])) { @@ -608,7 +627,9 @@ get_wchar(constant char *p) * Store a character into a UTF-8 string. */ public void -put_wchar(char **pp, LWCHAR ch) +put_wchar(pp, ch) + char **pp; + LWCHAR ch; { if (!utf_mode || ch < 0x80) { @@ -656,11 +677,14 @@ put_wchar(char **pp, LWCHAR ch) * Step forward or backward one character in a string. */ public LWCHAR -step_char(constant char **pp, signed int dir, constant char *limit) +step_char(pp, dir, limit) + char **pp; + signed int dir; + constant char *limit; { LWCHAR ch; int len; - constant char *p = *pp; + char *p = *pp; if (!utf_mode) { @@ -675,7 +699,7 @@ step_char(constant char **pp, signed int dir, constant char *limit) if (p + len > limit) { ch = 0; - p = limit; + p = (char *) limit; } else { ch = get_wchar(p); @@ -723,7 +747,9 @@ static struct wchar_range comb_table[] = { static int -is_in_table(LWCHAR ch, struct wchar_range_table *table) +is_in_table(ch, table) + LWCHAR ch; + struct wchar_range_table *table; { int hi; int lo; @@ -751,7 +777,8 @@ is_in_table(LWCHAR ch, struct wchar_range_table *table) * If a composing character follows any char, the two combine into one glyph. */ public int -is_composing_char(LWCHAR ch) +is_composing_char(ch) + LWCHAR ch; { return is_in_table(ch, &compose_table); } @@ -760,7 +787,8 @@ is_composing_char(LWCHAR ch) * Should this UTF-8 character be treated as binary? */ public int -is_ubin_char(LWCHAR ch) +is_ubin_char(ch) + LWCHAR ch; { return is_in_table(ch, &ubin_table); } @@ -769,7 +797,8 @@ is_ubin_char(LWCHAR ch) * Is this a double width UTF-8 character? */ public int -is_wide_char(LWCHAR ch) +is_wide_char(ch) + LWCHAR ch; { return is_in_table(ch, &wide_table); } @@ -780,7 +809,9 @@ is_wide_char(LWCHAR ch) * a specific char (not any char), the two combine into one glyph. */ public int -is_combining_char(LWCHAR ch1, LWCHAR ch2) +is_combining_char(ch1, ch2) + LWCHAR ch1; + LWCHAR ch2; { /* The table is small; use linear search. */ int i; diff --git a/contrib/less/charset.h b/contrib/less/charset.h index bb1e4376cbd50..1e3f0691069a4 100644 --- a/contrib/less/charset.h +++ b/contrib/less/charset.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. diff --git a/contrib/less/cmd.h b/contrib/less/cmd.h index 8a943d1522dcb..0c221b83459a0 100644 --- a/contrib/less/cmd.h +++ b/contrib/less/cmd.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -67,6 +67,8 @@ #define A_FILTER 55 #define A_F_UNTIL_HILITE 56 #define A_GOEND_BUF 57 +#define A_LLSHIFT 58 +#define A_RRSHIFT 59 #define A_INVALID 100 #define A_NOACTION 101 diff --git a/contrib/less/cmdbuf.c b/contrib/less/cmdbuf.c index 9631e18daafea..3df0bac3d8d25 100644 --- a/contrib/less/cmdbuf.c +++ b/contrib/less/cmdbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -32,7 +32,7 @@ static int literal; /* Next input char should not be interpreted */ static int updown_match = -1; /* Prefix length in up/down movement */ #if TAB_COMPLETE_FILENAME -static int cmd_complete(int action); +static int cmd_complete(); /* * These variables are statics used by cmd_complete. */ @@ -76,25 +76,25 @@ struct mlist */ struct mlist mlist_search = { &mlist_search, &mlist_search, &mlist_search, NULL, 0 }; -public void * constant ml_search = (void *) &mlist_search; +public void *ml_search = (void *) &mlist_search; struct mlist mlist_examine = { &mlist_examine, &mlist_examine, &mlist_examine, NULL, 0 }; -public void * constant ml_examine = (void *) &mlist_examine; +public void *ml_examine = (void *) &mlist_examine; #if SHELL_ESCAPE || PIPEC struct mlist mlist_shell = { &mlist_shell, &mlist_shell, &mlist_shell, NULL, 0 }; -public void * constant ml_shell = (void *) &mlist_shell; +public void *ml_shell = (void *) &mlist_shell; #endif #else /* CMD_HISTORY */ /* If CMD_HISTORY is off, these are just flags. */ -public void * constant ml_search = (void *)1; -public void * constant ml_examine = (void *)2; +public void *ml_search = (void *)1; +public void *ml_examine = (void *)2; #if SHELL_ESCAPE || PIPEC -public void * constant ml_shell = (void *)3; +public void *ml_shell = (void *)3; #endif #endif /* CMD_HISTORY */ @@ -114,7 +114,7 @@ static int cmd_mbc_buf_index; * Reset command buffer (to empty). */ public void -cmd_reset(void) +cmd_reset() { cp = cmdbuf; *cp = '\0'; @@ -129,7 +129,7 @@ cmd_reset(void) * Clear command line. */ public void -clear_cmd(void) +clear_cmd() { cmd_col = prompt_col = 0; cmd_mbc_buf_len = 0; @@ -140,28 +140,27 @@ clear_cmd(void) * Display a string, usually as a prompt for input into the command buffer. */ public void -cmd_putstr(constant char *s) +cmd_putstr(s) + constant char *s; { LWCHAR prev_ch = 0; LWCHAR ch; constant char *endline = s + strlen(s); while (*s != '\0') { - constant char *ns = s; + char *ns = (char *) s; + int width; ch = step_char(&ns, +1, endline); while (s < ns) putchr(*s++); if (!utf_mode) - { - cmd_col++; - prompt_col++; - } else if (!is_composing_char(ch) && - !is_combining_char(prev_ch, ch)) - { - int width = is_wide_char(ch) ? 2 : 1; - cmd_col += width; - prompt_col += width; - } + width = 1; + else if (is_composing_char(ch) || is_combining_char(prev_ch, ch)) + width = 0; + else + width = is_wide_char(ch) ? 2 : 1; + cmd_col += width; + prompt_col += width; prev_ch = ch; } } @@ -170,10 +169,10 @@ cmd_putstr(constant char *s) * How many characters are in the command buffer? */ public int -len_cmdbuf(void) +len_cmdbuf() { - constant char *s = cmdbuf; - constant char *endline = s + strlen(s); + char *s = cmdbuf; + char *endline = s + strlen(s); int len = 0; while (*s != '\0') @@ -186,63 +185,44 @@ len_cmdbuf(void) /* * Common part of cmd_step_right() and cmd_step_left(). + * {{ Returning pwidth and bswidth separately is a historical artifact + * since they're always the same. Maybe clean this up someday. }} */ static char * -cmd_step_common(constant char *p, LWCHAR ch, int len, int *pwidth, int *bswidth) +cmd_step_common(p, ch, len, pwidth, bswidth) + char *p; + LWCHAR ch; + int len; + int *pwidth; + int *bswidth; { char *pr; + int width; if (len == 1) { pr = prchar((int) ch); - if (pwidth != NULL || bswidth != NULL) - { - int len = (int) strlen(pr); - if (pwidth != NULL) - *pwidth = len; - if (bswidth != NULL) - *bswidth = len; - } + width = (int) strlen(pr); } else { pr = prutfchar(ch); - if (pwidth != NULL || bswidth != NULL) + if (is_composing_char(ch)) + width = 0; + else if (is_ubin_char(ch)) + width = (int) strlen(pr); + else { - if (is_composing_char(ch)) - { - if (pwidth != NULL) - *pwidth = 0; - if (bswidth != NULL) - *bswidth = 0; - } else if (is_ubin_char(ch)) - { - int len = (int) strlen(pr); - if (pwidth != NULL) - *pwidth = len; - if (bswidth != NULL) - *bswidth = len; - } else - { - LWCHAR prev_ch = step_char(&p, -1, cmdbuf); - if (is_combining_char(prev_ch, ch)) - { - if (pwidth != NULL) - *pwidth = 0; - if (bswidth != NULL) - *bswidth = 0; - } else - { - if (pwidth != NULL) - *pwidth = is_wide_char(ch) - ? 2 - : 1; - if (bswidth != NULL) - *bswidth = 1; - } - } + LWCHAR prev_ch = step_char(&p, -1, cmdbuf); + if (is_combining_char(prev_ch, ch)) + width = 0; + else + width = is_wide_char(ch) ? 2 : 1; } } - + if (pwidth != NULL) + *pwidth = width; + if (bswidth != NULL) + *bswidth = width; return (pr); } @@ -250,10 +230,13 @@ cmd_step_common(constant char *p, LWCHAR ch, int len, int *pwidth, int *bswidth) * Step a pointer one character right in the command buffer. */ static char * -cmd_step_right(char **pp, int *pwidth, int *bswidth) +cmd_step_right(pp, pwidth, bswidth) + char **pp; + int *pwidth; + int *bswidth; { char *p = *pp; - LWCHAR ch = step_char((constant char **)pp, +1, p + strlen(p)); + LWCHAR ch = step_char(pp, +1, p + strlen(p)); return cmd_step_common(p, ch, *pp - p, pwidth, bswidth); } @@ -262,10 +245,13 @@ cmd_step_right(char **pp, int *pwidth, int *bswidth) * Step a pointer one character left in the command buffer. */ static char * -cmd_step_left(char **pp, int *pwidth, int *bswidth) +cmd_step_left(pp, pwidth, bswidth) + char **pp; + int *pwidth; + int *bswidth; { char *p = *pp; - LWCHAR ch = step_char((constant char **)pp, -1, cmdbuf); + LWCHAR ch = step_char(pp, -1, cmdbuf); return cmd_step_common(*pp, ch, p - *pp, pwidth, bswidth); } @@ -275,7 +261,8 @@ cmd_step_left(char **pp, int *pwidth, int *bswidth) * Then position the cursor just after the char old_cp (a pointer into cmdbuf). */ static void -cmd_repaint(char *old_cp) +cmd_repaint(old_cp) + constant char *old_cp; { /* * Repaint the line from the current position. @@ -285,7 +272,7 @@ cmd_repaint(char *old_cp) { char *np = cp; int width; - constant char *pr = cmd_step_right(&np, &width, NULL); + char *pr = cmd_step_right(&np, &width, NULL); if (cmd_col + width >= sc_width) break; cp = np; @@ -315,7 +302,7 @@ cmd_repaint(char *old_cp) * and set cp to the corresponding char in cmdbuf. */ static void -cmd_home(void) +cmd_home() { while (cmd_col > prompt_col) { @@ -334,7 +321,7 @@ cmd_home(void) * Shift the cmdbuf display left a half-screen. */ static void -cmd_lshift(void) +cmd_lshift() { char *s; char *save_cp; @@ -372,7 +359,7 @@ cmd_lshift(void) * Shift the cmdbuf display right a half-screen. */ static void -cmd_rshift(void) +cmd_rshift() { char *s; char *save_cp; @@ -402,7 +389,7 @@ cmd_rshift(void) * Move cursor right one character. */ static int -cmd_right(void) +cmd_right() { char *pr; char *ncp; @@ -437,7 +424,7 @@ cmd_right(void) * Move cursor left one character. */ static int -cmd_left(void) +cmd_left() { char *ncp; int width, bswidth; @@ -467,7 +454,9 @@ cmd_left(void) * Insert a char into the command buffer, at the current position. */ static int -cmd_ichar(char *cs, int clen) +cmd_ichar(cs, clen) + char *cs; + int clen; { char *s; @@ -502,7 +491,7 @@ cmd_ichar(char *cs, int clen) * Delete the char to the left of the cursor. */ static int -cmd_erase(void) +cmd_erase() { char *s; int clen; @@ -551,7 +540,7 @@ cmd_erase(void) * Delete the char under the cursor. */ static int -cmd_delete(void) +cmd_delete() { if (*cp == '\0') { @@ -570,7 +559,7 @@ cmd_delete(void) * Delete the "word" to the left of the cursor. */ static int -cmd_werase(void) +cmd_werase() { if (cp > cmdbuf && cp[-1] == ' ') { @@ -596,7 +585,7 @@ cmd_werase(void) * Delete the "word" under the cursor. */ static int -cmd_wdelete(void) +cmd_wdelete() { if (*cp == ' ') { @@ -622,7 +611,7 @@ cmd_wdelete(void) * Delete all chars in the command buffer. */ static int -cmd_kill(void) +cmd_kill() { if (cmdbuf[0] == '\0') { @@ -648,7 +637,9 @@ cmd_kill(void) * Select an mlist structure to be the current command history. */ public void -set_mlist(constant void *mlist, int cmdflags) +set_mlist(mlist, cmdflags) + void *mlist; + int cmdflags; { #if CMD_HISTORY curr_mlist = (struct mlist *) mlist; @@ -667,9 +658,10 @@ set_mlist(constant void *mlist, int cmdflags) * cmdbuf's corresponding chars. */ static int -cmd_updown(int action) +cmd_updown(action) + int action; { - char *s; + constant char *s; struct mlist *ml; if (curr_mlist == NULL) @@ -729,7 +721,10 @@ cmd_updown(int action) * Add a string to an mlist. */ public void -cmd_addhist(struct mlist *constant mlist, char *cmd, int modified) +cmd_addhist(mlist, cmd, modified) + struct mlist *mlist; + constant char *cmd; + int modified; { #if CMD_HISTORY struct mlist *ml; @@ -772,7 +767,7 @@ cmd_addhist(struct mlist *constant mlist, char *cmd, int modified) * Add it to the currently selected history list. */ public void -cmd_accept(void) +cmd_accept() { #if CMD_HISTORY /* @@ -794,7 +789,8 @@ cmd_accept(void) * CC_QUIT The char requests the current command to be aborted. */ static int -cmd_edit(int c) +cmd_edit(c) + int c; { int action; int flags; @@ -909,7 +905,8 @@ cmd_edit(int c) * Insert a string into the command buffer, at the current position. */ static int -cmd_istr(char *str) +cmd_istr(str) + char *str; { char *s; int action; @@ -918,7 +915,7 @@ cmd_istr(char *str) for (s = str; *s != '\0'; ) { char *os = s; - step_char((constant char **)&s, +1, endline); + step_char(&s, +1, endline); action = cmd_ichar(os, s - os); if (action != CC_OK) { @@ -936,14 +933,14 @@ cmd_istr(char *str) * cursor at the end of the word. */ static char * -delimit_word(void) +delimit_word() { char *word; #if SPACES_IN_FILENAMES char *p; int delim_quoted = 0; int meta_quoted = 0; - char *esc = get_meta_escape(); + constant char *esc = get_meta_escape(); int esclen = (int) strlen(esc); #endif @@ -1023,7 +1020,7 @@ delimit_word(void) * which start with that word, and set tk_text to that list. */ static void -init_compl(void) +init_compl() { char *word; char c; @@ -1086,7 +1083,9 @@ init_compl(void) * Return the next word in the current completion list. */ static char * -next_compl(int action, char *prev) +next_compl(action, prev) + int action; + char *prev; { switch (action) { @@ -1106,7 +1105,8 @@ next_compl(int action, char *prev) * or a subsequent time (step thru the list). */ static int -cmd_complete(int action) +cmd_complete(action) + int action; { char *s; @@ -1203,7 +1203,8 @@ fail: * CC_ERROR The char could not be accepted due to an error. */ public int -cmd_char(int c) +cmd_char(c) + int c; { int action; int len; @@ -1292,7 +1293,8 @@ cmd_char(int c) * Return the number currently in the command buffer. */ public LINENUM -cmd_int(long *frac) +cmd_int(frac) + long *frac; { char *p; LINENUM n = 0; @@ -1313,7 +1315,7 @@ cmd_int(long *frac) * Return a pointer to the command buffer. */ public char * -get_cmdbuf(void) +get_cmdbuf() { return (cmdbuf); } @@ -1323,7 +1325,7 @@ get_cmdbuf(void) * Return the last (most recent) string in the current command history. */ public char * -cmd_lastpattern(void) +cmd_lastpattern() { if (curr_mlist == NULL) return (NULL); @@ -1335,7 +1337,8 @@ cmd_lastpattern(void) /* */ static int -mlist_size(struct mlist *ml) +mlist_size(ml) + struct mlist *ml; { int size = 0; for (ml = ml->next; ml->string != NULL; ml = ml->next) @@ -1347,7 +1350,7 @@ mlist_size(struct mlist *ml) * Get the name of the history file. */ static char * -histfile_name(void) +histfile_name() { char *home; char *name; @@ -1387,8 +1390,11 @@ histfile_name(void) * Read a .lesshst file and call a callback for each line in the file. */ static void -read_cmdhist2(void (*action)(void*,struct mlist*,char*), void *uparam, - int skip_search, int skip_shell) +read_cmdhist2(action, uparam, skip_search, skip_shell) + void (*action)(void*,struct mlist*,char*); + void *uparam; + int skip_search; + int skip_shell; { struct mlist *ml = NULL; char line[CMDBUF_SIZE]; @@ -1448,8 +1454,11 @@ read_cmdhist2(void (*action)(void*,struct mlist*,char*), void *uparam, } static void -read_cmdhist(void (*action)(void*,struct mlist*,char*), void *uparam, - int skip_search, int skip_shell) +read_cmdhist(action, uparam, skip_search, skip_shell) + void (*action)(void*,struct mlist*,char*); + void *uparam; + int skip_search; + int skip_shell; { read_cmdhist2(action, uparam, skip_search, skip_shell); (*action)(uparam, NULL, NULL); /* signal end of file */ @@ -1468,7 +1477,7 @@ addhist_init(void *uparam, struct mlist *ml, char *string) * Initialize history from a .lesshist file. */ public void -init_cmdhist(void) +init_cmdhist() { #if CMD_HISTORY read_cmdhist(&addhist_init, NULL, 0, 0); @@ -1480,7 +1489,9 @@ init_cmdhist(void) */ #if CMD_HISTORY static void -write_mlist_header(struct mlist *ml, FILE *f) +write_mlist_header(ml, f) + struct mlist *ml; + FILE *f; { if (ml == &mlist_search) fprintf(f, "%s\n", HISTFILE_SEARCH_SECTION); @@ -1494,7 +1505,9 @@ write_mlist_header(struct mlist *ml, FILE *f) * Write all modified entries in an mlist to the history file. */ static void -write_mlist(struct mlist *ml, FILE *f) +write_mlist(ml, f) + struct mlist *ml; + FILE *f; { for (ml = ml->next; ml->string != NULL; ml = ml->next) { @@ -1510,7 +1523,8 @@ write_mlist(struct mlist *ml, FILE *f) * Make a temp name in the same directory as filename. */ static char * -make_tempname(char *filename) +make_tempname(filename) + char *filename; { char lastch; char *tempname = ecalloc(1, strlen(filename)+1); @@ -1573,7 +1587,8 @@ copy_hist(void *uparam, struct mlist *ml, char *string) * Make a file readable only by its owner. */ static void -make_file_private(FILE *f) +make_file_private(f) + FILE *f; { #if HAVE_FCHMOD int do_chmod = 1; @@ -1593,7 +1608,7 @@ make_file_private(FILE *f) * Does the history file need to be updated? */ static int -histfile_modified(void) +histfile_modified() { if (mlist_search.modified) return 1; @@ -1608,7 +1623,7 @@ histfile_modified(void) * Update the .lesshst file. */ public void -save_cmdhist(void) +save_cmdhist() { #if CMD_HISTORY char *histname; diff --git a/contrib/less/command.c b/contrib/less/command.c index a8fc00753cecf..72d7c9d432808 100644 --- a/contrib/less/command.c +++ b/contrib/less/command.c @@ -1,6 +1,6 @@ /* $FreeBSD$ */ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -44,10 +44,10 @@ extern char *curr_altfilename; extern char version[]; extern struct scrpos initial_scrpos; extern IFILE curr_ifile; -extern void constant *ml_search; -extern void constant *ml_examine; +extern void *ml_search; +extern void *ml_examine; #if SHELL_ESCAPE || PIPEC -extern void constant *ml_shell; +extern void *ml_shell; #endif #if EDITOR extern char *editor; @@ -84,7 +84,7 @@ struct ungot { }; static struct ungot* ungot = NULL; -static void multi_search(char *pattern, int n, int silent); +static void multi_search(); /* * Move the cursor to start of prompt line before executing a command. @@ -92,7 +92,7 @@ static void multi_search(char *pattern, int n, int silent); * updating the screen. */ static void -cmd_exec(void) +cmd_exec() { #if HILITE_SEARCH clear_attn(); @@ -105,7 +105,11 @@ cmd_exec(void) * Set up the display to start a new multi-character command. */ static void -start_mca(int action, constant char *prompt, constant void *mlist, int cmdflags) +start_mca(action, prompt, mlist, cmdflags) + int action; + constant char *prompt; + void *mlist; + int cmdflags; { mca = action; clear_bot(); @@ -115,7 +119,7 @@ start_mca(int action, constant char *prompt, constant void *mlist, int cmdflags) } public int -in_mca(void) +in_mca() { return (mca != 0 && mca != A_PREFIX); } @@ -124,7 +128,7 @@ in_mca(void) * Set up the display to start a new search command. */ static void -mca_search(void) +mca_search() { #if HILITE_SEARCH if (search_type & SRCH_FILTER) @@ -167,7 +171,7 @@ mca_search(void) * Set up the display to start a new toggle-option command. */ static void -mca_opt_toggle(void) +mca_opt_toggle() { int no_prompt; int flag; @@ -202,7 +206,7 @@ mca_opt_toggle(void) * Execute a multicharacter command. */ static void -exec_mca(void) +exec_mca() { char *cbuf; @@ -292,7 +296,8 @@ exec_mca(void) * Is a character an erase or kill char? */ static int -is_erase_char(int c) +is_erase_char(c) + int c; { return (c == erase_char || c == erase2_char || c == kill_char); } @@ -301,7 +306,8 @@ is_erase_char(int c) * Handle the first char of an option (after the initial dash). */ static int -mca_opt_first_char(int c) +mca_opt_first_char(c) + int c; { int flag = (optflag & ~OPT_NO_PROMPT); if (flag == OPT_NO_TOGGLE) @@ -352,7 +358,8 @@ mca_opt_first_char(int c) * accepting chars until user hits RETURN. */ static int -mca_opt_nonfirst_char(int c) +mca_opt_nonfirst_char(c) + int c; { char *p; char *oname; @@ -401,7 +408,8 @@ mca_opt_nonfirst_char(int c) * Handle a char of an option toggle command. */ static int -mca_opt_char(int c) +mca_opt_char(c) + int c; { PARG parg; @@ -466,7 +474,8 @@ mca_opt_char(int c) * Handle a char of a search command. */ static int -mca_search_char(int c) +mca_search_char(c) + int c; { int flag = 0; @@ -522,7 +531,8 @@ mca_search_char(int c) * Handle a character of a multi-character command. */ static int -mca_char(int c) +mca_char(c) + int c; { int ret; @@ -624,7 +634,7 @@ mca_char(int c) * Discard any buffered file data. */ static void -clear_buffers(void) +clear_buffers() { if (!(ch_getflags() & CH_CANSEEK)) return; @@ -639,7 +649,7 @@ clear_buffers(void) * Make sure the screen is displayed. */ static void -make_display(void) +make_display() { /* * If nothing is displayed yet, display starting from initial_scrpos. @@ -679,7 +689,7 @@ make_display(void) * Display the appropriate prompt. */ static void -prompt(void) +prompt() { constant char *p; @@ -756,7 +766,7 @@ prompt(void) * Display the less version message. */ public void -dispversion(void) +dispversion() { PARG parg; @@ -771,7 +781,7 @@ dispversion(void) * (characters previously given to ungetcc or ungetsc). */ public int -getcc(void) +getcc() { if (ungot == NULL) { @@ -826,7 +836,8 @@ getcc(void) * The next getcc() will return this character. */ public void -ungetcc(int c) +ungetcc(c) + int c; { struct ungot *ug = (struct ungot *) ecalloc(1, sizeof(struct ungot)); @@ -841,7 +852,8 @@ ungetcc(int c) * The next sequence of getcc()'s will return this string. */ public void -ungetsc(char *s) +ungetsc(s) + char *s; { char *p; @@ -855,7 +867,10 @@ ungetsc(char *s) * If SRCH_PAST_EOF is set, continue the search thru multiple files. */ static void -multi_search(char *pattern, int n, int silent) +multi_search(pattern, n, silent) + char *pattern; + int n; + int silent; { int nomore; IFILE save_ifile; @@ -949,7 +964,8 @@ multi_search(char *pattern, int n, int silent) * Forward forever, or until a highlighted line appears. */ static int -forw_loop(int until_hilite) +forw_loop(until_hilite) + int until_hilite; { POSITION curr_len; @@ -989,7 +1005,7 @@ forw_loop(int until_hilite) * Accept and execute commands until a quit command. */ public void -commands(void) +commands() { int c; int action; @@ -1769,6 +1785,16 @@ commands(void) screen_trashed = 1; break; + case A_LLSHIFT: + hshift = 0; + screen_trashed = 1; + break; + + case A_RRSHIFT: + hshift = rrshift(); + screen_trashed = 1; + break; + case A_PREFIX: /* * The command is incomplete (more chars are needed). diff --git a/contrib/less/compose.uni b/contrib/less/compose.uni index b814ce9916ae0..e3e1fa4017ebb 100644 --- a/contrib/less/compose.uni +++ b/contrib/less/compose.uni @@ -1,4 +1,4 @@ -/* Generated by "./mkutable -f2 Mn Me -- unicode/UnicodeData.txt" on Mon Jul 14 16:21:21 PDT 2014 */ +/* Generated by "./mkutable -f2 Mn Me -- unicode/UnicodeData.txt" on Tue Sep 20 10:51:43 PDT 2016 */ { 0x0300, 0x036f }, /* Mn */ { 0x0483, 0x0487 }, /* Mn */ { 0x0488, 0x0489 }, /* Me */ @@ -23,7 +23,8 @@ { 0x0825, 0x0827 }, /* Mn */ { 0x0829, 0x082d }, /* Mn */ { 0x0859, 0x085b }, /* Mn */ - { 0x08e4, 0x0902 }, /* Mn */ + { 0x08d4, 0x08e1 }, /* Mn */ + { 0x08e3, 0x0902 }, /* Mn */ { 0x093a, 0x093a }, /* Mn */ { 0x093c, 0x093c }, /* Mn */ { 0x0941, 0x0948 }, /* Mn */ @@ -117,6 +118,7 @@ { 0x17c9, 0x17d3 }, /* Mn */ { 0x17dd, 0x17dd }, /* Mn */ { 0x180b, 0x180d }, /* Mn */ + { 0x1885, 0x1886 }, /* Mn */ { 0x18a9, 0x18a9 }, /* Mn */ { 0x1920, 0x1922 }, /* Mn */ { 0x1927, 0x1928 }, /* Mn */ @@ -156,7 +158,7 @@ { 0x1cf4, 0x1cf4 }, /* Mn */ { 0x1cf8, 0x1cf9 }, /* Mn */ { 0x1dc0, 0x1df5 }, /* Mn */ - { 0x1dfc, 0x1dff }, /* Mn */ + { 0x1dfb, 0x1dff }, /* Mn */ { 0x20d0, 0x20dc }, /* Mn */ { 0x20dd, 0x20e0 }, /* Me */ { 0x20e1, 0x20e1 }, /* Mn */ @@ -170,13 +172,13 @@ { 0xa66f, 0xa66f }, /* Mn */ { 0xa670, 0xa672 }, /* Me */ { 0xa674, 0xa67d }, /* Mn */ - { 0xa69f, 0xa69f }, /* Mn */ + { 0xa69e, 0xa69f }, /* Mn */ { 0xa6f0, 0xa6f1 }, /* Mn */ { 0xa802, 0xa802 }, /* Mn */ { 0xa806, 0xa806 }, /* Mn */ { 0xa80b, 0xa80b }, /* Mn */ { 0xa825, 0xa826 }, /* Mn */ - { 0xa8c4, 0xa8c4 }, /* Mn */ + { 0xa8c4, 0xa8c5 }, /* Mn */ { 0xa8e0, 0xa8f1 }, /* Mn */ { 0xa926, 0xa92d }, /* Mn */ { 0xa947, 0xa951 }, /* Mn */ @@ -203,7 +205,7 @@ { 0xabed, 0xabed }, /* Mn */ { 0xfb1e, 0xfb1e }, /* Mn */ { 0xfe00, 0xfe0f }, /* Mn */ - { 0xfe20, 0xfe2d }, /* Mn */ + { 0xfe20, 0xfe2f }, /* Mn */ { 0x101fd, 0x101fd }, /* Mn */ { 0x102e0, 0x102e0 }, /* Mn */ { 0x10376, 0x1037a }, /* Mn */ @@ -224,16 +226,21 @@ { 0x11173, 0x11173 }, /* Mn */ { 0x11180, 0x11181 }, /* Mn */ { 0x111b6, 0x111be }, /* Mn */ + { 0x111ca, 0x111cc }, /* Mn */ { 0x1122f, 0x11231 }, /* Mn */ { 0x11234, 0x11234 }, /* Mn */ { 0x11236, 0x11237 }, /* Mn */ + { 0x1123e, 0x1123e }, /* Mn */ { 0x112df, 0x112df }, /* Mn */ { 0x112e3, 0x112ea }, /* Mn */ - { 0x11301, 0x11301 }, /* Mn */ + { 0x11300, 0x11301 }, /* Mn */ { 0x1133c, 0x1133c }, /* Mn */ { 0x11340, 0x11340 }, /* Mn */ { 0x11366, 0x1136c }, /* Mn */ { 0x11370, 0x11374 }, /* Mn */ + { 0x11438, 0x1143f }, /* Mn */ + { 0x11442, 0x11444 }, /* Mn */ + { 0x11446, 0x11446 }, /* Mn */ { 0x114b3, 0x114b8 }, /* Mn */ { 0x114ba, 0x114ba }, /* Mn */ { 0x114bf, 0x114c0 }, /* Mn */ @@ -241,6 +248,7 @@ { 0x115b2, 0x115b5 }, /* Mn */ { 0x115bc, 0x115bd }, /* Mn */ { 0x115bf, 0x115c0 }, /* Mn */ + { 0x115dc, 0x115dd }, /* Mn */ { 0x11633, 0x1163a }, /* Mn */ { 0x1163d, 0x1163d }, /* Mn */ { 0x1163f, 0x11640 }, /* Mn */ @@ -248,6 +256,16 @@ { 0x116ad, 0x116ad }, /* Mn */ { 0x116b0, 0x116b5 }, /* Mn */ { 0x116b7, 0x116b7 }, /* Mn */ + { 0x1171d, 0x1171f }, /* Mn */ + { 0x11722, 0x11725 }, /* Mn */ + { 0x11727, 0x1172b }, /* Mn */ + { 0x11c30, 0x11c36 }, /* Mn */ + { 0x11c38, 0x11c3d }, /* Mn */ + { 0x11c3f, 0x11c3f }, /* Mn */ + { 0x11c92, 0x11ca7 }, /* Mn */ + { 0x11caa, 0x11cb0 }, /* Mn */ + { 0x11cb2, 0x11cb3 }, /* Mn */ + { 0x11cb5, 0x11cb6 }, /* Mn */ { 0x16af0, 0x16af4 }, /* Mn */ { 0x16b30, 0x16b36 }, /* Mn */ { 0x16f8f, 0x16f92 }, /* Mn */ @@ -257,5 +275,17 @@ { 0x1d185, 0x1d18b }, /* Mn */ { 0x1d1aa, 0x1d1ad }, /* Mn */ { 0x1d242, 0x1d244 }, /* Mn */ + { 0x1da00, 0x1da36 }, /* Mn */ + { 0x1da3b, 0x1da6c }, /* Mn */ + { 0x1da75, 0x1da75 }, /* Mn */ + { 0x1da84, 0x1da84 }, /* Mn */ + { 0x1da9b, 0x1da9f }, /* Mn */ + { 0x1daa1, 0x1daaf }, /* Mn */ + { 0x1e000, 0x1e006 }, /* Mn */ + { 0x1e008, 0x1e018 }, /* Mn */ + { 0x1e01b, 0x1e021 }, /* Mn */ + { 0x1e023, 0x1e024 }, /* Mn */ + { 0x1e026, 0x1e02a }, /* Mn */ { 0x1e8d0, 0x1e8d6 }, /* Mn */ + { 0x1e944, 0x1e94a }, /* Mn */ { 0xe0100, 0xe01ef }, /* Mn */ diff --git a/contrib/less/cvt.c b/contrib/less/cvt.c index 0c0b08be56002..54473e0c08636 100644 --- a/contrib/less/cvt.c +++ b/contrib/less/cvt.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -20,7 +20,9 @@ extern int utf_mode; * Get the length of a buffer needed to convert a string. */ public int -cvt_length(int len, int ops) +cvt_length(len, ops) + int len; + int ops; { if (utf_mode) /* @@ -36,7 +38,8 @@ cvt_length(int len, int ops) * Allocate a chpos array for use by cvt_text. */ public int * -cvt_alloc_chpos(int len) +cvt_alloc_chpos(len) + int len; { int i; int *chpos = (int *) ecalloc(sizeof(int), len); @@ -52,7 +55,12 @@ cvt_alloc_chpos(int len) * odst character (when it was in osrc) is returned in the chpos array. */ public void -cvt_text(char *odst, char *osrc, int *chpos, int *lenp, int ops) +cvt_text(odst, osrc, chpos, lenp, ops) + char *odst; + char *osrc; + int *chpos; + int *lenp; + int ops; { char *dst; char *edst = odst; @@ -69,7 +77,7 @@ cvt_text(char *odst, char *osrc, int *chpos, int *lenp, int ops) { int src_pos = (int) (src - osrc); int dst_pos = (int) (dst - odst); - ch = step_char((constant char **)&src, +1, src_end); + ch = step_char(&src, +1, src_end); if ((ops & CVT_BS) && ch == '\b' && dst > odst) { /* Delete backspace and preceding char. */ diff --git a/contrib/less/decode.c b/contrib/less/decode.c index 7f857fd52ceec..851672a1b0df8 100644 --- a/contrib/less/decode.c +++ b/contrib/less/decode.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -94,8 +94,12 @@ static unsigned char cmdtable[] = ESC,']',0, A_RSHIFT, ESC,'(',0, A_LSHIFT, ESC,')',0, A_RSHIFT, + ESC,'{',0, A_LLSHIFT, + ESC,'}',0, A_RRSHIFT, SK(SK_RIGHT_ARROW),0, A_RSHIFT, SK(SK_LEFT_ARROW),0, A_LSHIFT, + SK(SK_CTL_RIGHT_ARROW),0, A_RRSHIFT, + SK(SK_CTL_LEFT_ARROW),0, A_LLSHIFT, '{',0, A_F_BRACKET|A_EXTRA, '{','}',0, '}',0, A_B_BRACKET|A_EXTRA, '{','}',0, '(',0, A_F_BRACKET|A_EXTRA, '(',')',0, @@ -229,7 +233,9 @@ static struct tablelist *list_sysvar_tables = NULL; * Expand special key abbreviations in a command table. */ static void -expand_special_keys(char *table, int len) +expand_special_keys(table, len) + char *table; + int len; { char *fm; char *to; @@ -288,7 +294,7 @@ expand_special_keys(char *table, int len) * Initialize the command lists. */ public void -init_cmds(void) +init_cmds() { /* * Add the default command tables. @@ -318,7 +324,10 @@ init_cmds(void) * Add a command table. */ static int -add_cmd_table(struct tablelist **tlist, char *buf, int len) +add_cmd_table(tlist, buf, len) + struct tablelist **tlist; + char *buf; + int len; { struct tablelist *t; @@ -345,7 +354,9 @@ add_cmd_table(struct tablelist **tlist, char *buf, int len) * Add a command table. */ public void -add_fcmd_table(char *buf, int len) +add_fcmd_table(buf, len) + char *buf; + int len; { if (add_cmd_table(&list_fcmd_tables, buf, len) < 0) error("Warning: some commands disabled", NULL_PARG); @@ -355,7 +366,9 @@ add_fcmd_table(char *buf, int len) * Add an editing command table. */ public void -add_ecmd_table(char *buf, int len) +add_ecmd_table(buf, len) + char *buf; + int len; { if (add_cmd_table(&list_ecmd_tables, buf, len) < 0) error("Warning: some edit commands disabled", NULL_PARG); @@ -365,7 +378,10 @@ add_ecmd_table(char *buf, int len) * Add an environment variable table. */ static void -add_var_table(struct tablelist **tlist, char *buf, int len) +add_var_table(tlist, buf, len) + struct tablelist **tlist; + char *buf; + int len; { if (add_cmd_table(tlist, buf, len) < 0) error("Warning: environment variables from lesskey file unavailable", NULL_PARG); @@ -375,7 +391,11 @@ add_var_table(struct tablelist **tlist, char *buf, int len) * Search a single command table for the command string in cmd. */ static int -cmd_search(char *cmd, char *table, char *endtable, char **sp) +cmd_search(cmd, table, endtable, sp) + char *cmd; + char *table; + char *endtable; + char **sp; { char *p; char *q; @@ -463,7 +483,10 @@ cmd_search(char *cmd, char *table, char *endtable, char **sp) * The "extra" string, if any, is returned in sp. */ static int -cmd_decode(struct tablelist *tlist, char *cmd, char **sp) +cmd_decode(tlist, cmd, sp) + struct tablelist *tlist; + char *cmd; + char **sp; { struct tablelist *t; int action = A_INVALID; @@ -487,7 +510,9 @@ cmd_decode(struct tablelist *tlist, char *cmd, char **sp) * Decode a command from the cmdtables list. */ public int -fcmd_decode(char *cmd, char **sp) +fcmd_decode(cmd, sp) + char *cmd; + char **sp; { return (cmd_decode(list_fcmd_tables, cmd, sp)); } @@ -496,7 +521,9 @@ fcmd_decode(char *cmd, char **sp) * Decode a command from the edittables list. */ public int -ecmd_decode(char *cmd, char **sp) +ecmd_decode(cmd, sp) + char *cmd; + char **sp; { return (cmd_decode(list_ecmd_tables, cmd, sp)); } @@ -506,7 +533,8 @@ ecmd_decode(char *cmd, char **sp) * Looks first in the lesskey file, then in the real environment. */ public char * -lgetenv(char *var) +lgetenv(var) + char *var; { int a; char *s; @@ -530,7 +558,8 @@ lgetenv(char *var) * two bytes, low order first, in radix KRADIX. */ static int -gint(char **sp) +gint(sp) + char **sp; { int n; @@ -543,7 +572,9 @@ gint(char **sp) * Process an old (pre-v241) lesskey file. */ static int -old_lesskey(char *buf, int len) +old_lesskey(buf, len) + char *buf; + int len; { /* * Old-style lesskey file. @@ -562,7 +593,10 @@ old_lesskey(char *buf, int len) * Process a new (post-v241) lesskey file. */ static int -new_lesskey(char *buf, int len, int sysvar) +new_lesskey(buf, len, sysvar) + char *buf; + int len; + int sysvar; { char *p; int c; @@ -613,7 +647,9 @@ new_lesskey(char *buf, int len, int sysvar) * Set up a user command table, based on a "lesskey" file. */ public int -lesskey(char *filename, int sysvar) +lesskey(filename, sysvar) + char *filename; + int sysvar; { char *buf; POSITION len; @@ -681,7 +717,10 @@ lesskey(char *filename, int sysvar) * Add the standard lesskey file "$HOME/.less" */ public void -add_hometable(char *envname, char *def_filename, int sysvar) +add_hometable(envname, def_filename, sysvar) + char *envname; + char *def_filename; + int sysvar; { char *filename; PARG parg; @@ -707,7 +746,9 @@ add_hometable(char *envname, char *def_filename, int sysvar) * See if a char is a special line-editing command. */ public int -editchar(int c, int flags) +editchar(c, flags) + int c; + int flags; { int action; int nch; diff --git a/contrib/less/edit.c b/contrib/less/edit.c index 9a89314ef13d1..f2a74a049200a 100644 --- a/contrib/less/edit.c +++ b/contrib/less/edit.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -26,7 +26,7 @@ extern int sigs; extern IFILE curr_ifile; extern IFILE old_ifile; extern struct scrpos initial_scrpos; -extern void * constant ml_examine; +extern void *ml_examine; #if SPACES_IN_FILENAMES extern char openquote; extern char closequote; @@ -55,7 +55,9 @@ static void *curr_altpipe; * back_textlist does the same, but runs thru the list backwards. */ public void -init_textlist(struct textlist *tlist, char *str) +init_textlist(tlist, str) + struct textlist *tlist; + char *str; { char *s; #if SPACES_IN_FILENAMES @@ -97,7 +99,9 @@ init_textlist(struct textlist *tlist, char *str) } public char * -forw_textlist(struct textlist *tlist, char *prev) +forw_textlist(tlist, prev) + struct textlist *tlist; + char *prev; { char *s; @@ -119,7 +123,9 @@ forw_textlist(struct textlist *tlist, char *prev) } public char * -back_textlist(struct textlist *tlist, char *prev) +back_textlist(tlist, prev) + struct textlist *tlist; + char *prev; { char *s; @@ -146,7 +152,7 @@ back_textlist(struct textlist *tlist, char *prev) * Close the current input file. */ static void -close_file(void) +close_file() { struct scrpos scrpos; @@ -190,7 +196,8 @@ close_file(void) * Filename == NULL means just close the current file. */ public int -edit(char *filename) +edit(filename) + char *filename; { if (filename == NULL) return (edit_ifile(NULL_IFILE)); @@ -202,7 +209,8 @@ edit(char *filename) * ifile == NULL means just close the current file. */ public int -edit_ifile(IFILE ifile) +edit_ifile(ifile) + IFILE ifile; { int f; int answer; @@ -452,7 +460,8 @@ edit_ifile(IFILE ifile) * Then edit the first one. */ public int -edit_list(char *filelist) +edit_list(filelist) + char *filelist; { IFILE save_ifile; char *good_filename; @@ -509,7 +518,7 @@ edit_list(char *filelist) * Edit the first file in the command line (ifile) list. */ public int -edit_first(void) +edit_first() { curr_ifile = NULL_IFILE; return (edit_next(1)); @@ -519,7 +528,7 @@ edit_first(void) * Edit the last file in the command line (ifile) list. */ public int -edit_last(void) +edit_last() { curr_ifile = NULL_IFILE; return (edit_prev(1)); @@ -530,7 +539,10 @@ edit_last(void) * Edit the n-th next or previous file in the command line (ifile) list. */ static int -edit_istep(IFILE h, int n, int dir) +edit_istep(h, n, dir) + IFILE h; + int n; + int dir; { IFILE next; @@ -569,25 +581,31 @@ edit_istep(IFILE h, int n, int dir) } static int -edit_inext(IFILE h, int n) +edit_inext(h, n) + IFILE h; + int n; { return (edit_istep(h, n, +1)); } public int -edit_next(int n) +edit_next(n) + int n; { return edit_istep(curr_ifile, n, +1); } static int -edit_iprev(IFILE h, int n) +edit_iprev(h, n) + IFILE h; + int n; { return (edit_istep(h, n, -1)); } public int -edit_prev(int n) +edit_prev(n) + int n; { return edit_istep(curr_ifile, n, -1); } @@ -596,7 +614,8 @@ edit_prev(int n) * Edit a specific file in the command line (ifile) list. */ public int -edit_index(int n) +edit_index(n) + int n; { IFILE h; @@ -616,7 +635,7 @@ edit_index(int n) } public IFILE -save_curr_ifile(void) +save_curr_ifile() { if (curr_ifile != NULL_IFILE) hold_ifile(curr_ifile, 1); @@ -624,7 +643,8 @@ save_curr_ifile(void) } public void -unsave_ifile(IFILE save_ifile) +unsave_ifile(save_ifile) + IFILE save_ifile; { if (save_ifile != NULL_IFILE) hold_ifile(save_ifile, -1); @@ -634,7 +654,8 @@ unsave_ifile(IFILE save_ifile) * Reedit the ifile which was previously open. */ public void -reedit_ifile(IFILE save_ifile) +reedit_ifile(save_ifile) + IFILE save_ifile; { IFILE next; IFILE prev; @@ -667,7 +688,7 @@ reedit_ifile(IFILE save_ifile) } public void -reopen_curr_ifile(void) +reopen_curr_ifile() { IFILE save_ifile = save_curr_ifile(); close_file(); @@ -678,7 +699,7 @@ reopen_curr_ifile(void) * Edit standard input. */ public int -edit_stdin(void) +edit_stdin() { if (isatty(fd0)) { @@ -693,7 +714,7 @@ edit_stdin(void) * Used if standard output is not a tty. */ public void -cat_file(void) +cat_file() { int c; @@ -710,7 +731,8 @@ cat_file(void) * We take care not to blindly overwrite an existing file. */ public void -use_logfile(char *filename) +use_logfile(filename) + char *filename; { int exists; int answer; diff --git a/contrib/less/filename.c b/contrib/less/filename.c index 715a96a85a78f..a153207ba6359 100644 --- a/contrib/less/filename.c +++ b/contrib/less/filename.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -63,7 +63,8 @@ extern char closequote; * Remove quotes around a filename. */ public char * -shell_unquote(char *str) +shell_unquote(str) + char *str; { char *name; char *p; @@ -101,7 +102,7 @@ shell_unquote(char *str) * Get the shell's escape character. */ public char * -get_meta_escape(void) +get_meta_escape() { char *s; @@ -115,7 +116,7 @@ get_meta_escape(void) * Get the characters which the shell considers to be "metacharacters". */ static char * -metachars(void) +metachars() { static char *mchars = NULL; @@ -132,7 +133,8 @@ metachars(void) * Is this a shell metacharacter? */ static int -metachar(char c) +metachar(c) + char c; { return (strchr(metachars(), c) != NULL); } @@ -141,7 +143,8 @@ metachar(char c) * Insert a backslash before each metacharacter in a string. */ public char * -shell_quote(char *s) +shell_quote(s) + char *s; { char *p; char *newstr; @@ -218,7 +221,9 @@ shell_quote(char *s) * Return NULL if the file does not exist in the directory. */ static char * -dirfile(char *dirname, char *filename) +dirfile(dirname, filename) + char *dirname; + char *filename; { char *pathname; char *qpathname; @@ -256,7 +261,8 @@ dirfile(char *dirname, char *filename) * Return the full pathname of the given file in the "home directory". */ public char * -homefile(char *filename) +homefile(filename) + char *filename; { char *pathname; @@ -305,7 +311,8 @@ homefile(char *filename) * {{ This is a lot of work just to support % and #. }} */ public char * -fexpand(char *s) +fexpand(s) + char *s; { char *fr, *to; int n; @@ -400,7 +407,8 @@ fexpand(char *s) * the given string. */ public char * -fcomplete(char *s) +fcomplete(s) + char *s; { char *fpat; char *qs; @@ -459,12 +467,13 @@ fcomplete(char *s) * This is just a guess, and we need not try too hard to make it accurate. */ public int -bin_file(int f) +bin_file(f) + int f; { int n; int bin_count = 0; char data[256]; - constant char* p; + char* p; char* pend; if (!seekable(f)) @@ -503,7 +512,8 @@ bin_file(int f) * Try to determine the size of a file by seeking to the end. */ static POSITION -seek_filesize(int f) +seek_filesize(f) + int f; { off_t spos; @@ -518,7 +528,8 @@ seek_filesize(int f) * Return a pointer to the string in memory. */ static char * -readfd(FILE *fd) +readfd(fd) + FILE *fd; { int len; int ch; @@ -566,7 +577,8 @@ FILE *popen(); * Return a pointer to a pipe connected to the shell command's standard output. */ static FILE * -shellcmd(char *cmd) +shellcmd(cmd) + char *cmd; { FILE *fd; @@ -616,7 +628,8 @@ shellcmd(char *cmd) * Expand a filename, doing any system-specific metacharacter substitutions. */ public char * -lglob(char *filename) +lglob(filename) + char *filename; { char *gfilename; char *ofilename; @@ -805,7 +818,8 @@ lglob(char *filename) * Return a large number if there are any other % escapes besides %s. */ static int -num_pct_s(char *lessopen) +num_pct_s(lessopen) + char *lessopen; { int num = 0; @@ -830,7 +844,10 @@ num_pct_s(char *lessopen) * instead of the file we're about to open. */ public char * -open_altfile(char *filename, int *pf, void **pfd) +open_altfile(filename, pf, pfd) + char *filename; + int *pf; + void **pfd; { #if !HAVE_POPEN return (NULL); @@ -939,7 +956,10 @@ open_altfile(char *filename, int *pf, void **pfd) * Close a replacement file. */ public void -close_altfile(char *altfilename, char *filename, void *pipefd) +close_altfile(altfilename, filename, pipefd) + char *altfilename; + char *filename; + void *pipefd; { #if HAVE_POPEN char *lessclose; @@ -981,7 +1001,8 @@ close_altfile(char *altfilename, char *filename, void *pipefd) * Is the specified file a directory? */ public int -is_dir(char *filename) +is_dir(filename) + char *filename; { int isdir = 0; @@ -1016,7 +1037,8 @@ is_dir(char *filename) * (if it cannot be opened or is a directory, etc.) */ public char * -bad_file(char *filename) +bad_file(filename) + char *filename; { char *m = NULL; @@ -1061,7 +1083,8 @@ bad_file(char *filename) * In Unix, we can stat the file. */ public POSITION -filesize(int f) +filesize(f) + int f; { #if HAVE_STAT struct stat statbuf; @@ -1083,7 +1106,7 @@ filesize(int f) * */ public char * -shell_coption(void) +shell_coption() { return ("-c"); } @@ -1092,7 +1115,8 @@ shell_coption(void) * Return last component of a pathname. */ public char * -last_component(char *name) +last_component(name) + char *name; { char *slash; diff --git a/contrib/less/forwback.c b/contrib/less/forwback.c index 4624f6f0c1ba0..9fe0cb07d0e78 100644 --- a/contrib/less/forwback.c +++ b/contrib/less/forwback.c @@ -1,6 +1,6 @@ /* $FreeBSD$ */ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -48,7 +48,7 @@ extern char *tagoption; * Sound the bell to indicate user is trying to move past end of file. */ static void -eof_bell(void) +eof_bell() { if (quiet == NOT_QUIET) bell(); @@ -60,7 +60,7 @@ eof_bell(void) * Check to see if the end of file is currently displayed. */ public int -eof_displayed(void) +eof_displayed() { POSITION pos; @@ -87,7 +87,7 @@ eof_displayed(void) * Check to see if the entire file is currently displayed. */ public int -entire_file_displayed(void) +entire_file_displayed() { POSITION pos; @@ -107,7 +107,7 @@ entire_file_displayed(void) * for the first time. */ public void -squish_check(void) +squish_check() { if (!squished) return; @@ -125,7 +125,12 @@ squish_check(void) * The first real line after the blanks will start at ch_zero(). */ public void -forw(int n, POSITION pos, int force, int only_last, int nblank) +forw(n, pos, force, only_last, nblank) + int n; + POSITION pos; + int force; + int only_last; + int nblank; { int nlines = 0; int do_repaint; @@ -297,7 +302,11 @@ forw(int n, POSITION pos, int force, int only_last, int nblank) * Display n lines, scrolling backward. */ public void -back(int n, POSITION pos, int force, int only_last) +back(n, pos, force, only_last) + int n; + POSITION pos; + int force; + int only_last; { int nlines = 0; int do_repaint; @@ -355,7 +364,10 @@ back(int n, POSITION pos, int force, int only_last) * Start just after the line currently displayed at the bottom of the screen. */ public void -forward(int n, int force, int only_last) +forward(n, force, only_last) + int n; + int force; + int only_last; { POSITION pos; @@ -404,7 +416,10 @@ forward(int n, int force, int only_last) * Start just before the line currently displayed at the top of the screen. */ public void -backward(int n, int force, int only_last) +backward(n, force, only_last) + int n; + int force; + int only_last; { POSITION pos; @@ -424,7 +439,7 @@ backward(int n, int force, int only_last) * top_scroll, as well as back_scroll. */ public int -get_back_scroll(void) +get_back_scroll() { if (no_back_scroll) return (0); @@ -434,3 +449,21 @@ get_back_scroll(void) return (sc_height - 2); return (10000); /* infinity */ } + +/* + * Get line count of file up to the screen height + 1 char + */ + public int +get_line_count() +{ + int nlines; + POSITION pos; + + pos = ch_zero(); + for (nlines = 0; nlines <= sc_height; nlines++) + { + pos = forw_line(pos); + if (pos == NULL_POSITION) break; + } + return nlines; +} diff --git a/contrib/less/funcs.h b/contrib/less/funcs.h index fa8ecc345cdb1..1fe28c8494743 100644 --- a/contrib/less/funcs.h +++ b/contrib/less/funcs.h @@ -1,302 +1,300 @@ - public char * save (constant char *s); - public VOID_POINTER ecalloc (int count, unsigned int size); - public char * skipsp (char *s); - public char * skipnsp(char *s); - public int sprefix (char *ps, char *s, int uppercase); - public void quit (int status); - public void raw_mode (int on); - public void scrsize (void); - public char * special_key_str (int key); - public void get_term (void); - public void init (void); - public void deinit (void); - public void home (void); - public void add_line (void); - public void remove_top (int n); - public void win32_scroll_up (int n); - public void lower_left (void); - public void line_left (void); - public void check_winch (void); - public void goto_line (int slinenum); - public void vbell (void); - public void bell (void); - public void clear (void); - public void clear_eol (void); - public void clear_bot (void); - public void at_enter (int attr); - public void at_exit (void); - public void at_switch (int attr); - public int is_at_equiv (int attr1, int attr2); - public int apply_at_specials (int attr); - public void backspace (void); - public void putbs (void); - public char WIN32getch (int tty); - public void WIN32setcolors (int fg, int bg); - public void WIN32textout (char *text, int len); - public void match_brac(int obrac, int cbrac, int forwdir, int n); - public void ch_ungetchar (int c); - public void end_logfile (void); - public void sync_logfile (void); - public int ch_seek (POSITION pos); - public int ch_end_seek (void); - public int ch_end_buffer_seek (void); - public int ch_beg_seek (void); - public POSITION ch_length (void); - public POSITION ch_tell (void); - public int ch_forw_get (void); - public int ch_back_get (void); - public void ch_setbufspace (int bufspace); - public void ch_flush (void); - public int seekable (int f); - public void ch_set_eof (void); - public void ch_init (int f, int flags); - public void ch_close (void); - public int ch_getflags (void); -struct filestate; - public void ch_dump (struct filestate *fs); - public void init_charset (void); - public int binary_char (LWCHAR c); - public int control_char (LWCHAR c); - public char * prchar (LWCHAR c); - public char * prutfchar (LWCHAR ch); - public int utf_len (char ch); - public int is_utf8_well_formed (unsigned char *s, int slen); - public int utf_bin_count (unsigned char *data, int len); - public LWCHAR get_wchar (constant char *p); - public void put_wchar (char **pp, LWCHAR ch); - public LWCHAR step_char (constant char **pp, signed int dir, constant char *limit); - public int is_composing_char (LWCHAR ch); - public int is_ubin_char (LWCHAR ch); - public int is_wide_char (LWCHAR ch); - public int is_combining_char (LWCHAR ch1, LWCHAR ch2); - public void cmd_reset (void); - public void clear_cmd (void); - public void cmd_putstr (constant char *s); - public int len_cmdbuf (void); - public void set_mlist (constant void *mlist, int cmdflags); -struct mlist; - public void cmd_addhist (struct mlist *constant mlist, char *cmd, int modified); - public void cmd_accept (void); - public int cmd_char (int c); - public LINENUM cmd_int (long *frac); - public char * get_cmdbuf (void); - public char * cmd_lastpattern (void); - public void init_cmdhist (void); - public void save_cmdhist (void); - public int in_mca (void); - public void dispversion (void); - public int getcc (void); - public void ungetcc (int c); - public void ungetsc (char *s); - public void commands (void); - public int cvt_length (int len, int ops); - public int * cvt_alloc_chpos (int len); - public void cvt_text (char *odst, char *osrc, int *chpos, int *lenp, int ops); - public void init_cmds (void); - public void add_fcmd_table (char *buf, int len); - public void add_ecmd_table (char *buf, int len); - public int fcmd_decode (char *cmd, char **sp); - public int ecmd_decode (char *cmd, char **sp); - public char * lgetenv (char *var); - public int lesskey (char *filename, int sysvar); - public void add_hometable (char *envname, char *def_filename, int sysvar); - public int editchar (int c, int flags); - public void init_textlist (struct textlist *tlist, char *str); - public char * forw_textlist (struct textlist *tlist, char *prev); - public char * back_textlist (struct textlist *tlist, char *prev); - public int edit (char *filename); - public int edit_ifile (IFILE ifile); - public int edit_list (char *filelist); - public int edit_first (void); - public int edit_last (void); - public int edit_next (int n); - public int edit_prev (int n); - public int edit_index (int n); - public IFILE save_curr_ifile (void); - public void unsave_ifile (IFILE save_ifile); - public void reedit_ifile (IFILE save_ifiler); - public void reopen_curr_ifile (void); - public int edit_stdin (void); - public void cat_file (void); - public void use_logfile (char *filename); - public char * shell_unquote (char *str); - public char * get_meta_escape (void); - public char * shell_quote (char *s); - public char * homefile (char *filename); - public char * fexpand (char *s); - public char * fcomplete (char *s); - public int bin_file (int f); - public char * lglob (char *filename); - public char * open_altfile (char *filename, int *pf, void **pfd); - public void close_altfile (char *altfilename, char *filename, void *pipefd); - public int is_dir (char *filename); - public char * bad_file (char *filename); - public POSITION filesize (int f); - public char * shell_coption (void); - public char * last_component (char *name); - public int eof_displayed (void); - public int entire_file_displayed (void); - public void squish_check (void); - public void forw (int n, POSITION pos, int force, int only_last, int nblank); - public void back (int n, POSITION pos, int force, int only_last); - public void forward (int n, int force, int only_last); - public void backward (int n, int force, int only_last); - public int get_back_scroll (void); - public void del_ifile (IFILE h); - public IFILE next_ifile (IFILE h); - public IFILE prev_ifile (IFILE h); - public IFILE getoff_ifile (IFILE ifile); - public int nifile (void); - public IFILE get_ifile (char *filename, IFILE prev); - public char * get_filename (IFILE ifile); - public int get_index (IFILE ifile); - public void store_pos (IFILE ifile, struct scrpos *scrpos); - public void get_pos (IFILE ifile, struct scrpos *scrpos); - public void set_open (IFILE ifile); - public int opened (IFILE ifile); - public void hold_ifile (IFILE ifile, int incr); - public int held_ifile (IFILE ifile); - public void * get_filestate (IFILE ifile); - public void set_filestate (IFILE ifile, void *filestate); - public void if_dump (void); - public POSITION forw_line (POSITION curr_pos); - public POSITION back_line (POSITION curr_pos); - public void set_attnpos (POSITION pos); - public void jump_forw (void); - public void jump_forw_buffered (void); - public void jump_back (LINENUM linenum); - public void repaint (void); - public void jump_percent (int percent, long fraction); - public void jump_line_loc (POSITION pos, int sline); - public void jump_loc (POSITION pos, int sline); - public void init_line (void); - public int is_ascii_char (LWCHAR ch); - public void prewind (void); - public void plinenum (POSITION pos); - public void pshift_all (void); - public int is_ansi_end (LWCHAR ch); - public int is_ansi_middle (LWCHAR ch); - public int pappend (unsigned char c, POSITION pos); - public int pflushmbc (void); - public void pdone (int endline, int forw); - public void set_status_col (char c); - public int gline (int i, int *ap); - public void null_line (void); - public POSITION forw_raw_line (POSITION curr_pos, char **linep, int *line_lenp); - public POSITION back_raw_line (POSITION curr_pos, char **linep, int *line_lenp); - public void clr_linenum (void); - public void add_lnum (LINENUM linenum, POSITION pos); - public LINENUM find_linenum (POSITION pos); - public POSITION find_pos (LINENUM linenum); - public LINENUM currline (int where); - public void lsystem (char *cmd, char *donemsg); - public int pipe_mark (int c, char *cmd); - public int pipe_data (char *cmd, POSITION spos, POSITION epos); - public void init_mark (void); - public int badmark (int c); - public void setmark (int c); - public void lastmark (void); - public void gomark (int c); - public POSITION markpos (int c); - public void unmark (IFILE ifile); - public void opt_o (int type, char *s); - public void opt__O (int type, char *s); - public void opt_j (int type, char *s); - public void calc_jump_sline (void); - public void opt_shift (int type, char *s); - public void calc_shift_count (void); - public void opt_k (int type, char *s); - public void opt_t (int type, char *s); - public void opt__T (int type, char *s); - public void opt_p (int type, char *s); - public void opt__P (int type, char *s); - public void opt_b (int type, char *s); - public void opt_i (int type, char *s); - public void opt__V (int type, char *s); - public void opt_D (int type, char *s); - public void opt_x (int type, char *s); - public void opt_quote (int type, char *s); - public void opt_query (int type, char *s); - public int get_swindow (void); - public char * propt (int c); - public void scan_option (char *s); -struct loption; - public void toggle_option (struct loption *o, int lower, char *s, int how_toggle); - public int opt_has_param (struct loption *o); - public char * opt_prompt (struct loption *o); - public int isoptpending (void); - public void nopendopt (void); - public int getnum (char **sp, char *printopt, int *errp); - public long getfraction (char **sp, char *printopt, int *errp); - public int get_quit_at_eof (void); - public void init_option (void); - public struct loption * findopt (int c); - public struct loption * findopt_name (char **p_optname, char **p_oname, int *p_err); - public int iread (int fd, char *buf, unsigned int len); - public void intread (void); - public time_type get_time (void); - public char * errno_message (char *filename); - public int percentage (POSITION num, POSITION den); - public POSITION percent_pos (POSITION pos, int percent, long fraction); - public int os9_signal (int type, RETSIGTYPE (*handler)()); - public void put_line (void); - public void flush (void); - public int putchr (int c); - public void putstr (constant char *s); - public void get_return (void); - public void error (char *fmt, PARG *parg); - public void ierror (char *fmt, PARG *parg); - public int query (char *fmt, PARG *parg); - public int compile_pattern (char *pattern, int search_type, void **comp_pattern); - public void uncompile_pattern (void **pattern); - public int valid_pattern (char *pattern); - public int is_null_pattern (void *pattern); - public int match_pattern (void *pattern, char *tpattern, char *line, int line_len, char **sp, char **ep, int notbol, int search_type); - public POSITION position (int where); - public void add_forw_pos (POSITION pos); - public void add_back_pos (POSITION pos); - public void pos_clear (void); - public void pos_init (void); - public int onscreen (POSITION pos); - public int empty_screen (void); - public int empty_lines (int s, int e); - public void get_scrpos (struct scrpos *scrpos); - public int adjsline (int sline); - public void init_prompt (void); - public char * pr_expand (constant char *proto, int maxwidth); - public char * eq_message (void); - public char * pr_string (void); - public char * wait_message (void); - public void init_search (void); - public void repaint_hilite (int on); - public void clear_attn (void); - public void undo_search (void); -struct hilite_tree; - public void clr_hlist (struct hilite_tree *anchor); - public void clr_hilite (void); - public void clr_filter (void); - public int is_filtered (POSITION pos); - public POSITION next_unfiltered (POSITION pos); - public POSITION prev_unfiltered (POSITION pos); - public int is_hilited (POSITION pos, POSITION epos, int nohide, int *p_matches); - public void chg_caseless (void); - public void chg_hilite (void); - public int search (int search_type, char *pattern, int n); - public void prep_hilite (POSITION spos, POSITION epos, int maxlines); - public void set_filter_pattern (char *pattern, int search_type); - public int is_filtering (void); - public RETSIGTYPE winch (int type); - public void init_signals (int on); - public void psignals (void); - public void cleantags (void); - public int gettagtype (void); - public void findtag (char *tag); - public POSITION tagsearch (void); - public char * nexttag (int n); - public char * prevtag (int n); - public int ntags (void); - public int curr_tag (void); - public int edit_tagfile (void); - public void open_getchr (void); - public void close_getchr (void); - public int getchr (void); +public char * save LESSPARAMS((constant char *s)); +public VOID_POINTER ecalloc LESSPARAMS((int count, unsigned int size)); +public char * skipsp LESSPARAMS((char *s)); +public int sprefix LESSPARAMS((char *ps, char *s, int uppercase)); +public void quit LESSPARAMS((int status)); +public void raw_mode LESSPARAMS((int on)); +public void scrsize LESSPARAMS((VOID_PARAM)); +public char * special_key_str LESSPARAMS((int key)); +public void get_term LESSPARAMS((VOID_PARAM)); +public void init LESSPARAMS((VOID_PARAM)); +public void deinit LESSPARAMS((VOID_PARAM)); +public void home LESSPARAMS((VOID_PARAM)); +public void add_line LESSPARAMS((VOID_PARAM)); +public void remove_top LESSPARAMS((int n)); +public void win32_scroll_up LESSPARAMS((int n)); +public void lower_left LESSPARAMS((VOID_PARAM)); +public void line_left LESSPARAMS((VOID_PARAM)); +public void check_winch LESSPARAMS((VOID_PARAM)); +public void goto_line LESSPARAMS((int slinenum)); +public void vbell LESSPARAMS((VOID_PARAM)); +public void bell LESSPARAMS((VOID_PARAM)); +public void clear LESSPARAMS((VOID_PARAM)); +public void clear_eol LESSPARAMS((VOID_PARAM)); +public void clear_bot LESSPARAMS((VOID_PARAM)); +public void at_enter LESSPARAMS((int attr)); +public void at_exit LESSPARAMS((VOID_PARAM)); +public void at_switch LESSPARAMS((int attr)); +public int is_at_equiv LESSPARAMS((int attr1, int attr2)); +public int apply_at_specials LESSPARAMS((int attr)); +public void backspace LESSPARAMS((VOID_PARAM)); +public void putbs LESSPARAMS((VOID_PARAM)); +public char WIN32getch LESSPARAMS((int tty)); +public void WIN32setcolors LESSPARAMS((int fg, int bg)); +public void WIN32textout LESSPARAMS((char *text, int len)); +public void match_brac LESSPARAMS((int obrac, int cbrac, int forwdir, int n)); +public void ch_ungetchar LESSPARAMS((int c)); +public void end_logfile LESSPARAMS((VOID_PARAM)); +public void sync_logfile LESSPARAMS((VOID_PARAM)); +public int ch_seek LESSPARAMS((POSITION pos)); +public int ch_end_seek LESSPARAMS((VOID_PARAM)); +public int ch_end_buffer_seek LESSPARAMS((VOID_PARAM)); +public int ch_beg_seek LESSPARAMS((VOID_PARAM)); +public POSITION ch_length LESSPARAMS((VOID_PARAM)); +public POSITION ch_tell LESSPARAMS((VOID_PARAM)); +public int ch_forw_get LESSPARAMS((VOID_PARAM)); +public int ch_back_get LESSPARAMS((VOID_PARAM)); +public void ch_setbufspace LESSPARAMS((int bufspace)); +public void ch_flush LESSPARAMS((VOID_PARAM)); +public int seekable LESSPARAMS((int f)); +public void ch_set_eof LESSPARAMS((VOID_PARAM)); +public void ch_init LESSPARAMS((int f, int flags)); +public void ch_close LESSPARAMS((VOID_PARAM)); +public int ch_getflags LESSPARAMS((VOID_PARAM)); +public void ch_dump LESSPARAMS((VOID_PARAM)); +public void init_charset LESSPARAMS((VOID_PARAM)); +public int binary_char LESSPARAMS((LWCHAR c)); +public int control_char LESSPARAMS((LWCHAR c)); +public char * prchar LESSPARAMS((LWCHAR c)); +public char * prutfchar LESSPARAMS((LWCHAR ch)); +public int utf_len LESSPARAMS((unsigned char ch)); +public int is_utf8_well_formed LESSPARAMS((char *ss, int slen)); +public int utf_bin_count LESSPARAMS((char *data, int len)); +public LWCHAR get_wchar LESSPARAMS((constant char *p)); +public void put_wchar LESSPARAMS((char **pp, LWCHAR ch)); +public LWCHAR step_char LESSPARAMS((char **pp, signed int dir, constant char *limit)); +public int is_composing_char LESSPARAMS((LWCHAR ch)); +public int is_ubin_char LESSPARAMS((LWCHAR ch)); +public int is_wide_char LESSPARAMS((LWCHAR ch)); +public int is_combining_char LESSPARAMS((LWCHAR ch1, LWCHAR ch2)); +public void cmd_reset LESSPARAMS((VOID_PARAM)); +public void clear_cmd LESSPARAMS((VOID_PARAM)); +public void cmd_putstr LESSPARAMS((constant char *s)); +public int len_cmdbuf LESSPARAMS((VOID_PARAM)); +public void set_mlist LESSPARAMS((void *mlist, int cmdflags)); +public void cmd_addhist LESSPARAMS((struct mlist *mlist, constant char *cmd, int modified)); +public void cmd_accept LESSPARAMS((VOID_PARAM)); +public int cmd_char LESSPARAMS((int c)); +public LINENUM cmd_int LESSPARAMS((long *frac)); +public char * get_cmdbuf LESSPARAMS((VOID_PARAM)); +public char * cmd_lastpattern LESSPARAMS((VOID_PARAM)); +public void init_cmdhist LESSPARAMS((VOID_PARAM)); +public void save_cmdhist LESSPARAMS((VOID_PARAM)); +public int in_mca LESSPARAMS((VOID_PARAM)); +public void dispversion LESSPARAMS((VOID_PARAM)); +public int getcc LESSPARAMS((VOID_PARAM)); +public void ungetcc LESSPARAMS((int c)); +public void ungetsc LESSPARAMS((char *s)); +public void commands LESSPARAMS((VOID_PARAM)); +public int cvt_length LESSPARAMS((int len, int ops)); +public int * cvt_alloc_chpos LESSPARAMS((int len)); +public void cvt_text LESSPARAMS((char *odst, char *osrc, int *chpos, int *lenp, int ops)); +public void init_cmds LESSPARAMS((VOID_PARAM)); +public void add_fcmd_table LESSPARAMS((char *buf, int len)); +public void add_ecmd_table LESSPARAMS((char *buf, int len)); +public int fcmd_decode LESSPARAMS((char *cmd, char **sp)); +public int ecmd_decode LESSPARAMS((char *cmd, char **sp)); +public char * lgetenv LESSPARAMS((char *var)); +public int lesskey LESSPARAMS((char *filename, int sysvar)); +public void add_hometable LESSPARAMS((char *envname, char *def_filename, int sysvar)); +public int editchar LESSPARAMS((int c, int flags)); +public void init_textlist LESSPARAMS((struct textlist *tlist, char *str)); +public char * forw_textlist LESSPARAMS((struct textlist *tlist, char *prev)); +public char * back_textlist LESSPARAMS((struct textlist *tlist, char *prev)); +public int edit LESSPARAMS((char *filename)); +public int edit_ifile LESSPARAMS((IFILE ifile)); +public int edit_list LESSPARAMS((char *filelist)); +public int edit_first LESSPARAMS((VOID_PARAM)); +public int edit_last LESSPARAMS((VOID_PARAM)); +public int edit_next LESSPARAMS((int n)); +public int edit_prev LESSPARAMS((int n)); +public int edit_index LESSPARAMS((int n)); +public IFILE save_curr_ifile LESSPARAMS((VOID_PARAM)); +public void unsave_ifile LESSPARAMS((IFILE save_ifile)); +public void reedit_ifile LESSPARAMS((IFILE save_ifile)); +public void reopen_curr_ifile LESSPARAMS((VOID_PARAM)); +public int edit_stdin LESSPARAMS((VOID_PARAM)); +public void cat_file LESSPARAMS((VOID_PARAM)); +public void use_logfile LESSPARAMS((char *filename)); +public char * shell_unquote LESSPARAMS((char *str)); +public char * get_meta_escape LESSPARAMS((VOID_PARAM)); +public char * shell_quote LESSPARAMS((char *s)); +public char * homefile LESSPARAMS((char *filename)); +public char * fexpand LESSPARAMS((char *s)); +public char * fcomplete LESSPARAMS((char *s)); +public int bin_file LESSPARAMS((int f)); +public char * lglob LESSPARAMS((char *filename)); +public char * open_altfile LESSPARAMS((char *filename, int *pf, void **pfd)); +public void close_altfile LESSPARAMS((char *altfilename, char *filename, void *pipefd)); +public int is_dir LESSPARAMS((char *filename)); +public char * bad_file LESSPARAMS((char *filename)); +public POSITION filesize LESSPARAMS((int f)); +public char * shell_coption LESSPARAMS((VOID_PARAM)); +public char * last_component LESSPARAMS((char *name)); +public int eof_displayed LESSPARAMS((VOID_PARAM)); +public int entire_file_displayed LESSPARAMS((VOID_PARAM)); +public void squish_check LESSPARAMS((VOID_PARAM)); +public void forw LESSPARAMS((int n, POSITION pos, int force, int only_last, int nblank)); +public void back LESSPARAMS((int n, POSITION pos, int force, int only_last)); +public void forward LESSPARAMS((int n, int force, int only_last)); +public void backward LESSPARAMS((int n, int force, int only_last)); +public int get_back_scroll LESSPARAMS((VOID_PARAM)); +public int get_line_count LESSPARAMS((VOID_PARAM)); +public void del_ifile LESSPARAMS((IFILE h)); +public IFILE next_ifile LESSPARAMS((IFILE h)); +public IFILE prev_ifile LESSPARAMS((IFILE h)); +public IFILE getoff_ifile LESSPARAMS((IFILE ifile)); +public int nifile LESSPARAMS((VOID_PARAM)); +public IFILE get_ifile LESSPARAMS((char *filename, IFILE prev)); +public char * get_filename LESSPARAMS((IFILE ifile)); +public int get_index LESSPARAMS((IFILE ifile)); +public void store_pos LESSPARAMS((IFILE ifile, struct scrpos *scrpos)); +public void get_pos LESSPARAMS((IFILE ifile, struct scrpos *scrpos)); +public void set_open LESSPARAMS((IFILE ifile)); +public int opened LESSPARAMS((IFILE ifile)); +public void hold_ifile LESSPARAMS((IFILE ifile, int incr)); +public int held_ifile LESSPARAMS((IFILE ifile)); +public void * get_filestate LESSPARAMS((IFILE ifile)); +public void set_filestate LESSPARAMS((IFILE ifile, void *filestate)); +public void if_dump LESSPARAMS((VOID_PARAM)); +public POSITION forw_line LESSPARAMS((POSITION curr_pos)); +public POSITION back_line LESSPARAMS((POSITION curr_pos)); +public void set_attnpos LESSPARAMS((POSITION pos)); +public void jump_forw LESSPARAMS((VOID_PARAM)); +public void jump_forw_buffered LESSPARAMS((VOID_PARAM)); +public void jump_back LESSPARAMS((LINENUM linenum)); +public void repaint LESSPARAMS((VOID_PARAM)); +public void jump_percent LESSPARAMS((int percent, long fraction)); +public void jump_line_loc LESSPARAMS((POSITION pos, int sline)); +public void jump_loc LESSPARAMS((POSITION pos, int sline)); +public void init_line LESSPARAMS((VOID_PARAM)); +public int is_ascii_char LESSPARAMS((LWCHAR ch)); +public void prewind LESSPARAMS((VOID_PARAM)); +public void plinenum LESSPARAMS((POSITION pos)); +public void pshift_all LESSPARAMS((VOID_PARAM)); +public int is_ansi_end LESSPARAMS((LWCHAR ch)); +public int is_ansi_middle LESSPARAMS((LWCHAR ch)); +public int pappend LESSPARAMS((unsigned char c, POSITION pos)); +public int pflushmbc LESSPARAMS((VOID_PARAM)); +public void pdone LESSPARAMS((int endline, int forw)); +public void set_status_col LESSPARAMS((char c)); +public int gline LESSPARAMS((int i, int *ap)); +public void null_line LESSPARAMS((VOID_PARAM)); +public POSITION forw_raw_line LESSPARAMS((POSITION curr_pos, char **linep, int *line_lenp)); +public POSITION back_raw_line LESSPARAMS((POSITION curr_pos, char **linep, int *line_lenp)); +public int rrshift LESSPARAMS((VOID_PARAM)); +public void clr_linenum LESSPARAMS((VOID_PARAM)); +public void add_lnum LESSPARAMS((LINENUM linenum, POSITION pos)); +public LINENUM find_linenum LESSPARAMS((POSITION pos)); +public POSITION find_pos LESSPARAMS((LINENUM linenum)); +public LINENUM currline LESSPARAMS((int where)); +public void lsystem LESSPARAMS((char *cmd, char *donemsg)); +public int pipe_mark LESSPARAMS((int c, char *cmd)); +public int pipe_data LESSPARAMS((char *cmd, POSITION spos, POSITION epos)); +public void init_mark LESSPARAMS((VOID_PARAM)); +public int badmark LESSPARAMS((int c)); +public void setmark LESSPARAMS((int c)); +public void lastmark LESSPARAMS((VOID_PARAM)); +public void gomark LESSPARAMS((int c)); +public POSITION markpos LESSPARAMS((int c)); +public void unmark LESSPARAMS((IFILE ifile)); +public void opt_o LESSPARAMS((int type, char *s)); +public void opt__O LESSPARAMS((int type, char *s)); +public void opt_j LESSPARAMS((int type, char *s)); +public void calc_jump_sline LESSPARAMS((VOID_PARAM)); +public void opt_shift LESSPARAMS((int type, char *s)); +public void calc_shift_count LESSPARAMS((VOID_PARAM)); +public void opt_k LESSPARAMS((int type, char *s)); +public void opt_t LESSPARAMS((int type, char *s)); +public void opt__T LESSPARAMS((int type, char *s)); +public void opt_p LESSPARAMS((int type, char *s)); +public void opt__P LESSPARAMS((int type, char *s)); +public void opt_b LESSPARAMS((int type, char *s)); +public void opt_i LESSPARAMS((int type, char *s)); +public void opt__V LESSPARAMS((int type, char *s)); +public void opt_D LESSPARAMS((int type, char *s)); +public void opt_x LESSPARAMS((int type, char *s)); +public void opt_quote LESSPARAMS((int type, char *s)); +public void opt_query LESSPARAMS((int type, char *s)); +public int get_swindow LESSPARAMS((VOID_PARAM)); +public char * propt LESSPARAMS((int c)); +public void scan_option LESSPARAMS((char *s)); +public void toggle_option LESSPARAMS((struct loption *o, int lower, char *s, int how_toggle)); +public int opt_has_param LESSPARAMS((struct loption *o)); +public char * opt_prompt LESSPARAMS((struct loption *o)); +public int isoptpending LESSPARAMS((VOID_PARAM)); +public void nopendopt LESSPARAMS((VOID_PARAM)); +public int getnum LESSPARAMS((char **sp, char *printopt, int *errp)); +public long getfraction LESSPARAMS((char **sp, char *printopt, int *errp)); +public int get_quit_at_eof LESSPARAMS((VOID_PARAM)); +public void init_option LESSPARAMS((VOID_PARAM)); +public struct loption * findopt LESSPARAMS((int c)); +public struct loption * findopt_name LESSPARAMS((char **p_optname, char **p_oname, int *p_err)); +public int iread LESSPARAMS((int fd, unsigned char *buf, unsigned int len)); +public void intread LESSPARAMS((VOID_PARAM)); +public time_type get_time LESSPARAMS((VOID_PARAM)); +public char * errno_message LESSPARAMS((char *filename)); +public int percentage LESSPARAMS((POSITION num, POSITION den)); +public POSITION percent_pos LESSPARAMS((POSITION pos, int percent, long fraction)); +public int os9_signal LESSPARAMS((int type, RETSIGTYPE (*handler)())); +public void put_line LESSPARAMS((VOID_PARAM)); +public void flush LESSPARAMS((VOID_PARAM)); +public int putchr LESSPARAMS((int c)); +public void putstr LESSPARAMS((constant char *s)); +public void get_return LESSPARAMS((VOID_PARAM)); +public void error LESSPARAMS((char *fmt, PARG *parg)); +public void ierror LESSPARAMS((char *fmt, PARG *parg)); +public int query LESSPARAMS((char *fmt, PARG *parg)); +public int compile_pattern LESSPARAMS((char *pattern, int search_type, PATTERN_TYPE *comp_pattern)); +public void uncompile_pattern LESSPARAMS((PATTERN_TYPE *pattern)); +public int valid_pattern LESSPARAMS((char *pattern)); +public int is_null_pattern LESSPARAMS((PATTERN_TYPE pattern)); +public int match_pattern LESSPARAMS((PATTERN_TYPE pattern, char *tpattern, char *line, int line_len, char **sp, char **ep, int notbol, int search_type)); +public POSITION position LESSPARAMS((int where)); +public void add_forw_pos LESSPARAMS((POSITION pos)); +public void add_back_pos LESSPARAMS((POSITION pos)); +public void pos_clear LESSPARAMS((VOID_PARAM)); +public void pos_init LESSPARAMS((VOID_PARAM)); +public int onscreen LESSPARAMS((POSITION pos)); +public int empty_screen LESSPARAMS((VOID_PARAM)); +public int empty_lines LESSPARAMS((int s, int e)); +public void get_scrpos LESSPARAMS((struct scrpos *scrpos)); +public int adjsline LESSPARAMS((int sline)); +public void init_prompt LESSPARAMS((VOID_PARAM)); +public char * pr_expand LESSPARAMS((constant char *proto, int maxwidth)); +public char * eq_message LESSPARAMS((VOID_PARAM)); +public char * pr_string LESSPARAMS((VOID_PARAM)); +public char * wait_message LESSPARAMS((VOID_PARAM)); +public void init_search LESSPARAMS((VOID_PARAM)); +public void repaint_hilite LESSPARAMS((int on)); +public void clear_attn LESSPARAMS((VOID_PARAM)); +public void undo_search LESSPARAMS((VOID_PARAM)); +public void clr_hlist LESSPARAMS((struct hilite_tree *anchor)); +public void clr_hilite LESSPARAMS((VOID_PARAM)); +public void clr_filter LESSPARAMS((VOID_PARAM)); +public int is_filtered LESSPARAMS((POSITION pos)); +public POSITION next_unfiltered LESSPARAMS((POSITION pos)); +public POSITION prev_unfiltered LESSPARAMS((POSITION pos)); +public int is_hilited LESSPARAMS((POSITION pos, POSITION epos, int nohide, int *p_matches)); +public void chg_hilite LESSPARAMS((VOID_PARAM)); +public void chg_caseless LESSPARAMS((VOID_PARAM)); +public int search LESSPARAMS((int search_type, char *pattern, int n)); +public void prep_hilite LESSPARAMS((POSITION spos, POSITION epos, int maxlines)); +public void set_filter_pattern LESSPARAMS((char *pattern, int search_type)); +public int is_filtering LESSPARAMS((VOID_PARAM)); +public RETSIGTYPE winch LESSPARAMS((int type)); +public RETSIGTYPE winch LESSPARAMS((int type)); +public void init_signals LESSPARAMS((int on)); +public void psignals LESSPARAMS((VOID_PARAM)); +public void cleantags LESSPARAMS((VOID_PARAM)); +public int gettagtype LESSPARAMS((VOID_PARAM)); +public void findtag LESSPARAMS((char *tag)); +public POSITION tagsearch LESSPARAMS((VOID_PARAM)); +public char * nexttag LESSPARAMS((int n)); +public char * prevtag LESSPARAMS((int n)); +public int ntags LESSPARAMS((VOID_PARAM)); +public int curr_tag LESSPARAMS((VOID_PARAM)); +public int edit_tagfile LESSPARAMS((VOID_PARAM)); +public void open_getchr LESSPARAMS((VOID_PARAM)); +public void close_getchr LESSPARAMS((VOID_PARAM)); +public int getchr LESSPARAMS((VOID_PARAM)); diff --git a/contrib/less/help.c b/contrib/less/help.c index aba711677f7aa..3b0d1bd94f72a 100644 --- a/contrib/less/help.c +++ b/contrib/less/help.c @@ -23,8 +23,10 @@ constant char helpdata[] = { ' ',' ','E','S','C','-','S','P','A','C','E',' ',' ',' ',' ',' ',' ',' ',' ',' ','*',' ',' ','F','o','r','w','a','r','d',' ',' ','o','n','e',' ','w','i','n','d','o','w',',',' ','b','u','t',' ','d','o','n','\'','t',' ','s','t','o','p',' ','a','t',' ','e','n','d','-','o','f','-','f','i','l','e','.','\n', ' ',' ','d',' ',' ','^','D',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','*',' ',' ','F','o','r','w','a','r','d',' ',' ','o','n','e',' ','h','a','l','f','-','w','i','n','d','o','w',' ','(','a','n','d',' ','s','e','t',' ','h','a','l','f','-','w','i','n','d','o','w',' ','t','o',' ','_','\b','N',')','.','\n', ' ',' ','u',' ',' ','^','U',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','*',' ',' ','B','a','c','k','w','a','r','d',' ','o','n','e',' ','h','a','l','f','-','w','i','n','d','o','w',' ','(','a','n','d',' ','s','e','t',' ','h','a','l','f','-','w','i','n','d','o','w',' ','t','o',' ','_','\b','N',')','.','\n', -' ',' ','E','S','C','-',')',' ',' ','R','i','g','h','t','A','r','r','o','w',' ','*',' ',' ','L','e','f','t',' ',' ','o','n','e',' ','h','a','l','f',' ','s','c','r','e','e','n',' ','w','i','d','t','h',' ','(','o','r',' ','_','\b','N',' ','p','o','s','i','t','i','o','n','s',')','.','\n', -' ',' ','E','S','C','-','(',' ',' ','L','e','f','t','A','r','r','o','w',' ',' ','*',' ',' ','R','i','g','h','t',' ','o','n','e',' ','h','a','l','f',' ','s','c','r','e','e','n',' ','w','i','d','t','h',' ','(','o','r',' ','_','\b','N',' ','p','o','s','i','t','i','o','n','s',')','.','\n', +' ',' ','E','S','C','-',')',' ',' ','R','i','g','h','t','A','r','r','o','w',' ','*',' ',' ','R','i','g','h','t',' ','o','n','e',' ','h','a','l','f',' ','s','c','r','e','e','n',' ','w','i','d','t','h',' ','(','o','r',' ','_','\b','N',' ','p','o','s','i','t','i','o','n','s',')','.','\n', +' ',' ','E','S','C','-','(',' ',' ','L','e','f','t','A','r','r','o','w',' ',' ','*',' ',' ','L','e','f','t',' ',' ','o','n','e',' ','h','a','l','f',' ','s','c','r','e','e','n',' ','w','i','d','t','h',' ','(','o','r',' ','_','\b','N',' ','p','o','s','i','t','i','o','n','s',')','.','\n', +' ',' ','E','S','C','-','}',' ',' ','^','R','i','g','h','t','A','r','r','o','w',' ',' ',' ','R','i','g','h','t',' ','t','o',' ','l','a','s','t',' ','c','o','l','u','m','n',' ','d','i','s','p','l','a','y','e','d','.','\n', +' ',' ','E','S','C','-','{',' ',' ','^','L','e','f','t','A','r','r','o','w',' ',' ',' ',' ','L','e','f','t',' ',' ','t','o',' ','f','i','r','s','t',' ','c','o','l','u','m','n','.','\n', ' ',' ','F',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','F','o','r','w','a','r','d',' ','f','o','r','e','v','e','r',';',' ','l','i','k','e',' ','"','t','a','i','l',' ','-','f','"','.','\n', ' ',' ','E','S','C','-','F',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','L','i','k','e',' ','F',' ','b','u','t',' ','s','t','o','p',' ','w','h','e','n',' ','s','e','a','r','c','h',' ','p','a','t','t','e','r','n',' ','i','s',' ','f','o','u','n','d','.','\n', ' ',' ','r',' ',' ','^','R',' ',' ','^','L',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','R','e','p','a','i','n','t',' ','s','c','r','e','e','n','.','\n', @@ -102,6 +104,7 @@ constant char helpdata[] = { '\n', ' ',' ','!','_','\b','c','_','\b','o','_','\b','m','_','\b','m','_','\b','a','_','\b','n','_','\b','d',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','E','x','e','c','u','t','e',' ','t','h','e',' ','s','h','e','l','l',' ','c','o','m','m','a','n','d',' ','w','i','t','h',' ','$','S','H','E','L','L','.','\n', ' ',' ','|','X','\b','X','_','\b','c','_','\b','o','_','\b','m','_','\b','m','_','\b','a','_','\b','n','_','\b','d',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','P','i','p','e',' ','f','i','l','e',' ','b','e','t','w','e','e','n',' ','c','u','r','r','e','n','t',' ','p','o','s',' ','&',' ','m','a','r','k',' ','X','\b','X',' ','t','o',' ','s','h','e','l','l',' ','c','o','m','m','a','n','d','.','\n', +' ',' ','s',' ','_','\b','f','_','\b','i','_','\b','l','_','\b','e',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','S','a','v','e',' ','i','n','p','u','t',' ','t','o',' ','a',' ','f','i','l','e','.','\n', ' ',' ','v',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','E','d','i','t',' ','t','h','e',' ','c','u','r','r','e','n','t',' ','f','i','l','e',' ','w','i','t','h',' ','$','V','I','S','U','A','L',' ','o','r',' ','$','E','D','I','T','O','R','.','\n', ' ',' ','V',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ',' ','P','r','i','n','t',' ','v','e','r','s','i','o','n',' ','n','u','m','b','e','r',' ','o','f',' ','"','l','e','s','s','"','.','\n', ' ','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','-','\n', diff --git a/contrib/less/ifile.c b/contrib/less/ifile.c index 0d921558444c9..53a280c406c4b 100644 --- a/contrib/less/ifile.c +++ b/contrib/less/ifile.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -49,7 +49,9 @@ static struct ifile anchor = { &anchor, &anchor, NULL, NULL, 0, 0, '\0', static int ifiles = 0; static void -incr_index(struct ifile *p, int incr) +incr_index(p, incr) + struct ifile *p; + int incr; { for (; p != &anchor; p = p->h_next) p->h_index += incr; @@ -59,7 +61,9 @@ incr_index(struct ifile *p, int incr) * Link an ifile into the ifile list. */ static void -link_ifile(struct ifile *p, struct ifile *prev) +link_ifile(p, prev) + struct ifile *p; + struct ifile *prev; { /* * Link into list. @@ -83,7 +87,8 @@ link_ifile(struct ifile *p, struct ifile *prev) * Unlink an ifile from the ifile list. */ static void -unlink_ifile(struct ifile *p) +unlink_ifile(p) + struct ifile *p; { p->h_next->h_prev = p->h_prev; p->h_prev->h_next = p->h_next; @@ -98,7 +103,9 @@ unlink_ifile(struct ifile *p) * Return a pointer to the new ifile structure. */ static struct ifile * -new_ifile(char *filename, struct ifile *prev) +new_ifile(filename, prev) + char *filename; + struct ifile *prev; { struct ifile *p; @@ -119,7 +126,8 @@ new_ifile(char *filename, struct ifile *prev) * Delete an existing ifile structure. */ public void -del_ifile(IFILE h) +del_ifile(h) + IFILE h; { struct ifile *p; @@ -142,7 +150,8 @@ del_ifile(IFILE h) * Get the ifile after a given one in the list. */ public IFILE -next_ifile(IFILE h) +next_ifile(h) + IFILE h; { struct ifile *p; @@ -156,7 +165,8 @@ next_ifile(IFILE h) * Get the ifile before a given one in the list. */ public IFILE -prev_ifile(IFILE h) +prev_ifile(h) + IFILE h; { struct ifile *p; @@ -170,7 +180,8 @@ prev_ifile(IFILE h) * Return a different ifile from the given one. */ public IFILE -getoff_ifile(IFILE ifile) +getoff_ifile(ifile) + IFILE ifile; { IFILE newifile; @@ -185,7 +196,7 @@ getoff_ifile(IFILE ifile) * Return the number of ifiles. */ public int -nifile(void) +nifile() { return (ifiles); } @@ -194,7 +205,8 @@ nifile(void) * Find an ifile structure, given a filename. */ static struct ifile * -find_ifile(char *filename) +find_ifile(filename) + char *filename; { struct ifile *p; @@ -210,7 +222,9 @@ find_ifile(char *filename) * insert the new ifile after "prev" in the list. */ public IFILE -get_ifile(char *filename, IFILE prev) +get_ifile(filename, prev) + char *filename; + IFILE prev; { struct ifile *p; @@ -223,7 +237,8 @@ get_ifile(char *filename, IFILE prev) * Get the filename associated with a ifile. */ public char * -get_filename(IFILE ifile) +get_filename(ifile) + IFILE ifile; { if (ifile == NULL) return (NULL); @@ -234,7 +249,8 @@ get_filename(IFILE ifile) * Get the index of the file associated with a ifile. */ public int -get_index(IFILE ifile) +get_index(ifile) + IFILE ifile; { return (int_ifile(ifile)->h_index); } @@ -243,7 +259,9 @@ get_index(IFILE ifile) * Save the file position to be associated with a given file. */ public void -store_pos(IFILE ifile, struct scrpos *scrpos) +store_pos(ifile, scrpos) + IFILE ifile; + struct scrpos *scrpos; { int_ifile(ifile)->h_scrpos = *scrpos; } @@ -253,7 +271,9 @@ store_pos(IFILE ifile, struct scrpos *scrpos) * If no position has been associated with the file, return NULL_POSITION. */ public void -get_pos(IFILE ifile, struct scrpos *scrpos) +get_pos(ifile, scrpos) + IFILE ifile; + struct scrpos *scrpos; { *scrpos = int_ifile(ifile)->h_scrpos; } @@ -262,7 +282,8 @@ get_pos(IFILE ifile, struct scrpos *scrpos) * Mark the ifile as "opened". */ public void -set_open(IFILE ifile) +set_open(ifile) + IFILE ifile; { int_ifile(ifile)->h_opened = 1; } @@ -271,38 +292,45 @@ set_open(IFILE ifile) * Return whether the ifile has been opened previously. */ public int -opened(IFILE ifile) +opened(ifile) + IFILE ifile; { return (int_ifile(ifile)->h_opened); } public void -hold_ifile(IFILE ifile, int incr) +hold_ifile(ifile, incr) + IFILE ifile; + int incr; { int_ifile(ifile)->h_hold += incr; } public int -held_ifile(IFILE ifile) +held_ifile(ifile) + IFILE ifile; { return (int_ifile(ifile)->h_hold); } public void * -get_filestate(IFILE ifile) +get_filestate(ifile) + IFILE ifile; { return (int_ifile(ifile)->h_filestate); } public void -set_filestate(IFILE ifile, void *filestate) +set_filestate(ifile, filestate) + IFILE ifile; + void *filestate; { int_ifile(ifile)->h_filestate = filestate; } #if 0 public void -if_dump(void) +if_dump() { struct ifile *p; diff --git a/contrib/less/input.c b/contrib/less/input.c index b9e5de931b1b5..34689bf6a5a5e 100644 --- a/contrib/less/input.c +++ b/contrib/less/input.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -42,7 +42,8 @@ extern int size_linebuf; * of the NEXT line. The line obtained is the line starting at curr_pos. */ public POSITION -forw_line(POSITION curr_pos) +forw_line(curr_pos) + POSITION curr_pos; { POSITION base_pos; POSITION new_pos; @@ -248,7 +249,8 @@ get_forw_line: * of the PREVIOUS line. The line obtained is the one starting at new_pos. */ public POSITION -back_line(POSITION curr_pos) +back_line(curr_pos) + POSITION curr_pos; { POSITION new_pos, begin_new_pos, base_pos; int c; @@ -427,7 +429,8 @@ get_back_line: * Set attnpos. */ public void -set_attnpos(POSITION pos) +set_attnpos(pos) + POSITION pos; { int c; diff --git a/contrib/less/jump.c b/contrib/less/jump.c index 16bd22d58036c..618d0b87105b6 100644 --- a/contrib/less/jump.c +++ b/contrib/less/jump.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -26,7 +26,7 @@ extern int top_scroll; * Jump to the end of the file. */ public void -jump_forw(void) +jump_forw() { POSITION pos; POSITION end_pos; @@ -64,7 +64,7 @@ jump_forw(void) * Jump to the last buffered line in the file. */ public void -jump_forw_buffered(void) +jump_forw_buffered() { POSITION end; @@ -82,7 +82,8 @@ jump_forw_buffered(void) * Jump to line n in the file. */ public void -jump_back(LINENUM linenum) +jump_back(linenum) + LINENUM linenum; { POSITION pos; PARG parg; @@ -114,7 +115,7 @@ jump_back(LINENUM linenum) * Repaint the screen. */ public void -repaint(void) +repaint() { struct scrpos scrpos; /* @@ -123,14 +124,20 @@ repaint(void) */ get_scrpos(&scrpos); pos_clear(); - jump_loc(scrpos.pos, scrpos.ln); + if (scrpos.pos == NULL_POSITION) + /* Screen hasn't been drawn yet. */ + jump_loc(0, 0); + else + jump_loc(scrpos.pos, scrpos.ln); } /* * Jump to a specified percentage into the file. */ public void -jump_percent(int percent, long fraction) +jump_percent(percent, fraction) + int percent; + long fraction; { POSITION pos, len; @@ -161,7 +168,9 @@ jump_percent(int percent, long fraction) * the first character in a line. */ public void -jump_line_loc(POSITION pos, int sline) +jump_line_loc(pos, sline) + POSITION pos; + int sline; { int c; @@ -187,7 +196,9 @@ jump_line_loc(POSITION pos, int sline) * Place the target line on a specified line on the screen. */ public void -jump_loc(POSITION pos, int sline) +jump_loc(pos, sline) + POSITION pos; + int sline; { int nline; POSITION tpos; diff --git a/contrib/less/less.h b/contrib/less/less.h index 4123c352d2581..69576bbde4cd0 100644 --- a/contrib/less/less.h +++ b/contrib/less/less.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -39,10 +39,17 @@ /* * Language details. */ +#if HAVE_ANSI_PROTOS +#define LESSPARAMS(a) a +#else +#define LESSPARAMS(a) () +#endif #if HAVE_VOID #define VOID_POINTER void * +#define VOID_PARAM void #else #define VOID_POINTER char * +#define VOID_PARAM #define void int #endif #if HAVE_CONST @@ -525,9 +532,13 @@ struct wchar_range_table #define time_type long #endif +struct mlist; +struct loption; +struct hilite_tree; +#include "pattern.h" #include "funcs.h" /* Functions not included in funcs.h */ -void postoa(POSITION num, char *buf); -void linenumtoa(LINENUM num, char *buf); -void inttoa(int num, char *buf); +void postoa(); +void linenumtoa(); +void inttoa(); diff --git a/contrib/less/less.hlp b/contrib/less/less.hlp index 2a8e3e7233e71..255dc7e0ab2b4 100644 --- a/contrib/less/less.hlp +++ b/contrib/less/less.hlp @@ -20,8 +20,10 @@ ESC-SPACE * Forward one window, but don't stop at end-of-file. d ^D * Forward one half-window (and set half-window to _N). u ^U * Backward one half-window (and set half-window to _N). - ESC-) RightArrow * Left one half screen width (or _N positions). - ESC-( LeftArrow * Right one half screen width (or _N positions). + ESC-) RightArrow * Right one half screen width (or _N positions). + ESC-( LeftArrow * Left one half screen width (or _N positions). + ESC-} ^RightArrow Right to last column displayed. + ESC-{ ^LeftArrow Left to first column. F Forward forever; like "tail -f". ESC-F Like F but stop when search pattern is found. r ^R ^L Repaint screen. @@ -99,6 +101,7 @@ !_c_o_m_m_a_n_d Execute the shell command with $SHELL. |XX_c_o_m_m_a_n_d Pipe file between current pos & mark XX to shell command. + s _f_i_l_e Save input to a file. v Edit the current file with $VISUAL or $EDITOR. V Print version number of "less". --------------------------------------------------------------------------- diff --git a/contrib/less/less.nro b/contrib/less/less.nro index b3d98746aad9f..2f7bb7faa1de1 100644 --- a/contrib/less/less.nro +++ b/contrib/less/less.nro @@ -1,4 +1,4 @@ -.TH LESS 1 "Version 481: 31 Aug 2015" +.TH LESS 1 "Version 491: 07 Apr 2017" .SH NAME less \- opposite of more .SH SYNOPSIS @@ -101,6 +101,10 @@ Scroll horizontally left N characters, default half the screen width (see the \-# option). If a number N is specified, it becomes the default for future RIGHTARROW and LEFTARROW commands. +.IP "ESC-} or ^RIGHTARROW" +Scroll horizontally right to show the end of the longest displayed line. +.IP "ESC-{ or ^LEFTARROW" +Scroll horizontally left back to the first column. .IP "r or ^R or ^L" Repaint the screen. .IP R @@ -547,6 +551,7 @@ The first number selects the foreground color and the second selects the background color of the text. A single number \fIN\fP is the same as \fIN.M\fP, where \fIM\fP is the normal background color. +\fBx\fP may also be \fBa\fP to toggle strict ANSI sequence rendering (SGR mode). .IP "\-e or \-\-quit-at-eof" Causes @@ -1740,7 +1745,7 @@ The name of the editor (used for the v command). lesskey(1) .SH COPYRIGHT -Copyright (C) 1984-2015 Mark Nudelman +Copyright (C) 1984-2017 Mark Nudelman .PP less is part of the GNU project and is free software. You can redistribute it and/or modify it diff --git a/contrib/less/lessecho.c b/contrib/less/lessecho.c index e4a23f5f39c54..4ed4830e16414 100644 --- a/contrib/less/lessecho.c +++ b/contrib/less/lessecho.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -38,14 +38,14 @@ static char metachars[64] = ""; static int num_metachars = 0; static void -pr_usage(void) +pr_usage() { fprintf(stderr, "usage: lessecho [-ox] [-cx] [-pn] [-dn] [-mx] [-nn] [-ex] [-fn] [-a] file ...\n"); } static void -pr_version(void) +pr_version() { char *p; char buf[10]; @@ -61,14 +61,18 @@ pr_version(void) } static void -pr_error(char *s) +pr_error(s) + char *s; { fprintf(stderr, "%s\n", s); exit(1); } static long -lstrtol(char *s, int radix, char **pend) +lstrtol(s, radix, pend) + char *s; + int radix; + char **pend; { int v; int neg = 0; diff --git a/contrib/less/lessecho.nro b/contrib/less/lessecho.nro index 78b893f18d977..f5922fa4ab79a 100644 --- a/contrib/less/lessecho.nro +++ b/contrib/less/lessecho.nro @@ -1,4 +1,4 @@ -.TH LESSECHO 1 "Version 481: 31 Aug 2015" +.TH LESSECHO 1 "Version 491: 07 Apr 2017" .SH NAME lessecho \- expand metacharacters .SH SYNOPSIS diff --git a/contrib/less/lesskey.c b/contrib/less/lesskey.c index 47b765424fbc8..686f6e4ffdada 100644 --- a/contrib/less/lesskey.c +++ b/contrib/less/lesskey.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -104,6 +104,7 @@ struct cmdname cmdnames[] = { "display-flag", A_DISP_OPTION }, { "display-option", A_DISP_OPTION }, { "end", A_GOEND }, + { "end-scroll", A_RRSHIFT }, { "examine", A_EXAMINE }, { "filter", A_FILTER }, { "first-cmd", A_FIRSTCMD }, @@ -130,6 +131,7 @@ struct cmdname cmdnames[] = { "next-file", A_NEXT_FILE }, { "next-tag", A_NEXT_TAG }, { "noaction", A_NOACTION }, + { "no-scroll", A_LLSHIFT }, { "percent", A_PERCENT }, { "pipe", A_PIPE }, { "prev-file", A_PREV_FILE }, @@ -213,19 +215,19 @@ char *outfile = NULL ; int linenum; int errors; -static void lk_error(char *s); - extern char version[]; void -usage(void) +usage() { fprintf(stderr, "usage: lesskey [-o output] [input]\n"); exit(1); } char * -mkpathname(char *dirname, char *filename) +mkpathname(dirname, filename) + char *dirname; + char *filename; { char *pathname; @@ -240,7 +242,8 @@ mkpathname(char *dirname, char *filename) * Figure out the name of a default file (in the user's HOME directory). */ char * -homefile(char *filename) +homefile(filename) + char *filename; { char *p; char *pathname; @@ -263,7 +266,9 @@ homefile(char *filename) * Parse command line arguments. */ void -parse_args(int argc, char **argv) +parse_args(argc, argv) + int argc; + char **argv; { char *arg; @@ -336,7 +341,7 @@ parse_args(int argc, char **argv) * Initialize data structures. */ void -init_tables(void) +init_tables() { cmdtable.names = cmdnames; cmdtable.pbuffer = cmdtable.buffer; @@ -352,7 +357,9 @@ init_tables(void) * Parse one character of a string. */ char * -tstr(char **pp, int xlate) +tstr(pp, xlate) + char **pp; + int xlate; { char *p; char ch; @@ -416,7 +423,7 @@ tstr(char **pp, int xlate) case 'e': ch = SK_END; break; case 'x': ch = SK_DELETE; break; default: - lk_error("illegal char after \\k"); + error("illegal char after \\k", NULL_PARG); *pp = p+1; return (""); } @@ -466,7 +473,8 @@ tstr(char **pp, int xlate) * Skip leading spaces in a string. */ public char * -skipsp(char *s) +skipsp(s) + char *s; { while (*s == ' ' || *s == '\t') s++; @@ -477,7 +485,8 @@ skipsp(char *s) * Skip non-space characters in a string. */ public char * -skipnsp(char *s) +skipnsp(s) + char *s; { while (*s != '\0' && *s != ' ' && *s != '\t') s++; @@ -489,7 +498,8 @@ skipnsp(char *s) * strip off the trailing newline & any trailing # comment. */ char * -clean_line(char *s) +clean_line(s) + char *s; { int i; @@ -505,11 +515,12 @@ clean_line(char *s) * Add a byte to the output command table. */ void -add_cmd_char(int c) +add_cmd_char(c) + int c; { if (currtable->pbuffer >= currtable->buffer + MAX_USERCMD) { - lk_error("too many commands"); + error("too many commands", NULL_PARG); exit(1); } *(currtable->pbuffer)++ = c; @@ -519,7 +530,8 @@ add_cmd_char(int c) * Add a string to the output command table. */ void -add_cmd_str(char *s) +add_cmd_str(s) + char *s; { for ( ; *s != '\0'; s++) add_cmd_char(*s); @@ -529,7 +541,8 @@ add_cmd_str(char *s) * See if we have a special "control" line. */ int -control_line(char *s) +control_line(s) + char *s; { #define PREFIX(str,pat) (strncmp(str,pat,strlen(pat)) == 0) @@ -561,7 +574,10 @@ control_line(char *s) * Output some bytes. */ void -fputbytes(FILE *fd, char *buf, int len) +fputbytes(fd, buf, len) + FILE *fd; + char *buf; + int len; { while (len-- > 0) { @@ -574,7 +590,9 @@ fputbytes(FILE *fd, char *buf, int len) * Output an integer, in special KRADIX form. */ void -fputint(FILE *fd, unsigned int val) +fputint(fd, val) + FILE *fd; + unsigned int val; { char c; @@ -594,27 +612,32 @@ fputint(FILE *fd, unsigned int val) * Find an action, given the name of the action. */ int -findaction(char *actname) +findaction(actname) + char *actname; { int i; for (i = 0; currtable->names[i].cn_name != NULL; i++) if (strcmp(currtable->names[i].cn_name, actname) == 0) return (currtable->names[i].cn_action); - lk_error("unknown action"); + error("unknown action", NULL_PARG); return (A_INVALID); } - static void -lk_error(char *s) + void +error(s, parg) + char *s; + PARG *parg; { fprintf(stderr, "line %d: %s\n", linenum, s); errors++; + (void) parg; } void -parse_cmdline(char *p) +parse_cmdline(p) + char *p; { int cmdlen; char *actname; @@ -631,7 +654,7 @@ parse_cmdline(char *p) s = tstr(&p, 1); cmdlen += (int) strlen(s); if (cmdlen > MAX_CMDLEN) - lk_error("command too long"); + error("command too long", NULL_PARG); else add_cmd_str(s); } while (*p != ' ' && *p != '\t' && *p != '\0'); @@ -648,7 +671,7 @@ parse_cmdline(char *p) p = skipsp(p); if (*p == '\0') { - lk_error("missing action"); + error("missing action", NULL_PARG); return; } actname = p; @@ -683,7 +706,8 @@ parse_cmdline(char *p) } void -parse_varline(char *p) +parse_varline(p) + char *p; { char *s; @@ -700,7 +724,7 @@ parse_varline(char *p) p = skipsp(p); if (*p++ != '=') { - lk_error("missing ="); + error("missing =", NULL_PARG); return; } @@ -719,7 +743,8 @@ parse_varline(char *p) * Parse a line from the lesskey file. */ void -parse_line(char *line) +parse_line(line) + char *line; { char *p; @@ -744,7 +769,9 @@ parse_line(char *line) } int -main(int argc, char *argv[]) +main(argc, argv) + int argc; + char *argv[]; { FILE *desc; FILE *out; diff --git a/contrib/less/lesskey.h b/contrib/less/lesskey.h index 34b8c1758c7b6..92d4d93af1793 100644 --- a/contrib/less/lesskey.h +++ b/contrib/less/lesskey.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. diff --git a/contrib/less/lesskey.nro b/contrib/less/lesskey.nro index 826332de5eff8..a13be9329c97d 100644 --- a/contrib/less/lesskey.nro +++ b/contrib/less/lesskey.nro @@ -1,4 +1,4 @@ -.TH LESSKEY 1 "Version 481: 31 Aug 2015" +.TH LESSKEY 1 "Version 491: 07 Apr 2017" .SH NAME lesskey \- specify key bindings for less .SH SYNOPSIS @@ -134,7 +134,7 @@ default command keys used by less: \en forw-line e forw-line j forw-line - \ekd forw-line + \ekd forw-line ^E forw-line ^N forw-line k back-line @@ -149,15 +149,15 @@ default command keys used by less: ^D forw-scroll u back-scroll ^U back-scroll - \e40 forw-screen + \e40 forw-screen f forw-screen ^F forw-screen ^V forw-screen - \ekD forw-screen + \ekD forw-screen b back-screen ^B back-screen \eev back-screen - \ekU back-screen + \ekU back-screen z forw-window w back-window \ee\e40 forw-screen-force @@ -169,7 +169,7 @@ default command keys used by less: ^L repaint \eeu undo-hilite g goto-line - \ekh goto-line + \ekh goto-line < goto-line \ee< goto-line p percent @@ -180,6 +180,8 @@ default command keys used by less: \ee) right-scroll \ekl left-scroll \ekr right-scroll + \ee{ no-scroll + \ee} end-scroll { forw-bracket {} } back-bracket {} ( forw-bracket () @@ -191,7 +193,7 @@ default command keys used by less: G goto-end \ee> goto-end > goto-end - \eke goto-end + \eke goto-end \eeG goto-end-buffered = status ^G status @@ -359,7 +361,7 @@ which start with a NUL character (0). This NUL character should be represented as \e340 in a lesskey file. .SH COPYRIGHT -Copyright (C) 1984-2015 Mark Nudelman +Copyright (C) 1984-2017 Mark Nudelman .PP less is part of the GNU project and is free software. You can redistribute it and/or modify it diff --git a/contrib/less/lglob.h b/contrib/less/lglob.h index 87b5fb45cc7c0..0d9fb5b989bce 100644 --- a/contrib/less/lglob.h +++ b/contrib/less/lglob.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. diff --git a/contrib/less/line.c b/contrib/less/line.c index 464afda3b8853..3715d5805af48 100644 --- a/contrib/less/line.c +++ b/contrib/less/line.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -16,6 +16,7 @@ #include "less.h" #include "charset.h" +#include "position.h" static char *linebuf = NULL; /* Buffer which holds the current output line */ static char *attr = NULL; /* Extension of linebuf to hold attributes */ @@ -40,9 +41,9 @@ static POSITION pendpos; static char *end_ansi_chars; static char *mid_ansi_chars; -static int attr_swidth(int); -static int attr_ewidth(int); -static int do_append(LWCHAR, char *, POSITION); +static int attr_swidth(); +static int attr_ewidth(); +static int do_append(); extern int sigs; extern int bs_mode; @@ -70,7 +71,7 @@ static POSITION mbc_pos; * Initialize from environment variables. */ public void -init_line(void) +init_line() { end_ansi_chars = lgetenv("LESSANSIENDCHARS"); if (end_ansi_chars == NULL || *end_ansi_chars == '\0') @@ -89,7 +90,7 @@ init_line(void) * Expand the line buffer. */ static int -expand_linebuf(void) +expand_linebuf() { /* Double the size of the line buffer. */ int new_size = size_linebuf * 2; @@ -137,7 +138,8 @@ expand_linebuf(void) * Is a character ASCII? */ public int -is_ascii_char(LWCHAR ch) +is_ascii_char(ch) + LWCHAR ch; { return (ch <= 0x7F); } @@ -146,7 +148,7 @@ is_ascii_char(LWCHAR ch) * Rewind the line buffer. */ public void -prewind(void) +prewind() { curr = 0; column = 0; @@ -165,7 +167,8 @@ prewind(void) * Insert the line number (of the given position) into the line buffer. */ public void -plinenum(POSITION pos) +plinenum(pos) + POSITION pos; { LINENUM linenum = 0; int i; @@ -213,7 +216,7 @@ plinenum(POSITION pos) sprintf(linebuf+curr, "%*s ", n, buf); n++; /* One space after the line number. */ for (i = 0; i < n; i++) - attr[curr+i] = AT_NORMAL; + attr[curr+i] = AT_BOLD; curr += n; column += n; lmargin += n; @@ -235,7 +238,8 @@ plinenum(POSITION pos) * This means discarding N printable chars at the start of the buffer. */ static void -pshift(int shift) +pshift(shift) + int shift; { LWCHAR prev_ch = 0; unsigned char c; @@ -352,7 +356,7 @@ pshift(int shift) * */ public void -pshift_all(void) +pshift_all() { pshift(column); } @@ -362,7 +366,8 @@ pshift_all(void) * for a given character attribute. */ static int -attr_swidth(int a) +attr_swidth(a) + int a; { int w = 0; @@ -385,7 +390,8 @@ attr_swidth(int a) * for a given character attribute. */ static int -attr_ewidth(int a) +attr_ewidth(a) + int a; { int w = 0; @@ -410,7 +416,10 @@ attr_ewidth(int a) * attribute sequence to be inserted, so this must be taken into account. */ static int -pwidth(LWCHAR ch, int a, LWCHAR prev_ch) +pwidth(ch, a, prev_ch) + LWCHAR ch; + int a; + LWCHAR prev_ch; { int w; @@ -471,10 +480,10 @@ pwidth(LWCHAR ch, int a, LWCHAR prev_ch) * Return 1 if one is found. */ static int -backc(void) +backc() { LWCHAR prev_ch; - constant char *p = linebuf + curr; + char *p = linebuf + curr; LWCHAR ch = step_char(&p, -1, linebuf + lmargin); int width; @@ -499,9 +508,9 @@ backc(void) * Are we currently within a recognized ANSI escape sequence? */ static int -in_ansi_esc_seq(void) +in_ansi_esc_seq() { - constant char *p; + char *p; /* * Search backwards for either an ESC (which means we ARE in a seq); @@ -522,7 +531,8 @@ in_ansi_esc_seq(void) * Is a character the end of an ANSI escape sequence? */ public int -is_ansi_end(LWCHAR ch) +is_ansi_end(ch) + LWCHAR ch; { if (!is_ascii_char(ch)) return (0); @@ -533,7 +543,8 @@ is_ansi_end(LWCHAR ch) * */ public int -is_ansi_middle(LWCHAR ch) +is_ansi_middle(ch) + LWCHAR ch; { if (!is_ascii_char(ch)) return (0); @@ -551,7 +562,11 @@ is_ansi_middle(LWCHAR ch) } while (0) static int -store_char(LWCHAR ch, int a, char *rep, POSITION pos) +store_char(ch, a, rep, pos) + LWCHAR ch; + int a; + char *rep; + POSITION pos; { int w; int replen; @@ -585,7 +600,7 @@ store_char(LWCHAR ch, int a, char *rep, POSITION pos) { if (!is_ansi_end(ch) && !is_ansi_middle(ch)) { /* Remove whole unrecognized sequence. */ - constant char *p = &linebuf[curr]; + char *p = &linebuf[curr]; LWCHAR bch; do { bch = step_char(&p, -1, linebuf); @@ -603,7 +618,7 @@ store_char(LWCHAR ch, int a, char *rep, POSITION pos) } else { - constant char *p = &linebuf[curr]; + char *p = &linebuf[curr]; LWCHAR prev_ch = step_char(&p, -1, linebuf); w = pwidth(ch, a, prev_ch); } @@ -651,7 +666,9 @@ store_char(LWCHAR ch, int a, char *rep, POSITION pos) do { if (store_tab((a),(pos))) return (1); } while (0) static int -store_tab(int attr, POSITION pos) +store_tab(attr, pos) + int attr; + POSITION pos; { int to_tab = column + cshift - lmargin; int i; @@ -680,7 +697,9 @@ store_tab(int attr, POSITION pos) do { if (store_prchar((c), (pos))) return 1; } while (0) static int -store_prchar(LWCHAR c, POSITION pos) +store_prchar(c, pos) + LWCHAR c; + POSITION pos; { char *s; @@ -704,7 +723,8 @@ store_prchar(LWCHAR c, POSITION pos) } static int -flush_mbc_buf(POSITION pos) +flush_mbc_buf(pos) + POSITION pos; { int i; @@ -721,7 +741,9 @@ flush_mbc_buf(POSITION pos) * Returns 0 if ok, 1 if couldn't fit in buffer. */ public int -pappend(unsigned char c, POSITION pos) +pappend(c, pos) + unsigned char c; + POSITION pos; { int r; @@ -824,7 +846,10 @@ pappend(unsigned char c, POSITION pos) } static int -do_append(LWCHAR ch, char *rep, POSITION pos) +do_append(ch, rep, pos) + LWCHAR ch; + char *rep; + POSITION pos; { int a; LWCHAR prev_ch; @@ -961,7 +986,7 @@ do_append(LWCHAR ch, char *rep, POSITION pos) * */ public int -pflushmbc(void) +pflushmbc() { int r = 0; @@ -978,7 +1003,9 @@ pflushmbc(void) * Terminate the line in the line buffer. */ public void -pdone(int endline, int forw) +pdone(endline, forw) + int endline; + int forw; { (void) pflushmbc(); @@ -1055,7 +1082,8 @@ pdone(int endline, int forw) * */ public void -set_status_col(char c) +set_status_col(c) + char c; { linebuf[0] = c; attr[0] = AT_NORMAL|AT_HILITE; @@ -1067,7 +1095,9 @@ set_status_col(char c) * and the character attribute in *ap. */ public int -gline(int i, int *ap) +gline(i, ap) + int i; + int *ap; { if (is_null_line) { @@ -1097,7 +1127,7 @@ gline(int i, int *ap) * Indicate that there is no current line. */ public void -null_line(void) +null_line() { is_null_line = 1; cshift = 0; @@ -1109,7 +1139,10 @@ null_line(void) * {{ This is supposed to be more efficient than forw_line(). }} */ public POSITION -forw_raw_line(POSITION curr_pos, char **linep, int *line_lenp) +forw_raw_line(curr_pos, linep, line_lenp) + POSITION curr_pos; + char **linep; + int *line_lenp; { int n; int c; @@ -1155,7 +1188,10 @@ forw_raw_line(POSITION curr_pos, char **linep, int *line_lenp) * {{ This is supposed to be more efficient than back_line(). }} */ public POSITION -back_raw_line(POSITION curr_pos, char **linep, int *line_lenp) +back_raw_line(curr_pos, linep, line_lenp) + POSITION curr_pos; + char **linep; + int *line_lenp; { int n; int c; @@ -1220,3 +1256,30 @@ back_raw_line(POSITION curr_pos, char **linep, int *line_lenp) *line_lenp = size_linebuf - 1 - n; return (new_pos); } + +/* + * Find the shift necessary to show the end of the longest displayed line. + */ + public int +rrshift() +{ + POSITION pos; + int save_width; + int line; + int longest = 0; + + save_width = sc_width; + sc_width = INT_MAX; + hshift = 0; + pos = position(TOP); + for (line = 0; line < sc_height && pos != NULL_POSITION; line++) + { + pos = forw_line(pos); + if (column > longest) + longest = column; + } + sc_width = save_width; + if (longest < sc_width) + return 0; + return longest - sc_width; +} diff --git a/contrib/less/linenum.c b/contrib/less/linenum.c index 776db34352c53..4e726377e38dd 100644 --- a/contrib/less/linenum.c +++ b/contrib/less/linenum.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -73,7 +73,7 @@ extern int screen_trashed; * Initialize the line number structures. */ public void -clr_linenum(void) +clr_linenum() { struct linenum_info *p; @@ -101,7 +101,8 @@ clr_linenum(void) * Calculate the gap for an entry. */ static void -calcgap(struct linenum_info *p) +calcgap(p) + struct linenum_info *p; { /* * Don't bother to compute a gap for the anchor. @@ -120,7 +121,9 @@ calcgap(struct linenum_info *p) * FIRST character in the specified line. */ public void -add_lnum(LINENUM linenum, POSITION pos) +add_lnum(linenum, pos) + LINENUM linenum; + POSITION pos; { struct linenum_info *p; struct linenum_info *new; @@ -206,7 +209,7 @@ add_lnum(LINENUM linenum, POSITION pos) * line number, print a message to tell the user what we're doing. */ static void -longloopmessage(void) +longloopmessage() { ierror("Calculating line numbers", NULL_PARG); } @@ -217,7 +220,7 @@ static time_type startime; #endif static void -longish(void) +longish() { #if HAVE_TIME if (loopcount >= 0 && ++loopcount > 100) @@ -243,7 +246,7 @@ longish(void) * a lengthy line number calculation. */ static void -abort_long(void) +abort_long() { if (linenums == OPT_ONPLUS) /* @@ -259,7 +262,8 @@ abort_long(void) * Return 0 if we can't figure it out. */ public LINENUM -find_linenum(POSITION pos) +find_linenum(pos) + POSITION pos; { struct linenum_info *p; LINENUM linenum; @@ -373,7 +377,8 @@ find_linenum(POSITION pos) * Return NULL_POSITION if we can't figure it out. */ public POSITION -find_pos(LINENUM linenum) +find_pos(linenum) + LINENUM linenum; { struct linenum_info *p; POSITION cpos; @@ -445,7 +450,8 @@ find_pos(LINENUM linenum) * the "current" line (e.g. TOP, BOTTOM, MIDDLE, etc). */ public LINENUM -currline(int where) +currline(where) + int where; { POSITION pos; POSITION len; diff --git a/contrib/less/lsystem.c b/contrib/less/lsystem.c index 93a5e03ae5904..10911ebe15d40 100644 --- a/contrib/less/lsystem.c +++ b/contrib/less/lsystem.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -38,7 +38,9 @@ extern IFILE curr_ifile; * Like plain "system()", but handles resetting terminal modes, etc. */ public void -lsystem(char *cmd, char *donemsg) +lsystem(cmd, donemsg) + char *cmd; + char *donemsg; { int inp; #if HAVE_SHELL @@ -248,7 +250,9 @@ lsystem(char *cmd, char *donemsg) * the whole current screen is piped. */ public int -pipe_mark(int c, char *cmd) +pipe_mark(c, cmd) + int c; + char *cmd; { POSITION mpos, tpos, bpos; @@ -280,7 +284,10 @@ pipe_mark(int c, char *cmd) * Feed it the file contents between the positions spos and epos. */ public int -pipe_data(char *cmd, POSITION spos, POSITION epos) +pipe_data(cmd, spos, epos) + char *cmd; + POSITION spos; + POSITION epos; { FILE *f; int c; diff --git a/contrib/less/main.c b/contrib/less/main.c index 3c99db6c8f9b5..b4025656b027d 100644 --- a/contrib/less/main.c +++ b/contrib/less/main.c @@ -1,6 +1,6 @@ /* $FreeBSD$ */ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -54,11 +54,13 @@ extern int jump_sline; static char consoleTitle[256]; #endif +public int line_count; extern int less_is_more; extern int missing_cap; extern int know_dumb; extern int no_init; extern int pr_type; +extern int quit_if_one_screen; /* @@ -283,10 +285,25 @@ main(argc, argv) { if (edit_stdin()) /* Edit standard input */ quit(QUIT_ERROR); + if (quit_if_one_screen) + line_count = get_line_count(); } else { if (edit_first()) /* Edit first valid file in cmd line */ quit(QUIT_ERROR); + /* + * In case that we have only one file and -F, have to get a line + * count fot init(). If the line count is less then a height of a term, + * the content of the file is printed out and then less quits. Otherwise + * -F can not be used + */ + if (quit_if_one_screen) + { + if (nifile() == 1) + line_count = get_line_count(); + else /* In case more than one file, -F can not be used */ + quit_if_one_screen = FALSE; + } } init(); @@ -301,7 +318,8 @@ main(argc, argv) * (that is, to a buffer allocated by calloc). */ public char * -save(constant char *s) +save(s) + constant char *s; { char *p; @@ -315,7 +333,9 @@ save(constant char *s) * Like calloc(), but never returns an error (NULL). */ public VOID_POINTER -ecalloc(int count, unsigned int size) +ecalloc(count, size) + int count; + unsigned int size; { VOID_POINTER p; @@ -332,7 +352,8 @@ ecalloc(int count, unsigned int size) * Skip leading spaces in a string. */ public char * -skipsp(char *s) +skipsp(s) + char *s; { while (*s == ' ' || *s == '\t') s++; @@ -345,7 +366,10 @@ skipsp(char *s) * character; the remainder of the first string may be either case. */ public int -sprefix(char *ps, char *s, int uppercase) +sprefix(ps, s, uppercase) + char *ps; + char *s; + int uppercase; { int c; int sc; @@ -375,7 +399,8 @@ sprefix(char *ps, char *s, int uppercase) * Exit the program. */ public void -quit(int status) +quit(status) + int status; { static int save_status; diff --git a/contrib/less/mark.c b/contrib/less/mark.c index 59eda4b2d56e2..12a0b6c159ff8 100644 --- a/contrib/less/mark.c +++ b/contrib/less/mark.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -27,7 +27,7 @@ static struct mark marks[NMARKS]; * Initialize the mark table to show no marks are set. */ public void -init_mark(void) +init_mark() { int i; @@ -39,7 +39,8 @@ init_mark(void) * See if a mark letter is valid (between a and z). */ static struct mark * -getumark(int c) +getumark(c) + int c; { if (c >= 'a' && c <= 'z') return (&marks[c-'a']); @@ -57,7 +58,8 @@ getumark(int c) * or may be constructed on the fly for certain characters like ^, $. */ static struct mark * -getmark(int c) +getmark(c) + int c; { struct mark *m; static struct mark sm; @@ -122,7 +124,8 @@ getmark(int c) * Is a mark letter is invalid? */ public int -badmark(int c) +badmark(c) + int c; { return (getmark(c) == NULL); } @@ -131,7 +134,8 @@ badmark(int c) * Set a user-defined mark. */ public void -setmark(int c) +setmark(c) + int c; { struct mark *m; struct scrpos scrpos; @@ -148,7 +152,7 @@ setmark(int c) * Set lmark (the mark named by the apostrophe). */ public void -lastmark(void) +lastmark() { struct scrpos scrpos; @@ -165,7 +169,8 @@ lastmark(void) * Go to a mark. */ public void -gomark(int c) +gomark(c) + int c; { struct mark *m; struct scrpos scrpos; @@ -212,7 +217,8 @@ gomark(int c) * because it's always the first non-blank line on the screen. */ public POSITION -markpos(int c) +markpos(c) + int c; { struct mark *m; @@ -232,7 +238,8 @@ markpos(int c) * Clear the marks associated with a specified ifile. */ public void -unmark(IFILE ifile) +unmark(ifile) + IFILE ifile; { int i; diff --git a/contrib/less/mkhelp.c b/contrib/less/mkhelp.c index bc66e7d5ff1d6..16275d4bcf130 100644 --- a/contrib/less/mkhelp.c +++ b/contrib/less/mkhelp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -18,7 +18,9 @@ #include <stdio.h> int -main(int argc, char *argv[]) +main(argc, argv) + int argc; + char *argv[]; { int ch; int prevch; diff --git a/contrib/less/mkutable b/contrib/less/mkutable index 803335b924eb3..ff107b51f007c 100755 --- a/contrib/less/mkutable +++ b/contrib/less/mkutable @@ -4,14 +4,14 @@ use strict; my $USAGE = <<__EOF__; usage: mkutable [-n] [-f#] type... [--] [<] UnicodeData.txt -n = take non-matching types - -f = zero-based type field (default 2) + -f = zero-based type field (default 2) __EOF__ use vars qw( $opt_f $opt_n ); use Getopt::Std; my $type_field = 2; -exit (main() ? 1 : 0); +exit (main() ? 0 : 1); sub main { my $date = `date`; @@ -28,48 +28,58 @@ sub main { $types{$arg} = 1; } my %out = ( 'types' => \%types ); - my $last_code = 0; print $header; + my $last_code = 0; while (<>) { chomp; s/#.*//; my @fields = split /;/; next if not @fields; - my $code = hex $fields[0]; + my ($lo_code, $hi_code); + my $codes = $fields[0]; + if ($codes =~ /(\w+)\.\.(\w+)/) { + $lo_code = hex $1; + $hi_code = hex $2; + } else { + $lo_code = $hi_code = hex $fields[0]; + } my $type = $fields[$type_field]; $type =~ s/\s//g; - while (++$last_code < $code) { - output(\%out, $last_code, '?'); + for ($last_code = $lo_code; $last_code <= $hi_code; ++$last_code) { + output(\%out, $last_code, $type); } - output(\%out, $code, $type); } - output(\%out, $last_code+1, '?'); + output(\%out, $last_code); + return 1; } sub output { my ($out, $code, $type) = @_; - my $match = ${${$out}{types}}{$type}; - my $type_change = (not $$out{start_type} or $type ne $$out{start_type}); - $match = not $match if $opt_n; - if ($match and (not $$out{in_run} or $type_change)) { - end_run($out, $code-1); + my $type_ok = ($type and ${${$out}{types}}{$type}); + $type_ok = not $type_ok if $opt_n; + my $prev_code = $$out{prev_code}; + + if (not $type_ok) { + end_run($out, $prev_code); + } elsif (not $$out{in_run} or $type ne $$out{run_type} or $code != $prev_code+1) { + end_run($out, $prev_code); start_run($out, $code, $type); - } elsif (not $match and $$out{in_run}) { - end_run($out, $code-1); } + $$out{prev_code} = $code; } sub start_run { my ($out, $code, $type) = @_; $$out{start_code} = $code; - $$out{start_type} = $type; + $$out{prev_code} = $code; + $$out{run_type} = $type; $$out{in_run} = 1; } sub end_run { my ($out, $code) = @_; return if not $$out{in_run}; - printf "\t{ 0x%04x, 0x%04x }, /* %s */\n", $$out{start_code}, $code, $$out{start_type}; + printf "\t{ 0x%04x, 0x%04x }, /* %s */\n", $$out{start_code}, $code, $$out{run_type}; $$out{in_run} = 0; } diff --git a/contrib/less/optfunc.c b/contrib/less/optfunc.c index 20ff1ea4cc047..4c79fe3f16b95 100644 --- a/contrib/less/optfunc.c +++ b/contrib/less/optfunc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -47,9 +47,9 @@ extern char *every_first_cmd; extern IFILE curr_ifile; extern char version[]; extern int jump_sline; -extern int jump_sline_fraction; +extern long jump_sline_fraction; extern int shift_count; -extern int shift_count_fraction; +extern long shift_count_fraction; extern int less_is_more; #if LOGFILE extern char *namelogfile; @@ -67,6 +67,7 @@ extern int bo_fg_color, bo_bg_color; extern int ul_fg_color, ul_bg_color; extern int so_fg_color, so_bg_color; extern int bl_fg_color, bl_bg_color; +extern int sgr_mode; #endif @@ -75,7 +76,9 @@ extern int bl_fg_color, bl_bg_color; * Handler for -o option. */ public void -opt_o(int type, char *s) +opt_o(type, s) + int type; + char *s; { PARG parg; @@ -123,7 +126,9 @@ opt_o(int type, char *s) * Handler for -O option. */ public void -opt__O(int type, char *s) +opt__O(type, s) + int type; + char *s; { force_logfile = TRUE; opt_o(type, s); @@ -134,7 +139,9 @@ opt__O(int type, char *s) * Handlers for -j option. */ public void -opt_j(int type, char *s) +opt_j(type, s) + int type; + char *s; { PARG parg; char buf[16]; @@ -173,7 +180,7 @@ opt_j(int type, char *s) } else { - sprintf(buf, ".%06d", jump_sline_fraction); + sprintf(buf, ".%06ld", jump_sline_fraction); len = (int) strlen(buf); while (len > 2 && buf[len-1] == '0') len--; @@ -186,7 +193,7 @@ opt_j(int type, char *s) } public void -calc_jump_sline(void) +calc_jump_sline() { if (jump_sline_fraction < 0) return; @@ -197,7 +204,9 @@ calc_jump_sline(void) * Handlers for -# option. */ public void -opt_shift(int type, char *s) +opt_shift(type, s) + int type; + char *s; { PARG parg; char buf[16]; @@ -236,7 +245,7 @@ opt_shift(int type, char *s) } else { - sprintf(buf, ".%06d", shift_count_fraction); + sprintf(buf, ".%06ld", shift_count_fraction); len = (int) strlen(buf); while (len > 2 && buf[len-1] == '0') len--; @@ -248,7 +257,7 @@ opt_shift(int type, char *s) } } public void -calc_shift_count(void) +calc_shift_count() { if (shift_count_fraction < 0) return; @@ -257,7 +266,9 @@ calc_shift_count(void) #if USERFILE public void -opt_k(int type, char *s) +opt_k(type, s) + int type; + char *s; { PARG parg; @@ -279,7 +290,9 @@ opt_k(int type, char *s) * Handler for -t option. */ public void -opt_t(int type, char *s) +opt_t(type, s) + int type; + char *s; { IFILE save_ifile; POSITION pos; @@ -318,7 +331,9 @@ opt_t(int type, char *s) * Handler for -T option. */ public void -opt__T(int type, char *s) +opt__T(type, s) + int type; + char *s; { PARG parg; @@ -345,7 +360,9 @@ opt__T(int type, char *s) * Handler for -p option. */ public void -opt_p(int type, char *s) +opt_p(type, s) + int type; + char *s; { switch (type) { @@ -379,7 +396,9 @@ opt_p(int type, char *s) * Handler for -P option. */ public void -opt__P(int type, char *s) +opt__P(type, s) + int type; + char *s; { char **proto; PARG parg; @@ -416,7 +435,9 @@ opt__P(int type, char *s) */ /*ARGSUSED*/ public void -opt_b(int type, char *s) +opt_b(type, s) + int type; + char *s; { switch (type) { @@ -437,7 +458,9 @@ opt_b(int type, char *s) */ /*ARGSUSED*/ public void -opt_i(int type, char *s) +opt_i(type, s) + int type; + char *s; { switch (type) { @@ -455,7 +478,9 @@ opt_i(int type, char *s) */ /*ARGSUSED*/ public void -opt__V(int type, char *s) +opt__V(type, s) + int type; + char *s; { switch (type) { @@ -493,7 +518,7 @@ opt__V(int type, char *s) putstr("no "); #endif putstr("regular expressions)\n"); - putstr("Copyright (C) 1984-2015 Mark Nudelman\n\n"); + putstr("Copyright (C) 1984-2017 Mark Nudelman\n\n"); putstr("less comes with NO WARRANTY, to the extent permitted by law.\n"); putstr("For information about the terms of redistribution,\n"); putstr("see the file named README in the less distribution.\n"); @@ -508,7 +533,10 @@ opt__V(int type, char *s) * Parse an MSDOS color descriptor. */ static void -colordesc(char *s, int *fg_color, int *bg_color) +colordesc(s, fg_color, bg_color) + char *s; + int *fg_color; + int *bg_color; { int fg, bg; int err; @@ -542,8 +570,12 @@ colordesc(char *s, int *fg_color, int *bg_color) */ /*ARGSUSED*/ public void -opt_D(int type, char *s) +opt_D(type, s) + int type; + char *s; { + PARG p; + switch (type) { case INIT: @@ -565,8 +597,11 @@ opt_D(int type, char *s) case 's': colordesc(s, &so_fg_color, &so_bg_color); break; + case 'a': + sgr_mode = !sgr_mode; + break; default: - error("-D must be followed by n, d, u, k or s", NULL_PARG); + error("-D must be followed by n, d, u, k, s or a", NULL_PARG); break; } if (type == TOGGLE) @@ -576,6 +611,8 @@ opt_D(int type, char *s) } break; case QUERY: + p.p_string = (sgr_mode) ? "on" : "off"; + error("SGR mode is %s", &p); break; } } @@ -585,7 +622,9 @@ opt_D(int type, char *s) * Handler for the -x option. */ public void -opt_x(int type, char *s) +opt_x(type, s) + int type; + char *s; { extern int tabstops[]; extern int ntabstops; @@ -641,7 +680,9 @@ opt_x(int type, char *s) * Handler for the -" option. */ public void -opt_quote(int type, char *s) +opt_quote(type, s) + int type; + char *s; { char buf[3]; PARG parg; @@ -682,7 +723,9 @@ opt_quote(int type, char *s) */ /*ARGSUSED*/ public void -opt_query(int type, char *s) +opt_query(type, s) + int type; + char *s; { switch (type) { @@ -699,7 +742,7 @@ opt_query(int type, char *s) * Get the "screen window" size. */ public int -get_swindow(void) +get_swindow() { if (swindow > 0) return (swindow); diff --git a/contrib/less/option.c b/contrib/less/option.c index c5927e2c77223..8ba67bce31874 100644 --- a/contrib/less/option.c +++ b/contrib/less/option.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -23,8 +23,8 @@ static struct loption *pendopt; public int plusoption = FALSE; -static char *optstring(char *s, char **p_str, char *printopt, char *validchars); -static int flip_triple(int val, int lc); +static char *optstring(); +static int flip_triple(); extern int screen_trashed; extern int less_is_more; @@ -36,7 +36,8 @@ extern int opt_use_backslash; * Return a printable description of an option. */ static char * -opt_desc(struct loption *o) +opt_desc(o) + struct loption *o; { static char buf[OPTNAME_MAX + 10]; if (o->oletter == OLETTER_NONE) @@ -51,7 +52,8 @@ opt_desc(struct loption *o) * For example, if the option letter is 'x', just return "-x". */ public char * -propt(int c) +propt(c) + int c; { static char buf[8]; @@ -64,7 +66,8 @@ propt(int c) * LESS environment variable) and process it. */ public void -scan_option(char *s) +scan_option(s) + char *s; { struct loption *o; int optc; @@ -296,7 +299,11 @@ scan_option(char *s) * OPT_SET set to the inverse of the default value */ public void -toggle_option(struct loption *o, int lower, char *s, int how_toggle) +toggle_option(o, lower, s, how_toggle) + struct loption *o; + int lower; + char *s; + int how_toggle; { int num; int no_prompt; @@ -478,7 +485,9 @@ toggle_option(struct loption *o, int lower, char *s, int how_toggle) * "Toggle" a triple-valued option. */ static int -flip_triple(int val, int lc) +flip_triple(val, lc) + int val; + int lc; { if (lc) return ((val == OPT_ON) ? OPT_OFF : OPT_ON); @@ -490,7 +499,8 @@ flip_triple(int val, int lc) * Determine if an option takes a parameter. */ public int -opt_has_param(struct loption *o) +opt_has_param(o) + struct loption *o; { if (o == NULL) return (0); @@ -504,7 +514,8 @@ opt_has_param(struct loption *o) * Only string and number valued options have prompts. */ public char * -opt_prompt(struct loption *o) +opt_prompt(o) + struct loption *o; { if (o == NULL || (o->otype & (STRING|NUMBER)) == 0) return ("?"); @@ -519,7 +530,7 @@ opt_prompt(struct loption *o) * the previous option. */ public int -isoptpending(void) +isoptpending() { return (pendopt != NULL); } @@ -528,7 +539,8 @@ isoptpending(void) * Print error message about missing string. */ static void -nostring(char *printopt) +nostring(printopt) + char *printopt; { PARG parg; parg.p_string = printopt; @@ -539,7 +551,7 @@ nostring(char *printopt) * Print error message if a STRING type option is not followed by a string. */ public void -nopendopt(void) +nopendopt() { nostring(opt_desc(pendopt)); } @@ -550,7 +562,11 @@ nopendopt(void) * Return a pointer to the remainder of the string, if any. */ static char * -optstring(char *s, char **p_str, char *printopt, char *validchars) +optstring(s, p_str, printopt, validchars) + char *s; + char **p_str; + char *printopt; + char *validchars; { char *p; char *out; @@ -586,7 +602,9 @@ optstring(char *s, char **p_str, char *printopt, char *validchars) /* */ static int -num_error(char *printopt, int *errp) +num_error(printopt, errp) + char *printopt; + int *errp; { PARG parg; @@ -609,7 +627,10 @@ num_error(char *printopt, int *errp) * the char * to point after the translated number. */ public int -getnum(char **sp, char *printopt, int *errp) +getnum(sp, printopt, errp) + char **sp; + char *printopt; + int *errp; { char *s; int n; @@ -643,7 +664,10 @@ getnum(char **sp, char *printopt, int *errp) * That is, if "n" is returned, the fraction intended is n/NUM_FRAC_DENOM. */ public long -getfraction(char **sp, char *printopt, int *errp) +getfraction(sp, printopt, errp) + char **sp; + char *printopt; + int *errp; { char *s; long frac = 0; @@ -675,7 +699,7 @@ getfraction(char **sp, char *printopt, int *errp) * Get the value of the -e flag. */ public int -get_quit_at_eof(void) +get_quit_at_eof() { if (!less_is_more) return quit_at_eof; diff --git a/contrib/less/option.h b/contrib/less/option.h index dc97d75f77c96..6adcdde27dbe6 100644 --- a/contrib/less/option.h +++ b/contrib/less/option.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -60,7 +60,7 @@ struct loption int otype; /* Type of the option */ int odefault; /* Default value */ int *ovar; /* Pointer to the associated variable */ - void (*ofunc)(); /* Pointer to special handling function */ + void (*ofunc) LESSPARAMS((int, char*)); /* Pointer to special handling function */ char *odesc[3]; /* Description of each value */ }; diff --git a/contrib/less/opttbl.c b/contrib/less/opttbl.c index 096524f94d2b6..58ec347f4b68d 100644 --- a/contrib/less/opttbl.c +++ b/contrib/less/opttbl.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -176,10 +176,10 @@ static struct loption option[] = }, #if MSDOS_COMPILER { 'D', &D__optname, - STRING|REPAINT|NO_QUERY, 0, NULL, opt_D, + STRING|REPAINT, 0, NULL, opt_D, { "color desc: ", - "Ddknsu0123456789.", + "Dadknsu0123456789.", NULL } }, @@ -464,7 +464,7 @@ static struct loption option[] = * Initialize each option to its default value. */ public void -init_option(void) +init_option() { struct loption *o; char *p; @@ -489,7 +489,8 @@ init_option(void) * Find an option in the option table, given its option letter. */ public struct loption * -findopt(int c) +findopt(c) + int c; { struct loption *o; @@ -507,7 +508,8 @@ findopt(int c) * */ static int -is_optchar(char c) +is_optchar(c) + char c; { if (ASCII_IS_UPPER(c)) return 1; @@ -525,7 +527,10 @@ is_optchar(char c) * p_oname if non-NULL is set to point to the full option name. */ public struct loption * -findopt_name(char **p_optname, char **p_oname, int *p_err) +findopt_name(p_optname, p_oname, p_err) + char **p_optname; + char **p_oname; + int *p_err; { char *optname = *p_optname; struct loption *o; diff --git a/contrib/less/os.c b/contrib/less/os.c index 2daf7349fff76..df117610d82f1 100644 --- a/contrib/less/os.c +++ b/contrib/less/os.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -60,7 +60,10 @@ extern int sigs; * any pending iread(). */ public int -iread(int fd, char *buf, unsigned int len) +iread(fd, buf, len) + int fd; + unsigned char *buf; + unsigned int len; { int n; @@ -173,7 +176,7 @@ start: * Interrupt a pending iread(). */ public void -intread(void) +intread() { LONG_JUMP(read_label, 1); } @@ -183,7 +186,7 @@ intread(void) */ #if HAVE_TIME public time_type -get_time(void) +get_time() { time_type t; @@ -198,7 +201,8 @@ get_time(void) * Local version of strerror, if not available from the system. */ static char * -strerror(int err) +strerror(err) + int err; { #if HAVE_SYS_ERRLIST static char buf[16]; @@ -219,7 +223,8 @@ strerror(int err) * errno_message: Return an error message based on the value of "errno". */ public char * -errno_message(char *filename) +errno_message(filename) + char *filename; { char *p; char *m; @@ -241,7 +246,8 @@ errno_message(char *filename) /* #define HAVE_FLOAT 0 */ static POSITION -muldiv(POSITION val, POSITION num, POSITION den) +muldiv(val, num, den) + POSITION val, num, den; { #if HAVE_FLOAT double v = (((double) val) * num) / den; @@ -264,7 +270,9 @@ muldiv(POSITION val, POSITION num, POSITION den) * {{ Assumes a POSITION is a long int. }} */ public int -percentage(POSITION num, POSITION den) +percentage(num, den) + POSITION num; + POSITION den; { return (int) muldiv(num, (POSITION) 100, den); } @@ -273,7 +281,10 @@ percentage(POSITION num, POSITION den) * Return the specified percentage of a POSITION. */ public POSITION -percent_pos(POSITION pos, int percent, long fraction) +percent_pos(pos, percent, fraction) + POSITION pos; + int percent; + long fraction; { /* Change percent (parts per 100) to perden (parts per NUM_FRAC_DENOM). */ POSITION perden = (percent * (NUM_FRAC_DENOM / 100)) + (fraction / 100); @@ -324,7 +335,9 @@ memcpy(dst, src, len) * This implements an ANSI-style intercept setup for Microware C 3.2 */ public int -os9_signal(int type, RETSIGTYPE (*handler)()) +os9_signal(type, handler) + int type; + RETSIGTYPE (*handler)(); { intercept(handler); } diff --git a/contrib/less/output.c b/contrib/less/output.c index 4d1e2aaae5408..cd9ccc25e7e1a 100644 --- a/contrib/less/output.c +++ b/contrib/less/output.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -37,13 +37,14 @@ extern int bo_fg_color, bo_bg_color; extern int ul_fg_color, ul_bg_color; extern int so_fg_color, so_bg_color; extern int bl_fg_color, bl_bg_color; +extern int sgr_mode; #endif /* * Display the line which is in the line buffer. */ public void -put_line(void) +put_line() { int c; int i; @@ -93,7 +94,7 @@ static char *ob = obuf; * overwritten or scrolled away. */ public void -flush(void) +flush() { int n; int fd; @@ -126,8 +127,9 @@ flush(void) * the -D command-line option. */ char *anchor, *p, *p_next; - unsigned char fg, bg; + static unsigned char fg, fgi, bg, bgi; static unsigned char at; + unsigned char f, b; #if MSDOS_COMPILER==WIN32C /* Screen colors used by 3x and 4x SGR commands. */ static unsigned char screen_color[] = { @@ -147,6 +149,13 @@ flush(void) }; #endif + if (fg == 0 && bg == 0) + { + fg = nm_fg_color & 7; + fgi = nm_fg_color & 8; + bg = nm_bg_color & 7; + bgi = nm_bg_color & 8; + } for (anchor = p_next = obuf; (p_next = memchr(p_next, ESC, ob - p_next)) != NULL; ) { @@ -173,18 +182,21 @@ flush(void) */ p++; anchor = p_next = p; - at = 0; + fg = nm_fg_color & 7; + fgi = nm_fg_color & 8; + bg = nm_bg_color & 7; + bgi = nm_bg_color & 8; + at = 0; WIN32setcolors(nm_fg_color, nm_bg_color); continue; } p_next = p; + at &= ~32; /* * Select foreground/background colors * based on the escape sequence. */ - fg = nm_fg_color; - bg = nm_bg_color; while (!is_ansi_end(*p)) { char *q; @@ -212,17 +224,35 @@ flush(void) break; } if (*q == ';') + { q++; + at |= 32; + } switch (code) { default: /* case 0: all attrs off */ - fg = nm_fg_color; - bg = nm_bg_color; - at = 0; + fg = nm_fg_color & 7; + bg = nm_bg_color & 7; + at &= 32; + /* + * \e[0m use normal + * intensities, but + * \e[0;...m resets them + */ + if (at & 32) + { + fgi = 0; + bgi = 0; + } else + { + fgi = nm_fg_color & 8; + bgi = nm_bg_color & 8; + } break; case 1: /* bold on */ + fgi = 8; at |= 1; break; case 3: /* italic on */ @@ -230,16 +260,19 @@ flush(void) at |= 2; break; case 4: /* underline on */ + bgi = 8; at |= 4; break; case 5: /* slow blink on */ case 6: /* fast blink on */ + bgi = 8; at |= 8; break; case 8: /* concealed on */ - fg = (bg & 7) | 8; + at |= 16; break; case 22: /* bold off */ + fgi = 0; at &= ~1; break; case 23: /* italic off */ @@ -247,62 +280,84 @@ flush(void) at &= ~2; break; case 24: /* underline off */ + bgi = 0; at &= ~4; break; + case 28: /* concealed off */ + at &= ~16; + break; case 30: case 31: case 32: case 33: case 34: case 35: case 36: case 37: - fg = (fg & 8) | (screen_color[code - 30]); + fg = screen_color[code - 30]; + at |= 32; break; case 39: /* default fg */ - fg = nm_fg_color; + fg = nm_fg_color & 7; + at |= 32; break; case 40: case 41: case 42: case 43: case 44: case 45: case 46: case 47: - bg = (bg & 8) | (screen_color[code - 40]); + bg = screen_color[code - 40]; + at |= 32; break; - case 49: /* default fg */ - bg = nm_bg_color; + case 49: /* default bg */ + bg = nm_bg_color & 7; + at |= 32; break; } p = q; } if (!is_ansi_end(*p) || p == p_next) break; - if (at & 1) + /* + * In SGR mode, the ANSI sequence is + * always honored; otherwise if an attr + * is used by itself ("\e[1m" versus + * "\e[1;33m", for example), set the + * color assigned to that attribute. + */ + if (sgr_mode || (at & 32)) { - /* - * If \e[1m use defined bold - * color, else set intensity. - */ - if (p[-2] == '[') + if (at & 2) { -#if MSDOS_COMPILER==WIN32C - fg |= FOREGROUND_INTENSITY; - bg |= BACKGROUND_INTENSITY; -#else - fg = bo_fg_color; - bg = bo_bg_color; -#endif + f = bg | bgi; + b = fg | fgi; } else - fg |= 8; - } else if (at & 2) - { - fg = so_fg_color; - bg = so_bg_color; - } else if (at & 4) - { - fg = ul_fg_color; - bg = ul_bg_color; - } else if (at & 8) + { + f = fg | fgi; + b = bg | bgi; + } + } else { - fg = bl_fg_color; - bg = bl_bg_color; + if (at & 1) + { + f = bo_fg_color; + b = bo_bg_color; + } else if (at & 2) + { + f = so_fg_color; + b = so_bg_color; + } else if (at & 4) + { + f = ul_fg_color; + b = ul_bg_color; + } else if (at & 8) + { + f = bl_fg_color; + b = bl_bg_color; + } else + { + f = nm_fg_color; + b = nm_bg_color; + } } - fg &= 0xf; - bg &= 0xf; - WIN32setcolors(fg, bg); + if (at & 16) + f = b ^ 8; + f &= 0xf; + b &= 0xf; + WIN32setcolors(f, b); p_next = anchor = p + 1; } else p_next++; @@ -326,7 +381,8 @@ flush(void) * Output a character. */ public int -putchr(int c) +putchr(c) + int c; { #if 0 /* fake UTF-8 output for testing */ extern int utf_mode; @@ -379,7 +435,8 @@ putchr(int c) * Output a string. */ public void -putstr(constant char *s) +putstr(s) + constant char *s; { while (*s != '\0') putchr(*s++); @@ -414,7 +471,8 @@ TYPE_TO_A_FUNC(inttoa, int) * Output an integer in a given radix. */ static int -iprint_int(int num) +iprint_int(num) + int num; { char buf[INT_STRLEN_BOUND(num)]; @@ -427,7 +485,8 @@ iprint_int(int num) * Output a line number in a given radix. */ static int -iprint_linenum(LINENUM num) +iprint_linenum(num) + LINENUM num; { char buf[INT_STRLEN_BOUND(num)]; @@ -441,7 +500,9 @@ iprint_linenum(LINENUM num) * using a more portable argument list mechanism than printf's. */ static int -less_printf(char *fmt, PARG *parg) +less_printf(fmt, parg) + char *fmt; + PARG *parg; { char *s; int col; @@ -487,7 +548,7 @@ less_printf(char *fmt, PARG *parg) * become the next command. */ public void -get_return(void) +get_return() { int c; @@ -506,7 +567,9 @@ get_return(void) * and wait for carriage return. */ public void -error(char *fmt, PARG *parg) +error(fmt, parg) + char *fmt; + PARG *parg; { int col = 0; static char return_to_continue[] = " (press RETURN)"; @@ -559,7 +622,9 @@ static char intr_to_abort[] = "... (interrupt to abort)"; * time-consuming operation. */ public void -ierror(char *fmt, PARG *parg) +ierror(fmt, parg) + char *fmt; + PARG *parg; { at_exit(); clear_bot(); @@ -576,7 +641,9 @@ ierror(char *fmt, PARG *parg) * and return a single-character response. */ public int -query(char *fmt, PARG *parg) +query(fmt, parg) + char *fmt; + PARG *parg; { int c; int col = 0; diff --git a/contrib/less/pattern.c b/contrib/less/pattern.c index 0706f72588196..563bde0cc6973 100644 --- a/contrib/less/pattern.c +++ b/contrib/less/pattern.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -12,7 +12,6 @@ */ #include "less.h" -#include "pattern.h" extern int caseless; @@ -20,7 +19,11 @@ extern int caseless; * Compile a search pattern, for future use by match_pattern. */ static int -compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_error) +compile_pattern2(pattern, search_type, comp_pattern, show_error) + char *pattern; + int search_type; + PATTERN_TYPE *comp_pattern; + int show_error; { if (search_type & SRCH_NO_REGEX) return (0); @@ -28,8 +31,6 @@ compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_e #if HAVE_GNU_REGEX struct re_pattern_buffer *comp = (struct re_pattern_buffer *) ecalloc(1, sizeof(struct re_pattern_buffer)); - struct re_pattern_buffer **pcomp = - (struct re_pattern_buffer **) comp_pattern; re_set_syntax(RE_SYNTAX_POSIX_EXTENDED); if (re_compile_pattern(pattern, strlen(pattern), comp)) { @@ -38,13 +39,15 @@ compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_e error("Invalid pattern", NULL_PARG); return (-1); } - if (*pcomp != NULL) - regfree(*pcomp); - *pcomp = comp; + if (*comp_pattern != NULL) + { + regfree(*comp_pattern); + free(*comp_pattern); + } + *comp_pattern = comp; #endif #if HAVE_POSIX_REGCOMP regex_t *comp = (regex_t *) ecalloc(1, sizeof(regex_t)); - regex_t **pcomp = (regex_t **) comp_pattern; if (regcomp(comp, pattern, REGCOMP_FLAG)) { free(comp); @@ -52,13 +55,15 @@ compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_e error("Invalid pattern", NULL_PARG); return (-1); } - if (*pcomp != NULL) - regfree(*pcomp); - *pcomp = comp; + if (*comp_pattern != NULL) + { + regfree(*comp_pattern); + free(*comp_pattern); + } + *comp_pattern = comp; #endif #if HAVE_PCRE pcre *comp; - pcre **pcomp = (pcre **) comp_pattern; constant char *errstring; int erroffset; PARG parg; @@ -71,35 +76,32 @@ compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_e error("%s", &parg); return (-1); } - *pcomp = comp; + *comp_pattern = comp; #endif #if HAVE_RE_COMP PARG parg; - int *pcomp = (int *) comp_pattern; if ((parg.p_string = re_comp(pattern)) != NULL) { if (show_error) error("%s", &parg); return (-1); } - *pcomp = 1; + *comp_pattern = 1; #endif #if HAVE_REGCMP char *comp; - char **pcomp = (char **) comp_pattern; if ((comp = regcmp(pattern, 0)) == NULL) { if (show_error) error("Invalid pattern", NULL_PARG); return (-1); } - if (pcomp != NULL) - free(*pcomp); - *pcomp = comp; + if (comp_pattern != NULL) + free(*comp_pattern); + *comp_pattern = comp; #endif #if HAVE_V8_REGCOMP struct regexp *comp; - struct regexp **pcomp = (struct regexp **) comp_pattern; reg_show_error = show_error; comp = regcomp(pattern); reg_show_error = 1; @@ -111,9 +113,9 @@ compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_e */ return (-1); } - if (*pcomp != NULL) - free(*pcomp); - *pcomp = comp; + if (*comp_pattern != NULL) + free(*comp_pattern); + *comp_pattern = comp; #endif } return (0); @@ -123,7 +125,10 @@ compile_pattern2(char *pattern, int search_type, void **comp_pattern, int show_e * Like compile_pattern2, but convert the pattern to lowercase if necessary. */ public int -compile_pattern(char *pattern, int search_type, void **comp_pattern) +compile_pattern(pattern, search_type, comp_pattern) + char *pattern; + int search_type; + PATTERN_TYPE *comp_pattern; { char *cvt_pattern; int result; @@ -145,41 +150,42 @@ compile_pattern(char *pattern, int search_type, void **comp_pattern) * Forget that we have a compiled pattern. */ public void -uncompile_pattern(void **pattern) +uncompile_pattern(pattern) + PATTERN_TYPE *pattern; { #if HAVE_GNU_REGEX - struct re_pattern_buffer **pcomp = (struct re_pattern_buffer **) pattern; - if (*pcomp != NULL) - regfree(*pcomp); - *pcomp = NULL; + if (*pattern != NULL) + { + regfree(*pattern); + free(*pattern); + } + *pattern = NULL; #endif #if HAVE_POSIX_REGCOMP - regex_t **pcomp = (regex_t **) pattern; - if (*pcomp != NULL) - regfree(*pcomp); - *pcomp = NULL; + if (*pattern != NULL) + { + regfree(*pattern); + free(*pattern); + } + *pattern = NULL; #endif #if HAVE_PCRE - pcre **pcomp = (pcre **) pattern; - if (*pcomp != NULL) - pcre_free(*pcomp); - *pcomp = NULL; + if (*pattern != NULL) + pcre_free(*pattern); + *pattern = NULL; #endif #if HAVE_RE_COMP - int *pcomp = (int *) pattern; - *pcomp = 0; + *pattern = 0; #endif #if HAVE_REGCMP - char **pcomp = (char **) pattern; - if (*pcomp != NULL) - free(*pcomp); - *pcomp = NULL; + if (*pattern != NULL) + free(*pattern); + *pattern = NULL; #endif #if HAVE_V8_REGCOMP - struct regexp **pcomp = (struct regexp **) pattern; - if (*pcomp != NULL) - free(*pcomp); - *pcomp = NULL; + if (*pattern != NULL) + free(*pattern); + *pattern = NULL; #endif } @@ -187,9 +193,10 @@ uncompile_pattern(void **pattern) * Can a pattern be successfully compiled? */ public int -valid_pattern(char *pattern) +valid_pattern(pattern) + char *pattern; { - void *comp_pattern; + PATTERN_TYPE comp_pattern; int result; CLEAR_PATTERN(comp_pattern); @@ -204,7 +211,8 @@ valid_pattern(char *pattern) * Is a compiled pattern null? */ public int -is_null_pattern(void *pattern) +is_null_pattern(pattern) + PATTERN_TYPE pattern; { #if HAVE_GNU_REGEX return (pattern == NULL); @@ -234,7 +242,12 @@ is_null_pattern(void *pattern) * It supports no metacharacters like *, etc. */ static int -match(char *pattern, int pattern_len, char *buf, int buf_len, char **pfound, char **pend) +match(pattern, pattern_len, buf, buf_len, pfound, pend) + char *pattern; + int pattern_len; + char *buf; + int buf_len; + char **pfound, **pend; { char *pp, *lp; char *pattern_end = pattern + pattern_len; @@ -270,27 +283,17 @@ match(char *pattern, int pattern_len, char *buf, int buf_len, char **pfound, cha * Set sp and ep to the start and end of the matched string. */ public int -match_pattern(void *pattern, char *tpattern, char *line, int line_len, char **sp, char **ep, int notbol, int search_type) +match_pattern(pattern, tpattern, line, line_len, sp, ep, notbol, search_type) + PATTERN_TYPE pattern; + char *tpattern; + char *line; + int line_len; + char **sp; + char **ep; + int notbol; + int search_type; { int matched; -#if HAVE_GNU_REGEX - struct re_pattern_buffer *spattern = (struct re_pattern_buffer *) pattern; -#endif -#if HAVE_POSIX_REGCOMP - regex_t *spattern = (regex_t *) pattern; -#endif -#if HAVE_PCRE - pcre *spattern = (pcre *) pattern; -#endif -#if HAVE_RE_COMP - int spattern = (int) pattern; -#endif -#if HAVE_REGCMP - char *spattern = (char *) pattern; -#endif -#if HAVE_V8_REGCOMP - struct regexp *spattern = (struct regexp *) pattern; -#endif *sp = *ep = NULL; #if NO_REGEX @@ -303,9 +306,9 @@ match_pattern(void *pattern, char *tpattern, char *line, int line_len, char **sp #if HAVE_GNU_REGEX { struct re_registers search_regs; - spattern->not_bol = notbol; - spattern->regs_allocated = REGS_UNALLOCATED; - matched = re_search(spattern, line, line_len, 0, line_len, &search_regs) >= 0; + pattern->not_bol = notbol; + pattern->regs_allocated = REGS_UNALLOCATED; + matched = re_search(pattern, line, line_len, 0, line_len, &search_regs) >= 0; if (matched) { *sp = line + search_regs.start[0]; @@ -322,7 +325,7 @@ match_pattern(void *pattern, char *tpattern, char *line, int line_len, char **sp rm.rm_so = 0; rm.rm_eo = line_len; #endif - matched = !regexec(spattern, line, 1, &rm, flags); + matched = !regexec(pattern, line, 1, &rm, flags); if (matched) { #ifndef __WATCOMC__ @@ -339,7 +342,7 @@ match_pattern(void *pattern, char *tpattern, char *line, int line_len, char **sp { int flags = (notbol) ? PCRE_NOTBOL : 0; int ovector[3]; - matched = pcre_exec(spattern, NULL, line, line_len, + matched = pcre_exec(pattern, NULL, line, line_len, 0, flags, ovector, 3) >= 0; if (matched) { @@ -356,21 +359,21 @@ match_pattern(void *pattern, char *tpattern, char *line, int line_len, char **sp *sp = *ep = NULL; #endif #if HAVE_REGCMP - *ep = regex(spattern, line); + *ep = regex(pattern, line); matched = (*ep != NULL); if (matched) *sp = __loc1; #endif #if HAVE_V8_REGCOMP #if HAVE_REGEXEC2 - matched = regexec2(spattern, line, notbol); + matched = regexec2(pattern, line, notbol); #else - matched = regexec(spattern, line); + matched = regexec(pattern, line); #endif if (matched) { - *sp = spattern->startp[0]; - *ep = spattern->endp[0]; + *sp = pattern->startp[0]; + *ep = pattern->endp[0]; } #endif } diff --git a/contrib/less/pattern.h b/contrib/less/pattern.h index 712fdcf898c03..e226dc050ad41 100644 --- a/contrib/less/pattern.h +++ b/contrib/less/pattern.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -10,7 +10,7 @@ #if HAVE_GNU_REGEX #define __USE_GNU 1 #include <regex.h> -#define DEFINE_PATTERN(name) struct re_pattern_buffer *name +#define PATTERN_TYPE struct re_pattern_buffer * #define CLEAR_PATTERN(name) name = NULL #endif @@ -18,24 +18,24 @@ #include <regex.h> #ifdef REG_EXTENDED extern int less_is_more; -#define REGCOMP_FLAG (less_is_more ? 0 : REG_EXTENDED) +#define REGCOMP_FLAG (less_is_more ? 0 : REG_EXTENDED) #else -#define REGCOMP_FLAG 0 +#define REGCOMP_FLAG 0 #endif -#define DEFINE_PATTERN(name) regex_t *name +#define PATTERN_TYPE regex_t * #define CLEAR_PATTERN(name) name = NULL #endif #if HAVE_PCRE #include <pcre.h> -#define DEFINE_PATTERN(name) pcre *name +#define PATTERN_TYPE pcre * #define CLEAR_PATTERN(name) name = NULL #endif #if HAVE_RE_COMP char *re_comp(); int re_exec(); -#define DEFINE_PATTERN(name) int name +#define PATTERN_TYPE int #define CLEAR_PATTERN(name) name = 0 #endif @@ -43,18 +43,18 @@ int re_exec(); char *regcmp(); char *regex(); extern char *__loc1; -#define DEFINE_PATTERN(name) char *name +#define PATTERN_TYPE char ** #define CLEAR_PATTERN(name) name = NULL #endif #if HAVE_V8_REGCOMP #include "regexp.h" extern int reg_show_error; -#define DEFINE_PATTERN(name) struct regexp *name +#define PATTERN_TYPE struct regexp * #define CLEAR_PATTERN(name) name = NULL #endif #if NO_REGEX -#define DEFINE_PATTERN(name) +#define PATTERN_TYPE void * #define CLEAR_PATTERN(name) #endif diff --git a/contrib/less/pckeys.h b/contrib/less/pckeys.h index 8fb6800902c10..e0d86e7bdf362 100644 --- a/contrib/less/pckeys.h +++ b/contrib/less/pckeys.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. diff --git a/contrib/less/position.c b/contrib/less/position.c index a0e4e3a64417e..d5e949e2919c4 100644 --- a/contrib/less/position.c +++ b/contrib/less/position.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -36,7 +36,8 @@ extern int sc_width, sc_height; * the line after the bottom line on the screen */ public POSITION -position(int where) +position(where) + int where; { switch (where) { @@ -56,7 +57,8 @@ position(int where) * Add a new file position to the bottom of the position table. */ public void -add_forw_pos(POSITION pos) +add_forw_pos(pos) + POSITION pos; { int i; @@ -72,7 +74,8 @@ add_forw_pos(POSITION pos) * Add a new file position to the top of the position table. */ public void -add_back_pos(POSITION pos) +add_back_pos(pos) + POSITION pos; { int i; @@ -88,7 +91,7 @@ add_back_pos(POSITION pos) * Initialize the position table, done whenever we clear the screen. */ public void -pos_clear(void) +pos_clear() { int i; @@ -100,7 +103,7 @@ pos_clear(void) * Allocate or reallocate the position table. */ public void -pos_init(void) +pos_init() { struct scrpos scrpos; @@ -129,7 +132,8 @@ pos_init(void) * Return the position table entry if found, -1 if not. */ public int -onscreen(POSITION pos) +onscreen(pos) + POSITION pos; { int i; @@ -145,13 +149,15 @@ onscreen(POSITION pos) * See if the entire screen is empty. */ public int -empty_screen(void) +empty_screen() { return (empty_lines(0, sc_height-1)); } public int -empty_lines(int s, int e) +empty_lines(s, e) + int s; + int e; { int i; @@ -170,7 +176,8 @@ empty_lines(int s, int e) * the screen line to a number > 0. */ public void -get_scrpos(struct scrpos *scrpos) +get_scrpos(scrpos) + struct scrpos *scrpos; { int i; @@ -201,7 +208,8 @@ get_scrpos(struct scrpos *scrpos) * relative to the bottom of the screen. */ public int -adjsline(int sline) +adjsline(sline) + int sline; { /* * Negative screen line number means diff --git a/contrib/less/position.h b/contrib/less/position.h index e0e2bff490e4b..30678962ccc77 100644 --- a/contrib/less/position.h +++ b/contrib/less/position.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. diff --git a/contrib/less/prompt.c b/contrib/less/prompt.c index c53f27351bfb2..bf4f7291df287 100644 --- a/contrib/less/prompt.c +++ b/contrib/less/prompt.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -66,7 +66,7 @@ static char *mp; * Initialize the prompt prototype strings. */ public void -init_prompt(void) +init_prompt() { prproto[0] = save(s_proto); prproto[1] = save(less_is_more ? more_proto : m_proto); @@ -80,7 +80,8 @@ init_prompt(void) * Append a string to the end of the message. */ static void -ap_str(char *s) +ap_str(s) + char *s; { int len; @@ -96,7 +97,8 @@ ap_str(char *s) * Append a character to the end of the message. */ static void -ap_char(char c) +ap_char(c) + char c; { char buf[2]; @@ -109,7 +111,8 @@ ap_char(char c) * Append a POSITION (as a decimal integer) to the end of the message. */ static void -ap_pos(POSITION pos) +ap_pos(pos) + POSITION pos; { char buf[INT_STRLEN_BOUND(pos) + 2]; @@ -121,7 +124,8 @@ ap_pos(POSITION pos) * Append a line number to the end of the message. */ static void -ap_linenum(LINENUM linenum) +ap_linenum(linenum) + LINENUM linenum; { char buf[INT_STRLEN_BOUND(linenum) + 2]; @@ -133,7 +137,8 @@ ap_linenum(LINENUM linenum) * Append an integer to the end of the message. */ static void -ap_int(int num) +ap_int(num) + int num; { char buf[INT_STRLEN_BOUND(num) + 2]; @@ -145,7 +150,7 @@ ap_int(int num) * Append a question mark to the end of the message. */ static void -ap_quest(void) +ap_quest() { ap_str("?"); } @@ -154,7 +159,8 @@ ap_quest(void) * Return the "current" byte offset in the file. */ static POSITION -curr_byte(int where) +curr_byte(where) + int where; { POSITION pos; @@ -173,7 +179,9 @@ curr_byte(int where) * Here we decode that letter and return the appropriate boolean value. */ static int -cond(char c, int where) +cond(c, where) + char c; + int where; { POSITION len; @@ -235,7 +243,10 @@ cond(char c, int where) * usually by appending something to the message being built. */ static void -protochar(int c, int where, int iseditproto) +protochar(c, where, iseditproto) + int c; + int where; + int iseditproto; { POSITION pos; POSITION len; @@ -382,7 +393,8 @@ protochar(int c, int where, int iseditproto) * We must keep track of nested IFs and skip them properly. */ static constant char * -skipcond(constant char *p) +skipcond(p) + constant char *p; { int iflevel; @@ -439,7 +451,9 @@ skipcond(constant char *p) * Decode a char that represents a position on the screen. */ static constant char * -wherechar(char constant *p, int *wp) +wherechar(p, wp) + char constant *p; + int *wp; { switch (*p) { @@ -461,7 +475,9 @@ wherechar(char constant *p, int *wp) * Construct a message based on a prototype string. */ public char * -pr_expand(constant char *proto, int maxwidth) +pr_expand(proto, maxwidth) + constant char *proto; + int maxwidth; { constant char *p; int c; @@ -535,7 +551,7 @@ pr_expand(constant char *proto, int maxwidth) * Return a message suitable for printing by the "=" command. */ public char * -eq_message(void) +eq_message() { return (pr_expand(eqproto, 0)); } @@ -547,7 +563,7 @@ eq_message(void) * and the caller will prompt with a colon. */ public char * -pr_string(void) +pr_string() { char *prompt; int type; @@ -564,7 +580,7 @@ pr_string(void) * Return a message suitable for printing while waiting in the F command. */ public char * -wait_message(void) +wait_message() { return (pr_expand(wproto, sc_width-so_s_width-so_e_width-2)); } diff --git a/contrib/less/regexp.c b/contrib/less/regexp.c index b54c8f07ccc3f..fcf7c9a836050 100644 --- a/contrib/less/regexp.c +++ b/contrib/less/regexp.c @@ -207,12 +207,13 @@ STATIC int strcspn(); * of the structure of the compiled regexp. */ regexp * -regcomp(char *exp) +regcomp(exp) +char *exp; { - regexp *r; - char *scan; - char *longest; - int len; + register regexp *r; + register char *scan; + register char *longest; + register int len; int flags; if (exp == NULL) @@ -296,12 +297,14 @@ regcomp(char *exp) * follows makes it hard to avoid. */ static char * -reg(int paren, int *flagp) +reg(paren, flagp) +int paren; /* Parenthesized? */ +int *flagp; { - char *ret; - char *br; - char *ender; - int parno = 0; + register char *ret; + register char *br; + register char *ender; + register int parno = 0; int flags; *flagp = HASWIDTH; /* Tentatively. */ @@ -366,11 +369,12 @@ reg(int paren, int *flagp) * Implements the concatenation operator. */ static char * -regbranch(int *flagp) +regbranch(flagp) +int *flagp; { - char *ret; - char *chain; - char *latest; + register char *ret; + register char *chain; + register char *latest; int flags; *flagp = WORST; /* Tentatively. */ @@ -404,11 +408,12 @@ regbranch(int *flagp) * endmarker role is not redundant. */ static char * -regpiece(int *flagp) +regpiece(flagp) +int *flagp; { - char *ret; - char op; - char *next; + register char *ret; + register char op; + register char *next; int flags; ret = regatom(&flags); @@ -467,9 +472,10 @@ regpiece(int *flagp) * separate node; the code is simpler that way and it's not worth fixing. */ static char * -regatom(int *flagp) +regatom(flagp) +int *flagp; { - char *ret; + register char *ret; int flags; *flagp = WORST; /* Tentatively. */ @@ -486,8 +492,8 @@ regatom(int *flagp) *flagp |= HASWIDTH|SIMPLE; break; case '[': { - int clss; - int classend; + register int clss; + register int classend; if (*regparse == '^') { /* Complement of range. */ ret = regnode(ANYBUT); @@ -547,8 +553,8 @@ regatom(int *flagp) *flagp |= HASWIDTH|SIMPLE; break; default: { - int len; - char ender; + register int len; + register char ender; regparse--; len = (int) strcspn(regparse, META); @@ -577,10 +583,11 @@ regatom(int *flagp) - regnode - emit a node */ static char * /* Location. */ -regnode(char op) +regnode(op) +char op; { - char *ret; - char *ptr; + register char *ret; + register char *ptr; ret = regcode; if (ret == ®dummy) { @@ -601,7 +608,8 @@ regnode(char op) - regc - emit (if appropriate) a byte of code */ static void -regc(char b) +regc(b) +char b; { if (regcode != ®dummy) *regcode++ = b; @@ -615,11 +623,13 @@ regc(char b) * Means relocating the operand. */ static void -reginsert(char op, char *opnd) +reginsert(op, opnd) +char op; +char *opnd; { - char *src; - char *dst; - char *place; + register char *src; + register char *dst; + register char *place; if (regcode == ®dummy) { regsize += 3; @@ -642,11 +652,13 @@ reginsert(char op, char *opnd) - regtail - set the next-pointer at the end of a node chain */ static void -regtail(char *p, char *val) +regtail(p, val) +char *p; +char *val; { - char *scan; - char *temp; - int offset; + register char *scan; + register char *temp; + register int offset; if (p == ®dummy) return; @@ -672,7 +684,9 @@ regtail(char *p, char *val) - regoptail - regtail on operand of first argument; nop if operandless */ static void -regoptail(char *p, char *val) +regoptail(p, val) +char *p; +char *val; { /* "Operandless" and "op != BRANCH" are synonymous in practice. */ if (p == NULL || p == ®dummy || OP(p) != BRANCH) @@ -709,9 +723,12 @@ STATIC char *regprop(); - regexec - match a regexp against a string */ int -regexec2(regexp *prog, char *string, int notbol) +regexec2(prog, string, notbol) +register regexp *prog; +register char *string; +int notbol; { - char *s; + register char *s; /* Be paranoid... */ if (prog == NULL || string == NULL) { @@ -768,7 +785,9 @@ regexec2(regexp *prog, char *string, int notbol) } int -regexec(regexp *prog, char *string) +regexec(prog, string) +register regexp *prog; +register char *string; { return regexec2(prog, string, 0); } @@ -777,11 +796,13 @@ regexec(regexp *prog, char *string) - regtry - try match at specific point */ static int /* 0 failure, 1 success */ -regtry(regexp *prog, char *string) +regtry(prog, string) +regexp *prog; +char *string; { - int i; - char **sp; - char **ep; + register int i; + register char **sp; + register char **ep; reginput = string; regstartp = prog->startp; @@ -812,10 +833,11 @@ regtry(regexp *prog, char *string) * by recursion. */ static int /* 0 failure, 1 success */ -regmatch(char *prog) +regmatch(prog) +char *prog; { - char *scan; /* Current node. */ - char *next; /* Next node. */ + register char *scan; /* Current node. */ + char *next; /* Next node. */ scan = prog; #ifdef DEBUG @@ -844,8 +866,8 @@ regmatch(char *prog) reginput++; break; case EXACTLY: { - int len; - char *opnd; + register int len; + register char *opnd; opnd = OPERAND(scan); /* Inline the first character, for speed. */ @@ -880,8 +902,8 @@ regmatch(char *prog) case OPEN+7: case OPEN+8: case OPEN+9: { - int no; - char *save; + register int no; + register char *save; no = OP(scan) - OPEN; save = reginput; @@ -909,8 +931,8 @@ regmatch(char *prog) case CLOSE+7: case CLOSE+8: case CLOSE+9: { - int no; - char *save; + register int no; + register char *save; no = OP(scan) - CLOSE; save = reginput; @@ -930,7 +952,7 @@ regmatch(char *prog) /* NOTREACHED */ break; case BRANCH: { - char *save; + register char *save; if (OP(next) != BRANCH) /* No choice. */ next = OPERAND(scan); /* Avoid recursion. */ @@ -950,10 +972,10 @@ regmatch(char *prog) break; case STAR: case PLUS: { - char nextch; - int no; - char *save; - int min; + register char nextch; + register int no; + register char *save; + register int min; /* * Lookahead to avoid useless match attempts @@ -1004,11 +1026,12 @@ regmatch(char *prog) - regrepeat - repeatedly match something simple, report how many */ static int -regrepeat(char *p) +regrepeat(p) +char *p; { - int count = 0; - char *scan; - char *opnd; + register int count = 0; + register char *scan; + register char *opnd; scan = reginput; opnd = OPERAND(p); @@ -1049,9 +1072,10 @@ regrepeat(char *p) - regnext - dig the "next" pointer out of a node */ static char * -regnext(char *p) +regnext(p) +register char *p; { - int offset; + register int offset; if (p == ®dummy) return(NULL); @@ -1074,11 +1098,12 @@ STATIC char *regprop(); - regdump - dump a regexp onto stdout in vaguely comprehensible form */ void -regdump(regexp *r) +regdump(r) +regexp *r; { - char *s; - char op = EXACTLY; /* Arbitrary non-END op. */ - char *next; + register char *s; + register char op = EXACTLY; /* Arbitrary non-END op. */ + register char *next; s = r->program + 1; @@ -1116,9 +1141,10 @@ regdump(regexp *r) - regprop - printable representation of opcode */ static char * -regprop(char *op) +regprop(op) +char *op; { - char *p; + register char *p; static char buf[50]; (void) strcpy(buf, ":"); @@ -1207,11 +1233,13 @@ regprop(char *op) */ static int -strcspn(char *s1, char *s2) +strcspn(s1, s2) +char *s1; +char *s2; { - char *scan1; - char *scan2; - int count; + register char *scan1; + register char *scan2; + register int count; count = 0; for (scan1 = s1; *scan1 != '\0'; scan1++) { diff --git a/contrib/less/screen.c b/contrib/less/screen.c index 8ae52fac80d89..5b3682731d101 100644 --- a/contrib/less/screen.c +++ b/contrib/less/screen.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -151,6 +151,7 @@ public int bl_fg_color; /* Color of blinking text */ public int bl_bg_color; static int sy_fg_color; /* Color of system text (before less) */ static int sy_bg_color; +public int sgr_mode; /* Honor ANSI sequences rather than using above */ #else @@ -203,11 +204,11 @@ public int missing_cap = 0; /* Some capability is missing */ static int attrmode = AT_NORMAL; extern int binattr; +extern int line_count; #if !MSDOS_COMPILER -static char *cheaper(char *t1, char *t2, char *def); -static void tmodes(char *incap, char *outcap, char **instr, char **outstr, - char *def_instr, char *def_outstr, char **spp); +static char *cheaper(); +static void tmodes(); #endif /* @@ -233,6 +234,7 @@ extern int wscroll; extern int screen_trashed; extern int tty; extern int top_scroll; +extern int quit_if_one_screen; extern int oldbot; #if HILITE_SEARCH extern int hilite_search; @@ -254,7 +256,8 @@ extern char *tgoto(); * It doesn't matter whether an input \n is mapped to \r, or vice versa. */ public void -raw_mode(int on) +raw_mode(on) + int on; { static int curr_on = 0; @@ -618,7 +621,8 @@ raw_mode(int on) static int hardcopy; static char * -ltget_env(char *capname) +ltget_env(capname) + char *capname; { char name[16]; char *s; @@ -646,7 +650,8 @@ ltget_env(char *capname) } static int -ltgetflag(char *capname) +ltgetflag(capname) + char *capname; { char *s; @@ -658,7 +663,8 @@ ltgetflag(char *capname) } static int -ltgetnum(char *capname) +ltgetnum(capname) + char *capname; { char *s; @@ -670,7 +676,9 @@ ltgetnum(char *capname) } static char * -ltgetstr(char *capname, char **pp) +ltgetstr(capname, pp) + char *capname; + char **pp; { char *s; @@ -686,7 +694,7 @@ ltgetstr(char *capname, char **pp) * Get size of the output screen. */ public void -scrsize(void) +scrsize() { char *s; int sys_height; @@ -816,7 +824,7 @@ scrsize(void) * Figure out how many empty loops it takes to delay a millisecond. */ static void -get_clock(void) +get_clock() { clock_t start; @@ -844,14 +852,15 @@ get_clock(void) * Delay for a specified number of milliseconds. */ static void -dummy_func(void) +dummy_func() { static long delay_dummy = 0; delay_dummy++; } static void -delay(int msec) +delay(msec) + int msec; { long i; @@ -873,7 +882,8 @@ delay(int msec) * Return the characters actually input by a "special" key. */ public char * -special_key_str(int key) +special_key_str(key) + int key; { static char tbuf[40]; char *s; @@ -1045,7 +1055,7 @@ special_key_str(int key) * Get terminal capabilities via termcap. */ public void -get_term(void) +get_term() { #if MSDOS_COMPILER auto_wrap = 1; @@ -1099,6 +1109,7 @@ get_term(void) so_bg_color = 9; bl_fg_color = 15; bl_bg_color = 0; + sgr_mode = 0; /* * Get size of the screen. @@ -1342,14 +1353,16 @@ static int costcount; /*ARGSUSED*/ static int -inc_costcount(int c) +inc_costcount(c) + int c; { costcount++; return (c); } static int -cost(char *t) +cost(t) + char *t; { costcount = 0; tputs(t, sc_height, inc_costcount); @@ -1362,7 +1375,9 @@ cost(char *t) * cost (see cost() function). */ static char * -cheaper(char *t1, char *t2, char *def) +cheaper(t1, t2, def) + char *t1, *t2; + char *def; { if (*t1 == '\0' && *t2 == '\0') { @@ -1379,8 +1394,14 @@ cheaper(char *t1, char *t2, char *def) } static void -tmodes(char *incap, char *outcap, char **instr, char **outstr, char *def_instr, - char *def_outstr, char **spp) +tmodes(incap, outcap, instr, outstr, def_instr, def_outstr, spp) + char *incap; + char *outcap; + char **instr; + char **outstr; + char *def_instr; + char *def_outstr; + char **spp; { *instr = ltgetstr(incap, spp); if (*instr == NULL) @@ -1429,8 +1450,11 @@ _settextposition(int row, int col) * Initialize the screen to the correct color at startup. */ static void -initcolor(void) +initcolor() { +#if MSDOS_COMPILER==BORLANDC || MSDOS_COMPILER==DJGPPC + intensevideo(); +#endif SETCOLORS(nm_fg_color, nm_bg_color); #if 0 /* @@ -1462,7 +1486,7 @@ initcolor(void) * Termcap-like init with a private win32 console. */ static void -win32_init_term(void) +win32_init_term() { CONSOLE_SCREEN_BUFFER_INFO scr; COORD size; @@ -1513,10 +1537,12 @@ win32_deinit_term() * Initialize terminal */ public void -init(void) +init() { #if !MSDOS_COMPILER - if (!no_init) + if (quit_if_one_screen && line_count >= sc_height) + quit_if_one_screen = FALSE; + if (!no_init && !quit_if_one_screen) tputs(sc_init, sc_height, putchr); if (!no_keypad) tputs(sc_s_keypad, sc_height, putchr); @@ -1549,14 +1575,14 @@ init(void) * Deinitialize terminal */ public void -deinit(void) +deinit() { if (!init_done) return; #if !MSDOS_COMPILER if (!no_keypad) tputs(sc_e_keypad, sc_height, putchr); - if (!no_init) + if (!no_init && !quit_if_one_screen) tputs(sc_deinit, sc_height, putchr); #else /* Restore system colors. */ @@ -1576,7 +1602,7 @@ deinit(void) * Home cursor (move to upper left corner of screen). */ public void -home(void) +home() { #if !MSDOS_COMPILER tputs(sc_home, 1, putchr); @@ -1591,7 +1617,7 @@ home(void) * Should scroll the display down. */ public void -add_line(void) +add_line() { #if !MSDOS_COMPILER tputs(sc_addline, sc_height, putchr); @@ -1649,7 +1675,8 @@ add_line(void) * into the scrollback buffer when we go down-one-line (in WIN32). */ public void -remove_top(int n) +remove_top(n) + int n; { #if MSDOS_COMPILER==WIN32C SMALL_RECT rcSrc, rcClip; @@ -1702,7 +1729,7 @@ remove_top(int n) * Clear the screen. */ static void -win32_clear(void) +win32_clear() { /* * This will clear only the currently visible rows of the NT @@ -1733,7 +1760,8 @@ win32_clear(void) * window upward. */ public void -win32_scroll_up(int n) +win32_scroll_up(n) + int n; { SMALL_RECT rcSrc, rcClip; CHAR_INFO fillchar; @@ -1798,7 +1826,7 @@ win32_scroll_up(int n) * Move cursor to lower left corner of screen. */ public void -lower_left(void) +lower_left() { #if !MSDOS_COMPILER tputs(sc_lower_left, 1, putchr); @@ -1812,7 +1840,7 @@ lower_left(void) * Move cursor to left position of current line. */ public void -line_left(void) +line_left() { #if !MSDOS_COMPILER tputs(sc_return, 1, putchr); @@ -1844,7 +1872,7 @@ line_left(void) * (in lieu of SIGWINCH for WIN32). */ public void -check_winch(void) +check_winch() { #if MSDOS_COMPILER==WIN32C CONSOLE_SCREEN_BUFFER_INFO scr; @@ -1874,7 +1902,8 @@ check_winch(void) * Goto a specific line on the screen. */ public void -goto_line(int slinenum) +goto_line(slinenum) + int slinenum; { #if !MSDOS_COMPILER tputs(tgoto(sc_move, 0, slinenum), 1, putchr); @@ -1892,7 +1921,7 @@ goto_line(int slinenum) * {{ Yuck! There must be a better way to get a visual bell. }} */ static void -create_flash(void) +create_flash() { #if MSDOS_COMPILER==MSOFTC struct videoconfig w; @@ -1951,7 +1980,7 @@ create_flash(void) * Output the "visual bell", if there is one. */ public void -vbell(void) +vbell() { #if !MSDOS_COMPILER if (*sc_visual_bell == '\0') @@ -2015,7 +2044,7 @@ vbell(void) * Make a noise. */ static void -beep(void) +beep() { #if !MSDOS_COMPILER putchr(CONTROL('G')); @@ -2032,7 +2061,7 @@ beep(void) * Ring the terminal bell. */ public void -bell(void) +bell() { if (quiet == VERY_QUIET) vbell(); @@ -2044,7 +2073,7 @@ bell(void) * Clear the screen. */ public void -clear(void) +clear() { #if !MSDOS_COMPILER tputs(sc_clear, sc_height, putchr); @@ -2063,7 +2092,7 @@ clear(void) * {{ This must not move the cursor. }} */ public void -clear_eol(void) +clear_eol() { #if !MSDOS_COMPILER tputs(sc_eol_clear, 1, putchr); @@ -2122,7 +2151,7 @@ clear_eol(void) * Clear the screen if there's off-screen memory below the display. */ static void -clear_eol_bot(void) +clear_eol_bot() { #if MSDOS_COMPILER clear_eol(); @@ -2139,7 +2168,7 @@ clear_eol_bot(void) * Leave the cursor at the beginning of the bottom line. */ public void -clear_bot(void) +clear_bot() { /* * If we're in a non-normal attribute mode, temporarily exit @@ -2164,7 +2193,8 @@ clear_bot(void) } public void -at_enter(int attr) +at_enter(attr) + int attr; { attr = apply_at_specials(attr); @@ -2202,7 +2232,7 @@ at_enter(int attr) } public void -at_exit(void) +at_exit() { #if !MSDOS_COMPILER /* Undo things in the reverse order we did them. */ @@ -2223,7 +2253,8 @@ at_exit(void) } public void -at_switch(int attr) +at_switch(attr) + int attr; { int new_attrmode = apply_at_specials(attr); int ignore_modes = AT_ANSI; @@ -2236,7 +2267,9 @@ at_switch(int attr) } public int -is_at_equiv(int attr1, int attr2) +is_at_equiv(attr1, attr2) + int attr1; + int attr2; { attr1 = apply_at_specials(attr1); attr2 = apply_at_specials(attr2); @@ -2245,7 +2278,8 @@ is_at_equiv(int attr1, int attr2) } public int -apply_at_specials(int attr) +apply_at_specials(attr) + int attr; { if (attr & AT_BINARY) attr |= binattr; @@ -2262,7 +2296,7 @@ apply_at_specials(int attr) * and move the cursor left. */ public void -backspace(void) +backspace() { #if !MSDOS_COMPILER /* @@ -2311,7 +2345,7 @@ backspace(void) * Output a plain backspace, without erasing the previous char. */ public void -putbs(void) +putbs() { #if !MSDOS_COMPILER tputs(sc_backspace, 1, putchr); @@ -2350,7 +2384,8 @@ putbs(void) * Determine whether an input character is waiting to be read. */ static int -win32_kbhit(HANDLE tty) +win32_kbhit(tty) + HANDLE tty; { INPUT_RECORD ip; DWORD read; @@ -2406,7 +2441,16 @@ win32_kbhit(HANDLE tty) currentKey.scan = PCK_CTL_DELETE; break; } + } else if (ip.Event.KeyEvent.dwControlKeyState & SHIFT_PRESSED) + { + switch (currentKey.scan) + { + case PCK_SHIFT_TAB: /* tab */ + currentKey.ascii = 0; + break; + } } + return (TRUE); } @@ -2414,7 +2458,8 @@ win32_kbhit(HANDLE tty) * Read a character from the keyboard. */ public char -WIN32getch(int tty) +WIN32getch(tty) + int tty; { int ascii; @@ -2447,7 +2492,9 @@ WIN32getch(int tty) /* */ public void -WIN32setcolors(int fg, int bg) +WIN32setcolors(fg, bg) + int fg; + int bg; { SETCOLORS(fg, bg); } @@ -2455,7 +2502,9 @@ WIN32setcolors(int fg, int bg) /* */ public void -WIN32textout(char *text, int len) +WIN32textout(text, len) + char *text; + int len; { #if MSDOS_COMPILER==WIN32C DWORD written; diff --git a/contrib/less/scrsize.c b/contrib/less/scrsize.c index c08d666704b58..001c55ace3853 100644 --- a/contrib/less/scrsize.c +++ b/contrib/less/scrsize.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -45,7 +45,11 @@ #include <stdlib.h> #include <stdio.h> -static int get_winsize(Display *dpy, Window window, int *p_width, int *p_height) +static int get_winsize(dpy, window, p_width, p_height) + Display *dpy; + Window window; + int *p_width; + int *p_height; { XWindowAttributes win_attributes; XSizeHints hints; @@ -75,7 +79,9 @@ static int get_winsize(Display *dpy, Window window, int *p_width, int *p_height) return 0; } -int main(int argc, char *argv[]) +int main(argc, argv) + int argc; + char *argv[]; { char *cp; Display *dpy; diff --git a/contrib/less/search.c b/contrib/less/search.c index cb302a94e4118..84f9288a229ca 100644 --- a/contrib/less/search.c +++ b/contrib/less/search.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -13,7 +13,6 @@ */ #include "less.h" -#include "pattern.h" #include "position.h" #include "charset.h" @@ -29,7 +28,7 @@ extern int jump_sline; extern int bs_mode; extern int ctldisp; extern int status_col; -extern void * constant ml_search; +extern void *ml_search; extern POSITION start_attnpos; extern POSITION end_attnpos; extern int utf_mode; @@ -103,7 +102,7 @@ static struct hilite_tree filter_anchor = HILITE_INITIALIZER(); * search pattern and filter pattern. */ struct pattern_info { - DEFINE_PATTERN(compiled); + PATTERN_TYPE compiled; char* text; int search_type; }; @@ -121,9 +120,10 @@ static struct pattern_info filter_info; * Are there any uppercase letters in this string? */ static int -is_ucase(constant char *str) +is_ucase(str) + char *str; { - constant char *str_end = str + strlen(str); + char *str_end = str + strlen(str); LWCHAR ch; while (str < str_end) @@ -139,13 +139,15 @@ is_ucase(constant char *str) * Compile and save a search pattern. */ static int -set_pattern(struct pattern_info *info, char *pattern, int search_type) +set_pattern(info, pattern, search_type) + struct pattern_info *info; + char *pattern; + int search_type; { #if !NO_REGEX if (pattern == NULL) CLEAR_PATTERN(info->compiled); - else if (compile_pattern(pattern, search_type, - (void **)&info->compiled) < 0) + else if (compile_pattern(pattern, search_type, &info->compiled) < 0) return -1; #endif /* Pattern compiled successfully; save the text too. */ @@ -175,13 +177,14 @@ set_pattern(struct pattern_info *info, char *pattern, int search_type) * Discard a saved pattern. */ static void -clear_pattern(struct pattern_info *info) +clear_pattern(info) + struct pattern_info *info; { if (info->text != NULL) free(info->text); info->text = NULL; #if !NO_REGEX - uncompile_pattern((void **)&info->compiled); + uncompile_pattern(&info->compiled); #endif } @@ -189,7 +192,8 @@ clear_pattern(struct pattern_info *info) * Initialize saved pattern to nothing. */ static void -init_pattern(struct pattern_info *info) +init_pattern(info) + struct pattern_info *info; { CLEAR_PATTERN(info->compiled); info->text = NULL; @@ -200,7 +204,7 @@ init_pattern(struct pattern_info *info) * Initialize search variables. */ public void -init_search(void) +init_search() { init_pattern(&search_info); init_pattern(&filter_info); @@ -210,7 +214,7 @@ init_search(void) * Determine which text conversions to perform before pattern matching. */ static int -get_cvt_ops(void) +get_cvt_ops() { int ops = 0; if (is_caseless || bs_mode == BS_SPECIAL) @@ -234,7 +238,8 @@ get_cvt_ops(void) * Is there a previous (remembered) search pattern? */ static int -prev_pattern(struct pattern_info *info) +prev_pattern(info) + struct pattern_info *info; { #if !NO_REGEX if ((info->search_type & SRCH_NO_REGEX) == 0) @@ -250,7 +255,8 @@ prev_pattern(struct pattern_info *info) * If on==0, force all hilites off. */ public void -repaint_hilite(int on) +repaint_hilite(on) + int on; { int slinenum; POSITION pos; @@ -291,7 +297,7 @@ repaint_hilite(int on) * Clear the attn hilite. */ public void -clear_attn(void) +clear_attn() { int slinenum; POSITION old_start_attnpos; @@ -338,7 +344,7 @@ clear_attn(void) * Hide search string highlighting. */ public void -undo_search(void) +undo_search() { if (!prev_pattern(&search_info)) { @@ -356,7 +362,8 @@ undo_search(void) * Clear the hilite list. */ public void -clr_hlist(struct hilite_tree *anchor) +clr_hlist(anchor) + struct hilite_tree *anchor; { struct hilite_storage *hls; struct hilite_storage *nexthls; @@ -377,13 +384,13 @@ clr_hlist(struct hilite_tree *anchor) } public void -clr_hilite(void) +clr_hilite() { clr_hlist(&hilite_anchor); } public void -clr_filter(void) +clr_filter() { clr_hlist(&filter_anchor); } @@ -512,7 +519,9 @@ hlist_find(anchor, pos) * Should any characters in a specified range be highlighted? */ static int -is_hilited_range(POSITION pos, POSITION epos) +is_hilited_range(pos, epos) + POSITION pos; + POSITION epos; { struct hilite_node *n = hlist_find(&hilite_anchor, pos); return (n != NULL && (epos == NULL_POSITION || epos > n->r.hl_startpos)); @@ -522,7 +531,8 @@ is_hilited_range(POSITION pos, POSITION epos) * Is a line "filtered" -- that is, should it be hidden? */ public int -is_filtered(POSITION pos) +is_filtered(pos) + POSITION pos; { struct hilite_node *n; @@ -538,7 +548,8 @@ is_filtered(POSITION pos) * just return pos. */ public POSITION -next_unfiltered(POSITION pos) +next_unfiltered(pos) + POSITION pos; { struct hilite_node *n; @@ -559,7 +570,8 @@ next_unfiltered(POSITION pos) * we're filtered right to the beginning, otherwise just return pos. */ public POSITION -prev_unfiltered(POSITION pos) +prev_unfiltered(pos) + POSITION pos; { struct hilite_node *n; @@ -584,7 +596,11 @@ prev_unfiltered(POSITION pos) * If nohide is nonzero, don't consider hide_hilite. */ public int -is_hilited(POSITION pos, POSITION epos, int nohide, int *p_matches) +is_hilited(pos, epos, nohide, p_matches) + POSITION pos; + POSITION epos; + int nohide; + int *p_matches; { int match; @@ -630,7 +646,8 @@ is_hilited(POSITION pos, POSITION epos, int nohide, int *p_matches) * capacity, or create a new one if not. */ static struct hilite_storage* -hlist_getstorage(struct hilite_tree *anchor) +hlist_getstorage(anchor) + struct hilite_tree *anchor; { int capacity = 1; struct hilite_storage *s; @@ -660,7 +677,8 @@ hlist_getstorage(struct hilite_tree *anchor) * tree. */ static struct hilite_node* -hlist_getnode(struct hilite_tree *anchor) +hlist_getnode(anchor) + struct hilite_tree *anchor; { struct hilite_storage *s = hlist_getstorage(anchor); return &s->nodes[s->used++]; @@ -670,7 +688,9 @@ hlist_getnode(struct hilite_tree *anchor) * Rotate the tree left around a pivot node. */ static void -hlist_rotate_left(struct hilite_tree *anchor, struct hilite_node *n) +hlist_rotate_left(anchor, n) + struct hilite_tree *anchor; + struct hilite_node *n; { struct hilite_node *np = n->parent; struct hilite_node *nr = n->right; @@ -699,7 +719,9 @@ hlist_rotate_left(struct hilite_tree *anchor, struct hilite_node *n) * Rotate the tree right around a pivot node. */ static void -hlist_rotate_right(struct hilite_tree *anchor, struct hilite_node *n) +hlist_rotate_right(anchor, n) + struct hilite_tree *anchor; + struct hilite_node *n; { struct hilite_node *np = n->parent; struct hilite_node *nl = n->left; @@ -729,7 +751,9 @@ hlist_rotate_right(struct hilite_tree *anchor, struct hilite_node *n) * Add a new hilite to a hilite list. */ static void -add_hilite(struct hilite_tree *anchor, struct hilite *hl) +add_hilite(anchor, hl) + struct hilite_tree *anchor; + struct hilite *hl; { struct hilite_node *p, *n, *u; @@ -906,7 +930,11 @@ add_hilite(struct hilite_tree *anchor, struct hilite *hl) * Hilight every character in a range of displayed characters. */ static void -create_hilites(POSITION linepos, int start_index, int end_index, int *chpos) +create_hilites(linepos, start_index, end_index, chpos) + POSITION linepos; + int start_index; + int end_index; + int *chpos; { struct hilite hl; int i; @@ -943,8 +971,14 @@ create_hilites(POSITION linepos, int start_index, int end_index, int *chpos) * sp,ep delimit the first match already found. */ static void -hilite_line(POSITION linepos, char *line, int line_len, int *chpos, char *sp, - char *ep, int cvt_ops) +hilite_line(linepos, line, line_len, chpos, sp, ep, cvt_ops) + POSITION linepos; + char *line; + int line_len; + int *chpos; + char *sp; + char *ep; + int cvt_ops; { char *searchp; char *line_end = line + line_len; @@ -980,33 +1014,12 @@ hilite_line(POSITION linepos, char *line, int line_len, int *chpos, char *sp, } #endif -/* - * Change the caseless-ness of searches. - * Updates the internal search state to reflect a change in the -i flag. - */ - public void -chg_caseless(void) -{ - if (!is_ucase_pattern) - /* - * Pattern did not have uppercase. - * Just set the search caselessness to the global caselessness. - */ - is_caseless = caseless; - else - /* - * Pattern did have uppercase. - * Discard the pattern; we can't change search caselessness now. - */ - clear_pattern(&search_info); -} - #if HILITE_SEARCH /* * Find matching text which is currently on screen and highlight it. */ static void -hilite_screen(void) +hilite_screen() { struct scrpos scrpos; @@ -1021,7 +1034,7 @@ hilite_screen(void) * Change highlighting parameters. */ public void -chg_hilite(void) +chg_hilite() { /* * Erase any highlights currently on screen. @@ -1041,7 +1054,8 @@ chg_hilite(void) * Figure out where to start a search. */ static POSITION -search_pos(int search_type) +search_pos(search_type) + int search_type; { POSITION pos; int linenum; @@ -1078,18 +1092,18 @@ search_pos(int search_type) * Search does not include current screen. */ if (search_type & SRCH_FORW) - linenum = BOTTOM_PLUS_ONE; + linenum = sc_height-1; /* BOTTOM_PLUS_ONE */ else - linenum = TOP; + linenum = 0; /* TOP */ } else if (how_search == OPT_ONPLUS && !(search_type & SRCH_AFTER_TARGET)) { /* * Search includes all of displayed screen. */ if (search_type & SRCH_FORW) - linenum = TOP; + linenum = 0; /* TOP */ else - linenum = BOTTOM_PLUS_ONE; + linenum = sc_height-1; /* BOTTOM_PLUS_ONE */ } else { /* @@ -1133,8 +1147,14 @@ search_pos(int search_type) * Search a subset of the file, specified by start/end position. */ static int -search_range(POSITION pos, POSITION endpos, int search_type, int matches, - int maxlines, POSITION *plinepos, POSITION *pendpos) +search_range(pos, endpos, search_type, matches, maxlines, plinepos, pendpos) + POSITION pos; + POSITION endpos; + int search_type; + int matches; + int maxlines; + POSITION *plinepos; + POSITION *pendpos; { char *line; char *cline; @@ -1250,6 +1270,8 @@ search_range(POSITION pos, POSITION endpos, int search_type, int matches, hl.hl_startpos = linepos; hl.hl_endpos = pos; add_hilite(&filter_anchor, &hl); + free(cline); + free(chpos); continue; } } @@ -1313,7 +1335,8 @@ search_range(POSITION pos, POSITION endpos, int search_type, int matches, * search for a pattern in history. If found, compile that pattern. */ static int -hist_pattern(int search_type) +hist_pattern(search_type) + int search_type; { #if CMD_HISTORY char *pattern; @@ -1338,6 +1361,30 @@ hist_pattern(int search_type) } /* + * Change the caseless-ness of searches. + * Updates the internal search state to reflect a change in the -i flag. + */ + public void +chg_caseless() +{ + if (!is_ucase_pattern) + /* + * Pattern did not have uppercase. + * Just set the search caselessness to the global caselessness. + */ + is_caseless = caseless; + else + { + /* + * Pattern did have uppercase. + * Regenerate the pattern using the new state. + */ + clear_pattern(&search_info); + hist_pattern(search_info.search_type); + } +} + +/* * Search for the n-th occurrence of a specified pattern, * either forward or backward. * Return the number of matches not yet found in this file @@ -1347,7 +1394,10 @@ hist_pattern(int search_type) * if less than n matches are found in this file. */ public int -search(int search_type, char *pattern, int n) +search(search_type, pattern, n) + int search_type; + char *pattern; + int n; { POSITION pos; @@ -1481,7 +1531,10 @@ search(int search_type, char *pattern, int n) * prep_hilite asks that the range (spos,epos) be covered by the prep region. */ public void -prep_hilite(POSITION spos, POSITION epos, int maxlines) +prep_hilite(spos, epos, maxlines) + POSITION spos; + POSITION epos; + int maxlines; { POSITION nprep_startpos = prep_startpos; POSITION nprep_endpos = prep_endpos; @@ -1648,7 +1701,9 @@ prep_hilite(POSITION spos, POSITION epos, int maxlines) * Set the pattern to be used for line filtering. */ public void -set_filter_pattern(char *pattern, int search_type) +set_filter_pattern(pattern, search_type) + char *pattern; + int search_type; { clr_filter(); if (pattern == NULL || *pattern == '\0') @@ -1662,7 +1717,7 @@ set_filter_pattern(char *pattern, int search_type) * Is there a line filter in effect? */ public int -is_filtering(void) +is_filtering() { if (ch_getflags() & CH_HELPFILE) return (0); diff --git a/contrib/less/signal.c b/contrib/less/signal.c index 6766e1d874172..5cc4b33f92684 100644 --- a/contrib/less/signal.c +++ b/contrib/less/signal.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -42,7 +42,8 @@ extern long jump_sline_fraction; */ /* ARGSUSED*/ static RETSIGTYPE -u_interrupt(int type) +u_interrupt(type) + int type; { bell(); #if OS2 @@ -71,7 +72,8 @@ u_interrupt(int type) */ /* ARGSUSED*/ static RETSIGTYPE -stop(int type) +stop(type) + int type; { LSIGNAL(SIGTSTP, stop); sigs |= S_STOP; @@ -86,7 +88,8 @@ stop(int type) */ /* ARGSUSED*/ public RETSIGTYPE -winch(int type) +winch(type) + int type; { LSIGNAL(SIGWINCH, winch); sigs |= S_WINCH; @@ -100,7 +103,8 @@ winch(int type) */ /* ARGSUSED*/ public RETSIGTYPE -winch(int type) +winch(type) + int type; { LSIGNAL(SIGWIND, winch); sigs |= S_WINCH; @@ -117,7 +121,8 @@ winch(int type) #include "windows.h" static BOOL WINAPI -wbreak_handler(DWORD dwCtrlType) +wbreak_handler(dwCtrlType) + DWORD dwCtrlType; { switch (dwCtrlType) { @@ -136,7 +141,8 @@ wbreak_handler(DWORD dwCtrlType) * Set up the signal handlers. */ public void -init_signals(int on) +init_signals(on) + int on; { if (on) { @@ -188,7 +194,7 @@ init_signals(int on) * A received signal cause a bit to be set in "sigs". */ public void -psignals(void) +psignals() { int tsignals; diff --git a/contrib/less/tags.c b/contrib/less/tags.c index 3fb65b0443f90..f59d25b411f46 100644 --- a/contrib/less/tags.c +++ b/contrib/less/tags.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -43,13 +43,13 @@ enum { T_GPATH /* 'GPATH': path name (global) */ }; -static enum tag_result findctag(char *tag); -static enum tag_result findgtag(char *tag, int type); -static char *nextgtag(void); -static char *prevgtag(void); -static POSITION ctagsearch(void); -static POSITION gtagsearch(void); -static int getentry(char *buf, char **tag, char **file, char **line); +static enum tag_result findctag(); +static enum tag_result findgtag(); +static char *nextgtag(); +static char *prevgtag(); +static POSITION ctagsearch(); +static POSITION gtagsearch(); +static int getentry(); /* * The list of tags generated by the last findgtag() call. @@ -63,8 +63,6 @@ struct taglist { struct tag *tl_first; struct tag *tl_last; }; -#define TAG_END ((struct tag *) &taglist) -static struct taglist taglist = { TAG_END, TAG_END }; struct tag { struct tag *next, *prev; /* List links */ char *tag_file; /* Source file containing the tag */ @@ -72,6 +70,8 @@ struct tag { char *tag_pattern; /* Pattern used to find the tag */ char tag_endline; /* True if the pattern includes '$' */ }; +#define TAG_END ((struct tag *) &taglist) +static struct taglist taglist = { TAG_END, TAG_END }; static struct tag *curtag; #define TAG_INS(tp) \ @@ -88,7 +88,7 @@ static struct tag *curtag; * Delete tag structures. */ public void -cleantags(void) +cleantags() { struct tag *tp; @@ -110,7 +110,12 @@ cleantags(void) * Create a new tag entry. */ static struct tag * -maketagent(char *name, char *file, LINENUM linenum, char *pattern, int endline) +maketagent(name, file, linenum, pattern, endline) + char *name; + char *file; + LINENUM linenum; + char *pattern; + int endline; { struct tag *tp; @@ -133,7 +138,7 @@ maketagent(char *name, char *file, LINENUM linenum, char *pattern, int endline) * Get tag mode. */ public int -gettagtype(void) +gettagtype() { int f; @@ -164,7 +169,8 @@ gettagtype(void) * to find the tag. */ public void -findtag(char *tag) +findtag(tag) + char *tag; { int type = gettagtype(); enum tag_result result; @@ -194,7 +200,7 @@ findtag(char *tag) * Search for a tag. */ public POSITION -tagsearch(void) +tagsearch() { if (curtag == NULL) return (NULL_POSITION); /* No gtags loaded! */ @@ -208,7 +214,8 @@ tagsearch(void) * Go to the next tag. */ public char * -nexttag(int n) +nexttag(n) + int n; { char *tagfile = (char *) NULL; @@ -221,7 +228,8 @@ nexttag(int n) * Go to the previous tag. */ public char * -prevtag(int n) +prevtag(n) + int n; { char *tagfile = (char *) NULL; @@ -234,7 +242,7 @@ prevtag(int n) * Return the total number of tags. */ public int -ntags(void) +ntags() { return total; } @@ -243,7 +251,7 @@ ntags(void) * Return the sequence number of current tag. */ public int -curr_tag(void) +curr_tag() { return curseq; } @@ -257,7 +265,8 @@ curr_tag(void) * Sets curtag to the first tag entry. */ static enum tag_result -findctag(char *tag) +findctag(tag) + char *tag; { char *p; FILE *f; @@ -368,7 +377,7 @@ findctag(char *tag) * Edit current tagged file. */ public int -edit_tagfile(void) +edit_tagfile() { if (curtag == NULL) return (1); @@ -385,7 +394,7 @@ edit_tagfile(void) * parentheses (which are almost always found in a tag). */ static POSITION -ctagsearch(void) +ctagsearch() { POSITION pos, linepos; LINENUM linenum; @@ -461,7 +470,9 @@ ctagsearch(void) * Sets curtag to the first tag entry. */ static enum tag_result -findgtag(char *tag, int type) +findgtag(tag, type) + char *tag; /* tag to load */ + int type; /* tags type */ { char buf[256]; FILE *fp; @@ -594,7 +605,7 @@ static int circular = 0; /* 1: circular tag structure */ * appropriate tag. */ static char * -nextgtag(void) +nextgtag() { struct tag *tp; @@ -624,7 +635,7 @@ nextgtag(void) * at the appropriate tag. */ static char * -prevgtag(void) +prevgtag() { struct tag *tp; @@ -654,7 +665,7 @@ prevgtag(void) * if it was unable to position at the tag, 0 if successful. */ static POSITION -gtagsearch(void) +gtagsearch() { if (curtag == NULL) return (NULL_POSITION); /* No gtags loaded! */ @@ -690,7 +701,11 @@ gtagsearch(void) * into buf. */ static int -getentry(char *buf, char **tag, char **file, char **line) +getentry(buf, tag, file, line) + char *buf; /* standard or extended ctags -x format data */ + char **tag; /* name of the tag we actually found */ + char **file; /* file in which to find this tag */ + char **line; /* line number of file where this tag is found */ { char *p = buf; diff --git a/contrib/less/ttyin.c b/contrib/less/ttyin.c index 7a8364c8af5fd..87cffc28eb26d 100644 --- a/contrib/less/ttyin.c +++ b/contrib/less/ttyin.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -31,7 +31,7 @@ extern int utf_mode; * Open keyboard for input. */ public void -open_getchr(void) +open_getchr() { #if MSDOS_COMPILER==WIN32C /* Need this to let child processes inherit our console handle */ @@ -85,7 +85,7 @@ open_getchr(void) * Close the keyboard. */ public void -close_getchr(void) +close_getchr() { #if MSDOS_COMPILER==WIN32C SetConsoleMode((HANDLE)tty, console_mode); @@ -97,7 +97,7 @@ close_getchr(void) * Get a character from the keyboard. */ public int -getchr(void) +getchr() { char c; int result; @@ -120,7 +120,11 @@ getchr(void) if (c == '\003') return (READ_INTR); #else - result = iread(tty, &c, sizeof(char)); + { + unsigned char uc; + result = iread(tty, &uc, sizeof(char)); + c = (char) uc; + } if (result == READ_INTR) return (READ_INTR); if (result < 0) @@ -135,8 +139,8 @@ getchr(void) #if 0 /* allow entering arbitrary hex chars for testing */ /* ctrl-A followed by two hex chars makes a byte */ { - int hex_in = 0; - int hex_value = 0; + static int hex_in = 0; + static int hex_value = 0; if (c == CONTROL('A')) { hex_in = 2; diff --git a/contrib/less/ubin.uni b/contrib/less/ubin.uni index 39ec303f76505..17c45903003fc 100644 --- a/contrib/less/ubin.uni +++ b/contrib/less/ubin.uni @@ -1,4 +1,4 @@ -/* Generated by "./mkutable -f2 Cc Cf Cs Co Zl Zp -- unicode/UnicodeData.txt" on Mon Jul 14 16:21:22 PDT 2014 */ +/* Generated by "./mkutable -f2 Cc Cf Cs Co Zl Zp -- unicode/UnicodeData.txt" on Tue Sep 20 10:51:43 PDT 2016 */ { 0x0000, 0x001f }, /* Cc */ { 0x007f, 0x009f }, /* Cc */ { 0x00ad, 0x00ad }, /* Cf */ @@ -6,6 +6,7 @@ { 0x061c, 0x061c }, /* Cf */ { 0x06dd, 0x06dd }, /* Cf */ { 0x070f, 0x070f }, /* Cf */ + { 0x08e2, 0x08e2 }, /* Cf */ { 0x180e, 0x180e }, /* Cf */ { 0x200b, 0x200f }, /* Cf */ { 0x2028, 0x2028 }, /* Zl */ diff --git a/contrib/less/version.c b/contrib/less/version.c index 87b0b42bc8556..3240d4b2ce255 100644 --- a/contrib/less/version.c +++ b/contrib/less/version.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 1984-2015 Mark Nudelman + * Copyright (C) 1984-2017 Mark Nudelman * * You may distribute under the terms of either the GNU General Public * License or the Less License, as specified in the README file. @@ -789,8 +789,28 @@ v477 5/19/15 Fix off-by-one in jump_forw_buffered; don't add FAKE_* files to cmd history. v478 5/21/15 Fix nonportable pointer usage in hilite tree. v479 7/6/15 Allow %% escapes in LESSOPEN variable. -v480 7/24/15 Fix bug in no-regex searches; support MSVC v1900. -v481 8/20/15 Fix broken -g option. +v480 7/24/15 Fix bug in no-regex searches; support MSVC v1900. +v481 8/20/15 Fix broken -g option. +----------------------------------------------------------------- +v482 2/25/16 Update Unicode database to "2015-06-16, 20:24:00 GMT [KW]". +v483 2/27/16 Regenerate hilite when change search caselessness. + (Thanks to Jason Hood) + Fix bug when terminal has no "cm". (Thanks to Noel Cragg) +v484 9/20/16 Update to Unicode 9.0.0 database. +v485 10/21/16 Fix "nothing to search" bug when top/bottom line is empty; + Display line numbers in bold. (thanks to Jason Hood); + Fix incorrect display when entering double-width chars in + search string. +v486 10/22/16 New commands ESC-{ and ESC-} to shift to start/end of + displayed lines; new option -Da in Windows version to + enable SGR mode (thanks to Jason Hood). +v487 10/23/16 configure --help formatting. +----------------------------------------------------------------- +v488 2/23/17 Fix memory leaks in search (thanks to John Brooks). +v489 3/30/17 Make -F not do init/deinit if file fits on one screen + (thanks to Jindrich Novy). +v490 4/5/17 Switch to ANSI prototypes in funcs.h; remove "register". +v491 4/7/17 Fix signed char bug. */ -char version[] = "481"; +char version[] = "491"; diff --git a/contrib/less/wide.uni b/contrib/less/wide.uni index b43f7d3598fc4..17c385914f7a8 100644 --- a/contrib/less/wide.uni +++ b/contrib/less/wide.uni @@ -1,81 +1,108 @@ -/* Generated by "./mkutable -f1 W -- unicode/EastAsianWidth.txt" on Mon Jul 14 16:21:23 PDT 2014 */ - { 0x1100, 0x1100 }, /* W */ +/* Generated by "./mkutable -f1 W F -- unicode/EastAsianWidth.txt" on Tue Sep 20 10:51:43 PDT 2016 */ + { 0x1100, 0x115f }, /* W */ + { 0x231a, 0x231b }, /* W */ { 0x2329, 0x232a }, /* W */ - { 0x2e80, 0x2e80 }, /* W */ - { 0x2e9b, 0x2e9b }, /* W */ - { 0x2f00, 0x2f00 }, /* W */ - { 0x2ff0, 0x2ff0 }, /* W */ - { 0x3001, 0x3001 }, /* W */ - { 0x3004, 0x3012 }, /* W */ - { 0x3014, 0x301e }, /* W */ - { 0x3020, 0x3021 }, /* W */ - { 0x302a, 0x302a }, /* W */ - { 0x302e, 0x302e }, /* W */ - { 0x3030, 0x3031 }, /* W */ - { 0x3036, 0x3036 }, /* W */ - { 0x3038, 0x3038 }, /* W */ - { 0x303b, 0x303e }, /* W */ - { 0x3041, 0x3041 }, /* W */ - { 0x3099, 0x3099 }, /* W */ - { 0x309b, 0x309b }, /* W */ - { 0x309d, 0x309d }, /* W */ - { 0x309f, 0x30a1 }, /* W */ - { 0x30fb, 0x30fc }, /* W */ - { 0x30ff, 0x30ff }, /* W */ - { 0x3105, 0x3105 }, /* W */ - { 0x3131, 0x3131 }, /* W */ - { 0x3190, 0x3190 }, /* W */ - { 0x3192, 0x3192 }, /* W */ - { 0x3196, 0x3196 }, /* W */ - { 0x31a0, 0x31a0 }, /* W */ - { 0x31c0, 0x31c0 }, /* W */ - { 0x31f0, 0x31f0 }, /* W */ - { 0x3200, 0x3200 }, /* W */ - { 0x3220, 0x3220 }, /* W */ - { 0x322a, 0x322a }, /* W */ - { 0x3250, 0x3251 }, /* W */ - { 0x3260, 0x3260 }, /* W */ - { 0x3280, 0x3280 }, /* W */ - { 0x328a, 0x328a }, /* W */ - { 0x32b1, 0x32b1 }, /* W */ - { 0x32c0, 0x32c0 }, /* W */ - { 0x3300, 0x3300 }, /* W */ - { 0x3400, 0x3400 }, /* W */ - { 0x4db6, 0x4db6 }, /* W */ - { 0x4e00, 0x4e00 }, /* W */ - { 0x9fcd, 0x9fcd }, /* W */ - { 0xa000, 0xa000 }, /* W */ - { 0xa015, 0xa016 }, /* W */ - { 0xa490, 0xa490 }, /* W */ - { 0xa960, 0xa960 }, /* W */ - { 0xac00, 0xac00 }, /* W */ - { 0xf900, 0xf900 }, /* W */ - { 0xfa6e, 0xfa6e }, /* W */ - { 0xfa70, 0xfa70 }, /* W */ - { 0xfada, 0xfada }, /* W */ - { 0xfe10, 0xfe10 }, /* W */ - { 0xfe17, 0xfe19 }, /* W */ - { 0xfe30, 0xfe31 }, /* W */ - { 0xfe33, 0xfe33 }, /* W */ - { 0xfe35, 0xfe45 }, /* W */ - { 0xfe47, 0xfe49 }, /* W */ - { 0xfe4d, 0xfe4d }, /* W */ - { 0xfe50, 0xfe50 }, /* W */ - { 0xfe54, 0xfe54 }, /* W */ - { 0xfe58, 0xfe5f }, /* W */ - { 0xfe62, 0xfe64 }, /* W */ - { 0xfe68, 0xfe6a }, /* W */ - { 0x1b000, 0x1b000 }, /* W */ - { 0x1f200, 0x1f200 }, /* W */ - { 0x1f210, 0x1f210 }, /* W */ - { 0x1f240, 0x1f240 }, /* W */ - { 0x1f250, 0x1f250 }, /* W */ - { 0x20000, 0x20000 }, /* W */ - { 0x2a6d7, 0x2a6d7 }, /* W */ - { 0x2a700, 0x2a700 }, /* W */ - { 0x2b735, 0x2b735 }, /* W */ - { 0x2b740, 0x2b740 }, /* W */ - { 0x2b81e, 0x2b81e }, /* W */ - { 0x2f800, 0x2f800 }, /* W */ - { 0x2fa1e, 0x2fa1e }, /* W */ - { 0x30000, 0x30000 }, /* W */ + { 0x23e9, 0x23ec }, /* W */ + { 0x23f0, 0x23f0 }, /* W */ + { 0x23f3, 0x23f3 }, /* W */ + { 0x25fd, 0x25fe }, /* W */ + { 0x2614, 0x2615 }, /* W */ + { 0x2648, 0x2653 }, /* W */ + { 0x267f, 0x267f }, /* W */ + { 0x2693, 0x2693 }, /* W */ + { 0x26a1, 0x26a1 }, /* W */ + { 0x26aa, 0x26ab }, /* W */ + { 0x26bd, 0x26be }, /* W */ + { 0x26c4, 0x26c5 }, /* W */ + { 0x26ce, 0x26ce }, /* W */ + { 0x26d4, 0x26d4 }, /* W */ + { 0x26ea, 0x26ea }, /* W */ + { 0x26f2, 0x26f3 }, /* W */ + { 0x26f5, 0x26f5 }, /* W */ + { 0x26fa, 0x26fa }, /* W */ + { 0x26fd, 0x26fd }, /* W */ + { 0x2705, 0x2705 }, /* W */ + { 0x270a, 0x270b }, /* W */ + { 0x2728, 0x2728 }, /* W */ + { 0x274c, 0x274c }, /* W */ + { 0x274e, 0x274e }, /* W */ + { 0x2753, 0x2755 }, /* W */ + { 0x2757, 0x2757 }, /* W */ + { 0x2795, 0x2797 }, /* W */ + { 0x27b0, 0x27b0 }, /* W */ + { 0x27bf, 0x27bf }, /* W */ + { 0x2b1b, 0x2b1c }, /* W */ + { 0x2b50, 0x2b50 }, /* W */ + { 0x2b55, 0x2b55 }, /* W */ + { 0x2e80, 0x2e99 }, /* W */ + { 0x2e9b, 0x2ef3 }, /* W */ + { 0x2f00, 0x2fd5 }, /* W */ + { 0x2ff0, 0x2ffb }, /* W */ + { 0x3000, 0x3000 }, /* F */ + { 0x3001, 0x303e }, /* W */ + { 0x3041, 0x3096 }, /* W */ + { 0x3099, 0x30ff }, /* W */ + { 0x3105, 0x312d }, /* W */ + { 0x3131, 0x318e }, /* W */ + { 0x3190, 0x31ba }, /* W */ + { 0x31c0, 0x31e3 }, /* W */ + { 0x31f0, 0x321e }, /* W */ + { 0x3220, 0x3247 }, /* W */ + { 0x3250, 0x32fe }, /* W */ + { 0x3300, 0x4dbf }, /* W */ + { 0x4e00, 0xa48c }, /* W */ + { 0xa490, 0xa4c6 }, /* W */ + { 0xa960, 0xa97c }, /* W */ + { 0xac00, 0xd7a3 }, /* W */ + { 0xf900, 0xfaff }, /* W */ + { 0xfe10, 0xfe19 }, /* W */ + { 0xfe30, 0xfe52 }, /* W */ + { 0xfe54, 0xfe66 }, /* W */ + { 0xfe68, 0xfe6b }, /* W */ + { 0xff01, 0xff60 }, /* F */ + { 0xffe0, 0xffe6 }, /* F */ + { 0x16fe0, 0x16fe0 }, /* W */ + { 0x17000, 0x187ec }, /* W */ + { 0x18800, 0x18af2 }, /* W */ + { 0x1b000, 0x1b001 }, /* W */ + { 0x1f004, 0x1f004 }, /* W */ + { 0x1f0cf, 0x1f0cf }, /* W */ + { 0x1f18e, 0x1f18e }, /* W */ + { 0x1f191, 0x1f19a }, /* W */ + { 0x1f200, 0x1f202 }, /* W */ + { 0x1f210, 0x1f23b }, /* W */ + { 0x1f240, 0x1f248 }, /* W */ + { 0x1f250, 0x1f251 }, /* W */ + { 0x1f300, 0x1f320 }, /* W */ + { 0x1f32d, 0x1f335 }, /* W */ + { 0x1f337, 0x1f37c }, /* W */ + { 0x1f37e, 0x1f393 }, /* W */ + { 0x1f3a0, 0x1f3ca }, /* W */ + { 0x1f3cf, 0x1f3d3 }, /* W */ + { 0x1f3e0, 0x1f3f0 }, /* W */ + { 0x1f3f4, 0x1f3f4 }, /* W */ + { 0x1f3f8, 0x1f43e }, /* W */ + { 0x1f440, 0x1f440 }, /* W */ + { 0x1f442, 0x1f4fc }, /* W */ + { 0x1f4ff, 0x1f53d }, /* W */ + { 0x1f54b, 0x1f54e }, /* W */ + { 0x1f550, 0x1f567 }, /* W */ + { 0x1f57a, 0x1f57a }, /* W */ + { 0x1f595, 0x1f596 }, /* W */ + { 0x1f5a4, 0x1f5a4 }, /* W */ + { 0x1f5fb, 0x1f64f }, /* W */ + { 0x1f680, 0x1f6c5 }, /* W */ + { 0x1f6cc, 0x1f6cc }, /* W */ + { 0x1f6d0, 0x1f6d2 }, /* W */ + { 0x1f6eb, 0x1f6ec }, /* W */ + { 0x1f6f4, 0x1f6f6 }, /* W */ + { 0x1f910, 0x1f91e }, /* W */ + { 0x1f920, 0x1f927 }, /* W */ + { 0x1f930, 0x1f930 }, /* W */ + { 0x1f933, 0x1f93e }, /* W */ + { 0x1f940, 0x1f94b }, /* W */ + { 0x1f950, 0x1f95e }, /* W */ + { 0x1f980, 0x1f991 }, /* W */ + { 0x1f9c0, 0x1f9c0 }, /* W */ + { 0x20000, 0x2fffd }, /* W */ + { 0x30000, 0x3fffd }, /* W */ diff --git a/contrib/netbsd-tests/usr.bin/grep/t_grep.sh b/contrib/netbsd-tests/usr.bin/grep/t_grep.sh index 267ee87446ee0..57d17172d1bf5 100755 --- a/contrib/netbsd-tests/usr.bin/grep/t_grep.sh +++ b/contrib/netbsd-tests/usr.bin/grep/t_grep.sh @@ -494,6 +494,34 @@ wv_combo_break_body() atf_check -s exit:1 grep -v -w "x" test1 atf_check -s exit:1 grep -v -w "x" test2 } + +atf_test_case grep_nomatch_flags +grep_nomatch_flags_head() +{ + atf_set "descr" "Check for no match (-c, -l, -L, -q) flags not producing line matches or context (PR 219077)" +} + +grep_nomatch_flags_body() +{ + printf "A\nB\nC\n" > test1 + + atf_check -o inline:"1\n" grep -c -C 1 -e "B" test1 + atf_check -o inline:"1\n" grep -c -B 1 -e "B" test1 + atf_check -o inline:"1\n" grep -c -A 1 -e "B" test1 + atf_check -o inline:"1\n" grep -c -C 1 -e "B" test1 + + atf_check -o inline:"test1\n" grep -l -e "B" test1 + atf_check -o inline:"test1\n" grep -l -B 1 -e "B" test1 + atf_check -o inline:"test1\n" grep -l -A 1 -e "B" test1 + atf_check -o inline:"test1\n" grep -l -C 1 -e "B" test1 + + atf_check -s exit:1 -o inline:"test1\n" grep -L -e "D" test1 + + atf_check -o empty grep -q -e "B" test1 + atf_check -o empty grep -q -B 1 -e "B" test1 + atf_check -o empty grep -q -A 1 -e "B" test1 + atf_check -o empty grep -q -C 1 -e "B" test1 +} # End FreeBSD atf_init_test_cases() @@ -527,5 +555,6 @@ atf_init_test_cases() atf_add_test_case fgrep_sanity atf_add_test_case egrep_sanity atf_add_test_case grep_sanity + atf_add_test_case grep_nomatch_flags # End FreeBSD } diff --git a/contrib/openpam/HISTORY b/contrib/openpam/HISTORY index 12efa5564bd49..bfcc4726652f9 100644 --- a/contrib/openpam/HISTORY +++ b/contrib/openpam/HISTORY @@ -1,3 +1,23 @@ +OpenPAM Resedacea 2017-04-30 + + - BUGFIX: Reinstore the NULL check in pam_end(3) which was removed in + OpenPAM Radula, as it breaks common error-handling constructs. + + - BUGFIX: Return PAM_SYMBOL_ERR instead of PAM_SYSTEM_ERR from the + dispatcher when the required service function could not be found. + + - ENHANCE: Introduce the PAM_BAD_HANDLE error code for when pamh is + NULL in API functions that have a NULL check. + + - ENHANCE: Introduce the PAM_BAD_ITEM, PAM_BAD_FEATURE and + PAM_BAD_CONSTANT error codes for situations where we previously + incorrectly used PAM_SYMBOL_ERR to denote that an invalid constant + had been passed to an API function. + + - ENHANCE: Improve the RETURN VALUES section in API man pages, + especially for functions that cannot fail, which were incorrectly + documented as returning -1 on failure. +============================================================================ OpenPAM Radula 2017-02-19 - BUGFIX: Fix an inverted test which prevented pam_get_authtok(3) and diff --git a/contrib/openpam/Makefile.am b/contrib/openpam/Makefile.am index a716d6cf024c0..6355cefcd5bc7 100644 --- a/contrib/openpam/Makefile.am +++ b/contrib/openpam/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 917 2017-02-18 14:45:27Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ ACLOCAL_AMFLAGS = -I m4 diff --git a/contrib/openpam/Makefile.in b/contrib/openpam/Makefile.in index b70b7895112e4..80b8f14d11314 100644 --- a/contrib/openpam/Makefile.in +++ b/contrib/openpam/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 917 2017-02-18 14:45:27Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ @@ -165,7 +165,7 @@ CSCOPE = cscope DIST_SUBDIRS = misc include lib bin modules doc t am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in \ $(srcdir)/mkpkgng.in INSTALL README TODO compile config.guess \ - config.sub depcomp install-sh ltmain.sh missing + config.sub install-sh ltmain.sh missing DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) distdir = $(PACKAGE)-$(VERSION) top_distdir = $(distdir) diff --git a/contrib/openpam/RELNOTES b/contrib/openpam/RELNOTES index 87511047f0dec..23b159a29366a 100644 --- a/contrib/openpam/RELNOTES +++ b/contrib/openpam/RELNOTES @@ -1,6 +1,6 @@ - Release notes for OpenPAM Radula - ================================ + Release notes for OpenPAM Resedacea + =================================== OpenPAM is developed primarily on FreeBSD, but is expected to work on almost any POSIX-like platform that has GNU autotools, GNU make and diff --git a/contrib/openpam/autogen.sh b/contrib/openpam/autogen.sh index 469ee1159b371..4573429d4ee99 100755 --- a/contrib/openpam/autogen.sh +++ b/contrib/openpam/autogen.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# $Id: autogen.sh 814 2014-09-12 07:46:46Z des $ +# $OpenPAM: autogen.sh 938 2017-04-30 21:34:42Z des $ # libtoolize --copy --force diff --git a/contrib/openpam/bin/Makefile.am b/contrib/openpam/bin/Makefile.am index 4e24efb26fdc5..ce57b7a74ab71 100644 --- a/contrib/openpam/bin/Makefile.am +++ b/contrib/openpam/bin/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ SUBDIRS = openpam_dump_policy diff --git a/contrib/openpam/bin/Makefile.in b/contrib/openpam/bin/Makefile.in index d381ec35cb161..24ccda7b7865e 100644 --- a/contrib/openpam/bin/Makefile.in +++ b/contrib/openpam/bin/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ diff --git a/contrib/openpam/bin/openpam_dump_policy/Makefile.am b/contrib/openpam/bin/openpam_dump_policy/Makefile.am index 23f61df0c7712..3769cd124610e 100644 --- a/contrib/openpam/bin/openpam_dump_policy/Makefile.am +++ b/contrib/openpam/bin/openpam_dump_policy/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/lib/libpam diff --git a/contrib/openpam/bin/openpam_dump_policy/Makefile.in b/contrib/openpam/bin/openpam_dump_policy/Makefile.in index 24296379d57f8..bf89f6e5e878c 100644 --- a/contrib/openpam/bin/openpam_dump_policy/Makefile.in +++ b/contrib/openpam/bin/openpam_dump_policy/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/bin/openpam_dump_policy/openpam_dump_policy.c b/contrib/openpam/bin/openpam_dump_policy/openpam_dump_policy.c index 8047005d18622..fc78100fb581b 100644 --- a/contrib/openpam/bin/openpam_dump_policy/openpam_dump_policy.c +++ b/contrib/openpam/bin/openpam_dump_policy/openpam_dump_policy.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_dump_policy.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_dump_policy.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/bin/pamtest/Makefile.am b/contrib/openpam/bin/pamtest/Makefile.am index 7d2c7357544fb..b490fab8f6aef 100644 --- a/contrib/openpam/bin/pamtest/Makefile.am +++ b/contrib/openpam/bin/pamtest/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ AM_CPPFLAGS = -I$(top_srcdir)/include diff --git a/contrib/openpam/bin/pamtest/Makefile.in b/contrib/openpam/bin/pamtest/Makefile.in index f5080e6dd5188..6659af8a3ef54 100644 --- a/contrib/openpam/bin/pamtest/Makefile.in +++ b/contrib/openpam/bin/pamtest/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/bin/pamtest/pamtest.1 b/contrib/openpam/bin/pamtest/pamtest.1 index 3eee8bfd1dd1e..0b142abe07779 100644 --- a/contrib/openpam/bin/pamtest/pamtest.1 +++ b/contrib/openpam/bin/pamtest/pamtest.1 @@ -1,5 +1,5 @@ .\"- -.\" Copyright (c) 2011 Dag-Erling Smørgrav +.\" Copyright (c) 2011-2017 Dag-Erling Smørgrav .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -26,9 +26,9 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $Id: pamtest.1 924 2017-02-19 20:52:28Z des $ +.\" $OpenPAM: pamtest.1 939 2017-04-30 21:36:50Z des $ .\" -.Dd February 19, 2017 +.Dd April 30, 2017 .Dt PAMTEST 1 .Os .Sh NAME diff --git a/contrib/openpam/bin/pamtest/pamtest.c b/contrib/openpam/bin/pamtest/pamtest.c index e67bf7766f90b..3eb0768774d05 100644 --- a/contrib/openpam/bin/pamtest/pamtest.c +++ b/contrib/openpam/bin/pamtest/pamtest.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pamtest.c 685 2013-07-11 16:33:34Z des $ + * $OpenPAM: pamtest.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/bin/su/Makefile.am b/contrib/openpam/bin/su/Makefile.am index 0f95c001f8ad2..c9f3b23bb33b4 100644 --- a/contrib/openpam/bin/su/Makefile.am +++ b/contrib/openpam/bin/su/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ AM_CPPFLAGS = -I$(top_srcdir)/include diff --git a/contrib/openpam/bin/su/Makefile.in b/contrib/openpam/bin/su/Makefile.in index 6cb42e98874dc..82e6f99bc83e9 100644 --- a/contrib/openpam/bin/su/Makefile.in +++ b/contrib/openpam/bin/su/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/bin/su/su.1 b/contrib/openpam/bin/su/su.1 index 996fcba2766e7..e6fceb061d46b 100644 --- a/contrib/openpam/bin/su/su.1 +++ b/contrib/openpam/bin/su/su.1 @@ -1,5 +1,5 @@ .\"- -.\" Copyright (c) 2011 Dag-Erling Smørgrav +.\" Copyright (c) 2011-2017 Dag-Erling Smørgrav .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -26,9 +26,9 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $Id: su.1 924 2017-02-19 20:52:28Z des $ +.\" $OpenPAM: su.1 939 2017-04-30 21:36:50Z des $ .\" -.Dd February 19, 2017 +.Dd April 30, 2017 .Dt SU 1 .Os .Sh NAME diff --git a/contrib/openpam/bin/su/su.c b/contrib/openpam/bin/su/su.c index 18a3cee75d377..cb83628f3c652 100644 --- a/contrib/openpam/bin/su/su.c +++ b/contrib/openpam/bin/su/su.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: su.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: su.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/configure b/contrib/openpam/configure index 703ae7649da26..49db637d1ed77 100755 --- a/contrib/openpam/configure +++ b/contrib/openpam/configure @@ -1,7 +1,7 @@ #! /bin/sh -# From configure.ac Id: configure.ac 924 2017-02-19 20:52:28Z des . +# From configure.ac OpenPAM: configure.ac 939 2017-04-30 21:36:50Z des . # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for OpenPAM 20170219. +# Generated by GNU Autoconf 2.69 for OpenPAM 20170430. # # Report bugs to <des@des.no>. # @@ -590,8 +590,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='OpenPAM' PACKAGE_TARNAME='openpam' -PACKAGE_VERSION='20170219' -PACKAGE_STRING='OpenPAM 20170219' +PACKAGE_VERSION='20170430' +PACKAGE_STRING='OpenPAM 20170430' PACKAGE_BUGREPORT='des@des.no' PACKAGE_URL='http://www.openpam.org/' @@ -1345,7 +1345,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures OpenPAM 20170219 to adapt to many kinds of systems. +\`configure' configures OpenPAM 20170430 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1415,7 +1415,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of OpenPAM 20170219:";; + short | recursive ) echo "Configuration of OpenPAM 20170430:";; esac cat <<\_ACEOF @@ -1540,7 +1540,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -OpenPAM configure 20170219 +OpenPAM configure 20170430 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1909,7 +1909,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by OpenPAM $as_me 20170219, which was +It was created by OpenPAM $as_me 20170430, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2774,7 +2774,7 @@ fi # Define the identity of the package. PACKAGE='openpam' - VERSION='20170219' + VERSION='20170430' cat >>confdefs.h <<_ACEOF @@ -13577,7 +13577,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by OpenPAM $as_me 20170219, which was +This file was extended by OpenPAM $as_me 20170430, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -13644,7 +13644,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -OpenPAM config.status 20170219 +OpenPAM config.status 20170430 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/contrib/openpam/configure.ac b/contrib/openpam/configure.ac index 88feebc5394f7..57a8b9d02aa30 100644 --- a/contrib/openpam/configure.ac +++ b/contrib/openpam/configure.ac @@ -1,8 +1,8 @@ -dnl $Id: configure.ac 924 2017-02-19 20:52:28Z des $ +dnl $OpenPAM: configure.ac 939 2017-04-30 21:36:50Z des $ AC_PREREQ([2.62]) -AC_REVISION([$Id: configure.ac 924 2017-02-19 20:52:28Z des $]) -AC_INIT([OpenPAM], [20170219], [des@des.no], [openpam], [http://www.openpam.org/]) +AC_REVISION([$OpenPAM: configure.ac 939 2017-04-30 21:36:50Z des $]) +AC_INIT([OpenPAM], [20170430], [des@des.no], [openpam], [http://www.openpam.org/]) AC_CONFIG_SRCDIR([lib/libpam/pam_start.c]) AC_CONFIG_MACRO_DIR([m4]) AM_INIT_AUTOMAKE([foreign]) diff --git a/contrib/openpam/doc/Makefile.am b/contrib/openpam/doc/Makefile.am index cb9f04a1f6feb..62b1f3518d44f 100644 --- a/contrib/openpam/doc/Makefile.am +++ b/contrib/openpam/doc/Makefile.am @@ -1,3 +1,3 @@ -# $Id: Makefile.am 648 2013-03-05 17:54:27Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ SUBDIRS = man diff --git a/contrib/openpam/doc/Makefile.in b/contrib/openpam/doc/Makefile.in index 21da50599f8e9..5a310ff1d9822 100644 --- a/contrib/openpam/doc/Makefile.in +++ b/contrib/openpam/doc/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 648 2013-03-05 17:54:27Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ diff --git a/contrib/openpam/doc/man/Makefile.am b/contrib/openpam/doc/man/Makefile.am index b1a8133f0c353..a5431bcd19795 100644 --- a/contrib/openpam/doc/man/Makefile.am +++ b/contrib/openpam/doc/man/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ NULL = diff --git a/contrib/openpam/doc/man/Makefile.in b/contrib/openpam/doc/man/Makefile.in index 797058bad7aaf..14c1db300436a 100644 --- a/contrib/openpam/doc/man/Makefile.in +++ b/contrib/openpam/doc/man/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ false; \ diff --git a/contrib/openpam/doc/man/openpam.3 b/contrib/openpam/doc/man/openpam.3 index 78ec8db0aafe4..4c39ffe3483f3 100644 --- a/contrib/openpam/doc/man/openpam.3 +++ b/contrib/openpam/doc/man/openpam.3 @@ -1,5 +1,5 @@ .\" Generated by gendoc.pl -.Dd February 19, 2017 +.Dd April 30, 2017 .Dt OPENPAM 3 .Os .Sh NAME @@ -81,7 +81,7 @@ .Ft "int" .Fn pam_vprompt "const pam_handle_t *pamh" "int style" "char **resp" "const char *fmt" "va_list ap" .\" -.\" $Id: openpam.man 648 2013-03-05 17:54:27Z des $ +.\" $OpenPAM: openpam.man 938 2017-04-30 21:34:42Z des $ .\" .Sh DESCRIPTION These functions are OpenPAM extensions to the PAM API. diff --git a/contrib/openpam/doc/man/openpam.man b/contrib/openpam/doc/man/openpam.man index a7dd7f0a9ff8e..251d41a974226 100644 --- a/contrib/openpam/doc/man/openpam.man +++ b/contrib/openpam/doc/man/openpam.man @@ -1,5 +1,5 @@ .\" -.\" $Id: openpam.man 648 2013-03-05 17:54:27Z des $ +.\" $OpenPAM: openpam.man 938 2017-04-30 21:34:42Z des $ .\" .Sh DESCRIPTION These functions are OpenPAM extensions to the PAM API. diff --git a/contrib/openpam/doc/man/openpam_borrow_cred.3 b/contrib/openpam/doc/man/openpam_borrow_cred.3 index bf25a475fbd69..687ad425ed6f5 100644 --- a/contrib/openpam/doc/man/openpam_borrow_cred.3 +++ b/contrib/openpam/doc/man/openpam_borrow_cred.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_borrow_cred.c by gendoc.pl -.\" $Id: openpam_borrow_cred.c 649 2013-03-05 17:58:33Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_borrow_cred.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_BORROW_CRED 3 .Os .Sh NAME @@ -29,6 +29,8 @@ The .Fn openpam_borrow_cred function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_PERM_DENIED diff --git a/contrib/openpam/doc/man/openpam_free_data.3 b/contrib/openpam/doc/man/openpam_free_data.3 index ca2fe51dfd929..a804c2a51579a 100644 --- a/contrib/openpam/doc/man/openpam_free_data.3 +++ b/contrib/openpam/doc/man/openpam_free_data.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_free_data.c by gendoc.pl -.\" $Id: openpam_free_data.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_free_data.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_FREE_DATA 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_free_envlist.3 b/contrib/openpam/doc/man/openpam_free_envlist.3 index f8ed6f08e2c30..d069ba00407f0 100644 --- a/contrib/openpam/doc/man/openpam_free_envlist.3 +++ b/contrib/openpam/doc/man/openpam_free_envlist.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_free_envlist.c by gendoc.pl -.\" $Id: openpam_free_envlist.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_free_envlist.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_FREE_ENVLIST 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_get_feature.3 b/contrib/openpam/doc/man/openpam_get_feature.3 index 45b6bcc91d69b..daf49a4b63d1e 100644 --- a/contrib/openpam/doc/man/openpam_get_feature.3 +++ b/contrib/openpam/doc/man/openpam_get_feature.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_get_feature.c by gendoc.pl -.\" $Id: openpam_get_feature.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_get_feature.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_GET_FEATURE 3 .Os .Sh NAME @@ -50,8 +50,10 @@ The .Fn openpam_get_feature function returns one of the following values: .Bl -tag -width 18n -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_FEATURE +Unrecognized or restricted feature. .El .Sh SEE ALSO .Xr openpam_set_feature 3 , diff --git a/contrib/openpam/doc/man/openpam_get_option.3 b/contrib/openpam/doc/man/openpam_get_option.3 index fcfd9b83dd008..bdf73db9709c5 100644 --- a/contrib/openpam/doc/man/openpam_get_option.3 +++ b/contrib/openpam/doc/man/openpam_get_option.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_get_option.c by gendoc.pl -.\" $Id: openpam_get_option.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_get_option.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_GET_OPTION 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_log.3 b/contrib/openpam/doc/man/openpam_log.3 index 0930abb17ccba..a761547fc4345 100644 --- a/contrib/openpam/doc/man/openpam_log.3 +++ b/contrib/openpam/doc/man/openpam_log.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_log.c by gendoc.pl -.\" $Id: openpam_log.c 686 2013-07-11 16:36:02Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_log.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_LOG 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_nullconv.3 b/contrib/openpam/doc/man/openpam_nullconv.3 index 0ce1e44edeadc..7a8139ebe0ec9 100644 --- a/contrib/openpam/doc/man/openpam_nullconv.3 +++ b/contrib/openpam/doc/man/openpam_nullconv.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_nullconv.c by gendoc.pl -.\" $Id: openpam_nullconv.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_nullconv.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_NULLCONV 3 .Os .Sh NAME @@ -38,6 +38,8 @@ The .Fn openpam_nullconv function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_CONV_ERR Conversation failure. .El diff --git a/contrib/openpam/doc/man/openpam_readline.3 b/contrib/openpam/doc/man/openpam_readline.3 index db888ad97cfb0..e33167d82e666 100644 --- a/contrib/openpam/doc/man/openpam_readline.3 +++ b/contrib/openpam/doc/man/openpam_readline.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_readline.c by gendoc.pl -.\" $Id: openpam_readline.c 703 2013-08-16 11:57:54Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_readline.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_READLINE 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_readlinev.3 b/contrib/openpam/doc/man/openpam_readlinev.3 index 9fb28e18cdeea..6391ff50b417d 100644 --- a/contrib/openpam/doc/man/openpam_readlinev.3 +++ b/contrib/openpam/doc/man/openpam_readlinev.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_readlinev.c by gendoc.pl -.\" $Id: openpam_readlinev.c 890 2016-01-11 16:22:09Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_readlinev.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_READLINEV 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_readword.3 b/contrib/openpam/doc/man/openpam_readword.3 index 8bb0f492d799a..e7c8b0f9224b1 100644 --- a/contrib/openpam/doc/man/openpam_readword.3 +++ b/contrib/openpam/doc/man/openpam_readword.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_readword.c by gendoc.pl -.\" $Id: openpam_readword.c 916 2017-02-07 12:25:58Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_readword.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_READWORD 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_restore_cred.3 b/contrib/openpam/doc/man/openpam_restore_cred.3 index 3b1c27e361a5a..3319fd56b14b3 100644 --- a/contrib/openpam/doc/man/openpam_restore_cred.3 +++ b/contrib/openpam/doc/man/openpam_restore_cred.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_restore_cred.c by gendoc.pl -.\" $Id: openpam_restore_cred.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_restore_cred.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_RESTORE_CRED 3 .Os .Sh NAME @@ -23,6 +23,8 @@ The .Fn openpam_restore_cred function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_NO_MODULE_DATA Module data not found. .It Bq Er PAM_SYSTEM_ERR diff --git a/contrib/openpam/doc/man/openpam_set_feature.3 b/contrib/openpam/doc/man/openpam_set_feature.3 index bc58650969755..30774210df067 100644 --- a/contrib/openpam/doc/man/openpam_set_feature.3 +++ b/contrib/openpam/doc/man/openpam_set_feature.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_set_feature.c by gendoc.pl -.\" $Id: openpam_set_feature.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_set_feature.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_SET_FEATURE 3 .Os .Sh NAME @@ -32,8 +32,10 @@ The .Fn openpam_set_feature function returns one of the following values: .Bl -tag -width 18n -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_FEATURE +Unrecognized or restricted feature. .El .Sh SEE ALSO .Xr openpam_get_feature 3 , diff --git a/contrib/openpam/doc/man/openpam_set_option.3 b/contrib/openpam/doc/man/openpam_set_option.3 index 8cb0a7c65f838..751c3177425e1 100644 --- a/contrib/openpam/doc/man/openpam_set_option.3 +++ b/contrib/openpam/doc/man/openpam_set_option.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_set_option.c by gendoc.pl -.\" $Id: openpam_set_option.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_set_option.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_SET_OPTION 3 .Os .Sh NAME @@ -23,6 +23,8 @@ The .Fn openpam_set_option function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_SYSTEM_ERR diff --git a/contrib/openpam/doc/man/openpam_straddch.3 b/contrib/openpam/doc/man/openpam_straddch.3 index 127858f637a63..5f67891c814e1 100644 --- a/contrib/openpam/doc/man/openpam_straddch.3 +++ b/contrib/openpam/doc/man/openpam_straddch.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_straddch.c by gendoc.pl -.\" $Id: openpam_straddch.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_straddch.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_STRADDCH 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/openpam_subst.3 b/contrib/openpam/doc/man/openpam_subst.3 index 5c0c111e32ff7..ce55577854d53 100644 --- a/contrib/openpam/doc/man/openpam_subst.3 +++ b/contrib/openpam/doc/man/openpam_subst.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_subst.c by gendoc.pl -.\" $Id: openpam_subst.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_subst.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_SUBST 3 .Os .Sh NAME @@ -81,6 +81,10 @@ The .Fn openpam_subst function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .It Bq Er PAM_TRY_AGAIN Try again. .El diff --git a/contrib/openpam/doc/man/openpam_ttyconv.3 b/contrib/openpam/doc/man/openpam_ttyconv.3 index 6aeb9f3b3eeb7..4be2cd698dffb 100644 --- a/contrib/openpam/doc/man/openpam_ttyconv.3 +++ b/contrib/openpam/doc/man/openpam_ttyconv.3 @@ -1,6 +1,6 @@ .\" Generated from openpam_ttyconv.c by gendoc.pl -.\" $Id: openpam_ttyconv.c 890 2016-01-11 16:22:09Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: openpam_ttyconv.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt OPENPAM_TTYCONV 3 .Os .Sh NAME @@ -32,6 +32,8 @@ The .Fn openpam_ttyconv function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam.3 b/contrib/openpam/doc/man/pam.3 index 5b46b6d3a8d00..7707e5dbf7af8 100644 --- a/contrib/openpam/doc/man/pam.3 +++ b/contrib/openpam/doc/man/pam.3 @@ -1,5 +1,5 @@ .\" Generated by gendoc.pl -.Dd February 19, 2017 +.Dd April 30, 2017 .Dt PAM 3 .Os .Sh NAME @@ -60,7 +60,7 @@ .Ft "const char *" .Fn pam_strerror "const pam_handle_t *pamh" "int error_number" .\" -.\" $Id: pam.man 648 2013-03-05 17:54:27Z des $ +.\" $OpenPAM: pam.man 938 2017-04-30 21:34:42Z des $ .\" .Sh DESCRIPTION The Pluggable Authentication Modules (PAM) library abstracts a number @@ -180,6 +180,14 @@ Authentication token lock busy. Failed to recover old authentication token. .It Bq Er PAM_AUTH_ERR Authentication error. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. +.It Bq Er PAM_BAD_FEATURE +Unrecognized or restricted feature. +.It Bq Er PAM_BAD_HANDLE +Invalid PAM handle. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam.conf.5 b/contrib/openpam/doc/man/pam.conf.5 index 64adb9d238a21..777e839ea2caa 100644 --- a/contrib/openpam/doc/man/pam.conf.5 +++ b/contrib/openpam/doc/man/pam.conf.5 @@ -1,5 +1,5 @@ .\"- -.\" Copyright (c) 2005-2011 Dag-Erling Smørgrav +.\" Copyright (c) 2005-2017 Dag-Erling Smørgrav .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -26,9 +26,9 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $Id: pam.conf.5 924 2017-02-19 20:52:28Z des $ +.\" $OpenPAM: pam.conf.5 939 2017-04-30 21:36:50Z des $ .\" -.Dd February 19, 2017 +.Dd April 30, 2017 .Dt PAM.CONF 5 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/pam.man b/contrib/openpam/doc/man/pam.man index 24c5d32424997..16873f57de64c 100644 --- a/contrib/openpam/doc/man/pam.man +++ b/contrib/openpam/doc/man/pam.man @@ -1,5 +1,5 @@ .\" -.\" $Id: pam.man 648 2013-03-05 17:54:27Z des $ +.\" $OpenPAM: pam.man 938 2017-04-30 21:34:42Z des $ .\" .Sh DESCRIPTION The Pluggable Authentication Modules (PAM) library abstracts a number diff --git a/contrib/openpam/doc/man/pam_acct_mgmt.3 b/contrib/openpam/doc/man/pam_acct_mgmt.3 index b6d2d4f0fa5a8..bd32933f3721b 100644 --- a/contrib/openpam/doc/man/pam_acct_mgmt.3 +++ b/contrib/openpam/doc/man/pam_acct_mgmt.3 @@ -1,6 +1,6 @@ .\" Generated from pam_acct_mgmt.c by gendoc.pl -.\" $Id: pam_acct_mgmt.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_acct_mgmt.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_ACCT_MGMT 3 .Os .Sh NAME @@ -37,6 +37,8 @@ The .Fn pam_acct_mgmt function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_ACCT_EXPIRED diff --git a/contrib/openpam/doc/man/pam_authenticate.3 b/contrib/openpam/doc/man/pam_authenticate.3 index 46a9994c6800f..2701172abe58b 100644 --- a/contrib/openpam/doc/man/pam_authenticate.3 +++ b/contrib/openpam/doc/man/pam_authenticate.3 @@ -1,6 +1,6 @@ .\" Generated from pam_authenticate.c by gendoc.pl -.\" $Id: pam_authenticate.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_authenticate.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_AUTHENTICATE 3 .Os .Sh NAME @@ -41,18 +41,22 @@ Fail if the user's authentication token is null. If any other bits are set, .Fn pam_authenticate will return -.Dv PAM_SYMBOL_ERR . +.Dv PAM_BAD_CONSTANT . .Sh RETURN VALUES The .Fn pam_authenticate function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_AUTHINFO_UNAVAIL Authentication information is unavailable. .It Bq Er PAM_AUTH_ERR Authentication error. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR @@ -65,8 +69,6 @@ Maximum number of tries exceeded. Permission denied. .It Bq Er PAM_SERVICE_ERR Error in service module. -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. .It Bq Er PAM_SYSTEM_ERR System error. .It Bq Er PAM_USER_UNKNOWN diff --git a/contrib/openpam/doc/man/pam_chauthtok.3 b/contrib/openpam/doc/man/pam_chauthtok.3 index 63a53e618e8fb..89b7416db8dc9 100644 --- a/contrib/openpam/doc/man/pam_chauthtok.3 +++ b/contrib/openpam/doc/man/pam_chauthtok.3 @@ -1,6 +1,6 @@ .\" Generated from pam_chauthtok.c by gendoc.pl -.\" $Id: pam_chauthtok.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_chauthtok.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_CHAUTHTOK 3 .Os .Sh NAME @@ -33,12 +33,14 @@ Change only those authentication tokens that have expired. If any other bits are set, .Fn pam_chauthtok will return -.Dv PAM_SYMBOL_ERR . +.Dv PAM_BAD_CONSTANT . .Sh RETURN VALUES The .Fn pam_chauthtok function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_AUTHTOK_DISABLE_AGING @@ -49,6 +51,8 @@ Authentication token failure. Authentication token lock busy. .It Bq Er PAM_AUTHTOK_RECOVERY_ERR Failed to recover old authentication token. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR @@ -57,8 +61,6 @@ Conversation failure. Permission denied. .It Bq Er PAM_SERVICE_ERR Error in service module. -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. .It Bq Er PAM_SYSTEM_ERR System error. .It Bq Er PAM_TRY_AGAIN diff --git a/contrib/openpam/doc/man/pam_close_session.3 b/contrib/openpam/doc/man/pam_close_session.3 index 3d9ed399fad1c..749d2199f21c1 100644 --- a/contrib/openpam/doc/man/pam_close_session.3 +++ b/contrib/openpam/doc/man/pam_close_session.3 @@ -1,6 +1,6 @@ .\" Generated from pam_close_session.c by gendoc.pl -.\" $Id: pam_close_session.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_close_session.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_CLOSE_SESSION 3 .Os .Sh NAME @@ -30,14 +30,18 @@ Do not emit any messages. If any other bits are set, .Fn pam_close_session will return -.Dv PAM_SYMBOL_ERR . +.Dv PAM_BAD_CONSTANT . .Sh RETURN VALUES The .Fn pam_close_session function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR @@ -48,8 +52,6 @@ Permission denied. Error in service module. .It Bq Er PAM_SESSION_ERR Session failure. -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. .It Bq Er PAM_SYSTEM_ERR System error. .El diff --git a/contrib/openpam/doc/man/pam_conv.3 b/contrib/openpam/doc/man/pam_conv.3 index dcf34a25bffd5..bbf0a572cbe78 100644 --- a/contrib/openpam/doc/man/pam_conv.3 +++ b/contrib/openpam/doc/man/pam_conv.3 @@ -1,6 +1,6 @@ .\"- .\" Copyright (c) 2002-2003 Networks Associates Technology, Inc. -.\" Copyright (c) 2004-2014 Dag-Erling Smørgrav +.\" Copyright (c) 2004-2017 Dag-Erling Smørgrav .\" All rights reserved. .\" .\" This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,9 +32,9 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.\" $Id: pam_conv.3 924 2017-02-19 20:52:28Z des $ +.\" $OpenPAM: pam_conv.3 939 2017-04-30 21:36:50Z des $ .\" -.Dd February 19, 2017 +.Dd April 30, 2017 .Dt PAM_CONV 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/pam_end.3 b/contrib/openpam/doc/man/pam_end.3 index 80ff655d3ad8b..af4cbecffe628 100644 --- a/contrib/openpam/doc/man/pam_end.3 +++ b/contrib/openpam/doc/man/pam_end.3 @@ -1,6 +1,6 @@ .\" Generated from pam_end.c by gendoc.pl -.\" $Id: pam_end.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_end.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_END 3 .Os .Sh NAME @@ -25,7 +25,13 @@ last API call before the call to .Sh RETURN VALUES The .Fn pam_end -function returns 0 on success and -1 on failure. +function returns one of the following values: +.Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_HANDLE +Invalid PAM handle. +.El .Sh SEE ALSO .Xr pam 3 , .Xr pam_strerror 3 diff --git a/contrib/openpam/doc/man/pam_error.3 b/contrib/openpam/doc/man/pam_error.3 index de63c1ec02c85..cf3445ea906a7 100644 --- a/contrib/openpam/doc/man/pam_error.3 +++ b/contrib/openpam/doc/man/pam_error.3 @@ -1,6 +1,6 @@ .\" Generated from pam_error.c by gendoc.pl -.\" $Id: pam_error.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_error.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_ERROR 3 .Os .Sh NAME @@ -22,6 +22,8 @@ The .Fn pam_error function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_get_authtok.3 b/contrib/openpam/doc/man/pam_get_authtok.3 index dab3de8233052..bedeffbe033ca 100644 --- a/contrib/openpam/doc/man/pam_get_authtok.3 +++ b/contrib/openpam/doc/man/pam_get_authtok.3 @@ -1,6 +1,6 @@ .\" Generated from pam_get_authtok.c by gendoc.pl -.\" $Id: pam_get_authtok.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_get_authtok.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_GET_AUTHTOK 3 .Os .Sh NAME @@ -121,6 +121,12 @@ The .Fn pam_get_authtok function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_get_data.3 b/contrib/openpam/doc/man/pam_get_data.3 index 35b3cd17cc13d..104e25aeca96a 100644 --- a/contrib/openpam/doc/man/pam_get_data.3 +++ b/contrib/openpam/doc/man/pam_get_data.3 @@ -1,6 +1,6 @@ .\" Generated from pam_get_data.c by gendoc.pl -.\" $Id: pam_get_data.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_get_data.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_GET_DATA 3 .Os .Sh NAME @@ -39,6 +39,8 @@ The .Fn pam_get_data function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_NO_MODULE_DATA Module data not found. .El diff --git a/contrib/openpam/doc/man/pam_get_item.3 b/contrib/openpam/doc/man/pam_get_item.3 index c53e9228346bb..9f96e96b64070 100644 --- a/contrib/openpam/doc/man/pam_get_item.3 +++ b/contrib/openpam/doc/man/pam_get_item.3 @@ -1,6 +1,6 @@ .\" Generated from pam_get_item.c by gendoc.pl -.\" $Id: pam_get_item.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_get_item.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_GET_ITEM 3 .Os .Sh NAME @@ -74,8 +74,10 @@ The .Fn pam_get_item function returns one of the following values: .Bl -tag -width 18n -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .El .Sh SEE ALSO .Xr pam 3 , diff --git a/contrib/openpam/doc/man/pam_get_user.3 b/contrib/openpam/doc/man/pam_get_user.3 index 96c1af61dfd73..58a861d48075e 100644 --- a/contrib/openpam/doc/man/pam_get_user.3 +++ b/contrib/openpam/doc/man/pam_get_user.3 @@ -1,6 +1,6 @@ .\" Generated from pam_get_user.c by gendoc.pl -.\" $Id: pam_get_user.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_get_user.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_GET_USER 3 .Os .Sh NAME @@ -66,6 +66,10 @@ The .Fn pam_get_user function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_getenv.3 b/contrib/openpam/doc/man/pam_getenv.3 index d2dd7829bb261..dbbb484896f48 100644 --- a/contrib/openpam/doc/man/pam_getenv.3 +++ b/contrib/openpam/doc/man/pam_getenv.3 @@ -1,6 +1,6 @@ .\" Generated from pam_getenv.c by gendoc.pl -.\" $Id: pam_getenv.c 914 2017-01-21 15:15:29Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_getenv.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_GETENV 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/pam_getenvlist.3 b/contrib/openpam/doc/man/pam_getenvlist.3 index 9fa65e7469838..29827cd53a326 100644 --- a/contrib/openpam/doc/man/pam_getenvlist.3 +++ b/contrib/openpam/doc/man/pam_getenvlist.3 @@ -1,6 +1,6 @@ .\" Generated from pam_getenvlist.c by gendoc.pl -.\" $Id: pam_getenvlist.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_getenvlist.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_GETENVLIST 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/pam_info.3 b/contrib/openpam/doc/man/pam_info.3 index 5fe84b25d74ac..5fd8d5f57195d 100644 --- a/contrib/openpam/doc/man/pam_info.3 +++ b/contrib/openpam/doc/man/pam_info.3 @@ -1,6 +1,6 @@ .\" Generated from pam_info.c by gendoc.pl -.\" $Id: pam_info.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_info.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_INFO 3 .Os .Sh NAME @@ -22,6 +22,8 @@ The .Fn pam_info function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_open_session.3 b/contrib/openpam/doc/man/pam_open_session.3 index 052a17fe0f0a3..dd026071edac2 100644 --- a/contrib/openpam/doc/man/pam_open_session.3 +++ b/contrib/openpam/doc/man/pam_open_session.3 @@ -1,6 +1,6 @@ .\" Generated from pam_open_session.c by gendoc.pl -.\" $Id: pam_open_session.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_open_session.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_OPEN_SESSION 3 .Os .Sh NAME @@ -31,14 +31,18 @@ Do not emit any messages. If any other bits are set, .Fn pam_open_session will return -.Dv PAM_SYMBOL_ERR . +.Dv PAM_BAD_CONSTANT . .Sh RETURN VALUES The .Fn pam_open_session function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR @@ -49,8 +53,6 @@ Permission denied. Error in service module. .It Bq Er PAM_SESSION_ERR Session failure. -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. .It Bq Er PAM_SYSTEM_ERR System error. .El diff --git a/contrib/openpam/doc/man/pam_prompt.3 b/contrib/openpam/doc/man/pam_prompt.3 index d98696687f9c9..b78ce3a41b41a 100644 --- a/contrib/openpam/doc/man/pam_prompt.3 +++ b/contrib/openpam/doc/man/pam_prompt.3 @@ -1,6 +1,6 @@ .\" Generated from pam_prompt.c by gendoc.pl -.\" $Id: pam_prompt.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_prompt.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_PROMPT 3 .Os .Sh NAME @@ -34,6 +34,8 @@ The .Fn pam_prompt function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_putenv.3 b/contrib/openpam/doc/man/pam_putenv.3 index e177c584e28f6..2be772ffec3fb 100644 --- a/contrib/openpam/doc/man/pam_putenv.3 +++ b/contrib/openpam/doc/man/pam_putenv.3 @@ -1,6 +1,6 @@ .\" Generated from pam_putenv.c by gendoc.pl -.\" $Id: pam_putenv.c 914 2017-01-21 15:15:29Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_putenv.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_PUTENV 3 .Os .Sh NAME @@ -25,6 +25,8 @@ The .Fn pam_putenv function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_SYSTEM_ERR diff --git a/contrib/openpam/doc/man/pam_set_data.3 b/contrib/openpam/doc/man/pam_set_data.3 index 8232740f3a7de..253d014877fc7 100644 --- a/contrib/openpam/doc/man/pam_set_data.3 +++ b/contrib/openpam/doc/man/pam_set_data.3 @@ -1,6 +1,6 @@ .\" Generated from pam_set_data.c by gendoc.pl -.\" $Id: pam_set_data.c 913 2017-01-21 15:11:12Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_set_data.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SET_DATA 3 .Os .Sh NAME @@ -38,6 +38,8 @@ The .Fn pam_set_data function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_SYSTEM_ERR diff --git a/contrib/openpam/doc/man/pam_set_item.3 b/contrib/openpam/doc/man/pam_set_item.3 index 2165f5721538e..3d479c92dd1df 100644 --- a/contrib/openpam/doc/man/pam_set_item.3 +++ b/contrib/openpam/doc/man/pam_set_item.3 @@ -1,6 +1,6 @@ .\" Generated from pam_set_item.c by gendoc.pl -.\" $Id: pam_set_item.c 918 2017-02-19 17:46:22Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_set_item.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SET_ITEM 3 .Os .Sh NAME @@ -30,12 +30,12 @@ The .Fn pam_set_item function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .It Bq Er PAM_BUF_ERR Memory buffer error. -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. -.It Bq Er PAM_SYSTEM_ERR -System error. .El .Sh SEE ALSO .Xr pam 3 , diff --git a/contrib/openpam/doc/man/pam_setcred.3 b/contrib/openpam/doc/man/pam_setcred.3 index 830043b024e8d..fc5eaadc2b0c4 100644 --- a/contrib/openpam/doc/man/pam_setcred.3 +++ b/contrib/openpam/doc/man/pam_setcred.3 @@ -1,6 +1,6 @@ .\" Generated from pam_setcred.c by gendoc.pl -.\" $Id: pam_setcred.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_setcred.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SETCRED 3 .Os .Sh NAME @@ -38,14 +38,18 @@ The latter four are mutually exclusive. If any other bits are set, .Fn pam_setcred will return -.Dv PAM_SYMBOL_ERR . +.Dv PAM_BAD_CONSTANT . .Sh RETURN VALUES The .Fn pam_setcred function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. +.It Bq Er PAM_BAD_CONSTANT +Bad constant. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR @@ -60,8 +64,6 @@ Failed to retrieve user credentials. Permission denied. .It Bq Er PAM_SERVICE_ERR Error in service module. -.It Bq Er PAM_SYMBOL_ERR -Invalid symbol. .It Bq Er PAM_SYSTEM_ERR System error. .It Bq Er PAM_USER_UNKNOWN diff --git a/contrib/openpam/doc/man/pam_setenv.3 b/contrib/openpam/doc/man/pam_setenv.3 index 09e9ec5532ac1..8fced1e8cc5e7 100644 --- a/contrib/openpam/doc/man/pam_setenv.3 +++ b/contrib/openpam/doc/man/pam_setenv.3 @@ -1,6 +1,6 @@ .\" Generated from pam_setenv.c by gendoc.pl -.\" $Id: pam_setenv.c 914 2017-01-21 15:15:29Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_setenv.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SETENV 3 .Os .Sh NAME @@ -25,6 +25,8 @@ The .Fn pam_setenv function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_SYSTEM_ERR diff --git a/contrib/openpam/doc/man/pam_sm_acct_mgmt.3 b/contrib/openpam/doc/man/pam_sm_acct_mgmt.3 index e28174f753952..2df56b5aa29ae 100644 --- a/contrib/openpam/doc/man/pam_sm_acct_mgmt.3 +++ b/contrib/openpam/doc/man/pam_sm_acct_mgmt.3 @@ -1,6 +1,6 @@ .\" Generated from pam_sm_acct_mgmt.c by gendoc.pl -.\" $Id: pam_sm_acct_mgmt.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_sm_acct_mgmt.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SM_ACCT_MGMT 3 .Os .Sh NAME @@ -24,6 +24,8 @@ The .Fn pam_sm_acct_mgmt function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_ACCT_EXPIRED diff --git a/contrib/openpam/doc/man/pam_sm_authenticate.3 b/contrib/openpam/doc/man/pam_sm_authenticate.3 index 3d7d0ce3014a5..1dc6c69ca9b59 100644 --- a/contrib/openpam/doc/man/pam_sm_authenticate.3 +++ b/contrib/openpam/doc/man/pam_sm_authenticate.3 @@ -1,6 +1,6 @@ .\" Generated from pam_sm_authenticate.c by gendoc.pl -.\" $Id: pam_sm_authenticate.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_sm_authenticate.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SM_AUTHENTICATE 3 .Os .Sh NAME @@ -24,6 +24,8 @@ The .Fn pam_sm_authenticate function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_AUTHINFO_UNAVAIL diff --git a/contrib/openpam/doc/man/pam_sm_chauthtok.3 b/contrib/openpam/doc/man/pam_sm_chauthtok.3 index 3905973fa7c80..be1baa8d57914 100644 --- a/contrib/openpam/doc/man/pam_sm_chauthtok.3 +++ b/contrib/openpam/doc/man/pam_sm_chauthtok.3 @@ -1,6 +1,6 @@ .\" Generated from pam_sm_chauthtok.c by gendoc.pl -.\" $Id: pam_sm_chauthtok.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_sm_chauthtok.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SM_CHAUTHTOK 3 .Os .Sh NAME @@ -34,6 +34,8 @@ The .Fn pam_sm_chauthtok function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_AUTHTOK_DISABLE_AGING diff --git a/contrib/openpam/doc/man/pam_sm_close_session.3 b/contrib/openpam/doc/man/pam_sm_close_session.3 index d07ebf9aee6ec..5b2034e15c7e9 100644 --- a/contrib/openpam/doc/man/pam_sm_close_session.3 +++ b/contrib/openpam/doc/man/pam_sm_close_session.3 @@ -1,6 +1,6 @@ .\" Generated from pam_sm_close_session.c by gendoc.pl -.\" $Id: pam_sm_close_session.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_sm_close_session.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SM_CLOSE_SESSION 3 .Os .Sh NAME @@ -24,6 +24,8 @@ The .Fn pam_sm_close_session function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_BUF_ERR diff --git a/contrib/openpam/doc/man/pam_sm_open_session.3 b/contrib/openpam/doc/man/pam_sm_open_session.3 index d4bb88ba7d1c0..6b4ab9a409b38 100644 --- a/contrib/openpam/doc/man/pam_sm_open_session.3 +++ b/contrib/openpam/doc/man/pam_sm_open_session.3 @@ -1,6 +1,6 @@ .\" Generated from pam_sm_open_session.c by gendoc.pl -.\" $Id: pam_sm_open_session.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_sm_open_session.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SM_OPEN_SESSION 3 .Os .Sh NAME @@ -24,6 +24,8 @@ The .Fn pam_sm_open_session function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_BUF_ERR diff --git a/contrib/openpam/doc/man/pam_sm_setcred.3 b/contrib/openpam/doc/man/pam_sm_setcred.3 index 0b0140fcabc7c..cf1b76eb043b8 100644 --- a/contrib/openpam/doc/man/pam_sm_setcred.3 +++ b/contrib/openpam/doc/man/pam_sm_setcred.3 @@ -1,6 +1,6 @@ .\" Generated from pam_sm_setcred.c by gendoc.pl -.\" $Id: pam_sm_setcred.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_sm_setcred.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_SM_SETCRED 3 .Os .Sh NAME @@ -24,6 +24,8 @@ The .Fn pam_sm_setcred function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_ABORT General failure. .It Bq Er PAM_BUF_ERR diff --git a/contrib/openpam/doc/man/pam_start.3 b/contrib/openpam/doc/man/pam_start.3 index 72702cb7d8273..7a2ba5bb8f2ae 100644 --- a/contrib/openpam/doc/man/pam_start.3 +++ b/contrib/openpam/doc/man/pam_start.3 @@ -1,6 +1,6 @@ .\" Generated from pam_start.c by gendoc.pl -.\" $Id: pam_start.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_start.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_START 3 .Os .Sh NAME @@ -45,6 +45,10 @@ The .Fn pam_start function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. +.It Bq Er PAM_BAD_ITEM +Unrecognized or restricted item. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_SYSTEM_ERR diff --git a/contrib/openpam/doc/man/pam_strerror.3 b/contrib/openpam/doc/man/pam_strerror.3 index a6901fb8dfb9a..bdfb1c2649c4e 100644 --- a/contrib/openpam/doc/man/pam_strerror.3 +++ b/contrib/openpam/doc/man/pam_strerror.3 @@ -1,6 +1,6 @@ .\" Generated from pam_strerror.c by gendoc.pl -.\" $Id: pam_strerror.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_strerror.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_STRERROR 3 .Os .Sh NAME diff --git a/contrib/openpam/doc/man/pam_verror.3 b/contrib/openpam/doc/man/pam_verror.3 index a4aff7e396746..ed680b72ad6b6 100644 --- a/contrib/openpam/doc/man/pam_verror.3 +++ b/contrib/openpam/doc/man/pam_verror.3 @@ -1,6 +1,6 @@ .\" Generated from pam_verror.c by gendoc.pl -.\" $Id: pam_verror.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_verror.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_VERROR 3 .Os .Sh NAME @@ -26,6 +26,8 @@ The .Fn pam_verror function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_vinfo.3 b/contrib/openpam/doc/man/pam_vinfo.3 index b9f9d3f29e9b1..a5923cb8573cf 100644 --- a/contrib/openpam/doc/man/pam_vinfo.3 +++ b/contrib/openpam/doc/man/pam_vinfo.3 @@ -1,6 +1,6 @@ .\" Generated from pam_vinfo.c by gendoc.pl -.\" $Id: pam_vinfo.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_vinfo.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_VINFO 3 .Os .Sh NAME @@ -26,6 +26,8 @@ The .Fn pam_vinfo function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/doc/man/pam_vprompt.3 b/contrib/openpam/doc/man/pam_vprompt.3 index bd081fc776b31..4cfa9ed77dd12 100644 --- a/contrib/openpam/doc/man/pam_vprompt.3 +++ b/contrib/openpam/doc/man/pam_vprompt.3 @@ -1,6 +1,6 @@ .\" Generated from pam_vprompt.c by gendoc.pl -.\" $Id: pam_vprompt.c 648 2013-03-05 17:54:27Z des $ -.Dd February 19, 2017 +.\" $OpenPAM: pam_vprompt.c 938 2017-04-30 21:34:42Z des $ +.Dd April 30, 2017 .Dt PAM_VPROMPT 3 .Os .Sh NAME @@ -60,6 +60,8 @@ The .Fn pam_vprompt function returns one of the following values: .Bl -tag -width 18n +.It Bq Er PAM_SUCCESS +Success. .It Bq Er PAM_BUF_ERR Memory buffer error. .It Bq Er PAM_CONV_ERR diff --git a/contrib/openpam/include/Makefile.am b/contrib/openpam/include/Makefile.am index 8a2755a7c88ed..731c4a1ae1ff4 100644 --- a/contrib/openpam/include/Makefile.am +++ b/contrib/openpam/include/Makefile.am @@ -1,3 +1,3 @@ -# $Id: Makefile.am 648 2013-03-05 17:54:27Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ SUBDIRS = security diff --git a/contrib/openpam/include/Makefile.in b/contrib/openpam/include/Makefile.in index a2464ca91140e..3777407c61571 100644 --- a/contrib/openpam/include/Makefile.in +++ b/contrib/openpam/include/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 648 2013-03-05 17:54:27Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ diff --git a/contrib/openpam/include/security/Makefile.am b/contrib/openpam/include/security/Makefile.am index c3d10f32a08d7..6f36694e02fef 100644 --- a/contrib/openpam/include/security/Makefile.am +++ b/contrib/openpam/include/security/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ securitydir = $(includedir)/security diff --git a/contrib/openpam/include/security/Makefile.in b/contrib/openpam/include/security/Makefile.in index 93cd988730485..8c14050379892 100644 --- a/contrib/openpam/include/security/Makefile.in +++ b/contrib/openpam/include/security/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/include/security/openpam.h b/contrib/openpam/include/security/openpam.h index 77f0bb8eb7192..1e7bc1ef44b2b 100644 --- a/contrib/openpam/include/security/openpam.h +++ b/contrib/openpam/include/security/openpam.h @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam.h 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_OPENPAM_H_INCLUDED diff --git a/contrib/openpam/include/security/openpam_attr.h b/contrib/openpam/include/security/openpam_attr.h index 2d88ae5f25a36..b07768139d4cc 100644 --- a/contrib/openpam/include/security/openpam_attr.h +++ b/contrib/openpam/include/security/openpam_attr.h @@ -1,5 +1,5 @@ /* - * $Id: openpam_attr.h 656 2013-03-06 22:58:45Z des $ + * $OpenPAM: openpam_attr.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_OPENPAM_ATTR_H_INCLUDED diff --git a/contrib/openpam/include/security/openpam_version.h b/contrib/openpam/include/security/openpam_version.h index f10b6a945deae..9a2e1cb17a6e1 100644 --- a/contrib/openpam/include/security/openpam_version.h +++ b/contrib/openpam/include/security/openpam_version.h @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2014 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,14 +32,14 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_version.h 923 2017-02-19 19:31:16Z des $ + * $OpenPAM: openpam_version.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_OPENPAM_VERSION_H_INCLUDED #define SECURITY_OPENPAM_VERSION_H_INCLUDED #define OPENPAM -#define OPENPAM_VERSION 20170219 -#define OPENPAM_RELEASE "Radula" +#define OPENPAM_VERSION 20170430 +#define OPENPAM_RELEASE "Resedacea" #endif /* !SECURITY_OPENPAM_VERSION_H_INCLUDED */ diff --git a/contrib/openpam/include/security/pam_appl.h b/contrib/openpam/include/security/pam_appl.h index 5533bed233b9a..5f44d79555d99 100644 --- a/contrib/openpam/include/security/pam_appl.h +++ b/contrib/openpam/include/security/pam_appl.h @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_appl.h 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_appl.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_PAM_APPL_H_INCLUDED diff --git a/contrib/openpam/include/security/pam_constants.h b/contrib/openpam/include/security/pam_constants.h index ddd87ade41f6e..a76dfecd2406e 100644 --- a/contrib/openpam/include/security/pam_constants.h +++ b/contrib/openpam/include/security/pam_constants.h @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_constants.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_constants.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_PAM_CONSTANTS_H_INCLUDED @@ -78,6 +78,10 @@ enum { PAM_TRY_AGAIN = 27, PAM_MODULE_UNKNOWN = 28, PAM_DOMAIN_UNKNOWN = 29, + PAM_BAD_HANDLE = 30, /* OpenPAM extension */ + PAM_BAD_ITEM = 31, /* OpenPAM extension */ + PAM_BAD_FEATURE = 32, /* OpenPAM extension */ + PAM_BAD_CONSTANT = 33, /* OpenPAM extension */ PAM_NUM_ERRORS /* OpenPAM extension */ }; diff --git a/contrib/openpam/include/security/pam_modules.h b/contrib/openpam/include/security/pam_modules.h index 7f1db465f0391..a1e574515b55e 100644 --- a/contrib/openpam/include/security/pam_modules.h +++ b/contrib/openpam/include/security/pam_modules.h @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_modules.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_modules.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_PAM_MODULES_H_INCLUDED diff --git a/contrib/openpam/include/security/pam_types.h b/contrib/openpam/include/security/pam_types.h index fc834682a70c9..fc5a06d6a4df7 100644 --- a/contrib/openpam/include/security/pam_types.h +++ b/contrib/openpam/include/security/pam_types.h @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_types.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_types.h 938 2017-04-30 21:34:42Z des $ */ #ifndef SECURITY_PAM_TYPES_H_INCLUDED diff --git a/contrib/openpam/lib/Makefile.am b/contrib/openpam/lib/Makefile.am index 002d7bbddb1ec..9f9fb5fa6212c 100644 --- a/contrib/openpam/lib/Makefile.am +++ b/contrib/openpam/lib/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ SUBDIRS = diff --git a/contrib/openpam/lib/Makefile.in b/contrib/openpam/lib/Makefile.in index 74bf0a7d932da..ed2ff0f44813d 100644 --- a/contrib/openpam/lib/Makefile.in +++ b/contrib/openpam/lib/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ diff --git a/contrib/openpam/lib/libpam/Makefile.am b/contrib/openpam/lib/libpam/Makefile.am index faf0dd553f218..34d17d0cac680 100644 --- a/contrib/openpam/lib/libpam/Makefile.am +++ b/contrib/openpam/lib/libpam/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 833 2014-10-28 09:03:41Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ NULL = diff --git a/contrib/openpam/lib/libpam/Makefile.in b/contrib/openpam/lib/libpam/Makefile.in index 7e73926685d4b..c6e4b44f55c25 100644 --- a/contrib/openpam/lib/libpam/Makefile.in +++ b/contrib/openpam/lib/libpam/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 833 2014-10-28 09:03:41Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ diff --git a/contrib/openpam/lib/libpam/openpam_asprintf.c b/contrib/openpam/lib/libpam/openpam_asprintf.c index 3169f4baa4977..7b40aef284239 100644 --- a/contrib/openpam/lib/libpam/openpam_asprintf.c +++ b/contrib/openpam/lib/libpam/openpam_asprintf.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_asprintf.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_asprintf.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_asprintf.h b/contrib/openpam/lib/libpam/openpam_asprintf.h index 6d9e4e39826bc..79680ad6e1bf4 100644 --- a/contrib/openpam/lib/libpam/openpam_asprintf.h +++ b/contrib/openpam/lib/libpam/openpam_asprintf.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_asprintf.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_asprintf.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_ASPRINTF_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_borrow_cred.c b/contrib/openpam/lib/libpam/openpam_borrow_cred.c index 1b407bdfd27bc..79299ac4f1bc3 100644 --- a/contrib/openpam/lib/libpam/openpam_borrow_cred.c +++ b/contrib/openpam/lib/libpam/openpam_borrow_cred.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_borrow_cred.c 649 2013-03-05 17:58:33Z des $ + * $OpenPAM: openpam_borrow_cred.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_check_owner_perms.c b/contrib/openpam/lib/libpam/openpam_check_owner_perms.c index c8a598a9d448f..0f99b82d20872 100644 --- a/contrib/openpam/lib/libpam/openpam_check_owner_perms.c +++ b/contrib/openpam/lib/libpam/openpam_check_owner_perms.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_check_owner_perms.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_check_owner_perms.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_configure.c b/contrib/openpam/lib/libpam/openpam_configure.c index e06eba2859dc2..12f1a51633678 100644 --- a/contrib/openpam/lib/libpam/openpam_configure.c +++ b/contrib/openpam/lib/libpam/openpam_configure.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_configure.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_configure.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_constants.c b/contrib/openpam/lib/libpam/openpam_constants.c index b718af90f8a59..61359989fe452 100644 --- a/contrib/openpam/lib/libpam/openpam_constants.c +++ b/contrib/openpam/lib/libpam/openpam_constants.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2001-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_constants.c 690 2013-08-15 13:22:51Z des $ + * $OpenPAM: openpam_constants.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -44,86 +44,126 @@ #include "openpam_impl.h" const char *pam_err_name[PAM_NUM_ERRORS] = { - "PAM_SUCCESS", - "PAM_OPEN_ERR", - "PAM_SYMBOL_ERR", - "PAM_SERVICE_ERR", - "PAM_SYSTEM_ERR", - "PAM_BUF_ERR", - "PAM_CONV_ERR", - "PAM_PERM_DENIED", - "PAM_MAXTRIES", - "PAM_AUTH_ERR", - "PAM_NEW_AUTHTOK_REQD", - "PAM_CRED_INSUFFICIENT", - "PAM_AUTHINFO_UNAVAIL", - "PAM_USER_UNKNOWN", - "PAM_CRED_UNAVAIL", - "PAM_CRED_EXPIRED", - "PAM_CRED_ERR", - "PAM_ACCT_EXPIRED", - "PAM_AUTHTOK_EXPIRED", - "PAM_SESSION_ERR", - "PAM_AUTHTOK_ERR", - "PAM_AUTHTOK_RECOVERY_ERR", - "PAM_AUTHTOK_LOCK_BUSY", - "PAM_AUTHTOK_DISABLE_AGING", - "PAM_NO_MODULE_DATA", - "PAM_IGNORE", - "PAM_ABORT", - "PAM_TRY_AGAIN", - "PAM_MODULE_UNKNOWN", - "PAM_DOMAIN_UNKNOWN" + [PAM_SUCCESS] = "PAM_SUCCESS", + [PAM_OPEN_ERR] = "PAM_OPEN_ERR", + [PAM_SYMBOL_ERR] = "PAM_SYMBOL_ERR", + [PAM_SERVICE_ERR] = "PAM_SERVICE_ERR", + [PAM_SYSTEM_ERR] = "PAM_SYSTEM_ERR", + [PAM_BUF_ERR] = "PAM_BUF_ERR", + [PAM_CONV_ERR] = "PAM_CONV_ERR", + [PAM_PERM_DENIED] = "PAM_PERM_DENIED", + [PAM_MAXTRIES] = "PAM_MAXTRIES", + [PAM_AUTH_ERR] = "PAM_AUTH_ERR", + [PAM_NEW_AUTHTOK_REQD] = "PAM_NEW_AUTHTOK_REQD", + [PAM_CRED_INSUFFICIENT] = "PAM_CRED_INSUFFICIENT", + [PAM_AUTHINFO_UNAVAIL] = "PAM_AUTHINFO_UNAVAIL", + [PAM_USER_UNKNOWN] = "PAM_USER_UNKNOWN", + [PAM_CRED_UNAVAIL] = "PAM_CRED_UNAVAIL", + [PAM_CRED_EXPIRED] = "PAM_CRED_EXPIRED", + [PAM_CRED_ERR] = "PAM_CRED_ERR", + [PAM_ACCT_EXPIRED] = "PAM_ACCT_EXPIRED", + [PAM_AUTHTOK_EXPIRED] = "PAM_AUTHTOK_EXPIRED", + [PAM_SESSION_ERR] = "PAM_SESSION_ERR", + [PAM_AUTHTOK_ERR] = "PAM_AUTHTOK_ERR", + [PAM_AUTHTOK_RECOVERY_ERR] = "PAM_AUTHTOK_RECOVERY_ERR", + [PAM_AUTHTOK_LOCK_BUSY] = "PAM_AUTHTOK_LOCK_BUSY", + [PAM_AUTHTOK_DISABLE_AGING] = "PAM_AUTHTOK_DISABLE_AGING", + [PAM_NO_MODULE_DATA] = "PAM_NO_MODULE_DATA", + [PAM_IGNORE] = "PAM_IGNORE", + [PAM_ABORT] = "PAM_ABORT", + [PAM_TRY_AGAIN] = "PAM_TRY_AGAIN", + [PAM_MODULE_UNKNOWN] = "PAM_MODULE_UNKNOWN", + [PAM_DOMAIN_UNKNOWN] = "PAM_DOMAIN_UNKNOWN", + [PAM_BAD_HANDLE] = "PAM_BAD_HANDLE", + [PAM_BAD_ITEM] = "PAM_BAD_ITEM", + [PAM_BAD_FEATURE] = "PAM_BAD_FEATURE", + [PAM_BAD_CONSTANT] = "PAM_BAD_CONSTANT", +}; + +const char *pam_err_text[PAM_NUM_ERRORS] = { + [PAM_SUCCESS] = "Success", + [PAM_OPEN_ERR] = "Failed to load module", + [PAM_SYMBOL_ERR] = "Invalid symbol", + [PAM_SERVICE_ERR] = "Error in service module", + [PAM_SYSTEM_ERR] = "System error", + [PAM_BUF_ERR] = "Memory buffer error", + [PAM_CONV_ERR] = "Conversation failure", + [PAM_PERM_DENIED] = "Permission denied", + [PAM_MAXTRIES] = "Maximum number of tries exceeded", + [PAM_AUTH_ERR] = "Authentication error", + [PAM_NEW_AUTHTOK_REQD] = "New authentication token required", + [PAM_CRED_INSUFFICIENT] = "Insufficient credentials", + [PAM_AUTHINFO_UNAVAIL] = "Authentication information is unavailable", + [PAM_USER_UNKNOWN] = "Unknown user", + [PAM_CRED_UNAVAIL] = "Failed to retrieve user credentials", + [PAM_CRED_EXPIRED] = "User credentials have expired", + [PAM_CRED_ERR] = "Failed to set user credentials", + [PAM_ACCT_EXPIRED] = "User account has expired", + [PAM_AUTHTOK_EXPIRED] = "Password has expired", + [PAM_SESSION_ERR] = "Session failure", + [PAM_AUTHTOK_ERR] = "Authentication token failure", + [PAM_AUTHTOK_RECOVERY_ERR] = "Failed to recover old authentication token", + [PAM_AUTHTOK_LOCK_BUSY] = "Authentication token lock busy", + [PAM_AUTHTOK_DISABLE_AGING] = "Authentication token aging disabled", + [PAM_NO_MODULE_DATA] = "Module data not found", + [PAM_IGNORE] = "Ignore this module", + [PAM_ABORT] = "General failure", + [PAM_TRY_AGAIN] = "Try again", + [PAM_MODULE_UNKNOWN] = "Unknown module type", + [PAM_DOMAIN_UNKNOWN] = "Unknown authentication domain", + [PAM_BAD_HANDLE] = "Invalid PAM handle", + [PAM_BAD_ITEM] = "Unrecognized or restricted item", + [PAM_BAD_FEATURE] = "Unrecognized or restricted feature", + [PAM_BAD_CONSTANT] = "Invalid constant", }; const char *pam_item_name[PAM_NUM_ITEMS] = { - "(NO ITEM)", - "PAM_SERVICE", - "PAM_USER", - "PAM_TTY", - "PAM_RHOST", - "PAM_CONV", - "PAM_AUTHTOK", - "PAM_OLDAUTHTOK", - "PAM_RUSER", - "PAM_USER_PROMPT", - "PAM_REPOSITORY", - "PAM_AUTHTOK_PROMPT", - "PAM_OLDAUTHTOK_PROMPT", - "PAM_HOST", + [PAM_SERVICE] = "PAM_SERVICE", + [PAM_USER] = "PAM_USER", + [PAM_TTY] = "PAM_TTY", + [PAM_RHOST] = "PAM_RHOST", + [PAM_CONV] = "PAM_CONV", + [PAM_AUTHTOK] = "PAM_AUTHTOK", + [PAM_OLDAUTHTOK] = "PAM_OLDAUTHTOK", + [PAM_RUSER] = "PAM_RUSER", + [PAM_USER_PROMPT] = "PAM_USER_PROMPT", + [PAM_REPOSITORY] = "PAM_REPOSITORY", + [PAM_AUTHTOK_PROMPT] = "PAM_AUTHTOK_PROMPT", + [PAM_OLDAUTHTOK_PROMPT] = "PAM_OLDAUTHTOK_PROMPT", + [PAM_HOST] = "PAM_HOST", }; const char *pam_facility_name[PAM_NUM_FACILITIES] = { - [PAM_ACCOUNT] = "account", - [PAM_AUTH] = "auth", - [PAM_PASSWORD] = "password", - [PAM_SESSION] = "session", + [PAM_ACCOUNT] = "account", + [PAM_AUTH] = "auth", + [PAM_PASSWORD] = "password", + [PAM_SESSION] = "session", }; const char *pam_control_flag_name[PAM_NUM_CONTROL_FLAGS] = { - [PAM_BINDING] = "binding", - [PAM_OPTIONAL] = "optional", - [PAM_REQUIRED] = "required", - [PAM_REQUISITE] = "requisite", - [PAM_SUFFICIENT] = "sufficient", + [PAM_BINDING] = "binding", + [PAM_OPTIONAL] = "optional", + [PAM_REQUIRED] = "required", + [PAM_REQUISITE] = "requisite", + [PAM_SUFFICIENT] = "sufficient", }; const char *pam_func_name[PAM_NUM_PRIMITIVES] = { - "pam_authenticate", - "pam_setcred", - "pam_acct_mgmt", - "pam_open_session", - "pam_close_session", - "pam_chauthtok" + [PAM_SM_AUTHENTICATE] = "pam_authenticate", + [PAM_SM_SETCRED] = "pam_setcred", + [PAM_SM_ACCT_MGMT] = "pam_acct_mgmt", + [PAM_SM_OPEN_SESSION] = "pam_open_session", + [PAM_SM_CLOSE_SESSION] = "pam_close_session", + [PAM_SM_CHAUTHTOK] = "pam_chauthtok" }; const char *pam_sm_func_name[PAM_NUM_PRIMITIVES] = { - "pam_sm_authenticate", - "pam_sm_setcred", - "pam_sm_acct_mgmt", - "pam_sm_open_session", - "pam_sm_close_session", - "pam_sm_chauthtok" + [PAM_SM_AUTHENTICATE] = "pam_sm_authenticate", + [PAM_SM_SETCRED] = "pam_sm_setcred", + [PAM_SM_ACCT_MGMT] = "pam_sm_acct_mgmt", + [PAM_SM_OPEN_SESSION] = "pam_sm_open_session", + [PAM_SM_CLOSE_SESSION] = "pam_sm_close_session", + [PAM_SM_CHAUTHTOK] = "pam_sm_chauthtok" }; const char *openpam_policy_path[] = { diff --git a/contrib/openpam/lib/libpam/openpam_constants.h b/contrib/openpam/lib/libpam/openpam_constants.h index a42b4fa76f5b0..3ff3cd2a8b5fe 100644 --- a/contrib/openpam/lib/libpam/openpam_constants.h +++ b/contrib/openpam/lib/libpam/openpam_constants.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2011 Dag-Erling Smørgrav + * Copyright (c) 2011-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,13 +26,14 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_constants.h 659 2013-03-11 14:10:13Z des $ + * $OpenPAM: openpam_constants.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_CONSTANTS_H_INCLUDED #define OPENPAM_CONSTANTS_H_INCLUDED extern const char *pam_err_name[PAM_NUM_ERRORS]; +extern const char *pam_err_text[PAM_NUM_ERRORS]; extern const char *pam_item_name[PAM_NUM_ITEMS]; extern const char *pam_facility_name[PAM_NUM_FACILITIES]; extern const char *pam_control_flag_name[PAM_NUM_CONTROL_FLAGS]; diff --git a/contrib/openpam/lib/libpam/openpam_cred.h b/contrib/openpam/lib/libpam/openpam_cred.h index 68e2fd9d1c092..99d634630f0d3 100644 --- a/contrib/openpam/lib/libpam/openpam_cred.h +++ b/contrib/openpam/lib/libpam/openpam_cred.h @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_cred.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_cred.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_CRED_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_ctype.h b/contrib/openpam/lib/libpam/openpam_ctype.h index 671c2f5cf8c98..4920e4022c31b 100644 --- a/contrib/openpam/lib/libpam/openpam_ctype.h +++ b/contrib/openpam/lib/libpam/openpam_ctype.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_ctype.h 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_ctype.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_CTYPE_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_debug.h b/contrib/openpam/lib/libpam/openpam_debug.h index 1fe8346cca8c8..d980244444ef3 100644 --- a/contrib/openpam/lib/libpam/openpam_debug.h +++ b/contrib/openpam/lib/libpam/openpam_debug.h @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_debug.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_debug.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_DEBUG_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_dispatch.c b/contrib/openpam/lib/libpam/openpam_dispatch.c index 391ce8050d8fc..ff4c66ac8b947 100644 --- a/contrib/openpam/lib/libpam/openpam_dispatch.c +++ b/contrib/openpam/lib/libpam/openpam_dispatch.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2015 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_dispatch.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: openpam_dispatch.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -107,7 +107,7 @@ openpam_dispatch(pam_handle_t *pamh, if (chain->module->func[primitive] == NULL) { openpam_log(PAM_LOG_ERROR, "%s: no %s()", chain->module->path, pam_sm_func_name[primitive]); - r = PAM_SYSTEM_ERR; + r = PAM_SYMBOL_ERR; } else { pamh->primitive = primitive; pamh->current = chain; diff --git a/contrib/openpam/lib/libpam/openpam_dlfunc.h b/contrib/openpam/lib/libpam/openpam_dlfunc.h index 6f8724a65d4fe..a33de6bee48e3 100644 --- a/contrib/openpam/lib/libpam/openpam_dlfunc.h +++ b/contrib/openpam/lib/libpam/openpam_dlfunc.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_dlfunc.h 872 2015-12-01 19:25:07Z des $ + * $OpenPAM: openpam_dlfunc.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_DLFCN_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_dynamic.c b/contrib/openpam/lib/libpam/openpam_dynamic.c index 27cd4e6776dad..6a0f006a63949 100644 --- a/contrib/openpam/lib/libpam/openpam_dynamic.c +++ b/contrib/openpam/lib/libpam/openpam_dynamic.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_dynamic.c 683 2013-04-14 14:49:59Z des $ + * $OpenPAM: openpam_dynamic.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_features.c b/contrib/openpam/lib/libpam/openpam_features.c index 387d27bcd3848..afffe643fcbc7 100644 --- a/contrib/openpam/lib/libpam/openpam_features.c +++ b/contrib/openpam/lib/libpam/openpam_features.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_features.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_features.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_features.h b/contrib/openpam/lib/libpam/openpam_features.h index 00cac1c4b66fd..30c0d1a825674 100644 --- a/contrib/openpam/lib/libpam/openpam_features.h +++ b/contrib/openpam/lib/libpam/openpam_features.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_features.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_features.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_FEATURES_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_findenv.c b/contrib/openpam/lib/libpam/openpam_findenv.c index 3ad2c845794a4..99cdcd134363d 100644 --- a/contrib/openpam/lib/libpam/openpam_findenv.c +++ b/contrib/openpam/lib/libpam/openpam_findenv.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_findenv.c 914 2017-01-21 15:15:29Z des $ + * $OpenPAM: openpam_findenv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_free_data.c b/contrib/openpam/lib/libpam/openpam_free_data.c index 8f3d852b4905d..c4be65cee5694 100644 --- a/contrib/openpam/lib/libpam/openpam_free_data.c +++ b/contrib/openpam/lib/libpam/openpam_free_data.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_free_data.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_free_data.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_free_envlist.c b/contrib/openpam/lib/libpam/openpam_free_envlist.c index dda06153205ba..e91bc922fd52c 100644 --- a/contrib/openpam/lib/libpam/openpam_free_envlist.c +++ b/contrib/openpam/lib/libpam/openpam_free_envlist.c @@ -24,7 +24,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * $Id: openpam_free_envlist.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_free_envlist.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_get_feature.c b/contrib/openpam/lib/libpam/openpam_get_feature.c index ad3d9b3ade0fe..5f2b66df15417 100644 --- a/contrib/openpam/lib/libpam/openpam_get_feature.c +++ b/contrib/openpam/lib/libpam/openpam_get_feature.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012 Dag-Erling Smørgrav + * Copyright (c) 2012-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_get_feature.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_get_feature.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -50,7 +50,7 @@ openpam_get_feature(int feature, int *onoff) ENTERF(feature); if (feature < 0 || feature >= OPENPAM_NUM_FEATURES) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_FEATURE); *onoff = openpam_features[feature].onoff; RETURNC(PAM_SUCCESS); } @@ -58,7 +58,7 @@ openpam_get_feature(int feature, int *onoff) /* * Error codes: * - * PAM_SYMBOL_ERR + * PAM_BAD_FEATURE */ /** diff --git a/contrib/openpam/lib/libpam/openpam_get_option.c b/contrib/openpam/lib/libpam/openpam_get_option.c index 2cf5e87f3f404..3d158d2b14392 100644 --- a/contrib/openpam/lib/libpam/openpam_get_option.c +++ b/contrib/openpam/lib/libpam/openpam_get_option.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_get_option.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_get_option.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_impl.h b/contrib/openpam/lib/libpam/openpam_impl.h index 589a3b3255390..ad50bc14d3603 100644 --- a/contrib/openpam/lib/libpam/openpam_impl.h +++ b/contrib/openpam/lib/libpam/openpam_impl.h @@ -1,6 +1,6 @@ /*- * Copyright (c) 2001-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_impl.h 915 2017-02-07 12:03:19Z des $ + * $OpenPAM: openpam_impl.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_IMPL_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_load.c b/contrib/openpam/lib/libpam/openpam_load.c index 614c6fb409381..2edd5efb720fb 100644 --- a/contrib/openpam/lib/libpam/openpam_load.c +++ b/contrib/openpam/lib/libpam/openpam_load.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_load.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_load.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_log.c b/contrib/openpam/lib/libpam/openpam_log.c index e52ca95b210c0..ecb6758847db1 100644 --- a/contrib/openpam/lib/libpam/openpam_log.c +++ b/contrib/openpam/lib/libpam/openpam_log.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_log.c 686 2013-07-11 16:36:02Z des $ + * $OpenPAM: openpam_log.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_nullconv.c b/contrib/openpam/lib/libpam/openpam_nullconv.c index 49c71ba22d820..720b533b6db33 100644 --- a/contrib/openpam/lib/libpam/openpam_nullconv.c +++ b/contrib/openpam/lib/libpam/openpam_nullconv.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_nullconv.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_nullconv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_readline.c b/contrib/openpam/lib/libpam/openpam_readline.c index e3ed9b0a4f01b..e43291fbddc6d 100644 --- a/contrib/openpam/lib/libpam/openpam_readline.c +++ b/contrib/openpam/lib/libpam/openpam_readline.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_readline.c 703 2013-08-16 11:57:54Z des $ + * $OpenPAM: openpam_readline.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_readlinev.c b/contrib/openpam/lib/libpam/openpam_readlinev.c index d73fc5580e8f0..a0df81f7cd0e9 100644 --- a/contrib/openpam/lib/libpam/openpam_readlinev.c +++ b/contrib/openpam/lib/libpam/openpam_readlinev.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_readlinev.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_readlinev.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_readword.c b/contrib/openpam/lib/libpam/openpam_readword.c index a73b900cbacd2..4a58a819ea5a5 100644 --- a/contrib/openpam/lib/libpam/openpam_readword.c +++ b/contrib/openpam/lib/libpam/openpam_readword.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2014 Dag-Erling Smørgrav + * Copyright (c) 2012-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_readword.c 916 2017-02-07 12:25:58Z des $ + * $OpenPAM: openpam_readword.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_restore_cred.c b/contrib/openpam/lib/libpam/openpam_restore_cred.c index 8a150cc604e8e..59f808b023733 100644 --- a/contrib/openpam/lib/libpam/openpam_restore_cred.c +++ b/contrib/openpam/lib/libpam/openpam_restore_cred.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_restore_cred.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_restore_cred.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_set_feature.c b/contrib/openpam/lib/libpam/openpam_set_feature.c index 8e95f3eb057d4..f588b179ff27f 100644 --- a/contrib/openpam/lib/libpam/openpam_set_feature.c +++ b/contrib/openpam/lib/libpam/openpam_set_feature.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012 Dag-Erling Smørgrav + * Copyright (c) 2012-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_set_feature.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_set_feature.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -50,7 +50,7 @@ openpam_set_feature(int feature, int onoff) ENTERF(feature); if (feature < 0 || feature >= OPENPAM_NUM_FEATURES) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_FEATURE); openpam_features[feature].onoff = onoff; RETURNC(PAM_SUCCESS); } @@ -58,7 +58,7 @@ openpam_set_feature(int feature, int onoff) /* * Error codes: * - * PAM_SYMBOL_ERR + * PAM_BAD_FEATURE */ /** diff --git a/contrib/openpam/lib/libpam/openpam_set_option.c b/contrib/openpam/lib/libpam/openpam_set_option.c index 1c06d61d3cc92..e22fb3b492e50 100644 --- a/contrib/openpam/lib/libpam/openpam_set_option.c +++ b/contrib/openpam/lib/libpam/openpam_set_option.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_set_option.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_set_option.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_static.c b/contrib/openpam/lib/libpam/openpam_static.c index 302b3f4c4a86a..0816b4cc4a0ac 100644 --- a/contrib/openpam/lib/libpam/openpam_static.c +++ b/contrib/openpam/lib/libpam/openpam_static.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_static.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_static.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_straddch.c b/contrib/openpam/lib/libpam/openpam_straddch.c index 522405c0e23b3..0bb32d266adfb 100644 --- a/contrib/openpam/lib/libpam/openpam_straddch.c +++ b/contrib/openpam/lib/libpam/openpam_straddch.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_straddch.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_straddch.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_strlcat.c b/contrib/openpam/lib/libpam/openpam_strlcat.c index 64ba9ab00986f..0d2fd0989c492 100644 --- a/contrib/openpam/lib/libpam/openpam_strlcat.c +++ b/contrib/openpam/lib/libpam/openpam_strlcat.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlcat.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_strlcat.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_strlcat.h b/contrib/openpam/lib/libpam/openpam_strlcat.h index d099346879701..f561283afd964 100644 --- a/contrib/openpam/lib/libpam/openpam_strlcat.h +++ b/contrib/openpam/lib/libpam/openpam_strlcat.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlcat.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_strlcat.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_STRLCAT_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_strlcmp.h b/contrib/openpam/lib/libpam/openpam_strlcmp.h index d00383870e2cb..2b343b40a1385 100644 --- a/contrib/openpam/lib/libpam/openpam_strlcmp.h +++ b/contrib/openpam/lib/libpam/openpam_strlcmp.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlcmp.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_strlcmp.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_STRLCMP_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_strlcpy.c b/contrib/openpam/lib/libpam/openpam_strlcpy.c index 7a3d142bbcd1b..e4c66aefc4c8f 100644 --- a/contrib/openpam/lib/libpam/openpam_strlcpy.c +++ b/contrib/openpam/lib/libpam/openpam_strlcpy.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlcpy.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_strlcpy.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_strlcpy.h b/contrib/openpam/lib/libpam/openpam_strlcpy.h index 8260c5627e0db..40d300fb4901f 100644 --- a/contrib/openpam/lib/libpam/openpam_strlcpy.h +++ b/contrib/openpam/lib/libpam/openpam_strlcpy.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlcpy.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_strlcpy.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_STRLCPY_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_strlset.c b/contrib/openpam/lib/libpam/openpam_strlset.c index febdb5c4ebc87..e3a32dbc033d5 100644 --- a/contrib/openpam/lib/libpam/openpam_strlset.c +++ b/contrib/openpam/lib/libpam/openpam_strlset.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlset.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_strlset.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_strlset.h b/contrib/openpam/lib/libpam/openpam_strlset.h index 282a26bdbde26..73bfb61c78dbd 100644 --- a/contrib/openpam/lib/libpam/openpam_strlset.h +++ b/contrib/openpam/lib/libpam/openpam_strlset.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_strlset.h 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_strlset.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_STRLSET_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/openpam_subst.c b/contrib/openpam/lib/libpam/openpam_subst.c index 1a4973477dfb4..6456aee1442b0 100644 --- a/contrib/openpam/lib/libpam/openpam_subst.c +++ b/contrib/openpam/lib/libpam/openpam_subst.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_subst.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_subst.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_ttyconv.c b/contrib/openpam/lib/libpam/openpam_ttyconv.c index 7591eed682bc2..8066b3b672986 100644 --- a/contrib/openpam/lib/libpam/openpam_ttyconv.c +++ b/contrib/openpam/lib/libpam/openpam_ttyconv.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_ttyconv.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: openpam_ttyconv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_vasprintf.c b/contrib/openpam/lib/libpam/openpam_vasprintf.c index bedabf4fbd3df..dbdc5c6b15932 100644 --- a/contrib/openpam/lib/libpam/openpam_vasprintf.c +++ b/contrib/openpam/lib/libpam/openpam_vasprintf.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_vasprintf.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_vasprintf.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/openpam_vasprintf.h b/contrib/openpam/lib/libpam/openpam_vasprintf.h index e92ad0d2ce6f4..f78b6653167a9 100644 --- a/contrib/openpam/lib/libpam/openpam_vasprintf.h +++ b/contrib/openpam/lib/libpam/openpam_vasprintf.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: openpam_vasprintf.h 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: openpam_vasprintf.h 938 2017-04-30 21:34:42Z des $ */ #ifndef OPENPAM_VASPRINTF_H_INCLUDED diff --git a/contrib/openpam/lib/libpam/pam_acct_mgmt.c b/contrib/openpam/lib/libpam/pam_acct_mgmt.c index 87337a80ae6f1..8f6426d20bf1e 100644 --- a/contrib/openpam/lib/libpam/pam_acct_mgmt.c +++ b/contrib/openpam/lib/libpam/pam_acct_mgmt.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_acct_mgmt.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_acct_mgmt.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_authenticate.c b/contrib/openpam/lib/libpam/pam_authenticate.c index b24e39685d7ef..44cb4c86c4a7b 100644 --- a/contrib/openpam/lib/libpam/pam_authenticate.c +++ b/contrib/openpam/lib/libpam/pam_authenticate.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_authenticate.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_authenticate.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -60,7 +60,7 @@ pam_authenticate(pam_handle_t *pamh, ENTER(); if (flags & ~(PAM_SILENT|PAM_DISALLOW_NULL_AUTHTOK)) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_CONSTANT); r = openpam_dispatch(pamh, PAM_SM_AUTHENTICATE, flags); pam_set_item(pamh, PAM_AUTHTOK, NULL); RETURNC(r); @@ -72,7 +72,7 @@ pam_authenticate(pam_handle_t *pamh, * =openpam_dispatch * =pam_sm_authenticate * !PAM_IGNORE - * PAM_SYMBOL_ERR + * PAM_BAD_CONSTANT */ /** @@ -92,5 +92,5 @@ pam_authenticate(pam_handle_t *pamh, * Fail if the user's authentication token is null. * * If any other bits are set, =pam_authenticate will return - * =PAM_SYMBOL_ERR. + * =PAM_BAD_CONSTANT. */ diff --git a/contrib/openpam/lib/libpam/pam_authenticate_secondary.c b/contrib/openpam/lib/libpam/pam_authenticate_secondary.c index 9167867833dec..02623382f85d7 100644 --- a/contrib/openpam/lib/libpam/pam_authenticate_secondary.c +++ b/contrib/openpam/lib/libpam/pam_authenticate_secondary.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_authenticate_secondary.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_authenticate_secondary.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_chauthtok.c b/contrib/openpam/lib/libpam/pam_chauthtok.c index 163fec7cf42f9..68790b694dfc3 100644 --- a/contrib/openpam/lib/libpam/pam_chauthtok.c +++ b/contrib/openpam/lib/libpam/pam_chauthtok.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_chauthtok.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_chauthtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -60,7 +60,7 @@ pam_chauthtok(pam_handle_t *pamh, ENTER(); if (flags & ~(PAM_SILENT|PAM_CHANGE_EXPIRED_AUTHTOK)) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_CONSTANT); r = openpam_dispatch(pamh, PAM_SM_CHAUTHTOK, flags | PAM_PRELIM_CHECK); if (r == PAM_SUCCESS) @@ -77,7 +77,7 @@ pam_chauthtok(pam_handle_t *pamh, * =openpam_dispatch * =pam_sm_chauthtok * !PAM_IGNORE - * PAM_SYMBOL_ERR + * PAM_BAD_CONSTANT */ /** @@ -93,5 +93,5 @@ pam_chauthtok(pam_handle_t *pamh, * =PAM_CHANGE_EXPIRED_AUTHTOK: * Change only those authentication tokens that have expired. * - * If any other bits are set, =pam_chauthtok will return =PAM_SYMBOL_ERR. + * If any other bits are set, =pam_chauthtok will return =PAM_BAD_CONSTANT. */ diff --git a/contrib/openpam/lib/libpam/pam_close_session.c b/contrib/openpam/lib/libpam/pam_close_session.c index 8bc1efaebaaed..8181c829c2248 100644 --- a/contrib/openpam/lib/libpam/pam_close_session.c +++ b/contrib/openpam/lib/libpam/pam_close_session.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_close_session.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_close_session.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -60,7 +60,7 @@ pam_close_session(pam_handle_t *pamh, ENTER(); if (flags & ~(PAM_SILENT)) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_CONSTANT); r = openpam_dispatch(pamh, PAM_SM_CLOSE_SESSION, flags); RETURNC(r); } @@ -71,7 +71,7 @@ pam_close_session(pam_handle_t *pamh, * =openpam_dispatch * =pam_sm_close_session * !PAM_IGNORE - * PAM_SYMBOL_ERR + * PAM_BAD_CONSTANT */ /** @@ -85,5 +85,5 @@ pam_close_session(pam_handle_t *pamh, * Do not emit any messages. * * If any other bits are set, =pam_close_session will return - * =PAM_SYMBOL_ERR. + * =PAM_BAD_CONSTANT. */ diff --git a/contrib/openpam/lib/libpam/pam_end.c b/contrib/openpam/lib/libpam/pam_end.c index d1b3ad5898454..04541297d8a6e 100644 --- a/contrib/openpam/lib/libpam/pam_end.c +++ b/contrib/openpam/lib/libpam/pam_end.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_end.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_end.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -61,7 +61,7 @@ pam_end(pam_handle_t *pamh, ENTER(); if (pamh == NULL) - RETURNC(PAM_SYSTEM_ERR); + RETURNC(PAM_BAD_HANDLE); /* clear module data */ while ((dp = pamh->module_data) != NULL) { @@ -91,6 +91,12 @@ pam_end(pam_handle_t *pamh, RETURNC(PAM_SUCCESS); } +/* + * Error codes: + * + * PAM_BAD_HANDLE + */ + /** * The =pam_end function terminates a PAM transaction and destroys the * corresponding PAM context, releasing all resources allocated to it. diff --git a/contrib/openpam/lib/libpam/pam_error.c b/contrib/openpam/lib/libpam/pam_error.c index cd018f5389e5c..cba902a819f18 100644 --- a/contrib/openpam/lib/libpam/pam_error.c +++ b/contrib/openpam/lib/libpam/pam_error.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_error.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_error.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_get_authtok.c b/contrib/openpam/lib/libpam/pam_get_authtok.c index 83c6b7053fac5..6d3dc9c8fa6f2 100644 --- a/contrib/openpam/lib/libpam/pam_get_authtok.c +++ b/contrib/openpam/lib/libpam/pam_get_authtok.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_get_authtok.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_get_authtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -104,7 +104,7 @@ pam_get_authtok(pam_handle_t *pamh, twice = 0; break; default: - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_CONSTANT); } if (openpam_get_option(pamh, "try_first_pass") || openpam_get_option(pamh, "use_first_pass")) { @@ -170,6 +170,7 @@ pam_get_authtok(pam_handle_t *pamh, * =pam_prompt * =pam_set_item * !PAM_SYMBOL_ERR + * PAM_BAD_CONSTANT * PAM_TRY_AGAIN */ diff --git a/contrib/openpam/lib/libpam/pam_get_data.c b/contrib/openpam/lib/libpam/pam_get_data.c index a2f5072cb374c..1db50c071ca47 100644 --- a/contrib/openpam/lib/libpam/pam_get_data.c +++ b/contrib/openpam/lib/libpam/pam_get_data.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_get_data.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_get_data.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_get_item.c b/contrib/openpam/lib/libpam/pam_get_item.c index e28012ea26726..6babd0fc7aa88 100644 --- a/contrib/openpam/lib/libpam/pam_get_item.c +++ b/contrib/openpam/lib/libpam/pam_get_item.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_get_item.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_get_item.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -76,14 +76,14 @@ pam_get_item(const pam_handle_t *pamh, *item = pamh->item[item_type]; RETURNC(PAM_SUCCESS); default: - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_ITEM); } } /* * Error codes: * - * PAM_SYMBOL_ERR + * PAM_BAD_ITEM */ /** diff --git a/contrib/openpam/lib/libpam/pam_get_mapped_authtok.c b/contrib/openpam/lib/libpam/pam_get_mapped_authtok.c index 871405efb5dc4..01c283f32cbe0 100644 --- a/contrib/openpam/lib/libpam/pam_get_mapped_authtok.c +++ b/contrib/openpam/lib/libpam/pam_get_mapped_authtok.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_get_mapped_authtok.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_get_mapped_authtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_get_mapped_username.c b/contrib/openpam/lib/libpam/pam_get_mapped_username.c index 25305d2ac55c6..05a9a4c4337d6 100644 --- a/contrib/openpam/lib/libpam/pam_get_mapped_username.c +++ b/contrib/openpam/lib/libpam/pam_get_mapped_username.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_get_mapped_username.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_get_mapped_username.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_get_user.c b/contrib/openpam/lib/libpam/pam_get_user.c index f3fc4b60b8f22..7b1bd66b53162 100644 --- a/contrib/openpam/lib/libpam/pam_get_user.c +++ b/contrib/openpam/lib/libpam/pam_get_user.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_get_user.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_get_user.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_getenv.c b/contrib/openpam/lib/libpam/pam_getenv.c index 1e034468fbc29..c3bbcd8eac2f0 100644 --- a/contrib/openpam/lib/libpam/pam_getenv.c +++ b/contrib/openpam/lib/libpam/pam_getenv.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_getenv.c 914 2017-01-21 15:15:29Z des $ + * $OpenPAM: pam_getenv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_getenvlist.c b/contrib/openpam/lib/libpam/pam_getenvlist.c index 4139c2fb4d316..5b660e6b2bd31 100644 --- a/contrib/openpam/lib/libpam/pam_getenvlist.c +++ b/contrib/openpam/lib/libpam/pam_getenvlist.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_getenvlist.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_getenvlist.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -63,7 +63,7 @@ pam_getenvlist(pam_handle_t *pamh) envlist = malloc(sizeof(char *) * (pamh->env_count + 1)); if (envlist == NULL) { openpam_log(PAM_LOG_ERROR, "%s", - pam_strerror(pamh, PAM_BUF_ERR)); + pam_err_text[PAM_BUF_ERR]); RETURNP(NULL); } for (i = 0; i < pamh->env_count; ++i) { @@ -74,7 +74,7 @@ pam_getenvlist(pam_handle_t *pamh) } FREE(envlist); openpam_log(PAM_LOG_ERROR, "%s", - pam_strerror(pamh, PAM_BUF_ERR)); + pam_err_text[PAM_BUF_ERR]); RETURNP(NULL); } } diff --git a/contrib/openpam/lib/libpam/pam_info.c b/contrib/openpam/lib/libpam/pam_info.c index 550bb76c9594b..d417c6864ba9c 100644 --- a/contrib/openpam/lib/libpam/pam_info.c +++ b/contrib/openpam/lib/libpam/pam_info.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_info.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_info.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_open_session.c b/contrib/openpam/lib/libpam/pam_open_session.c index de12705088ebb..43da3ffe44c5c 100644 --- a/contrib/openpam/lib/libpam/pam_open_session.c +++ b/contrib/openpam/lib/libpam/pam_open_session.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_open_session.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_open_session.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -60,7 +60,7 @@ pam_open_session(pam_handle_t *pamh, ENTER(); if (flags & ~(PAM_SILENT)) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_CONSTANT); r = openpam_dispatch(pamh, PAM_SM_OPEN_SESSION, flags); RETURNC(r); } @@ -71,7 +71,7 @@ pam_open_session(pam_handle_t *pamh, * =openpam_dispatch * =pam_sm_open_session * !PAM_IGNORE - * PAM_SYMBOL_ERR + * PAM_BAD_CONSTANT */ /** @@ -86,5 +86,5 @@ pam_open_session(pam_handle_t *pamh, * Do not emit any messages. * * If any other bits are set, =pam_open_session will return - * =PAM_SYMBOL_ERR. + * =PAM_BAD_CONSTANT. */ diff --git a/contrib/openpam/lib/libpam/pam_prompt.c b/contrib/openpam/lib/libpam/pam_prompt.c index e3ba0f2b90e78..78b93402ce2a8 100644 --- a/contrib/openpam/lib/libpam/pam_prompt.c +++ b/contrib/openpam/lib/libpam/pam_prompt.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_prompt.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_prompt.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_putenv.c b/contrib/openpam/lib/libpam/pam_putenv.c index 605277fda6f49..2f3c936e1af82 100644 --- a/contrib/openpam/lib/libpam/pam_putenv.c +++ b/contrib/openpam/lib/libpam/pam_putenv.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_putenv.c 914 2017-01-21 15:15:29Z des $ + * $OpenPAM: pam_putenv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_set_data.c b/contrib/openpam/lib/libpam/pam_set_data.c index 6a26b6fb02e02..84871c17ccc06 100644 --- a/contrib/openpam/lib/libpam/pam_set_data.c +++ b/contrib/openpam/lib/libpam/pam_set_data.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_set_data.c 913 2017-01-21 15:11:12Z des $ + * $OpenPAM: pam_set_data.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_set_item.c b/contrib/openpam/lib/libpam/pam_set_item.c index 0e8f76f7fa6b8..02c6b49b3d276 100644 --- a/contrib/openpam/lib/libpam/pam_set_item.c +++ b/contrib/openpam/lib/libpam/pam_set_item.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_set_item.c 918 2017-02-19 17:46:22Z des $ + * $OpenPAM: pam_set_item.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -70,7 +70,7 @@ pam_set_item(pam_handle_t *pamh, case PAM_SERVICE: /* set once only, by pam_start() */ if (*slot != NULL && item != NULL) - RETURNC(PAM_SYSTEM_ERR); + RETURNC(PAM_BAD_ITEM); /* fall through */ case PAM_USER: case PAM_AUTHTOK: @@ -94,7 +94,7 @@ pam_set_item(pam_handle_t *pamh, osize = nsize = sizeof(struct pam_conv); break; default: - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_ITEM); } if (*slot != NULL) { memset(*slot, 0xd0, osize); @@ -113,8 +113,7 @@ pam_set_item(pam_handle_t *pamh, /* * Error codes: * - * PAM_SYMBOL_ERR - * PAM_SYSTEM_ERR + * PAM_BAD_ITEM * PAM_BUF_ERR */ diff --git a/contrib/openpam/lib/libpam/pam_set_mapped_authtok.c b/contrib/openpam/lib/libpam/pam_set_mapped_authtok.c index c8fbd91b4aac9..fa3d2fa4fdf4f 100644 --- a/contrib/openpam/lib/libpam/pam_set_mapped_authtok.c +++ b/contrib/openpam/lib/libpam/pam_set_mapped_authtok.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_set_mapped_authtok.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_set_mapped_authtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_set_mapped_username.c b/contrib/openpam/lib/libpam/pam_set_mapped_username.c index 8f62f53d235d7..ee8813b1e973b 100644 --- a/contrib/openpam/lib/libpam/pam_set_mapped_username.c +++ b/contrib/openpam/lib/libpam/pam_set_mapped_username.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_set_mapped_username.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_set_mapped_username.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_setcred.c b/contrib/openpam/lib/libpam/pam_setcred.c index 7a691765573b9..eb71ba808a840 100644 --- a/contrib/openpam/lib/libpam/pam_setcred.c +++ b/contrib/openpam/lib/libpam/pam_setcred.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_setcred.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_setcred.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -61,7 +61,7 @@ pam_setcred(pam_handle_t *pamh, ENTER(); if (flags & ~(PAM_SILENT|PAM_ESTABLISH_CRED|PAM_DELETE_CRED| PAM_REINITIALIZE_CRED|PAM_REFRESH_CRED)) - RETURNC(PAM_SYMBOL_ERR); + RETURNC(PAM_BAD_CONSTANT); /* XXX enforce exclusivity */ r = openpam_dispatch(pamh, PAM_SM_SETCRED, flags); RETURNC(r); @@ -73,7 +73,7 @@ pam_setcred(pam_handle_t *pamh, * =openpam_dispatch * =pam_sm_setcred * !PAM_IGNORE - * PAM_SYMBOL_ERR + * PAM_BAD_CONSTANT */ /** @@ -95,5 +95,5 @@ pam_setcred(pam_handle_t *pamh, * * The latter four are mutually exclusive. * - * If any other bits are set, =pam_setcred will return =PAM_SYMBOL_ERR. + * If any other bits are set, =pam_setcred will return =PAM_BAD_CONSTANT. */ diff --git a/contrib/openpam/lib/libpam/pam_setenv.c b/contrib/openpam/lib/libpam/pam_setenv.c index 3c2209c1c0dd9..f05faf2366086 100644 --- a/contrib/openpam/lib/libpam/pam_setenv.c +++ b/contrib/openpam/lib/libpam/pam_setenv.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_setenv.c 914 2017-01-21 15:15:29Z des $ + * $OpenPAM: pam_setenv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_acct_mgmt.c b/contrib/openpam/lib/libpam/pam_sm_acct_mgmt.c index 6420fbac0fb2d..0919c26bd54d7 100644 --- a/contrib/openpam/lib/libpam/pam_sm_acct_mgmt.c +++ b/contrib/openpam/lib/libpam/pam_sm_acct_mgmt.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_acct_mgmt.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_acct_mgmt.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_authenticate.c b/contrib/openpam/lib/libpam/pam_sm_authenticate.c index f31b6ac2db24f..79c134866d87d 100644 --- a/contrib/openpam/lib/libpam/pam_sm_authenticate.c +++ b/contrib/openpam/lib/libpam/pam_sm_authenticate.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_authenticate.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_authenticate.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_authenticate_secondary.c b/contrib/openpam/lib/libpam/pam_sm_authenticate_secondary.c index 4a232fbd0fcbc..2956142965716 100644 --- a/contrib/openpam/lib/libpam/pam_sm_authenticate_secondary.c +++ b/contrib/openpam/lib/libpam/pam_sm_authenticate_secondary.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_authenticate_secondary.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_authenticate_secondary.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_chauthtok.c b/contrib/openpam/lib/libpam/pam_sm_chauthtok.c index 4f17562af288e..fcf6542628b3b 100644 --- a/contrib/openpam/lib/libpam/pam_sm_chauthtok.c +++ b/contrib/openpam/lib/libpam/pam_sm_chauthtok.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_chauthtok.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_chauthtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_close_session.c b/contrib/openpam/lib/libpam/pam_sm_close_session.c index 9247d8f5b1fc6..ffd78fc2abaa6 100644 --- a/contrib/openpam/lib/libpam/pam_sm_close_session.c +++ b/contrib/openpam/lib/libpam/pam_sm_close_session.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_close_session.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_close_session.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_get_mapped_authtok.c b/contrib/openpam/lib/libpam/pam_sm_get_mapped_authtok.c index 473a52e7742ed..f5fba4ec6770f 100644 --- a/contrib/openpam/lib/libpam/pam_sm_get_mapped_authtok.c +++ b/contrib/openpam/lib/libpam/pam_sm_get_mapped_authtok.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_get_mapped_authtok.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_get_mapped_authtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_get_mapped_username.c b/contrib/openpam/lib/libpam/pam_sm_get_mapped_username.c index 6de9fdb464378..286f6e4a444d9 100644 --- a/contrib/openpam/lib/libpam/pam_sm_get_mapped_username.c +++ b/contrib/openpam/lib/libpam/pam_sm_get_mapped_username.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_get_mapped_username.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_get_mapped_username.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_open_session.c b/contrib/openpam/lib/libpam/pam_sm_open_session.c index a576a00b1af98..293d40aadef20 100644 --- a/contrib/openpam/lib/libpam/pam_sm_open_session.c +++ b/contrib/openpam/lib/libpam/pam_sm_open_session.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_open_session.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_open_session.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_set_mapped_authtok.c b/contrib/openpam/lib/libpam/pam_sm_set_mapped_authtok.c index de0e91f86e5db..afcb55928198d 100644 --- a/contrib/openpam/lib/libpam/pam_sm_set_mapped_authtok.c +++ b/contrib/openpam/lib/libpam/pam_sm_set_mapped_authtok.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_set_mapped_authtok.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_set_mapped_authtok.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_set_mapped_username.c b/contrib/openpam/lib/libpam/pam_sm_set_mapped_username.c index 2696d3e182ece..ed844c3385a8f 100644 --- a/contrib/openpam/lib/libpam/pam_sm_set_mapped_username.c +++ b/contrib/openpam/lib/libpam/pam_sm_set_mapped_username.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_set_mapped_username.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_set_mapped_username.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_sm_setcred.c b/contrib/openpam/lib/libpam/pam_sm_setcred.c index 0836adc54f8e1..1926c56640318 100644 --- a/contrib/openpam/lib/libpam/pam_sm_setcred.c +++ b/contrib/openpam/lib/libpam/pam_sm_setcred.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_sm_setcred.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_sm_setcred.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_start.c b/contrib/openpam/lib/libpam/pam_start.c index 03fae416cad61..6605d9179ce2e 100644 --- a/contrib/openpam/lib/libpam/pam_start.c +++ b/contrib/openpam/lib/libpam/pam_start.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_start.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_start.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_strerror.c b/contrib/openpam/lib/libpam/pam_strerror.c index 8621ad6c3dd4a..2f6fd70a25ba1 100644 --- a/contrib/openpam/lib/libpam/pam_strerror.c +++ b/contrib/openpam/lib/libpam/pam_strerror.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2002-2003 Networks Associates Technology, Inc. - * Copyright (c) 2004-2011 Dag-Erling Smørgrav + * Copyright (c) 2004-2017 Dag-Erling Smørgrav * All rights reserved. * * This software was developed for the FreeBSD Project by ThinkSec AS and @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_strerror.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_strerror.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -59,69 +59,9 @@ pam_strerror(const pam_handle_t *pamh, static char unknown[16]; (void)pamh; - - switch (error_number) { - case PAM_SUCCESS: - return ("success"); - case PAM_OPEN_ERR: - return ("failed to load module"); - case PAM_SYMBOL_ERR: - return ("invalid symbol"); - case PAM_SERVICE_ERR: - return ("error in service module"); - case PAM_SYSTEM_ERR: - return ("system error"); - case PAM_BUF_ERR: - return ("memory buffer error"); - case PAM_CONV_ERR: - return ("conversation failure"); - case PAM_PERM_DENIED: - return ("permission denied"); - case PAM_MAXTRIES: - return ("maximum number of tries exceeded"); - case PAM_AUTH_ERR: - return ("authentication error"); - case PAM_NEW_AUTHTOK_REQD: - return ("new authentication token required"); - case PAM_CRED_INSUFFICIENT: - return ("insufficient credentials"); - case PAM_AUTHINFO_UNAVAIL: - return ("authentication information is unavailable"); - case PAM_USER_UNKNOWN: - return ("unknown user"); - case PAM_CRED_UNAVAIL: - return ("failed to retrieve user credentials"); - case PAM_CRED_EXPIRED: - return ("user credentials have expired"); - case PAM_CRED_ERR: - return ("failed to set user credentials"); - case PAM_ACCT_EXPIRED: - return ("user account has expired"); - case PAM_AUTHTOK_EXPIRED: - return ("password has expired"); - case PAM_SESSION_ERR: - return ("session failure"); - case PAM_AUTHTOK_ERR: - return ("authentication token failure"); - case PAM_AUTHTOK_RECOVERY_ERR: - return ("failed to recover old authentication token"); - case PAM_AUTHTOK_LOCK_BUSY: - return ("authentication token lock busy"); - case PAM_AUTHTOK_DISABLE_AGING: - return ("authentication token aging disabled"); - case PAM_NO_MODULE_DATA: - return ("module data not found"); - case PAM_IGNORE: - return ("ignore this module"); - case PAM_ABORT: - return ("general failure"); - case PAM_TRY_AGAIN: - return ("try again"); - case PAM_MODULE_UNKNOWN: - return ("unknown module type"); - case PAM_DOMAIN_UNKNOWN: - return ("unknown authentication domain"); - default: + if (error_number >= 0 && error_number < PAM_NUM_ERRORS) { + return (pam_err_text[error_number]); + } else { snprintf(unknown, sizeof unknown, "#%d", error_number); return (unknown); } diff --git a/contrib/openpam/lib/libpam/pam_verror.c b/contrib/openpam/lib/libpam/pam_verror.c index 683abbc0b2fbe..67950cf711592 100644 --- a/contrib/openpam/lib/libpam/pam_verror.c +++ b/contrib/openpam/lib/libpam/pam_verror.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_verror.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_verror.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_vinfo.c b/contrib/openpam/lib/libpam/pam_vinfo.c index 3ae2c7faa19ac..3e027914630ae 100644 --- a/contrib/openpam/lib/libpam/pam_vinfo.c +++ b/contrib/openpam/lib/libpam/pam_vinfo.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_vinfo.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_vinfo.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/lib/libpam/pam_vprompt.c b/contrib/openpam/lib/libpam/pam_vprompt.c index eb337af0dc068..982e6ae5a5cea 100644 --- a/contrib/openpam/lib/libpam/pam_vprompt.c +++ b/contrib/openpam/lib/libpam/pam_vprompt.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_vprompt.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_vprompt.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/misc/gendoc.pl b/contrib/openpam/misc/gendoc.pl index 3202dc6f5529b..b8e6c96af428d 100644 --- a/contrib/openpam/misc/gendoc.pl +++ b/contrib/openpam/misc/gendoc.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -w #- # Copyright (c) 2002-2003 Networks Associates Technology, Inc. -# Copyright (c) 2004-2014 Dag-Erling Smørgrav +# Copyright (c) 2004-2017 Dag-Erling Smørgrav # All rights reserved. # # This software was developed for the FreeBSD Project by ThinkSec AS and @@ -33,7 +33,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # -# $Id: gendoc.pl 910 2017-01-21 12:22:08Z des $ +# $OpenPAM: gendoc.pl 938 2017-04-30 21:34:42Z des $ # use strict; @@ -93,6 +93,10 @@ The OpenPAM library is maintained by PAM_TRY_AGAIN => "Try again", PAM_MODULE_UNKNOWN => "Unknown module type", PAM_DOMAIN_UNKNOWN => "Unknown authentication domain", + PAM_BAD_HANDLE => "Invalid PAM handle", + PAM_BAD_ITEM => "Unrecognized or restricted item", + PAM_BAD_FEATURE => "Unrecognized or restricted feature", + PAM_BAD_CONSTANT => "Bad constant", ); sub parse_source($) { @@ -114,7 +118,7 @@ sub parse_source($) { my $experimental; my $version; my %xref; - my @errors; + my %errors; my $author; if ($fn !~ m,\.c$,) { @@ -130,7 +134,7 @@ sub parse_source($) { return undef if ($source =~ m/^ \* NOPARSE\s*$/m); - if ($source =~ m/(\$Id:[^\$]+\$)/) { + if ($source =~ m/(\$OpenPAM:[^\$]+\$)/) { $version = $1; } @@ -168,8 +172,8 @@ sub parse_source($) { if ($type eq "int") { foreach (split("\n", $source)) { - next unless (m/^ \*\s+(!?PAM_[A-Z_]+|=[a-z_]+)\s*$/); - push(@errors, $1); + next unless (m/^ \*\t(!?PAM_[A-Z_]+|=[a-z_]+)\s*(.*?)\s*$/); + $errors{$1} = $2; } ++$xref{3}->{pam_strerror}; } @@ -335,7 +339,7 @@ sub parse_source($) { 'args' => $args, 'man' => $man, 'xref' => \%xref, - 'errors' => \@errors, + 'errors' => \%errors, 'author' => $author, 'customrv' => $customrv, 'deprecated' => $deprecated, @@ -365,13 +369,13 @@ sub expand_errors($) { } $$func{recursed} = 1; - foreach (@{$$func{errors}}) { + foreach (keys %{$$func{errors}}) { if (m/^(PAM_[A-Z_]+)$/) { if (!defined($PAMERR{$1})) { warn("$$func{name}(): unrecognized error: $1\n"); next; } - $errors{$1} = 1; + $errors{$1} = $$func{errors}->{$_}; } elsif (m/^!(PAM_[A-Z_]+)$/) { # treat negations separately } elsif (m/^=([a-z_]+)$/) { @@ -385,20 +389,20 @@ sub expand_errors($) { warn("$$func{name}(): reference to unknown $ref()\n"); next; } - foreach (@{$FUNCTIONS{$ref}->{errors}}) { - $errors{$_} = 1; + foreach (keys %{$FUNCTIONS{$ref}->{errors}}) { + $errors{$_} //= $FUNCTIONS{$ref}->{errors}->{$_}; } } else { warn("$$func{name}(): invalid error specification: $_\n"); } } - foreach (@{$$func{errors}}) { + foreach (keys %{$$func{errors}}) { if (m/^!(PAM_[A-Z_]+)$/) { delete($errors{$1}); } } delete($$func{recursed}); - $$func{errors} = [ sort(keys(%errors)) ]; + $$func{errors} = \%errors; } sub dictionary_order($$) { @@ -430,6 +434,7 @@ sub gendoc($) { my $func = shift; # Ref to function hash local *FILE; + my %errors; my $mdoc; my $fn; @@ -489,18 +494,21 @@ sub gendoc($) { $mdoc .= ".Ef\n.Pp\n"; } $mdoc .= "$$func{man}\n"; - my @errors = @{$$func{errors}}; + %errors = %{$$func{errors}}; if ($$func{customrv}) { # leave it - } elsif ($$func{type} eq "int" && @errors) { + } elsif ($$func{type} eq "int" && %errors) { $mdoc .= ".Sh RETURN VALUES The .Fn $$func{name} function returns one of the following values: .Bl -tag -width 18n "; - foreach (@errors) { - $mdoc .= ".It Bq Er $_\n$PAMERR{$_}.\n"; + delete($errors{PAM_SUCCESS}); + foreach ('PAM_SUCCESS', sort keys %errors) { + $mdoc .= ".It Bq Er $_\n" . + ($errors{$_} || $PAMERR{$_}) . + ".\n"; } $mdoc .= ".El\n"; } elsif ($$func{type} eq "int") { diff --git a/contrib/openpam/mkpkgng.in b/contrib/openpam/mkpkgng.in index 3c57b7dc184f8..da4e75919bef8 100644 --- a/contrib/openpam/mkpkgng.in +++ b/contrib/openpam/mkpkgng.in @@ -27,7 +27,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # -# $Id: mkpkgng.in 890 2016-01-11 16:22:09Z des $ +# $OpenPAM: mkpkgng.in 938 2017-04-30 21:34:42Z des $ # # Print an informational message diff --git a/contrib/openpam/modules/Makefile.am b/contrib/openpam/modules/Makefile.am index 2ce129e5175c9..99dd788c9d4d9 100644 --- a/contrib/openpam/modules/Makefile.am +++ b/contrib/openpam/modules/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ SUBDIRS = pam_deny pam_permit pam_return diff --git a/contrib/openpam/modules/Makefile.in b/contrib/openpam/modules/Makefile.in index 172cda967b8f5..683fe342cd366 100644 --- a/contrib/openpam/modules/Makefile.in +++ b/contrib/openpam/modules/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 907 2017-01-18 09:39:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ diff --git a/contrib/openpam/modules/pam_deny/Makefile.am b/contrib/openpam/modules/pam_deny/Makefile.am index d48418cf258c8..f2b4fcb07e8a2 100644 --- a/contrib/openpam/modules/pam_deny/Makefile.am +++ b/contrib/openpam/modules/pam_deny/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ if CUSTOM_MODULES_DIR moduledir = $(OPENPAM_MODULES_DIR) diff --git a/contrib/openpam/modules/pam_deny/Makefile.in b/contrib/openpam/modules/pam_deny/Makefile.in index 17437e8e3ad1a..7f92f93fef040 100644 --- a/contrib/openpam/modules/pam_deny/Makefile.in +++ b/contrib/openpam/modules/pam_deny/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/modules/pam_deny/pam_deny.c b/contrib/openpam/modules/pam_deny/pam_deny.c index d6983c5a35218..e0b1762bd2005 100644 --- a/contrib/openpam/modules/pam_deny/pam_deny.c +++ b/contrib/openpam/modules/pam_deny/pam_deny.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_deny.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_deny.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/modules/pam_permit/Makefile.am b/contrib/openpam/modules/pam_permit/Makefile.am index b058c1e1aa021..faec653ccf590 100644 --- a/contrib/openpam/modules/pam_permit/Makefile.am +++ b/contrib/openpam/modules/pam_permit/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ if CUSTOM_MODULES_DIR moduledir = $(OPENPAM_MODULES_DIR) diff --git a/contrib/openpam/modules/pam_permit/Makefile.in b/contrib/openpam/modules/pam_permit/Makefile.in index ab35870379242..dbe448aaaa95a 100644 --- a/contrib/openpam/modules/pam_permit/Makefile.in +++ b/contrib/openpam/modules/pam_permit/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/modules/pam_permit/pam_permit.c b/contrib/openpam/modules/pam_permit/pam_permit.c index f42b6c1a95caa..454853456d752 100644 --- a/contrib/openpam/modules/pam_permit/pam_permit.c +++ b/contrib/openpam/modules/pam_permit/pam_permit.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_permit.c 648 2013-03-05 17:54:27Z des $ + * $OpenPAM: pam_permit.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/modules/pam_return/Makefile.am b/contrib/openpam/modules/pam_return/Makefile.am index 33e67262596a4..eaecc212fb04b 100644 --- a/contrib/openpam/modules/pam_return/Makefile.am +++ b/contrib/openpam/modules/pam_return/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 873 2015-12-01 19:38:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ if CUSTOM_MODULES_DIR moduledir = $(OPENPAM_MODULES_DIR) diff --git a/contrib/openpam/modules/pam_return/Makefile.in b/contrib/openpam/modules/pam_return/Makefile.in index cbb884be8565d..47c52cb5cb07f 100644 --- a/contrib/openpam/modules/pam_return/Makefile.in +++ b/contrib/openpam/modules/pam_return/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 873 2015-12-01 19:38:01Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/modules/pam_return/pam_return.c b/contrib/openpam/modules/pam_return/pam_return.c index 9455e338cc8e5..dcbfe95b82682 100644 --- a/contrib/openpam/modules/pam_return/pam_return.c +++ b/contrib/openpam/modules/pam_return/pam_return.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_return.c 873 2015-12-01 19:38:01Z des $ + * $OpenPAM: pam_return.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/modules/pam_unix/Makefile.am b/contrib/openpam/modules/pam_unix/Makefile.am index a1c97c737cecb..418f315244621 100644 --- a/contrib/openpam/modules/pam_unix/Makefile.am +++ b/contrib/openpam/modules/pam_unix/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ if CUSTOM_MODULES_DIR moduledir = $(OPENPAM_MODULES_DIR) diff --git a/contrib/openpam/modules/pam_unix/Makefile.in b/contrib/openpam/modules/pam_unix/Makefile.in index 1a2507d9bd803..fda413dd55154 100644 --- a/contrib/openpam/modules/pam_unix/Makefile.in +++ b/contrib/openpam/modules/pam_unix/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 834 2014-10-28 10:25:58Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/modules/pam_unix/pam_unix.c b/contrib/openpam/modules/pam_unix/pam_unix.c index 06d020bc3a22d..ab8134d7a85b7 100644 --- a/contrib/openpam/modules/pam_unix/pam_unix.c +++ b/contrib/openpam/modules/pam_unix/pam_unix.c @@ -32,7 +32,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: pam_unix.c 890 2016-01-11 16:22:09Z des $ + * $OpenPAM: pam_unix.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H diff --git a/contrib/openpam/t/Makefile.am b/contrib/openpam/t/Makefile.am index 9c538a4dd959a..55fdb48b7e68f 100644 --- a/contrib/openpam/t/Makefile.am +++ b/contrib/openpam/t/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am 922 2017-02-19 19:28:30Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ if WITH_TEST diff --git a/contrib/openpam/t/Makefile.in b/contrib/openpam/t/Makefile.in index 119d686e9c00b..002fdd8404fa6 100644 --- a/contrib/openpam/t/Makefile.in +++ b/contrib/openpam/t/Makefile.in @@ -14,7 +14,7 @@ @SET_MAKE@ -# $Id: Makefile.am 922 2017-02-19 19:28:30Z des $ +# $OpenPAM: Makefile.am 938 2017-04-30 21:34:42Z des $ VPATH = @srcdir@ am__is_gnu_make = { \ diff --git a/contrib/openpam/t/t_openpam_ctype.c b/contrib/openpam/t/t_openpam_ctype.c index 3fd617d4c7afd..ec7e807d0e603 100644 --- a/contrib/openpam/t/t_openpam_ctype.c +++ b/contrib/openpam/t/t_openpam_ctype.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2014-2015 Dag-Erling Smørgrav + * Copyright (c) 2014-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: t_openpam_ctype.c 922 2017-02-19 19:28:30Z des $ + * $OpenPAM: t_openpam_ctype.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -75,7 +75,7 @@ static const char oc_pfcs[] = OC_PFCS; crib[(int)oc_##set[i]] = 1; \ for (i = ret = 0; i < sizeof crib; ++i) { \ if (is_##set(i) != crib[i]) { \ - t_verbose("is_%s() incorrect " \ + t_printv("is_%s() incorrect " \ "for %#02x\n", #set, i); \ ++ret; \ } \ diff --git a/contrib/openpam/t/t_openpam_dispatch.c b/contrib/openpam/t/t_openpam_dispatch.c index f198a994e63e4..f2f3053a726cc 100644 --- a/contrib/openpam/t/t_openpam_dispatch.c +++ b/contrib/openpam/t/t_openpam_dispatch.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 Dag-Erling Smørgrav + * Copyright (c) 2015-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: t_openpam_dispatch.c 922 2017-02-19 19:28:30Z des $ + * $OpenPAM: t_openpam_dispatch.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -54,7 +54,7 @@ OPENPAM_UNUSED(void *arg)) #define T(n) \ - t_add_test(&t_ ## n ## _func, NULL, t_ ## n ## _desc) + t_add_test(&t_ ## n ## _func, NULL, "%s", t_ ## n ## _desc) const char *pam_return_so; @@ -73,7 +73,7 @@ T_FUNC(empty_policy, "empty policy") t_fprintf(tf, "# empty policy\n"); pam_err = pam_start(tf->name, "test", &pamc, &pamh); if (pam_err != PAM_SUCCESS) { - t_verbose("pam_start() returned %d\n", pam_err); + t_printv("pam_start() returned %d\n", pam_err); return (0); } /* @@ -82,22 +82,22 @@ T_FUNC(empty_policy, "empty policy") * instead. */ pam_err = pam_authenticate(pamh, 0); - t_verbose("pam_authenticate() returned %d\n", pam_err); + t_printv("pam_authenticate() returned %d\n", pam_err); ret = (pam_err == PAM_SYSTEM_ERR); pam_err = pam_setcred(pamh, 0); - t_verbose("pam_setcred() returned %d\n", pam_err); + t_printv("pam_setcred() returned %d\n", pam_err); ret &= (pam_err == PAM_SYSTEM_ERR); pam_err = pam_acct_mgmt(pamh, 0); - t_verbose("pam_acct_mgmt() returned %d\n", pam_err); + t_printv("pam_acct_mgmt() returned %d\n", pam_err); ret &= (pam_err == PAM_SYSTEM_ERR); pam_err = pam_chauthtok(pamh, 0); - t_verbose("pam_chauthtok() returned %d\n", pam_err); + t_printv("pam_chauthtok() returned %d\n", pam_err); ret &= (pam_err == PAM_SYSTEM_ERR); pam_err = pam_open_session(pamh, 0); - t_verbose("pam_open_session() returned %d\n", pam_err); + t_printv("pam_open_session() returned %d\n", pam_err); ret &= (pam_err == PAM_SYSTEM_ERR); pam_err = pam_close_session(pamh, 0); - t_verbose("pam_close_session() returned %d\n", pam_err); + t_printv("pam_close_session() returned %d\n", pam_err); ret &= (pam_err == PAM_SYSTEM_ERR); pam_end(pamh, pam_err); t_fclose(tf); @@ -150,7 +150,7 @@ T_FUNC(mod_return, "module return value") } pam_err = pam_start(tf->name, "test", &pamc, &pamh); if (pam_err != PAM_SUCCESS) { - t_verbose("pam_start() returned %d\n", pam_err); + t_printv("pam_start() returned %d\n", pam_err); t_fclose(tf); continue; } @@ -174,10 +174,10 @@ T_FUNC(mod_return, "module return value") pam_err = pam_chauthtok(pamh, tc->flags); break; } - t_verbose("%s returned %d\n", + t_printv("%s returned %d\n", pam_func_name[tc->primitive], pam_err); pam_end(pamh, pam_err); - t_verbose("here\n"); + t_printv("here\n"); t_fclose(tf); } return (1); @@ -196,7 +196,7 @@ t_prepare(int argc, char *argv[]) (void)argv; if ((pam_return_so = getenv("PAM_RETURN_SO")) == NULL) { - t_verbose("define PAM_RETURN_SO before running these tests\n"); + t_printv("define PAM_RETURN_SO before running these tests\n"); return (0); } diff --git a/contrib/openpam/t/t_openpam_readlinev.c b/contrib/openpam/t/t_openpam_readlinev.c index abc29e26cd557..e32f338794532 100644 --- a/contrib/openpam/t/t_openpam_readlinev.c +++ b/contrib/openpam/t/t_openpam_readlinev.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2015 Dag-Erling Smørgrav + * Copyright (c) 2012-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: t_openpam_readlinev.c 922 2017-02-19 19:28:30Z des $ + * $OpenPAM: t_openpam_readlinev.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -52,7 +52,7 @@ OPENPAM_UNUSED(void *arg)) #define T(n) \ - t_add_test(&t_ ## n ## _func, NULL, t_ ## n ## _desc) + t_add_test(&t_ ## n ## _func, NULL, "%s", t_ ## n ## _desc) /* * Read a line from the temp file and verify that the result matches our @@ -76,20 +76,20 @@ orlv_expect(struct t_file *tf, const char **expectedv, int lines, int eof) if (t_ferror(tf)) err(1, "%s(): %s", __func__, tf->name); if (expectedv != NULL && gotv == NULL) { - t_verbose("expected %d words, got nothing\n", expectedc); + t_printv("expected %d words, got nothing\n", expectedc); ret = 0; } else if (expectedv == NULL && gotv != NULL) { - t_verbose("expected nothing, got %d words\n", gotc); + t_printv("expected nothing, got %d words\n", gotc); ret = 0; } else if (expectedv != NULL && gotv != NULL) { if (expectedc != gotc) { - t_verbose("expected %d words, got %d\n", + t_printv("expected %d words, got %d\n", expectedc, gotc); ret = 0; } for (i = 0; i < gotc; ++i) { if (strcmp(expectedv[i], gotv[i]) != 0) { - t_verbose("word %d: expected <<%s>>, " + t_printv("word %d: expected <<%s>>, " "got <<%s>>\n", i, expectedv[i], gotv[i]); ret = 0; } @@ -97,15 +97,15 @@ orlv_expect(struct t_file *tf, const char **expectedv, int lines, int eof) } FREEV(gotc, gotv); if (lineno != lines) { - t_verbose("expected to advance %d lines, advanced %d lines\n", + t_printv("expected to advance %d lines, advanced %d lines\n", lines, lineno); ret = 0; } if (eof && !t_feof(tf)) { - t_verbose("expected EOF, but didn't get it\n"); + t_printv("expected EOF, but didn't get it\n"); ret = 0; } else if (!eof && t_feof(tf)) { - t_verbose("didn't expect EOF, but got it anyway\n"); + t_printv("didn't expect EOF, but got it anyway\n"); ret = 0; } return (ret); diff --git a/contrib/openpam/t/t_openpam_readword.c b/contrib/openpam/t/t_openpam_readword.c index 767696e52a550..e7e9871f45ca6 100644 --- a/contrib/openpam/t/t_openpam_readword.c +++ b/contrib/openpam/t/t_openpam_readword.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2015 Dag-Erling Smørgrav + * Copyright (c) 2012-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: t_openpam_readword.c 922 2017-02-19 19:28:30Z des $ + * $OpenPAM: t_openpam_readword.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -34,6 +34,7 @@ #endif #include <err.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -50,7 +51,7 @@ OPENPAM_UNUSED(void *arg)) #define T(n) \ - t_add_test(&t_ ## n ## _func, NULL, t_ ## n ## _desc) + t_add_test(&t_ ## n ## _func, NULL, "%s", t_ ## n ## _desc) /* * Read a word from the temp file and verify that the result matches our @@ -71,37 +72,37 @@ orw_expect(struct t_file *tf, const char *expected, int lines, int eof, int eol) if (t_ferror(tf)) err(1, "%s(): %s", __func__, tf->name); if (expected != NULL && got == NULL) { - t_verbose("expected <<%s>>, got nothing\n", expected); + t_printv("expected <<%s>>, got nothing\n", expected); ret = 0; } else if (expected == NULL && got != NULL) { - t_verbose("expected nothing, got <<%s>>\n", got); + t_printv("expected nothing, got <<%s>>\n", got); ret = 0; } else if (expected != NULL && got != NULL && strcmp(expected, got) != 0) { - t_verbose("expected <<%s>>, got <<%s>>\n", expected, got); + t_printv("expected <<%s>>, got <<%s>>\n", expected, got); ret = 0; } free(got); if (lineno != lines) { - t_verbose("expected to advance %d lines, advanced %d lines\n", + t_printv("expected to advance %d lines, advanced %d lines\n", lines, lineno); ret = 0; } if (eof && !t_feof(tf)) { - t_verbose("expected EOF, but didn't get it\n"); + t_printv("expected EOF, but didn't get it\n"); ret = 0; } if (!eof && t_feof(tf)) { - t_verbose("didn't expect EOF, but got it anyway\n"); + t_printv("didn't expect EOF, but got it anyway\n"); ret = 0; } ch = fgetc(tf->file); if (t_ferror(tf)) err(1, "%s(): %s", __func__, tf->name); if (eol && ch != '\n') { - t_verbose("expected EOL, but didn't get it\n"); + t_printv("expected EOL, but didn't get it\n"); ret = 0; } else if (!eol && ch == '\n') { - t_verbose("didn't expect EOL, but got it anyway\n"); + t_printv("didn't expect EOL, but got it anyway\n"); ret = 0; } if (ch != EOF) diff --git a/contrib/openpam/t/t_pam_conv.c b/contrib/openpam/t/t_pam_conv.c index 58e5c086c1b5e..243d96df3f766 100644 --- a/contrib/openpam/t/t_pam_conv.c +++ b/contrib/openpam/t/t_pam_conv.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 Dag-Erling Smørgrav + * Copyright (c) 2015-2017 Dag-Erling Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: t_pam_conv.c 922 2017-02-19 19:28:30Z des $ + * $OpenPAM: t_pam_conv.c 938 2017-04-30 21:34:42Z des $ */ #ifdef HAVE_CONFIG_H @@ -99,16 +99,16 @@ t_pam_conv(int nm, const struct pam_message **msgs, } switch (msgs[i]->msg_style) { case PAM_PROMPT_ECHO_OFF: - t_verbose("[PAM_PROMPT_ECHO_OFF] %s\n", msgs[i]->msg); + t_printv("[PAM_PROMPT_ECHO_OFF] %s\n", msgs[i]->msg); break; case PAM_PROMPT_ECHO_ON: - t_verbose("[PAM_PROMPT_ECHO_ON] %s\n", msgs[i]->msg); + t_printv("[PAM_PROMPT_ECHO_ON] %s\n", msgs[i]->msg); break; case PAM_ERROR_MSG: - t_verbose("[PAM_ERROR_MSG] %s\n", msgs[i]->msg); + t_printv("[PAM_ERROR_MSG] %s\n", msgs[i]->msg); break; case PAM_TEXT_INFO: - t_verbose("[PAM_TEXT_INFO] %s\n", msgs[i]->msg); + t_printv("[PAM_TEXT_INFO] %s\n", msgs[i]->msg); break; default: asprintf(&s->comment, "invalid message style %d", diff --git a/contrib/openpam/t/t_pam_conv.h b/contrib/openpam/t/t_pam_conv.h index ab07497445941..07ba702c940aa 100644 --- a/contrib/openpam/t/t_pam_conv.h +++ b/contrib/openpam/t/t_pam_conv.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: t_pam_conv.h 863 2015-07-30 23:44:31Z des $ + * $OpenPAM: t_pam_conv.h 938 2017-04-30 21:34:42Z des $ */ #ifndef T_PAM_CONV_H_INCLUDED diff --git a/contrib/zstd/.gitignore b/contrib/zstd/.gitignore deleted file mode 100644 index e02119883a7df..0000000000000 --- a/contrib/zstd/.gitignore +++ /dev/null @@ -1,41 +0,0 @@ -# Object files -*.o -*.ko -*.dSYM - -# Libraries -*.lib -*.a - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib - -# Executables -zstd -zstdmt -*.exe -*.out -*.app - -# Test artefacts -tmp* -dictionary* - -# Other files -.directory -_codelite/ -_zstdbench/ -.clang_complete -*.idea -*.swp -.DS_Store -googletest/ -*.d - -# Directories -bin/ -.buckd/ -buck-out/ diff --git a/contrib/zstd/.travis.yml b/contrib/zstd/.travis.yml deleted file mode 100644 index 3f98ea04dd75e..0000000000000 --- a/contrib/zstd/.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Long tests: run on commits to master branch/cron builds - -language: c -sudo: required -dist: trusty -matrix: - include: - # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test' - - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test32' - - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test' - - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' - - - env: Cmd='make arminstall && make armtest' - - env: Cmd='make arminstall && make aarch64test' - - env: Cmd='make ppcinstall && make ppctest' - - env: Cmd='make ppcinstall && make ppc64test' - - - - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest' - - env: Cmd='make -C tests versionsTest' - - env: Cmd='make gpp6install && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' - install: - - export CXX="g++-6" CC="gcc-6" - - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zstd-nolegacy' - - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zbuff' - - # OS X Mavericks - - env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install" - os: osx - -git: - depth: 1 - -branches: - only: - - dev - - master - -script: - - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST - - export FUZZERTEST=-T5mn; - export ZSTREAM_TESTTIME=-T5mn; - export DECODECORPUS_TESTTIME=-T1mn; - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then - git fetch origin dev; - git checkout -f FETCH_HEAD; - fi; - sh -c "$Cmd" || travis_terminate 1; diff --git a/contrib/zstd/Makefile b/contrib/zstd/Makefile index 49f29d782276a..54652665bc767 100644 --- a/contrib/zstd/Makefile +++ b/contrib/zstd/Makefile @@ -90,6 +90,10 @@ examples: manual: $(MAKE) -C contrib/gen_html $@ +.PHONY: cleanTabs +cleanTabs: + cd contrib; ./cleanTabs + .PHONY: clean clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @@ -105,9 +109,15 @@ clean: # make install is validated only for Linux, OSX, Hurd and some BSD targets #------------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) + HOST_OS = POSIX -.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + +.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan install: @$(MAKE) -C $(ZSTDDIR) $@ @$(MAKE) -C $(PRGDIR) $@ @@ -152,16 +162,16 @@ ppc64build: clean CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch armfuzz: clean - CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest aarch64fuzz: clean - CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest ppcfuzz: clean - CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest ppc64fuzz: clean - CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest gpptest: clean CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" @@ -180,19 +190,19 @@ clangtest: clean armtest: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests aarch64test: $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests ppctest: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests ppc64test: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests arm-ppc-compilation: $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" @@ -263,7 +273,7 @@ endif ifneq (,$(filter MSYS%,$(shell uname))) HOST_OS = MSYS -CMAKE_PARAMS = -G"MSYS Makefiles" +CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON endif @@ -275,7 +285,7 @@ cmakebuild: cmake --version $(RM) -r $(BUILDIR)/cmake/build mkdir $(BUILDIR)/cmake/build - cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90build: clean gcc -v diff --git a/contrib/zstd/NEWS b/contrib/zstd/NEWS index bbd9e16883864..7d9c9c94e186a 100644 --- a/contrib/zstd/NEWS +++ b/contrib/zstd/NEWS @@ -1,3 +1,25 @@ +v1.2.0 +cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) +cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell +cli : new : zstdmt symlink hardwired to `zstd -T0` +cli : new : command --threads=# (#671) +cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell +cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters +cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell +cli : fix : does not output compressed data on console +cli : fix : ignore symbolic links unless --force specified, +API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument +API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. +API : improved: ZSTDMT_compressCCtx() reduced memory usage +API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) +API : fix : src size stored in frame header is controlled at end of frame +API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) +API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate +build: improved cmake script, by @Majlen +build: enabled Multi-threading support for *BSD, by Baptiste Daroussin +tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. +new : contrib/linux-kernel version, by Nick Terrell + v1.1.4 cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski cli : new : advanced benchmark command --priority=rt @@ -5,13 +27,13 @@ cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 cli : fix : --rm remains silent when input is stdin cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski speed : improved decompression speed in streaming mode for single shot scenarios (+5%) -memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB -arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) -build: new: meson build system in contrib/meson, by Dima Krasner -build: improved cmake script, by @Majlen -build: added -Wformat-security flag, as recommended by Padraig Brady +build : new: meson build system in contrib/meson, by Dima Krasner +build : improved cmake script, by @Majlen +build : added -Wformat-security flag, as recommended by Padraig Brady doc : new : educational decoder, by Sean Purcell v1.1.3 diff --git a/contrib/zstd/README.md b/contrib/zstd/README.md index 6de5a10790db0..7caee5fd3f674 100644 --- a/contrib/zstd/README.md +++ b/contrib/zstd/README.md @@ -12,13 +12,13 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne |dev | [](https://travis-ci.org/facebook/zstd) | As a reference, several fast compression algorithms were tested and compared -on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`), +on a server running Linux Debian (`Linux version 4.8.0-1-amd64`), with a Core i7-6700K CPU @ 4.0GHz, -using [lzbench v1.6], an open-source in-memory benchmark by @inikep +using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 6.3.0, on the [Silesia compression corpus]. -[lzbench v1.6]: https://github.com/inikep/lzbench +[lzbench]: https://github.com/inikep/lzbench [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia | Compressor name | Ratio | Compression| Decompress.| @@ -38,7 +38,12 @@ on the [Silesia compression corpus]. Zstd can also offer stronger compression ratios at the cost of compression speed. Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma. -The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus]. +The following tests were run +on a server running Linux Debian (`Linux version 4.8.0-1-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with GCC 6.3.0, +on the [Silesia compression corpus]. Compression Speed vs Ratio | Decompression Speed ---------------------------|-------------------- diff --git a/contrib/zstd/appveyor.yml b/contrib/zstd/appveyor.yml index 9507fec6e24c0..67c4f72d7995b 100644 --- a/contrib/zstd/appveyor.yml +++ b/contrib/zstd/appveyor.yml @@ -1,66 +1,103 @@ -version: 1.0.{build} -environment: - matrix: - - COMPILER: "gcc" - PLATFORM: "mingw64" - MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' - - COMPILER: "gcc" - PLATFORM: "mingw32" - MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"' - - COMPILER: "gcc" - PLATFORM: "clang" - MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" +- + version: 1.0.{build} + branches: + only: + - dev + - master + environment: + matrix: + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "make allarch && make -C tests test-symbols fullbench-dll fullbench-lib" + ARTIFACT: "true" + BUILD: "true" + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x86" + SCRIPT: "make allarch" + ARTIFACT: "true" + BUILD: "true" + - COMPILER: "clang" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch" + BUILD: "true" -install: + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "" + TEST: "cmake" + + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "" + TEST: "pzstd" + + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Release" + + install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - - MKDIR bin - - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - - if [%COMPILER%]==[gcc] ( - SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && - SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && - COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe && - COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe - ) else ( - IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") + - SET PATH_ORIGINAL=%PATH% + - if [%HOST%]==[mingw] ( + SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && + SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe ) - -build_script: - - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%COMPILER%]==[gcc] ( - ECHO *** && - ECHO *** Building %PLATFORM% && - ECHO *** && - make -v && - cc -v && - ECHO %MAKE_PARAMS% && - sh -c %MAKE_PARAMS% + - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( + SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" ) - - if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe - - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] ( - COPY programs\zstd.exe bin\zstd.exe && - appveyor PushArtifact bin\zstd.exe + + build_script: + - if [%HOST%]==[mingw] ( + ( if [%PLATFORM%]==[x64] ( + SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%" + ) else if [%PLATFORM%]==[x86] ( + SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%" + ) ) ) - - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] ( - COPY programs\zstd.exe bin\zstd32.exe && - appveyor PushArtifact bin\zstd32.exe + - if [%HOST%]==[mingw] if [%BUILD%]==[true] ( + make -v && + sh -c "%COMPILER% -v" && + ECHO Building zlib to static link && + SET "CC=%COMPILER%" && + sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" && + sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a" + ECHO Building zstd && + SET "CPPFLAGS=-I../../zlib" && + SET "LDFLAGS=../../zlib/libz.a" && + sh -c "%SCRIPT%" && + ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true] + lib\dll\example\build_package.bat && + make -C programs DEBUGFLAGS= clean zstd && + cp programs\zstd.exe zstd_%PLATFORM%.exe && + appveyor PushArtifact zstd_%PLATFORM%.exe && + cp programs\zstd.exe bin\zstd.exe && + make -C programs DEBUGFLAGS= clean zstdmt && + cp programs\zstd.exe bin\zstdmt.exe && + cd bin\ && 7z a -tzip zstd-win-release-%PLATFORM%.zip * && + appveyor PushArtifact zstd-win-release-%PLATFORM%.zip + ) ) - - if [%COMPILER%]==[gcc] make clean - - if [%COMPILER%]==[visual] ( + - if [%HOST%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% && ECHO *** && @@ -111,29 +148,26 @@ build_script: COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) -test_script: + test_script: - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - - SET FUZZERTEST=-T1mn - - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] ( - tests\fuzzer_clang.exe %FUZZERTEST% && - ECHO *** && - ECHO *** Building cmake for %PLATFORM% && - ECHO *** && + - SET "CC=gcc" + - SET "CXX=g++" + - if [%TEST%]==[cmake] ( mkdir build\cmake\build && cd build\cmake\build && cmake -G "Visual Studio 14 2015 Win64" .. && cd ..\..\.. && - make clean && - ECHO *** && - ECHO *** Building pzstd for %PLATFORM% && - ECHO *** && + make clean + ) + - if [%TEST%]==[pzstd] ( make -C contrib\pzstd googletest-mingw64 && make -C contrib\pzstd pzstd.exe && make -C contrib\pzstd tests && make -C contrib\pzstd check && make -C contrib\pzstd clean ) - - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( + - SET "FUZZERTEST=-T30s" + - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] ( CD tests && SET ZSTD=./zstd.exe && sh -e playTests.sh --test-large-data && @@ -146,33 +180,76 @@ test_script: fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) -branches: - only: - - dev - - master +- + version: 1.0.{build} + environment: + matrix: + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "make allarch" + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x86" + SCRIPT: "make allarch" + - COMPILER: "clang" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch" -artifacts: - - path: bin\zstd.exe - - path: bin\zstd32.exe + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Release" + + install: + - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% + - SET PATH_ORIGINAL=%PATH% + - if [%HOST%]==[mingw] ( + SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && + SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe + ) + - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( + SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + ) -deploy: -- provider: GitHub - auth_token: - secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - artifact: bin\zstd.exe - force_update: true - on: - branch: autobuild - COMPILER: gcc - PLATFORM: "mingw64" - appveyor_repo_tag: true -- provider: GitHub - auth_token: - secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - artifact: bin\zstd32.exe - force_update: true - on: - branch: autobuild - COMPILER: gcc - PLATFORM: "mingw32" - appveyor_repo_tag: true + build_script: + - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%HOST%]==[mingw] ( + ( if [%PLATFORM%]==[x64] ( + SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%" + ) else if [%PLATFORM%]==[x86] ( + SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%" + ) ) && + make -v && + sh -c "%COMPILER% -v" && + set "CC=%COMPILER%" && + sh -c "%SCRIPT%" + ) + - if [%HOST%]==[visual] ( + ECHO *** && + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && + COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ + ) diff --git a/contrib/zstd/circle.yml b/contrib/zstd/circle.yml index 298569d145511..218e33bfc333b 100644 --- a/contrib/zstd/circle.yml +++ b/contrib/zstd/circle.yml @@ -3,7 +3,7 @@ dependencies: - sudo dpkg --add-architecture i386 - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 + - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386 test: diff --git a/contrib/zstd/contrib/cleanTabs b/contrib/zstd/contrib/cleanTabs new file mode 100755 index 0000000000000..215913a90ace4 --- /dev/null +++ b/contrib/zstd/contrib/cleanTabs @@ -0,0 +1,2 @@ +#!/bin/sh +sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c} diff --git a/contrib/zstd/contrib/pzstd/Options.cpp b/contrib/zstd/contrib/pzstd/Options.cpp index a0d969393c9b4..1f53f2bff78a8 100644 --- a/contrib/zstd/contrib/pzstd/Options.cpp +++ b/contrib/zstd/contrib/pzstd/Options.cpp @@ -91,7 +91,7 @@ void usage() { std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel); std::fprintf(stderr, " -d, --decompress : decompression\n"); std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n"); - std::fprintf(stderr, " -f, --force : overwrite output without prompting\n"); + std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n"); std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n"); std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n"); std::fprintf(stderr, " -h, --help : display help and exit\n"); @@ -121,6 +121,7 @@ Options::Status Options::parse(int argc, const char **argv) { bool recursive = false; bool ultra = false; bool forceStdout = false; + bool followLinks = false; // Local copy of input files, which are pointers into argv. std::vector<const char *> localInputFiles; for (int i = 1; i < argc; ++i) { @@ -255,6 +256,7 @@ Options::Status Options::parse(int argc, const char **argv) { case 'f': overwrite = true; forceStdout = true; + followLinks = true; break; case 't': test = true; @@ -328,13 +330,29 @@ Options::Status Options::parse(int argc, const char **argv) { } } + g_utilDisplayLevel = verbosity; + // Remove local input files that are symbolic links + if (!followLinks) { + std::remove_if(localInputFiles.begin(), localInputFiles.end(), + [&](const char *path) { + bool isLink = UTIL_isLink(path); + if (isLink && verbosity >= 2) { + std::fprintf( + stderr, + "Warning : %s is symbolic link, ignoring\n", + path); + } + return isLink; + }); + } + // Translate input files/directories into files to (de)compress if (recursive) { char *scratchBuffer = nullptr; unsigned numFiles = 0; const char **files = UTIL_createFileList(localInputFiles.data(), localInputFiles.size(), - &scratchBuffer, &numFiles); + &scratchBuffer, &numFiles, followLinks); if (files == nullptr) { std::fprintf(stderr, "Error traversing directories\n"); return Status::Failure; diff --git a/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp index 1d857aae808da..89085afd434ca 100644 --- a/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp +++ b/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp @@ -10,6 +10,7 @@ #include <gtest/gtest.h> #include <atomic> +#include <iostream> #include <thread> #include <vector> @@ -34,16 +35,19 @@ TEST(ThreadPool, AllJobsFinished) { std::atomic<unsigned> numFinished{0}; std::atomic<bool> start{false}; { + std::cerr << "Creating executor" << std::endl; ThreadPool executor(5); for (int i = 0; i < 10; ++i) { executor.add([ &numFinished, &start ] { while (!start.load()) { - // spin + std::this_thread::yield(); } ++numFinished; }); } + std::cerr << "Starting" << std::endl; start.store(true); + std::cerr << "Finishing" << std::endl; } EXPECT_EQ(10, numFinished.load()); } diff --git a/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp index 7f58ccb3f1997..8caf170d2948d 100644 --- a/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp +++ b/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp @@ -10,6 +10,7 @@ #include "utils/WorkQueue.h" #include <gtest/gtest.h> +#include <iostream> #include <memory> #include <mutex> #include <thread> @@ -201,11 +202,13 @@ TEST(WorkQueue, BoundedSizeMPMC) { WorkQueue<int> queue(10); std::vector<int> results(200, -1); std::mutex mutex; + std::cerr << "Creating popperThreads" << std::endl; std::vector<std::thread> popperThreads; for (int i = 0; i < 4; ++i) { popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); } + std::cerr << "Creating pusherThreads" << std::endl; std::vector<std::thread> pusherThreads; for (int i = 0; i < 2; ++i) { auto min = i * 100; @@ -218,15 +221,19 @@ TEST(WorkQueue, BoundedSizeMPMC) { }); } + std::cerr << "Joining pusherThreads" << std::endl; for (auto& thread : pusherThreads) { thread.join(); } + std::cerr << "Finishing queue" << std::endl; queue.finish(); + std::cerr << "Joining popperThreads" << std::endl; for (auto& thread : popperThreads) { thread.join(); } + std::cerr << "Inspecting results" << std::endl; for (int i = 0; i < 200; ++i) { EXPECT_EQ(i, results[i]); } diff --git a/contrib/zstd/doc/educational_decoder/zstd_decompress.c b/contrib/zstd/doc/educational_decoder/zstd_decompress.c index ae4eaa81c6aec..7c8d8114d4016 100644 --- a/contrib/zstd/doc/educational_decoder/zstd_decompress.c +++ b/contrib/zstd/doc/educational_decoder/zstd_decompress.c @@ -27,16 +27,19 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, /// Get the decompressed size of an input stream so memory can be allocated in /// advance +/// Returns -1 if the size can't be determined size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); /******* UTILITY MACROS AND TYPES *********************************************/ -// Max block size decompressed size is 128 KB and literal blocks must be smaller -// than that +// Max block size decompressed size is 128 KB and literal blocks can't be +// larger than their block #define MAX_LITERALS_SIZE ((size_t)128 * 1024) #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b)) +/// This decoder calls exit(1) when it encounters an error, however a production +/// library should propagate error codes #define ERROR(s) \ do { \ fprintf(stderr, "Error: %s\n", s); \ @@ -67,29 +70,31 @@ typedef int64_t i64; /// decompression functions. /*** IO STREAM OPERATIONS *************/ -/// These structs are the interface for IO, and do bounds checking on all -/// operations. They should be used opaquely to ensure safety. -/// Output is always done byte-by-byte +/// ostream_t/istream_t are used to wrap the pointers/length data passed into +/// ZSTD_decompress, so that all IO operations are safely bounds checked +/// They are written/read forward, and reads are treated as little-endian +/// They should be used opaquely to ensure safety typedef struct { u8 *ptr; size_t len; } ostream_t; -/// Input often reads a few bits at a time, so maintain an internal offset typedef struct { const u8 *ptr; - int bit_offset; size_t len; + + // Input often reads a few bits at a time, so maintain an internal offset + int bit_offset; } istream_t; /// The following two functions are the only ones that allow the istream to be /// non-byte aligned /// Reads `num` bits from a bitstream, and updates the internal offset -static inline u64 IO_read_bits(istream_t *const in, const int num); -/// Rewinds the stream by `num` bits -static inline void IO_rewind_bits(istream_t *const in, const int num); +static inline u64 IO_read_bits(istream_t *const in, const int num_bits); +/// Backs-up the stream by `num` bits so they can be read again +static inline void IO_rewind_bits(istream_t *const in, const int num_bits); /// If the remaining bits in a byte will be unused, advance to the end of the /// byte static inline void IO_align_stream(istream_t *const in); @@ -101,30 +106,31 @@ static inline void IO_write_byte(ostream_t *const out, u8 symb); /// be byte aligned. static inline size_t IO_istream_len(const istream_t *const in); -/// Returns a pointer where `len` bytes can be read, and advances the internal -/// state. The stream must be byte aligned. +/// Advances the stream by `len` bytes, and returns a pointer to the chunk that +/// was skipped. The stream must be byte aligned. static inline const u8 *IO_read_bytes(istream_t *const in, size_t len); -/// Returns a pointer where `len` bytes can be written, and advances the internal -/// state. The stream must be byte aligned. +/// Advances the stream by `len` bytes, and returns a pointer to the chunk that +/// was skipped so it can be written to. static inline u8 *IO_write_bytes(ostream_t *const out, size_t len); /// Advance the inner state by `len` bytes. The stream must be byte aligned. static inline void IO_advance_input(istream_t *const in, size_t len); -/// Returns an `ostream_t` constructed from the given pointer and length +/// Returns an `ostream_t` constructed from the given pointer and length. static inline ostream_t IO_make_ostream(u8 *out, size_t len); -/// Returns an `istream_t` constructed from the given pointer and length +/// Returns an `istream_t` constructed from the given pointer and length. static inline istream_t IO_make_istream(const u8 *in, size_t len); -/// Returns an `istream_t` with the same base as `in`, and length `len` -/// Then, advance `in` to account for the consumed bytes -/// `in` must be byte aligned +/// Returns an `istream_t` with the same base as `in`, and length `len`. +/// Then, advance `in` to account for the consumed bytes. +/// `in` must be byte aligned. static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len); /*** END IO STREAM OPERATIONS *********/ /*** BITSTREAM OPERATIONS *************/ -/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits -static inline u64 read_bits_LE(const u8 *src, const int num, +/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits, +/// and return them interpreted as a little-endian unsigned integer. +static inline u64 read_bits_LE(const u8 *src, const int num_bits, const size_t offset); /// Read bits from the end of a HUF or FSE bitstream. `offset` is in bits, so @@ -136,9 +142,8 @@ static inline u64 STREAM_read_bits(const u8 *src, const int bits, /*** END BITSTREAM OPERATIONS *********/ /*** BIT COUNTING OPERATIONS **********/ -/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to -/// `num`, or `-1` if `num == 0`. -static inline int log2inf(const u64 num); +/// Returns the index of the highest set bit in `num`, or `-1` if `num == 0` +static inline int highest_set_bit(const u64 num); /*** END BIT COUNTING OPERATIONS ******/ /*** HUFFMAN PRIMITIVES ***************/ @@ -384,8 +389,8 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len); } - istream_t in = {(const u8 *)src, 0, src_len}; - ostream_t out = {(u8 *)dst, dst_len}; + istream_t in = IO_make_istream(src, src_len); + ostream_t out = IO_make_ostream(dst, dst_len); // "A content compressed by Zstandard is transformed into a Zstandard frame. // Multiple frames can be appended into a single file or stream. A frame is @@ -633,6 +638,7 @@ static void frame_context_apply_dict(frame_context_t *const ctx, FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable); FSE_copy_dtable(&ctx->ml_dtable, &dict->ml_dtable); + // Copy the repeated offsets memcpy(ctx->previous_offsets, dict->previous_offsets, sizeof(ctx->previous_offsets)); } @@ -668,7 +674,7 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, // number of bytes to read and copy." const u8 *const read_ptr = IO_read_bytes(in, block_len); u8 *const write_ptr = IO_write_bytes(out, block_len); - // + // Copy the raw data into the output memcpy(write_ptr, read_ptr, block_len); @@ -682,7 +688,7 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, const u8 *const read_ptr = IO_read_bytes(in, 1); u8 *const write_ptr = IO_write_bytes(out, block_len); - // Copy `block_len` copies of `streams->src[0]` to the output + // Copy `block_len` copies of `read_ptr[0]` to the output memset(write_ptr, read_ptr[0], block_len); ctx->current_total_output += block_len; @@ -751,7 +757,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx, u8 **const literals, const int block_type, const int size_format); -static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable); +static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in); static void fse_decode_hufweights(ostream_t *weights, istream_t *const in, int *const num_symbs); @@ -894,12 +900,12 @@ static size_t decode_literals_compressed(frame_context_t *const ctx, istream_t huf_stream = IO_make_sub_istream(in, compressed_size); if (block_type == 2) { - // Decode provided Huffman table + // Decode the provided Huffman table // "This section is only present when Literals_Block_Type type is // Compressed_Literals_Block (2)." HUF_free_dtable(&ctx->literals_dtable); - decode_huf_table(&huf_stream, &ctx->literals_dtable); + decode_huf_table(&ctx->literals_dtable, &huf_stream); } else { // If the previous Huffman table is being repeated, ensure it exists if (!ctx->literals_dtable.symbols) { @@ -922,13 +928,13 @@ static size_t decode_literals_compressed(frame_context_t *const ctx, } // Decode the Huffman table description -static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) { - const u8 header = IO_read_bits(in, 8); - +static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in) { // "All literal values from zero (included) to last present one (excluded) // are represented by Weight with values from 0 to Max_Number_of_Bits." // "This is a single byte value (0-255), which describes how to decode the list of weights." + const u8 header = IO_read_bits(in, 8); + u8 weights[HUF_MAX_SYMBS]; memset(weights, 0, sizeof(weights)); @@ -997,7 +1003,7 @@ typedef struct { u16 ll_state; u16 of_state; u16 ml_state; -} sequence_state_t; +} sequence_states_t; /// Different modes to signal to decode_seq_tables what to do typedef enum { @@ -1052,10 +1058,10 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *const in, sequence_command_t *const sequences, const size_t num_sequences); -static sequence_command_t decode_sequence(sequence_state_t *const state, +static sequence_command_t decode_sequence(sequence_states_t *const state, const u8 *const src, i64 *const offset); -static void decode_seq_table(istream_t *const in, FSE_dtable *const table, +static void decode_seq_table(FSE_dtable *const table, istream_t *const in, const seq_part_t type, const seq_mode_t mode); static size_t decode_sequences(frame_context_t *const ctx, istream_t *in, @@ -1131,34 +1137,33 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in, // Offsets // Match Lengths" // Update the tables we have stored in the context - decode_seq_table(in, &ctx->ll_dtable, seq_literal_length, + decode_seq_table(&ctx->ll_dtable, in, seq_literal_length, (compression_modes >> 6) & 3); - decode_seq_table(in, &ctx->of_dtable, seq_offset, + decode_seq_table(&ctx->of_dtable, in, seq_offset, (compression_modes >> 4) & 3); - decode_seq_table(in, &ctx->ml_dtable, seq_match_length, + decode_seq_table(&ctx->ml_dtable, in, seq_match_length, (compression_modes >> 2) & 3); - // Check to make sure none of the tables are uninitialized - if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols || - !ctx->ml_dtable.symbols) { - CORRUPTION(); - } - sequence_state_t state; - // Copy the context's tables into the local state - memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable)); - memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable)); - memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable)); + sequence_states_t states; - size_t len = IO_istream_len(in); + // Initialize the decoding tables + { + states.ll_table = ctx->ll_dtable; + states.of_table = ctx->of_dtable; + states.ml_table = ctx->ml_dtable; + } + + const size_t len = IO_istream_len(in); const u8 *const src = IO_read_bytes(in, len); // "After writing the last bit containing information, the compressor writes // a single 1-bit and then fills the byte with 0-7 0 bits of padding." - const int padding = 8 - log2inf(src[len - 1]); - i64 offset = len * 8 - padding; + const int padding = 8 - highest_set_bit(src[len - 1]); + // The offset starts at the end because FSE streams are read backwards + i64 bit_offset = len * 8 - padding; // "The bitstream starts with initial state values, each using the required // number of bits in their respective accuracy, decoded previously from @@ -1166,24 +1171,22 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in, // // It starts by Literals_Length_State, followed by Offset_State, and finally // Match_Length_State." - FSE_init_state(&state.ll_table, &state.ll_state, src, &offset); - FSE_init_state(&state.of_table, &state.of_state, src, &offset); - FSE_init_state(&state.ml_table, &state.ml_state, src, &offset); + FSE_init_state(&states.ll_table, &states.ll_state, src, &bit_offset); + FSE_init_state(&states.of_table, &states.of_state, src, &bit_offset); + FSE_init_state(&states.ml_table, &states.ml_state, src, &bit_offset); for (size_t i = 0; i < num_sequences; i++) { // Decode sequences one by one - sequences[i] = decode_sequence(&state, src, &offset); + sequences[i] = decode_sequence(&states, src, &bit_offset); } - if (offset != 0) { + if (bit_offset != 0) { CORRUPTION(); } - - // Don't free tables so they can be used in the next block } // Decode a single sequence and update the state -static sequence_command_t decode_sequence(sequence_state_t *const state, +static sequence_command_t decode_sequence(sequence_states_t *const states, const u8 *const src, i64 *const offset) { // "Each symbol is a code in its own context, which specifies Baseline and @@ -1191,9 +1194,9 @@ static sequence_command_t decode_sequence(sequence_state_t *const state, // additional bits in the same bitstream." // Decode symbols, but don't update states - const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state); - const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state); - const u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state); + const u8 of_code = FSE_peek_symbol(&states->of_table, states->of_state); + const u8 ll_code = FSE_peek_symbol(&states->ll_table, states->ll_state); + const u8 ml_code = FSE_peek_symbol(&states->ml_table, states->ml_state); // Offset doesn't need a max value as it's not decoded using a table if (ll_code > SEQ_MAX_CODES[seq_literal_length] || @@ -1221,17 +1224,18 @@ static sequence_command_t decode_sequence(sequence_state_t *const state, // then Offset_State." // If the stream is complete don't read bits to update state if (*offset != 0) { - FSE_update_state(&state->ll_table, &state->ll_state, src, offset); - FSE_update_state(&state->ml_table, &state->ml_state, src, offset); - FSE_update_state(&state->of_table, &state->of_state, src, offset); + FSE_update_state(&states->ll_table, &states->ll_state, src, offset); + FSE_update_state(&states->ml_table, &states->ml_state, src, offset); + FSE_update_state(&states->of_table, &states->of_state, src, offset); } return seq; } /// Given a sequence part and table mode, decode the FSE distribution -static void decode_seq_table(istream_t *const in, FSE_dtable *const table, - const seq_part_t type, const seq_mode_t mode) { +/// Errors if the mode is `seq_repeat` without a pre-existing table in `table` +static void decode_seq_table(FSE_dtable *const table, istream_t *const in, + const seq_part_t type, const seq_mode_t mode) { // Constant arrays indexed by seq_part_t const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST, SEQ_OFFSET_DEFAULT_DIST, @@ -1272,12 +1276,17 @@ static void decode_seq_table(istream_t *const in, FSE_dtable *const table, // "Repeat_Mode : re-use distribution table from previous compressed // block." // Nothing to do here, table will be unchanged + if (!table->symbols) { + // This mode is invalid if we don't already have a table + CORRUPTION(); + } break; default: // Impossible, as mode is from 0-3 IMPOSSIBLE(); break; } + } /******* END SEQUENCE DECODING ************************************************/ @@ -1296,6 +1305,8 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, const sequence_command_t seq = sequences[i]; { + // If the sequence asks for more literals than are left, the + // sequence must be corrupted if (seq.literal_length > IO_istream_len(&litstream)) { CORRUPTION(); } @@ -1336,7 +1347,8 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, // as per the exception listed above offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; - // If idx == 1 we don't need to modify offset_hist[2] + // If idx == 1 we don't need to modify offset_hist[2], since + // we're using the second-most recent code if (idx > 1) { offset_hist[2] = offset_hist[1]; } @@ -1344,6 +1356,8 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, offset_hist[0] = offset; } } else { + // When it's not a repeat offset: + // "if (Offset_Value > 3) offset = Offset_Value - 3;" offset = seq.offset - 3; // Shift back history @@ -1391,11 +1405,11 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, total_output += seq.match_length; } + // Copy any leftover literals { size_t len = IO_istream_len(&litstream); u8 *const write_ptr = IO_write_bytes(out, len); const u8 *const read_ptr = IO_read_bytes(&litstream, len); - // Copy any leftover literals memcpy(write_ptr, read_ptr, len); total_output += len; @@ -1517,10 +1531,10 @@ static void parse_dictionary(dictionary_t *const dict, const u8 *src, // recent offsets (instead of using {1,4,8}), stored in order, 4-bytes // little-endian each, for a total of 12 bytes. Each recent offset must have // a value < dictionary size." - decode_huf_table(&in, &dict->literals_dtable); - decode_seq_table(&in, &dict->of_dtable, seq_offset, seq_fse); - decode_seq_table(&in, &dict->ml_dtable, seq_match_length, seq_fse); - decode_seq_table(&in, &dict->ll_dtable, seq_literal_length, seq_fse); + decode_huf_table(&dict->literals_dtable, &in); + decode_seq_table(&dict->of_dtable, &in, seq_offset, seq_fse); + decode_seq_table(&dict->ml_dtable, &in, seq_match_length, seq_fse); + decode_seq_table(&dict->ll_dtable, &in, seq_literal_length, seq_fse); // Read in the previous offset history dict->previous_offsets[0] = IO_read_bits(&in, 32); @@ -1571,20 +1585,20 @@ static void free_dictionary(dictionary_t *const dict) { /******* IO STREAM OPERATIONS *************************************************/ #define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream") /// Reads `num` bits from a bitstream, and updates the internal offset -static inline u64 IO_read_bits(istream_t *const in, const int num) { - if (num > 64 || num <= 0) { +static inline u64 IO_read_bits(istream_t *const in, const int num_bits) { + if (num_bits > 64 || num_bits <= 0) { ERROR("Attempt to read an invalid number of bits"); } - const size_t bytes = (num + in->bit_offset + 7) / 8; - const size_t full_bytes = (num + in->bit_offset) / 8; + const size_t bytes = (num_bits + in->bit_offset + 7) / 8; + const size_t full_bytes = (num_bits + in->bit_offset) / 8; if (bytes > in->len) { INP_SIZE(); } - const u64 result = read_bits_LE(in->ptr, num, in->bit_offset); + const u64 result = read_bits_LE(in->ptr, num_bits, in->bit_offset); - in->bit_offset = (num + in->bit_offset) % 8; + in->bit_offset = (num_bits + in->bit_offset) % 8; in->ptr += full_bytes; in->len -= full_bytes; @@ -1593,16 +1607,21 @@ static inline u64 IO_read_bits(istream_t *const in, const int num) { /// If a non-zero number of bits have been read from the current byte, advance /// the offset to the next byte -static inline void IO_rewind_bits(istream_t *const in, int num) { - if (num < 0) { +static inline void IO_rewind_bits(istream_t *const in, int num_bits) { + if (num_bits < 0) { ERROR("Attempting to rewind stream by a negative number of bits"); } - const int new_offset = in->bit_offset - num; - const i64 bytes = (new_offset - 7) / 8; + // move the offset back by `num_bits` bits + const int new_offset = in->bit_offset - num_bits; + // determine the number of whole bytes we have to rewind, rounding up to an + // integer number (e.g. if `new_offset == -5`, `bytes == 1`) + const i64 bytes = -(new_offset - 7) / 8; - in->ptr += bytes; - in->len -= bytes; + in->ptr -= bytes; + in->len += bytes; + // make sure the resulting `bit_offset` is positive, as mod in C does not + // convert numbers from negative to positive (e.g. -22 % 8 == -6) in->bit_offset = ((new_offset % 8) + 8) % 8; } @@ -1683,33 +1702,26 @@ static inline ostream_t IO_make_ostream(u8 *out, size_t len) { /// Returns an `istream_t` constructed from the given pointer and length static inline istream_t IO_make_istream(const u8 *in, size_t len) { - return (istream_t) { in, 0, len }; + return (istream_t) { in, len, 0 }; } /// Returns an `istream_t` with the same base as `in`, and length `len` /// Then, advance `in` to account for the consumed bytes /// `in` must be byte aligned static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len) { - if (len > in->len) { - INP_SIZE(); - } - if (in->bit_offset != 0) { - UNALIGNED(); - } - const istream_t sub = { in->ptr, in->bit_offset, len }; + // Consume `len` bytes of the parent stream + const u8 *const ptr = IO_read_bytes(in, len); - in->ptr += len; - in->len -= len; - - return sub; + // Make a substream using the pointer to those `len` bytes + return IO_make_istream(ptr, len); } /******* END IO STREAM OPERATIONS *********************************************/ /******* BITSTREAM OPERATIONS *************************************************/ /// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits -static inline u64 read_bits_LE(const u8 *src, const int num, +static inline u64 read_bits_LE(const u8 *src, const int num_bits, const size_t offset) { - if (num > 64) { + if (num_bits > 64) { ERROR("Attempt to read an invalid number of bits"); } @@ -1719,10 +1731,10 @@ static inline u64 read_bits_LE(const u8 *src, const int num, u64 res = 0; int shift = 0; - int left = num; + int left = num_bits; while (left > 0) { u64 mask = left >= 8 ? 0xff : (((u64)1 << left) - 1); - // Dead the next byte, shift it to account for the offset, and then mask + // Read the next byte, shift it to account for the offset, and then mask // out the top part if we don't need all the bits res += (((u64)*src++ >> bit_offset) & mask) << shift; shift += 8 - bit_offset; @@ -1761,7 +1773,7 @@ static inline u64 STREAM_read_bits(const u8 *const src, const int bits, /******* BIT COUNTING OPERATIONS **********************************************/ /// Returns `x`, where `2^x` is the largest power of 2 less than or equal to /// `num`, or `-1` if `num == 0`. -static inline int log2inf(const u64 num) { +static inline int highest_set_bit(const u64 num) { for (int i = 63; i >= 0; i--) { if (((u64)1 << i) <= num) { return i; @@ -1813,17 +1825,18 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, // final-bit-flag. Consequently, a last byte of 0 is not possible. And the // final-bit-flag itself is not part of the useful bitstream. Hence, the // last byte contains between 0 and 7 useful bits." - const int padding = 8 - log2inf(src[len - 1]); + const int padding = 8 - highest_set_bit(src[len - 1]); - i64 offset = len * 8 - padding; + // Offset starts at the end because HUF streams are read backwards + i64 bit_offset = len * 8 - padding; u16 state; - HUF_init_state(dtable, &state, src, &offset); + HUF_init_state(dtable, &state, src, &bit_offset); size_t symbols_written = 0; - while (offset > -dtable->max_bits) { + while (bit_offset > -dtable->max_bits) { // Iterate over the stream, decoding one symbol at a time - IO_write_byte(out, HUF_decode_symbol(dtable, &state, src, &offset)); + IO_write_byte(out, HUF_decode_symbol(dtable, &state, src, &bit_offset)); symbols_written++; } // "The process continues up to reading the required number of symbols per @@ -1836,7 +1849,7 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, // before the start of `src` // Therefore `offset`, the edge to start reading new bits at, should be // dtable->max_bits before the start of the stream - if (offset != -dtable->max_bits) { + if (bit_offset != -dtable->max_bits) { CORRUPTION(); } @@ -1960,7 +1973,7 @@ static void HUF_init_dtable_usingweights(HUF_dtable *const table, } // Find the first power of 2 larger than the sum - const int max_bits = log2inf(weight_sum) + 1; + const int max_bits = highest_set_bit(weight_sum) + 1; const u64 left_over = ((u64)1 << max_bits) - weight_sum; // If the left over isn't a power of 2, the weights are invalid if (left_over & (left_over - 1)) { @@ -1969,7 +1982,7 @@ static void HUF_init_dtable_usingweights(HUF_dtable *const table, // left_over is used to find the last weight as it's not transmitted // by inverting 2^(weight - 1) we can determine the value of last_weight - const int last_weight = log2inf(left_over) + 1; + const int last_weight = highest_set_bit(left_over) + 1; for (int i = 0; i < num_symbs; i++) { // "Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0" @@ -2063,7 +2076,7 @@ static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, // final-bit-flag. Consequently, a last byte of 0 is not possible. And the // final-bit-flag itself is not part of the useful bitstream. Hence, the // last byte contains between 0 and 7 useful bits." - const int padding = 8 - log2inf(src[len - 1]); + const int padding = 8 - highest_set_bit(src[len - 1]); i64 offset = len * 8 - padding; u16 state1, state2; @@ -2184,7 +2197,7 @@ static void FSE_init_dtable(FSE_dtable *const dtable, u16 next_state_desc = state_desc[symbol]++; // Fills in the table appropriately, next_state_desc increases by symbol // over time, decreasing number of bits - dtable->num_bits[i] = (u8)(accuracy_log - log2inf(next_state_desc)); + dtable->num_bits[i] = (u8)(accuracy_log - highest_set_bit(next_state_desc)); // Baseline increases until the bit threshold is passed, at which point // it resets to 0 dtable->new_state_base[i] = @@ -2235,7 +2248,7 @@ static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in, int symb = 0; while (remaining > 0 && symb < FSE_MAX_SYMBS) { // Log of the number of possible values we could read - int bits = log2inf(remaining + 1) + 1; + int bits = highest_set_bit(remaining + 1) + 1; u16 val = IO_read_bits(in, bits); diff --git a/contrib/zstd/doc/images/Cspeed4.png b/contrib/zstd/doc/images/Cspeed4.png Binary files differindex f0ca0ffba9c44..318204c00e963 100644 --- a/contrib/zstd/doc/images/Cspeed4.png +++ b/contrib/zstd/doc/images/Cspeed4.png diff --git a/contrib/zstd/doc/images/Dspeed4.png b/contrib/zstd/doc/images/Dspeed4.png Binary files differindex eba485d0d1ed7..b7baef1ff3f7a 100644 --- a/contrib/zstd/doc/images/Dspeed4.png +++ b/contrib/zstd/doc/images/Dspeed4.png diff --git a/contrib/zstd/doc/images/dict-cr.png b/contrib/zstd/doc/images/dict-cr.png Binary files differindex f555a46c7b99b..f3a9ce2bdda64 100644 --- a/contrib/zstd/doc/images/dict-cr.png +++ b/contrib/zstd/doc/images/dict-cr.png diff --git a/contrib/zstd/doc/images/dict-cs.png b/contrib/zstd/doc/images/dict-cs.png Binary files differindex ccc02b0d17134..55e5ef518fed8 100644 --- a/contrib/zstd/doc/images/dict-cs.png +++ b/contrib/zstd/doc/images/dict-cs.png diff --git a/contrib/zstd/doc/images/dict-ds.png b/contrib/zstd/doc/images/dict-ds.png Binary files differindex 858cad685509b..1153f1b95fd59 100644 --- a/contrib/zstd/doc/images/dict-ds.png +++ b/contrib/zstd/doc/images/dict-ds.png diff --git a/contrib/zstd/doc/zstd_compression_format.md b/contrib/zstd/doc/zstd_compression_format.md index d4b46548a3d2b..1f212fea23057 100644 --- a/contrib/zstd/doc/zstd_compression_format.md +++ b/contrib/zstd/doc/zstd_compression_format.md @@ -16,7 +16,8 @@ Distribution of this document is unlimited. ### Version -0.2.4 (17/02/17) +0.2.5 (31/03/17) + Introduction ------------ @@ -109,7 +110,7 @@ The structure of a single Zstandard frame is following: __`Magic_Number`__ -4 Bytes, little-endian format. +4 Bytes, __little-endian__ format. Value : 0xFD2FB528 __`Frame_Header`__ @@ -127,7 +128,7 @@ An optional 32-bit checksum, only present if `Content_Checksum_flag` is set. The content checksum is the result of [xxh64() hash function](http://www.xxhash.org) digesting the original (decoded) data as input, and a seed of zero. -The low 4 bytes of the checksum are stored in little endian format. +The low 4 bytes of the checksum are stored in __little-endian__ format. ### `Frame_Header` @@ -154,41 +155,42 @@ Decoding this byte is enough to tell the size of `Frame_Header`. | 2 | `Content_Checksum_flag` | | 1-0 | `Dictionary_ID_flag` | -In this table, bit 7 the is highest bit, while bit 0 the is lowest. +In this table, bit 7 is the highest bit, while bit 0 is the lowest one. __`Frame_Content_Size_flag`__ This is a 2-bits flag (`= Frame_Header_Descriptor >> 6`), -specifying if decompressed data size is provided within the header. -The `Flag_Value` can be converted into `Field_Size`, +specifying if `Frame_Content_Size` (the decompressed data size) +is provided within the header. +`Flag_Value` provides `FCS_Field_Size`, which is the number of bytes used by `Frame_Content_Size` according to the following table: -|`Flag_Value`| 0 | 1 | 2 | 3 | -| ---------- | ------ | --- | --- | --- | -|`Field_Size`| 0 or 1 | 2 | 4 | 8 | +| `Flag_Value` | 0 | 1 | 2 | 3 | +| -------------- | ------ | --- | --- | --- | +|`FCS_Field_Size`| 0 or 1 | 2 | 4 | 8 | -When `Flag_Value` is `0`, `Field_Size` depends on `Single_Segment_flag` : +When `Flag_Value` is `0`, `FCS_Field_Size` depends on `Single_Segment_flag` : if `Single_Segment_flag` is set, `Field_Size` is 1. -Otherwise, `Field_Size` is 0 (content size not provided). +Otherwise, `Field_Size` is 0 : `Frame_Content_Size` is not provided. __`Single_Segment_flag`__ If this flag is set, data must be regenerated within a single continuous memory segment. -In this case, `Frame_Content_Size` is necessarily present, -but `Window_Descriptor` byte is skipped. +In this case, `Window_Descriptor` byte is skipped, +but `Frame_Content_Size` is necessarily present. As a consequence, the decoder must allocate a memory segment of size equal or bigger than `Frame_Content_Size`. In order to preserve the decoder from unreasonable memory requirements, -a decoder can reject a compressed frame +a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. -This is just a recommendation, +This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. @@ -224,37 +226,38 @@ It also specifies the size of this field as `Field_Size`. #### `Window_Descriptor` -Provides guarantees on maximum back-reference distance -that will be used within compressed data. +Provides guarantees on minimum memory buffer required to decompress a frame. This information is important for decoders to allocate enough memory. -The `Window_Descriptor` byte is optional. It is absent when `Single_Segment_flag` is set. -In this case, the maximum back-reference distance is the content size itself, -which can be any value from 1 to 2^64-1 bytes (16 EB). +The `Window_Descriptor` byte is optional. +When `Single_Segment_flag` is set, `Window_Descriptor` is not present. +In this case, `Window_Size` is `Frame_Content_Size`, +which can be any value from 0 to 2^64-1 bytes (16 ExaBytes). | Bit numbers | 7-3 | 2-0 | | ----------- | ---------- | ---------- | | Field name | `Exponent` | `Mantissa` | -Maximum distance is given by the following formulas : +The minimum memory buffer size is called `Window_Size`. +It is described by the following formulas : ``` windowLog = 10 + Exponent; windowBase = 1 << windowLog; windowAdd = (windowBase / 8) * Mantissa; Window_Size = windowBase + windowAdd; ``` -The minimum window size is 1 KB. -The maximum size is `15*(1<<38)` bytes, which is 1.875 TB. +The minimum `Window_Size` is 1 KB. +The maximum `Window_Size` is `(1<<41) + 7*(1<<38)` bytes, which is 3.75 TB. To properly decode compressed data, a decoder will need to allocate a buffer of at least `Window_Size` bytes. In order to preserve decoder from unreasonable memory requirements, -a decoder can refuse a compressed frame +a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. For improved interoperability, -decoders are recommended to be compatible with window sizes of 8 MB, +decoders are recommended to be compatible with `Window_Size >= 8 MB`, and encoders are recommended to not request more than 8 MB. It's merely a recommendation though, decoders are free to support larger or lower limits, @@ -264,112 +267,118 @@ depending on local limitations. This is a variable size field, which contains the ID of the dictionary required to properly decode the frame. -Note that this field is optional. When it's not present, +`Dictionary_ID` field is optional. When it's not present, it's up to the decoder to make sure it uses the correct dictionary. -Format is little-endian. Field size depends on `Dictionary_ID_flag`. 1 byte can represent an ID 0-255. 2 bytes can represent an ID 0-65535. 4 bytes can represent an ID 0-4294967295. +Format is __little-endian__. It's allowed to represent a small ID (for example `13`) -with a large 4-bytes dictionary ID, losing some compacity in the process. +with a large 4-bytes dictionary ID, even if it is less efficient. _Reserved ranges :_ If the frame is going to be distributed in a private environment, any dictionary ID can be used. However, for public distribution of compressed frames using a dictionary, -the following ranges are reserved for future use and should not be used : -- low range : 1 - 32767 -- high range : >= (2^31) - +the following ranges are reserved and shall not be used : +- low range : `<= 32767` +- high range : `>= (1 << 31)` #### `Frame_Content_Size` This is the original (uncompressed) size. This information is optional. -The `Field_Size` is provided according to value of `Frame_Content_Size_flag`. -The `Field_Size` can be equal to 0 (not present), 1, 2, 4 or 8 bytes. -Format is little-endian. +`Frame_Content_Size` uses a variable number of bytes, provided by `FCS_Field_Size`. +`FCS_Field_Size` is provided by the value of `Frame_Content_Size_flag`. +`FCS_Field_Size` can be equal to 0 (not present), 1, 2, 4 or 8 bytes. -| `Field_Size` | Range | -| ------------ | ---------- | -| 1 | 0 - 255 | -| 2 | 256 - 65791| -| 4 | 0 - 2^32-1 | -| 8 | 0 - 2^64-1 | +| `FCS_Field_Size` | Range | +| ---------------- | ---------- | +| 0 | unknown | +| 1 | 0 - 255 | +| 2 | 256 - 65791| +| 4 | 0 - 2^32-1 | +| 8 | 0 - 2^64-1 | -When `Field_Size` is 1, 4 or 8 bytes, the value is read directly. -When `Field_Size` is 2, _the offset of 256 is added_. +`Frame_Content_Size` format is __little-endian__. +When `FCS_Field_Size` is 1, 4 or 8 bytes, the value is read directly. +When `FCS_Field_Size` is 2, _the offset of 256 is added_. It's allowed to represent a small size (for example `18`) using any compatible variant. + Blocks ------- -After the magic number and header of each block, -there are some number of blocks. -Each frame must have at least one block but there is no upper limit -on the number of blocks per frame. + +After `Magic_Number` and `Frame_Header`, there are some number of blocks. +Each frame must have at least one block, +but there is no upper limit on the number of blocks per frame. The structure of a block is as follows: -| `Last_Block` | `Block_Type` | `Block_Size` | `Block_Content` | -|:------------:|:------------:|:------------:|:---------------:| -| 1 bit | 2 bits | 21 bits | n bytes | +| `Block_Header` | `Block_Content` | +|:--------------:|:---------------:| +| 3 bytes | n bytes | + +`Block_Header` uses 3 bytes, written using __little-endian__ convention. +It contains 3 fields : -The block header (`Last_Block`, `Block_Type`, and `Block_Size`) uses 3-bytes. +| `Last_Block` | `Block_Type` | `Block_Size` | +|:------------:|:------------:|:------------:| +| bit 0 | bits 1-2 | bits 3-23 | __`Last_Block`__ The lowest bit signals if this block is the last one. -The frame will end after this one. +The frame will end after this last block. It may be followed by an optional `Content_Checksum` (see [Zstandard Frames](#zstandard-frames)). -__`Block_Type` and `Block_Size`__ - -The next 2 bits represent the `Block_Type`, -while the remaining 21 bits represent the `Block_Size`. -Format is __little-endian__. +__`Block_Type`__ +The next 2 bits represent the `Block_Type`. There are 4 block types : -| Value | 0 | 1 | 2 | 3 | +| Value | 0 | 1 | 2 | 3 | | ------------ | ----------- | ----------- | ------------------ | --------- | | `Block_Type` | `Raw_Block` | `RLE_Block` | `Compressed_Block` | `Reserved`| - `Raw_Block` - this is an uncompressed block. - `Block_Content` contains `Block_Size` bytes to read and copy - as decoded data. + `Block_Content` contains `Block_Size` bytes. -- `RLE_Block` - this is a single byte, repeated N times. - `Block_Content` consists of a single byte, - and `Block_Size` is the number of times this byte should be repeated. +- `RLE_Block` - this is a single byte, repeated `Block_Size` times. + `Block_Content` consists of a single byte. + On the decompression side, this byte must be repeated `Block_Size` times. - `Compressed_Block` - this is a [Zstandard compressed block](#compressed-blocks), explained later on. `Block_Size` is the length of `Block_Content`, the compressed data. - The decompressed size is unknown, + The decompressed size is not known, but its maximum possible value is guaranteed (see below) - `Reserved` - this is not a block. This value cannot be used with current version of this specification. +__`Block_Size`__ + +The upper 21 bits of `Block_Header` represent the `Block_Size`. + Block sizes must respect a few rules : -- In compressed mode, compressed size is always strictly less than decompressed size. -- Block decompressed size is always <= maximum back-reference distance. +- For `Compressed_Block`, `Block_Size` is always strictly less than decompressed size. +- Block decompressed size is always <= `Window_Size` - Block decompressed size is always <= 128 KB. -A data block is not necessarily "full" : -since an arbitrary “flush” may happen anytime, -block decompressed content can be any size (even empty), +A block can contain any number of bytes (even empty), up to `Block_Maximum_Decompressed_Size`, which is the smallest of : -- Maximum back-reference distance +- `Window_Size` - 128 KB + Compressed Blocks ----------------- -To decompress a compressed block, the compressed size must be provided from -`Block_Size` field in the block header. +To decompress a compressed block, the compressed size must be provided +from `Block_Size` field within `Block_Header`. A compressed block consists of 2 sections : - [Literals Section](#literals-section) @@ -381,36 +390,34 @@ data in [Sequence Execution](#sequence-execution) #### Prerequisites To decode a compressed block, the following elements are necessary : - Previous decoded data, up to a distance of `Window_Size`, - or all previous data when `Single_Segment_flag` is set. -- List of "recent offsets" from the previous compressed block. -- Decoding tables of the previous compressed block for each symbol type + or all previously decoded data when `Single_Segment_flag` is set. +- List of "recent offsets" from previous `Compressed_Block`. +- Decoding tables of previous `Compressed_Block` for each symbol type (literals, literals lengths, match lengths, offsets). Literals Section ---------------- -During sequence execution, symbols from the literals section -During sequence phase, literals will be entangled with match copy operations. All literals are regrouped in the first part of the block. -They can be decoded first, and then copied during sequence operations, -or they can be decoded on the flow, as needed by sequence commands. - -| `Literals_Section_Header` | [`Huffman_Tree_Description`] | Stream1 | [Stream2] | [Stream3] | [Stream4] | -| ------------------------- | ---------------------------- | ------- | --------- | --------- | --------- | +They can be decoded first, and then copied during [Sequence Execution], +or they can be decoded on the flow during [Sequence Execution]. Literals can be stored uncompressed or compressed using Huffman prefix codes. When compressed, an optional tree description can be present, followed by 1 or 4 streams. +| `Literals_Section_Header` | [`Huffman_Tree_Description`] | Stream1 | [Stream2] | [Stream3] | [Stream4] | +| ------------------------- | ---------------------------- | ------- | --------- | --------- | --------- | + #### `Literals_Section_Header` Header is in charge of describing how literals are packed. It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes, -using little-endian convention. +using __little-endian__ convention. | `Literals_Block_Type` | `Size_Format` | `Regenerated_Size` | [`Compressed_Size`] | -| --------------------- | ------------- | ------------------ | ----------------- | -| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits | +| --------------------- | ------------- | ------------------ | ------------------- | +| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits | In this representation, bits on the left are the lowest bits. @@ -418,33 +425,38 @@ __`Literals_Block_Type`__ This field uses 2 lowest bits of first byte, describing 4 different block types : -| `Literals_Block_Type` | Value | -| ----------------------------- | ----- | -| `Raw_Literals_Block` | 0 | -| `RLE_Literals_Block` | 1 | -| `Compressed_Literals_Block` | 2 | -| `Repeat_Stats_Literals_Block` | 3 | +| `Literals_Block_Type` | Value | +| --------------------------- | ----- | +| `Raw_Literals_Block` | 0 | +| `RLE_Literals_Block` | 1 | +| `Compressed_Literals_Block` | 2 | +| `Treeless_Literals_Block` | 3 | - `Raw_Literals_Block` - Literals are stored uncompressed. -- `RLE_Literals_Block` - Literals consist of a single byte value repeated N times. +- `RLE_Literals_Block` - Literals consist of a single byte value + repeated `Regenerated_Size` times. - `Compressed_Literals_Block` - This is a standard Huffman-compressed block, starting with a Huffman tree description. See details below. -- `Repeat_Stats_Literals_Block` - This is a Huffman-compressed block, +- `Treeless_Literals_Block` - This is a Huffman-compressed block, using Huffman tree _from previous Huffman-compressed literals block_. - Huffman tree description will be skipped. - Note: If this mode is used without any previous Huffman-table in the frame - (or [dictionary](#dictionary-format)), this should be treated as corruption. + `Huffman_Tree_Description` will be skipped. + Note: If this mode is triggered without any previous Huffman-table in the frame + (or [dictionary](#dictionary-format)), this should be treated as data corruption. __`Size_Format`__ `Size_Format` is divided into 2 families : -- For `Raw_Literals_Block` and `RLE_Literals_Block` it's enough to decode `Regenerated_Size`. -- For `Compressed_Block`, its required to decode both `Compressed_Size` - and `Regenerated_Size` (the decompressed size). It will also decode the number of streams. +- For `Raw_Literals_Block` and `RLE_Literals_Block`, + it's only necessary to decode `Regenerated_Size`. + There is no `Compressed_Size` field. +- For `Compressed_Block` and `Treeless_Literals_Block`, + it's required to decode both `Compressed_Size` + and `Regenerated_Size` (the decompressed size). + It's also necessary to decode the number of streams (1 or 4). -For values spanning several bytes, convention is little-endian. +For values spanning several bytes, convention is __little-endian__. __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ : @@ -463,9 +475,9 @@ __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ : Only Stream1 is present for these cases. Note : it's allowed to represent a short value (for example `13`) -using a long format, accepting the increased compressed data size. +using a long format, even if it's less efficient. -__`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ : +__`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__ : - Value 00 : _A single stream_. Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023). @@ -480,67 +492,68 @@ __`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block Both `Regenerated_Size` and `Compressed_Size` use 18 bits (0-262143). `Literals_Section_Header` has 5 bytes. -Both `Compressed_Size` and `Regenerated_Size` fields follow little-endian convention. -Note: `Compressed_Size` __includes__ the size of the Huffman Tree description if it -is present. +Both `Compressed_Size` and `Regenerated_Size` fields follow __little-endian__ convention. +Note: `Compressed_Size` __includes__ the size of the Huffman Tree description +_when_ it is present. ### Raw Literals Block -The data in Stream1 is `Regenerated_Size` bytes long, and contains the raw literals data -to be used in sequence execution. +The data in Stream1 is `Regenerated_Size` bytes long, +it contains the raw literals data to be used during [Sequence Execution]. ### RLE Literals Block Stream1 consists of a single byte which should be repeated `Regenerated_Size` times to generate the decoded literals. -### Compressed Literals Block and Repeat Stats Literals Block -Both of these modes contain Huffman encoded data +### Compressed Literals Block and Treeless Literals Block +Both of these modes contain Huffman encoded data. +`Treeless_Literals_Block` does not have a `Huffman_Tree_Description`. #### `Huffman_Tree_Description` This section is only present when `Literals_Block_Type` type is `Compressed_Literals_Block` (`2`). The format of the Huffman tree description can be found at [Huffman Tree description](#huffman-tree-description). -The size Huffman Tree description will be determined during the decoding process, -and must be used to determine where the compressed Huffman streams begin. +The size of `Huffman_Tree_Description` is determined during decoding process, +it must be used to determine where streams begin. +`Total_Streams_Size = Compressed_Size - Huffman_Tree_Description_Size`. -If repeat stats mode is used, the Huffman table used in the previous compressed block will -be used to decompress this block as well. +For `Treeless_Literals_Block`, +the Huffman table comes from previously compressed literals block. -Huffman compressed data consists either 1 or 4 Huffman-coded streams. +Huffman compressed data consists of either 1 or 4 Huffman-coded streams. If only one stream is present, it is a single bitstream occupying the entire -remaining portion of the literals block, encoded as described at +remaining portion of the literals block, encoded as described within [Huffman-Coded Streams](#huffman-coded-streams). If there are four streams, the literals section header only provides enough -information to know the regenerated and compressed sizes of all four streams combined. -The regenerated size of each stream is equal to `(totalSize+3)/4`, except for the last stream, -which may be up to 3 bytes smaller, to reach a total decompressed size match that described -in the literals header. +information to know the decompressed and compressed sizes of all four streams _combined_. +The decompressed size of each stream is equal to `(Regenerated_Size+3)/4`, +except for the last stream which may be up to 3 bytes smaller, +to reach a total decompressed size as specified in `Regenerated_Size`. -The compressed size of each stream is provided explicitly: the first 6 bytes of the compressed -data consist of three 2-byte little endian fields, describing the compressed sizes -of the first three streams. -The last streams size is computed from the total compressed size and the size of the other -three streams. +The compressed size of each stream is provided explicitly: +the first 6 bytes of the compressed data consist of three 2-byte __little-endian__ fields, +describing the compressed sizes of the first three streams. +`Stream4_Size` is computed from total `Total_Streams_Size` minus sizes of other streams. -`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`. +`Stream4_Size = Total_Streams_Size - 6 - Stream1_Size - Stream2_Size - Stream3_Size`. -Note: remember that totalCSize may be smaller than the `Compressed_Size` found in the literals -block header as `Compressed_Size` also contains the size of the Huffman Tree description if it -is present. +Note: remember that `Total_Streams_Size` can be smaller than `Compressed_Size` in header, +because `Compressed_Size` also contains `Huffman_Tree_Description_Size` when it is present. Each of these 4 bitstreams is then decoded independently as a Huffman-Coded stream, as described at [Huffman-Coded Streams](#huffman-coded-streams) + Sequences Section ----------------- A compressed block is a succession of _sequences_ . A sequence is a literal copy command, followed by a match copy command. A literal copy command specifies a length. -It is the number of bytes to be copied (or extracted) from the literal section. +It is the number of bytes to be copied (or extracted) from the Literals Section. A match copy command specifies an offset and a length. When all _sequences_ are decoded, -if there is are any literals left in the _literal section_, +if there are literals left in the _literal section_, these bytes are added at the end of the block. This is described in more detail in [Sequence Execution](#sequence-execution) @@ -557,7 +570,7 @@ followed by the bitstream. | -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- | To decode the `Sequences_Section`, it's required to know its size. -This size is deduced from `blockSize - literalSectionSize`. +This size is deduced from `Block_Size - Literals_Section_Size`. #### `Sequences_Section_Header` @@ -572,7 +585,7 @@ This is a variable size field using between 1 and 3 bytes. Let's call its first byte `byte0`. - `if (byte0 == 0)` : there are no sequences. The sequence section stops there. - Regenerated content is defined entirely by literals section. + Decompressed content is defined entirely as Literals Section content. - `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte. - `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes. - `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes. @@ -581,14 +594,14 @@ __Symbol compression modes__ This is a single byte, defining the compression mode of each symbol type. -|Bit number| 7-6 | 5-4 | 3-2 | 1-0 | +|Bit number| 7-6 | 5-4 | 3-2 | 1-0 | | -------- | ----------------------- | -------------- | -------------------- | ---------- | |Field name| `Literals_Lengths_Mode` | `Offsets_Mode` | `Match_Lengths_Mode` | `Reserved` | The last field, `Reserved`, must be all-zeroes. `Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the `Compression_Mode` of -literals lengths, offsets, and match lengths respectively. +literals lengths, offsets, and match lengths symbols respectively. They follow the same enumeration : @@ -598,17 +611,17 @@ They follow the same enumeration : - `Predefined_Mode` : A predefined FSE distribution table is used, defined in [default distributions](#default-distributions). - The table takes no space in the compressed data. + No distribution table will be present. - `RLE_Mode` : The table description consists of a single byte. - This code will be repeated for every sequence. + This code will be repeated for all sequences. - `Repeat_Mode` : The table used in the previous compressed block will be used again. No distribution table will be present. - Note: this includes RLE mode, so if repeat_mode follows rle_mode the same symbol will be repeated. + Note: this includes RLE mode, so if `Repeat_Mode` follows `RLE_Mode`, the same symbol will be repeated. If this mode is used without any previous sequence table in the frame (or [dictionary](#dictionary-format)) to repeat, this should be treated as corruption. - `FSE_Compressed_Mode` : standard FSE compression. A distribution table will be present. - The format of this distribution table is described in (FSE Table Description)[#fse-table-description]. + The format of this distribution table is described in [FSE Table Description](#fse-table-description). Note that the maximum allowed accuracy log for literals length and match length tables is 9, and the maximum accuracy log for the offsets table is 8. @@ -625,7 +638,7 @@ Literals length codes are values ranging from `0` to `35` included. They define lengths from 0 to 131071 bytes. The literals length is equal to the decoded `Baseline` plus the result of reading `Number_of_Bits` bits from the bitstream, -as a little-endian value. +as a __little-endian__ value. | `Literals_Length_Code` | 0-15 | | ---------------------- | ---------------------- | @@ -654,7 +667,7 @@ Match length codes are values ranging from `0` to `52` included. They define lengths from 3 to 131074 bytes. The match length is equal to the decoded `Baseline` plus the result of reading `Number_of_Bits` bits from the bitstream, -as a little-endian value. +as a __little-endian__ value. | `Match_Length_Code` | 0-31 | | ------------------- | ----------------------- | @@ -685,7 +698,7 @@ Recommendation is to support at least up to `22`. For information, at the time of this writing. the reference decoder supports a maximum `N` value of `28` in 64-bits mode. -An offset code is also the number of additional bits to read in little-endian fashion, +An offset code is also the number of additional bits to read in __little-endian__ fashion, and can be translated into an `Offset_Value` using the following formulas : ``` @@ -720,8 +733,8 @@ begins. FSE decoding requires a 'state' to be carried from symbol to symbol. For more explanation on FSE decoding, see the [FSE section](#fse). -For sequence decoding, a separate state must be kept track of for each of -literal lengths, offsets, and match lengths. +For sequence decoding, a separate state keeps track of each +literal lengths, offsets, and match lengths symbols. Some FSE primitives are also used. For more details on the operation of these primitives, see the [FSE section](#fse). @@ -753,8 +766,7 @@ See the [description of the codes] for how to determine these values. [description of the codes]: #the-codes-for-literals-lengths-match-lengths-and-offsets Decoding starts by reading the `Number_of_Bits` required to decode `Offset`. -It then does the same for `Match_Length`, -and then for `Literals_Length`. +It then does the same for `Match_Length`, and then for `Literals_Length`. This sequence is then used for [sequence execution](#sequence-execution). If it is not the last sequence in the block, @@ -807,6 +819,7 @@ short offsetCodes_defaultDistribution[29] = 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; ``` + Sequence Execution ------------------ Once literals and sequences have been decoded, @@ -826,7 +839,8 @@ in this case. The offset is defined as from the current position, so an offset of 6 and a match length of 3 means that 3 bytes should be copied from 6 bytes back. -Note that all offsets must be at most equal to the window size defined by the frame header. +Note that all offsets leading to previously decoded data +must be smaller than `Window_Size` defined in `Frame_Header_Descriptor`. #### Repeat offsets As seen in [Sequence Execution](#sequence-execution), @@ -842,11 +856,10 @@ so an `offset_value` of 1 means `Repeated_Offset2`, an `offset_value` of 2 means `Repeated_Offset3`, and an `offset_value` of 3 means `Repeated_Offset1 - 1_byte`. -In the first block, the offset history is populated with the following values : 1, 4 and 8 (in order). +For the first block, the starting offset history is populated with the following values : 1, 4 and 8 (in order). -Then each block gets its starting offset history from the ending values of the most recent compressed block. -Note that non-compressed blocks are skipped, -they do not contribute to offset history. +Then each block gets its starting offset history from the ending values of the most recent `Compressed_Block`. +Note that blocks which are not `Compressed_Block` are skipped, they do not contribute to offset history. [Offset Codes]: #offset-codes @@ -859,6 +872,7 @@ This means that when `Repeated_Offset1` (most recent) is used, history is unmodi When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`. If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one. + Skippable Frames ---------------- @@ -878,7 +892,7 @@ Skippable frames defined in this specification are compatible with [LZ4] ones. __`Magic_Number`__ -4 Bytes, little-endian format. +4 Bytes, __little-endian__ format. Value : 0x184D2A5?, which means any value from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a skippable frame. @@ -886,13 +900,14 @@ __`Frame_Size`__ This is the size, in bytes, of the following `User_Data` (without including the magic number nor the size field itself). -This field is represented using 4 Bytes, little-endian format, unsigned 32-bits. +This field is represented using 4 Bytes, __little-endian__ format, unsigned 32-bits. This means `User_Data` can’t be bigger than (2^32-1) bytes. __`User_Data`__ The `User_Data` can be anything. Data will just be skipped by the decoder. + Entropy Encoding ---------------- Two types of entropy encoding are used by the Zstandard format: @@ -900,7 +915,7 @@ FSE, and Huffman coding. FSE --- -FSE, or FiniteStateEntropy is an entropy coding based on [ANS]. +FSE, short for Finite State Entropy, is an entropy codec based on [ANS]. FSE encoding/decoding involves a state that is carried over between symbols, so decoding must be done in the opposite direction as encoding. Therefore, all FSE bitstreams are read from end to beginning. @@ -909,15 +924,15 @@ For additional details on FSE, see [Finite State Entropy]. [Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/ -FSE decoding involves a decoding table which has a power of 2 size and three elements: +FSE decoding involves a decoding table which has a power of 2 size, and contain three elements: `Symbol`, `Num_Bits`, and `Baseline`. The `log2` of the table size is its `Accuracy_Log`. The FSE state represents an index in this table. -The next symbol in the stream is the symbol indicated by the table value for that state. -To obtain the next state value, -the decoder should consume `Num_Bits` bits from the stream as a little endian value and add it to baseline. -To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a little endian value. +To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a __little-endian__ value. +The next symbol in the stream is the `Symbol` indicated in the table for that state. +To obtain the next state value, +the decoder should consume `Num_Bits` bits from the stream as a __little-endian__ value and add it to `Baseline`. [ANS]: https://en.wikipedia.org/wiki/Asymmetric_Numeral_Systems @@ -929,7 +944,7 @@ An FSE distribution table describes the probabilities of all symbols from `0` to the last present one (included) on a normalized scale of `1 << Accuracy_Log` . -It's a bitstream which is read forward, in little-endian fashion. +It's a bitstream which is read forward, in __little-endian__ fashion. It's not necessary to know its exact size, since it will be discovered and reported by the decoding process. @@ -1064,7 +1079,7 @@ Huffman Coding -------------- Zstandard Huffman-coded streams are read backwards, similar to the FSE bitstreams. -Therefore, to find the start of the bitstream it is therefore necessary to +Therefore, to find the start of the bitstream, it is therefore to know the offset of the last byte of the Huffman-coded stream. After writing the last bit containing information, the compressor @@ -1077,7 +1092,7 @@ byte to read. The decompressor needs to skip 0-7 initial `0`-bits and the first `1`-bit it occurs. Afterwards, the useful part of the bitstream begins. -The bitstream contains Huffman-coded symbols in little-endian order, +The bitstream contains Huffman-coded symbols in __little-endian__ order, with the codes defined by the method below. ### Huffman Tree Description @@ -1182,14 +1197,14 @@ The Huffman header compression uses 2 states, which share the same FSE distribution table. The first state (`State1`) encodes the even indexed symbols, and the second (`State2`) encodes the odd indexes. -State1 is initialized first, and then State2, and they take turns decoding -a single symbol and updating their state. +`State1` is initialized first, and then `State2`, and they take turns +decoding a single symbol and updating their state. For more details on these FSE operations, see the [FSE section](#fse). The number of symbols to decode is determined by tracking bitStream overflow condition: If updating state after decoding a symbol would require more bits than -remain in the stream, it is assumed the extra bits are 0. Then, +remain in the stream, it is assumed that extra bits are 0. Then, the symbols for each of the final states are decoded and the process is complete. ##### Conversion from weights to Huffman prefix codes @@ -1245,7 +1260,7 @@ it would be encoded as: |Encoding|`0000`|`0001`|`01`|`1`| `10000` | Starting from the end, -it's possible to read the bitstream in a little-endian fashion, +it's possible to read the bitstream in a __little-endian__ fashion, keeping track of already used bits. Since the bitstream is encoded in reverse order, by starting at the end the symbols can be read in forward order. @@ -1258,13 +1273,14 @@ If a bitstream is not entirely and exactly consumed, hence reaching exactly its beginning position with _all_ bits consumed, the decoding process is considered faulty. + Dictionary Format ----------------- -Zstandard is compatible with "raw content" dictionaries, free of any format restriction, -except that they must be at least 8 bytes. -These dictionaries function as if they were just the `Content` block of a formatted -dictionary. +Zstandard is compatible with "raw content" dictionaries, +free of any format restriction, except that they must be at least 8 bytes. +These dictionaries function as if they were just the `Content` part +of a formatted dictionary. But dictionaries created by `zstd --train` follow a format, described here. @@ -1274,9 +1290,9 @@ __Pre-requisites__ : a dictionary has a size, | `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` | | -------------- | --------------- | ---------------- | --------- | -__`Magic_Number`__ : 4 bytes ID, value 0xEC30A437, little-endian format +__`Magic_Number`__ : 4 bytes ID, value 0xEC30A437, __little-endian__ format -__`Dictionary_ID`__ : 4 bytes, stored in little-endian format. +__`Dictionary_ID`__ : 4 bytes, stored in __little-endian__ format. `Dictionary_ID` can be any value, except 0 (which means no `Dictionary_ID`). It's used by decoders to check if they use the correct dictionary. @@ -1284,9 +1300,9 @@ _Reserved ranges :_ If the frame is going to be distributed in a private environment, any `Dictionary_ID` can be used. However, for public distribution of compressed frames, - the following ranges are reserved for future use and should not be used : + the following ranges are reserved and shall not be used : - - low range : 1 - 32767 + - low range : <= 32767 - high range : >= (2^31) __`Entropy_Tables`__ : following the same format as the tables in compressed blocks. @@ -1298,26 +1314,30 @@ __`Entropy_Tables`__ : following the same format as the tables in compressed blo These tables populate the Repeat Stats literals mode and Repeat distribution mode for sequence decoding. It's finally followed by 3 offset values, populating recent offsets (instead of using `{1,4,8}`), - stored in order, 4-bytes little-endian each, for a total of 12 bytes. + stored in order, 4-bytes __little-endian__ each, for a total of 12 bytes. Each recent offset must have a value < dictionary size. __`Content`__ : The rest of the dictionary is its content. The content act as a "past" in front of data to compress or decompress, so it can be referenced in sequence commands. As long as the amount of data decoded from this frame is less than or - equal to the window-size, sequence commands may specify offsets longer - than the lenght of total decoded output so far to reference back to the - dictionary. After the total output has surpassed the window size however, + equal to `Window_Size`, sequence commands may specify offsets longer + than the total length of decoded output so far to reference back to the + dictionary. After the total output has surpassed `Window_Size` however, this is no longer allowed and the dictionary is no longer accessible. [compressed blocks]: #the-format-of-compressed_block + + Appendix A - Decoding tables for predefined codes ------------------------------------------------- -This appendix contains FSE decoding tables for the predefined literal length, match length, and offset -codes. The tables have been constructed using the algorithm as given above in the -"from normalized distribution to decoding tables" chapter. The tables here can be used as examples -to crosscheck that an implementation implements the decoding table generation algorithm correctly. +This appendix contains FSE decoding tables +for the predefined literal length, match length, and offset codes. +The tables have been constructed using the algorithm as given above in chapter +"from normalized distribution to decoding tables". +The tables here can be used as examples +to crosscheck that an implementation build its decoding tables correctly. #### Literal Length Code: @@ -1496,6 +1516,7 @@ to crosscheck that an implementation implements the decoding table generation al Version changes --------------- +- 0.2.5 : minor typos and clarifications - 0.2.4 : section restructuring, by Sean Purcell - 0.2.3 : clarified several details, by Sean Purcell - 0.2.2 : added predefined codes, by Johannes Rudolph diff --git a/contrib/zstd/doc/zstd_manual.html b/contrib/zstd/doc/zstd_manual.html index 204f56ea5f21e..2e77e7742f60e 100644 --- a/contrib/zstd/doc/zstd_manual.html +++ b/contrib/zstd/doc/zstd_manual.html @@ -1,10 +1,10 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>zstd 1.1.4 Manual</title> +<title>zstd 1.2.0 Manual</title> </head> <body> -<h1>zstd 1.1.4 Manual</h1> +<h1>zstd 1.2.0 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> @@ -57,46 +57,46 @@ <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -</b><p> Compresses `src` content as a single zstd compressed frame into already allocated `dst`. - Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - @return : compressed size written into `dst` (<= `dstCapacity), - or an error code if it fails (which can be tested using ZSTD_isError()). +</b><p> Compresses `src` content as a single zstd compressed frame into already allocated `dst`. + Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + @return : compressed size written into `dst` (<= `dstCapacity), + or an error code if it fails (which can be tested using ZSTD_isError()). </p></pre><BR> <pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); -</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - `dstCapacity` is an upper bound of originalSize. - If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTD_isError()). +</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + `dstCapacity` is an upper bound of originalSize. + If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + or an errorCode if it fails (which can be tested using ZSTD_isError()). </p></pre><BR> <pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); -</b><p> NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. - ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single - frame, but distinguishes empty frames from frames with an unknown size, or errors. +</b><p> NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. + ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single + frame, but distinguishes empty frames from frames with an unknown size, or errors. - Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple - concatenated frames in one buffer, and so is more general. - As a result however, it requires more computation and entire frames to be passed to it, - as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. + Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple + concatenated frames in one buffer, and so is more general. + As a result however, it requires more computation and entire frames to be passed to it, + as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. - 'src' is the start of a zstd compressed frame. - @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. - note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - When `return==0`, data to decompress could be any size. - In which case, it's necessary to use streaming mode to decompress data. - Optionally, application can still use ZSTD_decompress() while relying on implied limits. - (For example, data may be necessarily cut into blocks <= 16 KB). - note 2 : decompressed size is always present when compression is done with ZSTD_compress() - note 3 : decompressed size can be very large (64-bits value), - potentially larger than what local system can handle as a single memory segment. - In which case, it's necessary to use streaming mode to decompress data. - note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - Always ensure result fits within application's authorized limits. - Each application can set its own limits. - note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. + 'src' is the start of a zstd compressed frame. + @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. + note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + When `return==0`, data to decompress could be any size. + In which case, it's necessary to use streaming mode to decompress data. + Optionally, application can still use ZSTD_decompress() while relying on implied limits. + (For example, data may be necessarily cut into blocks <= 16 KB). + note 2 : decompressed size is always present when compression is done with ZSTD_compress() + note 3 : decompressed size can be very large (64-bits value), + potentially larger than what local system can handle as a single memory segment. + In which case, it's necessary to use streaming mode to decompress data. + note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + Always ensure result fits within application's authorized limits. + Each application can set its own limits. + note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. </p></pre><BR> <h3>Helper functions</h3><pre></pre><b><pre>int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b> @@ -106,28 +106,28 @@ const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable strin </pre></b><BR> <a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre> -<h3>Compression context</h3><pre> When compressing many times, - it is recommended to allocate a context just once, and re-use it for each successive compression operation. - This will make workload friendlier for system's memory. - Use one context per thread for parallel execution in multi-threaded environments. +<h3>Compression context</h3><pre> When compressing many times, + it is recommended to allocate a context just once, and re-use it for each successive compression operation. + This will make workload friendlier for system's memory. + Use one context per thread for parallel execution in multi-threaded environments. </pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTD_CCtx* ZSTD_createCCtx(void); size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); </pre></b><BR> <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). +</b><p> Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). </p></pre><BR> -<h3>Decompression context</h3><pre> When decompressing many times, - it is recommended to allocate a context just once, and re-use it for each successive compression operation. - This will make workload friendlier for system's memory. - Use one context per thread for parallel execution in multi-threaded environments. +<h3>Decompression context</h3><pre> When decompressing many times, + it is recommended to allocate a context just once, and re-use it for each successive compression operation. + This will make workload friendlier for system's memory. + Use one context per thread for parallel execution in multi-threaded environments. </pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTD_DCtx* ZSTD_createDCtx(void); size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </pre></b><BR> <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). +</b><p> Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). </p></pre><BR> <a name="Chapter5"></a><h2>Simple dictionary API</h2><pre></pre> @@ -169,9 +169,10 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict); -</b><p> Compression using a digested Dictionary. - Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. - Note that compression level is decided during dictionary creation. +</b><p> Compression using a digested Dictionary. + Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + Note that compression level is decided during dictionary creation. + Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) </p></pre><BR> <pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); @@ -399,7 +400,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v </p></pre><BR> <pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference, - ZSTD_parameters params, ZSTD_customMem customMem); + ZSTD_compressionParameters cParams, ZSTD_customMem customMem); </b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters </p></pre><BR> @@ -426,12 +427,19 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v both values are optional, select `0` if unknown. </p></pre><BR> -<pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); -</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter +<pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); +</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter +</p></pre><BR> + +<pre><b>size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams); +</b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters </p></pre><BR> <a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre> @@ -491,20 +499,29 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v Note : this use case also happens when using a non-conformant dictionary. - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - This is not a Zstandard frame. - When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. + When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. </p></pre><BR> <a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre> <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); </b>/**< size of CStream is variable, depending primarily on compression level */<b> size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b> size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b> size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b> size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); </b>/**< note : cdict will just be referenced, and must outlive compression session */<b> -size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); </b>/**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b> -size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); </b>/**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */<b> </pre></b><BR> +<pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); +</b><p> start a new compression job, using same parameters from previous job. + This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. + Note that zcs must be init at least once before using ZSTD_resetCStream(). + pledgedSrcSize==0 means "srcSize unknown". + If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + @return : 0, or an error code (which can be tested using ZSTD_isError()) +</p></pre><BR> + <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8 */<b> @@ -552,10 +569,9 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); <h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */<b> +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b> +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b> -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */<b> -size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); </pre></b><BR> <a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> A ZSTD_DCtx object is required to track streaming operations. @@ -640,19 +656,20 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - Compressing and decompressing require a context structure + Use ZSTD_createCCtx() and ZSTD_createDCtx() - It is necessary to init context before starting - + compression : ZSTD_compressBegin() - + decompression : ZSTD_decompressBegin() - + variants _usingDict() are also allowed - + copyCCtx() and copyDCtx() work too - - Block size is limited, it must be <= ZSTD_getBlockSizeMax() - + If you need to compress more, cut data into multiple blocks - + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + + copyCCtx() and copyDCtx() can be used too + - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. In which case, nothing is produced into `dst`. + User must test for such outcome and deal directly with uncompressed data + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! - + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. - Use ZSTD_insertBlock() in such a case. + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. <BR></pre> <h3>Raw zstd block functions</h3><pre></pre><b><pre>size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); diff --git a/contrib/zstd/examples/simple_compression.c b/contrib/zstd/examples/simple_compression.c index 9d448712ec34f..ab11314755759 100644 --- a/contrib/zstd/examples/simple_compression.c +++ b/contrib/zstd/examples/simple_compression.c @@ -116,7 +116,6 @@ static char* createOutFilename_orDie(const char* filename) int main(int argc, const char** argv) { const char* const exeName = argv[0]; - const char* const inFilename = argv[1]; if (argc!=2) { printf("wrong arguments\n"); @@ -125,6 +124,8 @@ int main(int argc, const char** argv) return 1; } + const char* const inFilename = argv[1]; + char* const outFilename = createOutFilename_orDie(inFilename); compress_orDie(inFilename, outFilename); free(outFilename); diff --git a/contrib/zstd/examples/streaming_compression.c b/contrib/zstd/examples/streaming_compression.c index 4c2c1a1d8bc72..24ad15bd614ce 100644 --- a/contrib/zstd/examples/streaming_compression.c +++ b/contrib/zstd/examples/streaming_compression.c @@ -112,7 +112,6 @@ static const char* createOutFilename_orDie(const char* filename) int main(int argc, const char** argv) { const char* const exeName = argv[0]; - const char* const inFilename = argv[1]; if (argc!=2) { printf("wrong arguments\n"); @@ -121,6 +120,8 @@ int main(int argc, const char** argv) return 1; } + const char* const inFilename = argv[1]; + const char* const outFilename = createOutFilename_orDie(inFilename); compressFile_orDie(inFilename, outFilename, 1); diff --git a/contrib/zstd/examples/streaming_decompression.c b/contrib/zstd/examples/streaming_decompression.c index 400aa673d64be..bb2d809870810 100644 --- a/contrib/zstd/examples/streaming_decompression.c +++ b/contrib/zstd/examples/streaming_decompression.c @@ -99,7 +99,6 @@ static void decompressFile_orDie(const char* fname) int main(int argc, const char** argv) { const char* const exeName = argv[0]; - const char* const inFilename = argv[1]; if (argc!=2) { fprintf(stderr, "wrong arguments\n"); @@ -108,6 +107,8 @@ int main(int argc, const char** argv) return 1; } + const char* const inFilename = argv[1]; + decompressFile_orDie(inFilename); return 0; } diff --git a/contrib/zstd/lib/Makefile b/contrib/zstd/lib/Makefile index 197fdeeea033c..d8d8e179d2058 100644 --- a/contrib/zstd/lib/Makefile +++ b/contrib/zstd/lib/Makefile @@ -71,6 +71,9 @@ libzstd.a: $(ZSTD_OBJ) @echo compiling static library @$(AR) $(ARFLAGS) $@ $^ +libzstd.a-mt: CPPFLAGS += -DZSTD_MULTHREAD +libzstd.a-mt: libzstd.a + $(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden $(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) @@ -86,10 +89,17 @@ endif libzstd : $(LIBZSTD) +libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD +libzstd-mt : libzstd + lib: libzstd.a libzstd -lib-release: DEBUGFLAGS := +lib-mt: CPPFLAGS += -DZSTD_MULTITHREAD +lib-mt: lib + +lib-release lib-release-mt: DEBUGFLAGS := lib-release: lib +lib-release-mt: lib-mt clean: @$(RM) -r *.dSYM # Mac OS-X specific diff --git a/contrib/zstd/lib/README.md b/contrib/zstd/lib/README.md index 3357e3d870967..79b6fd50014d4 100644 --- a/contrib/zstd/lib/README.md +++ b/contrib/zstd/lib/README.md @@ -22,6 +22,14 @@ Some additional API may be useful if you're looking into advanced features : They are not "stable", their definition may change in the future. Only static linking is allowed. +#### ZSTDMT API + +To enable multithreaded compression within the library, invoke `make lib-mt` target. +Prototypes are defined in header file `compress/zstdmt_compress.h`. +When linking a program that uses ZSTDMT API against libzstd.a on a POSIX system, +`-pthread` flag must be provided to the compiler and linker. +Note : ZSTDMT prototypes can still be used with a library built without multithread support, +but in this case, they will be single threaded only. #### Modular build diff --git a/contrib/zstd/lib/common/bitstream.h b/contrib/zstd/lib/common/bitstream.h index d3873002ebd1a..ca42850df3244 100644 --- a/contrib/zstd/lib/common/bitstream.h +++ b/contrib/zstd/lib/common/bitstream.h @@ -2,7 +2,7 @@ bitstream Part of FSE library header file (to include) - Copyright (C) 2013-2016, Yann Collet. + Copyright (C) 2013-2017, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -53,6 +53,16 @@ extern "C" { #include "error_private.h" /* error codes and messages */ +/*-************************************* +* Debug +***************************************/ +#if defined(BIT_DEBUG) && (BIT_DEBUG>=1) +# include <assert.h> +#else +# define assert(condition) ((void)0) +#endif + + /*========================================= * Target specific =========================================*/ @@ -74,7 +84,7 @@ extern "C" { typedef struct { size_t bitContainer; - int bitPos; + unsigned bitPos; char* startPtr; char* ptr; char* endPtr; @@ -112,6 +122,7 @@ typedef struct unsigned bitsConsumed; const char* ptr; const char* start; + const char* limitPtr; } BIT_DStream_t; typedef enum { BIT_DStream_unfinished = 0, @@ -163,7 +174,10 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val) # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ return 31 - __builtin_clz (val); # else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; v |= v >> 1; v |= v >> 2; @@ -175,31 +189,36 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val) } /*===== Local Constants =====*/ -static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ +static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, + 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, + 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ /*-************************************************************** * bitStream encoding ****************************************************************/ /*! BIT_initCStream() : - * `dstCapacity` must be > sizeof(void*) + * `dstCapacity` must be > sizeof(size_t) * @return : 0 if success, otherwise an error code (can be tested using ERR_isError() ) */ -MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) { bitC->bitContainer = 0; bitC->bitPos = 0; bitC->startPtr = (char*)startPtr; bitC->ptr = bitC->startPtr; - bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); - if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); return 0; } /*! BIT_addBits() : can add up to 26 bits into `bitC`. Does not check for register overflow ! */ -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) { bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; bitC->bitPos += nbBits; @@ -207,34 +226,42 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) /*! BIT_addBitsFast() : * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) { + assert((value>>nbBits) == 0); bitC->bitContainer |= value << bitC->bitPos; bitC->bitPos += nbBits; } /*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed * unsafe version; does not check buffer overflow */ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; + assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) ); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; + assert(bitC->ptr <= bitC->endPtr); bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ + bitC->bitContainer >>= nbBytes*8; } /*! BIT_flushBits() : + * assumption : bitContainer has not overflowed * safe version; check for buffer overflow, and prevents it. - * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; + assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) ); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ + bitC->bitContainer >>= nbBytes*8; } /*! BIT_closeCStream() : @@ -244,9 +271,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) { BIT_addBitsFast(bitC, 1, 1); /* endMark */ BIT_flushBits(bitC); - - if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ - + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); } @@ -264,15 +289,16 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si { if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char*)srcBuffer; bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { - bitD->start = (const char*)srcBuffer; bitD->ptr = bitD->start; bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) @@ -330,17 +356,18 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) #if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); #else - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); #endif } /*! BIT_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ + * unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) { - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); } MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) @@ -365,6 +392,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) { size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); BIT_skipBits(bitD, nbBits); return value; } @@ -376,10 +404,10 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ - return BIT_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { + if (bitD->ptr >= bitD->limitPtr) { bitD->ptr -= bitD->bitsConsumed >> 3; bitD->bitsConsumed &= 7; bitD->bitContainer = MEM_readLEST(bitD->ptr); @@ -389,6 +417,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; return BIT_DStream_completed; } + /* start < ptr < limitPtr */ { U32 nbBytes = bitD->bitsConsumed >> 3; BIT_DStream_status result = BIT_DStream_unfinished; if (bitD->ptr - nbBytes < bitD->start) { @@ -397,7 +426,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) } bitD->ptr -= nbBytes; bitD->bitsConsumed -= nbBytes*8; - bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ return result; } } diff --git a/contrib/zstd/lib/common/error_private.c b/contrib/zstd/lib/common/error_private.c index a0fa1724aee8b..b3287245f1ee0 100644 --- a/contrib/zstd/lib/common/error_private.c +++ b/contrib/zstd/lib/common/error_private.c @@ -29,7 +29,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(memory_allocation): return "Allocation error : not enough memory"; case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(corruption_detected): return "Corrupted block detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; @@ -37,6 +37,7 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(maxCode): default: return notErrorCode; } diff --git a/contrib/zstd/lib/common/fse.h b/contrib/zstd/lib/common/fse.h index baac390326751..6d5d41def19b9 100644 --- a/contrib/zstd/lib/common/fse.h +++ b/contrib/zstd/lib/common/fse.h @@ -316,6 +316,10 @@ If there is an error, the function will return an error code, which can be teste #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + /* ***************************************** * FSE advanced API @@ -353,7 +357,7 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. */ -#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) ) +#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); @@ -550,9 +554,9 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3 MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) { - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); BIT_addBits(bitC, statePtr->value, nbBitsOut); statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; } diff --git a/contrib/zstd/lib/common/huf.h b/contrib/zstd/lib/common/huf.h index e5572760a5485..7873ca3d42a57 100644 --- a/contrib/zstd/lib/common/huf.h +++ b/contrib/zstd/lib/common/huf.h @@ -43,6 +43,21 @@ extern "C" { #include <stddef.h> /* size_t */ +/* *** library symbols visibility *** */ +/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, + * HUF symbols remain "private" (internal symbols for library only). + * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define HUF_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ +#else +# define HUF_PUBLIC_API +#endif + + /* *** simple functions *** */ /** HUF_compress() : @@ -55,8 +70,8 @@ HUF_compress() : if return == 1, srcData is a single repeated byte symbol (RLE compression). if HUF_isError(return), compression failed (more details using HUF_getErrorName()) */ -size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); /** HUF_decompress() : @@ -69,32 +84,42 @@ HUF_decompress() : @return : size of regenerated data (== originalSize), or an error code, which can be tested using HUF_isError() */ -size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); /* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ /* Error Management */ -unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ /* *** Advanced function *** */ /** HUF_compress2() : - * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog`. + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); /** HUF_compress4X_wksp() : -* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ -size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ + * Same as HUF_compress2(), but uses externally allocated `workSpace`. + * `workspace` must have minimum alignment of 4, and be at least as large as following macro */ +#define HUF_WORKSPACE_SIZE (6 << 10) +#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) +HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/* ****************************************************************** + * WARNING !! + * The following section contains advanced and experimental definitions + * which shall never be used in the context of dll + * because they are not guaranteed to remain stable in the future. + * Only consider them in association with static linking. + *******************************************************************/ #ifdef HUF_STATIC_LINKING_ONLY /* *** Dependencies *** */ @@ -117,12 +142,14 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s ******************************************/ /* HUF buffer bounds */ #define HUF_CTABLEBOUND 129 -#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* static allocation of HUF's Compression Table */ +#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue+1]; \ + U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \ void* name##hv = &(name##hb); \ HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ @@ -134,10 +161,6 @@ typedef U32 HUF_DTable; #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } -/* The workspace must have alignment at least 4 and be at least this large */ -#define HUF_WORKSPACE_SIZE (6 << 10) -#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) - /* **************************************** * Advanced decompression functions diff --git a/contrib/zstd/lib/common/mem.h b/contrib/zstd/lib/common/mem.h index 3cacd216aa02a..4773a8b9309ef 100644 --- a/contrib/zstd/lib/common/mem.h +++ b/contrib/zstd/lib/common/mem.h @@ -89,8 +89,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# elif defined(__INTEL_COMPILER) || defined(__GNUC__) # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -122,7 +121,7 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) + __pragma( pack(push, 1) ) typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; __pragma( pack(pop) ) #else diff --git a/contrib/zstd/lib/common/zstd_errors.h b/contrib/zstd/lib/common/zstd_errors.h index 949dbd0fffac9..3d579d9693630 100644 --- a/contrib/zstd/lib/common/zstd_errors.h +++ b/contrib/zstd/lib/common/zstd_errors.h @@ -57,6 +57,7 @@ typedef enum { ZSTD_error_maxSymbolValue_tooSmall, ZSTD_error_dictionary_corrupted, ZSTD_error_dictionary_wrong, + ZSTD_error_dictionaryCreation_failed, ZSTD_error_maxCode } ZSTD_ErrorCode; diff --git a/contrib/zstd/lib/common/zstd_internal.h b/contrib/zstd/lib/common/zstd_internal.h index 5c5b287329759..2533333ba83cc 100644 --- a/contrib/zstd/lib/common/zstd_internal.h +++ b/contrib/zstd/lib/common/zstd_internal.h @@ -16,9 +16,9 @@ #ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline # include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4324) /* disable: C4324: padded structure */ -# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ #else # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ @@ -58,6 +58,8 @@ /*-************************************* * shared macros ***************************************/ +#undef MIN +#undef MAX #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) #define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ @@ -104,7 +106,6 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy #define LONGNBSEQ 0x7F00 #define MINMATCH 3 -#define EQUAL_READ32 4 #define Litbits 8 #define MaxLit ((1<<Litbits) - 1) diff --git a/contrib/zstd/lib/compress/fse_compress.c b/contrib/zstd/lib/compress/fse_compress.c index 6708fb9d78eaa..26e8052ddccea 100644 --- a/contrib/zstd/lib/compress/fse_compress.c +++ b/contrib/zstd/lib/compress/fse_compress.c @@ -291,7 +291,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) @@ -476,20 +476,20 @@ void FSE_freeCTable (FSE_CTable* ct) { free(ct); } /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; + U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + return minBits; } unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) { - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; return tableLog; @@ -808,7 +808,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; /* Scan input and build symbol stats */ - { CHECK_V_F(maxCount, FSE_count(count, &maxSymbolValue, src, srcSize) ); + { CHECK_V_F(maxCount, FSE_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) ); if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ diff --git a/contrib/zstd/lib/compress/zstd_compress.c b/contrib/zstd/lib/compress/zstd_compress.c index 450e5970a4a5e..c08b315dab935 100644 --- a/contrib/zstd/lib/compress/zstd_compress.c +++ b/contrib/zstd/lib/compress/zstd_compress.c @@ -21,18 +21,48 @@ /*-************************************* +* Debug +***************************************/ +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) +# include <assert.h> +#else +# define assert(condition) ((void)0) +#endif + +#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } + +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) +# include <stdio.h> + static unsigned g_debugLevel = ZSTD_DEBUG; +# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +/*-************************************* * Constants ***************************************/ static const U32 g_searchStrength = 8; /* control skip over incompressible data */ #define HASH_READ_SIZE 8 typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +/* entropy tables always have same size */ +static size_t const hufCTable_size = HUF_CTABLE_SIZE(255); +static size_t const litlengthCTable_size = FSE_CTABLE_SIZE(LLFSELog, MaxLL); +static size_t const offcodeCTable_size = FSE_CTABLE_SIZE(OffFSELog, MaxOff); +static size_t const matchlengthCTable_size = FSE_CTABLE_SIZE(MLFSELog, MaxML); +static size_t const entropyScratchSpace_size = HUF_WORKSPACE_SIZE; + /*-************************************* * Helper functions ***************************************/ -#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } -size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } +size_t ZSTD_compressBound(size_t srcSize) { + size_t const lowLimit = 256 KB; + size_t const margin = (srcSize < lowLimit) ? (lowLimit-srcSize) >> 12 : 0; /* from 64 to 0 */ + return srcSize + (srcSize >> 8) + margin; +} /*-************************************* @@ -70,6 +100,7 @@ struct ZSTD_CCtx_s { size_t workSpaceSize; size_t blockSize; U64 frameContentSize; + U64 consumedSrcSize; XXH64_state_t xxhState; ZSTD_customMem customMem; @@ -77,13 +108,13 @@ struct ZSTD_CCtx_s { U32* hashTable; U32* hashTable3; U32* chainTable; - HUF_CElt* hufTable; - U32 flagStaticTables; - HUF_repeat flagStaticHufTable; - FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; - FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; - FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; - unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32]; + HUF_repeat hufCTable_repeatMode; + HUF_CElt* hufCTable; + U32 fseCTables_ready; + FSE_CTable* offcodeCTable; + FSE_CTable* matchlengthCTable; + FSE_CTable* litlengthCTable; + unsigned* entropyScratchSpace; }; ZSTD_CCtx* ZSTD_createCCtx(void) @@ -150,9 +181,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - { U32 const searchLengthMin = ((cParams.strategy == ZSTD_fast) | (cParams.strategy == ZSTD_greedy)) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; - U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; - CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } + CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported); return 0; @@ -206,11 +235,14 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) size_t const hSize = ((size_t)1) << cParams.hashLog; U32 const hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); size_t const h3Size = ((size_t)1) << hashLog3; + size_t const entropySpace = hufCTable_size + litlengthCTable_size + + offcodeCTable_size + matchlengthCTable_size + + entropyScratchSpace_size; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + size_t const neededSpace = entropySpace + tableSpace + tokenSpace + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); return sizeof(ZSTD_CCtx) + neededSpace; @@ -232,6 +264,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra U32 const end = (U32)(cctx->nextSrc - cctx->base); cctx->params = params; cctx->frameContentSize = frameContentSize; + cctx->consumedSrcSize = 0; cctx->lowLimit = end; cctx->dictLimit = end; cctx->nextToUpdate = end+1; @@ -246,16 +279,16 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; -/*! ZSTD_resetCCtx_advanced() : +/*! ZSTD_resetCCtx_internal() : note : `params` must be validated */ -static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, +static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc, ZSTD_parameters params, U64 frameContentSize, ZSTD_compResetPolicy_e const crp) { if (crp == ZSTDcrp_continue) if (ZSTD_equivalentParams(params, zc->params)) { - zc->flagStaticTables = 0; - zc->flagStaticHufTable = HUF_repeat_none; + zc->fseCTables_ready = 0; + zc->hufCTable_repeatMode = HUF_repeat_none; return ZSTD_continueCCtx(zc, params, frameContentSize); } @@ -271,41 +304,67 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, void* ptr; /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) - + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace - + (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); + { size_t const entropySpace = hufCTable_size + litlengthCTable_size + + offcodeCTable_size + matchlengthCTable_size + + entropyScratchSpace_size; + size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32) + + (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t)); + size_t const optSpace = ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optPotentialSpace : 0; + size_t const neededSpace = entropySpace + optSpace + tableSpace + tokenSpace; if (zc->workSpaceSize < neededSpace) { + zc->workSpaceSize = 0; ZSTD_free(zc->workSpace, zc->customMem); zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; + ptr = zc->workSpace; + + /* entropy space */ + zc->hufCTable = (HUF_CElt*)ptr; + ptr = (char*)zc->hufCTable + hufCTable_size; /* note : HUF_CElt* is incomplete type, size is estimated via macro */ + zc->offcodeCTable = (FSE_CTable*) ptr; + ptr = (char*)ptr + offcodeCTable_size; + zc->matchlengthCTable = (FSE_CTable*) ptr; + ptr = (char*)ptr + matchlengthCTable_size; + zc->litlengthCTable = (FSE_CTable*) ptr; + ptr = (char*)ptr + litlengthCTable_size; + assert(((size_t)ptr & 3) == 0); /* ensure correct alignment */ + zc->entropyScratchSpace = (unsigned*) ptr; } } - if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */ - XXH64_reset(&zc->xxhState, 0); - zc->hashLog3 = hashLog3; - zc->hashTable = (U32*)(zc->workSpace); - zc->chainTable = zc->hashTable + hSize; - zc->hashTable3 = zc->chainTable + chainSize; - ptr = zc->hashTable3 + h3Size; - zc->hufTable = (HUF_CElt*)ptr; - zc->flagStaticTables = 0; - zc->flagStaticHufTable = HUF_repeat_none; - ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ + /* init params */ + zc->params = params; + zc->blockSize = blockSize; + zc->frameContentSize = frameContentSize; + zc->consumedSrcSize = 0; + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->loadedDictEnd = 0; + zc->fseCTables_ready = 0; + zc->hufCTable_repeatMode = HUF_repeat_none; zc->nextToUpdate = 1; zc->nextSrc = NULL; zc->base = NULL; zc->dictBase = NULL; zc->dictLimit = 0; zc->lowLimit = 0; - zc->params = params; - zc->blockSize = blockSize; - zc->frameContentSize = frameContentSize; { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; } + zc->hashLog3 = hashLog3; + zc->seqStore.litLengthSum = 0; + /* ensure entropy tables are close together at the beginning */ + assert((void*)zc->hufCTable == zc->workSpace); + assert((char*)zc->offcodeCTable == (char*)zc->hufCTable + hufCTable_size); + assert((char*)zc->matchlengthCTable == (char*)zc->offcodeCTable + offcodeCTable_size); + assert((char*)zc->litlengthCTable == (char*)zc->matchlengthCTable + matchlengthCTable_size); + assert((char*)zc->entropyScratchSpace == (char*)zc->litlengthCTable + litlengthCTable_size); + ptr = (char*)zc->entropyScratchSpace + entropyScratchSpace_size; + + /* opt parser space */ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ zc->seqStore.litFreq = (U32*)ptr; zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); @@ -315,8 +374,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; - zc->seqStore.litLengthSum = 0; } + + /* table Space */ + if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + zc->hashTable = (U32*)(ptr); + zc->chainTable = zc->hashTable + hSize; + zc->hashTable3 = zc->chainTable + chainSize; + ptr = zc->hashTable3 + h3Size; + + /* sequences storage */ zc->seqStore.sequencesStart = (seqDef*)ptr; ptr = zc->seqStore.sequencesStart + maxNbSeq; zc->seqStore.llCode = (BYTE*) ptr; @@ -324,10 +392,6 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; - zc->stage = ZSTDcs_init; - zc->dictID = 0; - zc->loadedDictEnd = 0; - return 0; } } @@ -341,27 +405,32 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0; } -/*! ZSTD_copyCCtx() : -* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. -* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). -* @return : 0, or an error code */ -size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1 + * @return : 0, or an error code */ +size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize) { if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); - memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); { ZSTD_parameters params = srcCCtx->params; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); + params.fParams = fParams; + DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u \n", !fParams.noDictIDFlag); + ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); } /* copy tables */ { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; + size_t const hSize = (size_t)1 << srcCCtx->params.cParams.hashLog; size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); + assert((U32*)dstCCtx->chainTable == (U32*)dstCCtx->hashTable + hSize); /* chainTable must follow hashTable */ + assert((U32*)dstCCtx->hashTable3 == (U32*)dstCCtx->chainTable + chainSize); + memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); /* presumes all tables follow each other */ } /* copy dictionary offsets */ @@ -376,23 +445,36 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long dstCCtx->dictID = srcCCtx->dictID; /* copy entropy tables */ - dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; - dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable; - if (srcCCtx->flagStaticTables) { - memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable)); - memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable)); - memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); + dstCCtx->fseCTables_ready = srcCCtx->fseCTables_ready; + if (srcCCtx->fseCTables_ready) { + memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, litlengthCTable_size); + memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, matchlengthCTable_size); + memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, offcodeCTable_size); } - if (srcCCtx->flagStaticHufTable) { - memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4); + dstCCtx->hufCTable_repeatMode = srcCCtx->hufCTable_repeatMode; + if (srcCCtx->hufCTable_repeatMode) { + memcpy(dstCCtx->hufCTable, srcCCtx->hufCTable, hufCTable_size); } return 0; } +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + fParams.contentSizeFlag = pledgedSrcSize>0; + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize); +} + /*! ZSTD_reduceTable() : -* reduce table indexes by `reducerValue` */ + * reduce table indexes by `reducerValue` */ static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue) { U32 u; @@ -499,26 +581,28 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, /* small ? don't even attempt compression (speed opt) */ # define LITERAL_NOENTROPY 63 - { size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; + { size_t const minLitSize = zc->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); } if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ - { HUF_repeat repeat = zc->flagStaticHufTable; + { HUF_repeat repeat = zc->hufCTable_repeatMode; int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat); + cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + zc->entropyScratchSpace, entropyScratchSpace_size, zc->hufCTable, &repeat, preferRepeat) + : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + zc->entropyScratchSpace, entropyScratchSpace_size, zc->hufCTable, &repeat, preferRepeat); if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */ - else { zc->flagStaticHufTable = HUF_repeat_check; } /* now have a table to reuse */ + else { zc->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */ } if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) { - zc->flagStaticHufTable = HUF_repeat_none; + zc->hufCTable_repeatMode = HUF_repeat_none; return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); } if (cLitSize==1) { - zc->flagStaticHufTable = HUF_repeat_none; + zc->hufCTable_repeatMode = HUF_repeat_none; return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); } @@ -637,12 +721,12 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, /* CTable for Literal Lengths */ { U32 max = MaxLL; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters); + size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->entropyScratchSpace); if ((mostFrequent == nbSeq) && (nbSeq > 2)) { *op++ = llCodeTable[0]; FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); LLtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + } else if ((zc->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { LLtype = set_repeat; } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) { FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); @@ -653,7 +737,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return ERROR(GENERIC); + if (FSE_isError(NCountSize)) return NCountSize; op += NCountSize; } FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); LLtype = set_compressed; @@ -661,12 +745,12 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, /* CTable for Offsets */ { U32 max = MaxOff; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters); + size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->entropyScratchSpace); if ((mostFrequent == nbSeq) && (nbSeq > 2)) { *op++ = ofCodeTable[0]; FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); Offtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + } else if ((zc->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { Offtype = set_repeat; } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) { FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); @@ -677,7 +761,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return ERROR(GENERIC); + if (FSE_isError(NCountSize)) return NCountSize; op += NCountSize; } FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); Offtype = set_compressed; @@ -685,12 +769,12 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, /* CTable for MatchLengths */ { U32 max = MaxML; - size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters); + size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->entropyScratchSpace); if ((mostFrequent == nbSeq) && (nbSeq > 2)) { *op++ = *mlCodeTable; FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); MLtype = set_rle; - } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + } else if ((zc->fseCTables_ready) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { MLtype = set_repeat; } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) { FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); @@ -701,14 +785,14 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return ERROR(GENERIC); + if (FSE_isError(NCountSize)) return NCountSize; op += NCountSize; } FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); MLtype = set_compressed; } } *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - zc->flagStaticTables = 0; + zc->fseCTables_ready = 0; /* Encoding Sequences */ { BIT_CStream_t blockStream; @@ -787,7 +871,7 @@ _check_compressibility: { size_t const minGain = ZSTD_minGain(srcSize); size_t const maxCSize = srcSize - minGain; if ((size_t)(op-ostart) >= maxCSize) { - zc->flagStaticHufTable = HUF_repeat_none; + zc->hufCTable_repeatMode = HUF_repeat_none; return 0; } } @@ -816,7 +900,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v const U32 pos = (U32)((const BYTE*)literals - g_start); if (g_start==NULL) g_start = (const BYTE*)literals; if ((pos > 1895000) && (pos < 1895300)) - fprintf(stderr, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", + DEBUGLOG(5, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); } #endif @@ -825,14 +909,20 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v seqStorePtr->lit += litLength; /* literal Length */ - if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + if (litLength>0xFFFF) { + seqStorePtr->longLengthID = 1; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } seqStorePtr->sequences[0].litLength = (U16)litLength; /* match offset */ seqStorePtr->sequences[0].offset = offsetCode + 1; /* match Length */ - if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + if (matchCode>0xFFFF) { + seqStorePtr->longLengthID = 2; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } seqStorePtr->sequences[0].matchLength = (U16)matchCode; seqStorePtr->sequences++; @@ -853,7 +943,14 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif } else { /* 32 bits */ @@ -864,7 +961,10 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif } @@ -936,7 +1036,7 @@ static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE ***************************************/ static const U32 prime3bytes = 506832829U; static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } -MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ static const U32 prime4bytes = 2654435761U; static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } @@ -1085,7 +1185,7 @@ static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, const U32 mls = ctx->params.cParams.searchLength; switch(mls) { - default: + default: /* includes case 3 */ case 4 : ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; case 5 : @@ -1135,7 +1235,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; ip++; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); } else { @@ -1147,7 +1247,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; U32 offset; - mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset = current - matchIndex; offset_2 = offset_1; @@ -1171,7 +1271,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; @@ -1199,7 +1299,7 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, U32 const mls = ctx->params.cParams.searchLength; switch(mls) { - default: + default: /* includes case 3 */ case 4 : ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; case 5 : @@ -1274,7 +1374,9 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, const BYTE* match = base + matchIndexS; hashLong[h2] = hashSmall[h] = current; /* update hash tables */ - if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */ + assert(offset_1 <= current); /* supposed guaranteed by construction */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + /* favor repcode */ mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); @@ -1285,15 +1387,15 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, offset = (U32)(ip-matchLong); while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { - size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE* match3 = base + matchIndex3; - hashLong[h3] = current + 1; - if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { - mLength = ZSTD_count(ip+9, match3+8, iend) + 8; + size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = current + 1; + if ( (matchIndexL3 > lowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1)) ) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; ip++; - offset = (U32)(ip-match3); - while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>lowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ } else { mLength = ZSTD_count(ip+4, match+4, iend) + 4; offset = (U32)(ip-match); @@ -1353,7 +1455,7 @@ static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_ const U32 mls = ctx->params.cParams.searchLength; switch(mls) { - default: + default: /* includes case 3 */ case 4 : ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; case 5 : @@ -1462,8 +1564,8 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, if (ip <= ilimit) { /* Fill Table */ - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); /* check immediate repcode */ @@ -1474,7 +1576,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; @@ -1503,7 +1605,7 @@ static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, U32 const mls = ctx->params.cParams.searchLength; switch(mls) { - default: + default: /* includes case 3 */ case 4 : ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; case 5 : @@ -1588,7 +1690,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co match = dictBase + matchIndex; matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ } if (matchLength > bestLength) { @@ -1667,7 +1769,7 @@ static size_t ZSTD_insertBtAndFindBestMatch ( match = dictBase + matchIndex; matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ } if (matchLength > bestLength) { @@ -1733,9 +1835,10 @@ static size_t ZSTD_BtFindBestMatch_selectMLS ( { switch(matchLengthSearch) { - default : + default : /* includes case 3 */ case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 7 : case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); } } @@ -1772,9 +1875,10 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( { switch(matchLengthSearch) { - default : + default : /* includes case 3 */ case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 7 : case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); } } @@ -1831,7 +1935,7 @@ size_t ZSTD_HcFindBestMatch_generic ( const U32 current = (U32)(ip-base); const U32 minChain = current > chainSize ? current - chainSize : 0; int nbAttempts=maxNbAttempts; - size_t ml=EQUAL_READ32-1; + size_t ml=4-1; /* HC4 match finder */ U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); @@ -1846,11 +1950,15 @@ size_t ZSTD_HcFindBestMatch_generic ( } else { match = dictBase + matchIndex; if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; } /* save best solution */ - if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } if (matchIndex <= minChain) break; matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); @@ -1868,9 +1976,10 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( { switch(matchLengthSearch) { - default : + default : /* includes case 3 */ case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); + case 7 : case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); } } @@ -1884,9 +1993,10 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( { switch(matchLengthSearch) { - default : + default : /* includes case 3 */ case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); + case 7 : case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); } } @@ -1934,7 +2044,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, /* check repCode */ if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) { /* repcode : we take it */ - matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; if (depth==0) goto _storeSequence; } @@ -1945,7 +2055,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, matchLength = ml2, start = ip, offset=offsetFound; } - if (matchLength < EQUAL_READ32) { + if (matchLength < 4) { ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ continue; } @@ -1955,17 +2065,17 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, while (ip<ilimit) { ip ++; if ((offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { - size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(mlRep * 3); int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); - if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) + if ((mlRep >= 4) && (gain2 > gain1)) matchLength = mlRep, offset = 0, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + if ((ml2 >= 4) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; /* search a better one */ } } @@ -1974,17 +2084,17 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if ((depth==2) && (ip<ilimit)) { ip ++; if ((offset) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { - size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + size_t const ml2 = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(ml2 * 4); int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) + if ((ml2 >= 4) && (gain2 > gain1)) matchLength = ml2, offset = 0, start = ip; } { size_t offset2=99999999; size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + if ((ml2 >= 4) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; } } } @@ -1993,7 +2103,9 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, /* catch up */ if (offset) { - while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ + while ( (start > anchor) + && (start > base+offset-ZSTD_REP_MOVE) + && (start[-1] == start[-1-offset+ZSTD_REP_MOVE]) ) /* only search for offset within prefix */ { start--; matchLength++; } offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); } @@ -2010,7 +2122,7 @@ _storeSequence: && ((offset_2>0) & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { /* store sequence */ - matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32; + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; @@ -2099,7 +2211,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, if (MEM_read32(ip+1) == MEM_read32(repMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; if (depth==0) goto _storeSequence; } } @@ -2110,7 +2222,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, matchLength = ml2, start = ip, offset=offsetFound; } - if (matchLength < EQUAL_READ32) { + if (matchLength < 4) { ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ continue; } @@ -2129,10 +2241,10 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; int const gain2 = (int)(repLength * 3); int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); - if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) + if ((repLength >= 4) && (gain2 > gain1)) matchLength = repLength, offset = 0, start = ip; } } @@ -2141,7 +2253,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + if ((ml2 >= 4) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; /* search a better one */ } } @@ -2159,10 +2271,10 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; - int gain2 = (int)(repLength * 4); - int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); - if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) matchLength = repLength, offset = 0, start = ip; } } @@ -2171,7 +2283,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); - if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + if ((ml2 >= 4) && (gain2 > gain1)) { matchLength = ml2, offset = offset2, start = ip; continue; } } } @@ -2203,7 +2315,7 @@ _storeSequence: if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; @@ -2294,8 +2406,12 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) { static const ZSTD_blockCompressor blockCompressor[2][8] = { - { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, - { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } + { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, + { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } }; return blockCompressor[extDict][(U32)strat]; @@ -2311,7 +2427,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ ZSTD_resetSeqStore(&(zc->seqStore)); if (current > zc->nextToUpdate + 384) - zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* limited update after finding a very long match */ blockCompressor(zc, src, srcSize); return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); } @@ -2343,7 +2459,8 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); size_t cSize; - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) + return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; /* preemptive overflow correction */ @@ -2398,7 +2515,8 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; - U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ U32 const checksumFlag = params.fParams.checksumFlag>0; U32 const windowSize = 1U << params.cParams.windowLog; U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); @@ -2410,6 +2528,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, size_t pos; if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); + DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u \n", !params.fParams.noDictIDFlag); + DEBUGLOG(5, "ZSTD_writeFrameHeader : dictID : %u \n", dictID); + DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDSizeCode : %u \n", dictIDSizeCode); MEM_writeLE32(dst, ZSTD_MAGICNUMBER); op[4] = frameHeaderDecriptionByte; pos=5; @@ -2478,6 +2599,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(cSize)) return cSize; + cctx->consumedSrcSize += srcSize; return cSize + fhSize; } else return fhSize; @@ -2488,7 +2610,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } @@ -2501,10 +2623,12 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const { size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); - return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } - +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize) { const BYTE* const ip = (const BYTE*) src; @@ -2534,13 +2658,15 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t case ZSTD_greedy: case ZSTD_lazy: case ZSTD_lazy2: - ZSTD_insertAndFindFirstIndex (zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength); + if (srcSize >= HASH_READ_SIZE) + ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength); break; case ZSTD_btlazy2: case ZSTD_btopt: case ZSTD_btopt2: - ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); + if (srcSize >= HASH_READ_SIZE) + ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); break; default: @@ -2567,18 +2693,15 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym /* Dictionary format : - Magic == ZSTD_DICT_MAGIC (4 bytes) - HUF_writeCTable(256) - FSE_writeNCount(off) - FSE_writeNCount(ml) - FSE_writeNCount(ll) - RepOffsets - Dictionary content -*/ -/*! ZSTD_loadDictEntropyStats() : - @return : size read from dictionary - note : magic number supposed already checked */ -static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) + * See : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : 0, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed > 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; @@ -2586,7 +2709,11 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ unsigned offcodeMaxValue = MaxOff; BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)]; - { size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize); + dictPtr += 4; /* skip magic number */ + cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); + dictPtr += 4; + + { size_t const hufHeaderSize = HUF_readCTable(cctx->hufCTable, 255, dictPtr, dictEnd-dictPtr); if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); dictPtr += hufHeaderSize; } @@ -2596,7 +2723,8 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ - CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); + CHECK_E( FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), + dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -2606,8 +2734,9 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); /* Every match length code must have non-zero probability */ - CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); + CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + CHECK_E( FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), + dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -2617,49 +2746,51 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); /* Every literal length code must have non-zero probability */ - CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); + CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + CHECK_E( FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), + dictionary_corrupted); dictPtr += litlengthHeaderSize; } if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); - cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); - cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[0] = MEM_readLE32(dictPtr+0); + cctx->rep[1] = MEM_readLE32(dictPtr+4); + cctx->rep[2] = MEM_readLE32(dictPtr+8); dictPtr += 12; - { U32 offcodeMax = MaxOff; - if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) { - U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */ - /* Calculate minimum offset code required to represent maxOffset */ - offcodeMax = ZSTD_highbit32(maxOffset); + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ } - /* Every possible supported offset <= dictContentSize + 128 KB must be representable */ + /* All offset values <= dictContentSize + 128 KB must be representable */ CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); - } + /* All repCodes must be <= dictContentSize and != 0*/ + { U32 u; + for (u=0; u<3; u++) { + if (cctx->rep[u] == 0) return ERROR(dictionary_corrupted); + if (cctx->rep[u] > dictContentSize) return ERROR(dictionary_corrupted); + } } - cctx->flagStaticTables = 1; - cctx->flagStaticHufTable = HUF_repeat_valid; - return dictPtr - (const BYTE*)dict; + cctx->fseCTables_ready = 1; + cctx->hufCTable_repeatMode = HUF_repeat_valid; + return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize); + } } /** ZSTD_compress_insertDictionary() : * @return : 0, or an error code */ -static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) +static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) { if ((dict==NULL) || (dictSize<=8)) return 0; /* dict as pure content */ - if ((MEM_readLE32(dict) != ZSTD_DICT_MAGIC) || (zc->forceRawDict)) - return ZSTD_loadDictionaryContent(zc, dict, dictSize); - zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); + if ((MEM_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) + return ZSTD_loadDictionaryContent(cctx, dict, dictSize); - /* known magic number : dict is parsed for entropy stats and content */ - { size_t const loadError = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); - size_t const eSize = loadError + 8; - if (ZSTD_isError(loadError)) return loadError; - return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); - } + /* dict as zstd dictionary */ + return ZSTD_loadZstdDictionary(cctx, dict, dictSize); } /*! ZSTD_compressBegin_internal() : @@ -2669,7 +2800,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 pledgedSrcSize) { ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; - CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, crp)); return ZSTD_compress_insertDictionary(cctx, dict, dictSize); } @@ -2745,10 +2877,13 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, const void* src, size_t srcSize) { size_t endResult; - size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); + size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */); if (ZSTD_isError(cSize)) return cSize; endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); if (ZSTD_isError(endResult)) return endResult; + if (cctx->params.fParams.contentSizeFlag) { /* control src size */ + if (cctx->frameContentSize != cctx->consumedSrcSize) return ERROR(srcSize_wrong); + } return cSize + endResult; } @@ -2773,7 +2908,8 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } -size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0); params.fParams.contentSizeFlag = 1; @@ -2812,8 +2948,16 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); } +static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) +{ + ZSTD_parameters params; + params.cParams = cParams; + params.fParams = fParams; + return params; +} + ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference, - ZSTD_parameters params, ZSTD_customMem customMem) + ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; @@ -2838,7 +2982,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u cdict->dictContent = internalBuffer; } - { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); + { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ + ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams); + size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(cdict->dictBuffer, customMem); ZSTD_free(cdict, customMem); @@ -2855,17 +3001,15 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_customMem const allocator = { NULL, NULL, NULL }; - ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); - params.fParams.contentSizeFlag = 1; - return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator); + ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_createCDict_advanced(dict, dictSize, 0, cParams, allocator); } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_customMem const allocator = { NULL, NULL, NULL }; - ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); - params.fParams.contentSizeFlag = 1; - return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator); + ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_createCDict_advanced(dict, dictSize, 1, cParams, allocator); } size_t ZSTD_freeCDict(ZSTD_CDict* cdict) @@ -2883,34 +3027,55 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); } -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize) +/* ZSTD_compressBegin_usingCDict_advanced() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { - if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) + if (cdict==NULL) return ERROR(GENERIC); /* does not support NULL cdict */ + DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u \n", !fParams.noDictIDFlag); + if (cdict->dictContentSize) + CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) ) else { ZSTD_parameters params = cdict->refContext->params; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize)); + params.fParams = fParams; + CHECK_F(ZSTD_compressBegin_internal(cctx, NULL, 0, params, pledgedSrcSize)); } return 0; } +/* ZSTD_compressBegin_usingCDict() : + * pledgedSrcSize=0 means "unknown" + * if pledgedSrcSize>0, it will enable contentSizeFlag */ +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u \n", !fParams.noDictIDFlag); + return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); +} + +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + /*! ZSTD_compress_usingCDict() : -* Compression using a digested Dictionary. -* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. -* Note that compression level is decided during dictionary creation */ + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict) { - CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); - - if (cdict->refContext->params.fParams.contentSizeFlag==1) { - cctx->params.fParams.contentSizeFlag = 1; - cctx->frameContentSize = srcSize; - } - - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } @@ -2939,7 +3104,6 @@ struct ZSTD_CStream_s { U32 checksum; U32 frameEnded; U64 pledgedSrcSize; - U64 inputProcessed; ZSTD_parameters params; ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CStream within "zstd.h" */ @@ -2970,9 +3134,13 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) if (zcs==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = zcs->customMem; ZSTD_freeCCtx(zcs->cctx); + zcs->cctx = NULL; ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = NULL; ZSTD_free(zcs->inBuff, cMem); + zcs->inBuff = NULL; ZSTD_free(zcs->outBuff, cMem); + zcs->outBuff = NULL; ZSTD_free(zcs, cMem); return 0; } @@ -2982,14 +3150,20 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) /*====== Initialization ======*/ size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } -size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ - if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) - else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); + DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag); + + if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize)) + else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); zcs->inToCompress = 0; zcs->inBuffPos = 0; @@ -2998,7 +3172,6 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long p zcs->stage = zcss_load; zcs->frameEnded = 0; zcs->pledgedSrcSize = pledgedSrcSize; - zcs->inputProcessed = 0; return 0; /* ready to go */ } @@ -3006,70 +3179,109 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - + DEBUGLOG(5, "ZSTD_resetCStream : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag); return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); } -size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) +/* ZSTD_initCStream_internal() : + * params are supposed validated at this stage + * and zcs->cdict is supposed to be correct */ +static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs, + const ZSTD_parameters params, + unsigned long long pledgedSrcSize) { + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + /* allocate buffers */ { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zcs->inBuffSize < neededInBuffSize) { - zcs->inBuffSize = neededInBuffSize; + zcs->inBuffSize = 0; ZSTD_free(zcs->inBuff, zcs->customMem); zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); if (zcs->inBuff == NULL) return ERROR(memory_allocation); + zcs->inBuffSize = neededInBuffSize; } zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); } if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { - zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; + size_t const outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; + zcs->outBuffSize = 0; ZSTD_free(zcs->outBuff, zcs->customMem); - zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); + zcs->outBuff = (char*) ZSTD_malloc(outBuffSize, zcs->customMem); if (zcs->outBuff == NULL) return ERROR(memory_allocation); + zcs->outBuffSize = outBuffSize; } - if (dict && dictSize >= 8) { - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); - if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); - zcs->cdict = zcs->cdictLocal; - } else zcs->cdict = NULL; - zcs->checksum = params.fParams.checksumFlag > 0; zcs->params = params; + DEBUGLOG(5, "ZSTD_initCStream_stage2 : dictIDFlag == %u \n", !params.fParams.noDictIDFlag); return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); } +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams) +{ + if (!cdict) return ERROR(GENERIC); /* cannot handle NULL cdict (does not know what to do) */ + { ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict); + params.fParams = fParams; + zcs->cdict = cdict; + return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize); + } +} + /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { - ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); - size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0); - zcs->cdict = cdict; - zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID; - return initError; + ZSTD_frameParameters const fParams = { 0 /* content */, 0 /* checksum */, 0 /* noDictID */ }; + return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, 0, fParams); +} + +static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + zcs->cdict = NULL; + + if (dict && dictSize >= 8) { + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem); + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + zcs->cdict = zcs->cdictLocal; + } + + DEBUGLOG(5, "ZSTD_initCStream_internal : dictIDFlag == %u \n", !params.fParams.noDictIDFlag); + return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize); +} + +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + CHECK_F( ZSTD_checkCParams(params.cParams) ); + DEBUGLOG(5, "ZSTD_initCStream_advanced : dictIDFlag == %u \n", !params.fParams.noDictIDFlag); + return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); + return ZSTD_initCStream_internal(zcs, dict, dictSize, params, 0); } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); - if (pledgedSrcSize) params.fParams.contentSizeFlag = 1; - return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); + params.fParams.contentSizeFlag = (pledgedSrcSize>0); + return ZSTD_initCStream_internal(zcs, NULL, 0, params, pledgedSrcSize); } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { - return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0); + return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0); } size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) @@ -3163,7 +3375,6 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, *srcSizePtr = ip - istart; *dstCapacityPtr = op - ostart; - zcs->inputProcessed += *srcSizePtr; if (zcs->frameEnded) return 0; { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; if (hintInSize==0) hintInSize = zcs->blockSize; @@ -3208,14 +3419,12 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) BYTE* const oend = (BYTE*)(output->dst) + output->size; BYTE* op = ostart; - if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) - return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ - if (zcs->stage != zcss_final) { /* flush whatever remains */ size_t srcSize = 0; size_t sizeWritten = output->size - output->pos; - size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ + size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, + &srcSize /* use a valid src address instead of NULL */, &srcSize, zsf_end); size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; op += sizeWritten; if (remainingToFlush) { @@ -3225,7 +3434,9 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /* create epilogue */ zcs->stage = zcss_final; zcs->outBuffContentSize = !notEnded ? 0 : - ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + /* write epilogue, including final empty block, into outBuff */ + ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); + if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize; } /* flush epilogue */ @@ -3268,7 +3479,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ - { 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */ + { 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */ { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */ { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */ diff --git a/contrib/zstd/lib/compress/zstd_opt.h b/contrib/zstd/lib/compress/zstd_opt.h index ac418b61c8341..54376119121d6 100644 --- a/contrib/zstd/lib/compress/zstd_opt.h +++ b/contrib/zstd/lib/compress/zstd_opt.h @@ -175,10 +175,10 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B } /* match offset */ - { BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); - seqStorePtr->offCodeSum++; - seqStorePtr->offCodeFreq[offCode]++; - } + { BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + seqStorePtr->offCodeSum++; + seqStorePtr->offCodeFreq[offCode]++; + } /* match Length */ { const BYTE ML_deltaCode = 36; @@ -360,6 +360,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS ( default : case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 7 : case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); } } @@ -387,6 +388,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( default : case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 7 : case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); } } diff --git a/contrib/zstd/lib/compress/zstdmt_compress.c b/contrib/zstd/lib/compress/zstdmt_compress.c index 45514a81a3af3..fc7f52a2902d3 100644 --- a/contrib/zstd/lib/compress/zstdmt_compress.c +++ b/contrib/zstd/lib/compress/zstdmt_compress.c @@ -33,7 +33,7 @@ # include <stdio.h> # include <unistd.h> # include <sys/times.h> - static unsigned g_debugLevel = 3; + static unsigned g_debugLevel = 5; # define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); } # define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); } @@ -44,26 +44,26 @@ DEBUGLOGRAW(l, " \n"); \ } -static unsigned long long GetCurrentClockTimeMicroseconds() +static unsigned long long GetCurrentClockTimeMicroseconds(void) { static clock_t _ticksPerSecond = 0; if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); - struct tms junk; clock_t newTicks = (clock_t) times(&junk); - return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); } } #define MUTEX_WAIT_TIME_DLEVEL 5 #define PTHREAD_MUTEX_LOCK(mutex) \ if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \ - unsigned long long beforeTime = GetCurrentClockTimeMicroseconds(); \ - pthread_mutex_lock(mutex); \ - unsigned long long afterTime = GetCurrentClockTimeMicroseconds(); \ - unsigned long long elapsedTime = (afterTime-beforeTime); \ - if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ - DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ elapsedTime, #mutex); \ - } \ + } } \ } else pthread_mutex_lock(mutex); #else @@ -228,17 +228,19 @@ void ZSTDMT_compressChunk(void* jobDescription) ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; const void* const src = (const char*)job->srcStart + job->dictSize; buffer_t const dstBuff = job->dstBuff; - DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); + DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", + job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); if (job->cdict) { /* should only happen for first segment */ - size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize); + size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize); if (job->cdict) DEBUGLOG(3, "using CDict "); if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } } else { /* srcStart points at reloaded section */ - size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1); /* Force loading dictionary in "content-only" mode (no header analysis) */ - size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, 0); - if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; } - ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1); - } + if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ + { size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1); /* Force loading dictionary in "content-only" mode (no header analysis) */ + size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize); + if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; } + ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1); + } } if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0); if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; } @@ -250,7 +252,9 @@ void ZSTDMT_compressChunk(void* jobDescription) job->cSize = (job->lastChunk) ? ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) : ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize); - DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk); + DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)", + (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk); + DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize)); _endJob: PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex); @@ -388,14 +392,17 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, int compressionLevel) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); + U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 0 : 3; + size_t const overlapSize = (size_t)1 << (params.cParams.windowLog - overlapLog); size_t const chunkTargetSize = (size_t)1 << (params.cParams.windowLog + 2); - unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + (srcSize < chunkTargetSize) /* min 1 */; + unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + 1; unsigned nbChunks = MIN(nbChunksMax, mtctx->nbThreads); size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks; size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0xFFFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */ size_t remainingSrcSize = srcSize; const char* const srcStart = (const char*)src; - size_t frameStartPos = 0; + unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize)); /* presumes avgChunkSize >= 256 KB, which should be the case */ + size_t frameStartPos = 0, dstBufferPos = 0; DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes ", params.cParams.windowLog, (U32)chunkTargetSize); DEBUGLOG(2, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); @@ -409,10 +416,11 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, { unsigned u; for (u=0; u<nbChunks; u++) { size_t const chunkSize = MIN(remainingSrcSize, avgChunkSize); - size_t const dstBufferCapacity = u ? ZSTD_compressBound(chunkSize) : dstCapacity; - buffer_t const dstAsBuffer = { dst, dstCapacity }; - buffer_t const dstBuffer = u ? ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity) : dstAsBuffer; + size_t const dstBufferCapacity = ZSTD_compressBound(chunkSize); + buffer_t const dstAsBuffer = { (char*)dst + dstBufferPos, dstBufferCapacity }; + buffer_t const dstBuffer = u < compressWithinDst ? dstAsBuffer : ZSTDMT_getBuffer(mtctx->buffPool, dstBufferCapacity); ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(mtctx->cctxPool); + size_t dictSize = u ? overlapSize : 0; if ((cctx==NULL) || (dstBuffer.start==NULL)) { mtctx->jobs[u].cSize = ERROR(memory_allocation); /* job result */ @@ -421,7 +429,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, break; /* let's wait for previous jobs to complete, but don't start new ones */ } - mtctx->jobs[u].srcStart = srcStart + frameStartPos; + mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize; + mtctx->jobs[u].dictSize = dictSize; mtctx->jobs[u].srcSize = chunkSize; mtctx->jobs[u].fullFrameSize = srcSize; mtctx->jobs[u].params = params; @@ -438,6 +447,7 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]); frameStartPos += chunkSize; + dstBufferPos += dstBufferCapacity; remainingSrcSize -= chunkSize; } } /* note : since nbChunks <= nbThreads, all jobs should be running immediately in parallel */ @@ -461,8 +471,10 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, if (ZSTD_isError(cSize)) error = cSize; if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall); if (chunkID) { /* note : chunk 0 is already written directly into dst */ - if (!error) memcpy((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize); - ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff); + if (!error) + memmove((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize); /* may overlap if chunk decompressed within dst */ + if (chunkID >= compressWithinDst) /* otherwise, it decompresses within dst */ + ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff); mtctx->jobs[chunkID].dstBuff = g_nullBuffer; } dstPos += cSize ; @@ -509,7 +521,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, if (updateDict) { ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL; if (dict && dictSize) { - zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params, cmem); + zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem); if (zcs->cdict == NULL) return ERROR(memory_allocation); } } zcs->frameContentSize = pledgedSrcSize; diff --git a/contrib/zstd/lib/decompress/zstd_decompress.c b/contrib/zstd/lib/decompress/zstd_decompress.c index 2aaa4a3df3c50..910f9ab783c31 100644 --- a/contrib/zstd/lib/decompress/zstd_decompress.c +++ b/contrib/zstd/lib/decompress/zstd_decompress.c @@ -177,30 +177,6 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ } -#if 0 -/* deprecated */ -static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) -{ - ZSTD_decompressBegin(dstDCtx); /* init */ - if (srcDCtx) { /* support refDCtx on NULL */ - dstDCtx->dictEnd = srcDCtx->dictEnd; - dstDCtx->vBase = srcDCtx->vBase; - dstDCtx->base = srcDCtx->base; - dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; - dstDCtx->dictID = srcDCtx->dictID; - dstDCtx->litEntropy = srcDCtx->litEntropy; - dstDCtx->fseEntropy = srcDCtx->fseEntropy; - dstDCtx->LLTptr = srcDCtx->entropy.LLTable; - dstDCtx->MLTptr = srcDCtx->entropy.MLTable; - dstDCtx->OFTptr = srcDCtx->entropy.OFTable; - dstDCtx->HUFptr = srcDCtx->entropy.hufTable; - dstDCtx->entropy.rep[0] = srcDCtx->entropy.rep[0]; - dstDCtx->entropy.rep[1] = srcDCtx->entropy.rep[1]; - dstDCtx->entropy.rep[2] = srcDCtx->entropy.rep[2]; - } -} -#endif - static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict); @@ -431,7 +407,8 @@ typedef struct /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) { if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); { U32 const cBlockHeader = MEM_readLE24(src); @@ -446,7 +423,8 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bp } -static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); @@ -454,7 +432,9 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, } -static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize) +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + size_t regenSize) { if (srcSize != 1) return ERROR(srcSize_wrong); if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); @@ -595,176 +575,70 @@ typedef union { U32 alignedBy4; } FSE_decode_t4; +/* Default FSE distribution table for Literal Lengths */ static const FSE_decode_t4 LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = { { { LL_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ - { { 0, 0, 4 } }, /* 0 : base, symbol, bits */ - { { 16, 0, 4 } }, - { { 32, 1, 5 } }, - { { 0, 3, 5 } }, - { { 0, 4, 5 } }, - { { 0, 6, 5 } }, - { { 0, 7, 5 } }, - { { 0, 9, 5 } }, - { { 0, 10, 5 } }, - { { 0, 12, 5 } }, - { { 0, 14, 6 } }, - { { 0, 16, 5 } }, - { { 0, 18, 5 } }, - { { 0, 19, 5 } }, - { { 0, 21, 5 } }, - { { 0, 22, 5 } }, - { { 0, 24, 5 } }, - { { 32, 25, 5 } }, - { { 0, 26, 5 } }, - { { 0, 27, 6 } }, - { { 0, 29, 6 } }, - { { 0, 31, 6 } }, - { { 32, 0, 4 } }, - { { 0, 1, 4 } }, - { { 0, 2, 5 } }, - { { 32, 4, 5 } }, - { { 0, 5, 5 } }, - { { 32, 7, 5 } }, - { { 0, 8, 5 } }, - { { 32, 10, 5 } }, - { { 0, 11, 5 } }, - { { 0, 13, 6 } }, - { { 32, 16, 5 } }, - { { 0, 17, 5 } }, - { { 32, 19, 5 } }, - { { 0, 20, 5 } }, - { { 32, 22, 5 } }, - { { 0, 23, 5 } }, - { { 0, 25, 4 } }, - { { 16, 25, 4 } }, - { { 32, 26, 5 } }, - { { 0, 28, 6 } }, - { { 0, 30, 6 } }, - { { 48, 0, 4 } }, - { { 16, 1, 4 } }, - { { 32, 2, 5 } }, - { { 32, 3, 5 } }, - { { 32, 5, 5 } }, - { { 32, 6, 5 } }, - { { 32, 8, 5 } }, - { { 32, 9, 5 } }, - { { 32, 11, 5 } }, - { { 32, 12, 5 } }, - { { 0, 15, 6 } }, - { { 32, 17, 5 } }, - { { 32, 18, 5 } }, - { { 32, 20, 5 } }, - { { 32, 21, 5 } }, - { { 32, 23, 5 } }, - { { 32, 24, 5 } }, - { { 0, 35, 6 } }, - { { 0, 34, 6 } }, - { { 0, 33, 6 } }, - { { 0, 32, 6 } }, + /* base, symbol, bits */ + { { 0, 0, 4 } }, { { 16, 0, 4 } }, { { 32, 1, 5 } }, { { 0, 3, 5 } }, + { { 0, 4, 5 } }, { { 0, 6, 5 } }, { { 0, 7, 5 } }, { { 0, 9, 5 } }, + { { 0, 10, 5 } }, { { 0, 12, 5 } }, { { 0, 14, 6 } }, { { 0, 16, 5 } }, + { { 0, 18, 5 } }, { { 0, 19, 5 } }, { { 0, 21, 5 } }, { { 0, 22, 5 } }, + { { 0, 24, 5 } }, { { 32, 25, 5 } }, { { 0, 26, 5 } }, { { 0, 27, 6 } }, + { { 0, 29, 6 } }, { { 0, 31, 6 } }, { { 32, 0, 4 } }, { { 0, 1, 4 } }, + { { 0, 2, 5 } }, { { 32, 4, 5 } }, { { 0, 5, 5 } }, { { 32, 7, 5 } }, + { { 0, 8, 5 } }, { { 32, 10, 5 } }, { { 0, 11, 5 } }, { { 0, 13, 6 } }, + { { 32, 16, 5 } }, { { 0, 17, 5 } }, { { 32, 19, 5 } }, { { 0, 20, 5 } }, + { { 32, 22, 5 } }, { { 0, 23, 5 } }, { { 0, 25, 4 } }, { { 16, 25, 4 } }, + { { 32, 26, 5 } }, { { 0, 28, 6 } }, { { 0, 30, 6 } }, { { 48, 0, 4 } }, + { { 16, 1, 4 } }, { { 32, 2, 5 } }, { { 32, 3, 5 } }, { { 32, 5, 5 } }, + { { 32, 6, 5 } }, { { 32, 8, 5 } }, { { 32, 9, 5 } }, { { 32, 11, 5 } }, + { { 32, 12, 5 } }, { { 0, 15, 6 } }, { { 32, 17, 5 } }, { { 32, 18, 5 } }, + { { 32, 20, 5 } }, { { 32, 21, 5 } }, { { 32, 23, 5 } }, { { 32, 24, 5 } }, + { { 0, 35, 6 } }, { { 0, 34, 6 } }, { { 0, 33, 6 } }, { { 0, 32, 6 } }, }; /* LL_defaultDTable */ +/* Default FSE distribution table for Match Lengths */ static const FSE_decode_t4 ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = { { { ML_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ - { { 0, 0, 6 } }, /* 0 : base, symbol, bits */ - { { 0, 1, 4 } }, - { { 32, 2, 5 } }, - { { 0, 3, 5 } }, - { { 0, 5, 5 } }, - { { 0, 6, 5 } }, - { { 0, 8, 5 } }, - { { 0, 10, 6 } }, - { { 0, 13, 6 } }, - { { 0, 16, 6 } }, - { { 0, 19, 6 } }, - { { 0, 22, 6 } }, - { { 0, 25, 6 } }, - { { 0, 28, 6 } }, - { { 0, 31, 6 } }, - { { 0, 33, 6 } }, - { { 0, 35, 6 } }, - { { 0, 37, 6 } }, - { { 0, 39, 6 } }, - { { 0, 41, 6 } }, - { { 0, 43, 6 } }, - { { 0, 45, 6 } }, - { { 16, 1, 4 } }, - { { 0, 2, 4 } }, - { { 32, 3, 5 } }, - { { 0, 4, 5 } }, - { { 32, 6, 5 } }, - { { 0, 7, 5 } }, - { { 0, 9, 6 } }, - { { 0, 12, 6 } }, - { { 0, 15, 6 } }, - { { 0, 18, 6 } }, - { { 0, 21, 6 } }, - { { 0, 24, 6 } }, - { { 0, 27, 6 } }, - { { 0, 30, 6 } }, - { { 0, 32, 6 } }, - { { 0, 34, 6 } }, - { { 0, 36, 6 } }, - { { 0, 38, 6 } }, - { { 0, 40, 6 } }, - { { 0, 42, 6 } }, - { { 0, 44, 6 } }, - { { 32, 1, 4 } }, - { { 48, 1, 4 } }, - { { 16, 2, 4 } }, - { { 32, 4, 5 } }, - { { 32, 5, 5 } }, - { { 32, 7, 5 } }, - { { 32, 8, 5 } }, - { { 0, 11, 6 } }, - { { 0, 14, 6 } }, - { { 0, 17, 6 } }, - { { 0, 20, 6 } }, - { { 0, 23, 6 } }, - { { 0, 26, 6 } }, - { { 0, 29, 6 } }, - { { 0, 52, 6 } }, - { { 0, 51, 6 } }, - { { 0, 50, 6 } }, - { { 0, 49, 6 } }, - { { 0, 48, 6 } }, - { { 0, 47, 6 } }, - { { 0, 46, 6 } }, + /* base, symbol, bits */ + { { 0, 0, 6 } }, { { 0, 1, 4 } }, { { 32, 2, 5 } }, { { 0, 3, 5 } }, + { { 0, 5, 5 } }, { { 0, 6, 5 } }, { { 0, 8, 5 } }, { { 0, 10, 6 } }, + { { 0, 13, 6 } }, { { 0, 16, 6 } }, { { 0, 19, 6 } }, { { 0, 22, 6 } }, + { { 0, 25, 6 } }, { { 0, 28, 6 } }, { { 0, 31, 6 } }, { { 0, 33, 6 } }, + { { 0, 35, 6 } }, { { 0, 37, 6 } }, { { 0, 39, 6 } }, { { 0, 41, 6 } }, + { { 0, 43, 6 } }, { { 0, 45, 6 } }, { { 16, 1, 4 } }, { { 0, 2, 4 } }, + { { 32, 3, 5 } }, { { 0, 4, 5 } }, { { 32, 6, 5 } }, { { 0, 7, 5 } }, + { { 0, 9, 6 } }, { { 0, 12, 6 } }, { { 0, 15, 6 } }, { { 0, 18, 6 } }, + { { 0, 21, 6 } }, { { 0, 24, 6 } }, { { 0, 27, 6 } }, { { 0, 30, 6 } }, + { { 0, 32, 6 } }, { { 0, 34, 6 } }, { { 0, 36, 6 } }, { { 0, 38, 6 } }, + { { 0, 40, 6 } }, { { 0, 42, 6 } }, { { 0, 44, 6 } }, { { 32, 1, 4 } }, + { { 48, 1, 4 } }, { { 16, 2, 4 } }, { { 32, 4, 5 } }, { { 32, 5, 5 } }, + { { 32, 7, 5 } }, { { 32, 8, 5 } }, { { 0, 11, 6 } }, { { 0, 14, 6 } }, + { { 0, 17, 6 } }, { { 0, 20, 6 } }, { { 0, 23, 6 } }, { { 0, 26, 6 } }, + { { 0, 29, 6 } }, { { 0, 52, 6 } }, { { 0, 51, 6 } }, { { 0, 50, 6 } }, + { { 0, 49, 6 } }, { { 0, 48, 6 } }, { { 0, 47, 6 } }, { { 0, 46, 6 } }, }; /* ML_defaultDTable */ +/* Default FSE distribution table for Offset Codes */ static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = { { { OF_DEFAULTNORMLOG, 1, 1 } }, /* header : tableLog, fastMode, fastMode */ - { { 0, 0, 5 } }, /* 0 : base, symbol, bits */ - { { 0, 6, 4 } }, - { { 0, 9, 5 } }, - { { 0, 15, 5 } }, - { { 0, 21, 5 } }, - { { 0, 3, 5 } }, - { { 0, 7, 4 } }, - { { 0, 12, 5 } }, - { { 0, 18, 5 } }, - { { 0, 23, 5 } }, - { { 0, 5, 5 } }, - { { 0, 8, 4 } }, - { { 0, 14, 5 } }, - { { 0, 20, 5 } }, - { { 0, 2, 5 } }, - { { 16, 7, 4 } }, - { { 0, 11, 5 } }, - { { 0, 17, 5 } }, - { { 0, 22, 5 } }, - { { 0, 4, 5 } }, - { { 16, 8, 4 } }, - { { 0, 13, 5 } }, - { { 0, 19, 5 } }, - { { 0, 1, 5 } }, - { { 16, 6, 4 } }, - { { 0, 10, 5 } }, - { { 0, 16, 5 } }, - { { 0, 28, 5 } }, - { { 0, 27, 5 } }, - { { 0, 26, 5 } }, - { { 0, 25, 5 } }, - { { 0, 24, 5 } }, + /* base, symbol, bits */ + { { 0, 0, 5 } }, { { 0, 6, 4 } }, + { { 0, 9, 5 } }, { { 0, 15, 5 } }, + { { 0, 21, 5 } }, { { 0, 3, 5 } }, + { { 0, 7, 4 } }, { { 0, 12, 5 } }, + { { 0, 18, 5 } }, { { 0, 23, 5 } }, + { { 0, 5, 5 } }, { { 0, 8, 4 } }, + { { 0, 14, 5 } }, { { 0, 20, 5 } }, + { { 0, 2, 5 } }, { { 16, 7, 4 } }, + { { 0, 11, 5 } }, { { 0, 17, 5 } }, + { { 0, 22, 5 } }, { { 0, 4, 5 } }, + { { 16, 8, 4 } }, { { 0, 13, 5 } }, + { { 0, 19, 5 } }, { { 0, 1, 5 } }, + { { 16, 6, 4 } }, { { 0, 10, 5 } }, + { { 0, 16, 5 } }, { { 0, 28, 5 } }, + { { 0, 27, 5 } }, { { 0, 26, 5 } }, + { { 0, 25, 5 } }, { { 0, 24, 5 } }, }; /* OF_defaultDTable */ /*! ZSTD_buildSeqTable() : @@ -927,8 +801,6 @@ size_t ZSTD_execSequenceLast7(BYTE* op, } - - static seq_t ZSTD_decodeSequence(seqState_t* seqState) { seq_t seq; @@ -943,21 +815,26 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) U32 const totalBits = llBits+mlBits+ofBits; static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000 }; static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; /* sequence */ { size_t offset; @@ -1031,7 +908,7 @@ size_t ZSTD_execSequence(BYTE* op, /* copy Match */ if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ + /* offset beyond prefix -> go into extDict */ if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); match = dictEnd + (match - base); if (match + sequence.matchLength <= dictEnd) { @@ -1156,21 +1033,26 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con U32 const totalBits = llBits+mlBits+ofBits; static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000 }; static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; /* sequence */ { size_t offset; @@ -1476,7 +1358,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); #endif if (srcSize >= ZSTD_skippableHeaderSize && - (MEM_readLE32(src) & 0xFFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4); } else { const BYTE* ip = (const BYTE*)src; @@ -2115,15 +1997,18 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) } /*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. + * Provides the dictID required to decompresse frame stored within `src`. * If @return == 0, the dictID could not be decoded. * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden information. * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameParams(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 }; @@ -2209,9 +2094,13 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) if (zds==NULL) return 0; /* support free on null */ { ZSTD_customMem const cMem = zds->customMem; ZSTD_freeDCtx(zds->dctx); + zds->dctx = NULL; ZSTD_freeDDict(zds->ddictLocal); + zds->ddictLocal = NULL; ZSTD_free(zds->inBuff, cMem); + zds->inBuff = NULL; ZSTD_free(zds->outBuff, cMem); + zds->outBuff = NULL; #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (zds->legacyContext) ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); @@ -2247,7 +2136,9 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } -size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */ +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) { size_t const initResult = ZSTD_initDStream(zds); zds->ddict = ddict; @@ -2277,8 +2168,11 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - if (zds==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize; + if (zds==NULL) return 0; /* support sizeof NULL */ + return sizeof(*zds) + + ZSTD_sizeof_DCtx(zds->dctx) + + ZSTD_sizeof_DDict(zds->ddictLocal) + + zds->inBuffSize + zds->outBuffSize; } @@ -2376,15 +2270,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { ZSTD_free(zds->inBuff, zds->customMem); - zds->inBuffSize = blockSize; + zds->inBuffSize = 0; zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); if (zds->inBuff == NULL) return ERROR(memory_allocation); + zds->inBuffSize = blockSize; } if (zds->outBuffSize < neededOutSize) { ZSTD_free(zds->outBuff, zds->customMem); - zds->outBuffSize = neededOutSize; + zds->outBuffSize = 0; zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); if (zds->outBuff == NULL) return ERROR(memory_allocation); + zds->outBuffSize = neededOutSize; } } zds->stage = zdss_read; /* pass-through */ diff --git a/contrib/zstd/lib/dictBuilder/cover.c b/contrib/zstd/lib/dictBuilder/cover.c index 3a7b9f39f9fd7..1863c8f34542e 100644 --- a/contrib/zstd/lib/dictBuilder/cover.c +++ b/contrib/zstd/lib/dictBuilder/cover.c @@ -59,8 +59,6 @@ static int g_displayLevel = 2; if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ g_time = clock(); \ DISPLAY(__VA_ARGS__); \ - if (displayLevel >= 4) \ - fflush(stdout); \ } \ } #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) @@ -236,10 +234,22 @@ static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { * Returns 1 if the dmer at lp is greater than the dmer at rp. */ static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) { - const U32 lhs = *(const U32 *)lp; - const U32 rhs = *(const U32 *)rp; + U32 const lhs = *(U32 const *)lp; + U32 const rhs = *(U32 const *)rp; return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d); } +/** + * Faster version for d <= 8. + */ +static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { + U64 const mask = (ctx->d == 8) ? (U64)-1 : (((U64)1 << (8 * ctx->d)) - 1); + U64 const lhs = MEM_readLE64(ctx->samples + *(U32 const *)lp) & mask; + U64 const rhs = MEM_readLE64(ctx->samples + *(U32 const *)rp) & mask; + if (lhs < rhs) { + return -1; + } + return (lhs > rhs); +} /** * Same as COVER_cmp() except ties are broken by pointer value @@ -253,6 +263,16 @@ static int COVER_strict_cmp(const void *lp, const void *rp) { } return result; } +/** + * Faster version for d <= 8. + */ +static int COVER_strict_cmp8(const void *lp, const void *rp) { + int result = COVER_cmp8(g_ctx, lp, rp); + if (result == 0) { + result = lp < rp ? -1 : 1; + } + return result; +} /** * Returns the first pointer in [first, last) whose element does not compare @@ -508,7 +528,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, const BYTE *const samples = (const BYTE *)samplesBuffer; const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); /* Checks */ - if (totalSamplesSize < d || + if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { DISPLAYLEVEL(1, "Total samples size is too large, maximum size is %u MB\n", (COVER_MAX_SAMPLES_SIZE >> 20)); @@ -522,7 +542,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, ctx->samplesSizes = samplesSizes; ctx->nbSamples = nbSamples; /* Partial suffix array */ - ctx->suffixSize = totalSamplesSize - d + 1; + ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1; ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); /* Maps index to the dmerID */ ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); @@ -556,7 +576,8 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, } /* qsort doesn't take an opaque pointer, so pass as a global */ g_ctx = ctx; - qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), &COVER_strict_cmp); + qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); } DISPLAYLEVEL(2, "Computing frequencies\n"); /* For each dmer group (group of positions with the same first d bytes): @@ -566,8 +587,8 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, * 2. We calculate how many samples the dmer occurs in and save it in * freqs[dmerId]. */ - COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, &COVER_cmp, - &COVER_group); + COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, + (ctx->d <= 8 ? &COVER_cmp8 : &COVER_cmp), &COVER_group); ctx->freqs = ctx->suffix; ctx->suffix = NULL; return 1; @@ -918,10 +939,10 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer, /* constants */ const unsigned nbThreads = parameters->nbThreads; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; - const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d; - const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k; - const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k; - const unsigned kSteps = parameters->steps == 0 ? 32 : parameters->steps; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); const unsigned kIterations = (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); diff --git a/contrib/zstd/lib/dictBuilder/zdict.c b/contrib/zstd/lib/dictBuilder/zdict.c index 0757dbbbb6430..179e02effa4db 100644 --- a/contrib/zstd/lib/dictBuilder/zdict.c +++ b/contrib/zstd/lib/dictBuilder/zdict.c @@ -11,8 +11,9 @@ /*-************************************** * Tuning parameters ****************************************/ +#define MINRATIO 4 /* minimum nb of apparition to be selected in dictionary */ #define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) -#define ZDICT_MIN_SAMPLES_SIZE 512 +#define ZDICT_MIN_SAMPLES_SIZE (ZDICT_CONTENTSIZE_MIN * MINRATIO) /*-************************************** @@ -59,11 +60,8 @@ #define NOISELENGTH 32 -#define MINRATIO 4 static const int g_compressionLevel_default = 6; static const U32 g_selectivity_default = 9; -static const size_t g_provision_entropySize = 200; -static const size_t g_min_fast_dictContent = 192; /*-************************************* @@ -308,10 +306,10 @@ static dictItem ZDICT_analyzePos( /* look backward */ length = MINMATCHLENGTH; while ((length >= MINMATCHLENGTH) & (start > 0)) { - length = ZDICT_count(b + pos, b + suffix[start - 1]); - if (length >= LLIMIT) length = LLIMIT - 1; - lengthList[length]++; - if (length >= MINMATCHLENGTH) start--; + length = ZDICT_count(b + pos, b + suffix[start - 1]); + if (length >= LLIMIT) length = LLIMIT - 1; + lengthList[length]++; + if (length >= MINMATCHLENGTH) start--; } /* largest useful length */ @@ -363,21 +361,35 @@ static dictItem ZDICT_analyzePos( } +static int isIncluded(const void* in, const void* container, size_t length) +{ + const char* const ip = (const char*) in; + const char* const into = (const char*) container; + size_t u; + + for (u=0; u<length; u++) { /* works because end of buffer is a noisy guard band */ + if (ip[u] != into[u]) break; + } + + return u==length; +} + /*! ZDICT_checkMerge check if dictItem can be merged, do it if possible @return : id of destination elt, 0 if not merged */ -static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) +static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const void* buffer) { const U32 tableSize = table->pos; const U32 eltEnd = elt.pos + elt.length; + const char* const buf = (const char*) buffer; /* tail overlap */ U32 u; for (u=1; u<tableSize; u++) { if (u==eltNbToSkip) continue; if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ /* append */ - U32 addedLength = table[u].pos - elt.pos; + U32 const addedLength = table[u].pos - elt.pos; table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ @@ -393,9 +405,10 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) /* front overlap */ for (u=1; u<tableSize; u++) { if (u==eltNbToSkip) continue; + if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); table[u].savings += elt.length / 8; /* rough approx bonus */ if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; @@ -407,7 +420,18 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) table[u] = table[u-1], u--; table[u] = elt; return u; - } } + } + + if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { + if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { + size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); + table[u].pos = elt.pos; + table[u].savings += (U32)(elt.savings * addedLength / elt.length); + table[u].length = MIN(elt.length, table[u].length + 1); + return u; + } + } + } return 0; } @@ -425,14 +449,14 @@ static void ZDICT_removeDictItem(dictItem* table, U32 id) } -static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt) +static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt, const void* buffer) { /* merge if possible */ - U32 mergeId = ZDICT_checkMerge(table, elt, 0); + U32 mergeId = ZDICT_tryMerge(table, elt, 0, buffer); if (mergeId) { U32 newMerge = 1; while (newMerge) { - newMerge = ZDICT_checkMerge(table, table[mergeId], mergeId); + newMerge = ZDICT_tryMerge(table, table[mergeId], mergeId, buffer); if (newMerge) ZDICT_removeDictItem(table, mergeId); mergeId = newMerge; } @@ -480,7 +504,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ if (ZDICT_clockSpan(displayClock) > refreshRate) \ { displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (notificationLevel>=4) fflush(stdout); } } + if (notificationLevel>=4) fflush(stderr); } } /* init */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ @@ -521,7 +545,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, if (doneMarks[cursor]) { cursor++; continue; } solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel); if (solution.length==0) { cursor++; continue; } - ZDICT_insertDictItem(dictList, dictListSize, solution); + ZDICT_insertDictItem(dictList, dictListSize, solution, buffer); cursor += solution.length; DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100); } } @@ -683,19 +707,19 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, goto _cleanup; } if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */ - for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */ - for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1; - for (u=0; u<=MaxML; u++) matchLengthCount[u]=1; - for (u=0; u<=MaxLL; u++) litLengthCount[u]=1; + for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */ + for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1; + for (u=0; u<=MaxML; u++) matchLengthCount[u] = 1; + for (u=0; u<=MaxLL; u++) litLengthCount[u] = 1; memset(repOffset, 0, sizeof(repOffset)); repOffset[1] = repOffset[4] = repOffset[8] = 1; memset(bestRepOffset, 0, sizeof(bestRepOffset)); - if (compressionLevel==0) compressionLevel=g_compressionLevel_default; + if (compressionLevel==0) compressionLevel = g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); - if (ZSTD_isError(beginResult)) { + if (ZSTD_isError(beginResult)) { + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult)); eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); goto _cleanup; } } @@ -812,7 +836,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); #endif - //dstPtr += 12; eSize += 12; _cleanup: @@ -831,7 +854,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, ZDICT_params_t params) { size_t hSize; -#define HBUFFSIZE 256 +#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */ BYTE header[HBUFFSIZE]; int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; @@ -877,20 +900,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t params) { - size_t hSize; int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; + size_t hSize = 8; - /* dictionary header */ - MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); - { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); - U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; - U32 const dictID = params.dictID ? params.dictID : compliantID; - MEM_writeLE32((char*)dictBuffer+4, dictID); - } - hSize = 8; - - /* entropy tables */ + /* calculate entropy tables */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ DISPLAYLEVEL(2, "statistics ... \n"); { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, @@ -902,6 +916,13 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo hSize += eSize; } + /* add dictionary header (after entropy tables) */ + MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); + { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); + U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; + U32 const dictID = params.dictID ? params.dictID : compliantID; + MEM_writeLE32((char*)dictBuffer+4, dictID); + } if (hSize + dictContentSize < dictBufferCapacity) memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); @@ -929,8 +950,8 @@ size_t ZDICT_trainFromBuffer_unsafe( /* checks */ if (!dictList) return ERROR(memory_allocation); - if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) { free(dictList); return ERROR(dstSize_tooSmall); } - if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return 0; } /* not enough source to create dictionary */ + if (maxDictSize < ZDICT_DICTSIZE_MIN) { free(dictList); return ERROR(dstSize_tooSmall); } /* requested dictionary size is too small */ + if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return ERROR(dictionaryCreation_failed); } /* not enough source to create dictionary */ /* init */ ZDICT_initDictItem(dictList); @@ -963,14 +984,15 @@ size_t ZDICT_trainFromBuffer_unsafe( /* create dictionary */ { U32 dictContentSize = ZDICT_dictSize(dictList); - if (dictContentSize < targetDictSize/3) { + if (dictContentSize < ZDICT_CONTENTSIZE_MIN) { free(dictList); return ERROR(dictionaryCreation_failed); } /* dictionary content too small */ + if (dictContentSize < targetDictSize/4) { DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize); + if (samplesBuffSize < 10 * targetDictSize) + DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20)); if (minRep > MINRATIO) { DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1); DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n"); } - if (samplesBuffSize < 10 * targetDictSize) - DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20)); } if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) { @@ -978,7 +1000,7 @@ size_t ZDICT_trainFromBuffer_unsafe( while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; } DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize); DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity); - DISPLAYLEVEL(2, "! always test dictionary efficiency on samples \n"); + DISPLAYLEVEL(2, "! always test dictionary efficiency on real samples \n"); } /* limit dictionary size */ diff --git a/contrib/zstd/lib/dictBuilder/zdict.h b/contrib/zstd/lib/dictBuilder/zdict.h index 4ead4474fa9b0..9b53de3465276 100644 --- a/contrib/zstd/lib/dictBuilder/zdict.h +++ b/contrib/zstd/lib/dictBuilder/zdict.h @@ -88,7 +88,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dict /*! COVER_params_t : For all values 0 means default. - kMin and d are the only required parameters. + k and d are the only required parameters. */ typedef struct { unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ @@ -147,18 +147,18 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictB Samples must be stored concatenated in a flat buffer `samplesBuffer`, supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. - dictContentSize must be > ZDICT_CONTENTSIZE_MIN bytes. - maxDictSize must be >= dictContentSize, and must be > ZDICT_DICTSIZE_MIN bytes. + dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes. + maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes. @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`), or an error code, which can be tested by ZDICT_isError(). note : ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0. - note 2 : dictBuffer and customDictContent can overlap + note 2 : dictBuffer and dictContent can overlap */ -#define ZDICT_CONTENTSIZE_MIN 256 -#define ZDICT_DICTSIZE_MIN 512 +#define ZDICT_CONTENTSIZE_MIN 128 +#define ZDICT_DICTSIZE_MIN 256 ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, - const void* customDictContent, size_t dictContentSize, + const void* dictContent, size_t dictContentSize, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t parameters); diff --git a/contrib/zstd/lib/legacy/zstd_v01.c b/contrib/zstd/lib/legacy/zstd_v01.c index bcacb8d5d7a28..cf5354d6a9b68 100644 --- a/contrib/zstd/lib/legacy/zstd_v01.c +++ b/contrib/zstd/lib/legacy/zstd_v01.c @@ -1432,7 +1432,7 @@ typedef struct ZSTD_Cctx_s #else U32 hashTable[HASH_TABLESIZE]; #endif - BYTE buffer[WORKPLACESIZE]; + BYTE buffer[WORKPLACESIZE]; } cctxi_t; diff --git a/contrib/zstd/lib/legacy/zstd_v02.c b/contrib/zstd/lib/legacy/zstd_v02.c index 2297b28c8b2e4..3cf8f4778250f 100644 --- a/contrib/zstd/lib/legacy/zstd_v02.c +++ b/contrib/zstd/lib/legacy/zstd_v02.c @@ -475,8 +475,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ - return BIT_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return BIT_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { @@ -1334,8 +1334,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi else { bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } + ip = iend - 4; + } bitStream = MEM_readLE32(ip) >> (bitCount & 31); } } @@ -2040,7 +2040,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } - /* Build rankVal */ + /* Build rankVal */ { const U32 minBits = tableLog+1 - maxW; U32 nextRankVal = 0; @@ -2374,7 +2374,7 @@ static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize) rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } - /* Build rankVal */ + /* Build rankVal */ { const U32 minBits = tableLog+1 - maxW; U32 nextRankVal = 0; @@ -2948,14 +2948,14 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ if (litSize > srcSize-11) /* risk of reading too far with wildcopy */ { - if (litSize > srcSize-3) return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, 8); - return litSize+3; - } - /* direct reference into compressed stream */ + if (litSize > srcSize-3) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); + return litSize+3; + } + /* direct reference into compressed stream */ dctx->litPtr = istart+3; dctx->litSize = litSize; return litSize+3; @@ -3515,13 +3515,13 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi unsigned ZSTDv02_isError(size_t code) { - return ZSTD_isError(code); + return ZSTD_isError(code); } size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize) { - return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); + return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize) @@ -3531,25 +3531,25 @@ size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize) ZSTDv02_Dctx* ZSTDv02_createDCtx(void) { - return (ZSTDv02_Dctx*)ZSTD_createDCtx(); + return (ZSTDv02_Dctx*)ZSTD_createDCtx(); } size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx) { - return ZSTD_freeDCtx((ZSTD_DCtx*)dctx); + return ZSTD_freeDCtx((ZSTD_DCtx*)dctx); } size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx) { - return ZSTD_resetDCtx((ZSTD_DCtx*)dctx); + return ZSTD_resetDCtx((ZSTD_DCtx*)dctx); } size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx) { - return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx); + return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx); } size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize); } diff --git a/contrib/zstd/lib/legacy/zstd_v03.c b/contrib/zstd/lib/legacy/zstd_v03.c index ef654931f5287..f438330a46920 100644 --- a/contrib/zstd/lib/legacy/zstd_v03.c +++ b/contrib/zstd/lib/legacy/zstd_v03.c @@ -477,8 +477,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ - return BIT_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return BIT_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { @@ -1335,8 +1335,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi else { bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } + ip = iend - 4; + } bitStream = MEM_readLE32(ip) >> (bitCount & 31); } } @@ -2037,7 +2037,7 @@ static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ } - /* Build rankVal */ + /* Build rankVal */ { const U32 minBits = tableLog+1 - maxW; U32 nextRankVal = 0; @@ -2589,14 +2589,14 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx, const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */ if (litSize > srcSize-11) /* risk of reading too far with wildcopy */ { - if (litSize > srcSize-3) return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, 8); - return litSize+3; - } - /* direct reference into compressed stream */ + if (litSize > srcSize-3) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, 8); + return litSize+3; + } + /* direct reference into compressed stream */ dctx->litPtr = istart+3; dctx->litSize = litSize; return litSize+3; @@ -3156,13 +3156,13 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi unsigned ZSTDv03_isError(size_t code) { - return ZSTD_isError(code); + return ZSTD_isError(code); } size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize) { - return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); + return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize) @@ -3172,25 +3172,25 @@ size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize) ZSTDv03_Dctx* ZSTDv03_createDCtx(void) { - return (ZSTDv03_Dctx*)ZSTD_createDCtx(); + return (ZSTDv03_Dctx*)ZSTD_createDCtx(); } size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx) { - return ZSTD_freeDCtx((ZSTD_DCtx*)dctx); + return ZSTD_freeDCtx((ZSTD_DCtx*)dctx); } size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx) { - return ZSTD_resetDCtx((ZSTD_DCtx*)dctx); + return ZSTD_resetDCtx((ZSTD_DCtx*)dctx); } size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx) { - return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx); + return ZSTD_nextSrcSizeToDecompress((ZSTD_DCtx*)dctx); } size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize); + return ZSTD_decompressContinue((ZSTD_DCtx*)dctx, dst, maxDstSize, src, srcSize); } diff --git a/contrib/zstd/lib/legacy/zstd_v04.c b/contrib/zstd/lib/legacy/zstd_v04.c index 09040e68ec56e..1a29da92d1e84 100644 --- a/contrib/zstd/lib/legacy/zstd_v04.c +++ b/contrib/zstd/lib/legacy/zstd_v04.c @@ -882,8 +882,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ - return BIT_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return BIT_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { @@ -1451,8 +1451,8 @@ static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsi else { bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } + ip = iend - 4; + } bitStream = MEM_readLE32(ip) >> (bitCount & 31); } } diff --git a/contrib/zstd/lib/legacy/zstd_v05.c b/contrib/zstd/lib/legacy/zstd_v05.c index a6f5f5dbbd16c..674f5b0e4a8e7 100644 --- a/contrib/zstd/lib/legacy/zstd_v05.c +++ b/contrib/zstd/lib/legacy/zstd_v05.c @@ -884,8 +884,8 @@ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits) MEM_STATIC BITv05_DStream_status BITv05_reloadDStream(BITv05_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ - return BITv05_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return BITv05_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { bitD->ptr -= bitD->bitsConsumed >> 3; diff --git a/contrib/zstd/lib/legacy/zstd_v06.c b/contrib/zstd/lib/legacy/zstd_v06.c index a4258b67a6180..ad8c4cd3186d0 100644 --- a/contrib/zstd/lib/legacy/zstd_v06.c +++ b/contrib/zstd/lib/legacy/zstd_v06.c @@ -982,8 +982,8 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, U32 nbBits) if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ - return BITv06_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return BITv06_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { bitD->ptr -= bitD->bitsConsumed >> 3; diff --git a/contrib/zstd/lib/zstd.h b/contrib/zstd/lib/zstd.h index a3237c77eebd8..f8050c1361044 100644 --- a/contrib/zstd/lib/zstd.h +++ b/contrib/zstd/lib/zstd.h @@ -55,8 +55,8 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 4 +#define ZSTD_VERSION_MINOR 2 +#define ZSTD_VERSION_RELEASE 0 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str @@ -71,48 +71,48 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to * Simple API ***************************************/ /*! ZSTD_compress() : - Compresses `src` content as a single zstd compressed frame into already allocated `dst`. - Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - @return : compressed size written into `dst` (<= `dstCapacity), - or an error code if it fails (which can be tested using ZSTD_isError()). */ + * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. + * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); /*! ZSTD_decompress() : - `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. - `dstCapacity` is an upper bound of originalSize. - If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTD_isError()). */ + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); /*! ZSTD_getDecompressedSize() : -* NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. -* ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single -* frame, but distinguishes empty frames from frames with an unknown size, or errors. -* -* Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple -* concatenated frames in one buffer, and so is more general. -* As a result however, it requires more computation and entire frames to be passed to it, -* as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. -* -* 'src' is the start of a zstd compressed frame. -* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. -* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. -* When `return==0`, data to decompress could be any size. -* In which case, it's necessary to use streaming mode to decompress data. -* Optionally, application can still use ZSTD_decompress() while relying on implied limits. -* (For example, data may be necessarily cut into blocks <= 16 KB). -* note 2 : decompressed size is always present when compression is done with ZSTD_compress() -* note 3 : decompressed size can be very large (64-bits value), -* potentially larger than what local system can handle as a single memory segment. -* In which case, it's necessary to use streaming mode to decompress data. -* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. -* Always ensure result fits within application's authorized limits. -* Each application can set its own limits. -* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ + * NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. + * ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single + * frame, but distinguishes empty frames from frames with an unknown size, or errors. + * + * Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple + * concatenated frames in one buffer, and so is more general. + * As a result however, it requires more computation and entire frames to be passed to it, + * as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. + * + * 'src' is the start of a zstd compressed frame. + * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==0`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can still use ZSTD_decompress() while relying on implied limits. + * (For example, data may be necessarily cut into blocks <= 16 KB). + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); @@ -127,29 +127,29 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readab * Explicit memory management ***************************************/ /*= Compression context -* When compressing many times, -* it is recommended to allocate a context just once, and re-use it for each successive compression operation. -* This will make workload friendlier for system's memory. -* Use one context per thread for parallel execution in multi-threaded environments. */ + * When compressing many times, + * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution in multi-threaded environments. */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /*! ZSTD_compressCCtx() : - Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */ + * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); /*= Decompression context -* When decompressing many times, -* it is recommended to allocate a context just once, and re-use it for each successive compression operation. -* This will make workload friendlier for system's memory. -* Use one context per thread for parallel execution in multi-threaded environments. */ + * When decompressing many times, + * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution in multi-threaded environments. */ typedef struct ZSTD_DCtx_s ZSTD_DCtx; ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*! ZSTD_decompressDCtx() : -* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */ + * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -194,9 +194,10 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); /*! ZSTD_compress_usingCDict() : -* Compression using a digested Dictionary. -* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. -* Note that compression level is decided during dictionary creation. */ + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression level is decided during dictionary creation. + * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -487,7 +488,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz /*! ZSTD_createCDict_advanced() : * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference, - ZSTD_parameters params, ZSTD_customMem customMem); + ZSTD_compressionParameters cParams, ZSTD_customMem customMem); /*! ZSTD_sizeof_CDict() : * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ @@ -513,12 +514,19 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : -* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ -ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); +* Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */ +ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : +* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams); /*--- Advanced decompression functions ---*/ @@ -578,7 +586,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); @@ -588,13 +596,22 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /*===== Advanced Streaming compression functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< size of CStream is variable, depending primarily on compression level */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ + +/*! ZSTD_resetCStream() : + * start a new compression job, using same parameters from previous job. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * pledgedSrcSize==0 means "srcSize unknown". + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /*===== Advanced Streaming decompression functions =====*/ @@ -650,8 +667,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize=0 means null-size */ ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */ + ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -745,19 +764,20 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - Compressing and decompressing require a context structure + Use ZSTD_createCCtx() and ZSTD_createDCtx() - It is necessary to init context before starting - + compression : ZSTD_compressBegin() - + decompression : ZSTD_decompressBegin() - + variants _usingDict() are also allowed - + copyCCtx() and copyDCtx() work too - - Block size is limited, it must be <= ZSTD_getBlockSizeMax() - + If you need to compress more, cut data into multiple blocks - + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + + copyCCtx() and copyDCtx() can be used too + - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. In which case, nothing is produced into `dst`. + User must test for such outcome and deal directly with uncompressed data + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! - + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. - Use ZSTD_insertBlock() in such a case. + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. */ #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ diff --git a/contrib/zstd/programs/Makefile b/contrib/zstd/programs/Makefile index 1475cb6109163..0c920a87bcbd8 100644 --- a/contrib/zstd/programs/Makefile +++ b/contrib/zstd/programs/Makefile @@ -18,6 +18,19 @@ ZSTDDIR = ../lib +# Version numbers +LIBVER_SRC := $(ZSTDDIR)/zstd.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) + +ZSTD_VERSION=$(LIBVER) + ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1) ALIGN_LOOP = -falign-loops=32 else @@ -69,10 +82,23 @@ else EXT = endif +VOID = /dev/null + +# thread detection +NO_THREAD_MSG := ==> no threads, building without multithreading support +HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0) +HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0) +ifeq ($(HAVE_THREAD), 1) +THREAD_MSG := ==> building with threading support +THREAD_CPP := -DZSTD_MULTITHREAD +THREAD_LD := -pthread +else +THREAD_MSG := $(NO_THREAD_MSG) +endif + # zlib detection NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support -VOID = /dev/null -HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0) +HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) ZLIB_MSG := ==> building zstd with .gz compression support ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS @@ -80,9 +106,10 @@ ZLIBLD = -lz else ZLIB_MSG := $(NO_ZLIB_MSG) endif + # lzma detection NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support -HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(){}' | $(CC) -o have_lzma -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0) +HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lzma$(EXT) -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0) ifeq ($(HAVE_LZMA), 1) LZMA_MSG := ==> building zstd with .xz/.lzma compression support LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS @@ -91,6 +118,16 @@ else LZMA_MSG := $(NO_LZMA_MSG) endif +# lz4 detection +NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support +HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lz4$(EXT) -x c - -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0) +ifeq ($(HAVE_LZ4), 1) +LZ4_MSG := ==> building zstd with .lz4 compression support +LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS +LZ4LD = -llz4 +else +LZ4_MSG := $(NO_LZ4_MSG) +endif .PHONY: default all clean clean_decomp_o install uninstall generate_res @@ -100,17 +137,20 @@ all: zstd $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) -zstd : CPPFLAGS += $(ZLIBCPP) -zstd : LDFLAGS += $(ZLIBLD) -zstd : LZMA_MSG := $(NO_LZMA_MSG) -zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG) -zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG) -xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP) -xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD) -zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o +zstd xzstd zstd4 xzstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) +zstd xzstd zstd4 xzstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) +xzstd xzstd4 : CPPFLAGS += $(LZMACPP) +xzstd xzstd4 : LDFLAGS += $(LZMALD) +zstd4 xzstd4 : CPPFLAGS += $(LZ4CPP) +zstd4 xzstd4 : LDFLAGS += $(LZ4LD) +zstd zstd4 : LZMA_MSG := - xz/lzma support is disabled +zstd xzstd : LZ4_MSG := - lz4 support is disabled +zstd xzstd zstd4 xzstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +zstd xzstd zstd4 xzstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o + @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" + @echo "$(LZ4_MSG)" ifneq (,$(filter Windows%,$(OS))) windres/generate_res.bat endif @@ -126,10 +166,20 @@ ifneq (,$(filter Windows%,$(OS))) endif $(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT) - zstd-nolegacy : clean_decomp_o $(MAKE) zstd ZSTD_LEGACY_SUPPORT=0 +zstd-nomt : THREAD_CPP := +zstd-nomt : THREAD_LD := +zstd-nomt : THREAD_MSG := - multi-threading disabled +zstd-nomt : zstd + +zstd-nogz : ZLIBCPP := +zstd-nogz : ZLIBLD := +zstd-nogz : ZLIB_MSG := - gzip support is disabled +zstd-nogz : zstd + + zstd-pgo : MOREFLAGS = -fprofile-generate zstd-pgo : clean zstd ./zstd -b19i1 $(PROFILE_WITH) @@ -142,22 +192,18 @@ zstd-pgo : clean zstd $(RM) $(ZSTDDECOMP_O) $(MAKE) zstd MOREFLAGS=-fprofile-use -zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c +# minimal target, with only zstd compression and decompression. no bench. no legacy. +zstd-small: CFLAGS = "-Os -s" +zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o zstd$(EXT) -zstd-small: - CFLAGS="-Os -s" $(MAKE) zstd-frugal - zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT) zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) -zstdmt: CPPFLAGS += -DZSTD_MULTITHREAD -ifeq (,$(filter Windows%,$(OS))) -zstdmt: LDFLAGS += -lpthread -endif +# zstd is now built with Multi-threading by default zstdmt: zstd generate_res: @@ -174,6 +220,19 @@ clean: clean_decomp_o: @$(RM) $(ZSTDDECOMP_O) +MD2ROFF = ronn +MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)" + +zstd.1: zstd.1.md + cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + +man: zstd.1 + +clean-man: + rm zstd.1 + +preview-man: clean-man man + man ./zstd.1 #----------------------------------------------------------------------------- # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets @@ -206,6 +265,7 @@ install: zstd @$(INSTALL_PROGRAM) zstd $(DESTDIR)$(BINDIR)/zstd @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdcat @ln -sf zstd $(DESTDIR)$(BINDIR)/unzstd + @ln -sf zstd $(DESTDIR)$(BINDIR)/zstdmt @$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless @$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep @echo Installing man pages diff --git a/contrib/zstd/programs/README.md b/contrib/zstd/programs/README.md index 203fd7b49bcee..d7922a0969e4c 100644 --- a/contrib/zstd/programs/README.md +++ b/contrib/zstd/programs/README.md @@ -11,8 +11,29 @@ There are however other Makefile targets that create different variations of CLI - `zstd-decompress` : decompressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions +#### Compilation variables +`zstd` tries to detect and use the following features automatically : + +- __HAVE_THREAD__ : multithreading is automatically enabled when `pthread` is detected. + It's possible to disable multithread support, by either compiling `zstd-nomt` target or using HAVE_THREAD=0 variable. + Example : make zstd HAVE_THREAD=0 + It's also possible to force compilation with multithread support, using HAVE_THREAD=1. + In which case, linking stage will fail if `pthread` library cannot be found. + This might be useful to prevent silent feature disabling. + +- __HAVE_ZLIB__ : `zstd` can compress and decompress files in `.gz` format. + This is done through command `--format=gzip`. + Alternatively, symlinks named `gzip` or `gunzip` will mimic intended behavior. + .gz support is automatically enabled when `zlib` library is detected at build time. + It's possible to disable .gz support, by either compiling `zstd-nogz` target or using HAVE_ZLIB=0 variable. + Example : make zstd HAVE_ZLIB=0 + It's also possible to force compilation with zlib support, using HAVE_ZLIB=1. + In which case, linking stage will fail if `zlib` library cannot be found. + This might be useful to prevent silent feature disabling. + + #### Aggregation of parameters -CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. +CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. #### Dictionary builder in Command Line Interface @@ -23,7 +44,7 @@ which can be loaded before compression and decompression. Using a dictionary, the compression ratio achievable on small data improves dramatically. These compression gains are achieved while simultaneously providing faster compression and decompression speeds. -Dictionary work if there is some correlation in a family of small data (there is no universal dictionary). +Dictionary work if there is some correlation in a family of small data (there is no universal dictionary). Hence, deploying one dictionary per type of data will provide the greater benefits. Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will rely more and more on previously decoded content to compress the rest of the file. @@ -35,7 +56,6 @@ Usage of the dictionary builder and created dictionaries with CLI: 3. Decompress with the dictionary: `zstd --decompress FILE.zst -D dictionaryName` - #### Benchmark in Command Line Interface CLI includes in-memory compression benchmark module for zstd. The benchmark is conducted using given filenames. The files are read into memory and joined together. @@ -48,7 +68,6 @@ One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels. - #### Usage of Command Line Interface The full list of options can be obtained with `-h` or `-H` parameter: ``` @@ -62,33 +81,40 @@ Arguments : -d : decompression -D file: use `file` as Dictionary -o file: result stored into `file` (only if 1 input file) - -f : overwrite output without prompting + -f : overwrite output without prompting and (de)compress links --rm : remove source file(s) after successful de/compression -k : preserve source file(s) (default) -h/-H : display help/long help and exit Advanced arguments : -V : display Version number and exit - -v : verbose mode; specify multiple times to increase log level (default:2) + -v : verbose mode; specify multiple times to increase verbosity -q : suppress warnings; specify twice to suppress errors too -c : force write to standard output, even if it is the console - -r : operate recursively on directories --ultra : enable levels beyond 19, up to 22 (requires more memory) + -T# : use # threads for compression (default:1) + -B# : select size of each job (default:0==automatic) --no-dictID : don't write dictID into header (dictionary compression) --[no-]check : integrity check (default:enabled) + -r : operate recursively on directories +--format=gzip : compress files to the .gz format --test : test compressed file integrity ---[no-]sparse : sparse mode (default:enabled on file, disabled on stdout) +--[no-]sparse : sparse mode (default:disabled) + -M# : Set a memory usage limit for decompression +-- : All arguments after "--" are treated as files Dictionary builder : --train ## : create a dictionary from a training set of files +--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args +--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) -o file : `file` is dictionary name (default: dictionary) ---maxdict ## : limit dictionary to specified size (default : 112640) - -s# : dictionary selectivity level (default: 9) ---dictID ## : force dictionary ID to specified value (default: random) +--maxdict=# : limit dictionary to specified size (default : 112640) +--dictID=# : force dictionary ID to specified value (default: random) Benchmark arguments : -b# : benchmark file(s), using # compression level (default : 1) -e# : test all compression levels from -bX to # (default: 1) -i# : minimum evaluation time in seconds (default : 3s) -B# : cut file into independent blocks of size # (default: no block) - ```
\ No newline at end of file +--priority=rt : set process priority to real-time +``` diff --git a/contrib/zstd/programs/bench.c b/contrib/zstd/programs/bench.c index 2dd1cfb0fab10..22b871952b8e4 100644 --- a/contrib/zstd/programs/bench.c +++ b/contrib/zstd/programs/bench.c @@ -70,12 +70,12 @@ static U32 g_compressibilityDefault = 50; ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; @@ -89,7 +89,7 @@ static clock_t g_time = 0; #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); #define EXM_THROW(error, ...) \ { \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "Error %i : ", error); \ DISPLAYLEVEL(1, __VA_ARGS__); \ DISPLAYLEVEL(1, " \n"); \ @@ -146,17 +146,20 @@ typedef struct { } blockParam_t; -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +#undef MIN +#undef MAX +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *comprParams) + const ZSTD_compressionParameters* comprParams) { size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; - size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); + size_t const avgSize = MIN(blockSize, (srcSize / nbFiles)); U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ @@ -176,22 +179,21 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, EXM_THROW(31, "allocation error : not enough memory"); /* init */ - if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* display last 17 characters */ UTIL_initTimer(&ticksPerSecond); - if (g_decodeOnly) { - const char* srcPtr = (const char*) srcBuffer; - U64 dSize64 = 0; + if (g_decodeOnly) { /* benchmark only decompression : source must be already compressed */ + const char* srcPtr = (const char*)srcBuffer; + U64 totalDSize64 = 0; U32 fileNb; for (fileNb=0; fileNb<nbFiles; fileNb++) { U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]); if (fSize64==0) EXM_THROW(32, "Impossible to determine original size "); - dSize64 += fSize64; + totalDSize64 += fSize64; srcPtr += fileSizes[fileNb]; } - { size_t const decodedSize = (size_t)dSize64; - if (dSize64 > decodedSize) EXM_THROW(32, "original size is too large"); - if (decodedSize==0) EXM_THROW(32, "Impossible to determine original size "); + { size_t const decodedSize = (size_t)totalDSize64; + if (totalDSize64 > decodedSize) EXM_THROW(32, "original size is too large"); /* size_t overflow */ free(resultBuffer); resultBuffer = malloc(decodedSize); if (!resultBuffer) EXM_THROW(33, "not enough memory"); @@ -260,12 +262,11 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_getTime(&clockStart); if (!cCompleted) { /* still some time to do compression tests */ - ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_customMem const cmem = { NULL, NULL, NULL }; - U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; + U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem); - if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); + ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); + ZSTD_CDict* cdict; if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog; if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog; @@ -273,6 +274,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); + cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); do { U32 blockNb; size_t rSize; diff --git a/contrib/zstd/programs/dibio.c b/contrib/zstd/programs/dibio.c index 5ef202c8abad5..aac36425cf752 100644 --- a/contrib/zstd/programs/dibio.c +++ b/contrib/zstd/programs/dibio.c @@ -53,12 +53,12 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ +static int g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((DIB_clockSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10; static clock_t g_time = 0; @@ -89,7 +89,8 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#undef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) /* ******************************************************** diff --git a/contrib/zstd/programs/fileio.c b/contrib/zstd/programs/fileio.c index cabb6ec524a2a..33850cb4dca46 100644 --- a/contrib/zstd/programs/fileio.c +++ b/contrib/zstd/programs/fileio.c @@ -31,6 +31,11 @@ #include <time.h> /* clock */ #include <errno.h> /* errno */ +#if defined (_MSC_VER) +# include <sys/stat.h> +# include <io.h> +#endif + #include "mem.h" #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ @@ -48,6 +53,12 @@ # include <lzma.h> #endif +#define LZ4_MAGICNUMBER 0x184D2204 +#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) +# include <lz4frame.h> +# include <lz4.h> +#endif + /*-************************************* * Constants @@ -71,7 +82,7 @@ #define CACHELINE 64 -#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */ +#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ #define FNSPACE 30 @@ -81,18 +92,20 @@ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } #define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } } + if (g_displayLevel>=4) fflush(stderr); } } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; +#undef MIN #define MIN(a,b) ((a) < (b) ? (a) : (b)) + /* ************************************************************ * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW ***************************************************************/ @@ -188,6 +201,18 @@ void FIO_setOverlapLog(unsigned overlapLog){ /*-************************************* * Functions ***************************************/ +/** FIO_remove() : + * @result : Unlink `fileName`, even if it's read-only */ +static int FIO_remove(const char* path) +{ +#if defined(_WIN32) || defined(WIN32) + /* windows doesn't allow remove read-only files, so try to make it + * writable first */ + chmod(path, _S_IWRITE); +#endif + return remove(path); +} + /** FIO_openSrcFile() : * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ @@ -227,23 +252,32 @@ static FILE* FIO_openDstFile(const char* dstFileName) DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); } } else { - if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ + if (g_sparseFileSupport == 1) { + g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; + } + if (strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ f = fopen( dstFileName, "rb" ); if (f != 0) { /* dest file exists, prompt for overwrite authorization */ fclose(f); - if (g_displayLevel <= 1) { - /* No interaction possible */ - DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName); - return NULL; - } - DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName); - { int ch = getchar(); - if ((ch!='Y') && (ch!='y')) { - DISPLAY(" not overwritten \n"); + if (!g_overwrite) { + if (g_displayLevel <= 1) { + /* No interaction possible */ + DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName); return NULL; } - while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ - } } } + DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName); + { int ch = getchar(); + if ((ch!='Y') && (ch!='y')) { + DISPLAY(" not overwritten \n"); + return NULL; + } + while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ + } + } + + /* need to unlink */ + FIO_remove(dstFileName); + } } f = fopen( dstFileName, "wb" ); if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } @@ -252,13 +286,13 @@ static FILE* FIO_openDstFile(const char* dstFileName) } -/*! FIO_loadFile() : -* creates a buffer, pointed by `*bufferPtr`, -* loads `filename` content into it, -* up to MAX_DICT_SIZE bytes. -* @return : loaded size -*/ -static size_t FIO_loadFile(void** bufferPtr, const char* fileName) +/*! FIO_createDictBuffer() : + * creates a buffer, pointed by `*bufferPtr`, + * loads `filename` content into it, up to DICTSIZE_MAX bytes. + * @return : loaded size + * if fileName==NULL, returns 0 and a NULL pointer + */ +static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) { FILE* fileHandle; U64 fileSize; @@ -270,14 +304,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName) fileHandle = fopen(fileName, "rb"); if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno)); fileSize = UTIL_getFileSize(fileName); - if (fileSize > MAX_DICT_SIZE) { - int seekResult; - if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */ - DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, (U32)MAX_DICT_SIZE); - seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */ - if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno)); - fileSize = MAX_DICT_SIZE; - } + if (fileSize > DICTSIZE_MAX) EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */ *bufferPtr = malloc((size_t)fileSize); if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno)); { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); @@ -330,7 +357,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, /* dictionary */ { void* dictBuffer; - size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName); + size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer"); { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); params.fParams.contentSizeFlag = srcRegFile; @@ -342,7 +369,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); + if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); /* 0 means : do not change */ #ifdef ZSTD_MULTITHREAD { size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); @@ -494,6 +521,84 @@ static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, const char* srcFi } #endif +#ifdef ZSTD_LZ4COMPRESS +static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } +static unsigned long long FIO_compressLz4Frame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + + LZ4F_preferences_t prefs; + LZ4F_compressionContext_t ctx; + + LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(31, "zstd: failed to create lz4 compression context"); + + memset(&prefs, 0, sizeof(prefs)); + +#if LZ4_VERSION_NUMBER <= 10600 +#define LZ4F_blockIndependent blockIndependent +#define LZ4F_max4MB max4MB +#endif + + prefs.autoFlush = 1; + prefs.compressionLevel = compressionLevel; + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* stick to defaults for lz4 cli */ + prefs.frameInfo.blockSizeID = LZ4F_max4MB; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; +#if LZ4_VERSION_NUMBER >= 10600 + prefs.frameInfo.contentSize = srcFileSize; +#endif + + { + size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max4MB); + size_t readSize; + size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); + if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); + if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } + outFileSize += headerSize; + + /* Read first block */ + readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); + inFileSize += readSize; + + /* Main Loop */ + while (readSize>0) { + size_t outSize; + + /* Compress Block */ + outSize = LZ4F_compressUpdate(ctx, ress->dstBuffer, ress->dstBufferSize, ress->srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", srcFileName, LZ4F_getErrorName(outSize)); + outFileSize += outSize; + if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100); + + /* Write Block */ + { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); + if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } + + /* Read next block */ + readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); + inFileSize += readSize; + } + if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); + + /* End of Stream mark */ + headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); + if (LZ4F_isError(headerSize)) EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", srcFileName, LZ4F_getErrorName(headerSize)); + + { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); + if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } + outFileSize += headerSize; + } + + *readsize = inFileSize; + LZ4F_freeCompressionContext(ctx); + + return outFileSize; +} +#endif + /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. @@ -512,6 +617,7 @@ static int FIO_compressFilename_internal(cRess_t ress, switch (g_compressionType) { case FIO_zstdCompression: break; + case FIO_gzipCompression: #ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); @@ -520,6 +626,7 @@ static int FIO_compressFilename_internal(cRess_t ress, EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); #endif goto finish; + case FIO_xzCompression: case FIO_lzmaCompression: #ifdef ZSTD_LZMACOMPRESS @@ -529,6 +636,15 @@ static int FIO_compressFilename_internal(cRess_t ress, EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName); #endif goto finish; + + case FIO_lz4Compression: +#ifdef ZSTD_LZ4COMPRESS + compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", srcFileName); +#endif + goto finish; } /* init */ @@ -548,8 +664,8 @@ static int FIO_compressFilename_internal(cRess_t ress, readsize += inSize; { ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; - while (inBuff.pos != inBuff.size) { /* note : is there any possibility of endless loop ? for example, if outBuff is not large enough ? */ - ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 }; + while (inBuff.pos != inBuff.size) { + ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; #ifdef ZSTD_MULTITHREAD size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff); #else @@ -563,13 +679,13 @@ static int FIO_compressFilename_internal(cRess_t ress, if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); compressedfilesize += outBuff.pos; } } } -#ifdef ZSTD_MULTITHREAD - if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20)) - else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20)); -#else - if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100) - else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100); -#endif + if (g_nbThreads > 1) { + if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20)) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20)); + } else { + if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100); + } } /* End of Frame */ @@ -750,7 +866,7 @@ static dRess_t FIO_createDResources(const char* dictFileName) /* dictionary */ { void* dictBuffer; - size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName); + size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName); size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize); if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError)); free(dictBuffer); @@ -1019,6 +1135,66 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, } #endif +#ifdef ZSTD_LZ4DECOMPRESS +static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, const char* srcFileName) +{ + unsigned long long filesize = 0; + LZ4F_errorCode_t nextToLoad; + LZ4F_decompressionContext_t dCtx; + LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); + + if (LZ4F_isError(errorCode)) EXM_THROW(61, "zstd: failed to create lz4 decompression context"); + + /* Init feed with magic number (already consumed from FILE* sFile) */ + { size_t inSize = 4; + size_t outSize= 0; + MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); + nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "zstd: %s: lz4 header error : %s", srcFileName, LZ4F_getErrorName(nextToLoad)); + } + + /* Main Loop */ + for (;nextToLoad;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = ress->dstBufferSize; + + /* Read input */ + if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; + readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ + /* Decode Input (at least partially) */ + size_t remaining = readSize - pos; + decodedBytes = ress->dstBufferSize; + nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "zstd: %s: decompression error : %s", srcFileName, LZ4F_getErrorName(nextToLoad)); + pos += remaining; + + /* Write Block */ + if (decodedBytes) { + if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) EXM_THROW(63, "Write error : cannot write to output file"); + filesize += decodedBytes; + DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + } + + if (!nextToLoad) break; + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) EXM_THROW(67, "zstd: %s: read error", srcFileName); + + if (nextToLoad!=0) EXM_THROW(68, "zstd: %s: unfinished stream", srcFileName); + + LZ4F_freeDecompressionContext(dCtx); + ress->srcBufferLoaded = 0; /* LZ4F will go to the frame boundary */ + + return filesize; +} +#endif + + /** FIO_decompressSrcFile() : Decompression `srcFileName` into `ress.dstFile` @@ -1071,6 +1247,15 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName); return 1; #endif + } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { +#ifdef ZSTD_LZ4DECOMPRESS + unsigned long long const result = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); + if (result == 0) return 1; + filesize += result; +#else + DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without ZSTD_LZ4DECOMPRESS) -- ignored \n", srcFileName); + return 1; +#endif } else { if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ @@ -1179,7 +1364,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles dstFileName = (char*)malloc(dfnSize); if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName"); } - if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION))) { + if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION) && strcmp(suffixPtr, LZ4_EXTENSION))) { DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION); skippedFiles++; continue; diff --git a/contrib/zstd/programs/fileio.h b/contrib/zstd/programs/fileio.h index 0dd58d625d44a..65da98d7fa88a 100644 --- a/contrib/zstd/programs/fileio.h +++ b/contrib/zstd/programs/fileio.h @@ -33,12 +33,13 @@ extern "C" { #define XZ_EXTENSION ".xz" #define GZ_EXTENSION ".gz" #define ZSTD_EXTENSION ".zst" +#define LZ4_EXTENSION ".lz4" /*-************************************* * Types ***************************************/ -typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression } FIO_compressionType_t; +typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t; /*-************************************* diff --git a/contrib/zstd/programs/platform.h b/contrib/zstd/programs/platform.h index 89a9f6cd42a01..74412cde332ea 100644 --- a/contrib/zstd/programs/platform.h +++ b/contrib/zstd/programs/platform.h @@ -100,9 +100,18 @@ extern "C" { #if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) # include <unistd.h> /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__) # include <io.h> /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#elif defined(WIN32) || defined(_WIN32) +# include <io.h> /* _isatty */ +# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include <stdio.h> /* FILE */ +static __inline int IS_CONSOLE(FILE* stdStream) +{ + DWORD dummy; + return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); +} #else # define IS_CONSOLE(stdStream) 0 #endif @@ -129,6 +138,14 @@ extern "C" { #endif +#ifndef ZSTD_SPARSE_DEFAULT +# if (defined(__APPLE__) && defined(__MACH__)) +# define ZSTD_SPARSE_DEFAULT 0 +# else +# define ZSTD_SPARSE_DEFAULT 1 +# endif +#endif + #if defined (__cplusplus) } diff --git a/contrib/zstd/programs/util.h b/contrib/zstd/programs/util.h index 59e19d027ccd2..5f437b2b268c4 100644 --- a/contrib/zstd/programs/util.h +++ b/contrib/zstd/programs/util.h @@ -1,6 +1,6 @@ /** * util.h - utility functions - * + * * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. * All rights reserved. * @@ -25,6 +25,7 @@ extern "C" { #include <stdlib.h> /* malloc */ #include <stddef.h> /* size_t, ptrdiff_t */ #include <stdio.h> /* fprintf */ +#include <string.h> /* strncmp */ #include <sys/types.h> /* stat, utime */ #include <sys/stat.h> /* stat */ #if defined(_MSC_VER) @@ -166,8 +167,8 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_freq_t ticksPerSecond) * File functions ******************************************/ #if defined(_MSC_VER) - #define chmod _chmod - typedef struct __stat64 stat_t; + #define chmod _chmod + typedef struct __stat64 stat_t; #else typedef struct stat stat_t; #endif @@ -178,9 +179,9 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) int res = 0; struct utimbuf timebuf; - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); /* set access and modification times */ #if !defined(_WIN32) res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ @@ -228,6 +229,20 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) return 0; } +UTIL_STATIC U32 UTIL_isLink(const char* infilename) +{ +#if defined(_WIN32) + /* no symlinks on windows */ + (void)infilename; +#else + int r; + stat_t statbuf; + r = lstat(infilename, &statbuf); + if (!r && S_ISLNK(statbuf.st_mode)) return 1; +#endif + return 0; +} + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { @@ -271,11 +286,14 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size) return NULL; } +static int g_utilDisplayLevel; +#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } #ifdef _WIN32 # define UTIL_HAS_CREATEFILELIST -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; @@ -311,7 +329,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ if (strcmp (cFile.cFileName, "..") == 0 || strcmp (cFile.cFileName, ".") == 0) continue; - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ + nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } } else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) { @@ -339,7 +357,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ # include <dirent.h> /* opendir, readdir */ # include <string.h> /* strerror, memcpy */ -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { DIR *dir; struct dirent *entry; @@ -360,13 +378,19 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { closedir(dir); return 0; } memcpy(path, dirName, dirLength); + path[dirLength] = '/'; memcpy(path+dirLength+1, entry->d_name, fnameLength); pathLength = dirLength+1+fnameLength; path[pathLength] = 0; + if (!followLinks && UTIL_isLink(path)) { + UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); + continue; + } + if (UTIL_isDirectory(path)) { - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ + nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ if (*bufStart == NULL) { free(path); closedir(dir); return 0; } } else { if (*bufStart + *pos + pathLength >= *bufEnd) { @@ -396,7 +420,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #else -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks) { (void)bufStart; (void)bufEnd; (void)pos; fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); @@ -411,7 +435,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) +UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb, int followLinks) { size_t pos; unsigned i, nbFiles; @@ -436,7 +460,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i nbFiles++; } } else { - nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); + nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); if (buf == NULL) return NULL; } } @@ -465,6 +489,201 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu if (filenameTable) free((void*)filenameTable); } +/* count the number of physical cores */ +#if defined(_WIN32) || defined(WIN32) + +#include <windows.h> + +typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); + +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores = 0; + if (numPhysicalCores != 0) return numPhysicalCores; + + { LPFN_GLPI glpi; + BOOL done = FALSE; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; + DWORD returnLength = 0; + size_t byteOffset = 0; + + glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), + "GetLogicalProcessorInformation"); + + if (glpi == NULL) { + goto failed; + } + + while(!done) { + DWORD rc = glpi(buffer, &returnLength); + if (FALSE == rc) { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + if (buffer) + free(buffer); + buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); + + if (buffer == NULL) { + perror("zstd"); + exit(1); + } + } else { + /* some other error */ + goto failed; + } + } else { + done = TRUE; + } + } + + ptr = buffer; + + while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { + + if (ptr->Relationship == RelationProcessorCore) { + numPhysicalCores++; + } + + ptr++; + byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + } + + free(buffer); + + return numPhysicalCores; + } + +failed: + /* try to fall back on GetSystemInfo */ + { SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + numPhysicalCores = sysinfo.dwNumberOfProcessors; + if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */ + } + return numPhysicalCores; +} + +#elif defined(__APPLE__) + +#include <sys/sysctl.h> + +/* Use apple-provided syscall + * see: man 3 sysctl */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static S32 numPhysicalCores = 0; /* apple specifies int32_t */ + if (numPhysicalCores != 0) return numPhysicalCores; + + { size_t size = sizeof(S32); + int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0); + if (ret != 0) { + if (errno == ENOENT) { + /* entry not present, fall back on 1 */ + numPhysicalCores = 1; + } else { + perror("zstd: can't get number of physical cpus"); + exit(1); + } + } + + return numPhysicalCores; + } +} + +#elif defined(__linux__) + +/* parse /proc/cpuinfo + * siblings / cpu cores should give hyperthreading ratio + * otherwise fall back on sysconf */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores = 0; + + if (numPhysicalCores != 0) return numPhysicalCores; + + numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (numPhysicalCores == -1) { + /* value not queryable, fall back on 1 */ + return numPhysicalCores = 1; + } + + /* try to determine if there's hyperthreading */ + { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); + size_t const BUF_SIZE = 80; + char buff[BUF_SIZE]; + + int siblings = 0; + int cpu_cores = 0; + int ratio = 1; + + if (cpuinfo == NULL) { + /* fall back on the sysconf value */ + return numPhysicalCores; + } + + /* assume the cpu cores/siblings values will be constant across all + * present processors */ + while (!feof(cpuinfo)) { + if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { + if (strncmp(buff, "siblings", 8) == 0) { + const char* const sep = strchr(buff, ':'); + if (*sep == '\0') { + /* formatting was broken? */ + goto failed; + } + + siblings = atoi(sep + 1); + } + if (strncmp(buff, "cpu cores", 9) == 0) { + const char* const sep = strchr(buff, ':'); + if (*sep == '\0') { + /* formatting was broken? */ + goto failed; + } + + cpu_cores = atoi(sep + 1); + } + } else if (ferror(cpuinfo)) { + /* fall back on the sysconf value */ + goto failed; + } + } + if (siblings && cpu_cores) { + ratio = siblings / cpu_cores; + } +failed: + fclose(cpuinfo); + return numPhysicalCores = numPhysicalCores / ratio; + } +} + +#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) + +/* Use apple-provided syscall + * see: man 3 sysctl */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores = 0; + + if (numPhysicalCores != 0) return numPhysicalCores; + + numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (numPhysicalCores == -1) { + /* value not queryable, fall back on 1 */ + return numPhysicalCores = 1; + } + return numPhysicalCores; +} + +#else + +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + /* assume 1 */ + return 1; +} + +#endif #if defined (__cplusplus) } diff --git a/contrib/zstd/programs/zstd.1 b/contrib/zstd/programs/zstd.1 index 684fb868aa309..6cc5f7e9d2f62 100644 --- a/contrib/zstd/programs/zstd.1 +++ b/contrib/zstd/programs/zstd.1 @@ -1,408 +1,334 @@ -\" -\" zstd.1: This is a manual page for 'zstd' program. This file is part of the -\" zstd <http://www.zstd.net/> project. -\" Author: Yann Collet -\" - -\" No hyphenation -.hy 0 -.nr HY 0 - -.TH zstd "1" "2015-08-22" "zstd" "User Commands" -.SH NAME -\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files - -.SH SYNOPSIS -.TP 5 -\fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] [-o <OUTPUT-FILE>] -.PP -.B unzstd -is equivalent to -.BR "zstd \-d" -.br -.B zstdcat -is equivalent to -.BR "zstd \-dcf" -.br - -.SH DESCRIPTION -.PP -\fBzstd\fR is a fast lossless compression algorithm -and data compression tool, -with command line syntax similar to \fB gzip (1) \fR and \fB xz (1) \fR . -It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages. -\fBzstd\fR offers highly configurable compression speed, -with fast modes at > 200 MB/s per core, -and strong modes nearing lzma compression ratios. -It also features a very fast decoder, with speeds > 500 MB/s per core. - -\fBzstd\fR command line syntax is generally similar to gzip, -but features the following differences : - - Source files are preserved by default. - It's possible to remove them automatically by using \fB--rm\fR command. - - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default. - Use \fB-q\fR to turn them off - -.PP -.B zstd -compresses or decompresses each -.I file -according to the selected operation mode. -If no -.I files -are given or -.I file -is -.BR \- , -.B zstd -reads from standard input and writes the processed data -to standard output. -.B zstd -will refuse (display an error and skip the -.IR file ) -to write compressed data to standard output if it is a terminal. -Similarly, -.B zstd -will refuse to read compressed data -from standard input if it is a terminal. - -.PP -Unless -.B \-\-stdout -or -.B \-o -is specified, -.I files -are written to a new file whose name is derived from the source -.I file -name: -.IP \(bu 3 -When compressing, the suffix -.B .zst -is appended to the source filename to get the target filename. -.IP \(bu 3 -When decompressing, the -.B .zst -suffix is removed from the filename to get the target filename. - -.SS "Concatenation with .zst files" -It is possible to concatenate -.B .zst -files as is. -.B zstd -will decompress such files as if they were a single -.B .zst -file. - - - -.SH OPTIONS - +. +.TH "ZSTD" "1" "May 2017" "zstd 1.2.0" "User Commands" +. +.SH "NAME" +\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files +. +.SH "SYNOPSIS" +\fBzstd\fR [\fIOPTIONS\fR] [\-|\fIINPUT\-FILE\fR] [\-o \fIOUTPUT\-FILE\fR] +. +.P +\fBzstdmt\fR is equivalent to \fBzstd \-T0\fR +. +.P +\fBunzstd\fR is equivalent to \fBzstd \-d\fR +. +.P +\fBzstdcat\fR is equivalent to \fBzstd \-dcf\fR +. +.SH "DESCRIPTION" +\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip (1)\fR and \fBxz (1)\fR\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, with fast modes at > 200 MB/s per code, and strong modes nearing lzma compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\. +. +.P +\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences : +. +.IP "\(bu" 4 +Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\. +. +.IP "\(bu" 4 +When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default\. Use \fB\-q\fR to turn them off\. +. +.IP "\(bu" 4 +\fBzstd\fR does not accept input from console, but it properly accepts \fBstdin\fR when it\'s not the console\. +. +.IP "\(bu" 4 +\fBzstd\fR displays a short help page when command line is an error\. Use \fB\-q\fR to turn it off\. +. +.IP "" 0 +. +.P +\fBzstd\fR compresses or decompresses each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal : it will display an error message and skip the \fIfile\fR\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\. +. +.P +Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name: +. +.IP "\(bu" 4 +When compressing, the suffix \fB\.zst\fR is appended to the source filename to get the target filename\. +. +.IP "\(bu" 4 +When decompressing, the \fB\.zst\fR suffix is removed from the source filename to get the target filename +. +.IP "" 0 +. +.SS "Concatenation with \.zst files" +It is possible to concatenate \fB\.zst\fR files as is\. \fBzstd\fR will decompress such files as if they were a single \fB\.zst\fR file\. +. +.SH "OPTIONS" . .SS "Integer suffixes and special values" -In most places where an integer argument is expected, -an optional suffix is supported to easily indicate large integers. -There must be no space between the integer and the suffix. +In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\. +. .TP -.B KiB -Multiply the integer by 1,024 (2^10). -.BR Ki , -.BR K , -and -.B KB -are accepted as synonyms for -.BR KiB . +\fBKiB\fR +Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\. +. .TP -.B MiB -Multiply the integer by 1,048,576 (2^20). -.BR Mi , -.BR M , -and -.B MB -are accepted as synonyms for -.BR MiB . - +\fBMiB\fR +Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\. . .SS "Operation mode" -If multiple operation mode options are given, -the last one takes effect. +If multiple operation mode options are given, the last one takes effect\. +. .TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode when no operation mode option -is specified and no other operation mode is implied from -the command name (for example, -.B unzstd -implies -.BR \-\-decompress ). +\fB\-z\fR, \fB\-\-compress\fR +Compress\. This is the default operation mode when no operation mode option is specified and no other operation mode is implied from the command name (for example, \fBunzstd\fR implies \fB\-\-decompress\fR)\. +. .TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. +\fB\-d\fR, \fB\-\-decompress\fR, \fB\-\-uncompress\fR +Decompress\. +. .TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.IR files . -This option is equivalent to -.B "\-\-decompress \-\-stdout" -except that the decompressed data is discarded instead of being -written to standard output. -No files are created or removed. +\fB\-t\fR, \fB\-\-test\fR +Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\-\-decompress \-\-stdout\fR except that the decompressed data is discarded instead of being written to standard output\. No files are created or removed\. +. .TP -.B \-b# - benchmark file(s) using compression level # +\fB\-b#\fR +Benchmark file(s) using compression level # +. .TP -.B \--train FILEs - use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100). - +\fB\-\-train FILEs\fR +Use FILEs as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\. . .SS "Operation modifiers" +. .TP -.B \-# - # compression level [1-19] (default:3) -.TP -.BR \--ultra - unlocks high compression levels 20+ (maximum 22), using a lot more memory. -Note that decompression will also require more memory when using these levels. -.TP -.B \-D file - use `file` as Dictionary to compress or decompress FILE(s) -.TP -.BR \--no-dictID - do not store dictionary ID within frame header (dictionary compression). - The decoder will have to rely on implicit knowledge about which dictionary to use, -it won't be able to check if it's correct. -.TP -.B \-o file - save result into `file` (only possible with a single INPUT-FILE) -.TP -.BR \-f ", " --force - overwrite output without prompting -.TP -.BR \-c ", " --stdout - force write to standard output, even if it is the console -.TP -.BR \--[no-]sparse - enable / disable sparse FS support, to make files with many zeroes smaller on disk. - Creating sparse files may save disk space and speed up the decompression -by reducing the amount of disk I/O. - default : enabled when output is into a file, and disabled when output is stdout. - This setting overrides default and can force sparse mode over stdout. -.TP -.BR \--rm - remove source file(s) after successful compression or decompression +\fB\-#\fR +\fB#\fR compression level [1\-19] (default: 3) +. .TP -.BR \-k ", " --keep - keep source file(s) after successful compression or decompression. - This is the default behavior. +\fB\-\-ultra\fR +unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\. +. .TP -.BR \-r - operate recursively on directories +\fB\-T#\fR, \fB\-\-threads=#\fR +Compress using \fB#\fR threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\. +. .TP -.BR \-h/\-H ", " --help - display help/long help and exit +\fB\-D file\fR +use \fBfile\fR as Dictionary to compress or decompress FILE(s) +. .TP -.BR \-V ", " --version - display Version number and exit +\fB\-\-nodictID\fR +do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\. +. .TP -.BR \-v ", " --verbose - verbose mode +\fB\-o file\fR +save result into \fBfile\fR (only possible with a single \fIINPUT\-FILE\fR) +. .TP -.BR \-q ", " --quiet - suppress warnings, interactivity and notifications. - specify twice to suppress errors too. +\fB\-f\fR, \fB\-\-force\fR +overwrite output without prompting, and (de)compress symbolic links +. .TP -.BR \-C ", " --[no-]check - add integrity check computed from uncompressed data (default : enabled) +\fB\-c\fR, \fB\-\-stdout\fR +force write to standard output, even if it is the console +. .TP -.BR \-t ", " --test - Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR. - No files are created or removed. +\fB\-\-[no\-]sparse\fR +enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default : enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\. +. .TP -.BR -- - All arguments after -- are treated as files - - -.SH DICTIONARY BUILDER -.PP -\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages. -It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`. -Then during compression and decompression, make reference to the same dictionary. -It will improve compression ratio of small files. -Typical gains range from ~10% (at 64KB) to x5 better (at <1KB). +\fB\-\-rm\fR +remove source file(s) after successful compression or decompression +. .TP -.B \--train FILEs - use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100), -and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary) +\fB\-k\fR, \fB\-\-keep\fR +keep source file(s) after successful compression or decompression\. This is the default behavior\. +. .TP -.B \-o file - dictionary saved into `file` (default: dictionary) +\fB\-r\fR +operate recursively on dictionaries +. .TP -.B \--maxdict # - limit dictionary to specified size (default : 112640) +\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR +display help/long help and exit +. .TP -.B \--dictID # - A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary. - By default, zstd will create a 4-bytes random number ID. - It's possible to give a precise number instead. - Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, - and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default. - However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries. +\fB\-V\fR, \fB\-\-version\fR +display version number and exit +. .TP -.B \-s# - dictionary selectivity level (default: 9) - the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size. +\fB\-v\fR +verbose mode +. .TP -.B \--cover=k=#,d=# - Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR. - Selects segments of size \fIk\fR with the highest score to put in the dictionary. - The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR. - Generally \fId\fR should be in the range [6, 24]. - Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048]. - Example: \fB--train --cover=k=64,d=8 FILEs\fR. +\fB\-q\fR, \fB\-\-quiet\fR +suppress warnings, interactivity, and notifications\. specify twice to suppress errors too\. +. .TP -.B \--optimize-cover[=steps=#,k=#,d=#] - If \fIsteps\fR is not specified, the default value of 32 is used. - If \fIk\fR is not specified, \fIsteps\fR values in [16, 2048] are checked for each value of \fId\fR. - If \fId\fR is not specified, the values checked are [6, 8, ..., 16]. - - Runs the cover dictionary builder for each parameter set saves the optimal parameters and dictionary. - Prints the optimal parameters and writes the optimal dictionary to the output file. - Supports multithreading if \fBzstd\fR is compiled with threading support. - - The parameter \fIk\fR is more sensitve than \fId\fR, and is faster to optimize over. - Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set. - Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024]). - \fBzstd --train --optimize-cover FILEs - \fBzstd --train --optimize-cover=d=d,steps=512 FILEs +\fB\-C\fR, \fB\-\-[no\-]check\fR +add integrity check computed from uncompressed data (default : enabled) +. .TP - -.SH BENCHMARK +\fB\-\-\fR +All arguments after \fB\-\-\fR are treated as files +. +.SH "DICTIONARY BUILDER" +\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages\. It\'s possible to train \fBzstd\fR with some samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary\. It will improve compression ratio of small files\. Typical gains range from 10% (at 64KB) to x5 better (at <1KB)\. +. .TP -.B \-b# - benchmark file(s) using compression level # +\fB\-\-train FILEs\fR +Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\. +. +.IP +Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-cover=d=8,steps=4\fR\. +. .TP -.B \-e# - benchmark file(s) using multiple compression levels, from -b# to -e# (included). +\fB\-o file\fR +Dictionary saved into \fBfile\fR (default name: dictionary)\. +. .TP -.B \-i# - minimum evaluation time, in seconds (default : 3s), benchmark mode only +\fB\-\-maxdict=#\fR +Limit dictionary to specified size (default: 112640)\. +. .TP -.B \-B# - cut file into independent blocks of size # (default: no block) -.B \--priority=rt - set process priority to real-time - -.SH ADVANCED COMPRESSION OPTIONS +\fB\-\-dictID=#\fR +A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. +. .TP -.B \--zstd[=\fIoptions\fR] -.PD -\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options. -The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level. -The list of available \fIoptions\fR: -.RS - +\fB\-\-train\-cover[=k#,d=#,steps=#]\fR +Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. Requires that \fId\fR <= \fIk\fR\. +. +.IP +Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\. +. +.IP +Examples: +. +.IP +\fBzstd \-\-train\-cover FILEs\fR +. +.IP +\fBzstd \-\-train\-cover=k=50,d=8 FILEs\fR +. +.IP +\fBzstd \-\-train\-cover=d=8,steps=500 FILEs\fR +. +.IP +\fBzstd \-\-train\-cover=k=50 FILEs\fR +. .TP -.BI strategy= strat -.PD 0 +\fB\-\-train\-legacy[=selectivity=#]\fR +Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\. +. +.IP +Examples: +. +.IP +\fBzstd \-\-train\-legacy FILEs\fR +. +.IP +\fBzstd \-\-train\-legacy=selectivity=8 FILEs\fR +. +.SH "BENCHMARK" +. .TP -.BI strat= strat -.PD -Specify a strategy used by a match finder. -.IP "" -There are 8 strategies numbered from 0 to 7, from faster to stronger: -0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2. -.IP "" - +\fB\-b#\fR +benchmark file(s) using compression level # +. .TP -.BI windowLog= wlog -.PD 0 +\fB\-e#\fR +benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\fR (inclusive) +. .TP -.BI wlog= wlog -.PD -Specify the maximum number of bits for a match distance. -.IP "" -The higher number of bits increases the chance to find a match what usually improves compression ratio. -It also increases memory requirements for compressor and decompressor. -.IP "" -The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. -.IP "" - +\fB\-i#\fR +minimum evaluation time, in seconds (default : 3s), benchmark mode only +. .TP -.BI hashLog= hlog -.PD 0 +\fB\-B#\fR +cut file into independent blocks of size # (default: no block) +. .TP -.BI hlog= hlog -.PD -Specify the maximum number of bits for a hash table. -.IP "" -The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression. -.IP "" -The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. - +\fB\-\-priority=rt\fR +set process priority to real\-time +. +.SH "ADVANCED COMPRESSION OPTIONS" +. +.SS "\-\-zstd[=options]:" +\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR: +. .TP -.BI chainLog= clog -.PD 0 +\fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR +Specify a strategy used by a match finder\. +. +.IP +There are 8 strategies numbered from 0 to 7, from faster to stronger: 0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2\. +. .TP -.BI clog= clog -.PD -Specify the maximum number of bits for a hash chain or a binary tree. -.IP "" -The higher number of bits increases the chance to find a match what usually improves compression ratio. -It also slows down compression speed and increases memory requirements for compression. -This option is ignored for the ZSTD_fast strategy. -.IP "" -The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation. -.IP "" - +\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR +Specify the maximum number of bits for a match distance\. +. +.IP +The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\. +. .TP -.BI searchLog= slog -.PD 0 +\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR +Specify the maximum number of bits for a hash table\. +. +.IP +Bigger hash tables cause less collisions which usually makes compression faster, but requires more memory during compression\. +. +.IP +The minimum \fIhlog\fR is 6 (64 B) and the maximum is 26 (128 MiB)\. +. .TP -.BI slog= slog -.PD -Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale. -.IP "" -The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed. -.IP "" -The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation. -.IP "" - +\fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR +Specify the maximum number of bits for a hash chain or a binary tree\. +. +.IP +Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\. +. +.IP +The minimum \fIclog\fR is 6 (64 B) and the maximum is 28 (256 MiB)\. +. .TP -.BI searchLength= slen -.PD 0 +\fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR +Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale\. +. +.IP +More searches increases the chance to find a match which usually increases compression ratio but decreases compression speed\. +. +.IP +The minimum \fIslog\fR is 1 and the maximum is 26\. +. .TP -.BI slen= slen -.PD -Specify the minimum searched length of a match in a hash table. -.IP "" -The bigger search length usually decreases compression ratio but improves decompression speed. -.IP "" -The minimum \fIslen\fR is 3 and the maximum is 7. -.IP "" - +\fBsearchLength\fR=\fIslen\fR, \fBslen\fR=\fIslen\fR +Specify the minimum searched length of a match in a hash table\. +. +.IP +Larger search lengths usually decrease compression ratio but improve decompression speed\. +. +.IP +The minimum \fIslen\fR is 3 and the maximum is 7\. +. .TP -.BI targetLength= tlen -.PD 0 +\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR +Specify the minimum match length that causes a match finder to stop searching for better matches\. +. +.IP +A larger minimum match length usually improves compression ratio but decreases compression speed\. This option is only used with strategies ZSTD_btopt and ZSTD_btopt2\. +. +.IP +The minimum \fItlen\fR is 4 and the maximum is 999\. +. .TP -.BI tlen= tlen -.PD -Specify the minimum match length that causes a match finder to interrupt searching of better matches. -.IP "" -The bigger minimum match length usually improves compression ratio but decreases compression speed. -This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies. -.IP "" -The minimum \fItlen\fR is 4 and the maximum is 999. -.IP "" - -.PP -.B An example -.br -The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB: -.IP "" -\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6 - -.SH BUGS -Report bugs at:- https://github.com/facebook/zstd/issues - -.SH AUTHOR +\fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR +Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\. +. +.IP +The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\. +. +.SS "\-B#:" +Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\. +. +.SS "Example" +The following parameters sets advanced compression options to those of predefined level 19 for files bigger than 256 KB: +. +.P +\fB\-\-zstd\fR=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6 +. +.SH "BUGS" +Report bugs at: https://github\.com/facebook/zstd/issues +. +.SH "AUTHOR" Yann Collet diff --git a/contrib/zstd/programs/zstd.1.md b/contrib/zstd/programs/zstd.1.md new file mode 100644 index 0000000000000..118c9f2f8ee58 --- /dev/null +++ b/contrib/zstd/programs/zstd.1.md @@ -0,0 +1,343 @@ +zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files +============================================================================ + +SYNOPSIS +-------- + +`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_] + +`zstdmt` is equivalent to `zstd -T0` + +`unzstd` is equivalent to `zstd -d` + +`zstdcat` is equivalent to `zstd -dcf` + + +DESCRIPTION +----------- +`zstd` is a fast lossless compression algorithm and data compression tool, +with command line syntax similar to `gzip (1)` and `xz (1)`. +It is based on the **LZ77** family, with further FSE & huff0 entropy stages. +`zstd` offers highly configurable compression speed, +with fast modes at > 200 MB/s per code, +and strong modes nearing lzma compression ratios. +It also features a very fast decoder, with speeds > 500 MB/s per core. + +`zstd` command line syntax is generally similar to gzip, +but features the following differences : + + - Source files are preserved by default. + It's possible to remove them automatically by using the `--rm` command. + - When compressing a single file, `zstd` displays progress notifications + and result summary by default. + Use `-q` to turn them off. + - `zstd` does not accept input from console, + but it properly accepts `stdin` when it's not the console. + - `zstd` displays a short help page when command line is an error. + Use `-q` to turn it off. + +`zstd` compresses or decompresses each _file_ according to the selected +operation mode. +If no _files_ are given or _file_ is `-`, `zstd` reads from standard input +and writes the processed data to standard output. +`zstd` will refuse to write compressed data to standard output +if it is a terminal : it will display an error message and skip the _file_. +Similarly, `zstd` will refuse to read compressed data from standard input +if it is a terminal. + +Unless `--stdout` or `-o` is specified, _files_ are written to a new file +whose name is derived from the source _file_ name: + +* When compressing, the suffix `.zst` is appended to the source filename to + get the target filename. +* When decompressing, the `.zst` suffix is removed from the source filename to + get the target filename + +### Concatenation with .zst files +It is possible to concatenate `.zst` files as is. +`zstd` will decompress such files as if they were a single `.zst` file. + +OPTIONS +------- + +### Integer suffixes and special values +In most places where an integer argument is expected, +an optional suffix is supported to easily indicate large integers. +There must be no space between the integer and the suffix. + +* `KiB`: + Multiply the integer by 1,024 (2\^10). + `Ki`, `K`, and `KB` are accepted as synonyms for `KiB`. +* `MiB`: + Multiply the integer by 1,048,576 (2\^20). + `Mi`, `M`, and `MB` are accepted as synonyms for `MiB`. + +### Operation mode +If multiple operation mode options are given, +the last one takes effect. + +* `-z`, `--compress`: + Compress. + This is the default operation mode when no operation mode option is specified + and no other operation mode is implied from the command name + (for example, `unzstd` implies `--decompress`). +* `-d`, `--decompress`, `--uncompress`: + Decompress. +* `-t`, `--test`: + Test the integrity of compressed _files_. + This option is equivalent to `--decompress --stdout` except that the + decompressed data is discarded instead of being written to standard output. + No files are created or removed. +* `-b#`: + Benchmark file(s) using compression level # +* `--train FILEs`: + Use FILEs as a training set to create a dictionary. + The training set should contain a lot of small files (> 100). + +### Operation modifiers + +* `-#`: + `#` compression level \[1-19] (default: 3) +* `--ultra`: + unlocks high compression levels 20+ (maximum 22), using a lot more memory. + Note that decompression will also require more memory when using these levels. +* `-T#`, `--threads=#`: + Compress using `#` threads (default: 1). + If `#` is 0, attempt to detect and use the number of physical CPU cores. + This modifier does nothing if `zstd` is compiled without multithread support. +* `-D file`: + use `file` as Dictionary to compress or decompress FILE(s) +* `--nodictID`: + do not store dictionary ID within frame header (dictionary compression). + The decoder will have to rely on implicit knowledge about which dictionary to use, + it won't be able to check if it's correct. +* `-o file`: + save result into `file` (only possible with a single _INPUT-FILE_) +* `-f`, `--force`: + overwrite output without prompting, and (de)compress symbolic links +* `-c`, `--stdout`: + force write to standard output, even if it is the console +* `--[no-]sparse`: + enable / disable sparse FS support, + to make files with many zeroes smaller on disk. + Creating sparse files may save disk space and speed up decompression by + reducing the amount of disk I/O. + default : enabled when output is into a file, + and disabled when output is stdout. + This setting overrides default and can force sparse mode over stdout. +* `--rm`: + remove source file(s) after successful compression or decompression +* `-k`, `--keep`: + keep source file(s) after successful compression or decompression. + This is the default behavior. +* `-r`: + operate recursively on dictionaries +* `-h`/`-H`, `--help`: + display help/long help and exit +* `-V`, `--version`: + display version number and exit +* `-v`: + verbose mode +* `-q`, `--quiet`: + suppress warnings, interactivity, and notifications. + specify twice to suppress errors too. +* `-C`, `--[no-]check`: + add integrity check computed from uncompressed data (default : enabled) +* `--`: + All arguments after `--` are treated as files + + +DICTIONARY BUILDER +------------------ +`zstd` offers _dictionary_ compression, +useful for very small files and messages. +It's possible to train `zstd` with some samples, +the result of which is saved into a file called a `dictionary`. +Then during compression and decompression, reference the same dictionary. +It will improve compression ratio of small files. +Typical gains range from 10% (at 64KB) to x5 better (at <1KB). + +* `--train FILEs`: + Use FILEs as training set to create a dictionary. + The training set should contain a lot of small files (> 100), + and weight typically 100x the target dictionary size + (for example, 10 MB for a 100 KB dictionary). + + Supports multithreading if `zstd` is compiled with threading support. + Additional parameters can be specified with `--train-cover`. + The legacy dictionary builder can be accessed with `--train-legacy`. + Equivalent to `--train-cover=d=8,steps=4`. +* `-o file`: + Dictionary saved into `file` (default name: dictionary). +* `--maxdict=#`: + Limit dictionary to specified size (default: 112640). +* `--dictID=#`: + A dictionary ID is a locally unique ID that a decoder can use to verify it is + using the right dictionary. + By default, zstd will create a 4-bytes random number ID. + It's possible to give a precise number instead. + Short numbers have an advantage : an ID < 256 will only need 1 byte in the + compressed frame header, and an ID < 65536 will only need 2 bytes. + This compares favorably to 4 bytes default. + However, it's up to the dictionary manager to not assign twice the same ID to + 2 different dictionaries. +* `--train-cover[=k#,d=#,steps=#]`: + Select parameters for the default dictionary builder algorithm named cover. + If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8. + If _k_ is not specified, then it tries _steps_ values in the range [50, 2000]. + If _steps_ is not specified, then the default value of 40 is used. + Requires that _d_ <= _k_. + + Selects segments of size _k_ with highest score to put in the dictionary. + The score of a segment is computed by the sum of the frequencies of all the + subsegments of size _d_. + Generally _d_ should be in the range [6, 8], occasionally up to 16, but the + algorithm will run faster with d <= _8_. + Good values for _k_ vary widely based on the input data, but a safe range is + [2 * _d_, 2000]. + Supports multithreading if `zstd` is compiled with threading support. + + Examples: + + `zstd --train-cover FILEs` + + `zstd --train-cover=k=50,d=8 FILEs` + + `zstd --train-cover=d=8,steps=500 FILEs` + + `zstd --train-cover=k=50 FILEs` + +* `--train-legacy[=selectivity=#]`: + Use legacy dictionary builder algorithm with the given dictionary + _selectivity_ (default: 9). + The smaller the _selectivity_ value, the denser the dictionary, + improving its efficiency but reducing its possible maximum size. + `--train-legacy=s=#` is also accepted. + + Examples: + + `zstd --train-legacy FILEs` + + `zstd --train-legacy=selectivity=8 FILEs` + + +BENCHMARK +--------- + +* `-b#`: + benchmark file(s) using compression level # +* `-e#`: + benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive) +* `-i#`: + minimum evaluation time, in seconds (default : 3s), benchmark mode only +* `-B#`: + cut file into independent blocks of size # (default: no block) +* `--priority=rt`: + set process priority to real-time + + +ADVANCED COMPRESSION OPTIONS +---------------------------- +### --zstd[=options]: +`zstd` provides 22 predefined compression levels. +The selected or default predefined compression level can be changed with +advanced compression options. +The _options_ are provided as a comma-separated list. +You may specify only the options you want to change and the rest will be +taken from the selected or default compression level. +The list of available _options_: + +- `strategy`=_strat_, `strat`=_strat_: + Specify a strategy used by a match finder. + + There are 8 strategies numbered from 0 to 7, from faster to stronger: + 0=ZSTD\_fast, 1=ZSTD\_dfast, 2=ZSTD\_greedy, 3=ZSTD\_lazy, + 4=ZSTD\_lazy2, 5=ZSTD\_btlazy2, 6=ZSTD\_btopt, 7=ZSTD\_btopt2. + +- `windowLog`=_wlog_, `wlog`=_wlog_: + Specify the maximum number of bits for a match distance. + + The higher number of increases the chance to find a match which usually + improves compression ratio. + It also increases memory requirements for the compressor and decompressor. + The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB). + +- `hashLog`=_hlog_, `hlog`=_hlog_: + Specify the maximum number of bits for a hash table. + + Bigger hash tables cause less collisions which usually makes compression + faster, but requires more memory during compression. + + The minimum _hlog_ is 6 (64 B) and the maximum is 26 (128 MiB). + +- `chainLog`=_clog_, `clog`=_clog_: + Specify the maximum number of bits for a hash chain or a binary tree. + + Higher numbers of bits increases the chance to find a match which usually + improves compression ratio. + It also slows down compression speed and increases memory requirements for + compression. + This option is ignored for the ZSTD_fast strategy. + + The minimum _clog_ is 6 (64 B) and the maximum is 28 (256 MiB). + +- `searchLog`=_slog_, `slog`=_slog_: + Specify the maximum number of searches in a hash chain or a binary tree + using logarithmic scale. + + More searches increases the chance to find a match which usually increases + compression ratio but decreases compression speed. + + The minimum _slog_ is 1 and the maximum is 26. + +- `searchLength`=_slen_, `slen`=_slen_: + Specify the minimum searched length of a match in a hash table. + + Larger search lengths usually decrease compression ratio but improve + decompression speed. + + The minimum _slen_ is 3 and the maximum is 7. + +- `targetLen`=_tlen_, `tlen`=_tlen_: + Specify the minimum match length that causes a match finder to stop + searching for better matches. + + A larger minimum match length usually improves compression ratio but + decreases compression speed. + This option is only used with strategies ZSTD_btopt and ZSTD_btopt2. + + The minimum _tlen_ is 4 and the maximum is 999. + +- `overlapLog`=_ovlog_, `ovlog`=_ovlog_: + Determine `overlapSize`, amount of data reloaded from previous job. + This parameter is only available when multithreading is enabled. + Reloading more data improves compression ratio, but decreases speed. + + The minimum _ovlog_ is 0, and the maximum is 9. + 0 means "no overlap", hence completely independent jobs. + 9 means "full overlap", meaning up to `windowSize` is reloaded from previous job. + Reducing _ovlog_ by 1 reduces the amount of reload by a factor 2. + Default _ovlog_ is 6, which means "reload `windowSize / 8`". + Exception : the maximum compression level (22) has a default _ovlog_ of 9. + +### -B#: +Select the size of each compression job. +This parameter is available only when multi-threading is enabled. +Default value is `4 * windowSize`, which means it varies depending on compression level. +`-B#` makes it possible to select a custom value. +Note that job size must respect a minimum value which is enforced transparently. +This minimum is either 1 MB, or `overlapSize`, whichever is largest. + +### Example +The following parameters sets advanced compression options to those of +predefined level 19 for files bigger than 256 KB: + +`--zstd`=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6 + +BUGS +---- +Report bugs at: https://github.com/facebook/zstd/issues + +AUTHOR +------ +Yann Collet diff --git a/contrib/zstd/programs/zstdcli.c b/contrib/zstd/programs/zstdcli.c index 05d29a989a96f..236338ab7e502 100644 --- a/contrib/zstd/programs/zstdcli.c +++ b/contrib/zstd/programs/zstdcli.c @@ -49,6 +49,7 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR +#define ZSTD_ZSTDMT "zstdmt" #define ZSTD_UNZSTD "unzstd" #define ZSTD_CAT "zstdcat" #define ZSTD_GZ "gzip" @@ -74,10 +75,10 @@ static U32 g_overlapLog = OVERLAP_LOG_DEFAULT; /*-************************************ * Display Macros **************************************/ -#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static FILE* displayOut; -static unsigned displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ +#define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } +static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ +static FILE* g_displayOut; /*-************************************ @@ -99,7 +100,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); - DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); DISPLAY( " -h/-H : display help/long help and exit\n"); @@ -113,19 +114,20 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL); + DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories \n"); -#endif #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); - DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); - DISPLAY( "--[no-]check : integrity check (default:enabled) \n"); #ifdef ZSTD_MULTITHREAD DISPLAY( " -T# : use # threads for compression (default:1) \n"); - DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n"); + DISPLAY( " -B# : select size of each job (default:0==automatic) \n"); +#endif + DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); + DISPLAY( "--[no-]check : integrity check (default:enabled) \n"); +#endif +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories \n"); #endif #ifdef ZSTD_GZCOMPRESS DISPLAY( "--format=gzip : compress files to the .gz format \n"); @@ -134,10 +136,16 @@ static int usage_advanced(const char* programName) DISPLAY( "--format=xz : compress files to the .xz format \n"); DISPLAY( "--format=lzma : compress files to the .lzma format \n"); #endif +#ifdef ZSTD_LZ4COMPRESS + DISPLAY( "--format=lz4 : compress files to the .lz4 format \n"); #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( "--test : test compressed file integrity \n"); +#if ZSTD_SPARSE_DEFAULT DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); +#else + DISPLAY( "--[no-]sparse : sparse mode (default:disabled)\n"); +#endif #endif DISPLAY( " -M# : Set a memory usage limit for decompression \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); @@ -145,12 +153,11 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); DISPLAY( "--train ## : create a dictionary from a training set of files \n"); - DISPLAY( "--cover=k=#,d=# : use the cover algorithm with parameters k and d \n"); - DISPLAY( "--optimize-cover[=steps=#,k=#,d=#] : optimize cover parameters with optional parameters\n"); + DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n"); + DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); - DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); - DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); - DISPLAY( "--dictID ## : force dictionary ID to specified value (default: random)\n"); + DISPLAY( "--maxdict=# : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); + DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n"); #endif #ifndef ZSTD_NOBENCH DISPLAY( "\n"); @@ -167,7 +174,7 @@ static int usage_advanced(const char* programName) static int badusage(const char* programName) { DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 1) usage(programName); + if (g_displayLevel >= 2) usage(programName); return 1; } @@ -179,6 +186,23 @@ static void waitEnter(void) (void)unused; } +static const char* lastNameFromPath(const char* path) +{ + const char* name = path; + if (strrchr(name, '/')) name = strrchr(name, '/') + 1; + if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */ + return name; +} + +/*! exeNameMatch() : + @return : a non-zero value if exeName matches test, excluding the extension + */ +static int exeNameMatch(const char* exeName, const char* test) +{ + return !strncmp(exeName, test, strlen(test)) && + (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.'); +} + /*! readU32FromChar() : @return : unsigned integer value read from input in `char` format allows and interprets K, KB, KiB, M, MB and MiB suffix. @@ -216,11 +240,11 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) #ifndef ZSTD_NODICT /** * parseCoverParameters() : - * reads cover parameters from *stringPtr (e.g. "--cover=smoothing=100,kmin=48,kstep=4,kmax=64,d=8") into *params + * reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params * @return 1 means that cover parameters were correct * @return 0 in case of malformed parameters */ -static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *params) +static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t* params) { memset(params, 0, sizeof(*params)); for (; ;) { @@ -230,9 +254,33 @@ static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *para return 0; } if (stringPtr[0] != 0) return 0; - DISPLAYLEVEL(4, "k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps); + DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps); + return 1; +} + +/** + * parseLegacyParameters() : + * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity + * @return 1 means that legacy dictionary builder parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity) +{ + if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; } + *selectivity = readU32FromChar(&stringPtr); + if (stringPtr[0] != 0) return 0; + DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity); return 1; } + +static COVER_params_t defaultCoverParams(void) +{ + COVER_params_t params; + memset(¶ms, 0, sizeof(params)); + params.d = 8; + params.steps = 4; + return params; +} #endif @@ -270,6 +318,7 @@ int main(int argCount, const char* argv[]) { int argNb, forceStdout=0, + followLinks=0, main_pause=0, nextEntryIsDictionary=0, operationResult=0, @@ -305,8 +354,8 @@ int main(int argCount, const char* argv[]) unsigned fileNamesNb; #endif #ifndef ZSTD_NODICT - COVER_params_t coverParams; - int cover = 0; + COVER_params_t coverParams = defaultCoverParams(); + int cover = 1; #endif /* init */ @@ -316,21 +365,19 @@ int main(int argCount, const char* argv[]) (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; - displayOut = stderr; - /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ - { size_t pos; - for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } - programName += pos; - } + g_displayOut = stderr; + + programName = lastNameFromPath(programName); /* preset behaviors */ - if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; - if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } - if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ - if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ - if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } /* behave like gzcat */ - if (!strcmp(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ - if (!strcmp(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ + if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbThreads=0; + if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; + if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } + if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ + if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ + if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat */ + if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ + if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ @@ -344,7 +391,7 @@ int main(int argCount, const char* argv[]) if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; outFileName=stdoutmark; - displayLevel-=(displayLevel==2); + g_displayLevel-=(g_displayLevel==2); continue; } } @@ -357,12 +404,12 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } - if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; } - if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } - if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } - if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } - if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } - if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } + if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; } + if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } + if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } + if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } + if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } + if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; } if (!strcmp(argument, "--ultra")) { ultra=1; continue; } if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } @@ -370,8 +417,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; } if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } - if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } + if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ + if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } @@ -383,26 +430,40 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; } if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; } #endif +#ifdef ZSTD_LZ4COMPRESS + if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; } +#endif /* long commands with arguments */ #ifndef ZSTD_NODICT - if (longCommandWArg(&argument, "--cover=")) { - cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName)); - continue; - } - if (longCommandWArg(&argument, "--optimize-cover")) { - cover=2; + if (longCommandWArg(&argument, "--train-cover")) { + operation = zom_train; + outFileName = g_defaultDictName; + cover = 1; /* Allow optional arguments following an = */ if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); } else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); } continue; } + if (longCommandWArg(&argument, "--train-legacy")) { + operation = zom_train; + outFileName = g_defaultDictName; + cover = 0; + /* Allow optional arguments following an = */ + if (*argument == 0) { continue; } + else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } + else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); } + continue; + } #endif + if (longCommandWArg(&argument, "--threads=")) { nbThreads = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; } /* fall-through, will trigger bad_usage() later on */ } @@ -424,9 +485,9 @@ int main(int argCount, const char* argv[]) switch(argument[0]) { /* Display help */ - case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ + case 'V': g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ case 'H': - case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); + case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); /* Compress */ case 'z': operation=zom_compress; argument++; break; @@ -445,19 +506,19 @@ int main(int argCount, const char* argv[]) case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; /* Overwrite */ - case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; + case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break; /* Verbose mode */ - case 'v': displayLevel++; argument++; break; + case 'v': g_displayLevel++; argument++; break; /* Quiet mode */ - case 'q': displayLevel--; argument++; break; + case 'q': g_displayLevel--; argument++; break; - /* keep source file (default); for gzip/xz compatibility */ + /* keep source file (default) */ case 'k': FIO_setRemoveSrcFile(0); argument++; break; /* Checksum */ - case 'C': argument++; FIO_setChecksumFlag(2); break; + case 'C': FIO_setChecksumFlag(2); argument++; break; /* test compressed file */ case 't': operation=zom_test; argument++; break; @@ -532,14 +593,14 @@ int main(int argCount, const char* argv[]) continue; } /* if (argument[0]=='-') */ - if (nextArgumentIsMaxDict) { + if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */ nextArgumentIsMaxDict = 0; lastCommand = 0; maxDictSize = readU32FromChar(&argument); continue; } - if (nextArgumentIsDictID) { + if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */ nextArgumentIsDictID = 0; lastCommand = 0; dictID = readU32FromChar(&argument); @@ -581,9 +642,27 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif + if (nbThreads == 0) { + /* try to guess */ + nbThreads = UTIL_countPhysicalCores(); + DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads); + } + + g_utilDisplayLevel = g_displayLevel; + if (!followLinks) { + unsigned u; + for (u=0, fileNamesNb=0; u<filenameIdx; u++) { + if (UTIL_isLink(filenameTable[u])) { + DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]); + } else { + filenameTable[fileNamesNb++] = filenameTable[u]; + } + } + filenameIdx = fileNamesNb; + } #ifdef UTIL_HAS_CREATEFILELIST if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ - extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks); if (extendedFileList) { unsigned u; for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]); @@ -597,7 +676,7 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - BMK_setNotificationLevel(displayLevel); + BMK_setNotificationLevel(g_displayLevel); BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); BMK_setNbSeconds(bench_nbSeconds); @@ -611,19 +690,20 @@ int main(int argCount, const char* argv[]) if (operation==zom_train) { #ifndef ZSTD_NODICT if (cover) { + int const optimize = !coverParams.k || !coverParams.d; coverParams.nbThreads = nbThreads; coverParams.compressionLevel = dictCLevel; - coverParams.notificationLevel = displayLevel; + coverParams.notificationLevel = g_displayLevel; coverParams.dictID = dictID; - DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, cover - 1); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, optimize); } else { ZDICT_params_t dictParams; memset(&dictParams, 0, sizeof(dictParams)); dictParams.compressionLevel = dictCLevel; dictParams.selectivityLevel = dictSelect; - dictParams.notificationLevel = displayLevel; + dictParams.notificationLevel = g_displayLevel; dictParams.dictID = dictID; - DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0); } #endif goto _end; @@ -635,7 +715,7 @@ int main(int argCount, const char* argv[]) /* Check if input/output defined as console; trigger an error in this case */ if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName)); - if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && (operation==zom_decompress))) + if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress) CLEAN_RETURN(badusage(programName)); /* user-selected output filename, only possible with a single file */ @@ -654,11 +734,11 @@ int main(int argCount, const char* argv[]) #endif /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; - if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1; + if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; + if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ - FIO_setNotificationLevel(displayLevel); + FIO_setNotificationLevel(g_displayLevel); if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS FIO_setNbThreads(nbThreads); diff --git a/contrib/zstd/tests/Makefile b/contrib/zstd/tests/Makefile index 9382fe80ddc29..ea58c0fe5119e 100644 --- a/contrib/zstd/tests/Makefile +++ b/contrib/zstd/tests/Makefile @@ -26,9 +26,12 @@ PYTHON ?= python3 TESTARTEFACT := versionsTest namespaceTest -CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +DEBUGFLAGS=-g -DZSTD_DEBUG=1 +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ + $(DEBUGFLAG) CFLAGS ?= -O3 -CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wformat-security CFLAGS += $(MOREFLAGS) @@ -65,14 +68,16 @@ FUZZERTEST ?= -T5mn ZSTDRTTEST = --test-large-data DECODECORPUS_TESTTIME ?= -T30 -.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest +.PHONY: default all all32 allnothread dll clean test test32 test-all namespaceTest versionsTest default: fullbench -all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest +all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest decodecorpus all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32 +allnothread: fullbench fuzzer paramgrill datagen zbufftest decodecorpus + dll: fuzzer-dll zstreamtest-dll zbufftest-dll zstd: @@ -152,6 +157,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c $(MAKE) -C $(ZSTDDIR) libzstd $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT) +paramgrill : DEBUGFLAG = paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) @@ -300,10 +306,10 @@ test-fullbench32: fullbench32 datagen $(QEMU_SYS) ./fullbench32 -i1 -P0 test-fuzzer: fuzzer - $(QEMU_SYS) ./fuzzer $(FUZZERTEST) + $(QEMU_SYS) ./fuzzer $(FUZZERTEST) $(FUZZER_FLAGS) test-fuzzer32: fuzzer32 - $(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) + $(QEMU_SYS) ./fuzzer32 $(FUZZERTEST) $(FUZZER_FLAGS) test-zbuff: zbufftest $(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME) @@ -312,10 +318,10 @@ test-zbuff32: zbufftest32 $(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME) test-zstream: zstreamtest - $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) + $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-zstream32: zstreamtest32 - $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) + $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-longmatch: longmatch $(QEMU_SYS) ./longmatch diff --git a/contrib/zstd/tests/decodecorpus.c b/contrib/zstd/tests/decodecorpus.c index 183d20f749e63..f7b3c854fdb25 100644 --- a/contrib/zstd/tests/decodecorpus.c +++ b/contrib/zstd/tests/decodecorpus.c @@ -98,7 +98,7 @@ static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb) BYTE* op = ptr; for (i = 0; i < size; i++) { - op[i] = RAND(seed) % (maxSymb + 1); + op[i] = (BYTE) (RAND(seed) % (maxSymb + 1)); } } @@ -134,8 +134,8 @@ static void RAND_genDist(U32* seed, BYTE* dist, double weight) { size_t i = 0; size_t statesLeft = DISTSIZE; - BYTE symb = RAND(seed) % 256; - BYTE step = (RAND(seed) % 256) | 1; /* force it to be odd so it's relatively prime to 256 */ + BYTE symb = (BYTE) (RAND(seed) % 256); + BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */ while (i < DISTSIZE) { size_t states = ((size_t)(weight * statesLeft)) + 1; @@ -259,7 +259,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame) /* Follow window algorithm from specification */ int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10); int const mantissa = RAND(seed) % 8; - windowByte = (exponent << 3) | mantissa; + windowByte = (BYTE) ((exponent << 3) | mantissa); fh.windowSize = (1U << (exponent + 10)); fh.windowSize += fh.windowSize / 8 * mantissa; } @@ -284,7 +284,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame) if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) { /* do single segment sometimes */ - fh.windowSize = fh.contentSize; + fh.windowSize = (U32) fh.contentSize; singleSegment = 1; } } @@ -307,7 +307,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame) { BYTE const frameHeaderDescriptor = - (fcsCode << 6) | (singleSegment << 5) | (1 << 2); + (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2)); op[pos++] = frameHeaderDescriptor; } @@ -318,14 +318,14 @@ static void writeFrameHeader(U32* seed, frame_t* frame) if (contentSizeFlag) { switch (fcsCode) { default: /* Impossible */ - case 0: op[pos++] = fh.contentSize; break; - case 1: MEM_writeLE16(op + pos, fh.contentSize - 256); pos += 2; break; - case 2: MEM_writeLE32(op + pos, fh.contentSize); pos += 4; break; - case 3: MEM_writeLE64(op + pos, fh.contentSize); pos += 8; break; + case 0: op[pos++] = (BYTE) fh.contentSize; break; + case 1: MEM_writeLE16(op + pos, (U16) (fh.contentSize - 256)); pos += 2; break; + case 2: MEM_writeLE32(op + pos, (U32) fh.contentSize); pos += 4; break; + case 3: MEM_writeLE64(op + pos, (U64) fh.contentSize); pos += 8; break; } } - DISPLAYLEVEL(2, " frame content size:\t%zu\n", fh.contentSize); + DISPLAYLEVEL(2, " frame content size:\t%u\n", (U32)fh.contentSize); DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize); DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag); DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment); @@ -385,7 +385,7 @@ static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t content op += litSize; } else { /* RLE literals */ - BYTE const symb = RAND(seed) % 256; + BYTE const symb = (BYTE) (RAND(seed) % 256); DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (U32)symb); @@ -542,8 +542,8 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con op += compressedSize; compressedSize += hufHeaderSize; - DISPLAYLEVEL(5, " regenerated size: %zu\n", litSize); - DISPLAYLEVEL(5, " compressed size: %zu\n", compressedSize); + DISPLAYLEVEL(5, " regenerated size: %u\n", (U32)litSize); + DISPLAYLEVEL(5, " compressed size: %u\n", (U32)compressedSize); if (compressedSize >= litSize) { DISPLAYLEVEL(5, " trying again\n"); /* if we have to try again, reset the stats so we don't accidentally @@ -611,7 +611,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, size_t const remainingMatch = contentSize - literalsSize; size_t excessMatch = 0; U32 numSequences = 0; - + U32 i; @@ -628,7 +628,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN; } - DISPLAYLEVEL(5, " total match lengths: %zu\n", remainingMatch); + DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch); for (i = 0; i < numSequences; i++) { /* Generate match and literal lengths by exponential distribution to @@ -647,10 +647,10 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, U32 offset, offsetCode, repIndex; /* bounds checks */ - matchLen = MIN(matchLen, excessMatch + MIN_SEQ_LEN); - literalLen = MIN(literalLen, literalsSize); + matchLen = (U32) MIN(matchLen, excessMatch + MIN_SEQ_LEN); + literalLen = MIN(literalLen, (U32) literalsSize); if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1; - if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + excessMatch; + if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + (U32) excessMatch; memcpy(srcPtr, literals, literalLen); srcPtr += literalLen; @@ -694,8 +694,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, } DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen); - DISPLAYLEVEL(7, " srcPos: %8tu seqNb: %3u", - (BYTE*)srcPtr - (BYTE*)frame->srcStart, i); + DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u", + (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart), i); DISPLAYLEVEL(6, "\n"); if (offsetCode < 3) { DISPLAYLEVEL(7, " repeat offset: %d\n", repIndex); @@ -711,8 +711,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, memcpy(srcPtr, literals, literalsSize); srcPtr += literalsSize; - DISPLAYLEVEL(6, " excess literals: %5zu", literalsSize); - DISPLAYLEVEL(7, " srcPos: %8tu", (BYTE*)srcPtr - (BYTE*)frame->srcStart); + DISPLAYLEVEL(6, " excess literals: %5u", (U32)literalsSize); + DISPLAYLEVEL(7, " srcPos: %8u", (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)); DISPLAYLEVEL(6, "\n"); return numSequences; @@ -957,11 +957,11 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize literalsSize = writeLiteralsBlock(seed, frame, contentSize); - DISPLAYLEVEL(4, " literals size: %zu\n", literalsSize); + DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize); nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize); - DISPLAYLEVEL(4, " number of sequences: %zu\n", nbSeq); + DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq); return (BYTE*)frame->data - blockStart; } @@ -977,7 +977,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, BYTE *op = header + 3; DISPLAYLEVEL(3, " block:\n"); - DISPLAYLEVEL(3, " block content size: %zu\n", contentSize); + DISPLAYLEVEL(3, " block content size: %u\n", (U32)contentSize); DISPLAYLEVEL(3, " last block: %s\n", lastBlock ? "yes" : "no"); if (blockTypeDesc == 0) { @@ -1025,10 +1025,10 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->src = (BYTE*)frame->src + contentSize; DISPLAYLEVEL(3, " block type: %s\n", BLOCK_TYPES[blockType]); - DISPLAYLEVEL(3, " block size field: %zu\n", blockSize); + DISPLAYLEVEL(3, " block size field: %u\n", (U32)blockSize); - header[0] = (lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff; - MEM_writeLE16(header + 1, blockSize >> 5); + header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff); + MEM_writeLE16(header + 1, (U16) (blockSize >> 5)); frame->data = op; } @@ -1300,7 +1300,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, *********************************************************/ static U32 makeSeed(void) { - U32 t = time(NULL); + U32 t = (U32) time(NULL); return XXH32(&t, sizeof(t), 0) % 65536; } diff --git a/contrib/zstd/tests/fullbench.c b/contrib/zstd/tests/fullbench.c index 940d315a7cd45..38cf22fa7ebce 100644 --- a/contrib/zstd/tests/fullbench.c +++ b/contrib/zstd/tests/fullbench.c @@ -251,14 +251,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) case 13: benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue"; break; - case 31: + case 31: benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock"; break; case 32: benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders"; break; #endif - case 41: + case 41: benchFunction = local_ZSTD_compressStream; benchName = "ZSTD_compressStream"; break; case 42: diff --git a/contrib/zstd/tests/fuzzer.c b/contrib/zstd/tests/fuzzer.c index def7542b5f0b5..a9dcf12e07026 100644 --- a/contrib/zstd/tests/fuzzer.c +++ b/contrib/zstd/tests/fuzzer.c @@ -28,6 +28,7 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ +#include "zstdmt_compress.h" #define ZDICT_STATIC_LINKING_ONLY #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ @@ -57,7 +58,7 @@ static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; @@ -133,13 +134,21 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK : %s \n", errorString); } + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize); CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize), CNBuffer, CNBuffSize, 1), cSize=r ); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++); + + DISPLAYLEVEL(4, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); + { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); if (rSize != CNBuffSize) goto _output_error; } @@ -157,6 +166,7 @@ static int basicUnitTests(U32 seed, double compressibility) } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++); { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0); if (r != CNBuffSize) goto _output_error; } @@ -179,6 +189,49 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); + + /* ZSTDMT simple MT compression test */ + DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++); + { ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2); + if (mtctx==NULL) { + DISPLAY("mtctx : mot enough memory, aborting \n"); + testResult = 1; + goto _end; + } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize); + CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx, + compressedBuffer, ZSTD_compressBound(CNBuffSize), + CNBuffer, CNBuffSize, + 1), + cSize=r ); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++); + { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) { + DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (U32)rSize, (U32)CNBuffSize); + goto _output_error; + } } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t u; + for (u=0; u<CNBuffSize; u++) { + if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;; + } } + DISPLAYLEVEL(4, "OK \n"); + + ZSTDMT_freeCCtx(mtctx); + } + + /* Simple API multiframe test */ DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++); { size_t off = 0; @@ -351,7 +404,58 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != CNBuffSize) goto _output_error); DISPLAYLEVEL(4, "OK \n"); - DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++); + DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + ZSTD_customMem customMem = { NULL, NULL, NULL }; + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem); + cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), + CNBuffer, CNBuffSize, cdict); + ZSTD_freeCDict(cdict); + if (ZSTD_isError(cSize)) goto _output_error; + } + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++); + { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ + } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); + { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + ZSTD_customMem const customMem = { NULL, NULL, NULL }; + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem); + cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), + CNBuffer, CNBuffSize, cdict, fParams); + ZSTD_freeCDict(cdict); + if (ZSTD_isError(cSize)) goto _output_error; + } + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(4, "test%3i : try retrieving contentSize from frame : ", testNb++); + { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); + if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; + } + DISPLAYLEVEL(4, "OK (unknown)\n"); + + DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); p.fParams.noDictIDFlag = 1; cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), @@ -438,9 +542,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Decompression defense tests */ DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ if (!ZSTD_isError(r)) goto _output_error; - if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; } + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++); @@ -449,6 +553,24 @@ static int basicUnitTests(U32 seed, double compressibility) if (!ZSTD_isError(r)) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); + /* content size verification test */ + DISPLAYLEVEL(4, "test%3i : Content size verification : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const srcSize = 5000; + size_t const wrongSrcSize = (srcSize + 1000); + ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); + params.fParams.contentSizeFlag = 1; + { size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize); + if (ZSTD_isError(result)) goto _output_error; + } + { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); + if (!ZSTD_isError(result)) goto _output_error; + if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; + DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result)); + } + ZSTD_freeCCtx(cctx); + } + /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); static const size_t dictSize = 65 KB; @@ -600,6 +722,14 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max) } +static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) +{ + ZSTD_parameters params; + params.cParams = cParams; + params.fParams = fParams; + return params; +} + static size_t FUZ_rLogLength(U32* seed, U32 logLength) { size_t const lengthMask = ((size_t)1 << logLength) - 1; @@ -616,7 +746,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility) +static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) { static const U32 maxSrcLog = 23; static const U32 maxSampleLog = 22; @@ -636,6 +766,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD U32 coreSeed = seed, lseed = 0; clock_t const startClock = clock(); clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC; + int const cLevelLimiter = bigTests ? 3 : 2; /* allocation */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -662,7 +793,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) { size_t sampleSize, maxTestSize, totalTestSize; size_t cSize, totalCSize, totalGenSize; - XXH64_state_t xxhState; U64 crcOrig; BYTE* sampleBuffer; const BYTE* dict; @@ -701,7 +831,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression tests */ - { unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1; + { unsigned const cLevel = + ( FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) ) + + 1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); @@ -801,7 +934,10 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1; + int const cLevel = (FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / cLevelLimiter))) + + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; @@ -813,22 +949,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); } else { ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); - ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, + ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ - ZSTD_parameters p; - size_t errorCode; - p.cParams = cPar; p.fParams = fpar; - errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); + ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); + size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode)); } { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0); CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } } - XXH64_reset(&xxhState, 0); ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1); + { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; U32 n; + XXH64_state_t xxhState; + XXH64_reset(&xxhState, 0); for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); @@ -843,13 +979,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); totalTestSize += segmentSize; - } } + } - { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); - CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); - cSize += flushResult; + { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); + CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); + cSize += flushResult; + } + crcOrig = XXH64_digest(&xxhState); } - crcOrig = XXH64_digest(&xxhState); /* streaming decompression test */ if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ @@ -899,7 +1036,7 @@ _output_error: /*_******************************************************* * Command line *********************************************************/ -int FUZ_usage(const char* programName) +static int FUZ_usage(const char* programName) { DISPLAY( "Usage :\n"); DISPLAY( " %s [args]\n", programName); @@ -915,19 +1052,39 @@ int FUZ_usage(const char* programName) return 0; } +/*! readU32FromChar() : + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : function result can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} int main(int argc, const char** argv) { - U32 seed=0; - int seedset=0; + U32 seed = 0; + int seedset = 0; int argNb; int nbTests = nbTestsDefault; int testNb = 0; U32 proba = FUZ_compressibility_default; - int result=0; + int result = 0; U32 mainPause = 0; U32 maxDuration = 0; - const char* programName = argv[0]; + int bigTests = 1; + const char* const programName = argv[0]; /* Check command line */ for (argNb=1; argNb<argc; argNb++) { @@ -936,81 +1093,64 @@ int main(int argc, const char** argv) /* Handle commands. Aggregated commands are allowed */ if (argument[0]=='-') { + + if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } + argument++; while (*argument!=0) { switch(*argument) { case 'h': return FUZ_usage(programName); + case 'v': argument++; - g_displayLevel=4; + g_displayLevel = 4; break; + case 'q': argument++; g_displayLevel--; break; + case 'p': /* pause at the end */ argument++; mainPause = 1; break; case 'i': - argument++; maxDuration=0; - nbTests=0; - while ((*argument>='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } + argument++; maxDuration = 0; + nbTests = readU32FromChar(&argument); break; case 'T': argument++; - nbTests=0; maxDuration=0; - while ((*argument>='0') && (*argument<='9')) { - maxDuration *= 10; - maxDuration += *argument - '0'; - argument++; - } - if (*argument=='m') maxDuration *=60, argument++; + nbTests = 0; + maxDuration = readU32FromChar(&argument); + if (*argument=='s') argument++; /* seconds */ + if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ if (*argument=='n') argument++; break; case 's': argument++; - seed=0; - seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } + seedset = 1; + seed = readU32FromChar(&argument); break; case 't': argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } + testNb = readU32FromChar(&argument); break; case 'P': /* compressibility % */ argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba>100) proba=100; + proba = readU32FromChar(&argument); + if (proba>100) proba = 100; break; default: - return FUZ_usage(programName); + return (FUZ_usage(programName), 1); } } } } /* for (argNb=1; argNb<argc; argNb++) */ /* Get Seed */ @@ -1030,7 +1170,7 @@ int main(int argc, const char** argv) if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ if (!result) - result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100); + result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests); if (mainPause) { int unused; DISPLAY("Press Enter \n"); diff --git a/contrib/zstd/tests/paramgrill.c b/contrib/zstd/tests/paramgrill.c index 5eabcba2b60b4..1913b54d0a54a 100644 --- a/contrib/zstd/tests/paramgrill.c +++ b/contrib/zstd/tests/paramgrill.c @@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64; **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#undef MIN +#undef MAX +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + /*-************************************ * Benchmark Parameters @@ -106,7 +111,11 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 FUZ_rotl32(U32 x, U32 r) +{ + return ((x << r) | (x >> (32 - r))); +} + U32 FUZ_rand(U32* src) { const U32 prime1 = 2654435761U; @@ -125,7 +134,7 @@ U32 FUZ_rand(U32* src) *********************************************************/ typedef struct { size_t cSize; - double cSpeed; + double cSpeed; /* bytes / sec */ double dSpeed; } BMK_result_t; @@ -141,8 +150,6 @@ typedef struct } blockParam_t; -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - static size_t BMK_benchParam(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, @@ -165,6 +172,11 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, char name[30] = { 0 }; U64 crcOrig; + /* init result for early exit */ + resultPtr->cSize = srcSize; + resultPtr->cSpeed = 0.; + resultPtr->dSpeed = 0.; + /* Memory allocation & restrictions */ snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat); if (!compressedBuffer || !resultBuffer || !blockTable) { @@ -206,7 +218,6 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, size_t cSize = 0; double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; - U64 crcCheck = 0; clock_t const benchStart = clock(); DISPLAY("\r%79s\r", ""); @@ -242,8 +253,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, cSize = 0; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; - if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops; ratio = (double)srcSize / (double)cSize; + if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops; DISPLAY("\r"); DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize); DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); @@ -273,18 +284,18 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, resultPtr->dSpeed = (double)srcSize / fastestD; /* CRC Checking */ - crcCheck = XXH64(resultBuffer, srcSize, 0); - if (crcOrig!=crcCheck) { - unsigned u; - unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize)); - DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u<srcSize; u++) { - if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) { - printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize); - break; - } } - break; - } + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + unsigned u; + unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize)); + DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u<srcSize; u++) { + if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) { + printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize); + break; + } } + break; + } } #endif } } @@ -505,8 +516,6 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] -#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) - static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, const void* srcBuffer, size_t srcSize, @@ -711,6 +720,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles) } +static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +{ + DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +} + +/* optimizeForSize(): + * targetSpeed : expressed in MB/s */ int optimizeForSize(const char* inFileName, U32 targetSpeed) { FILE* const inFile = fopen( inFileName, "rb" ); @@ -723,8 +740,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) /* Memory allocation & restrictions */ if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' \n", inFileName); + fclose(inFile); + return 11; + } /* Alloc */ origBuff = malloc(benchedSize); @@ -747,10 +767,9 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); - targetSpeed *= 1000; + targetSpeed *= 1000000; { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_compressionParameters params; winnerInfo_t winner; BMK_result_t candidate; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; @@ -764,26 +783,28 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, CParams); if (candidate.cSpeed < targetSpeed) break; if ( (candidate.cSize < winner.result.cSize) | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) { - winner.params = params; + winner.params = CParams; winner.result = candidate; BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); } } } BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); /* start tests */ { time_t const grillStart = time(NULL); do { - params = winner.params; + ZSTD_compressionParameters params = winner.params; paramVariation(¶ms); - if ((FUZ_rand(&g_rand) & 15) == 3) params = randomParams(); + if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ + params = ZSTD_adjustCParams(params, blockSize, 0); /* exclude faster if already played set of params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; @@ -800,6 +821,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) winner.params = params; winner.result = candidate; BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); } } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } @@ -833,7 +855,7 @@ static int usage_advanced(void) DISPLAY( " -T# : set level 1 speed objective \n"); DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); - DISPLAY( " -O# : find Optimized parameters for # target speed (default : 0) \n"); + DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); DISPLAY( " -S : Single run \n"); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); return 0; diff --git a/contrib/zstd/tests/playTests.sh b/contrib/zstd/tests/playTests.sh index c493fed55358d..021fd59fe2afe 100755 --- a/contrib/zstd/tests/playTests.sh +++ b/contrib/zstd/tests/playTests.sh @@ -11,6 +11,7 @@ roundTripTest() { local_p="$2" else local_c="$2" + local_p="" fi rm -f tmp1 tmp2 @@ -20,13 +21,36 @@ roundTripTest() { $DIFF -q tmp1 tmp2 } +fileRoundTripTest() { + if [ -n "$3" ]; then + local_c="$3" + local_p="$2" + else + local_c="$2" + local_p="" + fi + + rm -f tmp.zstd tmp.md5.1 tmp.md5.2 + $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d" + ./datagen $1 $local_p > tmp + cat tmp | $MD5SUM > tmp.md5.1 + $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2 + $DIFF -q tmp.md5.1 tmp.md5.2 +} + +isTerminal=false +if [ -t 0 ] && [ -t 1 ] +then + isTerminal=true +fi + isWindows=false -ECHO="echo" +ECHO="echo -e" INTOVOID="/dev/null" case "$OS" in Windows*) isWindows=true - ECHO="echo -e" + INTOVOID="NUL" ;; esac @@ -42,11 +66,17 @@ case "$UNAME" in SunOS) DIFF="gdiff" ;; esac - $ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" +if [ -n "$(echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep 'multi-threading is disabled')" ] +then + hasMT="" +else + hasMT="true" +fi + $ECHO "\n**** simple tests **** " ./datagen > tmp @@ -72,6 +102,12 @@ cp tmp tmp2 $ZSTD tmp2 -fo && die "-o must be followed by filename " $ECHO "test : implied stdout when input is stdin" $ECHO bob | $ZSTD | $ZSTD -d +if [ "$isTerminal" = true ]; then +$ECHO "test : compressed data to terminal" +$ECHO bob | $ZSTD && die "should have refused : compressed data to terminal" +$ECHO "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" +$ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal" +fi $ECHO "test : null-length file roundtrip" $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout $ECHO "test : decompress file with wrong suffix (must fail)" @@ -96,6 +132,14 @@ $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite" $ZSTD -q -f tmp $ZSTD -q --force tmp +$ECHO "test : overwrite readonly file" +rm -f tmpro tmpro.zst +$ECHO foo > tmpro.zst +$ECHO foo > tmpro +chmod 400 tmpro.zst +$ZSTD -q tmpro && die "should have refused to overwrite read-only file" +$ZSTD -q -f tmpro +rm -f tmpro tmpro.zst $ECHO "test : file removal" $ZSTD -f --rm tmp ls tmp && die "tmp should no longer be present" @@ -156,6 +200,19 @@ $ECHO "$ECHO foo | $ZSTD > /dev/full" $ECHO foo | $ZSTD > /dev/full && die "write error not detected!" $ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full" $ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + +$ECHO "\n**** symbolic link test **** " + +rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst +$ECHO "hello world" > hello.tmp +ln -s hello.tmp world.tmp +$ZSTD world.tmp hello.tmp +ls hello.tmp.zst || die "regular file should have been compressed!" +ls world.tmp.zst && die "symbolic link should not have been compressed!" +$ZSTD world.tmp hello.tmp -f +ls world.tmp.zst || die "symbol link should have been compressed with --force" +rm -f hello.tmp world.tmp hello.tmp.zst world.tmp.zst + fi @@ -227,12 +284,12 @@ $ECHO "- Create second (different) dictionary " $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" -$ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1 +$ZSTD --train *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with wrong dictID parameter order (must fail)" $ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " $ECHO "- Create dictionary with size limit" -$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v +$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict=4K -v $ECHO "- Create dictionary with wrong parameter order (must fail)" $ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " $ECHO "- Compress without dictID" @@ -240,7 +297,7 @@ $ZSTD -f tmp -D tmpDict1 --no-dictID $ZSTD -d tmp.zst -D tmpDict -fo result $DIFF $TESTFILE result $ECHO "- Compress with wrong argument order (must fail)" -$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name " +$ZSTD tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name " $ECHO "- Compress multiple files with dictionary" rm -rf dirTestDict mkdir dirTestDict @@ -255,6 +312,11 @@ case "$UNAME" in *) $MD5SUM -c tmph1 ;; esac rm -rf dirTestDict +$ECHO "- dictionary builder on bogus input" +$ECHO "Hello World" > tmp +$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" +./datagen -P0 -g10M > tmp +$ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" rm tmp* @@ -263,19 +325,39 @@ $ECHO "\n**** cover dictionary tests **** " TESTFILE=../programs/zstdcli.c ./datagen > tmpDict $ECHO "- Create first dictionary" -$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c -o tmpDict +$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c -o tmpDict cp $TESTFILE tmp $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result $DIFF $TESTFILE result $ECHO "- Create second (different) dictionary" -$ZSTD --train --cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" -$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1 +$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" -$ZSTD --train --optimize-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K +$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K +rm tmp* + +$ECHO "\n**** legacy dictionary tests **** " + +TESTFILE=../programs/zstdcli.c +./datagen > tmpDict +$ECHO "- Create first dictionary" +$ZSTD --train-legacy=selectivity=8 *.c ../programs/*.c -o tmpDict +cp $TESTFILE tmp +$ZSTD -f tmp -D tmpDict +$ZSTD -d tmp.zst -D tmpDict -fo result +$DIFF $TESTFILE result +$ECHO "- Create second (different) dictionary" +$ZSTD --train-legacy=s=5 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +$ECHO "- Create dictionary with short dictID" +$ZSTD --train-legacy -s5 *.c ../programs/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "- Create dictionary with size limit" +$ZSTD --train-legacy -s9 *.c ../programs/*.c -o tmpDict2 --maxdict=4K rm tmp* @@ -341,7 +423,7 @@ if [ $GZIPMODE -eq 1 ]; then $ZSTD -f --format=gzip tmp $ZSTD -f tmp cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.gz | $ZSTD -t && die "incomplete frame not detected !" + head -c -1 tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" rm tmp* else $ECHO "gzip mode not supported" @@ -383,13 +465,48 @@ if [ $LZMAMODE -eq 1 ]; then $ZSTD -f --format=lzma tmp $ZSTD -f tmp cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp - head -c -1 tmp.xz | $ZSTD -t && die "incomplete frame not detected !" - head -c -1 tmp.lzma | $ZSTD -t && die "incomplete frame not detected !" + head -c -1 tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" + head -c -1 tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" rm tmp* else $ECHO "xz mode not supported" fi +$ECHO "\n**** lz4 compatibility tests **** " + +LZ4MODE=1 +$ZSTD --format=lz4 -V || LZ4MODE=0 +if [ $LZ4MODE -eq 1 ]; then + $ECHO "lz4 support detected" + LZ4EXE=1 + lz4 -V || LZ4EXE=0 + if [ $LZ4EXE -eq 1 ]; then + ./datagen > tmp + $ZSTD --format=lz4 -f tmp + lz4 -t -v tmp.lz4 + lz4 -f tmp + $ZSTD -d -f -v tmp.lz4 + rm tmp* + else + $ECHO "lz4 binary not detected" + fi +else + $ECHO "lz4 mode not supported" +fi + + +$ECHO "\n**** lz4 frame tests **** " + +if [ $LZ4MODE -eq 1 ]; then + ./datagen > tmp + $ZSTD -f --format=lz4 tmp + $ZSTD -f tmp + cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp + head -c -1 tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !" + rm tmp* +else + $ECHO "lz4 mode not supported" +fi $ECHO "\n**** zstd round-trip tests **** " @@ -402,6 +519,19 @@ roundTripTest -g519K 6 # greedy, hash chain roundTripTest -g517K 16 # btlazy2 roundTripTest -g516K 19 # btopt +fileRoundTripTest -g500K + +if [ -n "$hasMT" ] +then + $ECHO "\n**** zstdmt round-trip tests **** " + roundTripTest -g4M "1 -T0" + roundTripTest -g8M "3 -T2" + roundTripTest -g8000K "2 --threads=2" + fileRoundTripTest -g4M "19 -T2 -B1M" +else + $ECHO "\n**** no multithreading, skipping zstdmt tests **** " +fi + rm tmp* if [ "$1" != "--test-large-data" ]; then @@ -437,4 +567,16 @@ roundTripTest -g50000000 -P94 19 roundTripTest -g99000000 -P99 20 roundTripTest -g6000000000 -P99 1 +fileRoundTripTest -g4193M -P99 1 + +if [ -n "$hasMT" ] +then + $ECHO "\n**** zstdmt long round-trip tests **** " + roundTripTest -g99000000 -P99 "20 -T2" + roundTripTest -g6000000000 -P99 "1 -T2" + fileRoundTripTest -g4193M -P98 " -T0" +else + $ECHO "\n**** no multithreading, skipping zstdmt tests **** " +fi + rm tmp* diff --git a/contrib/zstd/tests/test-zstd-speed.py b/contrib/zstd/tests/test-zstd-speed.py index 23d4f477c7f3d..56108a5cae4ca 100755 --- a/contrib/zstd/tests/test-zstd-speed.py +++ b/contrib/zstd/tests/test-zstd-speed.py @@ -14,14 +14,15 @@ # - dir1/zstd and dir2/zstd will be merged in a single results file import argparse -import os +import os # getloadavg import string import subprocess -import time +import time # strftime import traceback import hashlib +import platform # system -script_version = 'v1.1.1 (2016-10-28)' +script_version = 'v1.1.2 (2017-03-26)' default_repo_url = 'https://github.com/facebook/zstd.git' working_dir_name = 'speedTest' working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest @@ -152,10 +153,15 @@ def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5 % (os.getloadavg()[0], args.maxLoadAvg, sleepTime)) time.sleep(sleepTime) start_load = str(os.getloadavg()) + osType = platform.system() + if osType == 'Linux': + cpuSelector = "taskset --cpu-list 0" + else: + cpuSelector = "" if args.dictionary: - result = execute('programs/%s -rqi5b1e%s -D %s %s' % (executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True) + result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True) else: - result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True) + result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True) end_load = str(os.getloadavg()) linesExpected = args.lastCLevel + 1 if len(result) != linesExpected: @@ -291,7 +297,7 @@ if __name__ == '__main__': log("ERROR: e-mail senders 'mail' or 'mutt' not found") exit(1) - clang_version = execute("clang -v 2>&1 | grep 'clang version' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0]; + clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0]; gcc_version = execute("gcc -dumpversion", verbose)[0]; if verbose: diff --git a/contrib/zstd/tests/zbufftest.c b/contrib/zstd/tests/zbufftest.c index 14b73923311d1..601aa808d0278 100644 --- a/contrib/zstd/tests/zbufftest.c +++ b/contrib/zstd/tests/zbufftest.c @@ -60,7 +60,7 @@ static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_displayClock = 0; diff --git a/contrib/zstd/tests/zstreamtest.c b/contrib/zstd/tests/zstreamtest.c index aa7367bcfd318..0e09e185642be 100644 --- a/contrib/zstd/tests/zstreamtest.c +++ b/contrib/zstd/tests/zstreamtest.c @@ -59,7 +59,7 @@ static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stderr); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; @@ -131,7 +131,7 @@ static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blo } { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks); free(blockSizes); - if (ZDICT_isError(dictSize)) { free(dict.start); return (buffer_t){ NULL, 0, 0 }; } + if (ZDICT_isError(dictSize)) { free(dict.start); return g_nullBuffer; } dict.size = requestedDictSize; dict.filled = dictSize; return dict; /* how to return dictSize ? */ @@ -207,6 +207,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); } + /* Attempt bad compression parameters */ + DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); + { size_t r; + ZSTD_parameters params = ZSTD_getParams(1, 0, 0); + params.cParams.searchLength = 2; + r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); + if (!ZSTD_isError(r)) goto _output_error; + DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r)); + } + /* skippable frame test */ DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); @@ -438,11 +448,64 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } - /* Unknown srcSize */ + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled); + ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */}; + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, cParams, customMem); + size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, CNBufferSize, fParams); + if (ZSTD_isError(initError)) goto _output_error; + cSize = 0; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_isError(r)) goto _output_error; } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + cSize = outBuff.pos; + ZSTD_freeCDict(cdict); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); + } + + DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++); + { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (did != 0) goto _output_error; + } + DISPLAYLEVEL(3, "OK (not detected) \n"); + + DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ + DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); + } + + /* Empty srcSize */ + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); + { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); + params.fParams.contentSizeFlag = 1; + ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0); + } /* cstream advanced shall write content size = 0 */ + inBuff.src = CNBuffer; + inBuff.size = 0; + inBuff.pos = 0; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error; + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; + cSize = outBuff.pos; + if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); params.fParams.contentSizeFlag = 1; - ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */ + ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); + } /* cstream advanced shall write content size = 0 */ inBuff.src = CNBuffer; inBuff.size = 0; inBuff.pos = 0; @@ -552,7 +615,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } -static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) { static const U32 maxSrcLog = 24; static const U32 maxSampleLog = 19; @@ -574,6 +637,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ size_t dictSize = 0; U32 oldTestLog = 0; + int const cLevelLimiter = bigTests ? 3 : 2; /* allocations */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -638,7 +702,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ && oldTestLog /* at least one test happened */ && resetAllowed) { maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); - if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; + if (maxTestSize >= srcBufferSize) + maxTestSize = srcBufferSize-1; { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; size_t const resetError = ZSTD_resetCStream(zc, pledgedSrcSize); CHECK(ZSTD_isError(resetError), "ZSTD_resetCStream error : %s", ZSTD_getErrorName(resetError)); @@ -646,11 +711,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1; + U32 const cLevel = ( FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / cLevelLimiter))) + + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); oldTestLog = testLog; /* random dictionary selection */ - dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; + dictSize = ((FUZ_rand(&lseed)&1)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; } @@ -785,7 +853,7 @@ _output_error: /* Multi-threading version of fuzzer Tests */ -static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) { static const U32 maxSrcLog = 24; static const U32 maxSampleLog = 19; @@ -807,6 +875,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ size_t dictSize = 0; U32 oldTestLog = 0; + int const cLevelLimiter = bigTests ? 3 : 2; /* allocations */ cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); @@ -851,6 +920,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp /* some issues can only happen when reusing states */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { U32 const nbThreads = (FUZ_rand(&lseed) % 6) + 1; + DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); ZSTDMT_freeCCtx(zc); zc = ZSTDMT_createCCtx(nbThreads); resetAllowed=0; @@ -888,7 +958,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1; + U32 const cLevel = (FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / cLevelLimiter))) + + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); oldTestLog = testLog; /* random dictionary selection */ @@ -898,9 +971,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp } { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); - DISPLAYLEVEL(5, "Init with windowLog = %u \n", params.cParams.windowLog); + DISPLAYLEVEL(5, "Init with windowLog = %u and pledgedSrcSize = %u \n", + params.cParams.windowLog, (U32)pledgedSrcSize); params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + params.fParams.contentSizeFlag = pledgedSrcSize>0; + DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize); CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); } ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12); @@ -938,7 +1014,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp outBuff.size = outBuff.pos + adjustedDstSize; DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize); { size_t const flushError = ZSTDMT_flushStream(zc, &outBuff); - CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError)); + CHECK (ZSTD_isError(flushError), "ZSTDMT_flushStream error : %s", ZSTD_getErrorName(flushError)); } } } /* final frame epilogue */ @@ -949,12 +1025,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp outBuff.size = outBuff.pos + adjustedDstSize; DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (U32)adjustedDstSize); remainingToFlush = ZSTDMT_endStream(zc, &outBuff); - CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); + CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush)); DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (U32)remainingToFlush); } } - DISPLAYLEVEL(5, "Frame completed \n"); crcOrig = XXH64_digest(&xxhState); cSize = outBuff.pos; + DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize); } /* multi - fragments decompression test */ @@ -972,8 +1048,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); inBuff.size = inBuff.pos + readCSrcSize; outBuff.size = inBuff.pos + dstBuffSize; + DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize); } CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); @@ -1063,6 +1141,7 @@ int main(int argc, const char** argv) int result=0; int mainPause = 0; int mtOnly = 0; + int bigTests = 1; const char* const programName = argv[0]; ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL }; ZSTD_customMem const customNULL = { NULL, NULL, NULL }; @@ -1076,6 +1155,7 @@ int main(int argc, const char** argv) if (argument[0]=='-') { if (!strcmp(argument, "--mt")) { mtOnly=1; continue; } + if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } argument++; while (*argument!=0) { @@ -1181,8 +1261,8 @@ int main(int argc, const char** argv) result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */ } } - if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); - if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100); + if (!result && !mtOnly) result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests); + if (!result) result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests); if (mainPause) { int unused; diff --git a/contrib/zstd/zlibWrapper/examples/zwrapbench.c b/contrib/zstd/zlibWrapper/examples/zwrapbench.c index 23c3ca4dab8fa..a57ed51ec141b 100644 --- a/contrib/zstd/zlibWrapper/examples/zwrapbench.c +++ b/contrib/zstd/zlibWrapper/examples/zwrapbench.c @@ -73,13 +73,13 @@ static U32 g_compressibilityDefault = 50; #define DEFAULT_DISPLAY_LEVEL 2 #define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static int g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ static FILE* displayOut; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(displayOut); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; @@ -128,6 +128,11 @@ void BMK_SetBlockSize(size_t blockSize) /* ******************************************************** * Bench functions **********************************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + typedef struct { z_const char* srcPtr; @@ -142,9 +147,6 @@ typedef struct typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor; -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, @@ -234,7 +236,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, if (compressor == BMK_ZSTD) { ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_customMem const cmem = { NULL, NULL, NULL }; - ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); do { @@ -982,7 +984,7 @@ int main(int argCount, char** argv) #ifdef UTIL_HAS_CREATEFILELIST if (recursive) { - fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb); + fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, 1); if (fileNamesTable) { unsigned u; for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]); diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist index 615b40bad5f7e..52f4b00aaa918 100644 --- a/etc/mtree/BSD.tests.dist +++ b/etc/mtree/BSD.tests.dist @@ -612,6 +612,8 @@ .. comm .. + csplit + .. cut .. diff diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c index c4364407cdef4..a36c522c4bdda 100644 --- a/lib/libc/gen/glob.c +++ b/lib/libc/gen/glob.c @@ -581,7 +581,8 @@ glob0(const Char *pattern, glob_t *pglob, struct glob_limit *limit, case STAR: pglob->gl_flags |= GLOB_MAGCHAR; /* collapse adjacent stars to one, - * to avoid exponential behavior + * to ensure "**" at the end continues to match the + * empty string */ if (bufnext == patbuf || bufnext[-1] != M_ALL) *bufnext++ = M_ALL; diff --git a/lib/libc/sys/sigqueue.2 b/lib/libc/sys/sigqueue.2 index 04e101bc3443f..8377c67295b09 100644 --- a/lib/libc/sys/sigqueue.2 +++ b/lib/libc/sys/sigqueue.2 @@ -27,7 +27,7 @@ .\" .\" $FreeBSD$ .\" -.Dd March 10, 2012 +.Dd May 5, 2017 .Dt SIGQUEUE 2 .Os .Sh NAME @@ -129,7 +129,6 @@ does not exist. .Xr kill 2 , .Xr sigaction 2 , .Xr sigpending 2 , -.Xr sigqueue 2 , .Xr sigsuspend 2 , .Xr sigtimedwait 2 , .Xr sigwait 2 , @@ -147,3 +146,18 @@ Support for .Tn POSIX realtime signal queue first appeared in .Fx 7.0 . +.Sh CAVEATS +When using +.Nm +to send signals to a process which might have a different ABI +(for instance, one is 32-bit and the other 64-bit), +the +.Va sival_int +member of +.Fa value +can be delivered reliably, but the +.Va sival_ptr +may be truncated in endian dependent ways and must not be relied on. +Further, many pointer integrity schemes disallow sending pointers to other +processes, and this technique should not be used in programs intended to +be portable. diff --git a/lib/libstand/arp.c b/lib/libstand/arp.c index dac8f90886de2..841010c3195ce 100644 --- a/lib/libstand/arp.c +++ b/lib/libstand/arp.c @@ -65,17 +65,16 @@ int arp_num = 1; /* Local forwards */ static ssize_t arpsend(struct iodesc *, void *, size_t); -static ssize_t arprecv(struct iodesc *, void *, size_t, time_t); +static ssize_t arprecv(struct iodesc *, void **, void **, time_t); /* Broadcast an ARP packet, asking who has addr on interface d */ u_char * -arpwhohas(d, addr) - struct iodesc *d; - struct in_addr addr; +arpwhohas(struct iodesc *d, struct in_addr addr) { int i; struct ether_arp *ah; struct arp_list *al; + void *pkt; struct { struct ether_header eh; struct { @@ -83,13 +82,6 @@ arpwhohas(d, addr) u_char pad[18]; /* 60 - sizeof(...) */ } data; } wbuf; - struct { - struct ether_header eh; - struct { - struct ether_arp arp; - u_char pad[24]; /* extra space */ - } data; - } rbuf; /* Try for cached answer first */ for (i = 0, al = arp_list; i < arp_num; ++i, ++al) @@ -122,20 +114,24 @@ arpwhohas(d, addr) /* Store ip address in cache (incomplete entry). */ al->addr = addr; + pkt = NULL; + ah = NULL; i = sendrecv(d, arpsend, &wbuf.data, sizeof(wbuf.data), - arprecv, &rbuf.data, sizeof(rbuf.data)); + arprecv, &pkt, (void **)&ah); if (i == -1) { panic("arp: no response for %s\n", inet_ntoa(addr)); } /* Store ethernet address in cache */ - ah = &rbuf.data.arp; #ifdef ARP_DEBUG if (debug) { + struct ether_header *eh; + + eh = (struct ether_header *)((uintptr_t)pkt + ETHER_ALIGN); printf("arp: response from %s\n", - ether_sprintf(rbuf.eh.ether_shost)); + ether_sprintf(eh->ether_shost)); printf("arp: cacheing %s --> %s\n", inet_ntoa(addr), ether_sprintf(ah->arp_sha)); } @@ -143,14 +139,12 @@ arpwhohas(d, addr) MACPY(ah->arp_sha, al->ea); ++arp_num; + free(pkt); return (al->ea); } static ssize_t -arpsend(d, pkt, len) - struct iodesc *d; - void *pkt; - size_t len; +arpsend(struct iodesc *d, void *pkt, size_t len) { #ifdef ARP_DEBUG @@ -166,28 +160,27 @@ arpsend(d, pkt, len) * else -1 (and errno == 0) */ static ssize_t -arprecv(d, pkt, len, tleft) - struct iodesc *d; - void *pkt; - size_t len; - time_t tleft; +arprecv(struct iodesc *d, void **pkt, void **payload, time_t tleft) { ssize_t n; struct ether_arp *ah; u_int16_t etype; /* host order */ + void *ptr; #ifdef ARP_DEBUG if (debug) printf("arprecv: "); #endif - n = readether(d, pkt, len, tleft, &etype); + ptr = NULL; + n = readether(d, &ptr, (void **)&ah, tleft, &etype); errno = 0; /* XXX */ if (n == -1 || n < sizeof(struct ether_arp)) { #ifdef ARP_DEBUG if (debug) printf("bad len=%d\n", n); #endif + free(ptr); return (-1); } @@ -196,12 +189,11 @@ arprecv(d, pkt, len, tleft) if (debug) printf("not arp type=%d\n", etype); #endif + free(ptr); return (-1); } /* Ethernet address now checked in readether() */ - - ah = (struct ether_arp *)pkt; if (ah->arp_hrd != htons(ARPHRD_ETHER) || ah->arp_pro != htons(ETHERTYPE_IP) || ah->arp_hln != sizeof(ah->arp_sha) || @@ -211,6 +203,7 @@ arprecv(d, pkt, len, tleft) if (debug) printf("bad hrd/pro/hln/pln\n"); #endif + free(ptr); return (-1); } @@ -220,6 +213,7 @@ arprecv(d, pkt, len, tleft) printf("is request\n"); #endif arp_reply(d, ah); + free(ptr); return (-1); } @@ -228,6 +222,7 @@ arprecv(d, pkt, len, tleft) if (debug) printf("not ARP reply\n"); #endif + free(ptr); return (-1); } @@ -239,6 +234,7 @@ arprecv(d, pkt, len, tleft) if (debug) printf("unwanted address\n"); #endif + free(ptr); return (-1); } /* We don't care who the reply was sent to. */ @@ -248,6 +244,8 @@ arprecv(d, pkt, len, tleft) if (debug) printf("got it\n"); #endif + *pkt = ptr; + *payload = ah; return (n); } @@ -256,9 +254,7 @@ arprecv(d, pkt, len, tleft) * Notes: Re-uses buffer. Pad to length = 46. */ void -arp_reply(d, pkt) - struct iodesc *d; - void *pkt; /* the request */ +arp_reply(struct iodesc *d, void *pkt) { struct ether_arp *arp = pkt; diff --git a/lib/libstand/bootp.c b/lib/libstand/bootp.c index 479405debe22a..ec3cd7b5dc26b 100644 --- a/lib/libstand/bootp.c +++ b/lib/libstand/bootp.c @@ -38,6 +38,7 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); +#include <stddef.h> #include <sys/types.h> #include <sys/limits.h> #include <sys/endian.h> @@ -72,7 +73,7 @@ static char vm_cmu[4] = VM_CMU; /* Local forwards */ static ssize_t bootpsend(struct iodesc *, void *, size_t); -static ssize_t bootprecv(struct iodesc *, void *, size_t, time_t); +static ssize_t bootprecv(struct iodesc *, void **, void **, time_t); static int vend_rfc1048(u_char *, u_int); #ifdef BOOTP_VEND_CMU static void vend_cmu(u_char *); @@ -89,23 +90,21 @@ static void setenv_(u_char *cp, u_char *ep, struct dhcp_opt *opts); static char expected_dhcpmsgtype = -1, dhcp_ok; struct in_addr dhcp_serverip; #endif +struct bootp *bootp_response; +size_t bootp_response_size; /* Fetch required bootp infomation */ void -bootp(sock, flag) - int sock; - int flag; +bootp(int sock, int flag) { + void *pkt; struct iodesc *d; struct bootp *bp; struct { u_char header[HEADER_SIZE]; struct bootp wbootp; } wbuf; - struct { - u_char header[HEADER_SIZE]; - struct bootp rbootp; - } rbuf; + struct bootp *rbootp; #ifdef BOOTP_DEBUG if (debug) @@ -148,16 +147,15 @@ bootp(sock, flag) bp->bp_vend[8] = 9; bcopy("PXEClient", &bp->bp_vend[9], 9); bp->bp_vend[18] = TAG_PARAM_REQ; - bp->bp_vend[19] = 8; + bp->bp_vend[19] = 7; bp->bp_vend[20] = TAG_ROOTPATH; - bp->bp_vend[21] = TAG_TFTP_SERVER; - bp->bp_vend[22] = TAG_HOSTNAME; - bp->bp_vend[23] = TAG_SWAPSERVER; - bp->bp_vend[24] = TAG_GATEWAY; - bp->bp_vend[25] = TAG_SUBNET_MASK; - bp->bp_vend[26] = TAG_INTF_MTU; - bp->bp_vend[27] = TAG_SERVERID; - bp->bp_vend[28] = TAG_END; + bp->bp_vend[21] = TAG_HOSTNAME; + bp->bp_vend[22] = TAG_SWAPSERVER; + bp->bp_vend[23] = TAG_GATEWAY; + bp->bp_vend[24] = TAG_SUBNET_MASK; + bp->bp_vend[25] = TAG_INTF_MTU; + bp->bp_vend[26] = TAG_SERVERID; + bp->bp_vend[27] = TAG_END; } else bp->bp_vend[7] = TAG_END; #else @@ -176,8 +174,7 @@ bootp(sock, flag) if(sendrecv(d, bootpsend, bp, sizeof(*bp), - bootprecv, &rbuf.rbootp, sizeof(rbuf.rbootp)) - == -1) { + bootprecv, &pkt, (void **)&rbootp) == -1) { printf("bootp: no reply\n"); return; } @@ -188,7 +185,7 @@ bootp(sock, flag) bp->bp_vend[6] = DHCPREQUEST; bp->bp_vend[7] = TAG_REQ_ADDR; bp->bp_vend[8] = 4; - bcopy(&rbuf.rbootp.bp_yiaddr, &bp->bp_vend[9], 4); + bcopy(&rbootp->bp_yiaddr, &bp->bp_vend[9], 4); bp->bp_vend[13] = TAG_SERVERID; bp->bp_vend[14] = 4; bcopy(&dhcp_serverip.s_addr, &bp->bp_vend[15], 4); @@ -206,20 +203,21 @@ bootp(sock, flag) expected_dhcpmsgtype = DHCPACK; + free(pkt); if(sendrecv(d, bootpsend, bp, sizeof(*bp), - bootprecv, &rbuf.rbootp, sizeof(rbuf.rbootp)) - == -1) { + bootprecv, &pkt, (void **)&rbootp) == -1) { printf("DHCPREQUEST failed\n"); return; } } #endif - myip = d->myip = rbuf.rbootp.bp_yiaddr; - servip = rbuf.rbootp.bp_siaddr; - if(rootip.s_addr == INADDR_ANY) rootip = servip; - bcopy(rbuf.rbootp.bp_file, bootfile, sizeof(bootfile)); + myip = d->myip = rbootp->bp_yiaddr; + servip = rbootp->bp_siaddr; + if (rootip.s_addr == INADDR_ANY) + rootip = servip; + bcopy(rbootp->bp_file, bootfile, sizeof(bootfile)); bootfile[sizeof(bootfile) - 1] = '\0'; if (!netmask) { @@ -259,14 +257,12 @@ bootp(sock, flag) /* Bump xid so next request will be unique. */ ++d->xid; + free(pkt); } /* Transmit a bootp request */ static ssize_t -bootpsend(d, pkt, len) - struct iodesc *d; - void *pkt; - size_t len; +bootpsend(struct iodesc *d, void *pkt, size_t len) { struct bootp *bp; @@ -287,30 +283,25 @@ bootpsend(d, pkt, len) } static ssize_t -bootprecv(d, pkt, len, tleft) -struct iodesc *d; -void *pkt; -size_t len; -time_t tleft; +bootprecv(struct iodesc *d, void **pkt, void **payload, time_t tleft) { ssize_t n; struct bootp *bp; + void *ptr; -#ifdef BOOTP_DEBUGx +#ifdef BOOTP_DEBUG if (debug) printf("bootp_recvoffer: called\n"); #endif - n = readudp(d, pkt, len, tleft); + ptr = NULL; + n = readudp(d, &ptr, (void **)&bp, tleft); if (n == -1 || n < sizeof(struct bootp) - BOOTP_VENDSIZE) goto bad; - bp = (struct bootp *)pkt; - #ifdef BOOTP_DEBUG if (debug) - printf("bootprecv: checked. bp = 0x%lx, n = %d\n", - (long)bp, (int)n); + printf("bootprecv: checked. bp = %p, n = %zd\n", bp, n); #endif if (bp->bp_xid != htonl(d->xid)) { #ifdef BOOTP_DEBUG @@ -329,8 +320,21 @@ time_t tleft; /* Suck out vendor info */ if (bcmp(vm_rfc1048, bp->bp_vend, sizeof(vm_rfc1048)) == 0) { - if(vend_rfc1048(bp->bp_vend, sizeof(bp->bp_vend)) != 0) + int vsize = n - offsetof(struct bootp, bp_vend); + if (vend_rfc1048(bp->bp_vend, vsize) != 0) goto bad; + + /* Save copy of bootp reply or DHCP ACK message */ + if (bp->bp_op == BOOTREPLY && + ((dhcp_ok == 1 && expected_dhcpmsgtype == DHCPACK) || + dhcp_ok == 0)) { + free(bootp_response); + bootp_response = malloc(n); + if (bootp_response != NULL) { + bootp_response_size = n; + bcopy(bp, bootp_response, bootp_response_size); + } + } } #ifdef BOOTP_VEND_CMU else if (bcmp(vm_cmu, bp->bp_vend, sizeof(vm_cmu)) == 0) @@ -339,8 +343,11 @@ time_t tleft; else printf("bootprecv: unknown vendor 0x%lx\n", (long)bp->bp_vend); - return(n); + *pkt = ptr; + *payload = bp; + return (n); bad: + free(ptr); errno = 0; return (-1); } @@ -357,9 +364,7 @@ dhcp_try_rfc1048(u_char *cp, u_int len) } static int -vend_rfc1048(cp, len) - u_char *cp; - u_int len; +vend_rfc1048(u_char *cp, u_int len) { u_char *ep; int size; @@ -438,10 +443,6 @@ vend_rfc1048(cp, len) bcopy(cp, &dhcp_serverip.s_addr, sizeof(dhcp_serverip.s_addr)); } - if (tag == TAG_TFTP_SERVER) { - bcopy(cp, &tftpip.s_addr, - sizeof(tftpip.s_addr)); - } #endif cp += size; } @@ -450,8 +451,7 @@ vend_rfc1048(cp, len) #ifdef BOOTP_VEND_CMU static void -vend_cmu(cp) - u_char *cp; +vend_cmu(u_char *cp) { struct cmu_vend *vp; diff --git a/lib/libstand/bootp.h b/lib/libstand/bootp.h index 4a57df26e1bd6..9fb746f49163e 100644 --- a/lib/libstand/bootp.h +++ b/lib/libstand/bootp.h @@ -108,7 +108,6 @@ struct bootp { #define TAG_T2 ((unsigned char) 59) #define TAG_CLASSID ((unsigned char) 60) #define TAG_CLIENTID ((unsigned char) 61) -#define TAG_TFTP_SERVER ((unsigned char) 150) #endif #define TAG_END ((unsigned char) 255) @@ -148,6 +147,10 @@ struct cmu_vend { /* v_flags values */ #define VF_SMASK 1 /* Subnet mask field contains valid data */ +/* cached bootp response/dhcp ack */ +extern struct bootp *bootp_response; +extern size_t bootp_response_size; + int dhcp_try_rfc1048(u_char *cp, u_int len); #endif /* _BOOTP_H_ */ diff --git a/lib/libstand/bootparam.c b/lib/libstand/bootparam.c index e4d2f1022c925..1de2d53d77fe2 100644 --- a/lib/libstand/bootparam.c +++ b/lib/libstand/bootparam.c @@ -104,8 +104,7 @@ int xdr_string_decode(char **p, char *str, int *len_p); * know about us (don't want to broadcast a getport call). */ int -bp_whoami(sockfd) - int sockfd; +bp_whoami(int sockfd) { /* RPC structures for PMAPPROC_CALLIT */ struct args { @@ -126,22 +125,19 @@ bp_whoami(sockfd) n_long h[RPC_HEADER_WORDS]; struct args d; } sdata; - struct { - n_long h[RPC_HEADER_WORDS]; - struct repl d; - } rdata; char *send_tail, *recv_head; struct iodesc *d; - int len, x; + void *pkt; + int len, x, rc; RPC_PRINTF(("bp_whoami: myip=%s\n", inet_ntoa(myip))); + rc = -1; if (!(d = socktodesc(sockfd))) { RPC_PRINTF(("bp_whoami: bad socket. %d\n", sockfd)); - return (-1); + return (rc); } args = &sdata.d; - repl = &rdata.d; /* * Build request args for PMAPPROC_CALLIT. @@ -156,19 +152,19 @@ bp_whoami(sockfd) * append encapsulated data (client IP address) */ if (xdr_inaddr_encode(&send_tail, myip)) - return (-1); + return (rc); /* RPC: portmap/callit */ d->myport = htons(--rpc_port); d->destip.s_addr = INADDR_BROADCAST; /* XXX: subnet bcast? */ /* rpc_call will set d->destport */ + pkt = NULL; len = rpc_call(d, PMAPPROG, PMAPVERS, PMAPPROC_CALLIT, - args, send_tail - (char*)args, - repl, sizeof(*repl)); + args, send_tail - (char*)args, (void **)&repl, &pkt); if (len < 8) { printf("bootparamd: 'whoami' call failed\n"); - return (-1); + goto done; } /* Save bootparam server address (from IP header). */ @@ -196,7 +192,7 @@ bp_whoami(sockfd) x = ntohl(repl->encap_len); if (len < x) { printf("bp_whoami: short reply, %d < %d\n", len, x); - return (-1); + goto done; } recv_head = (char*) repl->capsule; @@ -204,24 +200,27 @@ bp_whoami(sockfd) hostnamelen = MAXHOSTNAMELEN-1; if (xdr_string_decode(&recv_head, hostname, &hostnamelen)) { RPC_PRINTF(("bp_whoami: bad hostname\n")); - return (-1); + goto done; } /* domain name */ domainnamelen = MAXHOSTNAMELEN-1; if (xdr_string_decode(&recv_head, domainname, &domainnamelen)) { RPC_PRINTF(("bp_whoami: bad domainname\n")); - return (-1); + goto done; } /* gateway address */ if (xdr_inaddr_decode(&recv_head, &gateip)) { RPC_PRINTF(("bp_whoami: bad gateway\n")); - return (-1); + goto done; } /* success */ - return(0); + rc = 0; +done: + free(pkt); + return (rc); } @@ -233,25 +232,18 @@ bp_whoami(sockfd) * server pathname */ int -bp_getfile(sockfd, key, serv_addr, pathname) - int sockfd; - char *key; - char *pathname; - struct in_addr *serv_addr; +bp_getfile(int sockfd, char *key, struct in_addr *serv_addr, char *pathname) { struct { n_long h[RPC_HEADER_WORDS]; n_long d[64]; } sdata; - struct { - n_long h[RPC_HEADER_WORDS]; - n_long d[128]; - } rdata; + void *pkt; char serv_name[FNAME_SIZE]; - char *send_tail, *recv_head; + char *rdata, *send_tail; /* misc... */ struct iodesc *d; - int sn_len, path_len, rlen; + int rc = -1, sn_len, path_len, rlen; if (!(d = socktodesc(sockfd))) { RPC_PRINTF(("bp_getfile: bad socket. %d\n", sockfd)); @@ -259,7 +251,6 @@ bp_getfile(sockfd, key, serv_addr, pathname) } send_tail = (char*) sdata.d; - recv_head = (char*) rdata.d; /* * Build request message. @@ -281,17 +272,16 @@ bp_getfile(sockfd, key, serv_addr, pathname) d->myport = htons(--rpc_port); d->destip = bp_server_addr; /* rpc_call will set d->destport */ - + pkt = NULL; rlen = rpc_call(d, BOOTPARAM_PROG, BOOTPARAM_VERS, BOOTPARAM_GETFILE, sdata.d, send_tail - (char*)sdata.d, - rdata.d, sizeof(rdata.d)); + (void **)&rdata, &pkt); if (rlen < 4) { RPC_PRINTF(("bp_getfile: short reply\n")); errno = EBADRPC; - return (-1); + goto done; } - recv_head = (char*) rdata.d; /* * Parse result message. @@ -299,26 +289,29 @@ bp_getfile(sockfd, key, serv_addr, pathname) /* server name */ sn_len = FNAME_SIZE-1; - if (xdr_string_decode(&recv_head, serv_name, &sn_len)) { + if (xdr_string_decode(&rdata, serv_name, &sn_len)) { RPC_PRINTF(("bp_getfile: bad server name\n")); - return (-1); + goto done; } /* server IP address (mountd/NFS) */ - if (xdr_inaddr_decode(&recv_head, serv_addr)) { + if (xdr_inaddr_decode(&rdata, serv_addr)) { RPC_PRINTF(("bp_getfile: bad server addr\n")); - return (-1); + goto done; } /* server pathname */ path_len = MAXPATHLEN-1; - if (xdr_string_decode(&recv_head, pathname, &path_len)) { + if (xdr_string_decode(&rdata, pathname, &path_len)) { RPC_PRINTF(("bp_getfile: bad server path\n")); - return (-1); + goto done; } /* success */ - return(0); + rc = 0; +done: + free(pkt); + return (rc); } @@ -329,17 +322,14 @@ bp_getfile(sockfd, key, serv_addr, pathname) int -xdr_string_encode(pkt, str, len) - char **pkt; - char *str; - int len; +xdr_string_encode(char **pkt, char *str, int len) { - u_int32_t *lenp; + uint32_t *lenp; char *datap; int padlen = (len + 3) & ~3; /* padded length */ /* The data will be int aligned. */ - lenp = (u_int32_t*) *pkt; + lenp = (uint32_t *) *pkt; *pkt += sizeof(*lenp); *lenp = htonl(len); @@ -351,18 +341,15 @@ xdr_string_encode(pkt, str, len) } int -xdr_string_decode(pkt, str, len_p) - char **pkt; - char *str; - int *len_p; /* bufsize - 1 */ +xdr_string_decode(char **pkt, char *str, int *len_p) { - u_int32_t *lenp; + uint32_t *lenp; char *datap; int slen; /* string length */ int plen; /* padded length */ /* The data will be int aligned. */ - lenp = (u_int32_t*) *pkt; + lenp = (uint32_t *) *pkt; *pkt += sizeof(*lenp); slen = ntohl(*lenp); plen = (slen + 3) & ~3; @@ -381,9 +368,7 @@ xdr_string_decode(pkt, str, len_p) int -xdr_inaddr_encode(pkt, ia) - char **pkt; - struct in_addr ia; /* network order */ +xdr_inaddr_encode(char **pkt, struct in_addr ia) { struct xdr_inaddr *xi; u_char *cp; @@ -414,9 +399,7 @@ xdr_inaddr_encode(pkt, ia) } int -xdr_inaddr_decode(pkt, ia) - char **pkt; - struct in_addr *ia; /* network order */ +xdr_inaddr_decode(char **pkt, struct in_addr *ia) { struct xdr_inaddr *xi; u_char *cp; diff --git a/lib/libstand/ether.c b/lib/libstand/ether.c index df42f31a8b71f..3616777e79554 100644 --- a/lib/libstand/ether.c +++ b/lib/libstand/ether.c @@ -54,12 +54,7 @@ __FBSDID("$FreeBSD$"); /* Caller must leave room for ethernet header in front!! */ ssize_t -sendether(d, pkt, len, dea, etype) - struct iodesc *d; - void *pkt; - size_t len; - u_char *dea; - int etype; +sendether(struct iodesc *d, void *pkt, size_t len, uint8_t *dea, int etype) { ssize_t n; struct ether_header *eh; @@ -86,32 +81,31 @@ sendether(d, pkt, len, dea, etype) /* * Get a packet of any Ethernet type, with our address or - * the broadcast address. Save the Ether type in arg 5. - * NOTE: Caller must leave room for the Ether header. + * the broadcast address. Save the Ether type in etype. + * Unless there is an error, we pass the whole packet and the unencapsulated + * data. */ ssize_t -readether(d, pkt, len, tleft, etype) - struct iodesc *d; - void *pkt; - size_t len; - time_t tleft; - u_int16_t *etype; +readether(struct iodesc *d, void **pkt, void **payload, time_t tleft, + uint16_t *etype) { ssize_t n; struct ether_header *eh; + void *ptr; #ifdef ETHER_DEBUG if (debug) printf("readether: called\n"); #endif - eh = (struct ether_header *)pkt - 1; - len += sizeof(*eh); - - n = netif_get(d, eh, len, tleft); - if (n == -1 || n < sizeof(*eh)) + ptr = NULL; + n = netif_get(d, &ptr, tleft); + if (n == -1 || n < sizeof(*eh)) { + free(ptr); return (-1); + } + eh = (struct ether_header *)((uintptr_t)ptr + ETHER_ALIGN); /* Validate Ethernet address. */ if (bcmp(d->myea, eh->ether_dhost, 6) != 0 && bcmp(bcea, eh->ether_dhost, 6) != 0) { @@ -120,8 +114,12 @@ readether(d, pkt, len, tleft, etype) printf("readether: not ours (ea=%s)\n", ether_sprintf(eh->ether_dhost)); #endif + free(ptr); return (-1); } + + *pkt = ptr; + *payload = (void *)((uintptr_t)eh + sizeof(*eh)); *etype = ntohs(eh->ether_type); n -= sizeof(*eh); @@ -133,8 +131,7 @@ readether(d, pkt, len, tleft, etype) */ static char digits[] = "0123456789abcdef"; char * -ether_sprintf(ap) - u_char *ap; +ether_sprintf(u_char *ap) { int i; static char etherbuf[18]; diff --git a/lib/libstand/globals.c b/lib/libstand/globals.c index 83b9f777a8574..8fb56cf23e6b0 100644 --- a/lib/libstand/globals.c +++ b/lib/libstand/globals.c @@ -32,7 +32,6 @@ struct in_addr nameip; /* DNS server ip address */ struct in_addr rootip; /* root ip address */ struct in_addr swapip; /* swap ip address */ struct in_addr gateip; /* gateway ip address */ -struct in_addr tftpip; /* TFTP ip address */ n_long netmask = 0xffffff00; /* subnet or net mask */ u_int intf_mtu; /* interface mtu from bootp/dhcp */ int errno; /* our old friend */ diff --git a/lib/libstand/net.c b/lib/libstand/net.c index 38759a8022737..ae6e74511e127 100644 --- a/lib/libstand/net.c +++ b/lib/libstand/net.c @@ -70,10 +70,10 @@ __FBSDID("$FreeBSD$"); */ ssize_t sendrecv(struct iodesc *d, - ssize_t (*sproc)(struct iodesc *, void *, size_t), - void *sbuf, size_t ssize, - ssize_t (*rproc)(struct iodesc *, void *, size_t, time_t), - void *rbuf, size_t rsize) + ssize_t (*sproc)(struct iodesc *, void *, size_t), + void *sbuf, size_t ssize, + ssize_t (*rproc)(struct iodesc *, void **, void **, time_t), + void **pkt, void **payload) { ssize_t cc; time_t t, tmo, tlast; @@ -116,7 +116,7 @@ sendrecv(struct iodesc *d, } /* Try to get a packet and process it. */ - cc = (*rproc)(d, rbuf, rsize, tleft); + cc = (*rproc)(d, pkt, payload, tleft); /* Return on data, EOF or real error. */ if (cc != -1 || errno != 0) return (cc); diff --git a/lib/libstand/net.h b/lib/libstand/net.h index 9962057d8ee99..740b5d970f875 100644 --- a/lib/libstand/net.h +++ b/lib/libstand/net.h @@ -91,7 +91,6 @@ extern struct in_addr rootip; extern struct in_addr swapip; extern struct in_addr gateip; extern struct in_addr nameip; -extern struct in_addr tftpip; extern n_long netmask; extern u_int intf_mtu; @@ -107,16 +106,15 @@ int rarp_getipaddress(int); /* Link functions: */ ssize_t sendether(struct iodesc *d, void *pkt, size_t len, u_char *dea, int etype); -ssize_t readether(struct iodesc *d, void *pkt, size_t len, - time_t tleft, u_int16_t *etype); +ssize_t readether(struct iodesc *, void **, void **, time_t, uint16_t *); ssize_t sendudp(struct iodesc *, void *, size_t); -ssize_t readudp(struct iodesc *, void *, size_t, time_t); +ssize_t readudp(struct iodesc *, void **, void **, time_t); ssize_t sendrecv(struct iodesc *, - ssize_t (*)(struct iodesc *, void *, size_t), + ssize_t (*)(struct iodesc *, void *, size_t), void *, size_t, - ssize_t (*)(struct iodesc *, void *, size_t, time_t), - void *, size_t); + ssize_t (*)(struct iodesc *, void **, void **, time_t), + void **, void **); /* bootp/DHCP */ void bootp(int, int); diff --git a/lib/libstand/netif.c b/lib/libstand/netif.c index c260690c503c7..105f9a31ab5da 100644 --- a/lib/libstand/netif.c +++ b/lib/libstand/netif.c @@ -59,7 +59,7 @@ int netif_debug = 0; */ void -netif_init() +netif_init(void) { struct netif_driver *drv; int d, i; @@ -76,13 +76,11 @@ netif_init() } int -netif_match(nif, machdep_hint) - struct netif *nif; - void *machdep_hint; +netif_match(struct netif *nif, void *machdep_hint) { struct netif_driver *drv = nif->nif_driver; -#if 0 +#if NETIF_DEBUG if (netif_debug) printf("%s%d: netif_match (%d)\n", drv->netif_bname, nif->nif_unit, nif->nif_sel); @@ -91,8 +89,7 @@ netif_match(nif, machdep_hint) } struct netif * -netif_select(machdep_hint) - void *machdep_hint; +netif_select(void *machdep_hint) { int d, u, unit_done, s; struct netif_driver *drv; @@ -162,9 +159,7 @@ netif_select(machdep_hint) } int -netif_probe(nif, machdep_hint) - struct netif *nif; - void *machdep_hint; +netif_probe(struct netif *nif, void *machdep_hint) { struct netif_driver *drv = nif->nif_driver; @@ -176,10 +171,7 @@ netif_probe(nif, machdep_hint) } void -netif_attach(nif, desc, machdep_hint) - struct netif *nif; - struct iodesc *desc; - void *machdep_hint; +netif_attach(struct netif *nif, struct iodesc *desc, void *machdep_hint) { struct netif_driver *drv = nif->nif_driver; @@ -199,8 +191,7 @@ netif_attach(nif, desc, machdep_hint) } void -netif_detach(nif) - struct netif *nif; +netif_detach(struct netif *nif) { struct netif_driver *drv = nif->nif_driver; @@ -217,11 +208,7 @@ netif_detach(nif) } ssize_t -netif_get(desc, pkt, len, timo) - struct iodesc *desc; - void *pkt; - size_t len; - time_t timo; +netif_get(struct iodesc *desc, void **pkt, time_t timo) { #ifdef NETIF_DEBUG struct netif *nif = desc->io_netif; @@ -238,20 +225,17 @@ netif_get(desc, pkt, len, timo) panic("%s%d: no netif_get support\n", drv->netif_bname, nif->nif_unit); #endif - rv = drv->netif_get(desc, pkt, len, timo); + rv = drv->netif_get(desc, pkt, timo); #ifdef NETIF_DEBUG if (netif_debug) printf("%s%d: netif_get returning %d\n", drv->netif_bname, nif->nif_unit, (int)rv); #endif - return rv; + return (rv); } ssize_t -netif_put(desc, pkt, len) - struct iodesc *desc; - void *pkt; - size_t len; +netif_put(struct iodesc *desc, void *pkt, size_t len) { #ifdef NETIF_DEBUG struct netif *nif = desc->io_netif; @@ -274,12 +258,11 @@ netif_put(desc, pkt, len) printf("%s%d: netif_put returning %d\n", drv->netif_bname, nif->nif_unit, (int)rv); #endif - return rv; + return (rv); } struct iodesc * -socktodesc(sock) - int sock; +socktodesc(int sock) { if (sock >= SOPEN_MAX) { errno = EBADF; @@ -289,8 +272,7 @@ socktodesc(sock) } int -netif_open(machdep_hint) - void *machdep_hint; +netif_open(void *machdep_hint) { int fd; struct iodesc *s; @@ -313,23 +295,22 @@ fnd: printf("netboot: couldn't probe %s%d\n", nif->nif_driver->netif_bname, nif->nif_unit); errno = EINVAL; - return(-1); + return (-1); } netif_attach(nif, s, machdep_hint); - return(fd); + return (fd); } int -netif_close(sock) - int sock; +netif_close(int sock) { if (sock >= SOPEN_MAX) { errno = EBADF; - return(-1); + return (-1); } netif_detach(sockets[sock].io_netif); sockets[sock].io_netif = (struct netif *)0; - return(0); + return (0); } diff --git a/lib/libstand/netif.h b/lib/libstand/netif.h index dac285107a735..44165ab0d88ac 100644 --- a/lib/libstand/netif.h +++ b/lib/libstand/netif.h @@ -6,15 +6,13 @@ #define __SYS_LIBNETBOOT_NETIF_H #include "iodesc.h" -#define NENTS(x) sizeof(x)/sizeof(x[0]) - struct netif_driver { const char *netif_bname; int (*netif_match)(struct netif *, void *); int (*netif_probe)(struct netif *, void *); void (*netif_init)(struct iodesc *, void *); - int (*netif_get)(struct iodesc *, void *, size_t, time_t); - int (*netif_put)(struct iodesc *, void *, size_t); + ssize_t (*netif_get)(struct iodesc *, void **, time_t); + ssize_t (*netif_put)(struct iodesc *, void *, size_t); void (*netif_end)(struct netif *); struct netif_dif *netif_ifs; int netif_nifs; @@ -56,7 +54,7 @@ struct netif *netif_select(void *); int netif_probe(struct netif *, void *); void netif_attach(struct netif *, struct iodesc *, void *); void netif_detach(struct netif *); -ssize_t netif_get(struct iodesc *, void *, size_t, time_t); +ssize_t netif_get(struct iodesc *, void **, time_t); ssize_t netif_put(struct iodesc *, void *, size_t); int netif_open(void *); diff --git a/lib/libstand/nfs.c b/lib/libstand/nfs.c index 8a7e111fcdd71..6c387b8d8ef7c 100644 --- a/lib/libstand/nfs.c +++ b/lib/libstand/nfs.c @@ -185,6 +185,7 @@ set_nfs_read_size(void) int nfs_getrootfh(struct iodesc *d, char *path, uint32_t *fhlenp, u_char *fhp) { + void *pkt = NULL; int len; struct args { uint32_t len; @@ -201,10 +202,6 @@ nfs_getrootfh(struct iodesc *d, char *path, uint32_t *fhlenp, u_char *fhp) uint32_t h[RPC_HEADER_WORDS]; struct args d; } sdata; - struct { - uint32_t h[RPC_HEADER_WORDS]; - struct repl d; - } rdata; size_t cc; #ifdef NFS_DEBUG @@ -213,7 +210,6 @@ nfs_getrootfh(struct iodesc *d, char *path, uint32_t *fhlenp, u_char *fhp) #endif args = &sdata.d; - repl = &rdata.d; bzero(args, sizeof(*args)); len = strlen(path); @@ -224,18 +220,25 @@ nfs_getrootfh(struct iodesc *d, char *path, uint32_t *fhlenp, u_char *fhp) len = sizeof(uint32_t) + roundup(len, sizeof(uint32_t)); cc = rpc_call(d, RPCPROG_MNT, RPCMNT_VER3, RPCMNT_MOUNT, - args, len, repl, sizeof(*repl)); - if (cc == -1) + args, len, (void **)&repl, &pkt); + if (cc == -1) { + free(pkt); /* errno was set by rpc_call */ return (errno); - if (cc < 2 * sizeof (uint32_t)) + } + if (cc < 2 * sizeof (uint32_t)) { + free(pkt); return (EBADRPC); - if (repl->errno != 0) + } + if (repl->errno != 0) { + free(pkt); return (ntohl(repl->errno)); + } *fhlenp = ntohl(repl->fhsize); bcopy(repl->fh, fhp, *fhlenp); set_nfs_read_size(); + free(pkt); return (0); } @@ -246,6 +249,7 @@ nfs_getrootfh(struct iodesc *d, char *path, uint32_t *fhlenp, u_char *fhp) int nfs_lookupfh(struct nfs_iodesc *d, const char *name, struct nfs_iodesc *newfd) { + void *pkt = NULL; int len, rlen, pos; struct args { uint32_t fhsize; @@ -263,10 +267,6 @@ nfs_lookupfh(struct nfs_iodesc *d, const char *name, struct nfs_iodesc *newfd) uint32_t h[RPC_HEADER_WORDS]; struct args d; } sdata; - struct { - uint32_t h[RPC_HEADER_WORDS]; - struct repl d; - } rdata; ssize_t cc; #ifdef NFS_DEBUG @@ -275,7 +275,6 @@ nfs_lookupfh(struct nfs_iodesc *d, const char *name, struct nfs_iodesc *newfd) #endif args = &sdata.d; - repl = &rdata.d; bzero(args, sizeof(*args)); args->fhsize = htonl(d->fhsize); @@ -289,23 +288,30 @@ nfs_lookupfh(struct nfs_iodesc *d, const char *name, struct nfs_iodesc *newfd) len = sizeof(uint32_t) + pos * sizeof(uint32_t) + roundup(len, sizeof(uint32_t)); - rlen = sizeof(*repl); - cc = rpc_call(d->iodesc, NFS_PROG, NFS_VER3, NFSPROCV3_LOOKUP, - args, len, repl, rlen); - if (cc == -1) + args, len, (void **)&repl, &pkt); + if (cc == -1) { + free(pkt); return (errno); /* XXX - from rpc_call */ - if (cc < 2 * sizeof(uint32_t)) + } + if (cc < 2 * sizeof(uint32_t)) { + free(pkt); return (EIO); - if (repl->errno != 0) + } + if (repl->errno != 0) { + free(pkt); /* saerrno.h now matches NFS error numbers. */ return (ntohl(repl->errno)); + } newfd->fhsize = ntohl(repl->fhsize); bcopy(repl->fhplusattr, &newfd->fh, newfd->fhsize); pos = roundup(newfd->fhsize, sizeof(uint32_t)) / sizeof(uint32_t); - if (repl->fhplusattr[pos++] == 0) + if (repl->fhplusattr[pos++] == 0) { + free(pkt); return (EIO); + } bcopy(&repl->fhplusattr[pos], &newfd->fa, sizeof(newfd->fa)); + free(pkt); return (0); } @@ -316,6 +322,7 @@ nfs_lookupfh(struct nfs_iodesc *d, const char *name, struct nfs_iodesc *newfd) int nfs_readlink(struct nfs_iodesc *d, char *buf) { + void *pkt = NULL; struct args { uint32_t fhsize; u_char fh[NFS_V3MAXFHSIZE]; @@ -331,11 +338,8 @@ nfs_readlink(struct nfs_iodesc *d, char *buf) uint32_t h[RPC_HEADER_WORDS]; struct args d; } sdata; - struct { - uint32_t h[RPC_HEADER_WORDS]; - struct repl d; - } rdata; ssize_t cc; + int rc = 0; #ifdef NFS_DEBUG if (debug) @@ -343,32 +347,41 @@ nfs_readlink(struct nfs_iodesc *d, char *buf) #endif args = &sdata.d; - repl = &rdata.d; bzero(args, sizeof(*args)); args->fhsize = htonl(d->fhsize); bcopy(d->fh, args->fh, d->fhsize); cc = rpc_call(d->iodesc, NFS_PROG, NFS_VER3, NFSPROCV3_READLINK, args, sizeof(uint32_t) + roundup(d->fhsize, sizeof(uint32_t)), - repl, sizeof(*repl)); + (void **)&repl, &pkt); if (cc == -1) return (errno); - if (cc < 2 * sizeof(uint32_t)) - return (EIO); + if (cc < 2 * sizeof(uint32_t)) { + rc = EIO; + goto done; + } - if (repl->errno != 0) - return (ntohl(repl->errno)); + if (repl->errno != 0) { + rc = ntohl(repl->errno); + goto done; + } - if (repl->ok == 0) - return (EIO); + if (repl->ok == 0) { + rc = EIO; + goto done; + } repl->len = ntohl(repl->len); - if (repl->len > NFS_MAXPATHLEN) - return (ENAMETOOLONG); + if (repl->len > NFS_MAXPATHLEN) { + rc = ENAMETOOLONG; + goto done; + } bcopy(repl->path, buf, repl->len); buf[repl->len] = 0; +done: + free(pkt); return (0); } #endif @@ -380,6 +393,7 @@ nfs_readlink(struct nfs_iodesc *d, char *buf) ssize_t nfs_readdata(struct nfs_iodesc *d, off_t off, void *addr, size_t len) { + void *pkt = NULL; struct args { uint32_t fhsize; uint32_t fhoffcnt[NFS_V3MAXFHSIZE / sizeof(uint32_t) + 3]; @@ -397,16 +411,11 @@ nfs_readdata(struct nfs_iodesc *d, off_t off, void *addr, size_t len) uint32_t h[RPC_HEADER_WORDS]; struct args d; } sdata; - struct { - uint32_t h[RPC_HEADER_WORDS]; - struct repl d; - } rdata; size_t cc; long x; int hlen, rlen, pos; args = &sdata.d; - repl = &rdata.d; bzero(args, sizeof(*args)); args->fhsize = htonl(d->fhsize); @@ -421,16 +430,19 @@ nfs_readdata(struct nfs_iodesc *d, off_t off, void *addr, size_t len) cc = rpc_call(d->iodesc, NFS_PROG, NFS_VER3, NFSPROCV3_READ, args, 4 * sizeof(uint32_t) + roundup(d->fhsize, sizeof(uint32_t)), - repl, sizeof(*repl)); - if (cc == -1) + (void **)&repl, &pkt); + if (cc == -1) { /* errno was already set by rpc_call */ return (-1); + } if (cc < hlen) { errno = EBADRPC; + free(pkt); return (-1); } if (repl->errno != 0) { errno = ntohl(repl->errno); + free(pkt); return (-1); } rlen = cc - hlen; @@ -438,9 +450,11 @@ nfs_readdata(struct nfs_iodesc *d, off_t off, void *addr, size_t len) if (rlen < x) { printf("nfsread: short packet, %d < %ld\n", rlen, x); errno = EBADRPC; + free(pkt); return (-1); } bcopy(repl->data, addr, x); + free(pkt); return (x); } @@ -481,17 +495,8 @@ nfs_open(const char *upath, struct open_file *f) return (ENXIO); } - /* - * This is silly - we should look at dv_type but that value is - * arch dependant and we can't use it here. - */ -#ifndef __i386__ - if (strcmp(f->f_dev->dv_name, "net") != 0) - return (EINVAL); -#else - if (strcmp(f->f_dev->dv_name, "pxe") != 0) + if (f->f_dev->dv_type != DEVT_NET) return (EINVAL); -#endif if (!(desc = socktodesc(*(int *)(f->f_devdata)))) return (EINVAL); @@ -660,9 +665,8 @@ nfs_close(struct open_file *f) printf("nfs_close: fp=0x%lx\n", (u_long)fp); #endif - if (fp) - free(fp); - f->f_fsdata = (void *)0; + free(fp); + f->f_fsdata = NULL; return (0); } @@ -773,11 +777,12 @@ nfs_readdir(struct open_file *f, struct dirent *d) struct nfs_iodesc *fp = (struct nfs_iodesc *)f->f_fsdata; struct nfsv3_readdir_repl *repl; struct nfsv3_readdir_entry *rent; + static void *pkt = NULL; static char *buf; static struct nfs_iodesc *pfp = NULL; static uint64_t cookie = 0; size_t cc; - int pos; + int pos, rc; struct args { uint32_t fhsize; @@ -787,14 +792,12 @@ nfs_readdir(struct open_file *f, struct dirent *d) uint32_t h[RPC_HEADER_WORDS]; struct args d; } sdata; - static struct { - uint32_t h[RPC_HEADER_WORDS]; - u_char d[NFS_READDIRSIZE]; - } rdata; if (fp != pfp || fp->off != cookie) { pfp = NULL; refill: + free(pkt); + pkt = NULL; args = &sdata.d; bzero(args, sizeof(*args)); @@ -810,11 +813,16 @@ nfs_readdir(struct open_file *f, struct dirent *d) cc = rpc_call(fp->iodesc, NFS_PROG, NFS_VER3, NFSPROCV3_READDIR, args, 6 * sizeof(uint32_t) + roundup(fp->fhsize, sizeof(uint32_t)), - rdata.d, sizeof(rdata.d)); - buf = rdata.d; + (void **)&buf, &pkt); + if (cc == -1) { + rc = errno; + goto err; + } repl = (struct nfsv3_readdir_repl *)buf; - if (repl->errno != 0) - return (ntohl(repl->errno)); + if (repl->errno != 0) { + rc = ntohl(repl->errno); + goto err; + } pfp = fp; cookie = fp->off; fp->cookie = ((uint64_t)ntohl(repl->cookiev0) << 32) | @@ -826,8 +834,8 @@ nfs_readdir(struct open_file *f, struct dirent *d) if (rent->follows == 0) { /* fid0 is actually eof */ if (rent->fid0 != 0) { - cookie = 0; - return (ENOENT); + rc = ENOENT; + goto err; } goto refill; } @@ -842,4 +850,11 @@ nfs_readdir(struct open_file *f, struct dirent *d) pos += 2; buf = (u_char *)&rent->nameplus[pos]; return (0); + +err: + free(pkt); + pkt = NULL; + pfp = NULL; + cookie = 0; + return (rc); } diff --git a/lib/libstand/rarp.c b/lib/libstand/rarp.c index 9ec050da1fbf4..8d996d04a3f6b 100644 --- a/lib/libstand/rarp.c +++ b/lib/libstand/rarp.c @@ -54,17 +54,17 @@ __FBSDID("$FreeBSD$"); static ssize_t rarpsend(struct iodesc *, void *, size_t); -static ssize_t rarprecv(struct iodesc *, void *, size_t, time_t); +static ssize_t rarprecv(struct iodesc *, void **, void **, time_t); /* * Ethernet (Reverse) Address Resolution Protocol (see RFC 903, and 826). */ int -rarp_getipaddress(sock) - int sock; +rarp_getipaddress(int sock) { struct iodesc *d; struct ether_arp *ap; + void *pkt; struct { u_char header[ETHER_SIZE]; struct { @@ -72,13 +72,6 @@ rarp_getipaddress(sock) u_char pad[18]; /* 60 - sizeof(arp) */ } data; } wbuf; - struct { - u_char header[ETHER_SIZE]; - struct { - struct ether_arp arp; - u_char pad[24]; /* extra space */ - } data; - } rbuf; #ifdef RARP_DEBUG if (debug) @@ -102,21 +95,21 @@ rarp_getipaddress(sock) ap->arp_op = htons(ARPOP_REVREQUEST); bcopy(d->myea, ap->arp_sha, 6); bcopy(d->myea, ap->arp_tha, 6); + pkt = NULL; if (sendrecv(d, rarpsend, &wbuf.data, sizeof(wbuf.data), - rarprecv, &rbuf.data, sizeof(rbuf.data)) < 0) - { + rarprecv, &pkt, (void *)&ap) < 0) { printf("No response for RARP request\n"); return (-1); } - ap = &rbuf.data.arp; bcopy(ap->arp_tpa, (char *)&myip, sizeof(myip)); #if 0 /* XXX - Can NOT assume this is our root server! */ bcopy(ap->arp_spa, (char *)&rootip, sizeof(rootip)); #endif + free(pkt); /* Compute our "natural" netmask. */ if (IN_CLASSA(myip.s_addr)) @@ -134,10 +127,7 @@ rarp_getipaddress(sock) * Broadcast a RARP request (i.e. who knows who I am) */ static ssize_t -rarpsend(d, pkt, len) - struct iodesc *d; - void *pkt; - size_t len; +rarpsend(struct iodesc *d, void *pkt, size_t len) { #ifdef RARP_DEBUG @@ -153,28 +143,26 @@ rarpsend(d, pkt, len) * else -1 (and errno == 0) */ static ssize_t -rarprecv(d, pkt, len, tleft) - struct iodesc *d; - void *pkt; - size_t len; - time_t tleft; +rarprecv(struct iodesc *d, void **pkt, void **payload, time_t tleft) { ssize_t n; struct ether_arp *ap; - u_int16_t etype; /* host order */ + void *ptr = NULL; + uint16_t etype; /* host order */ #ifdef RARP_DEBUG if (debug) printf("rarprecv: "); #endif - n = readether(d, pkt, len, tleft, &etype); + n = readether(d, ptr, (void **)&ap, tleft, &etype); errno = 0; /* XXX */ if (n == -1 || n < sizeof(struct ether_arp)) { #ifdef RARP_DEBUG if (debug) printf("bad len=%d\n", n); #endif + free(ptr); return (-1); } @@ -183,10 +171,10 @@ rarprecv(d, pkt, len, tleft) if (debug) printf("bad type=0x%x\n", etype); #endif + free(ptr); return (-1); } - ap = (struct ether_arp *)pkt; if (ap->arp_hrd != htons(ARPHRD_ETHER) || ap->arp_pro != htons(ETHERTYPE_IP) || ap->arp_hln != sizeof(ap->arp_sha) || @@ -196,6 +184,7 @@ rarprecv(d, pkt, len, tleft) if (debug) printf("bad hrd/pro/hln/pln\n"); #endif + free(ptr); return (-1); } @@ -204,6 +193,7 @@ rarprecv(d, pkt, len, tleft) if (debug) printf("bad op=0x%x\n", ntohs(ap->arp_op)); #endif + free(ptr); return (-1); } @@ -213,6 +203,7 @@ rarprecv(d, pkt, len, tleft) if (debug) printf("unwanted address\n"); #endif + free(ptr); return (-1); } @@ -221,5 +212,7 @@ rarprecv(d, pkt, len, tleft) if (debug) printf("got it\n"); #endif + *pkt = ptr; + *payload = ap; return (n); } diff --git a/lib/libstand/rpc.c b/lib/libstand/rpc.c index 79ee45af0ed5c..932fdc98b9036 100644 --- a/lib/libstand/rpc.c +++ b/lib/libstand/rpc.c @@ -97,7 +97,7 @@ struct rpc_reply { }; /* Local forwards */ -static ssize_t recvrpc(struct iodesc *, void *, size_t, time_t); +static ssize_t recvrpc(struct iodesc *, void **, void **, time_t); static int rpc_getport(struct iodesc *, n_long, n_long); int rpc_xid; @@ -109,14 +109,14 @@ int rpc_port = 0x400; /* predecrement */ */ ssize_t rpc_call(struct iodesc *d, n_long prog, n_long vers, n_long proc, - void *sdata, size_t slen, void *rdata, size_t rlen) + void *sdata, size_t slen, void **rdata, void **pkt) { - ssize_t cc; + ssize_t cc, rsize; struct auth_info *auth; struct rpc_call *call; struct rpc_reply *reply; char *send_head, *send_tail; - char *recv_head, *recv_tail; + void *ptr; n_long x; int port; /* host order */ @@ -145,7 +145,6 @@ rpc_call(struct iodesc *d, n_long prog, n_long vers, n_long proc, auth->authtype = htonl(RPCAUTH_NULL); auth->authlen = 0; -#if 1 /* Auth credentials: always auth unix (as root) */ send_head -= sizeof(struct auth_unix); bzero(send_head, sizeof(struct auth_unix)); @@ -153,13 +152,6 @@ rpc_call(struct iodesc *d, n_long prog, n_long vers, n_long proc, auth = (struct auth_info *)send_head; auth->authtype = htonl(RPCAUTH_UNIX); auth->authlen = htonl(sizeof(struct auth_unix)); -#else - /* Auth credentials: always auth_null (XXX OK?) */ - send_head -= sizeof(*auth); - auth = send_head; - auth->authtype = htonl(RPCAUTH_NULL); - auth->authlen = 0; -#endif /* RPC call structure. */ send_head -= sizeof(*call); @@ -172,34 +164,28 @@ rpc_call(struct iodesc *d, n_long prog, n_long vers, n_long proc, call->rp_vers = htonl(vers); call->rp_proc = htonl(proc); - /* Make room for the rpc_reply header. */ - recv_head = rdata; - recv_tail = (char *)rdata + rlen; - recv_head -= sizeof(*reply); - + ptr = NULL; cc = sendrecv(d, sendudp, send_head, send_tail - send_head, - recvrpc, recv_head, recv_tail - recv_head); + recvrpc, &ptr, (void **)&reply); #ifdef RPC_DEBUG if (debug) - printf("callrpc: cc=%ld rlen=%lu\n", (long)cc, (u_long)rlen); + printf("callrpc: cc=%zd\n", cc); #endif if (cc == -1) return (-1); if (cc <= sizeof(*reply)) { errno = EBADRPC; + free(ptr); return (-1); } - recv_tail = recv_head + cc; - /* * Check the RPC reply status. * The xid, dir, astatus were already checked. */ - reply = (struct rpc_reply *)recv_head; auth = &reply->rp_u.rpu_rok.rok_auth; x = ntohl(auth->authlen); if (x != 0) { @@ -208,17 +194,21 @@ rpc_call(struct iodesc *d, n_long prog, n_long vers, n_long proc, printf("callrpc: reply auth != NULL\n"); #endif errno = EBADRPC; - return(-1); + free(ptr); + return (-1); } x = ntohl(reply->rp_u.rpu_rok.rok_status); if (x != 0) { printf("callrpc: error = %ld\n", (long)x); errno = EBADRPC; - return(-1); + free(ptr); + return (-1); } - recv_head += sizeof(*reply); - return (ssize_t)(recv_tail - recv_head); + rsize = cc - sizeof(*reply); + *rdata = (void *)((uintptr_t)reply + sizeof(*reply)); + *pkt = ptr; + return (rsize); } /* @@ -227,8 +217,9 @@ rpc_call(struct iodesc *d, n_long prog, n_long vers, n_long proc, * Remaining checks are done by callrpc */ static ssize_t -recvrpc(struct iodesc *d, void *pkt, size_t len, time_t tleft) +recvrpc(struct iodesc *d, void **pkt, void **payload, time_t tleft) { + void *ptr; struct rpc_reply *reply; ssize_t n; int x; @@ -236,14 +227,15 @@ recvrpc(struct iodesc *d, void *pkt, size_t len, time_t tleft) errno = 0; #ifdef RPC_DEBUG if (debug) - printf("recvrpc: called len=%lu\n", (u_long)len); + printf("recvrpc: called\n"); #endif - n = readudp(d, pkt, len, tleft); - if (n <= (4 * 4)) - return -1; - - reply = (struct rpc_reply *)pkt; + ptr = NULL; + n = readudp(d, &ptr, (void **)&reply, tleft); + if (n <= (4 * 4)) { + free(ptr); + return (-1); + } x = ntohl(reply->rp_xid); if (x != rpc_xid) { @@ -251,7 +243,8 @@ recvrpc(struct iodesc *d, void *pkt, size_t len, time_t tleft) if (debug) printf("recvrpc: rp_xid %d != xid %d\n", x, rpc_xid); #endif - return -1; + free(ptr); + return (-1); } x = ntohl(reply->rp_direction); @@ -260,16 +253,20 @@ recvrpc(struct iodesc *d, void *pkt, size_t len, time_t tleft) if (debug) printf("recvrpc: rp_direction %d != REPLY\n", x); #endif - return -1; + free(ptr); + return (-1); } x = ntohl(reply->rp_astatus); if (x != RPC_MSGACCEPTED) { errno = ntohl(reply->rp_u.rpu_errno); printf("recvrpc: reject, astat=%d, errno=%d\n", x, errno); - return -1; + free(ptr); + return (-1); } + *pkt = ptr; + *payload = reply; /* Return data count (thus indicating success) */ return (n); } @@ -387,11 +384,7 @@ rpc_getport(struct iodesc *d, n_long prog, n_long vers) n_long h[RPC_HEADER_WORDS]; struct args d; } sdata; - struct { - n_long h[RPC_HEADER_WORDS]; - struct res d; - n_long pad; - } rdata; + void *pkt; ssize_t cc; int port; @@ -416,16 +409,18 @@ rpc_getport(struct iodesc *d, n_long prog, n_long vers) args->vers = htonl(vers); args->proto = htonl(IPPROTO_UDP); args->port = 0; - res = &rdata.d; + pkt = NULL; cc = rpc_call(d, PMAPPROG, PMAPVERS, PMAPPROC_GETPORT, - args, sizeof(*args), res, sizeof(*res)); + args, sizeof(*args), (void **)&res, &pkt); if (cc < sizeof(*res)) { printf("getport: %s", strerror(errno)); errno = EBADRPC; + free(pkt); return (-1); } port = (int)ntohl(res->port); + free(pkt); rpc_pmap_putcache(d->destip, prog, vers, port); diff --git a/lib/libstand/rpc.h b/lib/libstand/rpc.h index a892a6939dd20..5efe832101424 100644 --- a/lib/libstand/rpc.h +++ b/lib/libstand/rpc.h @@ -48,7 +48,7 @@ /* RPC functions: */ ssize_t rpc_call(struct iodesc *, n_long, n_long, n_long, - void *, size_t, void *, size_t); + void *, size_t, void **, void **); void rpc_fromaddr(void *, struct in_addr *, u_short *); int rpc_pmap_getcache(struct in_addr, u_int, u_int); void rpc_pmap_putcache(struct in_addr, u_int, u_int, int); diff --git a/lib/libstand/tftp.c b/lib/libstand/tftp.c index b3d2da7b0ed95..922142c2a807e 100644 --- a/lib/libstand/tftp.c +++ b/lib/libstand/tftp.c @@ -73,8 +73,8 @@ static int tftp_stat(struct open_file *f, struct stat *sb); static ssize_t sendrecv_tftp(struct tftp_handle *h, ssize_t (*sproc)(struct iodesc *, void *, size_t), void *sbuf, size_t ssize, - ssize_t (*rproc)(struct tftp_handle *h, void *, ssize_t, time_t, unsigned short *), - void *rbuf, size_t rsize, unsigned short *rtype); + ssize_t (*rproc)(struct tftp_handle *h, void **, void **, time_t, unsigned short *), + void **, void **, unsigned short *rtype); struct fs_ops tftp_fsops = { "tftp", @@ -114,11 +114,8 @@ struct tftp_handle { char *path; /* saved for re-requests */ unsigned int tftp_blksize; unsigned long tftp_tsize; - struct { - u_char header[HEADER_SIZE]; - struct tftphdr t; - u_char space[TFTP_MAX_BLKSIZE]; - } __packed __aligned(4) lastdata; + void *pkt; + struct tftphdr *tftp_hdr; }; #define TFTP_MAX_ERRCODE EOPTNEG @@ -181,20 +178,23 @@ tftp_sendack(struct tftp_handle *h) } static ssize_t -recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, +recvtftp(struct tftp_handle *h, void **pkt, void **payload, time_t tleft, unsigned short *rtype) { struct iodesc *d = h->iodesc; struct tftphdr *t; + void *ptr = NULL; + ssize_t len; errno = 0; - len = readudp(d, pkt, len, tleft); + len = readudp(d, &ptr, (void **)&t, tleft); - if (len < 4) + if (len < 4) { + free(ptr); return (-1); + } - t = (struct tftphdr *) pkt; *rtype = ntohs(t->th_opcode); switch (ntohs(t->th_opcode)) { case DATA: { @@ -204,6 +204,7 @@ recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, /* * Expected block? */ + free(ptr); return (-1); } if (d->xid == 1) { @@ -211,11 +212,13 @@ recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, * First data packet from new port. */ struct udphdr *uh; - uh = (struct udphdr *) pkt - 1; + uh = (struct udphdr *) t - 1; d->destport = uh->uh_sport; } /* else check uh_sport has not changed??? */ - got = len - (t->th_data - (char *) t); - return got; + got = len - (t->th_data - (char *)t); + *pkt = ptr; + *payload = t; + return (got); } case ERROR: if ((unsigned) ntohs(t->th_code) > TFTP_MAX_ERRCODE) { @@ -227,6 +230,7 @@ recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, #endif errno = tftperrors[ntohs(t->th_code)]; } + free(ptr); return (-1); case OACK: { struct udphdr *uh; @@ -237,6 +241,7 @@ recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, * Drop the pkt. */ if (d->xid != 1) { + free(ptr); return (-1); } @@ -244,7 +249,7 @@ recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, * Remember which port this OACK came from, because we need * to send the ACK or errors back to it. */ - uh = (struct udphdr *) pkt - 1; + uh = (struct udphdr *) t - 1; d->destport = uh->uh_sport; /* Parse options ACK-ed by the server. */ @@ -252,14 +257,18 @@ recvtftp(struct tftp_handle *h, void *pkt, ssize_t len, time_t tleft, if (tftp_parse_oack(h, t->th_u.tu_stuff, tftp_oack_len) != 0) { tftp_senderr(h, EOPTNEG, "Malformed OACK"); errno = EIO; + free(ptr); return (-1); } + *pkt = ptr; + *payload = t; return (0); } default: #ifdef TFTP_DEBUG printf("tftp type %d not handled\n", ntohs(t->th_opcode)); #endif + free(ptr); return (-1); } } @@ -276,6 +285,7 @@ tftp_makereq(struct tftp_handle *h) char *wtail; int l; ssize_t res; + void *pkt; struct tftphdr *t; char *tftp_blksize = NULL; int blksize_l; @@ -314,8 +324,6 @@ tftp_makereq(struct tftp_handle *h) bcopy("0", wtail, 2); wtail += 2; - t = &h->lastdata.t; - /* h->iodesc->myport = htons(--tftpport); */ h->iodesc->myport = htons(tftpport + (getsecs() & 0x3ff)); h->iodesc->destport = htons(IPPORT_TFTP); @@ -325,8 +333,17 @@ tftp_makereq(struct tftp_handle *h) h->islastblock = 0; h->validsize = 0; + pkt = NULL; res = sendrecv_tftp(h, &sendudp, &wbuf.t, wtail - (char *) &wbuf.t, - &recvtftp, t, sizeof(*t) + h->tftp_blksize, &rtype); + &recvtftp, &pkt, (void **)&t, &rtype); + if (res == -1) { + free(pkt); + return (errno); + } + + free(h->pkt); + h->pkt = pkt; + h->tftp_hdr = t; if (rtype == OACK) return (tftp_getnextblock(h)); @@ -362,6 +379,7 @@ tftp_getnextblock(struct tftp_handle *h) } __packed __aligned(4) wbuf; char *wtail; int res; + void *pkt; struct tftphdr *t; unsigned short rtype = 0; wbuf.t.th_opcode = htons((u_short) ACK); @@ -369,16 +387,20 @@ tftp_getnextblock(struct tftp_handle *h) wbuf.t.th_block = htons((u_short) h->currblock); wtail += 2; - t = &h->lastdata.t; - h->iodesc->xid = h->currblock + 1; /* expected block */ + pkt = NULL; res = sendrecv_tftp(h, &sendudp, &wbuf.t, wtail - (char *) &wbuf.t, - &recvtftp, t, sizeof(*t) + h->tftp_blksize, &rtype); + &recvtftp, &pkt, (void **)&t, &rtype); - if (res == -1) /* 0 is OK! */ + if (res == -1) { /* 0 is OK! */ + free(pkt); return (errno); + } + free(h->pkt); + h->pkt = pkt; + h->tftp_hdr = t; h->currblock++; h->validsize = res; if (res < h->tftp_blksize) @@ -405,14 +427,8 @@ tftp_open(const char *path, struct open_file *f) if (netproto != NET_TFTP) return (EINVAL); - if (strcmp(f->f_dev->dv_name, "net") != 0) { -#ifdef __i386__ - if (strcmp(f->f_dev->dv_name, "pxe") != 0) - return (EINVAL); -#else + if (f->f_dev->dv_type != DEVT_NET) return (EINVAL); -#endif - } if (is_open) return (EBUSY); @@ -507,7 +523,7 @@ tftp_read(struct open_file *f, void *addr, size_t size, return (EINVAL); } count = (size < inbuffer ? size : inbuffer); - bcopy(tftpfile->lastdata.t.th_data + offinblock, + bcopy(tftpfile->tftp_hdr->th_data + offinblock, addr, count); addr = (char *)addr + count; @@ -540,6 +556,7 @@ tftp_close(struct open_file *f) if (tftpfile) { free(tftpfile->path); + free(tftpfile->pkt); free(tftpfile); } is_open = 0; @@ -591,8 +608,9 @@ static ssize_t sendrecv_tftp(struct tftp_handle *h, ssize_t (*sproc)(struct iodesc *, void *, size_t), void *sbuf, size_t ssize, - ssize_t (*rproc)(struct tftp_handle *, void *, ssize_t, time_t, unsigned short *), - void *rbuf, size_t rsize, unsigned short *rtype) + ssize_t (*rproc)(struct tftp_handle *, void **, void **, time_t, + unsigned short *), + void **pkt, void **payload, unsigned short *rtype) { struct iodesc *d = h->iodesc; ssize_t cc; @@ -624,7 +642,7 @@ sendrecv_tftp(struct tftp_handle *h, recvnext: /* Try to get a packet and process it. */ - cc = (*rproc)(h, rbuf, rsize, tleft, rtype); + cc = (*rproc)(h, pkt, payload, tleft, rtype); /* Return on data, EOF or real error. */ if (cc != -1 || errno != 0) return (cc); diff --git a/lib/libstand/udp.c b/lib/libstand/udp.c index db5de892e8558..2eba7a18eefcb 100644 --- a/lib/libstand/udp.c +++ b/lib/libstand/udp.c @@ -59,10 +59,7 @@ __FBSDID("$FreeBSD$"); /* Caller must leave room for ethernet, ip and udp headers in front!! */ ssize_t -sendudp(d, pkt, len) - struct iodesc *d; - void *pkt; - size_t len; +sendudp(struct iodesc *d, void *pkt, size_t len) { ssize_t cc; struct ip *ip; @@ -131,32 +128,29 @@ sendudp(d, pkt, len) /* * Receive a UDP packet and validate it is for us. - * Caller leaves room for the headers (Ether, IP, UDP) */ ssize_t -readudp(d, pkt, len, tleft) - struct iodesc *d; - void *pkt; - size_t len; - time_t tleft; +readudp(struct iodesc *d, void **pkt, void **payload, time_t tleft) { ssize_t n; size_t hlen; struct ip *ip; struct udphdr *uh; - u_int16_t etype; /* host order */ + uint16_t etype; /* host order */ + void *ptr; #ifdef NET_DEBUG if (debug) printf("readudp: called\n"); #endif - uh = (struct udphdr *)pkt - 1; - ip = (struct ip *)uh - 1; - - n = readether(d, ip, len + sizeof(*ip) + sizeof(*uh), tleft, &etype); - if (n == -1 || n < sizeof(*ip) + sizeof(*uh)) - return -1; + ip = NULL; + ptr = NULL; + n = readether(d, &ptr, (void **)&ip, tleft, &etype); + if (n == -1 || n < sizeof(*ip) + sizeof(*uh)) { + free(ptr); + return (-1); + } /* Ethernet address checks now in readether() */ @@ -167,7 +161,8 @@ readudp(d, pkt, len, tleft) /* Send ARP reply */ arp_reply(d, ah); } - return -1; + free(ptr); + return (-1); } if (etype != ETHERTYPE_IP) { @@ -175,7 +170,8 @@ readudp(d, pkt, len, tleft) if (debug) printf("readudp: not IP. ether_type=%x\n", etype); #endif - return -1; + free(ptr); + return (-1); } /* Check ip header */ @@ -185,7 +181,8 @@ readudp(d, pkt, len, tleft) if (debug) printf("readudp: IP version or not UDP. ip_v=%d ip_p=%d\n", ip->ip_v, ip->ip_p); #endif - return -1; + free(ptr); + return (-1); } hlen = ip->ip_hl << 2; @@ -195,7 +192,8 @@ readudp(d, pkt, len, tleft) if (debug) printf("readudp: short hdr or bad cksum.\n"); #endif - return -1; + free(ptr); + return (-1); } if (n < ntohs(ip->ip_len)) { #ifdef NET_DEBUG @@ -203,7 +201,8 @@ readudp(d, pkt, len, tleft) printf("readudp: bad length %d < %d.\n", (int)n, ntohs(ip->ip_len)); #endif - return -1; + free(ptr); + return (-1); } if (d->myip.s_addr && ip->ip_dst.s_addr != d->myip.s_addr) { #ifdef NET_DEBUG @@ -212,12 +211,14 @@ readudp(d, pkt, len, tleft) printf("%s\n", inet_ntoa(ip->ip_dst)); } #endif - return -1; + free(ptr); + return (-1); } + uh = (struct udphdr *)((uintptr_t)ip + sizeof (*ip)); /* If there were ip options, make them go away */ if (hlen != sizeof(*ip)) { - bcopy(((u_char *)ip) + hlen, uh, len - hlen); + bcopy(((u_char *)ip) + hlen, uh, uh->uh_ulen - hlen); ip->ip_len = htons(sizeof(*ip)); n -= hlen - sizeof(*ip); } @@ -227,7 +228,8 @@ readudp(d, pkt, len, tleft) printf("readudp: bad dport %d != %d\n", d->myport, ntohs(uh->uh_dport)); #endif - return -1; + free(ptr); + return (-1); } #ifndef UDP_NO_CKSUM @@ -238,7 +240,8 @@ readudp(d, pkt, len, tleft) n = ntohs(uh->uh_ulen) + sizeof(*ip); if (n > RECV_SIZE - ETHER_SIZE) { printf("readudp: huge packet, udp len %d\n", (int)n); - return -1; + free(ptr); + return (-1); } /* Check checksum (must save and restore ip header) */ @@ -251,8 +254,8 @@ readudp(d, pkt, len, tleft) if (debug) printf("readudp: bad cksum\n"); #endif - *ip = tip; - return -1; + free(ptr); + return (-1); } *ip = tip; } @@ -263,10 +266,13 @@ readudp(d, pkt, len, tleft) printf("readudp: bad udp len %d < %d\n", ntohs(uh->uh_ulen), (int)sizeof(*uh)); #endif - return -1; + free(ptr); + return (-1); } n = (n > (ntohs(uh->uh_ulen) - sizeof(*uh))) ? ntohs(uh->uh_ulen) - sizeof(*uh) : n; + *pkt = ptr; + *payload = (void *)((uintptr_t)uh + sizeof(*uh)); return (n); } diff --git a/lib/libzstd/Makefile b/lib/libzstd/Makefile index 3d34ce9477af3..bb5ad0fdc5ed8 100644 --- a/lib/libzstd/Makefile +++ b/lib/libzstd/Makefile @@ -22,7 +22,9 @@ SRCS= entropy_common.c \ zdict.c WARNS= 2 INCS= zstd.h -CFLAGS+= -I${ZSTDDIR}/lib -I${ZSTDDIR}/lib/common -DXXH_NAMESPACE=ZSTD_ +CFLAGS+= -I${ZSTDDIR}/lib -I${ZSTDDIR}/lib/common -DXXH_NAMESPACE=ZSTD_ \ + -DZSTD_MULTITHREAD=1 +LIBADD= pthread PRIVATELIB= yes diff --git a/libexec/ftpd/blacklist.c b/libexec/ftpd/blacklist.c index 85f90b5352d70..7aca755f7d2d2 100644 --- a/libexec/ftpd/blacklist.c +++ b/libexec/ftpd/blacklist.c @@ -33,8 +33,8 @@ #include <stdlib.h> #include <unistd.h> -#include "blacklist_client.h" #include <blacklist.h> +#include "blacklist_client.h" static struct blacklist *blstate; extern int use_blacklist; @@ -48,7 +48,7 @@ blacklist_init(void) } void -blacklist_notify(int action, int fd, char *msg) +blacklist_notify(int action, int fd, const char *msg) { if (blstate == NULL) diff --git a/libexec/ftpd/blacklist_client.h b/libexec/ftpd/blacklist_client.h index 7ac6fd11ed279..391b49c9bdfda 100644 --- a/libexec/ftpd/blacklist_client.h +++ b/libexec/ftpd/blacklist_client.h @@ -31,14 +31,16 @@ #ifndef BLACKLIST_CLIENT_H #define BLACKLIST_CLIENT_H +#ifndef BLACKLIST_API_ENUM enum { BLACKLIST_AUTH_OK = 0, BLACKLIST_AUTH_FAIL }; +#endif #ifdef USE_BLACKLIST void blacklist_init(void); -void blacklist_notify(int, int, char *); +void blacklist_notify(int, int, const char *); #define BLACKLIST_INIT() blacklist_init() #define BLACKLIST_NOTIFY(x, y, z) blacklist_notify(x, y, z) diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile index 1d913b9adc700..a31d303bc848d 100644 --- a/rescue/rescue/Makefile +++ b/rescue/rescue/Makefile @@ -194,6 +194,10 @@ CRUNCH_ALIAS_less= more CRUNCH_PROGS_usr.bin+= xz CRUNCH_ALIAS_xz= unxz lzma unlzma xzcat lzcat +CRUNCH_PROGS_usr.bin+= zstd +CRUNCH_ALIAS_zstd= unzstd zstdcat zstdmt +CRUNCH_LIBS+= -lprivatezstd + CRUNCH_PROGS_usr.bin+= tar CRUNCH_LIBS+= -larchive .if ${MK_OPENSSL} != "no" diff --git a/sbin/camcontrol/modeedit.c b/sbin/camcontrol/modeedit.c index b6365158b7cdf..e7b877d7be855 100644 --- a/sbin/camcontrol/modeedit.c +++ b/sbin/camcontrol/modeedit.c @@ -629,8 +629,21 @@ editlist_save(struct cam_device *device, int dbd, int pc, int page, /* Recalculate headers & offsets. */ mh->data_length = 0; /* Reserved for MODE SELECT command. */ - mh->dev_spec = 0; /* Clear device-specific parameters. */ mh->blk_desc_len = 0; /* No block descriptors. */ + /* + * Tape drives include write protect (WP), Buffered Mode and Speed + * settings in the device-specific parameter. Clearing this + * parameter on a mode select can have the effect of turning off + * write protect or buffered mode, or changing the speed setting of + * the tape drive. + * + * Disks report DPO/FUA support via the device specific parameter + * for MODE SENSE, but the bit is reserved for MODE SELECT. So we + * clear this for disks (and other non-tape devices) to avoid + * potential errors from the target device. + */ + if (device->pd_type != T_SEQUENTIAL) + mh->dev_spec = 0; mph = MODE_PAGE_HEADER(mh); mph->page_code &= ~SMPH_PS; /* Reserved for MODE SELECT command. */ diff --git a/sbin/dhclient/dhclient.c b/sbin/dhclient/dhclient.c index bd7d4330bbfa2..c9c74144c1b8f 100644 --- a/sbin/dhclient/dhclient.c +++ b/sbin/dhclient/dhclient.c @@ -108,7 +108,11 @@ struct pidfh *pidfile; */ #define ASSERT_STATE(state_is, state_shouldbe) {} -#define TIME_MAX 2147483647 +/* + * We need to check that the expiry, renewal and rebind times are not beyond + * the end of time (~2038 when a 32-bit time_t is being used). + */ +#define TIME_MAX ((((time_t) 1 << (sizeof(time_t) * CHAR_BIT - 2)) - 1) * 2 + 1) int log_priority; int no_daemon; @@ -766,15 +770,17 @@ dhcpack(struct packet *packet) else ip->client->new->expiry = default_lease_time; /* A number that looks negative here is really just very large, - because the lease expiry offset is unsigned. */ - if (ip->client->new->expiry < 0) - ip->client->new->expiry = TIME_MAX; + because the lease expiry offset is unsigned. Also make sure that + the addition of cur_time below does not overflow (a 32 bit) time_t. */ + if (ip->client->new->expiry < 0 || + ip->client->new->expiry > TIME_MAX - cur_time) + ip->client->new->expiry = TIME_MAX - cur_time; /* XXX should be fixed by resetting the client state */ if (ip->client->new->expiry < 60) ip->client->new->expiry = 60; /* Unless overridden in the config, take the server-provided renewal - * time if there is one; otherwise figure it out according to the spec. + * time if there is one. Otherwise figure it out according to the spec. * Also make sure the renewal time does not exceed the expiry time. */ if (ip->client->config->default_actions[DHO_DHCP_RENEWAL_TIME] == @@ -786,7 +792,8 @@ dhcpack(struct packet *packet) ip->client->new->options[DHO_DHCP_RENEWAL_TIME].data); else ip->client->new->renewal = ip->client->new->expiry / 2; - if (ip->client->new->renewal > ip->client->new->expiry / 2) + if (ip->client->new->renewal < 0 || + ip->client->new->renewal > ip->client->new->expiry / 2) ip->client->new->renewal = ip->client->new->expiry / 2; /* Same deal with the rebind time. */ @@ -798,20 +805,15 @@ dhcpack(struct packet *packet) ip->client->new->rebind = getULong( ip->client->new->options[DHO_DHCP_REBINDING_TIME].data); else - ip->client->new->rebind = ip->client->new->renewal * 7 / 4; - if (ip->client->new->rebind > ip->client->new->renewal * 7 / 4) - ip->client->new->rebind = ip->client->new->renewal * 7 / 4; + ip->client->new->rebind = ip->client->new->renewal / 4 * 7; + if (ip->client->new->rebind < 0 || + ip->client->new->rebind > ip->client->new->renewal / 4 * 7) + ip->client->new->rebind = ip->client->new->renewal / 4 * 7; - ip->client->new->expiry += cur_time; - /* Lease lengths can never be negative. */ - if (ip->client->new->expiry < cur_time) - ip->client->new->expiry = TIME_MAX; - ip->client->new->renewal += cur_time; - if (ip->client->new->renewal < cur_time) - ip->client->new->renewal = TIME_MAX; - ip->client->new->rebind += cur_time; - if (ip->client->new->rebind < cur_time) - ip->client->new->rebind = TIME_MAX; + /* Convert the time offsets into seconds-since-the-epoch */ + ip->client->new->expiry += cur_time; + ip->client->new->renewal += cur_time; + ip->client->new->rebind += cur_time; bind_lease(ip); } diff --git a/sbin/dhclient/options.c b/sbin/dhclient/options.c index be073dae39cd5..fe025e6d6a40c 100644 --- a/sbin/dhclient/options.c +++ b/sbin/dhclient/options.c @@ -783,7 +783,7 @@ pretty_print_option(unsigned int code, unsigned char *data, int len, dp += 4; break; case 'L': - opcount = snprintf(op, opleft, "%ld", + opcount = snprintf(op, opleft, "%lu", (unsigned long)getULong(dp)); if (opcount >= opleft || opcount == -1) goto toobig; @@ -799,7 +799,7 @@ pretty_print_option(unsigned int code, unsigned char *data, int len, dp += 2; break; case 'S': - opcount = snprintf(op, opleft, "%d", + opcount = snprintf(op, opleft, "%u", getUShort(dp)); if (opcount >= opleft || opcount == -1) goto toobig; diff --git a/sbin/mount_nfs/mount_nfs.c b/sbin/mount_nfs/mount_nfs.c index f04e32b7d3d5c..93f6a9cb8347e 100644 --- a/sbin/mount_nfs/mount_nfs.c +++ b/sbin/mount_nfs/mount_nfs.c @@ -636,7 +636,7 @@ getnfsargs(char *spec, struct iovec **iov, int *iovlen) build_iovec(iov, iovlen, "hostname", nam, (size_t)-1); /* Add mounted file system to PATH_MOUNTTAB */ - if (!add_mtab(hostp, spec)) + if (mountmode != V4 && !add_mtab(hostp, spec)) warnx("can't update %s for %s:%s", PATH_MOUNTTAB, hostp, spec); return (1); } diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile index b58d128a956a2..492e78f218fde 100644 --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -22,6 +22,7 @@ MAN= aac.4 \ acpi_video.4 \ ${_acpi_wmi.4} \ ada.4 \ + adm6996fc.4 \ adv.4 \ adw.4 \ ae.4 \ @@ -152,6 +153,7 @@ MAN= aac.4 \ etherswitch.4 \ eventtimers.4 \ exca.4 \ + e6060sw.4 \ fd.4 \ fdc.4 \ fdt.4 \ @@ -245,6 +247,7 @@ MAN= aac.4 \ keyboard.4 \ kld.4 \ ksyms.4 \ + ksz8995ma.4 \ ktr.4 \ kue.4 \ lagg.4 \ diff --git a/share/man/man4/adm6996fc.4 b/share/man/man4/adm6996fc.4 new file mode 100644 index 0000000000000..81b96a1f6c689 --- /dev/null +++ b/share/man/man4/adm6996fc.4 @@ -0,0 +1,68 @@ +.\"- +.\" Copyright (c) 2017 Hiroki Mori +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd April 5, 2017 +.Dt ADM6996FC 4 +.Os +.Sh NAME +.Nm adm6996fc +.Nd driver for Infineon ADM6996FC fast ethernet switch chip +.Sh SYNOPSIS +.Cd "device mdio" +.Cd "device etherswitch" +.Cd "device adm6996fc" +.Pp +.Cd hint.adm6996fc.0.at="mdio0" +.Sh DESCRIPTION +The +.Nm +device driver provides a management interface to Infineon ADM6996FC fast ethernet switch chip. +This driver use smi interface by ethernet interface. +.Pp +This driver support is port and dot1q vlan. +dot1q support port base tag/untag. +.Sh EXAMPLES +Configure dot1q vlan by etherswitchcfg command. +.Pp +.Dl # etherswitchcfg config vlan_mode dot1q +.Pp +Configure port 5 is tagging port. +.Pp +.Dl # etherswitchcfg port5 addtag +.Sh SEE ALSO +.Xr etherswitch 4 , +.Xr etherswitchcfg 8 +.Sh HISTORY +The +.Nm +device driver first appeared in +.Fx 12.0 . +.Sh AUTHORS +The +.Nm +driver was written by +.An Hiroki Mori diff --git a/share/man/man4/e6060sw.4 b/share/man/man4/e6060sw.4 new file mode 100644 index 0000000000000..c5394106f5b9a --- /dev/null +++ b/share/man/man4/e6060sw.4 @@ -0,0 +1,70 @@ +.\"- +.\" Copyright (c) 2017 Hiroki Mori +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd April 5, 2017 +.Dt E6060SW 4 +.Os +.Sh NAME +.Nm e6060sw +.Nd driver for Marvell 88E6060 and 88E6065 fast ethernet switch chip +.Sh SYNOPSIS +.Cd "device mdio" +.Cd "device etherswitch" +.Cd "device e6060sw" +.Pp +.Cd hint.e6060sw.0.at="mdio0" +.Sh DESCRIPTION +The +.Nm +device driver provides a management interface to Marvell 88E6060 and 88E6065 fast ethernet switch chip. +This driver use smi interface by ethernet interface. +.Pp +88E6060 support is only port vlan. +88E6065 support is port and dot1q vlan. +dot1q support group base tag/untag. +.Sh EXAMPLES +Configure dot1q vlan on 88E6065 by etherswitchcfg command. +.Pp +.Dl # etherswitchcfg config vlan_mode dot1q +.Pp +Configure vlangroup with dot1q tag on 88E6065. +This example is port0 as vlan 2. +.Pp +.Dl # etherswitchcfg vlangroup2 vlan 2 members 0,5t port0 pvid 2 +.Sh SEE ALSO +.Xr etherswitch 4 , +.Xr etherswitchcfg 8 +.Sh HISTORY +The +.Nm +device driver first appeared in +.Fx 12.0 . +.Sh AUTHORS +The +.Nm +driver was written by +.An Hiroki Mori diff --git a/share/man/man4/etherswitch.4 b/share/man/man4/etherswitch.4 index 8d11dc60c2c1c..ea7ef49e6f4c7 100644 --- a/share/man/man4/etherswitch.4 +++ b/share/man/man4/etherswitch.4 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd January 5, 2015 +.Dd April 5, 2017 .Dt ETHERSWITCH 4 .Os .Sh NAME @@ -54,6 +54,9 @@ device nodes .Xr iicbus 4 , .Xr ip17x 4 , .Xr rtl8366rb 4 , +.Xr e6060sw 4 , +.Xr adm6996fc 4 , +.Xr ksz8995ma 4 , .Xr ukswitch 4 , .Xr etherswitchcfg 8 .Sh HISTORY diff --git a/share/man/man4/ksz8995ma.4 b/share/man/man4/ksz8995ma.4 new file mode 100644 index 0000000000000..758d6697722f4 --- /dev/null +++ b/share/man/man4/ksz8995ma.4 @@ -0,0 +1,69 @@ +.\"- +.\" Copyright (c) 2017 Hiroki Mori +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd April 6, 2017 +.Dt KSZ8995MA 4 +.Os +.Sh NAME +.Nm ksz8995ma +.Nd driver for Micrel KSZ8995MA fast ethernet switch chip +.Sh SYNOPSIS +.Cd "device spibus" +.Cd "device etherswitch" +.Cd "device ksz8995ma" +.Pp +.Cd hint.ksz8995ma.0.at="spibus0" +.Sh DESCRIPTION +The +.Nm +device driver provides a management interface to Micrel KSZ8995MA fast ethernet switch chip. +This driver use spi interface. +KSZ8995 series have many vertion but only support MA/FQ version. +.Pp +This driver support is port and dot1q vlan. +dot1q support port base tag/untag. +.Sh EXAMPLES +Configure dot1q vlan by etherswitchcfg command. +.Pp +.Dl # etherswitchcfg config vlan_mode dot1q +.Pp +Configure port 5 is tagging port. +.Pp +.Dl # etherswitchcfg port5 addtag +.Sh SEE ALSO +.Xr etherswitch 4 , +.Xr etherswitchcfg 8 +.Sh HISTORY +The +.Nm +device driver first appeared in +.Fx 12.0 . +.Sh AUTHORS +The +.Nm +driver was written by +.An Hiroki Mori diff --git a/share/man/man4/sa.4 b/share/man/man4/sa.4 index f50139e455ac3..4c93515b0a56a 100644 --- a/share/man/man4/sa.4 +++ b/share/man/man4/sa.4 @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd February 12, 2015 +.Dd May 5, 2017 .Dt SA 4 .Os .Sh NAME @@ -242,6 +242,87 @@ These devices include the QIC family of devices. block devices. This has not been determined yet, and they are treated as separate behaviors by the driver at this time.) +.Sh PARAMETERS +The +.Nm +driver supports a number of parameters. +The user can query parameters using +.Dq mt param -l +(which uses the +.Dv MTIOCPARAMGET +ioctl) and the user can set parameters using +.Dq mt param -s +(which uses the +.Dv MTIOCPARAMSET +ioctl). +See +.Xr mt 1 +and +.Xr mtio 4 +for more details on the interface. +.Pp +Supported parameters: +.Bl -tag -width 5n +.It sili +The default is 0. +When set to 1, it sets the Suppress Incorrect Length Indicator (SILI) bit +on tape reads. +Tape drives normally return sense data (which contains the residual) when the +application reads a block that is not the same length as the amount of data +requested. +The SILI bit supresses that notification in most cases. +See the SSC-5 spec (available at t10.org), specifically the section on the +READ(6) command, for more information. +.It eot_warn +The default is 0. +By default, the +.Nm +driver reports entering Programmable Early Warning, Early Warning and End +of Media conditions by returning a write with 0 bytes written, and +.Dv errno +set to 0. +If +.Va eot_warn +is set to 1, the +.Nm +driver will set +.Dv errno +to +.Dv ENOSPC +when it enters any of the out of space conditions. +.It protection.protection_supported +This is a read-only parameter, and is set to 1 if the tape drive supports +protection information. +.It protection.prot_method +If protection is supported, set this to the desired protection method +supported by the tape drive. +As of SSC-5r03 (available at t10.org), the protection method values are: +.Bl -tag -width 3n +.It 0 +No protection. +.It 1 +Reed-Solomon CRC, 4 bytes in length. +.It 2 +CRC32C, 4 bytes in length. +.El +.It protection.pi_length +Length of the protection information, see above for lengths. +.It protection.lbp_w +If set to 1, enable logical block protection on writes. +The CRC must be appended to the end of the block written to the tape driver. +The tape drive will verify the CRC when it receives the block. +.It protection.lbp_r +If set to 1, enable logical block protection on reads. +The CRC will be appended to the end of the block read from the tape driver. +The application should verify the CRC when it receives the block. +.It protection.rdbp +If set to 1, enable logical block protection on the RECOVER BUFFERED DATA +command. +The +.Nm +driver does not currently use the +RECOVER BUFFERED DATA command. +.El .Sh IOCTLS The .Nm @@ -262,7 +343,26 @@ Control mode device (to examine state while another program is accessing the device, e.g.). .El .Sh DIAGNOSTICS -None. +The +.Nm +driver supports injecting End Of Media (EOM) notification to aid +application development and testing. +EOM is indicated to the application by returning the read or write with 0 +bytes written. +In addition, when EOM is injected, the tape position status will be updated +to temporarily show Beyond of the Programmable Early Warning (BPEW) status. +To see BPEW status, use the +.Dv MTIOCEXTGET +ioctl, which is used by the +.Dq mt status +command. +To inject an EOM notification, set the +.Pp +.Va kern.cam.sa.%d.inject_eom +.Pp +sysctl variable to 1. +One EOM notification will be sent, BPEW status will be set for one position +query, and then the driver state will be reset to normal. .Sh SEE ALSO .Xr mt 1 , .Xr cam 4 diff --git a/share/man/man7/arch.7 b/share/man/man7/arch.7 index 286b473fbe216..a6e052f46f397 100644 --- a/share/man/man7/arch.7 +++ b/share/man/man7/arch.7 @@ -1,4 +1,4 @@ -.\" Copyright (c) 2016 The FreeBSD Foundation. All rights reserved. +.\" Copyright (c) 2016-2017 The FreeBSD Foundation. All rights reserved. .\" .\" This documentation was created by Ed Maste under sponsorship of .\" The FreeBSD Foundation. @@ -26,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd May 3, 2017 +.Dd May 5, 2017 .Dt ARCH 7 .Os .Sh NAME @@ -35,9 +35,139 @@ .Sh DESCRIPTION Differences between CPU architectures and platforms supported by .Fx . +.Ss Introduction +This document is a quick reference of key ABI details of +.Fx +architecture ports. +For full details consult the processor-specific ABI supplement +documentation. .Pp If not explicitly mentioned, sizes are in bytes. +The architecture details in this document apply to +.Fx 10.0 +and later, unless otherwise noted. +.Pp +.Fx +uses a flat address space. +Variables of types +.Vt unsigned long , +.Vt uintptr_t , +and +.Vt size_t +and pointers all have the same representation. +.Pp +In order to maximize compatibility with future pointer integrity mechanisms, +manipulations of pointers as integers should be performed via +.Vt uintptr_t +or +.Vt intptr_t +and no other types. +In particular, +.Vt long +and +.Vt ptrdiff_t +should be avoided. +.Pp +On some architectures, e.g. +.Dv sparc64 , +.Dv powerpc +and AIM variants of +.Dv powerpc64 , +the kernel uses a separate address space. +On other architectures, kernel and a user mode process share a +single address space. +The kernel is located at the highest addresses. +.Pp +On each architecture, the main user mode thread's stack starts near +the highest user address and grows down. +.Pp +.Fx +architecture support varies by release. +This table shows the first +.Fx +release to support each architecture, and, for discontinued +architectures, the final release. +.Pp +.Bl -column -offset indent "Sy Architecture" "Sy Initial Release" "Sy Final Release" +.It Sy Architecture Ta Sy Initial Release Ta Sy Final Release +.It alpha Ta 3.x Ta 6.4 +.It amd64 Ta 5.1 +.It arm Ta 6.0 +.It armeb Ta 8.0 +.It armv6 Ta 10.0 +.It arm64 Ta 11.0 +.It ia64 Ta 5.0 Ta 10.x +.It i386 Ta 1.0 +.It mips Ta 8.0 +.It mipsel Ta 9.0 +.It mipselhf Ta 12.0 +.It mipshf Ta 12.0 +.It mipsn32 Ta 9.0 +.It mips64 Ta 9.0 +.It mips64el Ta 9.0 +.It mips64elhf Ta 12.0 +.It mips64hf Ta 12.0 +.It pc98 Ta 2.2 Ta 11.x +.It powerpc Ta 6.0 +.It powerpcspe Ta 12.0 +.It powerpc64 Ta 6.0 +.It riscv64 Ta 12.0 +.It riscv64sf Ta 12.0 +.It sparc64 Ta 5.0 +.El .Ss Type sizes +All +.Fx +architectures use some variant of the ELF (see +.Xr elf 5 ) +.Sy Application Binary Interface +(ABI) for the machine processor. +All supported ABIs can be divided into two groups: +.Bl -tag -width "Dv ILP32" +.It Dv ILP32 +.Vt int , +.Vt long , +.Vt void * +types machine representations all have 4-byte size. +.It Dv LP64 +.Vt int +type machine representation uses 4 bytes, +while +.Vt long +and +.Vt void * +are 8 bytes. +.El +Compilers define the +.Dv _LP64 +symbol when compiling for an +.Dv LP64 +ABI. +.Pp +Some machines support more that one +.Fx +ABI. +Typically these are 64-bit machines, where the +.Dq native +.Dv LP64 +execution environment is accompanied by the +.Dq legacy +.Dv ILP32 +environment, which was historical 32-bit predecessor for 64-bit evolution. +Examples are: +.Bl -column -offset indent "Dv powerpc64" "Sy ILP32 counterpart" +.It Sy LP64 Ta Sy ILP32 counterpart +.It Dv amd64 Ta Dv i386 +.It Dv powerpc64 Ta Dv powerpc +.It Dv mips64* Ta Dv mips* +.El +.Dv arm64 +currently does not support execution of +.Dv armv6 +binaries, even if the CPU implements +.Dv AArch32 +execution state. +.Pp On all supported architectures: .Bl -column -offset -indent "long long" "Size" .It Sy Type Ta Sy Size @@ -48,6 +178,12 @@ On all supported architectures: .It float Ta 4 .It double Ta 8 .El +Integers are represented in two's complement. +Alignment of integer and pointer types is natural, that is, +the address of the variable must be congruent to zero modulo the type size. +Most ILP32 ABIs, except +.Dv arm , +require only 4-byte alignment for 64-bit integers. .Pp Machine-dependent type sizes: .Bl -column -offset indent "Sy Architecture" "Sy void *" "Sy long double" "Sy time_t" @@ -134,7 +270,7 @@ variants of powerpc. .It amd64 Ta hard Ta hard, 80 bit .It arm Ta soft Ta soft, double precision .It armeb Ta soft Ta soft, double precision -.It armv6 Ta hard Ta hard, double precision +.It armv6 Ta hard(1) Ta hard, double precision .It arm64 Ta hard Ta soft, quad precision .It i386 Ta hard Ta hard, 80 bit .It mips Ta soft Ta identical to double @@ -153,6 +289,11 @@ variants of powerpc. .It riscv64sf Ta soft Ta soft, double precision .It sparc64 Ta hard Ta hard, quad precision .El +.Pp +(1) Prior to +.Fx 11.0 , +armv6 used the softfp ABI even though it supported only processors +with a floating point unit. .Ss Predefined Macros The compiler provides a number of predefined macros. Some of these provide architecture-specific details and are explained below. diff --git a/share/man/man9/VOP_GETPAGES.9 b/share/man/man9/VOP_GETPAGES.9 index 4625d9b57115c..358d36c850942 100644 --- a/share/man/man9/VOP_GETPAGES.9 +++ b/share/man/man9/VOP_GETPAGES.9 @@ -29,7 +29,7 @@ .\" .\" $FreeBSD$ .\" -.Dd December 16, 2015 +.Dd May 7, 2017 .Dt VOP_GETPAGES 9 .Os .Sh NAME @@ -41,9 +41,21 @@ .In sys/vnode.h .In vm/vm.h .Ft int -.Fn VOP_GETPAGES "struct vnode *vp" "vm_page_t *ma" "int count" "int *rbehind" "int *rahead" +.Fo VOP_GETPAGES +.Fa "struct vnode *vp" +.Fa "vm_page_t *ma" +.Fa "int count" +.Fa "int *rbehind" +.Fa "int *rahead" +.Fc .Ft int -.Fn VOP_PUTPAGES "struct vnode *vp" "vm_page_t *ma" "int count" "int sync" "int *rtvals" +.Fo VOP_PUTPAGES +.Fa "struct vnode *vp" +.Fa "vm_page_t *ma" +.Fa "int bytecount" +.Fa "int flags" +.Fa "int *rtvals" +.Fc .Sh DESCRIPTION The .Fn VOP_GETPAGES @@ -70,10 +82,32 @@ The file to access. Pointer to the first element of an array of pages representing a contiguous region of the file to be read or written. .It Fa count -The number of bytes that should be read into the pages of the array. -.It Fa sync +The length of the +.Fa ma +array. +.It Fa bytecount +The number of bytes that should be written from the pages of the array. +.It Fa flags +A bitfield of flags affecting the function operation. +If .Dv VM_PAGER_PUT_SYNC -if the write should be synchronous. +is set, the write should be synchronous; control must not be returned +to the caller until after the write is finished. +If +.Dv VM_PAGER_PUT_INVAL +is set, the pages are to be invalidated after being written. +If +.Dv VM_PAGER_PUT_NOREUSE +is set, the I/O performed should set the IO_NOREUSE flag, to indicate +to the filesystem that pages should be marked for fast reuse if needed. +This could occur via a call to +.Xr vm_page_deactivate_noreuse 9 , +which puts such pages onto the head of the inactive queue. +If +.Dv VM_PAGER_CLUSTER_OK +is set, writes may be performed asynchronously, so that related writes +can be coalesced for efficiency, e.g., +using the clustering mechanism of the buffer cache. .It Fa rtvals An array of VM system result codes indicating the status of each page written by @@ -127,32 +161,33 @@ The page was not handled by this request. .Pp The .Fn VOP_GETPAGES -method is expected to release any pages in +method must populate and validate all requested pages in order to +return success. +It is expected to release any pages in .Fa ma that it does not successfully handle, by calling .Xr vm_page_free 9 . When it succeeds, .Fn VOP_GETPAGES must set the valid bits appropriately. +Upon entry to +.Fn VOP_GETPAGES , +all pages in +.Fa ma +are busied exclusively. +Upon successful return, the pages must all be busied exclusively +as well, but pages may be unbusied during processing. +The filesystem is responsible for activating paged-out pages, but this +does not necessarily need to be done within .Fn VOP_GETPAGES -must keep -.Fa reqpage -busy. -It must unbusy all other successfully handled pages and put them -on appropriate page queue(s). -For example, -.Fn VOP_GETPAGES -may either activate a page (if its wanted bit is set) -or deactivate it (otherwise), and finally call -.Xr vm_page_xunbusy 9 -to arouse any threads currently waiting for the page to be faulted in. +depending on the architecture of the particular filesystem. .Sh RETURN VALUES -If it successfully reads -.Fa ma[reqpage] , +If it successfully reads all pages in +.Fa ma , .Fn VOP_GETPAGES returns .Dv VM_PAGER_OK ; -otherwise, +otherwise, it returns .Dv VM_PAGER_ERROR . By convention, the return value of .Fn VOP_PUTPAGES diff --git a/share/misc/committers-ports.dot b/share/misc/committers-ports.dot index abe87bb695c72..db9ab4c74e8d5 100644 --- a/share/misc/committers-ports.dot +++ b/share/misc/committers-ports.dot @@ -84,6 +84,7 @@ daichi [label="Daichi Goto\ndaichi@FreeBSD.org\n2002/10/17"] danfe [label="Alexey Dokuchaev\ndanfe@FreeBSD.org\n2004/08/20"] danilo [label="Danilo E. Gondolfo\ndanilo@FreeBSD.org\n2013/09/23"] db [label="Diane Bruce\ndb@FreeBSD.org\n2007/01/18"] +dbaio [label="Danilo G. Baio\ndbaio@FreeBSD.org\n2017/05/03"] dbn [label="David Naylor\ndbn@FreeBSD.org\n2013/01/14"] decke [label="Bernhard Froehlich\ndecke@FreeBSD.org\n2010/03/21"] delphij [label="Xin Li\ndelphij@FreeBSD.org\n2006/05/01"] @@ -402,6 +403,7 @@ gabor -> scheidell garga -> acm garga -> alepulver +garga -> dbaio garga -> mandree garga -> mm garga -> rnoland diff --git a/share/mk/bsd.init.mk b/share/mk/bsd.init.mk index 5e8951a6e6054..5947322a77342 100644 --- a/share/mk/bsd.init.mk +++ b/share/mk/bsd.init.mk @@ -29,7 +29,7 @@ __<bsd.init.mk>__: .if ${MK_DIRDEPS_BUILD} == "yes" && ${.MAKE.LEVEL:U1} == 0 && \ ${BUILD_AT_LEVEL0:Uyes:tl} == "no" && !make(clean*) _SKIP_BUILD= not building at level 0 -.elseif !empty(.MAKEFLAGS:M-V${_V_DO_BUILD}) || \ +.elif !empty(.MAKEFLAGS:M-V${_V_DO_BUILD}) || \ ${.TARGETS:M*install*} == ${.TARGETS} || \ make(clean*) || make(obj) || make(analyze) || make(print-dir) || \ make(destroy*) diff --git a/share/mk/src.libnames.mk b/share/mk/src.libnames.mk index 65d3327269000..c949218154570 100644 --- a/share/mk/src.libnames.mk +++ b/share/mk/src.libnames.mk @@ -209,6 +209,7 @@ _LIBRARIES+= \ # 2nd+ order consumers. Auto-generating this would be better. _DP_80211= sbuf bsdxml _DP_archive= z bz2 lzma bsdxml +_DP_zstd= pthread .if ${MK_BLACKLIST} != "no" _DP_blacklist+= pthread .endif diff --git a/sys/arm/conf/RT1310 b/sys/arm/conf/RT1310 new file mode 100644 index 0000000000000..923c5b37423ac --- /dev/null +++ b/sys/arm/conf/RT1310 @@ -0,0 +1,80 @@ +# +# Custom kernel for RT1310 boards. +# +# $FreeBSD$ +# + +ident RT1310 +include "std.arm" +hints "RT1310.hints" + +# Flattened Device Tree +options FDT +options FDT_DTB_STATIC +makeoptions FDT_DTS_FILE=wzr2-g300n.dts + +makeoptions MODULES_OVERRIDE="" + +#makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols +makeoptions WERROR="-Werror" + +options SCHED_4BSD # 4BSD scheduler +options INET # InterNETworking +options FFS # Berkeley Fast Filesystem +options TMPFS # Efficient memory filesystem +options MSDOSFS + +options ROOTDEVNAME=\"cd9660:/dev/cfid0s.rootfs.uzip\" + +options SYSVSHM # SYSV-style shared memory +options SYSVMSG # SYSV-style message queues +options SYSVSEM # SYSV-style semaphores +options _KPOSIX_PRIORITY_SCHEDULING # Posix P1003_1B real-time extensions +options MUTEX_NOINLINE +options RWLOCK_NOINLINE +options NO_FFS_SNAPSHOT +options NO_SWAPPING + +# Debugging +options ALT_BREAK_TO_DEBUGGER +options DDB +#options DEADLKRES # Enable the deadlock resolver +#options DIAGNOSTIC +#options INVARIANTS # Enable calls of extra sanity checking +#options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS +options KDB +options WITNESS # Enable checks to detect deadlocks and cycles +options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +#options WITNESS_KDB + +# Pseudo devices +device loop +device md +device pty +device random + +# Serial ports +device uart +device uart_ns8250 + +# Flash +device cfi +device cfid + +# Networking +device ether +device mii +device bpf +device fv + +# etherswitch +device mdio +device etherswitch +device miiproxy +device ip17x + +# GPIO +device gpio +device gpioled +device rt1310gpio + diff --git a/sys/arm/ralink/files.ralink b/sys/arm/ralink/files.ralink new file mode 100644 index 0000000000000..10f18ddd6c025 --- /dev/null +++ b/sys/arm/ralink/files.ralink @@ -0,0 +1,9 @@ +# $FreeBSD$ +arm/ralink/rt1310_machdep.c standard +arm/ralink/rt1310_intc.c standard +arm/ralink/rt1310_gpio.c optional rt1310gpio +arm/ralink/rt1310_timer.c standard +arm/ralink/if_fv.c optional fv + +kern/kern_clocksource.c standard + diff --git a/sys/arm/ralink/if_fv.c b/sys/arm/ralink/if_fv.c new file mode 100644 index 0000000000000..31713f32d7b0f --- /dev/null +++ b/sys/arm/ralink/if_fv.c @@ -0,0 +1,1873 @@ +/*- + * Copyright (c) 2016 Hiroki Mori. All rights reserved. + * Copyright (C) 2007 + * Oleksandr Tymoshenko <gonzo@freebsd.org>. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWFV IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE FV DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWFV, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * $Id: $ + * + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +/* + * FV Ethernet interface driver + * copy from mips/idt/if_kr.c and netbsd code + */ +#include <sys/param.h> +#include <sys/endian.h> +#include <sys/systm.h> +#include <sys/sockio.h> +#include <sys/mbuf.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/module.h> +#include <sys/mutex.h> +#include <sys/socket.h> +#include <sys/taskqueue.h> + +#include <net/if.h> +#include <net/if_arp.h> +#include <net/ethernet.h> +#include <net/if_dl.h> +#include <net/if_media.h> +#include <net/if_types.h> +#include <net/if_var.h> + +#include <net/bpf.h> + +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> + +#include <machine/bus.h> +#include <machine/resource.h> +#include <sys/bus.h> +#include <sys/rman.h> + +#include <dev/mii/mii.h> +#include <dev/mii/miivar.h> + +/* Todo: move to options.arm */ +/*#define FV_MDIO*/ + +#ifdef FV_MDIO +#include <dev/mdio/mdio.h> +#include <dev/etherswitch/miiproxy.h> +#include "mdio_if.h" +#endif + +MODULE_DEPEND(are, ether, 1, 1, 1); +MODULE_DEPEND(are, miibus, 1, 1, 1); +#ifdef FV_MDIO +MODULE_DEPEND(are, mdio, 1, 1, 1); +#endif + +#include "miibus_if.h" + +#include <arm/ralink/if_fvreg.h> + +#ifdef FV_DEBUG +void dump_txdesc(struct fv_softc *, int); +void dump_status_reg(struct fv_softc *); +#endif + +static int fv_attach(device_t); +static int fv_detach(device_t); +static int fv_ifmedia_upd(struct ifnet *); +static void fv_ifmedia_sts(struct ifnet *, struct ifmediareq *); +static int fv_ioctl(struct ifnet *, u_long, caddr_t); +static void fv_init(void *); +static void fv_init_locked(struct fv_softc *); +static void fv_link_task(void *, int); +static int fv_miibus_readreg(device_t, int, int); +static void fv_miibus_statchg(device_t); +static int fv_miibus_writereg(device_t, int, int, int); +static int fv_probe(device_t); +static void fv_reset(struct fv_softc *); +static int fv_resume(device_t); +static int fv_rx_ring_init(struct fv_softc *); +static int fv_tx_ring_init(struct fv_softc *); +static int fv_shutdown(device_t); +static void fv_start(struct ifnet *); +static void fv_start_locked(struct ifnet *); +static void fv_stop(struct fv_softc *); +static int fv_suspend(device_t); + +static void fv_rx(struct fv_softc *); +static void fv_tx(struct fv_softc *); +static void fv_intr(void *); +static void fv_tick(void *); + +static void fv_dmamap_cb(void *, bus_dma_segment_t *, int, int); +static int fv_dma_alloc(struct fv_softc *); +static void fv_dma_free(struct fv_softc *); +static int fv_newbuf(struct fv_softc *, int); +static __inline void fv_fixup_rx(struct mbuf *); + +static void fv_hinted_child(device_t bus, const char *dname, int dunit); + +static void fv_setfilt(struct fv_softc *sc); + +static device_method_t fv_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, fv_probe), + DEVMETHOD(device_attach, fv_attach), + DEVMETHOD(device_detach, fv_detach), + DEVMETHOD(device_suspend, fv_suspend), + DEVMETHOD(device_resume, fv_resume), + DEVMETHOD(device_shutdown, fv_shutdown), + + /* MII interface */ + DEVMETHOD(miibus_readreg, fv_miibus_readreg), + DEVMETHOD(miibus_writereg, fv_miibus_writereg), +#if !defined(FV_MDIO) + DEVMETHOD(miibus_statchg, fv_miibus_statchg), +#endif + + /* bus interface */ + DEVMETHOD(bus_add_child, device_add_child_ordered), + DEVMETHOD(bus_hinted_child, fv_hinted_child), + + DEVMETHOD_END +}; + +static driver_t fv_driver = { + "fv", + fv_methods, + sizeof(struct fv_softc) +}; + +static devclass_t fv_devclass; + +DRIVER_MODULE(fv, simplebus, fv_driver, fv_devclass, 0, 0); +#ifdef MII +DRIVER_MODULE(miibus, fv, miibus_driver, miibus_devclass, 0, 0); +#endif + +static struct mtx miibus_mtx; +MTX_SYSINIT(miibus_mtx, &miibus_mtx, "are mii lock", MTX_DEF); + +#ifdef FV_MDIO +static int fvmdio_probe(device_t); +static int fvmdio_attach(device_t); +static int fvmdio_detach(device_t); + +/* + * Declare an additional, separate driver for accessing the MDIO bus. + */ +static device_method_t fvmdio_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, fvmdio_probe), + DEVMETHOD(device_attach, fvmdio_attach), + DEVMETHOD(device_detach, fvmdio_detach), + + /* bus interface */ + DEVMETHOD(bus_add_child, device_add_child_ordered), + + /* MDIO access */ + DEVMETHOD(mdio_readreg, fv_miibus_readreg), + DEVMETHOD(mdio_writereg, fv_miibus_writereg), +}; + +DEFINE_CLASS_0(fvmdio, fvmdio_driver, fvmdio_methods, + sizeof(struct fv_softc)); +static devclass_t fvmdio_devclass; + +DRIVER_MODULE(miiproxy, fv, miiproxy_driver, miiproxy_devclass, 0, 0); +DRIVER_MODULE(fvmdio, simplebus, fvmdio_driver, fvmdio_devclass, 0, 0); +DRIVER_MODULE(mdio, fvmdio, mdio_driver, mdio_devclass, 0, 0); +#endif + +/* setup frame code refer dc code */ + +static void +fv_setfilt(struct fv_softc *sc) +{ + uint16_t eaddr[(ETHER_ADDR_LEN+1)/2]; + struct fv_desc *sframe; + int i; + struct ifnet *ifp; + struct ifmultiaddr *ifma; + uint16_t *sp; + uint8_t *ma; + + ifp = sc->fv_ifp; + + i = sc->fv_cdata.fv_tx_prod; + FV_INC(sc->fv_cdata.fv_tx_prod, FV_TX_RING_CNT); + sc->fv_cdata.fv_tx_cnt++; + sframe = &sc->fv_rdata.fv_tx_ring[i]; + sp = (uint16_t *)sc->fv_cdata.fv_sf_buff; + memset(sp, 0xff, FV_SFRAME_LEN); + + sframe->fv_addr = sc->fv_rdata.fv_sf_paddr; + sframe->fv_devcs = ADCTL_Tx_SETUP | FV_DMASIZE(FV_SFRAME_LEN); + + i = 0; + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + ma = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); + sp[i] = sp[i+1] = (ma[1] << 8 | ma[0]); + i += 2; + sp[i] = sp[i+1] = (ma[3] << 8 | ma[2]); + i += 2; + sp[i] = sp[i+1] = (ma[5] << 8 | ma[4]); + i += 2; + } + if_maddr_runlock(ifp); + + bcopy(IF_LLADDR(sc->fv_ifp), eaddr, ETHER_ADDR_LEN); + sp[90] = sp[91] = eaddr[0]; + sp[92] = sp[93] = eaddr[1]; + sp[94] = sp[95] = eaddr[2]; + + sframe->fv_stat = ADSTAT_OWN; + bus_dmamap_sync(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map, BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(sc->fv_cdata.fv_sf_tag, + sc->fv_cdata.fv_sf_buff_map, BUS_DMASYNC_PREWRITE); + CSR_WRITE_4(sc, CSR_TXPOLL, 0xFFFFFFFF); + DELAY(10000); +} + +static int +fv_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "fv,ethernet")) + return (ENXIO); + + device_set_desc(dev, "FV Ethernet interface"); + return (BUS_PROBE_DEFAULT); +} + +static int +fv_attach(device_t dev) +{ + struct ifnet *ifp; + struct fv_softc *sc; + int error = 0, rid; + int unit; + int i; + + sc = device_get_softc(dev); + unit = device_get_unit(dev); + sc->fv_dev = dev; + sc->fv_ofw = ofw_bus_get_node(dev); + + i = OF_getprop(sc->fv_ofw, "local-mac-address", (void *)&sc->fv_eaddr, 6); + if (i != 6) { + /* hardcode macaddress */ + sc->fv_eaddr[0] = 0x00; + sc->fv_eaddr[1] = 0x0C; + sc->fv_eaddr[2] = 0x42; + sc->fv_eaddr[3] = 0x09; + sc->fv_eaddr[4] = 0x5E; + sc->fv_eaddr[5] = 0x6B; + } + + mtx_init(&sc->fv_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, + MTX_DEF); + callout_init_mtx(&sc->fv_stat_callout, &sc->fv_mtx, 0); + TASK_INIT(&sc->fv_link_task, 0, fv_link_task, sc); + + /* Map control/status registers. */ + sc->fv_rid = 0; + sc->fv_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->fv_rid, + RF_ACTIVE | RF_SHAREABLE); + + if (sc->fv_res == NULL) { + device_printf(dev, "couldn't map memory\n"); + error = ENXIO; + goto fail; + } + + sc->fv_btag = rman_get_bustag(sc->fv_res); + sc->fv_bhandle = rman_get_bushandle(sc->fv_res); + + /* Allocate interrupts */ + rid = 0; + sc->fv_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_SHAREABLE | RF_ACTIVE); + + if (sc->fv_irq == NULL) { + device_printf(dev, "couldn't map interrupt\n"); + error = ENXIO; + goto fail; + } + + /* Allocate ifnet structure. */ + ifp = sc->fv_ifp = if_alloc(IFT_ETHER); + + if (ifp == NULL) { + device_printf(dev, "couldn't allocate ifnet structure\n"); + error = ENOSPC; + goto fail; + } + ifp->if_softc = sc; + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_ioctl = fv_ioctl; + ifp->if_start = fv_start; + ifp->if_init = fv_init; + + /* ifqmaxlen is sysctl value in net/if.c */ + IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen); + ifp->if_snd.ifq_maxlen = ifqmaxlen; + IFQ_SET_READY(&ifp->if_snd); + + ifp->if_capenable = ifp->if_capabilities; + + if (fv_dma_alloc(sc) != 0) { + error = ENXIO; + goto fail; + } + + /* TODO: calculate prescale */ +/* + CSR_WRITE_4(sc, FV_ETHMCP, (165000000 / (1250000 + 1)) & ~1); + + CSR_WRITE_4(sc, FV_MIIMCFG, FV_MIIMCFG_R); + DELAY(1000); + CSR_WRITE_4(sc, FV_MIIMCFG, 0); +*/ + CSR_WRITE_4(sc, CSR_BUSMODE, BUSMODE_SWR); + DELAY(1000); + +#ifdef FV_MDIO + sc->fv_miiproxy = mii_attach_proxy(sc->fv_dev); +#endif + +#ifdef MII + /* Do MII setup. */ + error = mii_attach(dev, &sc->fv_miibus, ifp, fv_ifmedia_upd, + fv_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0); + if (error != 0) { + device_printf(dev, "attaching PHYs failed\n"); + goto fail; + } +#else + ifmedia_init(&sc->fv_ifmedia, 0, fv_ifmedia_upd, fv_ifmedia_sts); + + ifmedia_add(&sc->fv_ifmedia, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&sc->fv_ifmedia, IFM_ETHER | IFM_AUTO); +#endif + + /* Call MI attach routine. */ + ether_ifattach(ifp, sc->fv_eaddr); + + /* Hook interrupt last to avoid having to lock softc */ + error = bus_setup_intr(dev, sc->fv_irq, INTR_TYPE_NET | INTR_MPSAFE, + NULL, fv_intr, sc, &sc->fv_intrhand); + + if (error) { + device_printf(dev, "couldn't set up irq\n"); + ether_ifdetach(ifp); + goto fail; + } + +fail: + if (error) + fv_detach(dev); + + return (error); +} + +static int +fv_detach(device_t dev) +{ + struct fv_softc *sc = device_get_softc(dev); + struct ifnet *ifp = sc->fv_ifp; + + KASSERT(mtx_initialized(&sc->fv_mtx), ("vr mutex not initialized")); + + /* These should only be active if attach succeeded */ + if (device_is_attached(dev)) { + FV_LOCK(sc); + sc->fv_detach = 1; + fv_stop(sc); + FV_UNLOCK(sc); + taskqueue_drain(taskqueue_swi, &sc->fv_link_task); + ether_ifdetach(ifp); + } +#ifdef MII + if (sc->fv_miibus) + device_delete_child(dev, sc->fv_miibus); +#endif + bus_generic_detach(dev); + + if (sc->fv_intrhand) + bus_teardown_intr(dev, sc->fv_irq, sc->fv_intrhand); + if (sc->fv_irq) + bus_release_resource(dev, SYS_RES_IRQ, 0, sc->fv_irq); + + if (sc->fv_res) + bus_release_resource(dev, SYS_RES_MEMORY, sc->fv_rid, + sc->fv_res); + + if (ifp) + if_free(ifp); + + fv_dma_free(sc); + + mtx_destroy(&sc->fv_mtx); + + return (0); + +} + +static int +fv_suspend(device_t dev) +{ + + panic("%s", __func__); + return 0; +} + +static int +fv_resume(device_t dev) +{ + + panic("%s", __func__); + return 0; +} + +static int +fv_shutdown(device_t dev) +{ + struct fv_softc *sc; + + sc = device_get_softc(dev); + + FV_LOCK(sc); + fv_stop(sc); + FV_UNLOCK(sc); + + return (0); +} + +static int +fv_miibus_readbits(struct fv_softc *sc, int count) +{ + int result; + + result = 0; + while(count--) { + result <<= 1; + CSR_WRITE_4(sc, CSR_MIIMNG, MII_RD); + DELAY(10); + CSR_WRITE_4(sc, CSR_MIIMNG, MII_RD | MII_CLK); + DELAY(10); + if (CSR_READ_4(sc, CSR_MIIMNG) & MII_DIN) + result |= 1; + } + + return (result); +} + +static int +fv_miibus_writebits(struct fv_softc *sc, int data, int count) +{ + int bit; + + while(count--) { + bit = ((data) >> count) & 0x1 ? MII_DOUT : 0; + CSR_WRITE_4(sc, CSR_MIIMNG, bit | MII_WR); + DELAY(10); + CSR_WRITE_4(sc, CSR_MIIMNG, bit | MII_WR | MII_CLK); + DELAY(10); + } + + return (0); +} + +static void +fv_miibus_turnaround(struct fv_softc *sc, int cmd) +{ + if (cmd == MII_WRCMD) { + fv_miibus_writebits(sc, 0x02, 2); + } else { + fv_miibus_readbits(sc, 1); + } +} + +static int +fv_miibus_readreg(device_t dev, int phy, int reg) +{ + struct fv_softc * sc = device_get_softc(dev); + int result; + + mtx_lock(&miibus_mtx); + fv_miibus_writebits(sc, MII_PREAMBLE, 32); + fv_miibus_writebits(sc, MII_RDCMD, 4); + fv_miibus_writebits(sc, phy, 5); + fv_miibus_writebits(sc, reg, 5); + fv_miibus_turnaround(sc, MII_RDCMD); + result = fv_miibus_readbits(sc, 16); + fv_miibus_turnaround(sc, MII_RDCMD); + mtx_unlock(&miibus_mtx); + + return (result); +} + +static int +fv_miibus_writereg(device_t dev, int phy, int reg, int data) +{ + struct fv_softc * sc = device_get_softc(dev); + + mtx_lock(&miibus_mtx); + fv_miibus_writebits(sc, MII_PREAMBLE, 32); + fv_miibus_writebits(sc, MII_WRCMD, 4); + fv_miibus_writebits(sc, phy, 5); + fv_miibus_writebits(sc, reg, 5); + fv_miibus_turnaround(sc, MII_WRCMD); + fv_miibus_writebits(sc, data, 16); + mtx_unlock(&miibus_mtx); + + return (0); +} + +#if !defined(FV_MDIO) +static void +fv_miibus_statchg(device_t dev) +{ + struct fv_softc *sc; + + sc = device_get_softc(dev); + taskqueue_enqueue(taskqueue_swi, &sc->fv_link_task); +} +#endif + +static void +fv_link_task(void *arg, int pending) +{ +#ifdef MII + struct fv_softc *sc; + struct mii_data *mii; + struct ifnet *ifp; + /* int lfdx, mfdx; */ + + sc = (struct fv_softc *)arg; + + FV_LOCK(sc); + mii = device_get_softc(sc->fv_miibus); + ifp = sc->fv_ifp; + if (mii == NULL || ifp == NULL || + (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { + FV_UNLOCK(sc); + return; + } + + if (mii->mii_media_status & IFM_ACTIVE) { + if (IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) + sc->fv_link_status = 1; + } else + sc->fv_link_status = 0; + + FV_UNLOCK(sc); +#endif +} + +static void +fv_reset(struct fv_softc *sc) +{ + int i; + + CSR_WRITE_4(sc, CSR_BUSMODE, BUSMODE_SWR); + + /* + * The chip doesn't take itself out of reset automatically. + * We need to do so after 2us. + */ + DELAY(1000); + CSR_WRITE_4(sc, CSR_BUSMODE, 0); + + for (i = 0; i < 1000; i++) { + /* + * Wait a bit for the reset to complete before peeking + * at the chip again. + */ + DELAY(1000); + if ((CSR_READ_4(sc, CSR_BUSMODE) & BUSMODE_SWR) == 0) + break; + } + + if (CSR_READ_4(sc, CSR_BUSMODE) & BUSMODE_SWR) + device_printf(sc->fv_dev, "reset time out\n"); + + DELAY(1000); +} + +static void +fv_init(void *xsc) +{ + struct fv_softc *sc = xsc; + + FV_LOCK(sc); + fv_init_locked(sc); + FV_UNLOCK(sc); +} + +static void +fv_init_locked(struct fv_softc *sc) +{ + struct ifnet *ifp = sc->fv_ifp; +#ifdef MII + struct mii_data *mii; +#endif + + FV_LOCK_ASSERT(sc); + +#ifdef MII + mii = device_get_softc(sc->fv_miibus); +#endif + + fv_stop(sc); + fv_reset(sc); + + /* Init circular RX list. */ + if (fv_rx_ring_init(sc) != 0) { + device_printf(sc->fv_dev, + "initialization failed: no memory for rx buffers\n"); + fv_stop(sc); + return; + } + + /* Init tx descriptors. */ + fv_tx_ring_init(sc); + + /* + * Initialize the BUSMODE register. + */ + CSR_WRITE_4(sc, CSR_BUSMODE, + /* XXX: not sure if this is a good thing or not... */ + BUSMODE_BAR | BUSMODE_PBL_32LW); + + /* + * Initialize the interrupt mask and enable interrupts. + */ + /* normal interrupts */ + sc->sc_inten = STATUS_TI | STATUS_TU | STATUS_RI | STATUS_NIS; + + /* abnormal interrupts */ + sc->sc_inten |= STATUS_TPS | STATUS_TJT | STATUS_UNF | + STATUS_RU | STATUS_RPS | STATUS_SE | STATUS_AIS; + + sc->sc_rxint_mask = STATUS_RI|STATUS_RU; + sc->sc_txint_mask = STATUS_TI|STATUS_UNF|STATUS_TJT; + + sc->sc_rxint_mask &= sc->sc_inten; + sc->sc_txint_mask &= sc->sc_inten; + + CSR_WRITE_4(sc, CSR_INTEN, sc->sc_inten); + CSR_WRITE_4(sc, CSR_STATUS, 0xffffffff); + + /* + * Give the transmit and receive rings to the chip. + */ + CSR_WRITE_4(sc, CSR_TXLIST, FV_TX_RING_ADDR(sc, 0)); + CSR_WRITE_4(sc, CSR_RXLIST, FV_RX_RING_ADDR(sc, 0)); + + /* + * Set the station address. + */ + fv_setfilt(sc); + + + /* + * Write out the opmode. + */ + CSR_WRITE_4(sc, CSR_OPMODE, OPMODE_SR | OPMODE_ST | + OPMODE_TR_128 | OPMODE_FDX | OPMODE_SPEED); + /* + * Start the receive process. + */ + CSR_WRITE_4(sc, CSR_RXPOLL, RXPOLL_RPD); + + sc->fv_link_status = 1; +#ifdef MII + mii_mediachg(mii); +#endif + + ifp->if_drv_flags |= IFF_DRV_RUNNING; + ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + + callout_reset(&sc->fv_stat_callout, hz, fv_tick, sc); +} + +static void +fv_start(struct ifnet *ifp) +{ + struct fv_softc *sc; + + sc = ifp->if_softc; + + FV_LOCK(sc); + fv_start_locked(ifp); + FV_UNLOCK(sc); +} + +/* + * Encapsulate an mbuf chain in a descriptor by coupling the mbuf data + * pointers to the fragment pointers. + * Use Implicit Chain implementation. + */ +static int +fv_encap(struct fv_softc *sc, struct mbuf **m_head) +{ + struct fv_txdesc *txd; + struct fv_desc *desc; + struct mbuf *m; + bus_dma_segment_t txsegs[FV_MAXFRAGS]; + int error, i, nsegs, prod, si; + int padlen; + int txstat; + + FV_LOCK_ASSERT(sc); + + /* + * Some VIA Rhine wants packet buffers to be longword + * aligned, but very often our mbufs aren't. Rather than + * waste time trying to decide when to copy and when not + * to copy, just do it all the time. + */ + m = m_defrag(*m_head, M_NOWAIT); + if (m == NULL) { + device_printf(sc->fv_dev, "fv_encap m_defrag error\n"); + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + + /* + * The Rhine chip doesn't auto-pad, so we have to make + * sure to pad short frames out to the minimum frame length + * ourselves. + */ + if ((*m_head)->m_pkthdr.len < FV_MIN_FRAMELEN) { + m = *m_head; + padlen = FV_MIN_FRAMELEN - m->m_pkthdr.len; + if (M_WRITABLE(m) == 0) { + /* Get a writable copy. */ + m = m_dup(*m_head, M_NOWAIT); + m_freem(*m_head); + if (m == NULL) { + device_printf(sc->fv_dev, "fv_encap m_dup error\n"); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + } + if (m->m_next != NULL || M_TRAILINGSPACE(m) < padlen) { + m = m_defrag(m, M_NOWAIT); + if (m == NULL) { + device_printf(sc->fv_dev, "fv_encap m_defrag error\n"); + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + } + /* + * Manually pad short frames, and zero the pad space + * to avoid leaking data. + */ + bzero(mtod(m, char *) + m->m_pkthdr.len, padlen); + m->m_pkthdr.len += padlen; + m->m_len = m->m_pkthdr.len; + *m_head = m; + } + + prod = sc->fv_cdata.fv_tx_prod; + txd = &sc->fv_cdata.fv_txdesc[prod]; + error = bus_dmamap_load_mbuf_sg(sc->fv_cdata.fv_tx_tag, txd->tx_dmamap, + *m_head, txsegs, &nsegs, BUS_DMA_NOWAIT); + if (error == EFBIG) { + device_printf(sc->fv_dev, "fv_encap EFBIG error\n"); + m = m_defrag(*m_head, M_NOWAIT); + if (m == NULL) { + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + error = bus_dmamap_load_mbuf_sg(sc->fv_cdata.fv_tx_tag, + txd->tx_dmamap, *m_head, txsegs, &nsegs, BUS_DMA_NOWAIT); + if (error != 0) { + m_freem(*m_head); + *m_head = NULL; + return (error); + } + + } else if (error != 0) + return (error); + if (nsegs == 0) { + m_freem(*m_head); + *m_head = NULL; + return (EIO); + } + + /* Check number of available descriptors. */ + if (sc->fv_cdata.fv_tx_cnt + nsegs >= (FV_TX_RING_CNT - 1)) { + bus_dmamap_unload(sc->fv_cdata.fv_tx_tag, txd->tx_dmamap); + return (ENOBUFS); + } + + txd->tx_m = *m_head; + bus_dmamap_sync(sc->fv_cdata.fv_tx_tag, txd->tx_dmamap, + BUS_DMASYNC_PREWRITE); + + si = prod; + + /* + * Make a list of descriptors for this packet. + */ + desc = NULL; + for (i = 0; i < nsegs; i++) { + desc = &sc->fv_rdata.fv_tx_ring[prod]; + desc->fv_stat = ADSTAT_OWN; + desc->fv_devcs = txsegs[i].ds_len; + /* end of descriptor */ + if (prod == FV_TX_RING_CNT - 1) + desc->fv_devcs |= ADCTL_ER; + desc->fv_addr = txsegs[i].ds_addr; + + ++sc->fv_cdata.fv_tx_cnt; + FV_INC(prod, FV_TX_RING_CNT); + } + + /* + * Set mark last fragment with Last/Intr flag + */ + if (desc) { + desc->fv_devcs |= ADCTL_Tx_IC; + desc->fv_devcs |= ADCTL_Tx_LS; + } + + /* Update producer index. */ + sc->fv_cdata.fv_tx_prod = prod; + + /* Sync descriptors. */ + bus_dmamap_sync(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + txstat = (CSR_READ_4(sc, CSR_STATUS) >> 20) & 7; + if (txstat == 0 || txstat == 6) { + /* Transmit Process Stat is stop or suspended */ + desc = &sc->fv_rdata.fv_tx_ring[si]; + desc->fv_devcs |= ADCTL_Tx_FS; + } + else { + /* Get previous descriptor */ + si = (si + FV_TX_RING_CNT - 1) % FV_TX_RING_CNT; + desc = &sc->fv_rdata.fv_tx_ring[si]; + /* join remain data and flugs */ + desc->fv_devcs &= ~ADCTL_Tx_IC; + desc->fv_devcs &= ~ADCTL_Tx_LS; + } + + + return (0); +} + +static void +fv_start_locked(struct ifnet *ifp) +{ + struct fv_softc *sc; + struct mbuf *m_head; + int enq; + int txstat; + + sc = ifp->if_softc; + + FV_LOCK_ASSERT(sc); + + if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING || sc->fv_link_status == 0 ) + return; + + for (enq = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd) && + sc->fv_cdata.fv_tx_cnt < FV_TX_RING_CNT - 2; ) { + IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); + if (m_head == NULL) + break; + /* + * Pack the data into the transmit ring. If we + * don't have room, set the OACTIVE flag and wait + * for the NIC to drain the ring. + */ + if (fv_encap(sc, &m_head)) { + if (m_head == NULL) + break; + IFQ_DRV_PREPEND(&ifp->if_snd, m_head); + ifp->if_drv_flags |= IFF_DRV_OACTIVE; + break; + } + + enq++; + /* + * If there's a BPF listener, bounce a copy of this frame + * to him. + */ + ETHER_BPF_MTAP(ifp, m_head); + } + + if (enq > 0) { + txstat = (CSR_READ_4(sc, CSR_STATUS) >> 20) & 7; + if (txstat == 0 || txstat == 6) + CSR_WRITE_4(sc, CSR_TXPOLL, TXPOLL_TPD); + } +} + +static void +fv_stop(struct fv_softc *sc) +{ + struct ifnet *ifp; + + FV_LOCK_ASSERT(sc); + + ifp = sc->fv_ifp; + ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + callout_stop(&sc->fv_stat_callout); + + /* Disable interrupts. */ + CSR_WRITE_4(sc, CSR_INTEN, 0); + + /* Stop the transmit and receive processes. */ + CSR_WRITE_4(sc, CSR_OPMODE, 0); + CSR_WRITE_4(sc, CSR_RXLIST, 0); + CSR_WRITE_4(sc, CSR_TXLIST, 0); + +} + + +static int +fv_ioctl(struct ifnet *ifp, u_long command, caddr_t data) +{ + struct fv_softc *sc = ifp->if_softc; + struct ifreq *ifr = (struct ifreq *) data; +#ifdef MII + struct mii_data *mii; +#endif + int error; + int csr; + + switch (command) { + case SIOCSIFFLAGS: + FV_LOCK(sc); + if (ifp->if_flags & IFF_UP) { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if ((ifp->if_flags ^ sc->fv_if_flags) & + IFF_PROMISC) { + csr = CSR_READ_4(sc, CSR_OPMODE); + CSR_WRITE_4(sc, CSR_OPMODE, csr | + OPMODE_PM | OPMODE_PR); + } + if ((ifp->if_flags ^ sc->fv_if_flags) & + IFF_ALLMULTI) { + csr = CSR_READ_4(sc, CSR_OPMODE); + CSR_WRITE_4(sc, CSR_OPMODE, csr | + OPMODE_PM); + } + } else { + if (sc->fv_detach == 0) + fv_init_locked(sc); + } + } else { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + fv_stop(sc); + } + sc->fv_if_flags = ifp->if_flags; + FV_UNLOCK(sc); + error = 0; + break; + case SIOCADDMULTI: + case SIOCDELMULTI: +#if 0 + FV_LOCK(sc); + fv_set_filter(sc); + FV_UNLOCK(sc); +#endif + error = 0; + break; + case SIOCGIFMEDIA: + case SIOCSIFMEDIA: +#ifdef MII + mii = device_get_softc(sc->fv_miibus); + error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command); +#else + error = ifmedia_ioctl(ifp, ifr, &sc->fv_ifmedia, command); +#endif + break; + case SIOCSIFCAP: + error = 0; +#if 0 + mask = ifr->ifr_reqcap ^ ifp->if_capenable; + if ((mask & IFCAP_HWCSUM) != 0) { + ifp->if_capenable ^= IFCAP_HWCSUM; + if ((IFCAP_HWCSUM & ifp->if_capenable) && + (IFCAP_HWCSUM & ifp->if_capabilities)) + ifp->if_hwassist = FV_CSUM_FEATURES; + else + ifp->if_hwassist = 0; + } + if ((mask & IFCAP_VLAN_HWTAGGING) != 0) { + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if (IFCAP_VLAN_HWTAGGING & ifp->if_capenable && + IFCAP_VLAN_HWTAGGING & ifp->if_capabilities && + ifp->if_drv_flags & IFF_DRV_RUNNING) { + FV_LOCK(sc); + fv_vlan_setup(sc); + FV_UNLOCK(sc); + } + } + VLAN_CAPABILITIES(ifp); +#endif + break; + default: + error = ether_ioctl(ifp, command, data); + break; + } + + return (error); +} + +/* + * Set media options. + */ +static int +fv_ifmedia_upd(struct ifnet *ifp) +{ +#ifdef MII + struct fv_softc *sc; + struct mii_data *mii; + struct mii_softc *miisc; + int error; + + sc = ifp->if_softc; + FV_LOCK(sc); + mii = device_get_softc(sc->fv_miibus); + LIST_FOREACH(miisc, &mii->mii_phys, mii_list) + PHY_RESET(miisc); + error = mii_mediachg(mii); + FV_UNLOCK(sc); + + return (error); +#else + return (0); +#endif +} + +/* + * Report current media status. + */ +static void +fv_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) +{ +#ifdef MII + struct fv_softc *sc = ifp->if_softc; + struct mii_data *mii; + + mii = device_get_softc(sc->fv_miibus); + FV_LOCK(sc); + mii_pollstat(mii); + ifmr->ifm_active = mii->mii_media_active; + ifmr->ifm_status = mii->mii_media_status; + FV_UNLOCK(sc); +#else + ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; +#endif +} + +struct fv_dmamap_arg { + bus_addr_t fv_busaddr; +}; + +static void +fv_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct fv_dmamap_arg *ctx; + + if (error != 0) + return; + ctx = arg; + ctx->fv_busaddr = segs[0].ds_addr; +} + +static int +fv_dma_alloc(struct fv_softc *sc) +{ + struct fv_dmamap_arg ctx; + struct fv_txdesc *txd; + struct fv_rxdesc *rxd; + int error, i; + + /* Create parent DMA tag. */ + error = bus_dma_tag_create( + bus_get_dma_tag(sc->fv_dev), /* parent */ + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ + 0, /* nsegments */ + BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->fv_cdata.fv_parent_tag); + if (error != 0) { + device_printf(sc->fv_dev, "failed to create parent DMA tag\n"); + goto fail; + } + /* Create tag for Tx ring. */ + error = bus_dma_tag_create( + sc->fv_cdata.fv_parent_tag, /* parent */ + FV_RING_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + FV_TX_RING_SIZE, /* maxsize */ + 1, /* nsegments */ + FV_TX_RING_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->fv_cdata.fv_tx_ring_tag); + if (error != 0) { + device_printf(sc->fv_dev, "failed to create Tx ring DMA tag\n"); + goto fail; + } + + /* Create tag for Rx ring. */ + error = bus_dma_tag_create( + sc->fv_cdata.fv_parent_tag, /* parent */ + FV_RING_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + FV_RX_RING_SIZE, /* maxsize */ + 1, /* nsegments */ + FV_RX_RING_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->fv_cdata.fv_rx_ring_tag); + if (error != 0) { + device_printf(sc->fv_dev, "failed to create Rx ring DMA tag\n"); + goto fail; + } + + /* Create tag for Tx buffers. */ + error = bus_dma_tag_create( + sc->fv_cdata.fv_parent_tag, /* parent */ + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MCLBYTES * FV_MAXFRAGS, /* maxsize */ + FV_MAXFRAGS, /* nsegments */ + MCLBYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->fv_cdata.fv_tx_tag); + if (error != 0) { + device_printf(sc->fv_dev, "failed to create Tx DMA tag\n"); + goto fail; + } + + /* Create tag for Rx buffers. */ + error = bus_dma_tag_create( + sc->fv_cdata.fv_parent_tag, /* parent */ + FV_RX_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MCLBYTES, /* maxsize */ + 1, /* nsegments */ + MCLBYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->fv_cdata.fv_rx_tag); + if (error != 0) { + device_printf(sc->fv_dev, "failed to create Rx DMA tag\n"); + goto fail; + } + + /* Create tag for setup frame buffers. */ + error = bus_dma_tag_create( + sc->fv_cdata.fv_parent_tag, /* parent */ + sizeof(uint32_t), 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + FV_SFRAME_LEN + FV_MIN_FRAMELEN, /* maxsize */ + 1, /* nsegments */ + FV_SFRAME_LEN + FV_MIN_FRAMELEN, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->fv_cdata.fv_sf_tag); + if (error != 0) { + device_printf(sc->fv_dev, "failed to create setup frame DMA tag\n"); + goto fail; + } + + /* Allocate DMA'able memory and load the DMA map for Tx ring. */ + error = bus_dmamem_alloc(sc->fv_cdata.fv_tx_ring_tag, + (void **)&sc->fv_rdata.fv_tx_ring, BUS_DMA_WAITOK | + BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->fv_cdata.fv_tx_ring_map); + if (error != 0) { + device_printf(sc->fv_dev, + "failed to allocate DMA'able memory for Tx ring\n"); + goto fail; + } + + ctx.fv_busaddr = 0; + error = bus_dmamap_load(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map, sc->fv_rdata.fv_tx_ring, + FV_TX_RING_SIZE, fv_dmamap_cb, &ctx, 0); + if (error != 0 || ctx.fv_busaddr == 0) { + device_printf(sc->fv_dev, + "failed to load DMA'able memory for Tx ring\n"); + goto fail; + } + sc->fv_rdata.fv_tx_ring_paddr = ctx.fv_busaddr; + + /* Allocate DMA'able memory and load the DMA map for Rx ring. */ + error = bus_dmamem_alloc(sc->fv_cdata.fv_rx_ring_tag, + (void **)&sc->fv_rdata.fv_rx_ring, BUS_DMA_WAITOK | + BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->fv_cdata.fv_rx_ring_map); + if (error != 0) { + device_printf(sc->fv_dev, + "failed to allocate DMA'able memory for Rx ring\n"); + goto fail; + } + + ctx.fv_busaddr = 0; + error = bus_dmamap_load(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_cdata.fv_rx_ring_map, sc->fv_rdata.fv_rx_ring, + FV_RX_RING_SIZE, fv_dmamap_cb, &ctx, 0); + if (error != 0 || ctx.fv_busaddr == 0) { + device_printf(sc->fv_dev, + "failed to load DMA'able memory for Rx ring\n"); + goto fail; + } + sc->fv_rdata.fv_rx_ring_paddr = ctx.fv_busaddr; + + /* Allocate DMA'able memory and load the DMA map for setup frame. */ + error = bus_dmamem_alloc(sc->fv_cdata.fv_sf_tag, + (void **)&sc->fv_cdata.fv_sf_buff, BUS_DMA_WAITOK | + BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->fv_cdata.fv_sf_buff_map); + if (error != 0) { + device_printf(sc->fv_dev, + "failed to allocate DMA'able memory for setup frame\n"); + goto fail; + } + + ctx.fv_busaddr = 0; + error = bus_dmamap_load(sc->fv_cdata.fv_sf_tag, + sc->fv_cdata.fv_sf_buff_map, sc->fv_cdata.fv_sf_buff, + FV_SFRAME_LEN, fv_dmamap_cb, &ctx, 0); + if (error != 0 || ctx.fv_busaddr == 0) { + device_printf(sc->fv_dev, + "failed to load DMA'able memory for setup frame\n"); + goto fail; + } + sc->fv_rdata.fv_sf_paddr = ctx.fv_busaddr; + + /* Create DMA maps for Tx buffers. */ + for (i = 0; i < FV_TX_RING_CNT; i++) { + txd = &sc->fv_cdata.fv_txdesc[i]; + txd->tx_m = NULL; + txd->tx_dmamap = NULL; + error = bus_dmamap_create(sc->fv_cdata.fv_tx_tag, 0, + &txd->tx_dmamap); + if (error != 0) { + device_printf(sc->fv_dev, + "failed to create Tx dmamap\n"); + goto fail; + } + } + /* Create DMA maps for Rx buffers. */ + if ((error = bus_dmamap_create(sc->fv_cdata.fv_rx_tag, 0, + &sc->fv_cdata.fv_rx_sparemap)) != 0) { + device_printf(sc->fv_dev, + "failed to create spare Rx dmamap\n"); + goto fail; + } + for (i = 0; i < FV_RX_RING_CNT; i++) { + rxd = &sc->fv_cdata.fv_rxdesc[i]; + rxd->rx_m = NULL; + rxd->rx_dmamap = NULL; + error = bus_dmamap_create(sc->fv_cdata.fv_rx_tag, 0, + &rxd->rx_dmamap); + if (error != 0) { + device_printf(sc->fv_dev, + "failed to create Rx dmamap\n"); + goto fail; + } + } + +fail: + return (error); +} + +static void +fv_dma_free(struct fv_softc *sc) +{ + struct fv_txdesc *txd; + struct fv_rxdesc *rxd; + int i; + + /* Tx ring. */ + if (sc->fv_cdata.fv_tx_ring_tag) { + if (sc->fv_rdata.fv_tx_ring_paddr) + bus_dmamap_unload(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map); + if (sc->fv_rdata.fv_tx_ring) + bus_dmamem_free(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_rdata.fv_tx_ring, + sc->fv_cdata.fv_tx_ring_map); + sc->fv_rdata.fv_tx_ring = NULL; + sc->fv_rdata.fv_tx_ring_paddr = 0; + bus_dma_tag_destroy(sc->fv_cdata.fv_tx_ring_tag); + sc->fv_cdata.fv_tx_ring_tag = NULL; + } + /* Rx ring. */ + if (sc->fv_cdata.fv_rx_ring_tag) { + if (sc->fv_rdata.fv_rx_ring_paddr) + bus_dmamap_unload(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_cdata.fv_rx_ring_map); + if (sc->fv_rdata.fv_rx_ring) + bus_dmamem_free(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_rdata.fv_rx_ring, + sc->fv_cdata.fv_rx_ring_map); + sc->fv_rdata.fv_rx_ring = NULL; + sc->fv_rdata.fv_rx_ring_paddr = 0; + bus_dma_tag_destroy(sc->fv_cdata.fv_rx_ring_tag); + sc->fv_cdata.fv_rx_ring_tag = NULL; + } + /* Tx buffers. */ + if (sc->fv_cdata.fv_tx_tag) { + for (i = 0; i < FV_TX_RING_CNT; i++) { + txd = &sc->fv_cdata.fv_txdesc[i]; + if (txd->tx_dmamap) { + bus_dmamap_destroy(sc->fv_cdata.fv_tx_tag, + txd->tx_dmamap); + txd->tx_dmamap = NULL; + } + } + bus_dma_tag_destroy(sc->fv_cdata.fv_tx_tag); + sc->fv_cdata.fv_tx_tag = NULL; + } + /* Rx buffers. */ + if (sc->fv_cdata.fv_rx_tag) { + for (i = 0; i < FV_RX_RING_CNT; i++) { + rxd = &sc->fv_cdata.fv_rxdesc[i]; + if (rxd->rx_dmamap) { + bus_dmamap_destroy(sc->fv_cdata.fv_rx_tag, + rxd->rx_dmamap); + rxd->rx_dmamap = NULL; + } + } + if (sc->fv_cdata.fv_rx_sparemap) { + bus_dmamap_destroy(sc->fv_cdata.fv_rx_tag, + sc->fv_cdata.fv_rx_sparemap); + sc->fv_cdata.fv_rx_sparemap = 0; + } + bus_dma_tag_destroy(sc->fv_cdata.fv_rx_tag); + sc->fv_cdata.fv_rx_tag = NULL; + } + + if (sc->fv_cdata.fv_parent_tag) { + bus_dma_tag_destroy(sc->fv_cdata.fv_parent_tag); + sc->fv_cdata.fv_parent_tag = NULL; + } +} + +/* + * Initialize the transmit descriptors. + */ +static int +fv_tx_ring_init(struct fv_softc *sc) +{ + struct fv_ring_data *rd; + struct fv_txdesc *txd; + bus_addr_t addr; + int i; + + sc->fv_cdata.fv_tx_prod = 0; + sc->fv_cdata.fv_tx_cons = 0; + sc->fv_cdata.fv_tx_cnt = 0; + sc->fv_cdata.fv_tx_pkts = 0; + + rd = &sc->fv_rdata; + bzero(rd->fv_tx_ring, FV_TX_RING_SIZE); + for (i = 0; i < FV_TX_RING_CNT; i++) { + if (i == FV_TX_RING_CNT - 1) + addr = FV_TX_RING_ADDR(sc, 0); + else + addr = FV_TX_RING_ADDR(sc, i + 1); + rd->fv_tx_ring[i].fv_stat = 0; + rd->fv_tx_ring[i].fv_devcs = 0; + rd->fv_tx_ring[i].fv_addr = 0; + rd->fv_tx_ring[i].fv_link = addr; + txd = &sc->fv_cdata.fv_txdesc[i]; + txd->tx_m = NULL; + } + + bus_dmamap_sync(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + return (0); +} + +/* + * Initialize the RX descriptors and allocate mbufs for them. Note that + * we arrange the descriptors in a closed ring, so that the last descriptor + * points back to the first. + */ +static int +fv_rx_ring_init(struct fv_softc *sc) +{ + struct fv_ring_data *rd; + struct fv_rxdesc *rxd; + int i; + + sc->fv_cdata.fv_rx_cons = 0; + + rd = &sc->fv_rdata; + bzero(rd->fv_rx_ring, FV_RX_RING_SIZE); + for (i = 0; i < FV_RX_RING_CNT; i++) { + rxd = &sc->fv_cdata.fv_rxdesc[i]; + rxd->rx_m = NULL; + rxd->desc = &rd->fv_rx_ring[i]; + rd->fv_rx_ring[i].fv_stat = ADSTAT_OWN; + rd->fv_rx_ring[i].fv_devcs = 0; + if (i == FV_RX_RING_CNT - 1) + rd->fv_rx_ring[i].fv_devcs |= ADCTL_ER; + rd->fv_rx_ring[i].fv_addr = 0; + if (fv_newbuf(sc, i) != 0) + return (ENOBUFS); + } + + bus_dmamap_sync(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_cdata.fv_rx_ring_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + return (0); +} + +/* + * Initialize an RX descriptor and attach an MBUF cluster. + */ +static int +fv_newbuf(struct fv_softc *sc, int idx) +{ + struct fv_desc *desc; + struct fv_rxdesc *rxd; + struct mbuf *m; + bus_dma_segment_t segs[1]; + bus_dmamap_t map; + int nsegs; + + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + if (m == NULL) + return (ENOBUFS); + m->m_len = m->m_pkthdr.len = MCLBYTES; + + /* tcp header boundary alignment margin */ + m_adj(m, 4); + + if (bus_dmamap_load_mbuf_sg(sc->fv_cdata.fv_rx_tag, + sc->fv_cdata.fv_rx_sparemap, m, segs, &nsegs, 0) != 0) { + m_freem(m); + return (ENOBUFS); + } + KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); + + rxd = &sc->fv_cdata.fv_rxdesc[idx]; + if (rxd->rx_m != NULL) { +/* This code make bug. Make scranble on buffer data. + bus_dmamap_sync(sc->fv_cdata.fv_rx_tag, rxd->rx_dmamap, + BUS_DMASYNC_POSTREAD); +*/ + bus_dmamap_unload(sc->fv_cdata.fv_rx_tag, rxd->rx_dmamap); + } + map = rxd->rx_dmamap; + rxd->rx_dmamap = sc->fv_cdata.fv_rx_sparemap; + sc->fv_cdata.fv_rx_sparemap = map; + bus_dmamap_sync(sc->fv_cdata.fv_rx_tag, rxd->rx_dmamap, + BUS_DMASYNC_PREREAD); + rxd->rx_m = m; + desc = rxd->desc; + desc->fv_addr = segs[0].ds_addr; + desc->fv_devcs |= FV_DMASIZE(segs[0].ds_len); + rxd->saved_ca = desc->fv_addr ; + rxd->saved_ctl = desc->fv_stat ; + + return (0); +} + +static __inline void +fv_fixup_rx(struct mbuf *m) +{ + int i; + uint16_t *src, *dst; + + src = mtod(m, uint16_t *); + dst = src - 1; + + for (i = 0; i < m->m_len / sizeof(uint16_t); i++) { + *dst++ = *src++; + } + + if (m->m_len % sizeof(uint16_t)) + *(uint8_t *)dst = *(uint8_t *)src; + + m->m_data -= ETHER_ALIGN; +} + + +static void +fv_tx(struct fv_softc *sc) +{ + struct fv_txdesc *txd; + struct fv_desc *cur_tx; + struct ifnet *ifp; + uint32_t ctl, devcs; + int cons, prod, prev_cons; + + FV_LOCK_ASSERT(sc); + + cons = sc->fv_cdata.fv_tx_cons; + prod = sc->fv_cdata.fv_tx_prod; + if (cons == prod) + return; + + bus_dmamap_sync(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + ifp = sc->fv_ifp; + /* + * Go through our tx list and free mbufs for those + * frames that have been transmitted. + */ + prev_cons = cons; + for (; cons != prod; FV_INC(cons, FV_TX_RING_CNT)) { + cur_tx = &sc->fv_rdata.fv_tx_ring[cons]; + ctl = cur_tx->fv_stat; + devcs = cur_tx->fv_devcs; + /* Check if descriptor has "finished" flag */ + if (FV_DMASIZE(devcs) == 0) + break; + + sc->fv_cdata.fv_tx_cnt--; + ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + + txd = &sc->fv_cdata.fv_txdesc[cons]; + + if ((ctl & ADSTAT_Tx_ES) == 0) + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); + else if (ctl & ADSTAT_Tx_UF) { /* only underflow not check collision */ + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + } + + bus_dmamap_sync(sc->fv_cdata.fv_tx_tag, txd->tx_dmamap, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(sc->fv_cdata.fv_tx_tag, txd->tx_dmamap); + + /* Free only if it's first descriptor in list */ + if (txd->tx_m) + m_freem(txd->tx_m); + txd->tx_m = NULL; + + /* reset descriptor */ + cur_tx->fv_stat = 0; + cur_tx->fv_devcs = 0; + cur_tx->fv_addr = 0; + } + + sc->fv_cdata.fv_tx_cons = cons; + + bus_dmamap_sync(sc->fv_cdata.fv_tx_ring_tag, + sc->fv_cdata.fv_tx_ring_map, BUS_DMASYNC_PREWRITE); +} + + +static void +fv_rx(struct fv_softc *sc) +{ + struct fv_rxdesc *rxd; + struct ifnet *ifp = sc->fv_ifp; + int cons, prog, packet_len, error; + struct fv_desc *cur_rx; + struct mbuf *m; + + FV_LOCK_ASSERT(sc); + + cons = sc->fv_cdata.fv_rx_cons; + + bus_dmamap_sync(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_cdata.fv_rx_ring_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + for (prog = 0; prog < FV_RX_RING_CNT; FV_INC(cons, FV_RX_RING_CNT)) { + cur_rx = &sc->fv_rdata.fv_rx_ring[cons]; + rxd = &sc->fv_cdata.fv_rxdesc[cons]; + m = rxd->rx_m; + + if ((cur_rx->fv_stat & ADSTAT_OWN) == ADSTAT_OWN) + break; + + prog++; + + if (cur_rx->fv_stat & (ADSTAT_ES | ADSTAT_Rx_TL)) { + device_printf(sc->fv_dev, + "Receive Descriptor error %x\n", cur_rx->fv_stat); + if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); + packet_len = 0; + } else { + packet_len = ADSTAT_Rx_LENGTH(cur_rx->fv_stat); + } + + /* Assume it's error */ + error = 1; + + if (packet_len < 64) + if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); + else if ((cur_rx->fv_stat & ADSTAT_Rx_DE) == 0) { + error = 0; + bus_dmamap_sync(sc->fv_cdata.fv_rx_tag, rxd->rx_dmamap, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + m = rxd->rx_m; + /* Skip 4 bytes of CRC */ + m->m_pkthdr.len = m->m_len = packet_len - ETHER_CRC_LEN; + + fv_fixup_rx(m); + m->m_pkthdr.rcvif = ifp; + if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); + + FV_UNLOCK(sc); + (*ifp->if_input)(ifp, m); + FV_LOCK(sc); + } + + if (error) { + /* Restore CONTROL and CA values, reset DEVCS */ + cur_rx->fv_stat = rxd->saved_ctl; + cur_rx->fv_addr = rxd->saved_ca; + cur_rx->fv_devcs = 0; + } + else { + /* Reinit descriptor */ + cur_rx->fv_stat = ADSTAT_OWN; + cur_rx->fv_devcs = 0; + if (cons == FV_RX_RING_CNT - 1) + cur_rx->fv_devcs |= ADCTL_ER; + cur_rx->fv_addr = 0; + if (fv_newbuf(sc, cons) != 0) { + device_printf(sc->fv_dev, + "Failed to allocate buffer\n"); + break; + } + } + + bus_dmamap_sync(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_cdata.fv_rx_ring_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + } + + if (prog > 0) { + sc->fv_cdata.fv_rx_cons = cons; + + bus_dmamap_sync(sc->fv_cdata.fv_rx_ring_tag, + sc->fv_cdata.fv_rx_ring_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + } +} + +static void +fv_intr(void *arg) +{ + struct fv_softc *sc = arg; + uint32_t status; + struct ifnet *ifp = sc->fv_ifp; + + FV_LOCK(sc); + + status = CSR_READ_4(sc, CSR_STATUS); + /* mask out interrupts */ + while((status & sc->sc_inten) != 0) { + if (status) { + CSR_WRITE_4(sc, CSR_STATUS, status); + } + if (status & STATUS_UNF) { + device_printf(sc->fv_dev, "Transmit Underflow\n"); + } + if (status & sc->sc_rxint_mask) { + fv_rx(sc); + } + if (status & sc->sc_txint_mask) { + fv_tx(sc); + } + if (status & STATUS_AIS) { + device_printf(sc->fv_dev, "Abnormal Interrupt %x\n", + status); + } + CSR_WRITE_4(sc, CSR_FULLDUP, FULLDUP_CS | + (1 << FULLDUP_TT_SHIFT) | (3 << FULLDUP_NTP_SHIFT) | + (2 << FULLDUP_RT_SHIFT) | (2 << FULLDUP_NRP_SHIFT)); + + + status = CSR_READ_4(sc, CSR_STATUS); + } + + /* Try to get more packets going. */ + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + fv_start_locked(ifp); + + FV_UNLOCK(sc); +} + +static void +fv_tick(void *xsc) +{ + struct fv_softc *sc = xsc; +#ifdef MII + struct mii_data *mii; + + FV_LOCK_ASSERT(sc); + + mii = device_get_softc(sc->fv_miibus); + mii_tick(mii); +#endif + callout_reset(&sc->fv_stat_callout, hz, fv_tick, sc); +} + +static void +fv_hinted_child(device_t bus, const char *dname, int dunit) +{ + BUS_ADD_CHILD(bus, 0, dname, dunit); + device_printf(bus, "hinted child %s%d\n", dname, dunit); +} + +#ifdef FV_MDIO +static int +fvmdio_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "fv,mdio")) + return (ENXIO); + + device_set_desc(dev, "FV built-in ethernet interface, MDIO controller"); + return(0); +} + +static int +fvmdio_attach(device_t dev) +{ + struct fv_softc *sc; + int error; + + sc = device_get_softc(dev); + sc->fv_dev = dev; + sc->fv_rid = 0; + sc->fv_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->fv_rid, RF_ACTIVE | RF_SHAREABLE); + if (sc->fv_res == NULL) { + device_printf(dev, "couldn't map memory\n"); + error = ENXIO; + goto fail; + } + + sc->fv_btag = rman_get_bustag(sc->fv_res); + sc->fv_bhandle = rman_get_bushandle(sc->fv_res); + + bus_generic_probe(dev); + bus_enumerate_hinted_children(dev); + error = bus_generic_attach(dev); +fail: + return(error); +} + +static int +fvmdio_detach(device_t dev) +{ + return(0); +} +#endif + +#ifdef FV_DEBUG +void +dump_txdesc(struct fv_softc *sc, int pos) +{ + struct fv_desc *desc; + + desc = &sc->fv_rdata.fv_tx_ring[pos]; + device_printf(sc->fv_dev, "CSR_TXLIST %08x\n", CSR_READ_4(sc, CSR_TXLIST)); + device_printf(sc->fv_dev, "%d TDES0:%08x TDES1:%08x TDES2:%08x TDES3:%08x\n", + pos, desc->fv_stat, desc->fv_devcs, desc->fv_addr, desc->fv_link); +} + +void +dump_status_reg(struct fv_softc *sc) +{ + uint32_t status; + + /* mask out interrupts */ + + status = CSR_READ_4(sc, CSR_STATUS); + device_printf(sc->fv_dev, "CSR5 Status Register EB:%d TS:%d RS:%d NIS:%d AIS:%d ER:%d SE:%d LNF:%d TM:%d RWT:%d RPS:%d RU:%d RI:%d UNF:%d LNP/ANC:%d TJT:%d TU:%d TPS:%d TI:%d\n", + (status >> 23 ) & 7, + (status >> 20 ) & 7, + (status >> 17 ) & 7, + (status >> 16 ) & 1, + (status >> 15 ) & 1, + (status >> 14 ) & 1, + (status >> 13 ) & 1, + (status >> 12 ) & 1, + (status >> 11 ) & 1, + (status >> 9 ) & 1, + (status >> 8 ) & 1, + (status >> 7 ) & 1, + (status >> 6 ) & 1, + (status >> 5 ) & 1, + (status >> 4 ) & 1, + (status >> 3 ) & 1, + (status >> 2 ) & 1, + (status >> 1 ) & 1, + (status >> 0 ) & 1); + +} +#endif diff --git a/sys/arm/ralink/if_fvreg.h b/sys/arm/ralink/if_fvreg.h new file mode 100644 index 0000000000000..2c4d2ad9830d1 --- /dev/null +++ b/sys/arm/ralink/if_fvreg.h @@ -0,0 +1,452 @@ +/*- + * Copyright (C) 2007 + * Oleksandr Tymoshenko <gonzo@freebsd.org>. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWFV IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE FV DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWFV, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef __IF_FVREG_H__ +#define __IF_FVREG_H__ + +struct fv_desc { + uint32_t fv_stat; + uint32_t fv_devcs; + uint32_t fv_addr; + uint32_t fv_link; +}; + +#define FV_DMASIZE(len) ((len) & ((1 << 11)-1)) +#define FV_PKTSIZE(len) ((len & 0xffff0000) >> 16) + +#define FV_RX_RING_CNT 128 +#define FV_TX_RING_CNT 128 +#define FV_TX_RING_SIZE sizeof(struct fv_desc) * FV_TX_RING_CNT +#define FV_RX_RING_SIZE sizeof(struct fv_desc) * FV_RX_RING_CNT +#define FV_RING_ALIGN sizeof(struct fv_desc) +#define FV_RX_ALIGN sizeof(uint32_t) +#define FV_MAXFRAGS 8 +#define FV_TX_INTR_THRESH 8 + +#define FV_TX_RING_ADDR(sc, i) \ + ((sc)->fv_rdata.fv_tx_ring_paddr + sizeof(struct fv_desc) * (i)) +#define FV_RX_RING_ADDR(sc, i) \ + ((sc)->fv_rdata.fv_rx_ring_paddr + sizeof(struct fv_desc) * (i)) +#define FV_INC(x,y) (x) = (((x) + 1) % y) + +struct fv_txdesc { + struct mbuf *tx_m; + bus_dmamap_t tx_dmamap; +}; + +struct fv_rxdesc { + struct mbuf *rx_m; + bus_dmamap_t rx_dmamap; + struct fv_desc *desc; + /* Use this values on error instead of allocating new mbuf */ + uint32_t saved_ctl, saved_ca; +}; + +struct fv_chain_data { + bus_dma_tag_t fv_parent_tag; + bus_dma_tag_t fv_tx_tag; + struct fv_txdesc fv_txdesc[FV_TX_RING_CNT]; + bus_dma_tag_t fv_rx_tag; + struct fv_rxdesc fv_rxdesc[FV_RX_RING_CNT]; + bus_dma_tag_t fv_tx_ring_tag; + bus_dma_tag_t fv_rx_ring_tag; + bus_dmamap_t fv_tx_ring_map; + bus_dmamap_t fv_rx_ring_map; + bus_dmamap_t fv_rx_sparemap; + int fv_tx_pkts; + int fv_tx_prod; + int fv_tx_cons; + int fv_tx_cnt; + int fv_rx_cons; + + bus_dma_tag_t fv_sf_tag; + bus_dmamap_t fv_sf_buff_map; + uint32_t *fv_sf_buff; +}; + +struct fv_ring_data { + struct fv_desc *fv_rx_ring; + struct fv_desc *fv_tx_ring; + bus_addr_t fv_rx_ring_paddr; + bus_addr_t fv_tx_ring_paddr; + bus_addr_t fv_sf_paddr; +}; + +struct fv_softc { + struct ifnet *fv_ifp; /* interface info */ + bus_space_handle_t fv_bhandle; /* bus space handle */ + bus_space_tag_t fv_btag; /* bus space tag */ + device_t fv_dev; + uint8_t fv_eaddr[ETHER_ADDR_LEN]; + struct resource *fv_res; + int fv_rid; + struct resource *fv_irq; + void *fv_intrhand; + u_int32_t sc_inten; /* copy of CSR_INTEN */ + u_int32_t sc_rxint_mask; /* mask of Rx interrupts we want */ + u_int32_t sc_txint_mask; /* mask of Tx interrupts we want */ +#ifdef MII + device_t fv_miibus; +#else + struct ifmedia fv_ifmedia; +#endif +#ifdef FV_MDIO + device_t fv_miiproxy; +#endif + int fv_if_flags; + bus_dma_tag_t fv_parent_tag; + bus_dma_tag_t fv_tag; + struct mtx fv_mtx; + phandle_t fv_ofw; + struct callout fv_stat_callout; + struct task fv_link_task; + struct fv_chain_data fv_cdata; + struct fv_ring_data fv_rdata; + int fv_link_status; + int fv_detach; +}; + +#define FV_LOCK(_sc) mtx_lock(&(_sc)->fv_mtx) +#define FV_UNLOCK(_sc) mtx_unlock(&(_sc)->fv_mtx) +#define FV_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->fv_mtx, MA_OWNED) + +/* + * register space access macros + */ +#define CSR_WRITE_4(sc, reg, val) \ + bus_space_write_4(sc->fv_btag, sc->fv_bhandle, reg, val) + +#define CSR_READ_4(sc, reg) \ + bus_space_read_4(sc->fv_btag, sc->fv_bhandle, reg) + + +/* $NetBSD: aereg.h,v 1.2 2008/04/28 20:23:28 martin Exp $ */ + +/*- + * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, + * NASA Ames Research Center. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWFV IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE FV DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWFV, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Descriptor Status bits common to transmit and receive. + */ +#define ADSTAT_OWN 0x80000000 /* Tulip owns descriptor */ +#define ADSTAT_ES 0x00008000 /* Error Summary */ + +/* + * Descriptor Status bits for Receive Descriptor. + */ +#define ADSTAT_Rx_FF 0x40000000 /* Filtering Fail */ +#define ADSTAT_Rx_FL 0x3fff0000 /* Frame Length including CRC */ +#define ADSTAT_Rx_DE 0x00004000 /* Descriptor Error */ +#define ADSTAT_Rx_LE 0x00001000 /* Length Error */ +#define ADSTAT_Rx_RF 0x00000800 /* Runt Frame */ +#define ADSTAT_Rx_MF 0x00000400 /* Multicast Frame */ +#define ADSTAT_Rx_FS 0x00000200 /* First Descriptor */ +#define ADSTAT_Rx_LS 0x00000100 /* Last Descriptor */ +#define ADSTAT_Rx_TL 0x00000080 /* Frame Too Long */ +#define ADSTAT_Rx_CS 0x00000040 /* Collision Seen */ +#define ADSTAT_Rx_RT 0x00000020 /* Frame Type */ +#define ADSTAT_Rx_RW 0x00000010 /* Receive Watchdog */ +#define ADSTAT_Rx_RE 0x00000008 /* Report on MII Error */ +#define ADSTAT_Rx_DB 0x00000004 /* Dribbling Bit */ +#define ADSTAT_Rx_CE 0x00000002 /* CRC Error */ +#define ADSTAT_Rx_ZER 0x00000001 /* Zero (always 0) */ + +#define ADSTAT_Rx_LENGTH(x) (((x) & ADSTAT_Rx_FL) >> 16) + +/* + * Descriptor Status bits for Transmit Descriptor. + */ +#define ADSTAT_Tx_ES 0x00008000 /* Error Summary */ +#define ADSTAT_Tx_TO 0x00004000 /* Transmit Jabber Timeout */ +#define ADSTAT_Tx_LO 0x00000800 /* Loss of Carrier */ +#define ADSTAT_Tx_NC 0x00000400 /* No Carrier */ +#define ADSTAT_Tx_LC 0x00000200 /* Late Collision */ +#define ADSTAT_Tx_EC 0x00000100 /* Excessive Collisions */ +#define ADSTAT_Tx_HF 0x00000080 /* Heartbeat Fail */ +#define ADSTAT_Tx_CC 0x00000078 /* Collision Count */ +#define ADSTAT_Tx_ED 0x00000004 /* Excessive Deferral */ +#define ADSTAT_Tx_UF 0x00000002 /* Underflow Error */ +#define ADSTAT_Tx_DE 0x00000001 /* Deferred */ + +#define ADSTAT_Tx_COLLISIONS(x) (((x) & ADSTAT_Tx_CC) >> 3) + +/* + * Descriptor Control bits common to transmit and receive. + */ +#define ADCTL_SIZE1 0x000007ff /* Size of buffer 1 */ +#define ADCTL_SIZE1_SHIFT 0 + +#define ADCTL_SIZE2 0x003ff800 /* Size of buffer 2 */ +#define ADCTL_SIZE2_SHIFT 11 + +#define ADCTL_ER 0x02000000 /* End of Ring */ +#define ADCTL_CH 0x01000000 /* Second Address Chained */ + +/* + * Descriptor Control bits for Transmit Descriptor. + */ +#define ADCTL_Tx_IC 0x80000000 /* Interrupt on Completion */ +#define ADCTL_Tx_LS 0x40000000 /* Last Segment */ +#define ADCTL_Tx_FS 0x20000000 /* First Segment */ +#define ADCTL_Tx_SETUP 0x08000000 /* Setup frame */ +#define ADCTL_Tx_AC 0x04000000 /* Add CRC Disable */ +#define ADCTL_Tx_DPD 0x00800000 /* Disabled Padding */ + +/* + * Control registers. + */ + +/* tese are registers only found on this part */ +#ifdef NOTUSE +#define CSR_MACCTL 0x0000 /* mac control */ +#define CSR_MACHI 0x0004 +#define CSR_MACLO 0x0008 +#define CSR_HTHI 0x000C /* multicast table high */ +#define CSR_HTLO 0x0010 /* multicast table low */ +#define CSR_MIIADDR 0x0014 /* mii address */ +#define CSR_MIIDATA 0x0018 /* mii data */ +#define CSR_FLOWC 0x001C /* flow control */ +#define CSR_VL1 0x0020 /* vlan 1 tag */ +#endif + +/* these are more or less normal Tulip registers */ +#define CSR_BUSMODE (0x08*0) /* bus mode */ +#define CSR_TXPOLL (0x08*1) /* tx poll demand */ +#define CSR_RXPOLL (0x08*2) /* rx poll demand */ +#define CSR_RXLIST (0x08*3) /* rx base descriptor address */ +#define CSR_TXLIST (0x08*4) /* tx base descriptor address */ +#define CSR_STATUS (0x08*5) /* (interrupt) status */ +#define CSR_OPMODE (0x08*6) /* operation mode */ +#define CSR_INTEN (0x08*7) /* interrupt enable */ +#define CSR_MISSED (0x08*8) /* missed frame counter */ + +#ifdef NOTUSE +#define CSR_HTBA 0x1050 /* host tx buffer address (ro) */ +#define CSR_HRBA 0x1054 /* host rx buffer address (ro) */ +#endif + +#define CSR_MIIMNG (0x08*9) /* MII Management Register */ +#define CSR_FULLDUP (0x08*11) /* Full Duplex Register */ + +/* 21143 like register */ +#define FULLDUP_CS 0x80000000 /* Cycle Size */ +#define FULLDUP_TT_SHIFT 27 /* Transmit Timer */ +#define FULLDUP_NTP_SHIFT 24 /* Number of Transmit Packets */ +#define FULLDUP_RT_SHIFT 20 /* Receive Timer */ +#define FULLDUP_NRP_SHIFT 17 /* Number of Receive Packets */ +#define FULLDUP_CON_MODE 0x00010000 /* Continuous Mode */ +#define FULLDUP_TIM_SHIFT 0 /* Timer Value */ + +/* CSR_MACCTL - Mac Control */ +#define MACCTL_RE 0x00000004 /* rx enable */ +#define MACCTL_TE 0x00000008 /* tx enable */ +#define MACCTL_DC 0x00000020 /* deferral check */ +#define MACCTL_PSTR 0x00000100 /* automatic pad strip */ +#define MACCTL_DTRY 0x00000400 /* disable retry */ +#define MACCTL_DBF 0x00000800 /* disable broadcast frames */ +#define MACCTL_LCC 0x00001000 /* late collision control */ +#define MACCTL_HASH 0x00002000 /* hash filtering enable */ +#define MACCTL_HO 0x00008000 /* disable perfect filtering */ +#define MACCTL_PB 0x00010000 /* pass bad frames */ +#define MACCTL_IF 0x00020000 /* inverse filtering */ +#define MACCTL_PR 0x00040000 /* promiscuous mode */ +#define MACCTL_PM 0x00080000 /* pass all multicast */ +#define MACCTL_FDX 0x00100000 /* full duplex mode */ +#define MACCTL_LOOP 0x00600000 /* loopback mask */ +#define MACCTL_LOOP_INT 0x00200000 /* internal loopback */ +#define MACCTL_LOOP_EXT 0x00400000 /* external loopback */ +#define MACCTL_LOOP_NONE 0x00000000 +#define MACCTL_DRO 0x00800000 /* disable receive own */ +#define MACCTL_PS 0x08000000 /* port select, 0 = mii */ +#define MACCTL_HBD 0x10000000 /* heartbeat disable */ +#define MACCTL_BLE 0x40000000 /* mac big endian */ +#define MACCTL_RA 0x80000000 /* receive all packets */ + +/* CSR_MIIADDR - MII Addess */ +#define MIIADDR_BUSY 0x00000001 /* mii busy */ +#define MIIADDR_WRITE 0x00000002 /* mii write */ +#define MIIADDR_REG_MASK 0x000007C0 /* mii register */ +#define MIIADDR_REG_SHIFT 6 +#define MIIADDR_PHY_MASK 0x0000F800 /* mii phy */ +#define MIIADDR_PHY_SHIFT 11 + +#define MIIADDR_GETREG(x) (((x) & MIIADDR_REG) >> 6) +#define MIIADDR_PUTREG(x) (((x) << 6) & MIIADR_REG) +#define MIIADDR_GETPHY(x) (((x) & MIIADDR_PHY) >> 11) +#define MIIADDR_PUTPHY(x) (((x) << 6) & MIIADR_PHY) + +/* CSR_FLOWC - Flow Control */ +#define FLOWC_FCB 0x00000001 /* flow control busy */ +#define FLOWC_FCE 0x00000002 /* flow control enable */ +#define FLOWC_PCF 0x00000004 /* pass control frames */ +#define FLOWC_PT 0xffff0000 /* pause time */ + +/* CSR_BUSMODE - Bus Mode */ +#define BUSMODE_SWR 0x00000001 /* software reset */ +#define BUSMODE_BAR 0x00000002 /* bus arbitration */ +#define BUSMODE_DSL 0x0000007c /* descriptor skip length */ +#define BUSMODE_BLE 0x00000080 /* data buf endian */ + /* programmable burst length */ +#define BUSMODE_PBL_DEFAULT 0x00000000 /* default value */ +#define BUSMODE_PBL_1LW 0x00000100 /* 1 longword */ +#define BUSMODE_PBL_2LW 0x00000200 /* 2 longwords */ +#define BUSMODE_PBL_4LW 0x00000400 /* 4 longwords */ +#define BUSMODE_PBL_8LW 0x00000800 /* 8 longwords */ +#define BUSMODE_PBL_16LW 0x00001000 /* 16 longwords */ +#define BUSMODE_PBL_32LW 0x00002000 /* 32 longwords */ +#define BUSMODE_TAP_SHIFT 17 /* Transmit Automatic Polling */ +#define BUSMODE_DBO 0x00100000 /* descriptor endian */ +#define BUSMODE_ALIGN_16B 0x01000000 /* force oddhw rx buf align */ + +/* CSR_TXPOLL - Transmit Poll Demand */ +#define TXPOLL_TPD 0x00000001 /* transmit poll demand */ + + +/* CSR_RXPOLL - Receive Poll Demand */ +#define RXPOLL_RPD 0x00000001 /* receive poll demand */ + +/* CSR_STATUS - Status */ +#define STATUS_TI 0x00000001 /* transmit interrupt */ +#define STATUS_TPS 0x00000002 /* transmit process stopped */ +#define STATUS_TU 0x00000004 /* transmit buffer unavail */ +#define STATUS_TJT 0x00000008 /* transmit jabber timeout */ +#define STATUS_UNF 0x00000020 /* transmit underflow */ +#define STATUS_RI 0x00000040 /* receive interrupt */ +#define STATUS_RU 0x00000080 /* receive buffer unavail */ +#define STATUS_RPS 0x00000100 /* receive process stopped */ +#define STATUS_ETI 0x00000400 /* early transmit interrupt */ +#define STATUS_SE 0x00002000 /* system error */ +#define STATUS_ER 0x00004000 /* early receive (21041) */ +#define STATUS_AIS 0x00008000 /* abnormal intr summary */ +#define STATUS_NIS 0x00010000 /* normal interrupt summary */ +#define STATUS_RS 0x000e0000 /* receive process state */ +#define STATUS_RS_STOPPED 0x00000000 /* Stopped */ +#define STATUS_RS_FETCH 0x00020000 /* Running - fetch receive + descriptor */ +#define STATUS_RS_CHECK 0x00040000 /* Running - check for end + of receive */ +#define STATUS_RS_WAIT 0x00060000 /* Running - wait for packet */ +#define STATUS_RS_SUSPENDED 0x00080000 /* Suspended */ +#define STATUS_RS_CLOSE 0x000a0000 /* Running - close receive + descriptor */ +#define STATUS_RS_FLUSH 0x000c0000 /* Running - flush current + frame from FIFO */ +#define STATUS_RS_QUEUE 0x000e0000 /* Running - queue current + frame from FIFO into + buffer */ +#define STATUS_TS 0x00700000 /* transmit process state */ +#define STATUS_TS_STOPPED 0x00000000 /* Stopped */ +#define STATUS_TS_FETCH 0x00100000 /* Running - fetch transmit + descriptor */ +#define STATUS_TS_WAIT 0x00200000 /* Running - wait for end + of transmission */ +#define STATUS_TS_READING 0x00300000 /* Running - read buffer from + memory and queue into + FIFO */ +#define STATUS_TS_SUSPENDED 0x00600000 /* Suspended */ +#define STATUS_TS_CLOSE 0x00700000 /* Running - close transmit + descriptor */ +#define STATUS_TX_ABORT 0x00800000 /* Transmit bus abort */ +#define STATUS_RX_ABORT 0x01000000 /* Transmit bus abort */ + +/* CSR_OPMODE - Operation Mode */ +#define OPMODE_SR 0x00000002 /* start receive */ +#define OPMODE_OSF 0x00000004 /* operate on second frame */ +#define OPMODE_PR 0x00000040 /* promiscuous mode */ +#define OPMODE_PM 0x00000080 /* pass all multicast */ +#define OPMODE_FDX 0x00000200 /* full duplex mode */ +#define OPMODE_ST 0x00002000 /* start transmitter */ +#define OPMODE_TR 0x0000c000 /* threshold control */ +#define OPMODE_TR_32 0x00000000 /* 32 words */ +#define OPMODE_TR_64 0x00004000 /* 64 words */ +#define OPMODE_TR_128 0x00008000 /* 128 words */ +#define OPMODE_TR_256 0x0000c000 /* 256 words */ +#define OPMODE_SF 0x00200000 /* store and forward mode */ +#define OPMODE_SPEED 0x80000000 /* speed 100M:1 10M:0 */ + +/* CSR_INTEN - Interrupt Enable */ + /* See bits for CSR_STATUS -- Status */ + + +/* CSR_MISSED - Missed Frames */ +#define MISSED_MFC 0xffff0000 /* missed packet count */ +#define MISSED_FOC 0x0000ffff /* fifo overflow counter */ + +#define MISSED_GETMFC(x) ((x) & MISSED_MFC) +#define MISSED_GETFOC(x) (((x) & MISSED_FOC) >> 16) + +/* setup frame code refer dc code */ + +#define FV_SFRAME_LEN 192 +#define FV_MIN_FRAMELEN 60 + +/* + * MII Definitions for the 21041 and 21140/21140A/21142 + * copy from if_devar.h + */ +#define MII_PREAMBLE (~0) +#define MII_TEST 0xAAAAAAAA +#define MII_RDCMD 0x06 +#define MII_WRCMD 0x05 +#define MII_DIN 0x00080000 +#define MII_RD 0x00040000 +#define MII_WR 0x00000000 +#define MII_DOUT 0x00020000 +#define MII_CLK 0x00010000 +#define MII_CLKON MII_CLK +#define MII_CLKOFF MII_CLK + +#endif /* __IF_FVREG_H__ */ diff --git a/sys/arm/ralink/rt1310_gpio.c b/sys/arm/ralink/rt1310_gpio.c new file mode 100644 index 0000000000000..774c7c4696f65 --- /dev/null +++ b/sys/arm/ralink/rt1310_gpio.c @@ -0,0 +1,480 @@ +/*- + * Copyright (c) 2011 Jakub Wojciech Klama <jceel@FreeBSD.org> + * Copyright (c) 2015 Hiroki Mori + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/* + * GPIO on RT1310A consist of 2 ports: + * - PortA with 8 input/output pins + * - PortB with 4 input/output pins + * + * Pins are mapped to logical pin number as follows: + * [0..7] -> GPI_00..GPI_07 (port A) + * [8..11] -> GPI_08..GPI_11 (port B) + * + */ + + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bio.h> +#include <sys/bus.h> +#include <sys/conf.h> +#include <sys/endian.h> +#include <sys/kernel.h> +#include <sys/kthread.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/mutex.h> +#include <sys/queue.h> +#include <sys/resource.h> +#include <sys/rman.h> +#include <sys/time.h> +#include <sys/timetc.h> +#include <sys/watchdog.h> +#include <sys/gpio.h> + +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/cpufunc.h> +#include <machine/resource.h> +#include <machine/intr.h> +#include <machine/fdt.h> + +#include <dev/gpio/gpiobusvar.h> +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> + +#include <arm/ralink/rt1310reg.h> +#include <arm/ralink/rt1310var.h> + +#include "gpio_if.h" + +struct rt1310_gpio_softc +{ + device_t lg_dev; + device_t lg_busdev; + struct resource * lg_res; + bus_space_tag_t lg_bst; + bus_space_handle_t lg_bsh; +}; + +struct rt1310_gpio_pinmap +{ + int lp_start_idx; + int lp_pin_count; + int lp_port; + int lp_start_bit; + int lp_flags; +}; + +static const struct rt1310_gpio_pinmap rt1310_gpio_pins[] = { + { 0, 8, RT_GPIO_PORTA, 0, GPIO_PIN_INPUT | GPIO_PIN_OUTPUT }, + { 8, 4, RT_GPIO_PORTB, 0, GPIO_PIN_INPUT | GPIO_PIN_OUTPUT }, + { -1, -1, -1, -1, -1 }, +}; + +#define RT_GPIO_NPINS 12 + +#define RT_GPIO_PIN_IDX(_map, _idx) \ + (_idx - _map->lp_start_idx) + +#define RT_GPIO_PIN_BIT(_map, _idx) \ + (_map->lp_start_bit + RT_GPIO_PIN_IDX(_map, _idx)) + +static int rt1310_gpio_probe(device_t); +static int rt1310_gpio_attach(device_t); +static int rt1310_gpio_detach(device_t); + +static device_t rt1310_gpio_get_bus(device_t); +static int rt1310_gpio_pin_max(device_t, int *); +static int rt1310_gpio_pin_getcaps(device_t, uint32_t, uint32_t *); +static int rt1310_gpio_pin_getflags(device_t, uint32_t, uint32_t *); +static int rt1310_gpio_pin_setflags(device_t, uint32_t, uint32_t); +static int rt1310_gpio_pin_getname(device_t, uint32_t, char *); +static int rt1310_gpio_pin_get(device_t, uint32_t, uint32_t *); +static int rt1310_gpio_pin_set(device_t, uint32_t, uint32_t); +static int rt1310_gpio_pin_toggle(device_t, uint32_t); + +static const struct rt1310_gpio_pinmap *rt1310_gpio_get_pinmap(int); + +static struct rt1310_gpio_softc *rt1310_gpio_sc = NULL; + +#define rt1310_gpio_read_4(_sc, _reg) \ + bus_space_read_4(_sc->lg_bst, _sc->lg_bsh, _reg) +#define rt1310_gpio_write_4(_sc, _reg, _val) \ + bus_space_write_4(_sc->lg_bst, _sc->lg_bsh, _reg, _val) + +static int +rt1310_gpio_probe(device_t dev) +{ + phandle_t node; + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "ralink,rt1310-gpio")) + return (ENXIO); + + node = ofw_bus_get_node(dev); + if (!OF_hasprop(node, "gpio-controller")) + return (ENXIO); + + device_set_desc(dev, "RT1310 GPIO"); + return (BUS_PROBE_DEFAULT); +} + +static int +rt1310_gpio_attach(device_t dev) +{ + struct rt1310_gpio_softc *sc = device_get_softc(dev); + int rid; + + sc->lg_dev = dev; + + rid = 0; + sc->lg_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (!sc->lg_res) { + device_printf(dev, "cannot allocate memory window\n"); + return (ENXIO); + } + + sc->lg_bst = rman_get_bustag(sc->lg_res); + sc->lg_bsh = rman_get_bushandle(sc->lg_res); + + rt1310_gpio_sc = sc; + + sc->lg_busdev = gpiobus_attach_bus(dev); + if (sc->lg_busdev == NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->lg_res); + return (ENXIO); + } + + return (0); +} + +static int +rt1310_gpio_detach(device_t dev) +{ + return (EBUSY); +} + +static device_t +rt1310_gpio_get_bus(device_t dev) +{ + struct rt1310_gpio_softc *sc; + + sc = device_get_softc(dev); + + return (sc->lg_busdev); +} + +static int +rt1310_gpio_pin_max(device_t dev, int *npins) +{ + *npins = RT_GPIO_NPINS - 1; + return (0); +} + +static int +rt1310_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) +{ + const struct rt1310_gpio_pinmap *map; + + if (pin > RT_GPIO_NPINS) + return (ENODEV); + + map = rt1310_gpio_get_pinmap(pin); + + *caps = map->lp_flags; + return (0); +} + +static int +rt1310_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) +{ + struct rt1310_gpio_softc *sc = device_get_softc(dev); + const struct rt1310_gpio_pinmap *map; + uint32_t state; + int dir; + + if (pin > RT_GPIO_NPINS) + return (ENODEV); + + map = rt1310_gpio_get_pinmap(pin); + + /* Check whether it's bidirectional pin */ + if ((map->lp_flags & (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) != + (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) { + *flags = map->lp_flags; + return (0); + } + + switch (map->lp_port) { + case RT_GPIO_PORTA: + state = rt1310_gpio_read_4(sc, RT_GPIO_OFF_PADIR); + dir = (state & (1 << RT_GPIO_PIN_BIT(map, pin))); + break; + case RT_GPIO_PORTB: + state = rt1310_gpio_read_4(sc, RT_GPIO_OFF_PBDIR); + dir = (state & (1 << RT_GPIO_PIN_BIT(map, pin))); + break; + default: + panic("unknown GPIO port"); + } + + *flags = dir ? GPIO_PIN_OUTPUT : GPIO_PIN_INPUT; + + return (0); +} + +static int +rt1310_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) +{ + struct rt1310_gpio_softc *sc = device_get_softc(dev); + const struct rt1310_gpio_pinmap *map; + uint32_t dir, state; + uint32_t port; + + if (pin > RT_GPIO_NPINS) + return (ENODEV); + + map = rt1310_gpio_get_pinmap(pin); + + /* Check whether it's bidirectional pin */ + if ((map->lp_flags & (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) != + (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) + return (ENOTSUP); + + if (flags & GPIO_PIN_INPUT) + dir = 0; + + if (flags & GPIO_PIN_OUTPUT) + dir = 1; + + switch (map->lp_port) { + case RT_GPIO_PORTA: + port = RT_GPIO_OFF_PADIR; + break; + case RT_GPIO_PORTB: + port = RT_GPIO_OFF_PBDIR; + break; + } + + state = rt1310_gpio_read_4(sc, port); + if (flags & GPIO_PIN_INPUT) { + state &= ~(1 << RT_GPIO_PIN_IDX(map, pin)); + } else { + state |= (1 << RT_GPIO_PIN_IDX(map, pin)); + } + rt1310_gpio_write_4(sc, port, state); + + return (0); +} + +static int +rt1310_gpio_pin_getname(device_t dev, uint32_t pin, char *name) +{ + snprintf(name, GPIOMAXNAME - 1, "GPIO_%02d", pin); + + return (0); +} + +static int +rt1310_gpio_pin_get(device_t dev, uint32_t pin, uint32_t *value) +{ + struct rt1310_gpio_softc *sc = device_get_softc(dev); + const struct rt1310_gpio_pinmap *map; + uint32_t state, flags; + int dir; + + map = rt1310_gpio_get_pinmap(pin); + + if (rt1310_gpio_pin_getflags(dev, pin, &flags)) + return (ENXIO); + + if (flags & GPIO_PIN_OUTPUT) + dir = 1; + + if (flags & GPIO_PIN_INPUT) + dir = 0; + + switch (map->lp_port) { + case RT_GPIO_PORTA: + state = rt1310_gpio_read_4(sc, RT_GPIO_OFF_PADR); + *value = !!(state & (1 << RT_GPIO_PIN_BIT(map, pin))); + break; + case RT_GPIO_PORTB: + state = rt1310_gpio_read_4(sc, RT_GPIO_OFF_PBDR); + *value = !!(state & (1 << RT_GPIO_PIN_BIT(map, pin))); + break; + } + + return (0); +} + +static int +rt1310_gpio_pin_set(device_t dev, uint32_t pin, uint32_t value) +{ + struct rt1310_gpio_softc *sc = device_get_softc(dev); + const struct rt1310_gpio_pinmap *map; + uint32_t state, flags; + uint32_t port; + + map = rt1310_gpio_get_pinmap(pin); + + if (rt1310_gpio_pin_getflags(dev, pin, &flags)) + return (ENXIO); + + if ((flags & GPIO_PIN_OUTPUT) == 0) + return (EINVAL); + + switch (map->lp_port) { + case RT_GPIO_PORTA: + port = RT_GPIO_OFF_PADR; + break; + case RT_GPIO_PORTB: + port = RT_GPIO_OFF_PBDR; + break; + } + + state = rt1310_gpio_read_4(sc, port); + if (value == 1) { + state |= (1 << RT_GPIO_PIN_BIT(map, pin)); + } else { + state &= ~(1 << RT_GPIO_PIN_BIT(map, pin)); + } + rt1310_gpio_write_4(sc, port, state); + + return (0); +} + +static int +rt1310_gpio_pin_toggle(device_t dev, uint32_t pin) +{ + const struct rt1310_gpio_pinmap *map; + uint32_t flags; + + map = rt1310_gpio_get_pinmap(pin); + + if (rt1310_gpio_pin_getflags(dev, pin, &flags)) + return (ENXIO); + + if ((flags & GPIO_PIN_OUTPUT) == 0) + return (EINVAL); + + panic("not implemented yet"); + + return (0); + +} + +static const struct rt1310_gpio_pinmap * +rt1310_gpio_get_pinmap(int pin) +{ + const struct rt1310_gpio_pinmap *map; + + for (map = &rt1310_gpio_pins[0]; map->lp_start_idx != -1; map++) { + if (pin >= map->lp_start_idx && + pin < map->lp_start_idx + map->lp_pin_count) + return map; + } + + panic("pin number %d out of range", pin); +} + +int +rt1310_gpio_set_flags(device_t dev, int pin, int flags) +{ + if (rt1310_gpio_sc == NULL) + return (ENXIO); + + return rt1310_gpio_pin_setflags(rt1310_gpio_sc->lg_dev, pin, flags); +} + +int +rt1310_gpio_set_state(device_t dev, int pin, int state) +{ + if (rt1310_gpio_sc == NULL) + return (ENXIO); + + return rt1310_gpio_pin_set(rt1310_gpio_sc->lg_dev, pin, state); +} + +int +rt1310_gpio_get_state(device_t dev, int pin, int *state) +{ + if (rt1310_gpio_sc == NULL) + return (ENXIO); + + return rt1310_gpio_pin_get(rt1310_gpio_sc->lg_dev, pin, state); +} + +static phandle_t +rt1310_gpio_get_node(device_t bus, device_t dev) +{ + /* We only have one child, the GPIO bus, which needs our own node. */ + return (ofw_bus_get_node(bus)); +} + +static device_method_t rt1310_gpio_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, rt1310_gpio_probe), + DEVMETHOD(device_attach, rt1310_gpio_attach), + DEVMETHOD(device_detach, rt1310_gpio_detach), + + /* GPIO interface */ + DEVMETHOD(gpio_get_bus, rt1310_gpio_get_bus), + DEVMETHOD(gpio_pin_max, rt1310_gpio_pin_max), + DEVMETHOD(gpio_pin_getcaps, rt1310_gpio_pin_getcaps), + DEVMETHOD(gpio_pin_getflags, rt1310_gpio_pin_getflags), + DEVMETHOD(gpio_pin_setflags, rt1310_gpio_pin_setflags), + DEVMETHOD(gpio_pin_getname, rt1310_gpio_pin_getname), + DEVMETHOD(gpio_pin_set, rt1310_gpio_pin_set), + DEVMETHOD(gpio_pin_get, rt1310_gpio_pin_get), + DEVMETHOD(gpio_pin_toggle, rt1310_gpio_pin_toggle), + + /* ofw_bus interface */ + DEVMETHOD(ofw_bus_get_node, rt1310_gpio_get_node), + + { 0, 0 } +}; + +static devclass_t rt1310_gpio_devclass; + +static driver_t rt1310_gpio_driver = { + "gpio", + rt1310_gpio_methods, + sizeof(struct rt1310_gpio_softc), +}; + +DRIVER_MODULE(rt1310gpio, simplebus, rt1310_gpio_driver, rt1310_gpio_devclass, 0, 0); +MODULE_VERSION(rt1310gpio, 1); diff --git a/sys/arm/ralink/rt1310_intc.c b/sys/arm/ralink/rt1310_intc.c new file mode 100644 index 0000000000000..05c8bdc18f8c5 --- /dev/null +++ b/sys/arm/ralink/rt1310_intc.c @@ -0,0 +1,441 @@ +/*- + * Copyright (c) 2010 Jakub Wojciech Klama <jceel@FreeBSD.org> + * Copyright (c) 2015 Hiroki Mori + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_platform.h" + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/types.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/proc.h> +#include <sys/rman.h> +#include <vm/vm.h> +#include <vm/vm_kern.h> +#include <vm/pmap.h> +#include <vm/vm_page.h> +#include <vm/vm_extern.h> + +#define _ARM32_BUS_DMA_PRIVATE +#include <machine/bus.h> +#include <machine/intr.h> + +#include <dev/fdt/fdt_common.h> +#include <dev/ofw/openfirm.h> + +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> + +#include <arm/ralink/rt1310reg.h> + +#define INTC_NIRQS 32 + +#ifdef INTRNG +#include "pic_if.h" + +struct rt1310_irqsrc { + struct intr_irqsrc ri_isrc; + u_int ri_irq; +}; +#endif + +struct rt1310_intc_softc { + device_t dev; + struct resource * ri_res; + bus_space_tag_t ri_bst; + bus_space_handle_t ri_bsh; +#ifdef INTRNG + struct rt1310_irqsrc ri_isrcs[INTC_NIRQS]; +#endif +}; + +static int rt1310_intc_probe(device_t); +static int rt1310_intc_attach(device_t); +#ifndef INTRNG +static void rt1310_intc_eoi(void *); +#else +static int rt1310_pic_attach(struct rt1310_intc_softc *sc); +#endif + +static struct rt1310_intc_softc *intc_softc = NULL; + +#define intc_read_4(_sc, _reg) \ + bus_space_read_4((_sc)->ri_bst, (_sc)->ri_bsh, (_reg)) +#define intc_write_4(_sc, _reg, _val) \ + bus_space_write_4((_sc)->ri_bst, (_sc)->ri_bsh, (_reg), (_val)) + +struct rt1310_irqdef { + u_int ri_trig; + u_int ri_prio; +}; + +struct rt1310_irqdef irqdef[INTC_NIRQS] = { + {RT_INTC_TRIG_HIGH_LVL, 2}, /* 0 */ + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_HIGH_LVL, 1}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_HIGH_LVL, 1}, + {RT_INTC_TRIG_HIGH_LVL, 1}, + {RT_INTC_TRIG_HIGH_LVL, 1}, + {RT_INTC_TRIG_HIGH_LVL, 1}, /* 8 */ + {RT_INTC_TRIG_HIGH_LVL, 1}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 4}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_HIGH_LVL, 2}, /* 16 */ + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 2}, + {RT_INTC_TRIG_LOW_LVL, 2}, + {RT_INTC_TRIG_NEG_EDGE, 2}, + {RT_INTC_TRIG_HIGH_LVL, 3}, + {RT_INTC_TRIG_HIGH_LVL, 2}, /* 24 */ + {RT_INTC_TRIG_POS_EDGE, 2}, + {RT_INTC_TRIG_POS_EDGE, 2}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_HIGH_LVL, 2}, + {RT_INTC_TRIG_POS_EDGE, 2}, + {RT_INTC_TRIG_POS_EDGE, 3}, + {RT_INTC_TRIG_POS_EDGE, 3}, +}; + +static int +rt1310_intc_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "rt,pic")) + return (ENXIO); + +#ifdef INTRNG + device_set_desc(dev, "RT1310 INTRNG Interrupt Controller"); +#else + device_set_desc(dev, "RT1310 Interrupt Controller"); +#endif + return (BUS_PROBE_DEFAULT); +} + +static int +rt1310_intc_attach(device_t dev) +{ + struct rt1310_intc_softc *sc = device_get_softc(dev); + int rid = 0; + int i; + + if (intc_softc) + return (ENXIO); + + sc->dev = dev; + + sc->ri_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (!sc->ri_res) { + device_printf(dev, "could not alloc resources\n"); + return (ENXIO); + } + + sc->ri_bst = rman_get_bustag(sc->ri_res); + sc->ri_bsh = rman_get_bushandle(sc->ri_res); + intc_softc = sc; +#ifndef INTRNG + arm_post_filter = rt1310_intc_eoi; +#else + rt1310_pic_attach(sc); +#endif + + intc_write_4(sc, RT_INTC_IECR, 0); + intc_write_4(sc, RT_INTC_ICCR, ~0); + + for (i = 0; i <= INTC_NIRQS; ++i) { + intc_write_4(sc, RT_INTC_SCR0+i*4, + (irqdef[i].ri_trig << RT_INTC_TRIG_SHIF) | + irqdef[i].ri_prio); + intc_write_4(sc, RT_INTC_SVR0+i*4, i); + } + + /* Clear interrupt status registers and disable all interrupts */ + intc_write_4(sc, RT_INTC_ICCR, ~0); + intc_write_4(sc, RT_INTC_IMR, 0); + return (0); +} + +#ifndef INTRNG +int +arm_get_next_irq(int last) +{ + struct rt1310_intc_softc *sc = intc_softc; + uint32_t value; + int i; + value = intc_read_4(sc, RT_INTC_IPR); + for (i = 0; i < 32; i++) { + if (value & (1 << i)) + return (i); + } + + return (-1); +} + +void +arm_mask_irq(uintptr_t nb) +{ + struct rt1310_intc_softc *sc = intc_softc; + uint32_t value; + + /* Make sure that interrupt isn't active already */ + rt1310_intc_eoi((void *)nb); + + /* Clear bit in ER register */ + value = intc_read_4(sc, RT_INTC_IECR); + value &= ~(1 << nb); + intc_write_4(sc, RT_INTC_IECR, value); + intc_write_4(sc, RT_INTC_IMR, value); + + intc_write_4(sc, RT_INTC_ICCR, 1 << nb); +} + +void +arm_unmask_irq(uintptr_t nb) +{ + struct rt1310_intc_softc *sc = intc_softc; + uint32_t value; + + value = intc_read_4(sc, RT_INTC_IECR); + + value |= (1 << nb); + + intc_write_4(sc, RT_INTC_IMR, value); + intc_write_4(sc, RT_INTC_IECR, value); +} + +static void +rt1310_intc_eoi(void *data) +{ + struct rt1310_intc_softc *sc = intc_softc; + int nb = (int)data; + + intc_write_4(sc, RT_INTC_ICCR, 1 << nb); + if (nb == 0) { + uint32_t value; + value = intc_read_4(sc, RT_INTC_IECR); + value &= ~(1 << nb); + intc_write_4(sc, RT_INTC_IECR, value); + intc_write_4(sc, RT_INTC_IMR, value); + } +} + +#else + +static void +rt1310_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + unsigned int value; + struct rt1310_intc_softc *sc; + + sc = intc_softc; + irq = ((struct rt1310_irqsrc *)isrc)->ri_irq; + + value = intc_read_4(sc, RT_INTC_IECR); + + value |= (1 << irq); + + intc_write_4(sc, RT_INTC_IMR, value); + intc_write_4(sc, RT_INTC_IECR, value); +} + +static void +rt1310_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + unsigned int value; + struct rt1310_intc_softc *sc; + + sc = intc_softc; + irq = ((struct rt1310_irqsrc *)isrc)->ri_irq; + + /* Clear bit in ER register */ + value = intc_read_4(sc, RT_INTC_IECR); + value &= ~(1 << irq); + intc_write_4(sc, RT_INTC_IECR, value); + intc_write_4(sc, RT_INTC_IMR, value); + + intc_write_4(sc, RT_INTC_ICCR, 1 << irq); +} + +static int +rt1310_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct intr_map_data_fdt *daf; + struct rt1310_intc_softc *sc; + + if (data->type != INTR_MAP_DATA_FDT) + return (ENOTSUP); + + daf = (struct intr_map_data_fdt *)data; + + if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS) + return (EINVAL); + + sc = device_get_softc(dev); + *isrcp = &sc->ri_isrcs[daf->cells[0]].ri_isrc; + return (0); +} + +static void +rt1310_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + arm_irq_memory_barrier(0); + rt1310_disable_intr(dev, isrc); +} + +static void +rt1310_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + arm_irq_memory_barrier(0); + rt1310_enable_intr(dev, isrc); +} + +static void +rt1310_post_filter(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + struct rt1310_intc_softc *sc; + + arm_irq_memory_barrier(0); + sc = intc_softc; + irq = ((struct rt1310_irqsrc *)isrc)->ri_irq; + + intc_write_4(sc, RT_INTC_ICCR, 1 << irq); +} + +static int +rt1310_intr(void *arg) +{ + uint32_t irq; + struct rt1310_intc_softc *sc = arg; + + irq = ffs(intc_read_4(sc, RT_INTC_IPR)) - 1; + + if (intr_isrc_dispatch(&sc->ri_isrcs[irq].ri_isrc, + curthread->td_intr_frame) != 0) { + intc_write_4(sc, RT_INTC_ICCR, 1 << irq); + device_printf(sc->dev, "Stray irq %u disabled\n", irq); + } + + arm_irq_memory_barrier(0); + + return (FILTER_HANDLED); +} + +static int +rt1310_pic_attach(struct rt1310_intc_softc *sc) +{ + struct intr_pic *pic; + int error; + uint32_t irq; + const char *name; + intptr_t xref; + + name = device_get_nameunit(sc->dev); + for (irq = 0; irq < INTC_NIRQS; irq++) { + sc->ri_isrcs[irq].ri_irq = irq; + + error = intr_isrc_register(&sc->ri_isrcs[irq].ri_isrc, + sc->dev, 0, "%s,%u", name, irq); + if (error != 0) + return (error); + } + + xref = OF_xref_from_node(ofw_bus_get_node(sc->dev)); + pic = intr_pic_register(sc->dev, xref); + if (pic == NULL) + return (ENXIO); + + return (intr_pic_claim_root(sc->dev, xref, rt1310_intr, sc, 0)); +} +#endif + +struct fdt_fixup_entry fdt_fixup_table[] = { + { NULL, NULL } +}; + +#ifndef INTRNG +static int +fdt_pic_decode_ic(phandle_t node, pcell_t *intr, int *interrupt, int *trig, + int *pol) +{ + if (!fdt_is_compatible(node, "lpc,pic")) + return (ENXIO); + + *interrupt = fdt32_to_cpu(intr[0]); + *trig = INTR_TRIGGER_CONFORM; + *pol = INTR_POLARITY_CONFORM; + return (0); +} + +fdt_pic_decode_t fdt_pic_table[] = { + &fdt_pic_decode_ic, + NULL +}; +#endif + +static device_method_t rt1310_intc_methods[] = { + DEVMETHOD(device_probe, rt1310_intc_probe), + DEVMETHOD(device_attach, rt1310_intc_attach), +#ifdef INTRNG + DEVMETHOD(pic_disable_intr, rt1310_disable_intr), + DEVMETHOD(pic_enable_intr, rt1310_enable_intr), + DEVMETHOD(pic_map_intr, rt1310_map_intr), + DEVMETHOD(pic_post_filter, rt1310_post_filter), + DEVMETHOD(pic_post_ithread, rt1310_post_ithread), + DEVMETHOD(pic_pre_ithread, rt1310_pre_ithread), +#endif + { 0, 0 } +}; + +static driver_t rt1310_intc_driver = { + "pic", + rt1310_intc_methods, + sizeof(struct rt1310_intc_softc), +}; + +static devclass_t rt1310_intc_devclass; + +EARLY_DRIVER_MODULE(pic, simplebus, rt1310_intc_driver, rt1310_intc_devclass, 0, 0, BUS_PASS_INTERRUPT); diff --git a/sys/arm/ralink/rt1310_machdep.c b/sys/arm/ralink/rt1310_machdep.c new file mode 100644 index 0000000000000..2cd0322b96e68 --- /dev/null +++ b/sys/arm/ralink/rt1310_machdep.c @@ -0,0 +1,176 @@ +/*- + * Copyright (c) 1994-1998 Mark Brinicombe. + * Copyright (c) 1994 Brini. + * All rights reserved. + * + * This code is derived from software written for Brini by Mark Brinicombe + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Brini. + * 4. The name of the company nor the name of the author may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: FreeBSD: sys/arm/lpc/lpc_machdep.c + */ + +#include "opt_ddb.h" +#include "opt_platform.h" + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#define _ARM32_BUS_DMA_PRIVATE +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/reboot.h> +#include <sys/devmap.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include <machine/bus.h> +#include <machine/fdt.h> +#include <machine/machdep.h> +#include <machine/platform.h> +#include <machine/cpu.h> + +#include <arm/ralink/rt1310reg.h> +#include <arm/ralink/rt1310var.h> + +#include <dev/fdt/fdt_common.h> + +#ifdef EARLY_PRINTF +early_putc_t *early_putc; +#endif + + +uint32_t rt1310_master_clock; + +vm_offset_t +platform_lastaddr(void) +{ + + return (devmap_lastaddr()); +} + +void +platform_probe_and_attach(void) +{ +} + +void +platform_gpio_init(void) +{ + + /* + * Set initial values of GPIO output ports + */ +} + +void +platform_late_init(void) +{ + bootverbose = 1; +} + +/* + * Add a single static device mapping. + * The values used were taken from the ranges property of the SoC node in the + * dts file when this code was converted to arm_devmap_add_entry(). + */ +int +platform_devmap_init(void) +{ + devmap_add_entry(0x19C00000, 0xE0000); + devmap_add_entry(0x1e800000, 0x800000); + devmap_add_entry(0x1f000000, 0x400000); + return (0); +} + +struct arm32_dma_range * +bus_dma_get_range(void) +{ + + return (NULL); +} + +int +bus_dma_get_range_nb(void) +{ + + return (0); +} + +void +cpu_reset(void) +{ + bus_space_tag_t bst; + bus_space_handle_t bsh; + + bst = fdtbus_bs_tag; + + /* Enable WDT */ + /* Instant assert of RESETOUT_N with pulse length 1ms */ + bus_space_map(bst, 0x1e8c0000, 0x20000, 0, &bsh); + bus_space_write_4(bst, bsh, 0, 13000); + bus_space_write_4(bst, bsh, 8, (1<<3) | (1<<4) | 7); + bus_space_unmap(bst, bsh, 0x20000); + + for (;;) + continue; +} + +#ifdef RALINK_BOOT_DEBUG +void bootdebug1(int c); +void bootdebug1(int c) +{ + /* direct put uart physical address */ + uint8_t* uart_base_addr=(uint8_t*)0x1e840000; + *(uart_base_addr) = c; +} + +void bootdebug2(int c); +void bootdebug2(int c) +{ +#if defined(SOCDEV_PA) && defined(SOCDEV_VA) + /* direct put uart map address at locore-v4.S */ + uint8_t* uart_base_addr=(uint8_t*)0xce840000; + *(uart_base_addr) = c; +#endif +} + +void bootdebug3(int c); +void bootdebug3(int c) +{ + bus_space_tag_t bst; + bus_space_handle_t bsh; + + bst = fdtbus_bs_tag; + bus_space_map(bst, 0x1e840000, 0x20000, 0, &bsh); + bus_space_write_1(bst, bsh, 0, c); + bus_space_unmap(bst, bsh, 0x20000); +} +#endif diff --git a/sys/arm/ralink/rt1310_timer.c b/sys/arm/ralink/rt1310_timer.c new file mode 100644 index 0000000000000..6834bbdc388dc --- /dev/null +++ b/sys/arm/ralink/rt1310_timer.c @@ -0,0 +1,341 @@ +/*- + * Copyright (c) 2011 Jakub Wojciech Klama <jceel@FreeBSD.org> + * Copyright (c) 2015 Hiroki Mori + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/malloc.h> +#include <sys/rman.h> +#include <sys/timetc.h> +#include <sys/timeet.h> +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/intr.h> + +#include <dev/fdt/fdt_common.h> +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> + +#include <arm/ralink/rt1310reg.h> +#include <arm/ralink/rt1310var.h> + +struct rt1310_timer_softc { + device_t lt_dev; + struct eventtimer lt_et; + struct resource * lt_res[8]; + bus_space_tag_t lt_bst0; + bus_space_handle_t lt_bsh0; + bus_space_tag_t lt_bst1; + bus_space_handle_t lt_bsh1; + bus_space_tag_t lt_bst2; + bus_space_handle_t lt_bsh2; + bus_space_tag_t lt_bst3; + bus_space_handle_t lt_bsh3; + int lt_oneshot; + uint32_t lt_period; +}; + +static struct resource_spec rt1310_timer_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { SYS_RES_MEMORY, 1, RF_ACTIVE }, + { SYS_RES_MEMORY, 2, RF_ACTIVE }, + { SYS_RES_MEMORY, 3, RF_ACTIVE }, + { SYS_RES_IRQ, 0, RF_ACTIVE }, + { SYS_RES_IRQ, 1, RF_ACTIVE }, + { SYS_RES_IRQ, 2, RF_ACTIVE }, + { -1, 0 } +}; + +static struct rt1310_timer_softc *timer_softc = NULL; +static int rt1310_timer_initialized = 0; +static int rt1310_timer_probe(device_t); +static int rt1310_timer_attach(device_t); +static int rt1310_timer_start(struct eventtimer *, + sbintime_t first, sbintime_t period); +static int rt1310_timer_stop(struct eventtimer *et); +static unsigned rt1310_get_timecount(struct timecounter *); +static int rt1310_hardclock(void *); + +#define timer0_read_4(sc, reg) \ + bus_space_read_4(sc->lt_bst0, sc->lt_bsh0, reg) +#define timer0_write_4(sc, reg, val) \ + bus_space_write_4(sc->lt_bst0, sc->lt_bsh0, reg, val) +#define timer0_clear(sc) \ + do { \ + timer0_write_4(sc, RT_TIMER_LOAD, 0); \ + timer0_write_4(sc, RT_TIMER_VALUE, 0); \ + } while(0) + +#define timer1_read_4(sc, reg) \ + bus_space_read_4(sc->lt_bst1, sc->lt_bsh1, reg) +#define timer1_write_4(sc, reg, val) \ + bus_space_write_4(sc->lt_bst1, sc->lt_bsh1, reg, val) +#define timer1_clear(sc) \ + do { \ + timer1_write_4(sc, RT_TIMER_LOAD, 0); \ + timer1_write_4(sc, RT_TIMER_VALUE, 0); \ + } while(0) + +#define timer2_read_4(sc, reg) \ + bus_space_read_4(sc->lt_bst1, sc->lt_bsh2, reg) +#define timer2_write_4(sc, reg, val) \ + bus_space_write_4(sc->lt_bst2, sc->lt_bsh2, reg, val) +#define timer3_write_4(sc, reg, val) \ + bus_space_write_4(sc->lt_bst3, sc->lt_bsh3, reg, val) + + +static struct timecounter rt1310_timecounter = { + .tc_get_timecount = rt1310_get_timecount, + .tc_name = "RT1310ATimer1", + .tc_frequency = 0, /* will be filled later */ + .tc_counter_mask = ~0u, + .tc_quality = 1000, +}; + +static int +rt1310_timer_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "rt,timer")) + return (ENXIO); + + device_set_desc(dev, "RT1310 timer"); + return (BUS_PROBE_DEFAULT); +} + +static int +rt1310_timer_attach(device_t dev) +{ + void *intrcookie; + struct rt1310_timer_softc *sc = device_get_softc(dev); + phandle_t node; + uint32_t freq; + + if (timer_softc) + return (ENXIO); + + timer_softc = sc; + + if (bus_alloc_resources(dev, rt1310_timer_spec, sc->lt_res)) { + device_printf(dev, "could not allocate resources\n"); + return (ENXIO); + } + + sc->lt_bst0 = rman_get_bustag(sc->lt_res[0]); + sc->lt_bsh0 = rman_get_bushandle(sc->lt_res[0]); + sc->lt_bst1 = rman_get_bustag(sc->lt_res[1]); + sc->lt_bsh1 = rman_get_bushandle(sc->lt_res[1]); + sc->lt_bst2 = rman_get_bustag(sc->lt_res[2]); + sc->lt_bsh2 = rman_get_bushandle(sc->lt_res[2]); + sc->lt_bst3 = rman_get_bustag(sc->lt_res[3]); + sc->lt_bsh3 = rman_get_bushandle(sc->lt_res[3]); + + /* Timer2 interrupt */ + if (bus_setup_intr(dev, sc->lt_res[6], INTR_TYPE_CLK, + rt1310_hardclock, NULL, sc, &intrcookie)) { + device_printf(dev, "could not setup interrupt handler\n"); + bus_release_resources(dev, rt1310_timer_spec, sc->lt_res); + return (ENXIO); + } + + /* Enable timer clock */ +/* + rt1310_pwr_write(dev, LPC_CLKPWR_TIMCLK_CTRL1, + LPC_CLKPWR_TIMCLK_CTRL1_TIMER0 | + LPC_CLKPWR_TIMCLK_CTRL1_TIMER1); +*/ + + /* Get PERIPH_CLK encoded in parent bus 'bus-frequency' property */ + + node = ofw_bus_get_node(dev); + if (OF_getprop(OF_parent(node), "bus-frequency", &freq, + sizeof(pcell_t)) <= 0) { + bus_release_resources(dev, rt1310_timer_spec, sc->lt_res); + bus_teardown_intr(dev, sc->lt_res[2], intrcookie); + device_printf(dev, "could not obtain base clock frequency\n"); + return (ENXIO); + } + + freq = fdt32_to_cpu(freq); + + /* Set desired frequency in event timer and timecounter */ + sc->lt_et.et_frequency = (uint64_t)freq; + rt1310_timecounter.tc_frequency = (uint64_t)freq; + + sc->lt_et.et_name = "RT1310ATimer2"; + sc->lt_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT; + sc->lt_et.et_quality = 1000; + sc->lt_et.et_min_period = (0x00000002LLU << 32) / sc->lt_et.et_frequency; + sc->lt_et.et_max_period = (0xfffffffeLLU << 32) / sc->lt_et.et_frequency; + sc->lt_et.et_start = rt1310_timer_start; + sc->lt_et.et_stop = rt1310_timer_stop; + sc->lt_et.et_priv = sc; + + et_register(&sc->lt_et); + tc_init(&rt1310_timecounter); + + /* Reset and enable timecounter */ + + timer0_write_4(sc, RT_TIMER_CONTROL, 0); + timer1_write_4(sc, RT_TIMER_CONTROL, 0); + timer2_write_4(sc, RT_TIMER_CONTROL, 0); + timer3_write_4(sc, RT_TIMER_CONTROL, 0); + + timer1_write_4(sc, RT_TIMER_LOAD, ~0); + timer1_write_4(sc, RT_TIMER_VALUE, ~0); + timer1_write_4(sc, RT_TIMER_CONTROL, + RT_TIMER_CTRL_ENABLE | RT_TIMER_CTRL_PERIODCAL); + + /* DELAY() now can work properly */ + rt1310_timer_initialized = 1; + + return (0); +} + +static int +rt1310_timer_start(struct eventtimer *et, sbintime_t first, sbintime_t period) +{ + struct rt1310_timer_softc *sc = (struct rt1310_timer_softc *)et->et_priv; + uint32_t ticks; + + if (period == 0) { + sc->lt_oneshot = 1; + sc->lt_period = 0; + } else { + sc->lt_oneshot = 0; + sc->lt_period = ((uint32_t)et->et_frequency * period) >> 32; + } + + if (first == 0) + ticks = sc->lt_period; + else + ticks = ((uint32_t)et->et_frequency * first) >> 32; + + /* Reset timer */ + timer2_write_4(sc, RT_TIMER_CONTROL, 0); + + /* Start timer */ + timer2_write_4(sc, RT_TIMER_LOAD, ticks); + timer2_write_4(sc, RT_TIMER_VALUE, ticks); + timer2_write_4(sc, RT_TIMER_CONTROL, + RT_TIMER_CTRL_ENABLE | RT_TIMER_CTRL_INTCTL); + + return (0); +} + +static int +rt1310_timer_stop(struct eventtimer *et) +{ + struct rt1310_timer_softc *sc = (struct rt1310_timer_softc *)et->et_priv; + + timer2_write_4(sc, RT_TIMER_CONTROL, 0); + + return (0); +} + +static device_method_t rt1310_timer_methods[] = { + DEVMETHOD(device_probe, rt1310_timer_probe), + DEVMETHOD(device_attach, rt1310_timer_attach), + { 0, 0 } +}; + +static driver_t rt1310_timer_driver = { + "timer", + rt1310_timer_methods, + sizeof(struct rt1310_timer_softc), +}; + +static devclass_t rt1310_timer_devclass; + +EARLY_DRIVER_MODULE(timer, simplebus, rt1310_timer_driver, rt1310_timer_devclass, 0, 0, BUS_PASS_TIMER); + +static int +rt1310_hardclock(void *arg) +{ + struct rt1310_timer_softc *sc = (struct rt1310_timer_softc *)arg; + + /* Reset pending interrupt */ + timer2_write_4(sc, RT_TIMER_CONTROL, + timer2_read_4(sc, RT_TIMER_CONTROL) | 0x08); + timer2_write_4(sc, RT_TIMER_CONTROL, + timer2_read_4(sc, RT_TIMER_CONTROL) & 0x1fb); + + /* Start timer again */ + if (!sc->lt_oneshot) { + timer2_write_4(sc, RT_TIMER_LOAD, sc->lt_period); + timer2_write_4(sc, RT_TIMER_VALUE, sc->lt_period); + timer2_write_4(sc, RT_TIMER_CONTROL, + RT_TIMER_CTRL_ENABLE | RT_TIMER_CTRL_INTCTL); + } + + if (sc->lt_et.et_active) + sc->lt_et.et_event_cb(&sc->lt_et, sc->lt_et.et_arg); + + return (FILTER_HANDLED); +} + +static unsigned +rt1310_get_timecount(struct timecounter *tc) +{ + return ~timer1_read_4(timer_softc, RT_TIMER_VALUE); +} + +void +DELAY(int usec) +{ + uint32_t counter; + uint32_t first, last; + int val = (rt1310_timecounter.tc_frequency / 1000000 + 1) * usec; + + /* Timer is not initialized yet */ + if (!rt1310_timer_initialized) { + for (; usec > 0; usec--) + for (counter = 100; counter > 0; counter--) + ; + return; + } + + first = rt1310_get_timecount(&rt1310_timecounter); + while (val > 0) { + last = rt1310_get_timecount(&rt1310_timecounter); + if (last < first) { + /* Timer rolled over */ + last = first; + } + + val -= (last - first); + first = last; + } +} diff --git a/sys/arm/ralink/rt1310reg.h b/sys/arm/ralink/rt1310reg.h new file mode 100644 index 0000000000000..355ba117b2176 --- /dev/null +++ b/sys/arm/ralink/rt1310reg.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2011 Jakub Wojciech Klama <jceel@FreeBSD.org> + * Copyright (c) 2015 Hiroki Mori + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM_RALINK_RT1310REG_H +#define _ARM_RALINK_RT1310REG_H + +/* + * Interrupt controller + */ + +#define RT_INTC_SCR0 0x00 +#define RT_INTC_SVR0 0x80 +#define RT_INTC_ISR 0x104 +#define RT_INTC_IPR 0x108 +#define RT_INTC_IMR 0x10c +#define RT_INTC_IECR 0x114 +#define RT_INTC_ICCR 0x118 + +#define RT_INTC_TRIG_LOW_LVL (0) +#define RT_INTC_TRIG_HIGH_LVL (1) +#define RT_INTC_TRIG_NEG_EDGE (2) +#define RT_INTC_TRIG_POS_EDGE (3) + +#define RT_INTC_TRIG_SHIF 6 + +/* + * Timer 0|1|2|3. + */ + +#define RT_TIMER_LOAD 0x00 +#define RT_TIMER_VALUE 0x04 +#define RT_TIMER_CONTROL 0x08 + +#define RT_TIMER_CTRL_INTCTL (1 << 1) +#define RT_TIMER_CTRL_INTCLR (1 << 2) +#define RT_TIMER_CTRL_INTMASK (1 << 3) +#define RT_TIMER_CTRL_DIV16 (3 << 4) +#define RT_TIMER_CTRL_DIV256 (7 << 4) +#define RT_TIMER_CTRL_PERIODCAL (1 << 7) +#define RT_TIMER_CTRL_ENABLE (1 << 8) + +#define RT_TIMER_INTERVAL (5000*150) + +/* + * GPIO + */ + +#define RT_GPIO_PORTA (0) +#define RT_GPIO_PORTB (1) + +#define RT_GPIO_OFF_PADR (0x0) +#define RT_GPIO_OFF_PADIR (0x4) +#define RT_GPIO_OFF_PBDR (0x8) +#define RT_GPIO_OFF_PBDIR (0xC) + +#endif /* _ARM_RALINK_RT1310REG_H */ diff --git a/sys/arm/ralink/rt1310var.h b/sys/arm/ralink/rt1310var.h new file mode 100644 index 0000000000000..cebf275a951e6 --- /dev/null +++ b/sys/arm/ralink/rt1310var.h @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2011 Jakub Wojciech Klama <jceel@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM_RT_RTVAR_H +#define _ARM_RT_RTVAR_H + +#include <sys/types.h> +#include <sys/bus.h> +#include <machine/bus.h> + +/* Clocking and power control */ +uint32_t lpc_pwr_read(device_t, int); +void lpc_pwr_write(device_t, int, uint32_t); + +/* GPIO */ +void rt1310_gpio_init(void); +int rt1310_gpio_set_flags(device_t, int, int); +int rt1310_gpio_set_state(device_t, int, int); +int rt1310_gpio_get_state(device_t, int, int *); + +/* DMA */ +struct lpc_dmac_channel_config +{ + int ldc_fcntl; + int ldc_src_periph; + int ldc_src_width; + int ldc_src_incr; + int ldc_src_burst; + int ldc_dst_periph; + int ldc_dst_width; + int ldc_dst_incr; + int ldc_dst_burst; + void (*ldc_success_handler)(void *); + void (*ldc_error_handler)(void *); + void * ldc_handler_arg; +}; + +int lpc_dmac_config_channel(device_t, int, struct lpc_dmac_channel_config *); +int lpc_dmac_setup_transfer(device_t, int, bus_addr_t, bus_addr_t, bus_size_t, int); +int lpc_dmac_enable_channel(device_t, int); +int lpc_dmac_disable_channel(device_t, int); +int lpc_dmac_start_burst(device_t, int); + +extern uint32_t rt1310_master_clock; + +#endif /* _ARM_RT_RTVAR_H */ diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 08148876a7287..36bcd231c630f 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -1849,8 +1849,129 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) { + struct pch new_tail; + struct pv_chunk *pc; + struct md_page *pvh; + pd_entry_t *pde; + pmap_t pmap; + pt_entry_t *pte, tpte; + pv_entry_t pv; + vm_offset_t va; + vm_page_t m, m_pc; + struct spglist free; + uint64_t inuse; + int bit, field, freed, lvl; - panic("ARM64TODO: reclaim_pv_chunk"); + PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); + KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); + pmap = NULL; + m_pc = NULL; + SLIST_INIT(&free); + TAILQ_INIT(&new_tail); + mtx_lock(&pv_chunks_mutex); + while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && SLIST_EMPTY(&free)) { + TAILQ_REMOVE(&pv_chunks, pc, pc_lru); + mtx_unlock(&pv_chunks_mutex); + if (pmap != pc->pc_pmap) { + if (pmap != NULL && pmap != locked_pmap) + PMAP_UNLOCK(pmap); + pmap = pc->pc_pmap; + /* Avoid deadlock and lock recursion. */ + if (pmap > locked_pmap) { + RELEASE_PV_LIST_LOCK(lockp); + PMAP_LOCK(pmap); + } else if (pmap != locked_pmap && + !PMAP_TRYLOCK(pmap)) { + pmap = NULL; + TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); + mtx_lock(&pv_chunks_mutex); + continue; + } + } + + /* + * Destroy every non-wired, 4 KB page mapping in the chunk. + */ + freed = 0; + for (field = 0; field < _NPCM; field++) { + for (inuse = ~pc->pc_map[field] & pc_freemask[field]; + inuse != 0; inuse &= ~(1UL << bit)) { + bit = ffsl(inuse) - 1; + pv = &pc->pc_pventry[field * 64 + bit]; + va = pv->pv_va; + pde = pmap_pde(pmap, va, &lvl); + if (lvl != 2) + continue; + pte = pmap_l2_to_l3(pde, va); + tpte = pmap_load(pte); + if ((tpte & ATTR_SW_WIRED) != 0) + continue; + tpte = pmap_load_clear(pte); + PTE_SYNC(pte); + pmap_invalidate_page(pmap, va); + m = PHYS_TO_VM_PAGE(tpte & ~ATTR_MASK); + if (pmap_page_dirty(tpte)) + vm_page_dirty(m); + if ((tpte & ATTR_AF) != 0) + vm_page_aflag_set(m, PGA_REFERENCED); + CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); + TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); + m->md.pv_gen++; + if (TAILQ_EMPTY(&m->md.pv_list) && + (m->flags & PG_FICTITIOUS) == 0) { + pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + if (TAILQ_EMPTY(&pvh->pv_list)) { + vm_page_aflag_clear(m, + PGA_WRITEABLE); + } + } + pc->pc_map[field] |= 1UL << bit; + pmap_unuse_l3(pmap, va, pmap_load(pde), &free); + freed++; + } + } + if (freed == 0) { + TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); + mtx_lock(&pv_chunks_mutex); + continue; + } + /* Every freed mapping is for a 4 KB page. */ + pmap_resident_count_dec(pmap, freed); + PV_STAT(atomic_add_long(&pv_entry_frees, freed)); + PV_STAT(atomic_add_int(&pv_entry_spare, freed)); + PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 && + pc->pc_map[2] == PC_FREE2) { + PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); + PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); + PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); + /* Entire chunk is free; return it. */ + m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); + dump_drop_page(m_pc->phys_addr); + mtx_lock(&pv_chunks_mutex); + break; + } + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); + mtx_lock(&pv_chunks_mutex); + /* One freed pv entry in locked_pmap is sufficient. */ + if (pmap == locked_pmap) + break; + } + TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); + mtx_unlock(&pv_chunks_mutex); + if (pmap != NULL && pmap != locked_pmap) + PMAP_UNLOCK(pmap); + if (m_pc == NULL && !SLIST_EMPTY(&free)) { + m_pc = SLIST_FIRST(&free); + SLIST_REMOVE_HEAD(&free, plinks.s.ss); + /* Recycle a freed page table page. */ + m_pc->wire_count = 1; + atomic_add_int(&vm_cnt.v_wire_count, 1); + } + pmap_free_zero_pages(&free); + return (m_pc); } /* diff --git a/sys/boot/common/dev_net.c b/sys/boot/common/dev_net.c index 1795767dc5b32..b7127163c77e5 100644 --- a/sys/boot/common/dev_net.c +++ b/sys/boot/common/dev_net.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include <netinet/in_systm.h> #include <stand.h> +#include <stddef.h> #include <string.h> #include <net.h> #include <netif.h> @@ -79,7 +80,7 @@ static int net_init(void); static int net_open(struct open_file *, ...); static int net_close(struct open_file *); static void net_cleanup(void); -static int net_strategy(); +static int net_strategy(void *, int, daddr_t, size_t, char *, size_t *); static int net_print(int); static int net_getparams(int sock); @@ -216,7 +217,8 @@ net_cleanup(void) } static int -net_strategy() +net_strategy(void *devdata, int rw, daddr_t blk, size_t size, char *buf, + size_t *rsize) { return (EIO); @@ -246,6 +248,8 @@ net_getparams(int sock) { char buf[MAXHOSTNAMELEN]; n_long rootaddr, smask; + struct iodesc *d = socktodesc(sock); + extern struct in_addr servip; #ifdef SUPPORT_BOOTP /* @@ -254,8 +258,26 @@ net_getparams(int sock) * be initialized. If any remain uninitialized, we will * use RARP and RPC/bootparam (the Sun way) to get them. */ - if (try_bootp) - bootp(sock, BOOTP_NONE); + if (try_bootp) { + int rc = -1; + if (bootp_response != NULL) { + rc = dhcp_try_rfc1048(bootp_response->bp_vend, + bootp_response_size - + offsetof(struct bootp, bp_vend)); + + if (servip.s_addr == 0) + servip = bootp_response->bp_siaddr; + if (rootip.s_addr == 0) + rootip = bootp_response->bp_siaddr; + if (gateip.s_addr == 0) + gateip = bootp_response->bp_giaddr; + if (myip.s_addr == 0) + myip = bootp_response->bp_yiaddr; + d->myip = myip; + } + if (rc < 0) + bootp(sock, BOOTP_NONE); + } if (myip.s_addr != 0) goto exit; #ifdef NETIF_DEBUG @@ -312,8 +334,11 @@ net_getparams(int sock) return (EIO); } exit: - if ((rootaddr = net_parse_rootpath()) != INADDR_NONE) + netproto = NET_TFTP; + if ((rootaddr = net_parse_rootpath()) != INADDR_NONE) { + netproto = NET_NFS; rootip.s_addr = rootaddr; + } #ifdef NETIF_DEBUG if (debug) { @@ -365,13 +390,6 @@ net_parse_rootpath() int i; n_long addr = INADDR_NONE; - netproto = NET_NFS; - - if (tftpip.s_addr != 0) { - netproto = NET_TFTP; - addr = tftpip.s_addr; - } - for (i = 0; rootpath[i] != '\0' && i < FNAME_SIZE; i++) if (rootpath[i] == ':') break; diff --git a/sys/boot/efi/libefi/efinet.c b/sys/boot/efi/libefi/efinet.c index c598548ed9fca..b5c8ae74eadd6 100644 --- a/sys/boot/efi/libefi/efinet.c +++ b/sys/boot/efi/libefi/efinet.c @@ -29,6 +29,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> +#include <net/ethernet.h> #include <netinet/in.h> #include <netinet/in_systm.h> @@ -36,19 +37,17 @@ __FBSDID("$FreeBSD$"); #include <net.h> #include <netif.h> -#include <dev_net.c> - #include <efi.h> #include <efilib.h> static EFI_GUID sn_guid = EFI_SIMPLE_NETWORK_PROTOCOL; static void efinet_end(struct netif *); -static int efinet_get(struct iodesc *, void *, size_t, time_t); +static ssize_t efinet_get(struct iodesc *, void **, time_t); static void efinet_init(struct iodesc *, void *); static int efinet_match(struct netif *, void *); static int efinet_probe(struct netif *, void *); -static int efinet_put(struct iodesc *, void *, size_t); +static ssize_t efinet_put(struct iodesc *, void *, size_t); struct netif_driver efinetif = { .netif_bname = "efinet", @@ -113,7 +112,7 @@ efinet_probe(struct netif *nif, void *machdep_hint) return (0); } -static int +static ssize_t efinet_put(struct iodesc *desc, void *pkt, size_t len) { struct netif *nif = desc->io_netif; @@ -125,14 +124,14 @@ efinet_put(struct iodesc *desc, void *pkt, size_t len) if (net == NULL) return (-1); - status = net->Transmit(net, 0, len, pkt, 0, 0, 0); + status = net->Transmit(net, 0, len, pkt, NULL, NULL, NULL); if (status != EFI_SUCCESS) return (-1); /* Wait for the buffer to be transmitted */ do { buf = NULL; /* XXX Is this needed? */ - status = net->GetStatus(net, 0, &buf); + status = net->GetStatus(net, NULL, &buf); /* * XXX EFI1.1 and the E1000 card returns a different * address than we gave. Sigh. @@ -143,41 +142,42 @@ efinet_put(struct iodesc *desc, void *pkt, size_t len) return ((status == EFI_SUCCESS) ? len : -1); } -static int -efinet_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) +static ssize_t +efinet_get(struct iodesc *desc, void **pkt, time_t timeout) { struct netif *nif = desc->io_netif; EFI_SIMPLE_NETWORK *net; EFI_STATUS status; UINTN bufsz; time_t t; - char buf[2048]; + char *buf, *ptr; + ssize_t ret = -1; net = nif->nif_devdata; if (net == NULL) - return (0); + return (ret); + + bufsz = net->Mode->MaxPacketSize + ETHER_HDR_LEN + ETHER_CRC_LEN; + buf = malloc(bufsz + ETHER_ALIGN); + if (buf == NULL) + return (ret); + ptr = buf + ETHER_ALIGN; - t = time(0); - while ((time(0) - t) < timeout) { - bufsz = sizeof(buf); - status = net->Receive(net, 0, &bufsz, buf, 0, 0, 0); + t = getsecs(); + while ((getsecs() - t) < timeout) { + status = net->Receive(net, NULL, &bufsz, ptr, NULL, NULL, NULL); if (status == EFI_SUCCESS) { - /* - * XXX EFI1.1 and the E1000 card trash our - * workspace if we do not do this silly copy. - * Either they are not respecting the len - * value or do not like the alignment. - */ - if (bufsz > len) - bufsz = len; - bcopy(buf, pkt, bufsz); - return (bufsz); + *pkt = buf; + ret = (ssize_t)bufsz; + break; } if (status != EFI_NOT_READY) - return (0); + break; } - return (0); + if (ret == -1) + free(buf); + return (ret); } static void @@ -205,8 +205,8 @@ efinet_init(struct iodesc *desc, void *machdep_hint) if (net->Mode->State == EfiSimpleNetworkStopped) { status = net->Start(net); if (status != EFI_SUCCESS) { - printf("net%d: cannot start interface (status=%ld)\n", - nif->nif_unit, (long)status); + printf("net%d: cannot start interface (status=%lu)\n", + nif->nif_unit, EFI_ERROR_CODE(status)); return; } } @@ -214,8 +214,8 @@ efinet_init(struct iodesc *desc, void *machdep_hint) if (net->Mode->State != EfiSimpleNetworkInitialized) { status = net->Initialize(net, 0, 0); if (status != EFI_SUCCESS) { - printf("net%d: cannot init. interface (status=%ld)\n", - nif->nif_unit, (long)status); + printf("net%d: cannot init. interface (status=%lu)\n", + nif->nif_unit, EFI_ERROR_CODE(status)); return; } } @@ -224,10 +224,10 @@ efinet_init(struct iodesc *desc, void *machdep_hint) UINT32 mask = EFI_SIMPLE_NETWORK_RECEIVE_UNICAST | EFI_SIMPLE_NETWORK_RECEIVE_BROADCAST; - status = net->ReceiveFilters(net, mask, 0, FALSE, 0, 0); + status = net->ReceiveFilters(net, mask, 0, FALSE, 0, NULL); if (status != EFI_SUCCESS) { - printf("net%d: cannot set rx. filters (status=%ld)\n", - nif->nif_unit, (long)status); + printf("net%d: cannot set rx. filters (status=%lu)\n", + nif->nif_unit, EFI_ERROR_CODE(status)); return; } } @@ -258,9 +258,9 @@ struct devsw efinet_dev = { .dv_name = "net", .dv_type = DEVT_NET, .dv_init = efinet_dev_init, - .dv_strategy = net_strategy, - .dv_open = net_open, - .dv_close = net_close, + .dv_strategy = NULL, /* Will be set in efinet_dev_init */ + .dv_open = NULL, /* Will be set in efinet_dev_init */ + .dv_close = NULL, /* Will be set in efinet_dev_init */ .dv_ioctl = noioctl, .dv_print = efinet_dev_print, .dv_cleanup = NULL @@ -277,13 +277,14 @@ efinet_dev_init() EFI_STATUS status; UINTN sz; int err, i, nifs; + extern struct devsw netdev; sz = 0; handles = NULL; - status = BS->LocateHandle(ByProtocol, &sn_guid, 0, &sz, 0); + status = BS->LocateHandle(ByProtocol, &sn_guid, NULL, &sz, NULL); if (status == EFI_BUFFER_TOO_SMALL) { handles = (EFI_HANDLE *)malloc(sz); - status = BS->LocateHandle(ByProtocol, &sn_guid, 0, &sz, + status = BS->LocateHandle(ByProtocol, &sn_guid, NULL, &sz, handles); if (EFI_ERROR(status)) free(handles); @@ -313,10 +314,11 @@ efinet_dev_init() * pull packets off the network leading to lost packets. */ status = BS->OpenProtocol(handles[i], &sn_guid, (void **)&net, - IH, 0, EFI_OPEN_PROTOCOL_EXCLUSIVE); + IH, NULL, EFI_OPEN_PROTOCOL_EXCLUSIVE); if (status != EFI_SUCCESS) { printf("Unable to open network interface %d for " - "exclusive access: %d\n", i, EFI_ERROR(status)); + "exclusive access: %lu\n", i, + EFI_ERROR_CODE(status)); } handles2[nifs] = handles[i]; @@ -351,6 +353,11 @@ efinet_dev_init() dif->dif_stats = &stats[i]; dif->dif_private = handles2[i]; } + + efinet_dev.dv_open = netdev.dv_open; + efinet_dev.dv_close = netdev.dv_close; + efinet_dev.dv_strategy = netdev.dv_strategy; + done: free(handles2); return (err); diff --git a/sys/boot/efi/libefi/time.c b/sys/boot/efi/libefi/time.c index 99831e1cf1a8f..1f9ee6f58c06e 100644 --- a/sys/boot/efi/libefi/time.c +++ b/sys/boot/efi/libefi/time.c @@ -230,5 +230,5 @@ time(time_t *tloc) time_t getsecs(void) { - return time(0); + return time(NULL); } diff --git a/sys/boot/efi/loader/Makefile b/sys/boot/efi/loader/Makefile index 23845aab2ca27..0f54aa4f7675e 100644 --- a/sys/boot/efi/loader/Makefile +++ b/sys/boot/efi/loader/Makefile @@ -9,6 +9,7 @@ MK_SSP= no PROG= loader.sym INTERNALPROG= WARNS?= 3 +LOADER_NET_SUPPORT?= yes # architecture-specific loader code SRCS= autoload.c \ @@ -35,6 +36,10 @@ CWARNFLAGS.zfs.c+= -Wno-array-bounds CWARNFLAGS.zfs.c+= -Wno-missing-prototypes .endif +.if defined(LOADER_NET_SUPPORT) +CFLAGS+= -I${.CURDIR}/../../../../lib/libstand +.endif + .if ${COMPILER_TYPE} == "gcc" && ${COMPILER_VERSION} > 40201 CWARNFLAGS.self_reloc.c+= -Wno-error=maybe-uninitialized .endif diff --git a/sys/boot/fdt/dts/arm/rt1310a.dtsi b/sys/boot/fdt/dts/arm/rt1310a.dtsi new file mode 100644 index 0000000000000..6fc916b64d16c --- /dev/null +++ b/sys/boot/fdt/dts/arm/rt1310a.dtsi @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2011 Jakub Klama <jceel@FreeBSD.org> + * Copyright (c) 2015 Hiroki Mori + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Ralink RT1310A Device Tree Source. + * + * $FreeBSD$ + */ + +/ { + compatible = "ralink,rt1310a-soc"; + #address-cells = <1>; + #size-cells = <1>; + + aliases { + serial0 = &serial0; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "ARM,926EJ-S"; + reg = <0x0>; + d-cache-line-size = <32>; // 32 bytes + i-cache-line-size = <32>; // 32 bytes + d-cache-size = <0x4000>; // L1, 16K + i-cache-size = <0x4000>; // L1, 16K + timebase-frequency = <0>; + bus-frequency = <0>; + clock-frequency = <0>; + }; + }; + + memory { + device_type = "memory"; + reg = <0x40000000 0x1000000>; // 16M at 0x40000000 + }; + + localbus@1f000000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + ranges = <0x0 0x1f000000 0x400000>; + }; + + ahb@19C00000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + ranges = <0x0 0x19C00000 0xE0000>; + bus-frequency = <13000000>; + + PIC: pic@40000 { + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + reg = <0x40000 0x20000>; + compatible = "rt,pic"; + }; + + fvmdio@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "fv,mdio"; + reg = <0x80000 0x20000>; + }; + + enet0:fv_mac0@80000 { + compatible = "fv,ethernet"; + reg = <0x80000 0x20000>; + interrupts = <7>; + interrupt-parent = <&PIC>; + + }; + + enet1:fv_mac1@A0000 { + compatible = "fv,ethernet"; + reg = <0xA0000 0x20000>; + interrupts = <8>; + interrupt-parent = <&PIC>; + }; + + }; + + apb@1E800000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + ranges = <0x0 0x1E800000 0x800000>; + bus-frequency = <75000000>; + + timer@000000 { + compatible = "rt,timer"; + reg = <0x0 0x10 + 0x10 0x10 + 0x20 0x10 + 0x30 0x10>; + interrupts = <3 4 5>; + interrupt-parent = <&PIC>; + }; + + rtc@20000 { + compatible = "rt,rtc"; + interrupts = <6>; + reg = <0x20000 0x20000>; + }; + + serial0: serial@40000 { + compatible = "ns16550"; + reg = <0x40000 0x20000>; + interrupts = <1>; + reg-shift = <2>; + clock-frequency = <6758400>; + current-speed = <38400>; + interrupt-parent = <&PIC>; + }; + + gpio0: gpio@A0000 { + compatible = "ralink,rt1310-gpio"; + gpio-controller; + #gpio-cells = <2>; + interrupts = <8>; + reg = <0xA0000 0x20000>; + }; + }; + + +/* + chosen { + stdin = "serial0"; + stdout = "serial0"; + }; +*/ +}; + diff --git a/sys/boot/fdt/dts/arm/wzr2-g300n.dts b/sys/boot/fdt/dts/arm/wzr2-g300n.dts new file mode 100644 index 0000000000000..e6b8159ec8372 --- /dev/null +++ b/sys/boot/fdt/dts/arm/wzr2-g300n.dts @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2015 Hiroki Mori + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Buffalo WZR2-G300N Device Tree Source. + * + * $FreeBSD$ + */ + +/dts-v1/; + +#include "rt1310a.dtsi" + +/ { + compatible = "WZR2-G300N", "ralink,rt1310a-soc"; + model = "WZR2-G300N"; + + flash@1f000000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "cfi-flash"; + reg = <0x1f000000 0x400000>; // 4M at 0x1f000000 + + partition@0 { + reg = <0x00000000 0x0000e000>; + label = "uboot"; + }; + partition@1 { + reg = <0x0000e000 0x00002000>; + label = "uboot_env"; + }; + partition@2 { + reg = <0x00010000 0x000f0000>; + label = "kernel"; + }; + partition@3 { + reg = <0x00100000 0x002d0000>; + label = "rootfs"; + }; + partition@4 { + reg = <0x003d0000 0x00010000>; + label = "config"; + }; + partition@5 { + reg = <0x00010000 0x003c0000>; + label = "upgrade"; + }; + }; + + gpio-leds { + compatible = "gpio-leds"; + + status { + label = "status"; + gpios = <&gpio0 4 0>; + }; + }; + + ip17x@0 { + compatible = "icplus,ip17x"; + mii-poll = <0>; + }; + +}; + +&enet0 { + local-mac-address = [ 00 1a f1 01 1f 23 ]; +}; + +&enet1 { + local-mac-address = [ 00 1a f1 01 1f 24 ]; +}; diff --git a/sys/boot/i386/libi386/pxe.c b/sys/boot/i386/libi386/pxe.c index 31c89c202caa8..7cb483317924f 100644 --- a/sys/boot/i386/libi386/pxe.c +++ b/sys/boot/i386/libi386/pxe.c @@ -30,11 +30,15 @@ __FBSDID("$FreeBSD$"); #include <stand.h> +#include <stddef.h> #include <string.h> #include <stdarg.h> +#include <sys/param.h> +#include <net/ethernet.h> #include <netinet/in_systm.h> #include <netinet/in.h> +#include <netinet/ip.h> #include <netinet/udp.h> #include <net.h> @@ -53,17 +57,15 @@ __FBSDID("$FreeBSD$"); * the PXE BIOS, and the data buffer is used to receive data from the PXE BIOS. */ #define PXE_BUFFER_SIZE 0x2000 -#define PXE_TFTP_BUFFER_SIZE 512 static char scratch_buffer[PXE_BUFFER_SIZE]; static char data_buffer[PXE_BUFFER_SIZE]; static pxenv_t *pxenv_p = NULL; /* PXENV+ */ static pxe_t *pxe_p = NULL; /* !PXE */ -static BOOTPLAYER bootplayer; /* PXE Cached information. */ +#ifdef PXE_DEBUG static int pxe_debug = 0; -static int pxe_sock = -1; -static int pxe_opens = 0; +#endif void pxe_enable(void *pxeinfo); static void (*pxe_call)(int func); @@ -71,25 +73,17 @@ static void pxenv_call(int func); static void bangpxe_call(int func); static int pxe_init(void); -static int pxe_strategy(void *devdata, int flag, daddr_t dblk, - size_t size, char *buf, size_t *rsize); -static int pxe_open(struct open_file *f, ...); -static int pxe_close(struct open_file *f); static int pxe_print(int verbose); static void pxe_cleanup(void); -static void pxe_setnfshandle(char *rootpath); static void pxe_perror(int error); static int pxe_netif_match(struct netif *nif, void *machdep_hint); static int pxe_netif_probe(struct netif *nif, void *machdep_hint); static void pxe_netif_init(struct iodesc *desc, void *machdep_hint); -static int pxe_netif_get(struct iodesc *desc, void *pkt, size_t len, - time_t timeout); -static int pxe_netif_put(struct iodesc *desc, void *pkt, size_t len); +static ssize_t pxe_netif_get(struct iodesc *, void **, time_t); +static ssize_t pxe_netif_put(struct iodesc *desc, void *pkt, size_t len); static void pxe_netif_end(struct netif *nif); -int nfs_getrootfh(struct iodesc*, char*, uint32_t*, u_char*); - extern struct netif_stats pxe_st[]; extern u_int16_t __bangpxeseg; extern u_int16_t __bangpxeoff; @@ -97,25 +91,24 @@ extern void __bangpxeentry(void); extern u_int16_t __pxenvseg; extern u_int16_t __pxenvoff; extern void __pxenventry(void); -extern struct in_addr servip; struct netif_dif pxe_ifs[] = { /* dif_unit dif_nsel dif_stats dif_private */ {0, 1, &pxe_st[0], 0} }; -struct netif_stats pxe_st[NENTS(pxe_ifs)]; +struct netif_stats pxe_st[nitems(pxe_ifs)]; struct netif_driver pxenetif = { - "pxenet", - pxe_netif_match, - pxe_netif_probe, - pxe_netif_init, - pxe_netif_get, - pxe_netif_put, - pxe_netif_end, - pxe_ifs, - NENTS(pxe_ifs) + .netif_bname = "pxenet", + .netif_match = pxe_netif_match, + .netif_probe = pxe_netif_probe, + .netif_init = pxe_netif_init, + .netif_get = pxe_netif_get, + .netif_put = pxe_netif_put, + .netif_end = pxe_netif_end, + .netif_ifs = pxe_ifs, + .netif_nifs = nitems(pxe_ifs) }; struct netif_driver *netif_drivers[] = { @@ -124,15 +117,15 @@ struct netif_driver *netif_drivers[] = { }; struct devsw pxedisk = { - "pxe", - DEVT_NET, - pxe_init, - pxe_strategy, - pxe_open, - pxe_close, - noioctl, - pxe_print, - pxe_cleanup + .dv_name = "net", + .dv_type = DEVT_NET, + .dv_init = pxe_init, + .dv_strategy = NULL, /* Will be set in pxe_init */ + .dv_open = NULL, /* Will be set in pxe_init */ + .dv_close = NULL, /* Will be set in pxe_init */ + .dv_ioctl = noioctl, + .dv_print = pxe_print, + .dv_cleanup = pxe_cleanup }; /* @@ -160,6 +153,7 @@ pxe_init(void) int counter; uint8_t checksum; uint8_t *checkptr; + extern struct devsw netdev; if (pxenv_p == NULL) return (0); @@ -215,7 +209,11 @@ pxe_init(void) break; } } - + + pxedisk.dv_open = netdev.dv_open; + pxedisk.dv_close = netdev.dv_close; + pxedisk.dv_strategy = netdev.dv_strategy; + printf("\nPXE version %d.%d, real mode entry point ", (uint8_t) (pxenv_p->Version >> 8), (uint8_t) (pxenv_p->Version & 0xFF)); @@ -236,192 +234,29 @@ pxe_init(void) pxe_p = NULL; return (0); } - bcopy(PTOV((gci_p->Buffer.segment << 4) + gci_p->Buffer.offset), - &bootplayer, gci_p->BufferSize); - return (1); -} - - -static int -pxe_strategy(void *devdata, int flag, daddr_t dblk, size_t size, - char *buf, size_t *rsize) -{ - return (EIO); -} - -static int -pxe_open(struct open_file *f, ...) -{ - va_list args; - char *devname; /* Device part of file name (or NULL). */ - char temp[FNAME_SIZE]; - int error = 0; - int i; - - va_start(args, f); - devname = va_arg(args, char*); - va_end(args); - - /* On first open, do netif open, mount, etc. */ - if (pxe_opens == 0) { - /* Find network interface. */ - if (pxe_sock < 0) { - pxe_sock = netif_open(devname); - if (pxe_sock < 0) { - printf("pxe_open: netif_open() failed\n"); - return (ENXIO); - } - if (pxe_debug) - printf("pxe_open: netif_open() succeeded\n"); - - if (socktodesc(pxe_sock) == NULL) { - printf("pxe_open: bad socket %d\n", pxe_sock); - return (ENXIO); - } - - } - if (rootip.s_addr == 0) { - /* - * Try to extract the RFC1048 data from PXE. - * If fail do a bootp/dhcp request to find out where our - * NFS/TFTP server is. Even if we dont get back - * the proper information, fall back to the server - * which brought us to life and a default rootpath. - */ - - if (dhcp_try_rfc1048(bootplayer.vendor.d, BOOTP_DHCPVEND) < 0) { - if (pxe_debug) - printf("pxe_open: no RFC1048 data in PXE Cache\n"); - bootp(pxe_sock, BOOTP_PXE); - } else if (pxe_debug) { - printf("pxe_open: loaded RFC1048 data from PXE Cache\n"); - } - -#ifdef LOADER_TFTP_SUPPORT - bootp(pxe_sock, BOOTP_PXE); -#endif - if (rootip.s_addr == 0) - rootip.s_addr = bootplayer.sip; - if (gateip.s_addr == 0) - gateip.s_addr = bootplayer.gip; - if (myip.s_addr == 0) - myip.s_addr = bootplayer.yip; - if (servip.s_addr == 0) - servip = rootip; - - netproto = NET_NFS; - if (tftpip.s_addr != 0) { - netproto = NET_TFTP; - rootip.s_addr = tftpip.s_addr; - } - - if (netproto == NET_NFS && !rootpath[0]) - strcpy(rootpath, PXENFSROOTPATH); - - for (i = 0; rootpath[i] != '\0' && i < FNAME_SIZE; i++) - if (rootpath[i] == ':') - break; - if (i && i != FNAME_SIZE && rootpath[i] == ':') { - rootpath[i++] = '\0'; - if (inet_addr(&rootpath[0]) != INADDR_NONE) - rootip.s_addr = inet_addr(&rootpath[0]); - bcopy(&rootpath[i], &temp[0], strlen(&rootpath[i]) + 1); - bcopy(&temp[0], &rootpath[0], strlen(&rootpath[i]) + 1); - } - setenv("boot.netif.ip", inet_ntoa(myip), 1); - setenv("boot.netif.netmask", intoa(netmask), 1); - setenv("boot.netif.gateway", inet_ntoa(gateip), 1); - setenv("boot.netif.server", inet_ntoa(rootip), 1); - if (bootplayer.Hardware == ETHER_TYPE) { - sprintf(temp, "%6D", bootplayer.CAddr, ":"); - setenv("boot.netif.hwaddr", temp, 1); - } - if (intf_mtu != 0) { - char mtu[16]; - snprintf(sizeof(mtu), mtu, "%u", intf_mtu); - setenv("boot.netif.mtu", mtu, 1); - } - printf("pxe_open: server addr: %s\n", inet_ntoa(rootip)); - printf("pxe_open: server path: %s\n", rootpath); - printf("pxe_open: gateway ip: %s\n", inet_ntoa(gateip)); - printf("pxe_open: my ip: %s\n", inet_ntoa(myip)); - printf("pxe_open: netmask: %s\n", intoa(netmask)); - printf("pxe_open: servip: %s\n", inet_ntoa(servip)); - - if (netproto == NET_TFTP) { - setenv("boot.tftproot.server", inet_ntoa(rootip), 1); - setenv("boot.tftproot.path", rootpath, 1); - } else if (netproto == NET_NFS) { - setenv("boot.nfsroot.server", inet_ntoa(rootip), 1); - setenv("boot.nfsroot.path", rootpath, 1); - } - setenv("dhcp.host-name", hostname, 1); - - setenv("pxeboot.ip", inet_ntoa(myip), 1); - if (bootplayer.Hardware == ETHER_TYPE) { - sprintf(temp, "%6D", bootplayer.CAddr, ":"); - setenv("pxeboot.hwaddr", temp, 1); - } - } - } - pxe_opens++; - f->f_devdata = &pxe_sock; - return (error); -} - -static int -pxe_close(struct open_file *f) -{ - -#ifdef PXE_DEBUG - if (pxe_debug) - printf("pxe_close: opens=%d\n", pxe_opens); -#endif - - /* On last close, do netif close, etc. */ - f->f_devdata = NULL; - /* Extra close call? */ - if (pxe_opens <= 0) - return (0); - pxe_opens--; - /* Not last close? */ - if (pxe_opens > 0) - return (0); - - if (netproto == NET_NFS) { - /* get an NFS filehandle for our root filesystem */ - pxe_setnfshandle(rootpath); - } - - if (pxe_sock >= 0) { - -#ifdef PXE_DEBUG - if (pxe_debug) - printf("pxe_close: calling netif_close()\n"); -#endif - netif_close(pxe_sock); - pxe_sock = -1; + free(bootp_response); + if ((bootp_response = malloc(gci_p->BufferSize)) != NULL) { + bootp_response_size = gci_p->BufferSize; + bcopy(PTOV((gci_p->Buffer.segment << 4) + gci_p->Buffer.offset), + bootp_response, bootp_response_size); } - return (0); + return (1); } static int pxe_print(int verbose) { - char line[255]; if (pxe_call == NULL) return (0); printf("%s devices:", pxedisk.dv_name); if (pager_output("\n") != 0) return (1); + printf(" %s0:", pxedisk.dv_name); if (verbose) { - snprintf(line, sizeof(line), " pxe0: %s:%s\n", - inet_ntoa(rootip), rootpath); - } else { - snprintf(line, sizeof(line), " pxe0:\n"); + printf(" %s:%s", inet_ntoa(rootip), rootpath); } - return (pager_output(line)); + return (pager_output("\n")); } static void @@ -460,66 +295,6 @@ pxe_perror(int err) return; } -/* - * Reach inside the libstand NFS code and dig out an NFS handle - * for the root filesystem. - */ -#define NFS_V3MAXFHSIZE 64 - -struct nfs_iodesc { - struct iodesc *iodesc; - off_t off; - uint32_t fhsize; - u_char fh[NFS_V3MAXFHSIZE]; - /* structure truncated */ -}; -extern struct nfs_iodesc nfs_root_node; -extern int rpc_port; - -static void -pxe_rpcmountcall() -{ - struct iodesc *d; - int error; - - if (!(d = socktodesc(pxe_sock))) - return; - d->myport = htons(--rpc_port); - d->destip = rootip; - if ((error = nfs_getrootfh(d, rootpath, &nfs_root_node.fhsize, - nfs_root_node.fh)) != 0) { - printf("NFS MOUNT RPC error: %d\n", error); - nfs_root_node.fhsize = 0; - } - nfs_root_node.iodesc = d; -} - -static void -pxe_setnfshandle(char *rootpath) -{ - int i; - u_char *fh; - char buf[2 * NFS_V3MAXFHSIZE + 3], *cp; - - /* - * If NFS files were never opened, we need to do mount call - * ourselves. Use nfs_root_node.iodesc as flag indicating - * previous NFS usage. - */ - if (nfs_root_node.iodesc == NULL) - pxe_rpcmountcall(); - - fh = &nfs_root_node.fh[0]; - buf[0] = 'X'; - cp = &buf[1]; - for (i = 0; i < nfs_root_node.fhsize; i++, cp += 2) - sprintf(cp, "%02x", fh[i]); - sprintf(cp, "X"); - setenv("boot.nfsroot.nfshandle", buf, 1); - sprintf(buf, "%d", nfs_root_node.fhsize); - setenv("boot.nfsroot.nfshandlelen", buf, 1); -} - void pxenv_call(int func) { @@ -570,121 +345,196 @@ bangpxe_call(int func) static int pxe_netif_match(struct netif *nif, void *machdep_hint) { - return 1; + return (1); } static int pxe_netif_probe(struct netif *nif, void *machdep_hint) { - t_PXENV_UDP_OPEN *udpopen_p = (t_PXENV_UDP_OPEN *)scratch_buffer; - if (pxe_call == NULL) - return -1; - - bzero(udpopen_p, sizeof(*udpopen_p)); - udpopen_p->src_ip = bootplayer.yip; - pxe_call(PXENV_UDP_OPEN); + return (-1); - if (udpopen_p->status != 0) { - printf("pxe_netif_probe: failed %x\n", udpopen_p->status); - return -1; - } - return 0; + return (0); } static void pxe_netif_end(struct netif *nif) { - t_PXENV_UDP_CLOSE *udpclose_p = (t_PXENV_UDP_CLOSE *)scratch_buffer; - bzero(udpclose_p, sizeof(*udpclose_p)); + t_PXENV_UNDI_CLOSE *undi_close_p; - pxe_call(PXENV_UDP_CLOSE); - if (udpclose_p->status != 0) - printf("pxe_end failed %x\n", udpclose_p->status); + undi_close_p = (t_PXENV_UNDI_CLOSE *)scratch_buffer; + bzero(undi_close_p, sizeof(*undi_close_p)); + pxe_call(PXENV_UNDI_CLOSE); + if (undi_close_p->Status != 0) + printf("undi close failed: %x\n", undi_close_p->Status); } static void pxe_netif_init(struct iodesc *desc, void *machdep_hint) { - int i; - for (i = 0; i < 6; ++i) - desc->myea[i] = bootplayer.CAddr[i]; - desc->xid = bootplayer.ident; -} + t_PXENV_UNDI_GET_INFORMATION *undi_info_p; + t_PXENV_UNDI_OPEN *undi_open_p; + uint8_t *mac; + int i, len; -static int -pxe_netif_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) -{ - return len; + undi_info_p = (t_PXENV_UNDI_GET_INFORMATION *)scratch_buffer; + bzero(undi_info_p, sizeof(*undi_info_p)); + pxe_call(PXENV_UNDI_GET_INFORMATION); + if (undi_info_p->Status != 0) { + printf("undi get info failed: %x\n", undi_info_p->Status); + return; + } + + /* Make sure the CurrentNodeAddress is valid. */ + for (i = 0; i < undi_info_p->HwAddrLen; ++i) { + if (undi_info_p->CurrentNodeAddress[i] != 0) + break; + } + if (i < undi_info_p->HwAddrLen) { + for (i = 0; i < undi_info_p->HwAddrLen; ++i) { + if (undi_info_p->CurrentNodeAddress[i] != 0xff) + break; + } + } + if (i < undi_info_p->HwAddrLen) + mac = undi_info_p->CurrentNodeAddress; + else + mac = undi_info_p->PermNodeAddress; + + len = min(sizeof (desc->myea), undi_info_p->HwAddrLen); + for (i = 0; i < len; ++i) + desc->myea[i] = mac[i]; + + if (bootp_response != NULL) + desc->xid = bootp_response->bp_xid; + else + desc->xid = 0; + + undi_open_p = (t_PXENV_UNDI_OPEN *)scratch_buffer; + bzero(undi_open_p, sizeof(*undi_open_p)); + undi_open_p->PktFilter = FLTR_DIRECTED | FLTR_BRDCST; + pxe_call(PXENV_UNDI_OPEN); + if (undi_open_p->Status != 0) + printf("undi open failed: %x\n", undi_open_p->Status); } static int -pxe_netif_put(struct iodesc *desc, void *pkt, size_t len) +pxe_netif_receive(void **pkt) { - return len; -} + t_PXENV_UNDI_ISR *isr = (t_PXENV_UNDI_ISR *)scratch_buffer; + char *buf, *ptr, *frame; + size_t size, rsize; -ssize_t -sendudp(struct iodesc *h, void *pkt, size_t len) -{ - t_PXENV_UDP_WRITE *udpwrite_p = (t_PXENV_UDP_WRITE *)scratch_buffer; - bzero(udpwrite_p, sizeof(*udpwrite_p)); + bzero(isr, sizeof(*isr)); + isr->FuncFlag = PXENV_UNDI_ISR_IN_START; + pxe_call(PXENV_UNDI_ISR); + if (isr->Status != 0) + return (-1); - udpwrite_p->ip = h->destip.s_addr; - udpwrite_p->dst_port = h->destport; - udpwrite_p->src_port = h->myport; - udpwrite_p->buffer_size = len; - udpwrite_p->buffer.segment = VTOPSEG(pkt); - udpwrite_p->buffer.offset = VTOPOFF(pkt); + bzero(isr, sizeof(*isr)); + isr->FuncFlag = PXENV_UNDI_ISR_IN_PROCESS; + pxe_call(PXENV_UNDI_ISR); + if (isr->Status != 0) + return (-1); - if (netmask == 0 || SAMENET(myip, h->destip, netmask)) - udpwrite_p->gw = 0; - else - udpwrite_p->gw = gateip.s_addr; + while (isr->FuncFlag == PXENV_UNDI_ISR_OUT_TRANSMIT) { + /* + * Wait till transmit is done. + */ + bzero(isr, sizeof(*isr)); + isr->FuncFlag = PXENV_UNDI_ISR_IN_GET_NEXT; + pxe_call(PXENV_UNDI_ISR); + if (isr->Status != 0 || + isr->FuncFlag == PXENV_UNDI_ISR_OUT_DONE) + return (-1); + } - pxe_call(PXENV_UDP_WRITE); + while (isr->FuncFlag != PXENV_UNDI_ISR_OUT_RECEIVE) { + if (isr->Status != 0 || + isr->FuncFlag == PXENV_UNDI_ISR_OUT_DONE) { + return (-1); + } + bzero(isr, sizeof(*isr)); + isr->FuncFlag = PXENV_UNDI_ISR_IN_GET_NEXT; + pxe_call(PXENV_UNDI_ISR); + } -#if 0 - /* XXX - I dont know why we need this. */ - delay(1000); -#endif - if (udpwrite_p->status != 0) { - /* XXX: This happens a lot. It shouldn't. */ - if (udpwrite_p->status != 1) - printf("sendudp failed %x\n", udpwrite_p->status); - return -1; + size = isr->FrameLength; + buf = malloc(size + ETHER_ALIGN); + if (buf == NULL) + return (-1); + ptr = buf + ETHER_ALIGN; + rsize = 0; + + while (rsize < size) { + frame = (char *)((uintptr_t)isr->Frame.segment << 4); + frame += isr->Frame.offset; + bcopy(PTOV(frame), ptr, isr->BufferLength); + ptr += isr->BufferLength; + rsize += isr->BufferLength; + + bzero(isr, sizeof(*isr)); + isr->FuncFlag = PXENV_UNDI_ISR_IN_GET_NEXT; + pxe_call(PXENV_UNDI_ISR); + if (isr->Status != 0) { + free(buf); + return (-1); + } + + /* Did we got another update? */ + if (isr->FuncFlag == PXENV_UNDI_ISR_OUT_RECEIVE) + continue; + break; } - return len; + + *pkt = buf; + return (rsize); } -ssize_t -readudp(struct iodesc *h, void *pkt, size_t len, time_t timeout) +static ssize_t +pxe_netif_get(struct iodesc *desc, void **pkt, time_t timeout) { - t_PXENV_UDP_READ *udpread_p = (t_PXENV_UDP_READ *)scratch_buffer; - struct udphdr *uh = NULL; + time_t t; + void *ptr; + int ret = -1; - uh = (struct udphdr *) pkt - 1; - bzero(udpread_p, sizeof(*udpread_p)); + t = getsecs(); + while ((getsecs() - t) < timeout) { + ret = pxe_netif_receive(&ptr); + if (ret != -1) { + *pkt = ptr; + break; + } + } + return (ret); +} - udpread_p->dest_ip = h->myip.s_addr; - udpread_p->d_port = h->myport; - udpread_p->buffer_size = len; - udpread_p->buffer.segment = VTOPSEG(data_buffer); - udpread_p->buffer.offset = VTOPOFF(data_buffer); +static ssize_t +pxe_netif_put(struct iodesc *desc, void *pkt, size_t len) +{ + t_PXENV_UNDI_TRANSMIT *trans_p; + t_PXENV_UNDI_TBD *tbd_p; + char *data; - pxe_call(PXENV_UDP_READ); + trans_p = (t_PXENV_UNDI_TRANSMIT *)scratch_buffer; + bzero(trans_p, sizeof(*trans_p)); + tbd_p = (t_PXENV_UNDI_TBD *)(scratch_buffer + sizeof(*trans_p)); + bzero(tbd_p, sizeof(*tbd_p)); -#if 0 - /* XXX - I dont know why we need this. */ - delay(1000); -#endif - if (udpread_p->status != 0) { - /* XXX: This happens a lot. It shouldn't. */ - if (udpread_p->status != 1) - printf("readudp failed %x\n", udpread_p->status); - return -1; + data = scratch_buffer + sizeof(*trans_p) + sizeof(*tbd_p); + + trans_p->TBD.segment = VTOPSEG(tbd_p); + trans_p->TBD.offset = VTOPOFF(tbd_p); + + tbd_p->ImmedLength = len; + tbd_p->Xmit.segment = VTOPSEG(data); + tbd_p->Xmit.offset = VTOPOFF(data); + bcopy(pkt, data, len); + + pxe_call(PXENV_UNDI_TRANSMIT); + if (trans_p->Status != 0) { + return (-1); } - bcopy(data_buffer, pkt, udpread_p->buffer_size); - uh->uh_sport = udpread_p->s_port; - return udpread_p->buffer_size; + + return (len); } diff --git a/sys/boot/i386/libi386/pxe.h b/sys/boot/i386/libi386/pxe.h index 62b6aa79cca2a..119c86be519d5 100644 --- a/sys/boot/i386/libi386/pxe.h +++ b/sys/boot/i386/libi386/pxe.h @@ -349,7 +349,7 @@ typedef struct { */ # define PXENV_UNDI_ISR_OUT_DONE 0 # define PXENV_UNDI_ISR_OUT_TRANSMIT 2 -# define PXENV_UNDI_ISR_OUT_RECIEVE 3 +# define PXENV_UNDI_ISR_OUT_RECEIVE 3 # define PXENV_UNDI_ISR_OUT_BUSY 4 } PACKED t_PXENV_UNDI_ISR; diff --git a/sys/boot/i386/loader/Makefile b/sys/boot/i386/loader/Makefile index 31bf9834ddb9f..6bdffc147bc0c 100644 --- a/sys/boot/i386/loader/Makefile +++ b/sys/boot/i386/loader/Makefile @@ -9,6 +9,7 @@ MAN= INTERNALPROG= NEWVERSWHAT?= "bootstrap loader" x86 VERSION_FILE= ${.CURDIR}/../loader/version +LOADER_NET_SUPPORT?= yes # architecture-specific loader code SRCS= main.c conf.c vers.c @@ -25,6 +26,10 @@ CFLAGS+= -DLOADER_ZFS_SUPPORT LIBZFSBOOT= ${.OBJDIR}/../../zfs/libzfsboot.a .endif +.if defined(LOADER_NET_SUPPORT) +CFLAGS+= -I${.CURDIR}/../../../../lib/libstand +.endif + # Enable PXE TFTP or NFS support, not both. .if defined(LOADER_TFTP_SUPPORT) CFLAGS+= -DLOADER_TFTP_SUPPORT diff --git a/sys/boot/ofw/libofw/ofw_net.c b/sys/boot/ofw/libofw/ofw_net.c index 691dedab0525a..ea5e47e7ed0d3 100644 --- a/sys/boot/ofw/libofw/ofw_net.c +++ b/sys/boot/ofw/libofw/ofw_net.c @@ -46,8 +46,8 @@ __FBSDID("$FreeBSD$"); static int ofwn_probe(struct netif *, void *); static int ofwn_match(struct netif *, void *); static void ofwn_init(struct iodesc *, void *); -static int ofwn_get(struct iodesc *, void *, size_t, time_t); -static int ofwn_put(struct iodesc *, void *, size_t); +static ssize_t ofwn_get(struct iodesc *, void **, time_t); +static ssize_t ofwn_put(struct iodesc *, void *, size_t); static void ofwn_end(struct netif *); extern struct netif_stats ofwn_stats[]; @@ -57,7 +57,7 @@ struct netif_dif ofwn_ifs[] = { { 0, 1, &ofwn_stats[0], 0, }, }; -struct netif_stats ofwn_stats[NENTS(ofwn_ifs)]; +struct netif_stats ofwn_stats[nitems(ofwn_ifs)]; struct netif_driver ofwnet = { "net", /* netif_bname */ @@ -68,7 +68,7 @@ struct netif_driver ofwnet = { ofwn_put, /* netif_put */ ofwn_end, /* netif_end */ ofwn_ifs, /* netif_ifs */ - NENTS(ofwn_ifs) /* netif_nifs */ + nitems(ofwn_ifs) /* netif_nifs */ }; static ihandle_t netinstance; @@ -87,7 +87,7 @@ ofwn_probe(struct netif *nif, void *machdep_hint) return 0; } -static int +static ssize_t ofwn_put(struct iodesc *desc, void *pkt, size_t len) { size_t sendlen; @@ -124,20 +124,32 @@ ofwn_put(struct iodesc *desc, void *pkt, size_t len) return rv; } -static int -ofwn_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) +static ssize_t +ofwn_get(struct iodesc *desc, void **pkt, time_t timeout) { time_t t; - int length; + ssize_t length; + size_t len; + char *buf, *ptr; #if defined(NETIF_DEBUG) - printf("netif_get: pkt=%p, maxlen=%d, timeout=%d\n", pkt, len, - timeout); + printf("netif_get: pkt=%p, timeout=%d\n", pkt, timeout); #endif + /* + * We should read the "max-frame-size" int property instead, + * but at this time the iodesc does not have mtu, so we will take + * a small shortcut here. + */ + len = ETHER_MAX_LEN; + buf = malloc(len + ETHER_ALIGN); + if (buf == NULL) + return (-1); + ptr = buf + ETHER_ALIGN; + t = getsecs(); do { - length = OF_read(netinstance, pkt, len); + length = OF_read(netinstance, ptr, len); } while ((length == -2 || length == 0) && (getsecs() - t < timeout)); @@ -145,12 +157,14 @@ ofwn_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) printf("netif_get: received length=%d (%x)\n", length, length); #endif - if (length < 12) - return -1; + if (length < 12) { + free(buf); + return (-1); + } #if defined(NETIF_VERBOSE_DEBUG) { - char *ch = pkt; + char *ch = ptr; int i; for(i = 0; i < 96; i += 4) { @@ -163,7 +177,7 @@ ofwn_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) #if defined(NETIF_DEBUG) { - struct ether_header *eh = pkt; + struct ether_header *eh = ptr; printf("dst: %s ", ether_sprintf(eh->ether_dhost)); printf("src: %s ", ether_sprintf(eh->ether_shost)); @@ -171,7 +185,8 @@ ofwn_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) } #endif - return length; + *pkt = buf; + return (length); } extern char *strchr(); diff --git a/sys/boot/uboot/lib/net.c b/sys/boot/uboot/lib/net.c index 9bc0c07543706..a0d1cb4f45832 100644 --- a/sys/boot/uboot/lib/net.c +++ b/sys/boot/uboot/lib/net.c @@ -50,8 +50,8 @@ __FBSDID("$FreeBSD$"); static int net_probe(struct netif *, void *); static int net_match(struct netif *, void *); static void net_init(struct iodesc *, void *); -static int net_get(struct iodesc *, void *, size_t, time_t); -static int net_put(struct iodesc *, void *, size_t); +static ssize_t net_get(struct iodesc *, void **, time_t); +static ssize_t net_put(struct iodesc *, void *, size_t); static void net_end(struct netif *); extern struct netif_stats net_stats[]; @@ -61,7 +61,7 @@ struct netif_dif net_ifs[] = { { 0, 1, &net_stats[0], 0, }, }; -struct netif_stats net_stats[NENTS(net_ifs)]; +struct netif_stats net_stats[nitems(net_ifs)]; struct netif_driver uboot_net = { "uboot_eth", /* netif_bname */ @@ -72,7 +72,7 @@ struct netif_driver uboot_net = { net_put, /* netif_put */ net_end, /* netif_end */ net_ifs, /* netif_ifs */ - NENTS(net_ifs) /* netif_nifs */ + nitems(net_ifs) /* netif_nifs */ }; struct uboot_softc { @@ -232,7 +232,7 @@ net_probe(struct netif *nif, void *machdep_hint) return (0); } -static int +static ssize_t net_put(struct iodesc *desc, void *pkt, size_t len) { struct netif *nif = desc->io_netif; @@ -271,18 +271,21 @@ net_put(struct iodesc *desc, void *pkt, size_t len) return (rv); } -static int -net_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) +static ssize_t +net_get(struct iodesc *desc, void **pkt, time_t timeout) { struct netif *nif = desc->io_netif; struct uboot_softc *sc = nif->nif_devdata; time_t t; int err, rlen; + size_t len; + char *buf; #if defined(NETIF_DEBUG) - printf("net_get: pkt %p, len %d, timeout %d\n", pkt, len, timeout); + printf("net_get: pkt %p, timeout %d\n", pkt, timeout); #endif t = getsecs(); + len = sizeof(sc->sc_rxbuf); do { err = ub_dev_recv(sc->sc_handle, sc->sc_rxbuf, len, &rlen); @@ -299,13 +302,12 @@ net_get(struct iodesc *desc, void *pkt, size_t len, time_t timeout) #endif if (rlen > 0) { - memcpy(pkt, sc->sc_rxbuf, MIN(len, rlen)); - if (rlen != len) { -#if defined(NETIF_DEBUG) - printf("net_get: len %x, rlen %x\n", len, rlen); -#endif - } - return (rlen); + buf = malloc(rlen + ETHER_ALIGN); + if (buf == NULL) + return (-1); + memcpy(buf + ETHER_ALIGN, sc->sc_rxbuf, rlen); + *pkt = buf; + return ((ssize_t)rlen); } return (-1); diff --git a/sys/cam/scsi/scsi_sa.c b/sys/cam/scsi/scsi_sa.c index 2ed06abb820d4..8a8451c3bced6 100644 --- a/sys/cam/scsi/scsi_sa.c +++ b/sys/cam/scsi/scsi_sa.c @@ -337,6 +337,8 @@ struct sa_softc { u_int32_t maxio; u_int32_t cpi_maxio; int allow_io_split; + int inject_eom; + int set_pews_status; u_int32_t comp_algorithm; u_int32_t saved_comp_algorithm; u_int32_t media_blksize; @@ -2323,6 +2325,9 @@ sasysctlinit(void *context, int pending) SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "cpi_maxio", CTLFLAG_RD, &softc->cpi_maxio, 0, "Maximum Controller I/O size"); + SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "inject_eom", CTLFLAG_RW, + &softc->inject_eom, 0, "Queue EOM for the next write/read"); bailout: /* @@ -2588,8 +2593,27 @@ sastart(struct cam_periph *periph, union ccb *start_ccb) bp = bioq_first(&softc->bio_queue); if (bp == NULL) { xpt_release_ccb(start_ccb); - } else if ((softc->flags & SA_FLAG_ERR_PENDING) != 0) { + } else if (((softc->flags & SA_FLAG_ERR_PENDING) != 0) + || (softc->inject_eom != 0)) { struct bio *done_bp; + + if (softc->inject_eom != 0) { + softc->flags |= SA_FLAG_EOM_PENDING; + softc->inject_eom = 0; + /* + * If we're injecting EOM for writes, we + * need to keep PEWS set for 3 queries + * to cover 2 position requests from the + * kernel via sagetpos(), and then allow + * for one for the user to see the BPEW + * flag (e.g. via mt status). After that, + * it will be cleared. + */ + if (bp->bio_cmd == BIO_WRITE) + softc->set_pews_status = 3; + else + softc->set_pews_status = 1; + } again: softc->queue_count--; bioq_remove(&softc->bio_queue, bp); @@ -4842,9 +4866,12 @@ sagetpos(struct cam_periph *periph) else softc->eop = 0; - if (long_pos.flags & SA_RPOS_LONG_BPEW) + if ((long_pos.flags & SA_RPOS_LONG_BPEW) + || (softc->set_pews_status != 0)) { softc->bpew = 1; - else + if (softc->set_pews_status > 0) + softc->set_pews_status--; + } else softc->bpew = 0; } else if (error == EINVAL) { /* diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c index 08a072fbfa203..a8e69b731fe7a 100644 --- a/sys/compat/freebsd32/freebsd32_misc.c +++ b/sys/compat/freebsd32/freebsd32_misc.c @@ -2477,6 +2477,32 @@ siginfo_to_siginfo32(const siginfo_t *src, struct siginfo32 *dst) dst->si_overrun = src->si_overrun; } +#ifndef _FREEBSD32_SYSPROTO_H_ +struct freebsd32_sigqueue_args { + pid_t pid; + int signum; + /* union sigval32 */ int value; +}; +#endif +int +freebsd32_sigqueue(struct thread *td, struct freebsd32_sigqueue_args *uap) +{ + union sigval sv; + + /* + * On 32-bit ABIs, sival_int and sival_ptr are the same. + * On 64-bit little-endian ABIs, the low bits are the same. + * In 64-bit big-endian ABIs, sival_int overlaps with + * sival_ptr's HIGH bits. We choose to support sival_int + * rather than sival_ptr in this case as it seems to be + * more common. + */ + bzero(&sv, sizeof(sv)); + sv.sival_int = uap->value; + + return (kern_sigqueue(td, uap->pid, uap->signum, &sv)); +} + int freebsd32_sigtimedwait(struct thread *td, struct freebsd32_sigtimedwait_args *uap) { diff --git a/sys/compat/freebsd32/freebsd32_proto.h b/sys/compat/freebsd32/freebsd32_proto.h index fb02f3b96f354..1ec58c1c2c0d7 100644 --- a/sys/compat/freebsd32/freebsd32_proto.h +++ b/sys/compat/freebsd32/freebsd32_proto.h @@ -382,6 +382,11 @@ struct freebsd32_thr_new_args { char param_l_[PADL_(struct thr_param32 *)]; struct thr_param32 * param; char param_r_[PADR_(struct thr_param32 *)]; char param_size_l_[PADL_(int)]; int param_size; char param_size_r_[PADR_(int)]; }; +struct freebsd32_sigqueue_args { + char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)]; + char signum_l_[PADL_(int)]; int signum; char signum_r_[PADR_(int)]; + char value_l_[PADL_(int)]; int value; char value_r_[PADR_(int)]; +}; struct freebsd32_kmq_open_args { char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)]; char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)]; @@ -764,6 +769,7 @@ int freebsd32_ksem_timedwait(struct thread *, struct freebsd32_ksem_timedwait_ar int freebsd32_thr_suspend(struct thread *, struct freebsd32_thr_suspend_args *); int freebsd32_umtx_op(struct thread *, struct freebsd32_umtx_op_args *); int freebsd32_thr_new(struct thread *, struct freebsd32_thr_new_args *); +int freebsd32_sigqueue(struct thread *, struct freebsd32_sigqueue_args *); int freebsd32_kmq_open(struct thread *, struct freebsd32_kmq_open_args *); int freebsd32_kmq_setattr(struct thread *, struct freebsd32_kmq_setattr_args *); int freebsd32_kmq_timedreceive(struct thread *, struct freebsd32_kmq_timedreceive_args *); @@ -1230,6 +1236,7 @@ int freebsd10_freebsd32_pipe(struct thread *, struct freebsd10_freebsd32_pipe_ar #define FREEBSD32_SYS_AUE_freebsd32_thr_suspend AUE_NULL #define FREEBSD32_SYS_AUE_freebsd32_umtx_op AUE_NULL #define FREEBSD32_SYS_AUE_freebsd32_thr_new AUE_THR_NEW +#define FREEBSD32_SYS_AUE_freebsd32_sigqueue AUE_NULL #define FREEBSD32_SYS_AUE_freebsd32_kmq_open AUE_MQ_OPEN #define FREEBSD32_SYS_AUE_freebsd32_kmq_setattr AUE_MQ_SETATTR #define FREEBSD32_SYS_AUE_freebsd32_kmq_timedreceive AUE_MQ_TIMEDRECEIVE diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h index 739636991f641..e9ce1e3b164e0 100644 --- a/sys/compat/freebsd32/freebsd32_syscall.h +++ b/sys/compat/freebsd32/freebsd32_syscall.h @@ -358,7 +358,7 @@ #define FREEBSD32_SYS_auditctl 453 #define FREEBSD32_SYS_freebsd32_umtx_op 454 #define FREEBSD32_SYS_freebsd32_thr_new 455 -#define FREEBSD32_SYS_sigqueue 456 +#define FREEBSD32_SYS_freebsd32_sigqueue 456 #define FREEBSD32_SYS_freebsd32_kmq_open 457 #define FREEBSD32_SYS_freebsd32_kmq_setattr 458 #define FREEBSD32_SYS_freebsd32_kmq_timedreceive 459 diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c index 2088f3b973861..03debbb8f4c00 100644 --- a/sys/compat/freebsd32/freebsd32_syscalls.c +++ b/sys/compat/freebsd32/freebsd32_syscalls.c @@ -465,7 +465,7 @@ const char *freebsd32_syscallnames[] = { "auditctl", /* 453 = auditctl */ "freebsd32_umtx_op", /* 454 = freebsd32_umtx_op */ "freebsd32_thr_new", /* 455 = freebsd32_thr_new */ - "sigqueue", /* 456 = sigqueue */ + "freebsd32_sigqueue", /* 456 = freebsd32_sigqueue */ "freebsd32_kmq_open", /* 457 = freebsd32_kmq_open */ "freebsd32_kmq_setattr", /* 458 = freebsd32_kmq_setattr */ "freebsd32_kmq_timedreceive", /* 459 = freebsd32_kmq_timedreceive */ diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c index ca6fc583270d3..f35ee30114477 100644 --- a/sys/compat/freebsd32/freebsd32_sysent.c +++ b/sys/compat/freebsd32/freebsd32_sysent.c @@ -508,7 +508,7 @@ struct sysent freebsd32_sysent[] = { { AS(auditctl_args), (sy_call_t *)sys_auditctl, AUE_AUDITCTL, NULL, 0, 0, 0, SY_THR_STATIC }, /* 453 = auditctl */ { AS(freebsd32_umtx_op_args), (sy_call_t *)freebsd32_umtx_op, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 454 = freebsd32_umtx_op */ { AS(freebsd32_thr_new_args), (sy_call_t *)freebsd32_thr_new, AUE_THR_NEW, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 455 = freebsd32_thr_new */ - { AS(sigqueue_args), (sy_call_t *)sys_sigqueue, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC }, /* 456 = sigqueue */ + { AS(freebsd32_sigqueue_args), (sy_call_t *)freebsd32_sigqueue, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC }, /* 456 = freebsd32_sigqueue */ { AS(freebsd32_kmq_open_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT }, /* 457 = freebsd32_kmq_open */ { AS(freebsd32_kmq_setattr_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_ABSENT }, /* 458 = freebsd32_kmq_setattr */ { AS(freebsd32_kmq_timedreceive_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, SYF_CAPENABLED, SY_THR_ABSENT }, /* 459 = freebsd32_kmq_timedreceive */ diff --git a/sys/compat/freebsd32/freebsd32_systrace_args.c b/sys/compat/freebsd32/freebsd32_systrace_args.c index 86b94a80588c0..a07fdae612e58 100644 --- a/sys/compat/freebsd32/freebsd32_systrace_args.c +++ b/sys/compat/freebsd32/freebsd32_systrace_args.c @@ -2355,12 +2355,12 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 2; break; } - /* sigqueue */ + /* freebsd32_sigqueue */ case 456: { - struct sigqueue_args *p = params; + struct freebsd32_sigqueue_args *p = params; iarg[0] = p->pid; /* pid_t */ iarg[1] = p->signum; /* int */ - uarg[2] = (intptr_t) p->value; /* void * */ + iarg[2] = p->value; /* int */ *n_args = 3; break; } @@ -7130,7 +7130,7 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; - /* sigqueue */ + /* freebsd32_sigqueue */ case 456: switch(ndx) { case 0: @@ -7140,7 +7140,7 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) p = "int"; break; case 2: - p = "userland void *"; + p = "int"; break; default: break; @@ -10305,7 +10305,7 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; - /* sigqueue */ + /* freebsd32_sigqueue */ case 456: if (ndx == 0 || ndx == 1) p = "int"; diff --git a/sys/compat/freebsd32/syscalls.master b/sys/compat/freebsd32/syscalls.master index db0f725aba742..ac624e4f7df22 100644 --- a/sys/compat/freebsd32/syscalls.master +++ b/sys/compat/freebsd32/syscalls.master @@ -819,8 +819,8 @@ 455 AUE_THR_NEW STD { int freebsd32_thr_new( \ struct thr_param32 *param, \ int param_size); } -456 AUE_NULL NOPROTO { int sigqueue(pid_t pid, int signum, \ - void *value); } +456 AUE_NULL STD { int freebsd32_sigqueue(pid_t pid, \ + int signum, int value); } 457 AUE_MQ_OPEN NOSTD { int freebsd32_kmq_open( \ const char *path, int flags, mode_t mode, \ const struct mq_attr32 *attr); } diff --git a/sys/compat/linprocfs/linprocfs.c b/sys/compat/linprocfs/linprocfs.c index ea4d9465748bf..57853e6385c11 100644 --- a/sys/compat/linprocfs/linprocfs.c +++ b/sys/compat/linprocfs/linprocfs.c @@ -202,8 +202,9 @@ linprocfs_docpuinfo(PFS_FILL_ARGS) char model[128]; uint64_t freq; size_t size; + u_int cache_size[4]; int fqmhz, fqkhz; - int i; + int i, j; /* * We default the flags to include all non-conflicting flags, @@ -219,27 +220,20 @@ linprocfs_docpuinfo(PFS_FILL_ARGS) "3dnowext", "3dnow" }; + static char *power_flags[] = { + "ts", "fid", "vid", + "ttp", "tm", "stc", + "100mhzsteps", "hwpstate", "", + "cpb", "eff_freq_ro", "proc_feedback", + "acc_power", + }; + hw_model[0] = CTL_HW; hw_model[1] = HW_MODEL; model[0] = '\0'; size = sizeof(model); if (kernel_sysctl(td, hw_model, 2, &model, &size, 0, 0, 0, 0) != 0) strcpy(model, "unknown"); - for (i = 0; i < mp_ncpus; ++i) { - sbuf_printf(sb, - "processor\t: %d\n" - "vendor_id\t: %.20s\n" - "cpu family\t: %u\n" - "model\t\t: %u\n" - "model name\t: %s\n" - "stepping\t: %u\n\n", - i, cpu_vendor, CPUID_TO_FAMILY(cpu_id), - CPUID_TO_MODEL(cpu_id), model, cpu_id & CPUID_STEPPING); - /* XXX per-cpu vendor / class / model / id? */ - } - - sbuf_cat(sb, "flags\t\t:"); - #ifdef __i386__ switch (cpu_vendor_id) { case CPU_VENDOR_AMD: @@ -251,20 +245,70 @@ linprocfs_docpuinfo(PFS_FILL_ARGS) break; } #endif - - for (i = 0; i < 32; i++) - if (cpu_feature & (1 << i)) - sbuf_printf(sb, " %s", flags[i]); - sbuf_cat(sb, "\n"); - freq = atomic_load_acq_64(&tsc_freq); - if (freq != 0) { - fqmhz = (freq + 4999) / 1000000; - fqkhz = ((freq + 4999) / 10000) % 100; + do_cpuid(0x80000006, cache_size); + for (i = 0; i < mp_ncpus; ++i) { + fqmhz = 0; + fqkhz = 0; + freq = atomic_load_acq_64(&tsc_freq); + if (freq != 0) { + fqmhz = (freq + 4999) / 1000000; + fqkhz = ((freq + 4999) / 10000) % 100; + } sbuf_printf(sb, + "processor\t: %d\n" + "vendor_id\t: %.20s\n" + "cpu family\t: %u\n" + "model\t\t: %u\n" + "model name\t: %s\n" + "stepping\t: %u\n" "cpu MHz\t\t: %d.%02d\n" - "bogomips\t: %d.%02d\n", - fqmhz, fqkhz, fqmhz, fqkhz); + "cache size\t: %d KB\n" + "physical id\t: %d\n" + "siblings\t: %d\n" + "core id\t\t: %d\n" + "cpu cores\t: %d\n" + "apicid\t\t: %d\n" + "initial apicid\t: %d\n" + "fpu\t\t: %s\n" + "fpu_exception\t: %s\n" + "cpuid level\t: %d\n" + "wp\t\t: %s\n", + i, cpu_vendor, CPUID_TO_FAMILY(cpu_id), + CPUID_TO_MODEL(cpu_id), model, cpu_id & CPUID_STEPPING, + fqmhz, fqkhz, + (cache_size[2] >> 16), 0, mp_ncpus, i, mp_ncpus, + i, i, /*cpu_id & CPUID_LOCAL_APIC_ID ??*/ + (cpu_feature & CPUID_FPU) ? "yes" : "no", "yes", + CPUID_TO_FAMILY(cpu_id), "yes"); + sbuf_cat(sb, "flags\t\t:"); + for (j = 0; j < nitems(flags); j++) + if (cpu_feature & (1 << j)) + sbuf_printf(sb, " %s", flags[j]); + sbuf_cat(sb, "\n"); + sbuf_printf(sb, + "bugs\t\t: %s\n" + "bogomips\t: %d.%02d\n" + "clflush size\t: %d\n" + "cache_alignment\t: %d\n" + "address sizes\t: %d bits physical, %d bits virtual\n", +#if defined(I586_CPU) && !defined(NO_F00F_HACK) + (has_f00f_bug) ? "Intel F00F" : "", +#else + "", +#endif + fqmhz, fqkhz, + cpu_clflush_line_size, cpu_clflush_line_size, + cpu_maxphyaddr, + (cpu_maxphyaddr > 32) ? 48 : 0); + sbuf_cat(sb, "power management: "); + for (j = 0; j < nitems(power_flags); j++) + if (amd_pminfo & (1 << j)) + sbuf_printf(sb, " %s", power_flags[j]); + sbuf_cat(sb, "\n\n"); + + /* XXX per-cpu vendor / class / model / id? */ } + sbuf_cat(sb, "\n"); return (0); } diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c index b608d049567bb..ecea3a7531926 100644 --- a/sys/compat/linuxkpi/common/src/linux_compat.c +++ b/sys/compat/linuxkpi/common/src/linux_compat.c @@ -485,6 +485,16 @@ linux_cdev_handle_insert(void *handle, struct vm_area_struct *vmap) return (NULL); } } + /* + * The same VM object might be shared by multiple processes + * and the mm_struct is usually freed when a process exits. + * + * The atomic reference below makes sure the mm_struct is + * available as long as the vmap is in the linux_vma_head. + */ + if (atomic_inc_not_zero(&vmap->vm_mm->mm_users) == 0) + panic("linuxkpi: mm_users is zero\n"); + TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry); rw_wunlock(&linux_vma_lock); return (vmap); @@ -499,6 +509,9 @@ linux_cdev_handle_remove(struct vm_area_struct *vmap) rw_wlock(&linux_vma_lock); TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry); rw_wunlock(&linux_vma_lock); + + /* Drop reference on mm_struct */ + mmput(vmap->vm_mm); kfree(vmap); } @@ -1626,18 +1639,12 @@ linux_irq_handler(void *ent) } #if defined(__i386__) || defined(__amd64__) -static void -wbinvd_cb(void *arg __unused) -{ - - wbinvd(); -} - int linux_wbinvd_on_all_cpus(void) { - return (linux_on_each_cpu(wbinvd_cb, NULL)); + pmap_invalidate_cache(); + return (0); } #endif diff --git a/sys/conf/options.mips b/sys/conf/options.mips index b61b9aeb1242c..92e830ba341b7 100644 --- a/sys/conf/options.mips +++ b/sys/conf/options.mips @@ -152,6 +152,7 @@ RT3050F opt_rt305x.h RT305X opt_rt305x.h RT305X_UBOOT opt_rt305x.h RT305X_USE_UART opt_rt305x.h +RT_MDIO opt_rt305x.h # # Options that affect the pmap. diff --git a/sys/contrib/octeon-sdk/cvmx-app-init.h b/sys/contrib/octeon-sdk/cvmx-app-init.h index beab38e66e1e2..9870a6f0d0f13 100644 --- a/sys/contrib/octeon-sdk/cvmx-app-init.h +++ b/sys/contrib/octeon-sdk/cvmx-app-init.h @@ -311,6 +311,7 @@ enum cvmx_board_types_enum { #endif #if defined(OCTEON_VENDOR_UBIQUITI) CVMX_BOARD_TYPE_CUST_UBIQUITI_E100=20002, + CVMX_BOARD_TYPE_CUST_UBIQUITI_USG= 20004, #endif #if defined(OCTEON_VENDOR_RADISYS) CVMX_BOARD_TYPE_CUST_RADISYS_RSYS4GBE=20002, @@ -457,6 +458,7 @@ static inline const char *cvmx_board_type_to_string(enum cvmx_board_types_enum t #endif #if defined(OCTEON_VENDOR_UBIQUITI) ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_CUST_UBIQUITI_E100) + ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_CUST_UBIQUITI_USG) #endif #if defined(OCTEON_VENDOR_RADISYS) ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_CUST_RADISYS_RSYS4GBE) diff --git a/sys/contrib/octeon-sdk/cvmx-helper-board.c b/sys/contrib/octeon-sdk/cvmx-helper-board.c index 49e52d47d1bcc..b7375e55cdf9d 100644 --- a/sys/contrib/octeon-sdk/cvmx-helper-board.c +++ b/sys/contrib/octeon-sdk/cvmx-helper-board.c @@ -598,6 +598,7 @@ int cvmx_helper_board_get_mii_address(int ipd_port) #endif #if defined(OCTEON_VENDOR_UBIQUITI) case CVMX_BOARD_TYPE_CUST_UBIQUITI_E100: + case CVMX_BOARD_TYPE_CUST_UBIQUITI_USG: if (ipd_port > 2) return -1; return (7 - ipd_port); @@ -1499,7 +1500,8 @@ int __cvmx_helper_board_hardware_enable(int interface) } } #if defined(OCTEON_VENDOR_UBIQUITI) - else if (cvmx_sysinfo_get()->board_type == CVMX_BOARD_TYPE_CUST_UBIQUITI_E100) + else if (cvmx_sysinfo_get()->board_type == CVMX_BOARD_TYPE_CUST_UBIQUITI_E100 || + cvmx_sysinfo_get()->board_type == CVMX_BOARD_TYPE_CUST_UBIQUITI_USG) { /* Configure ASX cloks for all ports on interface 0. */ if (interface == 0) @@ -1590,6 +1592,7 @@ cvmx_helper_board_usb_clock_types_t __cvmx_helper_board_usb_get_clock_type(void) #endif #if defined(OCTEON_VENDOR_UBIQUITI) case CVMX_BOARD_TYPE_CUST_UBIQUITI_E100: + case CVMX_BOARD_TYPE_CUST_UBIQUITI_USG: #endif #if defined(OCTEON_BOARD_CAPK_0100ND) case CVMX_BOARD_TYPE_CN3010_EVB_HS5: diff --git a/sys/dev/atkbdc/psm.c b/sys/dev/atkbdc/psm.c index 807f2ff1cf1c1..94ea8209b6d44 100644 --- a/sys/dev/atkbdc/psm.c +++ b/sys/dev/atkbdc/psm.c @@ -81,7 +81,6 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <sys/time.h> #include <sys/uio.h> -#include <sys/libkern.h> #include <sys/limits.h> #include <sys/mouse.h> @@ -300,6 +299,8 @@ typedef struct elantechhw { int dpmmy; int ntracesx; int ntracesy; + int dptracex; + int dptracey; int issemimt; int isclickpad; int hascrc; @@ -366,6 +367,7 @@ enum { typedef struct elantechaction { finger_t fingers[ELANTECH_MAX_FINGERS]; int mask; + int mask_v4wait; } elantechaction_t; /* driver control block */ @@ -3016,13 +3018,15 @@ proc_synaptics(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, ms->button = touchpad_buttons; + psmgestures(sc, &f[0], nfingers, ms); + for (id = 0; id < PSM_FINGERS; id++) + psmsmoother(sc, &f[id], id, ms, x, y); + /* Palm detection doesn't terminate the current action. */ - if (!psmpalmdetect(sc, &f[0], nfingers)) { - psmgestures(sc, &f[0], nfingers, ms); - for (id = 0; id < PSM_FINGERS; id++) - psmsmoother(sc, &f[id], id, ms, x, y); - } else { - VLOG(2, (LOG_DEBUG, "synaptics: palm detected! (%d)\n", f[0].w)); + if (psmpalmdetect(sc, &f[0], nfingers)) { + *x = *y = *z = 0; + ms->button = ms->obutton; + return (0); } ms->button |= extended_buttons | guest_buttons; @@ -3062,8 +3066,9 @@ static int psmpalmdetect(struct psm_softc *sc, finger_t *f, int nfingers) { if (!( - ((sc->synhw.capMultiFinger || - sc->synhw.capAdvancedGestures) && nfingers > 1) || + ((sc->synhw.capMultiFinger || sc->synhw.capAdvancedGestures) && + !sc->synhw.capReportsV && nfingers > 1) || + (sc->synhw.capReportsV && nfingers > 2) || (sc->synhw.capPalmDetect && f->w <= sc->syninfo.max_width) || (!sc->synhw.capPalmDetect && f->p <= sc->syninfo.max_pressure) || (sc->synhw.capPen && f->flags & PSM_FINGER_IS_PEN))) { @@ -3076,6 +3081,7 @@ psmpalmdetect(struct psm_softc *sc, finger_t *f, int nfingers) * [min_pressure; max_pressure] * - pen aren't supported but PSM_FINGER_IS_PEN is set */ + VLOG(2, (LOG_DEBUG, "synaptics: palm detected! (%d)\n", f->w)); return (1); } return (0); @@ -3767,27 +3773,30 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, nfingers = (pb->ipacket[0] & 0xc0) >> 6; if (nfingers == 3 && (pb->ipacket[3] & 0x80)) nfingers = 4; - mask = (1 << nfingers) - 1; - fn = ELANTECH_FINGER_SET_XYP(pb); + if (nfingers == 0) { + mask = (1 << nfingers) - 1; /* = 0x00 */ + break; + } + + /* Map 3-rd and 4-th fingers to first finger */ + mask = (1 << 1) - 1; /* = 0x01 */ + f[0] = ELANTECH_FINGER_SET_XYP(pb); if (sc->elanhw.haspressure) { - fn.w = ((pb->ipacket[0] & 0x30) >> 2) | + f[0].w = ((pb->ipacket[0] & 0x30) >> 2) | ((pb->ipacket[3] & 0x30) >> 4); } else { - fn.p = PSM_FINGER_DEFAULT_P; - fn.w = PSM_FINGER_DEFAULT_W; + f[0].p = PSM_FINGER_DEFAULT_P; + f[0].w = PSM_FINGER_DEFAULT_W; } /* * HW v2 dont report exact finger positions when 3 or more - * fingers are on touchpad. Use reported value as fingers - * position as it is required for tap detection + * fingers are on touchpad. */ if (nfingers > 2) - fn.flags = PSM_FINGER_FUZZY; + f[0].flags = PSM_FINGER_FUZZY; - for (id = 0; id < imin(nfingers, ELANTECH_MAX_FINGERS); id++) - f[id] = fn; break; case ELANTECH_PKT_V2_2FINGER: /*HW V2. Two finger touch */ @@ -3833,8 +3842,12 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, * ------------------------------------------- */ nfingers = (pb->ipacket[0] & 0xc0) >> 6; - mask = (1 << nfingers) - 1; - id = nfingers - 1; + /* Map 3-rd finger to first finger */ + id = nfingers > 2 ? 0 : nfingers - 1; + mask = (1 << (id + 1)) - 1; + + if (nfingers == 0) + break; fn = ELANTECH_FINGER_SET_XYP(pb); fn.w = ((pb->ipacket[0] & 0x30) >> 2) | @@ -3842,15 +3855,11 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, /* * HW v3 dont report exact finger positions when 3 or more - * fingers are on touchpad. Use reported value as fingers - * position as it is required for tap detection + * fingers are on touchpad. */ if (nfingers > 1) fn.flags = PSM_FINGER_FUZZY; - for (id = 0; id < imin(nfingers, ELANTECH_MAX_FINGERS); id++) - f[id] = fn; - if (nfingers == 2) { if (ELANTECH_PKT_IS_V3_HEAD(pb, sc->elanhw.hascrc)) { sc->elanaction.fingers[0] = fn; @@ -3858,6 +3867,7 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, } else f[0] = sc->elanaction.fingers[0]; } + f[id] = fn; break; case ELANTECH_PKT_V4_STATUS: /* HW Version 4. Status packet */ @@ -3879,9 +3889,15 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, mask = pb->ipacket[1] & 0x1f; nfingers = bitcount(mask); + if (sc->elanaction.mask_v4wait != 0) + VLOG(3, (LOG_DEBUG, "elantech: HW v4 status packet" + " when not all previous head packets received\n")); + + /* Bitmap of fingers to receive before gesture processing */ + sc->elanaction.mask_v4wait = mask & ~sc->elanaction.mask; + /* Skip "new finger is on touchpad" packets */ - if ((sc->elanaction.mask & mask) == sc->elanaction.mask && - (mask & ~sc->elanaction.mask)) { + if (sc->elanaction.mask_v4wait) { sc->elanaction.mask = mask; return (0); } @@ -3906,11 +3922,33 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, mask = sc->elanaction.mask; nfingers = bitcount(mask); id = ((pb->ipacket[3] & 0xe0) >> 5) - 1; + fn = ELANTECH_FINGER_SET_XYP(pb); + fn.w =(pb->ipacket[0] & 0xf0) >> 4; + + if (id < 0) + return (0); - if (id >= 0 && id < ELANTECH_MAX_FINGERS) { - f[id] = ELANTECH_FINGER_SET_XYP(pb); - f[id].w = (pb->ipacket[0] & 0xf0) >> 4; + /* Packet is finger position update. Report it */ + if (sc->elanaction.mask_v4wait == 0) { + if (id < ELANTECH_MAX_FINGERS) + f[id] = fn; + break; } + + /* Remove finger from waiting bitmap and store into context */ + sc->elanaction.mask_v4wait &= ~(1 << id); + if (id < ELANTECH_MAX_FINGERS) + sc->elanaction.fingers[id] = fn; + + /* Wait for other fingers if needed */ + if (sc->elanaction.mask_v4wait != 0) + return (0); + + /* All new fingers are received. Report them from context */ + for (id = 0; id < ELANTECH_MAX_FINGERS; id++) + if (sc->elanaction.mask & (1 << id)) + f[id] = sc->elanaction.fingers[id]; + break; case ELANTECH_PKT_V4_MOTION: /* HW Version 4. Motion packet */ @@ -4006,20 +4044,14 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, ms->button = touchpad_button | trackpoint_button; - /* Palm detection doesn't terminate the current action. */ - if (!psmpalmdetect(sc, &f[0], nfingers)) { - /* Send finger 1 position to gesture processor */ - if (PSM_FINGER_IS_SET(f[0]) || PSM_FINGER_IS_SET(f[1]) || - nfingers == 0) - psmgestures(sc, &f[0], imin(nfingers, 3), ms); - /* Send fingers positions to movement smoothers */ - for (id = 0; id < PSM_FINGERS; id++) - if (PSM_FINGER_IS_SET(f[id]) || !(mask & (1 << id))) - psmsmoother(sc, &f[id], id, ms, x, y); - } else { - VLOG(2, (LOG_DEBUG, "elantech: palm detected! (%d)\n", - f[0].w)); - } + /* Send finger 1 position to gesture processor */ + if (PSM_FINGER_IS_SET(f[0]) || PSM_FINGER_IS_SET(f[1]) || + nfingers == 0) + psmgestures(sc, &f[0], imin(nfingers, 3), ms); + /* Send fingers positions to movement smoothers */ + for (id = 0; id < PSM_FINGERS; id++) + if (PSM_FINGER_IS_SET(f[id]) || !(mask & (1 << id))) + psmsmoother(sc, &f[id], id, ms, x, y); /* Store current finger positions in action context */ for (id = 0; id < ELANTECH_MAX_FINGERS; id++) { @@ -4030,6 +4062,13 @@ proc_elantech(struct psm_softc *sc, packetbuf_t *pb, mousestatus_t *ms, } sc->elanaction.mask = mask; + /* Palm detection doesn't terminate the current action. */ + if (psmpalmdetect(sc, &f[0], nfingers)) { + *x = *y = *z = 0; + ms->button = ms->obutton; + return (0); + } + /* Use the extra buttons as a scrollwheel */ if (ms->button & MOUSE_BUTTON4DOWN) *z = -1; @@ -5054,7 +5093,7 @@ synaptics_sysctl_create_tree(struct psm_softc *sc, const char *name, "Enable two finger scrolling"); /* hw.psm.synaptics.min_pressure. */ - sc->syninfo.min_pressure = 16; + sc->syninfo.min_pressure = 32; SYSCTL_ADD_PROC(&sc->syninfo.sysctl_ctx, SYSCTL_CHILDREN(sc->syninfo.sysctl_tree), OID_AUTO, "min_pressure", CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_ANYBODY, @@ -6155,7 +6194,10 @@ elantech_init_synaptics(struct psm_softc *sc) sc->synhw.capPassthrough = sc->elanhw.hastrackpoint; sc->synhw.capClickPad = sc->elanhw.isclickpad; sc->synhw.capMultiFinger = 1; - sc->synhw.capAdvancedGestures = 1; + if (sc->elanhw.issemimt) + sc->synhw.capAdvancedGestures = 1; + else + sc->synhw.capReportsV = 1; sc->synhw.capPalmDetect = 1; sc->synhw.capPen = 0; sc->synhw.capReportsMax = 1; @@ -6189,6 +6231,12 @@ elantech_init_synaptics(struct psm_softc *sc) /* Disable finger detection pressure threshold */ sc->syninfo.min_pressure = 1; + /* Adjust palm width to nearly match synaptics w=10 */ + sc->syninfo.max_width = 7; + + /* Elans often report double & triple taps as single event */ + sc->syninfo.tap_min_queue = 1; + /* Use full area of touchpad */ sc->syninfo.margin_top = 0; sc->syninfo.margin_right = 0; @@ -6233,8 +6281,17 @@ enable_elantech(struct psm_softc *sc, enum probearg arg) static const int ic2hw[] = /*IC: 0 1 2 3 4 5 6 7 8 9 a b c d e f */ { 0, 0, 2, 0, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0 }; + static const int fw_sizes[][3] = { + /* FW.vers MaxX MaxY */ + { 0x020030, 1152, 768 }, + { 0x020800, 1152, 768 }, + { 0x020b00, 1152, 768 }, + { 0x040215, 900, 500 }, + { 0x040216, 819, 405 }, + { 0x040219, 900, 500 }, + }; elantechhw_t elanhw; - int icversion, hwversion, dptracex, dptracey, id, resp[3], dpix, dpiy; + int icversion, hwversion, xtr, i, id, resp[3], dpix, dpiy; KBDC kbdc = sc->kbdc; VLOG(3, (LOG_DEBUG, "elantech: BEGIN init\n")); @@ -6285,8 +6342,8 @@ enable_elantech(struct psm_softc *sc, enum probearg arg) return (FALSE); } - elanhw.ntracesx = resp[1] - 1; - elanhw.ntracesy = resp[2] - 1; + elanhw.ntracesx = imax(resp[1], 3); + elanhw.ntracesy = imax(resp[2], 3); elanhw.hastrackpoint = (resp[0] & 0x80) != 0; /* Get the touchpad resolution */ @@ -6320,24 +6377,35 @@ enable_elantech(struct psm_softc *sc, enum probearg arg) * On HW v.3 touchpads it should be done after switching hardware * to real resolution mode (by setting bit 3 of reg10) */ + elanhw.dptracex = elanhw.dptracey = 64; + for (i = 0; i < nitems(fw_sizes); i++) { + if (elanhw.fwversion == fw_sizes[i][0]) { + elanhw.sizex = fw_sizes[i][1]; + elanhw.sizey = fw_sizes[i][2]; + goto found; + } + } if (elantech_cmd(kbdc, hwversion, ELANTECH_FW_ID, resp) != 0) { printf(" Failed to read touchpad size\n"); elanhw.sizex = 10000; /* Arbitrary high values to */ elanhw.sizey = 10000; /* prevent clipping in smoother */ } else if (hwversion == 2) { - dptracex = dptracey = 64; if ((elanhw.fwversion >> 16) == 0x14 && (resp[1] & 0x10) && !elantech_cmd(kbdc, hwversion, ELANTECH_SAMPLE, resp)) { - dptracex = resp[1] / 2; - dptracey = resp[2] / 2; + elanhw.dptracex = resp[1] / 2; + elanhw.dptracey = resp[2] / 2; } - elanhw.sizex = (elanhw.ntracesx - 1) * dptracex; - elanhw.sizey = (elanhw.ntracesy - 1) * dptracey; + xtr = ((elanhw.fwversion >> 8) == 0x0208) ? 1 : 2; + elanhw.sizex = (elanhw.ntracesx - xtr) * elanhw.dptracex; + elanhw.sizey = (elanhw.ntracesy - xtr) * elanhw.dptracey; } else { elanhw.sizex = (resp[0] & 0x0f) << 8 | resp[1]; elanhw.sizey = (resp[0] & 0xf0) << 4 | resp[2]; + xtr = (elanhw.sizex % (elanhw.ntracesx - 2) == 0) ? 2 : 1; + elanhw.dptracex = elanhw.sizex / (elanhw.ntracesx - xtr); + elanhw.dptracey = elanhw.sizey / (elanhw.ntracesy - xtr); } - +found: if (verbose >= 2) { printf(" Model information:\n"); printf(" MaxX: %d\n", elanhw.sizex); @@ -6346,6 +6414,8 @@ enable_elantech(struct psm_softc *sc, enum probearg arg) printf(" DpmmY: %d\n", elanhw.dpmmy); printf(" TracesX: %d\n", elanhw.ntracesx); printf(" TracesY: %d\n", elanhw.ntracesy); + printf(" DptraceX: %d\n", elanhw.dptracex); + printf(" DptraceY: %d\n", elanhw.dptracey); printf(" SemiMT: %d\n", elanhw.issemimt); printf(" Clickpad: %d\n", elanhw.isclickpad); printf(" Trackpoint: %d\n", elanhw.hastrackpoint); diff --git a/sys/dev/cfi/cfi_core.c b/sys/dev/cfi/cfi_core.c index d292e1a4188b1..1bf687e3ceea3 100644 --- a/sys/dev/cfi/cfi_core.c +++ b/sys/dev/cfi/cfi_core.c @@ -145,6 +145,17 @@ cfi_write(struct cfi_softc *sc, u_int ofs, u_int val) } } +/* + * This is same workaound as NetBSD sys/dev/nor/cfi.c cfi_reset_default() + */ +static void +cfi_reset_default(struct cfi_softc *sc) +{ + + cfi_write(sc, 0, CFI_BCS_READ_ARRAY2); + cfi_write(sc, 0, CFI_BCS_READ_ARRAY); +} + uint8_t cfi_read_qry(struct cfi_softc *sc, u_int ofs) { @@ -152,7 +163,7 @@ cfi_read_qry(struct cfi_softc *sc, u_int ofs) cfi_write(sc, CFI_QRY_CMD_ADDR * sc->sc_width, CFI_QRY_CMD_DATA); val = cfi_read(sc, ofs * sc->sc_width); - cfi_write(sc, 0, CFI_BCS_READ_ARRAY); + cfi_reset_default(sc); return (val); } @@ -745,7 +756,7 @@ cfi_write_block(struct cfi_softc *sc) /* error is 0. */ out: - cfi_write(sc, 0, CFI_BCS_READ_ARRAY); + cfi_reset_default(sc); /* Relock Intel flash */ switch (sc->sc_cmdset) { diff --git a/sys/dev/cfi/cfi_reg.h b/sys/dev/cfi/cfi_reg.h index c810e3f7448d4..8f4f3de66c381 100644 --- a/sys/dev/cfi/cfi_reg.h +++ b/sys/dev/cfi/cfi_reg.h @@ -113,6 +113,7 @@ struct cfi_qry { #define CFI_BCS_CONFIRM 0xd0 #define CFI_BCS_BUF_PROG_SETUP 0xe8 #define CFI_BCS_READ_ARRAY 0xff +#define CFI_BCS_READ_ARRAY2 0xf0 /* Intel commands. */ #define CFI_INTEL_LB 0x01 /* Lock Block */ diff --git a/sys/dev/cxgbe/t4_iov.c b/sys/dev/cxgbe/t4_iov.c index 0f7deb07723af..a5779bf657fc9 100644 --- a/sys/dev/cxgbe/t4_iov.c +++ b/sys/dev/cxgbe/t4_iov.c @@ -92,11 +92,25 @@ struct { {0x5013, "Chelsio T580-CHR"}, #endif }, t6iov_pciids[] = { + {0x6000, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */ {0x6001, "Chelsio T6225-CR"}, /* 2 x 10/25G */ {0x6002, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ + {0x6003, "Chelsio T6425-CR"}, /* 4 x 10/25G */ + {0x6004, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */ + {0x6005, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */ + {0x6006, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */ {0x6007, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ {0x6008, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */ + {0x6009, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */ {0x600d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */ + {0x6010, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */ + {0x6011, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */ + {0x6014, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */ + {0x6015, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */ + + /* Custom */ + {0x6080, "Chelsio T6225 80"}, + {0x6081, "Chelsio T62100 81"}, }; static int t4iov_attach_child(device_t dev); diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index e3eec95defc1e..02146f3aae24b 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -623,13 +623,25 @@ struct { #endif }, t6_pciids[] = { {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */ - {0x6400, "Chelsio T6225-DBG"}, /* 2 x 10/25G, debug */ + {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */ {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */ {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ + {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */ + {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */ + {0x6405, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */ + {0x6406, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */ {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */ + {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */ {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */ - {0x6410, "Chelsio T62100-DBG"}, /* 2 x 40/50/100G, debug */ + {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */ + {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */ + {0x6414, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */ + {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */ + + /* Custom */ + {0x6480, "Chelsio T6225 80"}, + {0x6481, "Chelsio T62100 81"}, }; #ifdef TCP_OFFLOAD diff --git a/sys/dev/cxgbe/t4_sched.c b/sys/dev/cxgbe/t4_sched.c index c8a3e7796b490..c2d3cdeadc446 100644 --- a/sys/dev/cxgbe/t4_sched.c +++ b/sys/dev/cxgbe/t4_sched.c @@ -357,7 +357,6 @@ t4_init_tx_sched(struct adapter *sc) n * sizeof(*tc), M_CXGBE, M_ZERO | M_WAITOK); tc = &pi->sched_params->cl_rl[0]; for (j = 0; j < n; j++, tc++) { - tc->flags = TX_CLRL_REFRESH; tc->refcount = 0; tc->ratemode = FW_SCHED_PARAMS_RATE_ABS; tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; @@ -365,8 +364,11 @@ t4_init_tx_sched(struct adapter *sc) tc->maxrate = init_kbps[min(j, nitems(init_kbps) - 1)]; tc->pktsize = ETHERMTU; /* XXX */ - t4_sched_params_cl_rl_kbps(sc, pi->tx_chan, j, tc->mode, - tc->maxrate, tc->pktsize, 1); + if (t4_sched_params_cl_rl_kbps(sc, pi->tx_chan, j, + tc->mode, tc->maxrate, tc->pktsize, 1) == 0) + tc->flags = 0; + else + tc->flags = TX_CLRL_ERROR; } } diff --git a/sys/dev/cxgbe/t4_vf.c b/sys/dev/cxgbe/t4_vf.c index 3791c710cc223..d6e399a91c0d3 100644 --- a/sys/dev/cxgbe/t4_vf.c +++ b/sys/dev/cxgbe/t4_vf.c @@ -112,11 +112,25 @@ struct { {0x5813, "Chelsio T580-CHR VF"}, #endif }, t6vf_pciids[] = { + {0x6800, "Chelsio T6-DBG-25 VF"}, /* 2 x 10/25G, debug */ {0x6801, "Chelsio T6225-CR VF"}, /* 2 x 10/25G */ {0x6802, "Chelsio T6225-SO-CR VF"}, /* 2 x 10/25G, nomem */ + {0x6803, "Chelsio T6425-CR VF"}, /* 4 x 10/25G */ + {0x6804, "Chelsio T6425-SO-CR VF"}, /* 4 x 10/25G, nomem */ + {0x6805, "Chelsio T6225-OCP-SO VF"}, /* 2 x 10/25G, nomem */ + {0x6806, "Chelsio T62100-OCP-SO VF"}, /* 2 x 40/50/100G, nomem */ {0x6807, "Chelsio T62100-LP-CR VF"}, /* 2 x 40/50/100G */ {0x6808, "Chelsio T62100-SO-CR VF"}, /* 2 x 40/50/100G, nomem */ + {0x6809, "Chelsio T6210-BT VF"}, /* 2 x 10GBASE-T */ {0x680d, "Chelsio T62100-CR VF"}, /* 2 x 40/50/100G */ + {0x6810, "Chelsio T6-DBG-100 VF"}, /* 2 x 40/50/100G, debug */ + {0x6811, "Chelsio T6225-LL-CR VF"}, /* 2 x 10/25G */ + {0x6814, "Chelsio T61100-OCP-SO VF"}, /* 1 x 40/50/100G, nomem */ + {0x6815, "Chelsio T6201-BT VF"}, /* 2 x 1000BASE-T */ + + /* Custom */ + {0x6880, "Chelsio T6225 80 VF"}, + {0x6881, "Chelsio T62100 81 VF"}, }; static d_ioctl_t t4vf_ioctl; diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 9742da01124cf..2cc7aa69a044a 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -29,6 +29,8 @@ __FBSDID("$FreeBSD$"); #include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_ratelimit.h" #ifdef TCP_OFFLOAD #include <sys/param.h> @@ -186,6 +188,76 @@ send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) t4_wrq_tx(sc, wr); } +#ifdef RATELIMIT +/* + * Input is Bytes/second (so_max_pacing-rate), chip counts in Kilobits/second. + */ +static int +update_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps) +{ + int tc_idx, rc; + const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000; + const int port_id = toep->vi->pi->port_id; + + CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps); + + if (kbps == 0) { + /* unbind */ + tc_idx = -1; + } else { + rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx); + if (rc != 0) + return (rc); + MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls); + } + + if (toep->tc_idx != tc_idx) { + struct wrqe *wr; + struct fw_flowc_wr *flowc; + int nparams = 1, flowclen, flowclen16; + struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + + flowclen = sizeof(*flowc) + nparams * sizeof(struct + fw_flowc_mnemval); + flowclen16 = howmany(flowclen, 16); + if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 || + (wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq)) == NULL) { + if (tc_idx >= 0) + t4_release_cl_rl_kbps(sc, port_id, tc_idx); + return (ENOMEM); + } + + flowc = wrtod(wr); + memset(flowc, 0, wr->wr_len); + + flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | + V_FW_FLOWC_WR_NPARAMS(nparams)); + flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) | + V_FW_WR_FLOWID(toep->tid)); + + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; + if (tc_idx == -1) + flowc->mnemval[0].val = htobe32(0xff); + else + flowc->mnemval[0].val = htobe32(tc_idx); + + txsd->tx_credits = flowclen16; + txsd->plen = 0; + toep->tx_credits -= txsd->tx_credits; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) + toep->txsd_pidx = 0; + toep->txsd_avail--; + t4_wrq_tx(sc, wr); + } + + if (toep->tc_idx >= 0) + t4_release_cl_rl_kbps(sc, port_id, toep->tc_idx); + toep->tc_idx = tc_idx; + + return (0); +} +#endif + void send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) { @@ -619,7 +691,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) struct socket *so = inp->inp_socket; struct sockbuf *sb = &so->so_snd; int tx_credits, shove, compl, sowwakeup; - struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + struct ofld_tx_sdesc *txsd; bool aiotx_mbuf_seen; INP_WLOCK_ASSERT(inp); @@ -638,6 +710,13 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; +#ifdef RATELIMIT + if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && + (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { + inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; + } +#endif + /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. @@ -648,6 +727,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) return; } + txsd = &toep->txsd[toep->txsd_pidx]; do { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits); diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c index e66210ec11951..64d8fd092a6e1 100644 --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" +#include "opt_ratelimit.h" #include <sys/param.h> #include <sys/types.h> @@ -156,6 +157,7 @@ alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags) refcount_init(&toep->refcount, 1); toep->td = sc->tom_softc; toep->vi = vi; + toep->tc_idx = -1; toep->tx_total = tx_credits; toep->tx_credits = tx_credits; toep->ofld_txq = &sc->sge.ofld_txq[txqid]; @@ -312,6 +314,10 @@ release_offload_resources(struct toepcb *toep) if (toep->ce) release_lip(td, toep->ce); +#ifdef RATELIMIT + if (toep->tc_idx != -1) + t4_release_cl_rl_kbps(sc, toep->vi->pi->port_id, toep->tc_idx); +#endif mtx_lock(&td->toep_list_lock); TAILQ_REMOVE(&td->toep_list, toep, link); mtx_unlock(&td->toep_list_lock); diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index 5a774fd824aeb..452d99ba46ebb 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -149,6 +149,7 @@ struct toepcb { struct l2t_entry *l2te; /* L2 table entry used by this connection */ struct clip_entry *ce; /* CLIP table entry used by this tid */ int tid; /* Connection identifier */ + int tc_idx; /* traffic class that this tid is bound to */ /* tx credit handling */ u_int tx_total; /* total tx WR credits (in 16B units) */ diff --git a/sys/dev/etherswitch/e6000sw/e6060sw.c b/sys/dev/etherswitch/e6000sw/e6060sw.c index cdd3fa5193408..fedc1e4ac9f62 100644 --- a/sys/dev/etherswitch/e6000sw/e6060sw.c +++ b/sys/dev/etherswitch/e6000sw/e6060sw.c @@ -169,6 +169,7 @@ e6060sw_probe(device_t dev) sc = device_get_softc(dev); bzero(sc, sizeof(*sc)); + devid = 0; for (i = 0; i < 2; ++i) { data = MDIO_READREG(device_get_parent(dev), CORE_REGISTER + i * 0x10, SWITCH_ID); @@ -184,8 +185,6 @@ e6060sw_probe(device_t dev) break; } } - if (i == 2) - return (ENXIO); if (devid == E6060) devname = "88E6060"; @@ -193,6 +192,9 @@ e6060sw_probe(device_t dev) devname = "88E6063"; else if (devid == E6065) devname = "88E6065"; + else + return (ENXIO); + sprintf(desc, "Marvell %s MDIO switch driver at 0x%02x", devname, sc->smi_offset); device_set_desc_copy(dev, desc); diff --git a/sys/dev/etherswitch/infineon/adm6996fc.c b/sys/dev/etherswitch/infineon/adm6996fc.c index da31b7a964de0..a2f7e9d90b2bd 100644 --- a/sys/dev/etherswitch/infineon/adm6996fc.c +++ b/sys/dev/etherswitch/infineon/adm6996fc.c @@ -35,6 +35,7 @@ * This code suppose ADM6996FC SDC/SDIO connect to SOC network interface * MDC/MDIO. * This code development on Netgear WGR614Cv7. + * etherswitchcfg command port option support addtag. */ #include <sys/param.h> @@ -462,8 +463,6 @@ adm6996fc_getport(device_t dev, etherswitch_port_t *p) p->es_pvid = ADM6996FC_PVIDBYDATA(data1, data2); if (((data1 >> ADM6996FC_OPTE_SHIFT) & 0x01) == 1) p->es_flags |= ETHERSWITCH_PORT_ADDTAG; - else - p->es_flags |= ETHERSWITCH_PORT_STRIPTAG; } else { p->es_pvid = 0; } @@ -517,6 +516,10 @@ adm6996fc_setport(device_t dev, etherswitch_port_t *p) data = ADM6996FC_READREG(parent, bcaddr[p->es_port]); data &= ~(0xf << 10); data |= (p->es_pvid & 0xf) << ADM6996FC_PVID_SHIFT; + if (p->es_flags & ETHERSWITCH_PORT_ADDTAG) + data |= 1 << ADM6996FC_OPTE_SHIFT; + else + data &= ~(1 << ADM6996FC_OPTE_SHIFT); ADM6996FC_WRITEREG(parent, bcaddr[p->es_port], data); data = ADM6996FC_READREG(parent, vidaddr[p->es_port]); /* only port 4 is hi bit */ @@ -670,9 +673,6 @@ adm6996fc_setconf(device_t dev, etherswitch_conf_t *conf) /* Private VID set 1 */ data &= ~(0xf << 10); data |= (1 << 10); - /* Output Packet Tagging Enable */ - if (i == 5) - data |= (1 << 4); ADM6996FC_WRITEREG(parent, bcaddr[i], data); } for (i = 2;i <= 15; ++i) { diff --git a/sys/dev/etherswitch/ip17x/ip17x.c b/sys/dev/etherswitch/ip17x/ip17x.c index e39663498679f..650dac2ddfd62 100644 --- a/sys/dev/etherswitch/ip17x/ip17x.c +++ b/sys/dev/etherswitch/ip17x/ip17x.c @@ -28,6 +28,8 @@ * $FreeBSD$ */ +#include "opt_platform.h" + #include <sys/param.h> #include <sys/bus.h> #include <sys/errno.h> @@ -61,6 +63,12 @@ #include <dev/etherswitch/ip17x/ip175c.h> #include <dev/etherswitch/ip17x/ip175d.h> +#ifdef FDT +#include <dev/fdt/fdt_common.h> +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> +#endif + #include "mdio_if.h" #include "miibus_if.h" #include "etherswitch_if.h" @@ -72,11 +80,28 @@ static void ip17x_tick(void *); static int ip17x_ifmedia_upd(struct ifnet *); static void ip17x_ifmedia_sts(struct ifnet *, struct ifmediareq *); +static void +ip17x_identify(driver_t *driver, device_t parent) +{ + if (device_find_child(parent, "ip17x", -1) == NULL) + BUS_ADD_CHILD(parent, 0, "ip17x", -1); +} + static int ip17x_probe(device_t dev) { struct ip17x_softc *sc; uint32_t oui, model, phy_id1, phy_id2; +#ifdef FDT + phandle_t ip17x_node; + pcell_t cell; + + ip17x_node = fdt_find_compatible(OF_finddevice("/"), + "icplus,ip17x", 0); + + if (ip17x_node == 0) + return (ENXIO); +#endif sc = device_get_softc(dev); @@ -118,6 +143,15 @@ ip17x_probe(device_t dev) sc->sc_switchtype = IP17X_SWITCH_IP178C; } + sc->miipoll = 1; +#ifdef FDT + if ((OF_getencprop(ip17x_node, "mii-poll", + &cell, sizeof(cell))) > 0) + sc->miipoll = cell ? 1 : 0; +#else + (void) resource_int_value(device_get_name(dev), device_get_unit(dev), + "mii-poll", &sc->miipoll); +#endif device_set_desc_copy(dev, "IC+ IP17x switch driver"); return (BUS_PROBE_DEFAULT); } @@ -229,9 +263,11 @@ ip17x_attach(device_t dev) if (err != 0) return (err); - callout_init(&sc->callout_tick, 0); + if (sc->miipoll) { + callout_init(&sc->callout_tick, 0); - ip17x_tick(sc); + ip17x_tick(sc); + } return (0); } @@ -243,7 +279,8 @@ ip17x_detach(device_t dev) int i, port; sc = device_get_softc(dev); - callout_drain(&sc->callout_tick); + if (sc->miipoll) + callout_drain(&sc->callout_tick); for (i=0; i < MII_NPHY; i++) { if (((1 << i) & sc->phymask) == 0) @@ -564,6 +601,7 @@ ip17x_setconf(device_t dev, etherswitch_conf_t *conf) static device_method_t ip17x_methods[] = { /* Device interface */ + DEVMETHOD(device_identify, ip17x_identify), DEVMETHOD(device_probe, ip17x_probe), DEVMETHOD(device_attach, ip17x_attach), DEVMETHOD(device_detach, ip17x_detach), @@ -604,8 +642,13 @@ static devclass_t ip17x_devclass; DRIVER_MODULE(ip17x, mdio, ip17x_driver, ip17x_devclass, 0, 0); DRIVER_MODULE(miibus, ip17x, miibus_driver, miibus_devclass, 0, 0); -DRIVER_MODULE(mdio, ip17x, mdio_driver, mdio_devclass, 0, 0); DRIVER_MODULE(etherswitch, ip17x, etherswitch_driver, etherswitch_devclass, 0, 0); MODULE_VERSION(ip17x, 1); + +#ifdef FDT +MODULE_DEPEND(ip17x, mdio, 1, 1, 1); /* XXX which versions? */ +#else +DRIVER_MODULE(mdio, ip17x, mdio_driver, mdio_devclass, 0, 0); MODULE_DEPEND(ip17x, miibus, 1, 1, 1); /* XXX which versions? */ MODULE_DEPEND(ip17x, etherswitch, 1, 1, 1); /* XXX which versions? */ +#endif diff --git a/sys/dev/etherswitch/ip17x/ip17x_var.h b/sys/dev/etherswitch/ip17x/ip17x_var.h index a1f418754d961..19e0abac340dc 100644 --- a/sys/dev/etherswitch/ip17x/ip17x_var.h +++ b/sys/dev/etherswitch/ip17x/ip17x_var.h @@ -53,6 +53,7 @@ struct ip17x_softc { int numports; /* number of ports */ int *portphy; device_t **miibus; + int miipoll; etherswitch_info_t info; ip17x_switch_type sc_switchtype; struct callout callout_tick; diff --git a/sys/dev/flash/mx25l.c b/sys/dev/flash/mx25l.c index 1da67603025bb..e7d6b229486d5 100644 --- a/sys/dev/flash/mx25l.c +++ b/sys/dev/flash/mx25l.c @@ -110,7 +110,9 @@ struct mx25l_flash_ident flash_devices[] = { { "en25f32", 0x1c, 0x3116, 64 * 1024, 64, FL_NONE }, { "en25p32", 0x1c, 0x2016, 64 * 1024, 64, FL_NONE }, { "en25p64", 0x1c, 0x2017, 64 * 1024, 128, FL_NONE }, + { "en25q32", 0x1c, 0x3016, 64 * 1024, 64, FL_NONE }, { "en25q64", 0x1c, 0x3017, 64 * 1024, 128, FL_ERASE_4K }, + { "m25p32", 0x20, 0x2016, 64 * 1024, 64, FL_NONE }, { "m25p64", 0x20, 0x2017, 64 * 1024, 128, FL_NONE }, { "mx25ll32", 0xc2, 0x2016, 64 * 1024, 64, FL_NONE }, { "mx25ll64", 0xc2, 0x2017, 64 * 1024, 128, FL_NONE }, diff --git a/sys/dev/hyperv/input/hv_kbd.c b/sys/dev/hyperv/input/hv_kbd.c index bc9ab111afdfd..38b0c9598d9c9 100644 --- a/sys/dev/hyperv/input/hv_kbd.c +++ b/sys/dev/hyperv/input/hv_kbd.c @@ -198,7 +198,7 @@ static void hvkbd_do_poll(hv_kbd_sc *sc, uint8_t wait) { while (!hv_kbd_prod_is_ready(sc)) { - hv_kbd_read_channel(NULL, sc); + hv_kbd_read_channel(sc->hs_chan, sc); if (!wait) break; } diff --git a/sys/dev/rt/if_rt.c b/sys/dev/rt/if_rt.c index bd78685b8301d..daa79b09a26c7 100644 --- a/sys/dev/rt/if_rt.c +++ b/sys/dev/rt/if_rt.c @@ -70,6 +70,12 @@ __FBSDID("$FreeBSD$"); #include <dev/mii/mii.h> #include <dev/mii/miivar.h> +#ifdef RT_MDIO +#include <dev/mdio/mdio.h> +#include <dev/etherswitch/miiproxy.h> +#include "mdio_if.h" +#endif + #if 0 #include <mips/rt305x/rt305x_sysctlvar.h> #include <mips/rt305x/rt305xreg.h> @@ -91,6 +97,7 @@ __FBSDID("$FreeBSD$"); #define RT_TX_WATCHDOG_TIMEOUT 5 +#define RT_CHIPID_RT2880 0x2880 #define RT_CHIPID_RT3050 0x3050 #define RT_CHIPID_RT5350 0x5350 #define RT_CHIPID_MT7620 0x7620 @@ -99,6 +106,7 @@ __FBSDID("$FreeBSD$"); #ifdef FDT /* more specific and new models should go first */ static const struct ofw_compat_data rt_compat_data[] = { + { "ralink,rt2880-eth", RT_CHIPID_RT2880 }, { "ralink,rt3050-eth", RT_CHIPID_RT3050 }, { "ralink,rt3352-eth", RT_CHIPID_RT3050 }, { "ralink,rt3883-eth", RT_CHIPID_RT3050 }, @@ -166,6 +174,8 @@ static void rt_dma_map_addr(void *arg, bus_dma_segment_t *segs, static void rt_sysctl_attach(struct rt_softc *sc); #ifdef IF_RT_PHY_SUPPORT void rt_miibus_statchg(device_t); +#endif +#if defined(IF_RT_PHY_SUPPORT) || defined(RT_MDIO) static int rt_miibus_readreg(device_t, int, int); static int rt_miibus_writereg(device_t, int, int, int); #endif @@ -351,7 +361,7 @@ rt_attach(device_t dev) sc->mem_rid = 0; sc->mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->mem_rid, - RF_ACTIVE); + RF_ACTIVE | RF_SHAREABLE); if (sc->mem == NULL) { device_printf(dev, "could not allocate memory resource\n"); error = ENXIO; @@ -467,6 +477,9 @@ rt_attach(device_t dev) GDM_DST_PORT_CPU << GDM_OFRC_P_SHIFT /* fwd Other to CPU */ )); + if (sc->rt_chipid == RT_CHIPID_RT2880) + RT_WRITE(sc, MDIO_CFG, MDIO_2880_100T_INIT); + /* allocate Tx and Rx rings */ for (i = 0; i < RT_SOFTC_TX_RING_COUNT; i++) { error = rt_alloc_tx_ring(sc, &sc->tx_ring[i], i); @@ -2733,16 +2746,20 @@ rt_sysctl_attach(struct rt_softc *sc) "Tx collision count for GDMA ports"); } -#ifdef IF_RT_PHY_SUPPORT +#if defined(IF_RT_PHY_SUPPORT) || defined(RT_MDIO) +/* This code is only work RT2880 and same chip. */ +/* TODO: make RT3052 and later support code. But nobody need it? */ static int rt_miibus_readreg(device_t dev, int phy, int reg) { struct rt_softc *sc = device_get_softc(dev); + int dat; /* * PSEUDO_PHYAD is a special value for indicate switch attached. * No one PHY use PSEUDO_PHYAD (0x1e) address. */ +#ifndef RT_MDIO if (phy == 31) { /* Fake PHY ID for bfeswitch attach */ switch (reg) { @@ -2754,13 +2771,14 @@ rt_miibus_readreg(device_t dev, int phy, int reg) return (0x6250); /* bfeswitch */ } } +#endif /* Wait prev command done if any */ while (RT_READ(sc, MDIO_ACCESS) & MDIO_CMD_ONGO); - RT_WRITE(sc, MDIO_ACCESS, - MDIO_CMD_ONGO || - ((phy << MDIO_PHY_ADDR_SHIFT) & MDIO_PHY_ADDR_MASK) || - ((reg << MDIO_PHYREG_ADDR_SHIFT) & MDIO_PHYREG_ADDR_MASK)); + dat = ((phy << MDIO_PHY_ADDR_SHIFT) & MDIO_PHY_ADDR_MASK) | + ((reg << MDIO_PHYREG_ADDR_SHIFT) & MDIO_PHYREG_ADDR_MASK); + RT_WRITE(sc, MDIO_ACCESS, dat); + RT_WRITE(sc, MDIO_ACCESS, dat | MDIO_CMD_ONGO); while (RT_READ(sc, MDIO_ACCESS) & MDIO_CMD_ONGO); return (RT_READ(sc, MDIO_ACCESS) & MDIO_PHY_DATA_MASK); @@ -2770,19 +2788,23 @@ static int rt_miibus_writereg(device_t dev, int phy, int reg, int val) { struct rt_softc *sc = device_get_softc(dev); + int dat; /* Wait prev command done if any */ while (RT_READ(sc, MDIO_ACCESS) & MDIO_CMD_ONGO); - RT_WRITE(sc, MDIO_ACCESS, - MDIO_CMD_ONGO || MDIO_CMD_WR || - ((phy << MDIO_PHY_ADDR_SHIFT) & MDIO_PHY_ADDR_MASK) || - ((reg << MDIO_PHYREG_ADDR_SHIFT) & MDIO_PHYREG_ADDR_MASK) || - (val & MDIO_PHY_DATA_MASK)); + dat = MDIO_CMD_WR | + ((phy << MDIO_PHY_ADDR_SHIFT) & MDIO_PHY_ADDR_MASK) | + ((reg << MDIO_PHYREG_ADDR_SHIFT) & MDIO_PHYREG_ADDR_MASK) | + (val & MDIO_PHY_DATA_MASK); + RT_WRITE(sc, MDIO_ACCESS, dat); + RT_WRITE(sc, MDIO_ACCESS, dat | MDIO_CMD_ONGO); while (RT_READ(sc, MDIO_ACCESS) & MDIO_CMD_ONGO); return (0); } +#endif +#ifdef IF_RT_PHY_SUPPORT void rt_miibus_statchg(device_t dev) { @@ -2842,3 +2864,85 @@ DRIVER_MODULE(rt, simplebus, rt_driver, rt_dev_class, 0, 0); MODULE_DEPEND(rt, ether, 1, 1, 1); MODULE_DEPEND(rt, miibus, 1, 1, 1); +#ifdef RT_MDIO +MODULE_DEPEND(rt, mdio, 1, 1, 1); + +static int rtmdio_probe(device_t); +static int rtmdio_attach(device_t); +static int rtmdio_detach(device_t); + +static struct mtx miibus_mtx; + +MTX_SYSINIT(miibus_mtx, &miibus_mtx, "rt mii lock", MTX_DEF); + +/* + * Declare an additional, separate driver for accessing the MDIO bus. + */ +static device_method_t rtmdio_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, rtmdio_probe), + DEVMETHOD(device_attach, rtmdio_attach), + DEVMETHOD(device_detach, rtmdio_detach), + + /* bus interface */ + DEVMETHOD(bus_add_child, device_add_child_ordered), + + /* MDIO access */ + DEVMETHOD(mdio_readreg, rt_miibus_readreg), + DEVMETHOD(mdio_writereg, rt_miibus_writereg), +}; + +DEFINE_CLASS_0(rtmdio, rtmdio_driver, rtmdio_methods, + sizeof(struct rt_softc)); +static devclass_t rtmdio_devclass; + +DRIVER_MODULE(miiproxy, rt, miiproxy_driver, miiproxy_devclass, 0, 0); +DRIVER_MODULE(rtmdio, simplebus, rtmdio_driver, rtmdio_devclass, 0, 0); +DRIVER_MODULE(mdio, rtmdio, mdio_driver, mdio_devclass, 0, 0); + +static int +rtmdio_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "ralink,rt2880-mdio")) + return (ENXIO); + + device_set_desc(dev, "FV built-in ethernet interface, MDIO controller"); + return(0); +} + +static int +rtmdio_attach(device_t dev) +{ + struct rt_softc *sc; + int error; + + sc = device_get_softc(dev); + sc->dev = dev; + sc->mem_rid = 0; + sc->mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->mem_rid, RF_ACTIVE | RF_SHAREABLE); + if (sc->mem == NULL) { + device_printf(dev, "couldn't map memory\n"); + error = ENXIO; + goto fail; + } + + sc->bst = rman_get_bustag(sc->mem); + sc->bsh = rman_get_bushandle(sc->mem); + + bus_generic_probe(dev); + bus_enumerate_hinted_children(dev); + error = bus_generic_attach(dev); +fail: + return(error); +} + +static int +rtmdio_detach(device_t dev) +{ + return(0); +} +#endif diff --git a/sys/dev/rt/if_rtreg.h b/sys/dev/rt/if_rtreg.h index 147dea3c520fe..7198556fd5425 100644 --- a/sys/dev/rt/if_rtreg.h +++ b/sys/dev/rt/if_rtreg.h @@ -48,6 +48,10 @@ #define MDIO_PHY_DATA_MASK 0x0000ffff #define MDIO_PHY_DATA_SHIFT 0 +#define MDIO_CFG 0x04 +#define MDIO_2880_100T_INIT 0x1001BC01 +#define MDIO_2880_GIGA_INIT 0x1F01DC01 + #define FE_GLO_CFG 0x08 /*Frame Engine Global Configuration */ #define EXT_VLAN_TYPE_MASK 0xffff0000 #define EXT_VLAN_TYPE_SHIFT 16 diff --git a/sys/gnu/dts/mips/MZK-W04N-XX.dts b/sys/gnu/dts/mips/MZK-W04N-XX.dts new file mode 100644 index 0000000000000..5c198c9c779e3 --- /dev/null +++ b/sys/gnu/dts/mips/MZK-W04N-XX.dts @@ -0,0 +1,94 @@ +/dts-v1/; + +#include "rt2880.dtsi" + +/ { + compatible = "MZK-WNH", "ralink,rt2880-soc"; + model = "Planex MZK-WNH"; + +/* + memory@0 { + device_type = "memory"; + reg = <0x0 0x2000000>; + }; +*/ + + cfi@1f000000 { + compatible = "cfi-flash"; + reg = <0x1f000000 0x800000>; + bank-width = <2>; + device-width = <2>; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "Bootloader"; + reg = <0x0 0x30000>; + read-only; + }; + + devdata: partition@30000 { + label = "Config"; + reg = <0x00030000 0x00010000>; + read-only; + }; + + factory: partition@40000 { + label = "Factory"; + reg = <0x00040000 0x00010000>; + read-only; + }; + kernel: partition@50000 { + label = "kernel"; + reg = <0x00050000 0x000f0000>; + read-only; + }; + rootfs: partition@160000 { + label = "rootfs"; + reg = <0x00140000 0x002c0000>; + read-only; + }; + upgrade: partition@400000 { + label = "upgrade"; + reg = <0x00050000 0x003b0000>; + read-only; + }; + + }; + + gpio-leds { + compatible = "gpio-leds"; + + status { + label = "status"; + gpios = <&gpio0 12 0>; + }; + + }; + gpio-keys-polled { + compatible = "gpio-keys-polled"; + #address-cells = <1>; + #size-cells = <0>; + poll-interval = <20>; + + reset { + label = "reset"; + gpios = <&gpio0 10 1>; + linux,code = <0x198>; + }; + + }; + + ip17x@0 { + compatible = "icplus,ip17x"; + }; + +}; + +ðernet { + mtd-mac-address = <&factory 0x28>; +}; + +&wmac { + ralink,mtd-eeprom = <&factory 0>; +}; diff --git a/sys/gnu/dts/mips/rt2880.dtsi b/sys/gnu/dts/mips/rt2880.dtsi index ad882547f794c..284a7e9b2c7dd 100644 --- a/sys/gnu/dts/mips/rt2880.dtsi +++ b/sys/gnu/dts/mips/rt2880.dtsi @@ -5,7 +5,7 @@ cpus { cpu@0 { - compatible = "mips,mips24KEc"; + compatible = "mips,mips4KEc"; }; }; @@ -80,6 +80,9 @@ ralink,register-map = [ 00 04 08 0c 20 24 28 2c 30 34 ]; + + interrupt-parent = <&intc>; + interrupts = <7>; }; gpio1: gpio@638 { @@ -182,13 +185,13 @@ compatible = "ralink,rt2880-port", "mediatek,eth-port"; reg = <0>; }; + }; - mdio-bus { - #address-cells = <1>; - #size-cells = <0>; - - status = "disabled"; - }; + mdio-bus { + compatible = "ralink,rt2880-mdio"; + reg = <0x00400000 0x10000>; + #address-cells = <1>; + #size-cells = <0>; }; wmac: wmac@480000 { diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c index e43cd6b8482f2..fe9a2e4c4d020 100644 --- a/sys/kern/kern_sig.c +++ b/sys/kern/kern_sig.c @@ -1844,33 +1844,43 @@ struct sigqueue_args { int sys_sigqueue(struct thread *td, struct sigqueue_args *uap) { + union sigval sv; + + sv.sival_ptr = uap->value; + + return (kern_sigqueue(td, uap->pid, uap->signum, &sv)); +} + +int +kern_sigqueue(struct thread *td, pid_t pid, int signum, union sigval *value) +{ ksiginfo_t ksi; struct proc *p; int error; - if ((u_int)uap->signum > _SIG_MAXSIG) + if ((u_int)signum > _SIG_MAXSIG) return (EINVAL); /* * Specification says sigqueue can only send signal to * single process. */ - if (uap->pid <= 0) + if (pid <= 0) return (EINVAL); - if ((p = pfind(uap->pid)) == NULL) { - if ((p = zpfind(uap->pid)) == NULL) + if ((p = pfind(pid)) == NULL) { + if ((p = zpfind(pid)) == NULL) return (ESRCH); } - error = p_cansignal(td, p, uap->signum); - if (error == 0 && uap->signum != 0) { + error = p_cansignal(td, p, signum); + if (error == 0 && signum != 0) { ksiginfo_init(&ksi); ksi.ksi_flags = KSI_SIGQ; - ksi.ksi_signo = uap->signum; + ksi.ksi_signo = signum; ksi.ksi_code = SI_QUEUE; ksi.ksi_pid = td->td_proc->p_pid; ksi.ksi_uid = td->td_ucred->cr_ruid; - ksi.ksi_value.sival_ptr = uap->value; + ksi.ksi_value = *value; error = pksignal(p, ksi.ksi_signo, &ksi); } PROC_UNLOCK(p); diff --git a/sys/mips/atheros/ar531x/if_are.c b/sys/mips/atheros/ar531x/if_are.c index 294961825f709..07e2a1e5a48b8 100644 --- a/sys/mips/atheros/ar531x/if_are.c +++ b/sys/mips/atheros/ar531x/if_are.c @@ -302,9 +302,9 @@ are_attach(device_t dev) ifp->if_init = are_init; sc->are_if_flags = ifp->if_flags; - /* XXX: add real size */ - IFQ_SET_MAXLEN(&ifp->if_snd, 9); - ifp->if_snd.ifq_maxlen = 9; + /* ifqmaxlen is sysctl value in net/if.c */ + IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen); + ifp->if_snd.ifq_maxlen = ifqmaxlen; IFQ_SET_READY(&ifp->if_snd); /* Tell the upper layer(s) we support long frames. */ @@ -686,19 +686,92 @@ are_encap(struct are_softc *sc, struct mbuf **m_head) { struct are_txdesc *txd; struct are_desc *desc, *prev_desc; + struct mbuf *m; bus_dma_segment_t txsegs[ARE_MAXFRAGS]; uint32_t link_addr; int error, i, nsegs, prod, si, prev_prod; int txstat; + int startcount; + int padlen; + + startcount = sc->are_cdata.are_tx_cnt; ARE_LOCK_ASSERT(sc); + /* + * Some VIA Rhine wants packet buffers to be longword + * aligned, but very often our mbufs aren't. Rather than + * waste time trying to decide when to copy and when not + * to copy, just do it all the time. + */ + m = m_defrag(*m_head, M_NOWAIT); + if (m == NULL) { + device_printf(sc->are_dev, "are_encap m_defrag error\n"); + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + + /* + * The Rhine chip doesn't auto-pad, so we have to make + * sure to pad short frames out to the minimum frame length + * ourselves. + */ + if ((*m_head)->m_pkthdr.len < ARE_MIN_FRAMELEN) { + m = *m_head; + padlen = ARE_MIN_FRAMELEN - m->m_pkthdr.len; + if (M_WRITABLE(m) == 0) { + /* Get a writable copy. */ + m = m_dup(*m_head, M_NOWAIT); + m_freem(*m_head); + if (m == NULL) { + device_printf(sc->are_dev, "are_encap m_dup error\n"); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + } + if (m->m_next != NULL || M_TRAILINGSPACE(m) < padlen) { + m = m_defrag(m, M_NOWAIT); + if (m == NULL) { + device_printf(sc->are_dev, "are_encap m_defrag error\n"); + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + } + /* + * Manually pad short frames, and zero the pad space + * to avoid leaking data. + */ + bzero(mtod(m, char *) + m->m_pkthdr.len, padlen); + m->m_pkthdr.len += padlen; + m->m_len = m->m_pkthdr.len; + *m_head = m; + } + prod = sc->are_cdata.are_tx_prod; txd = &sc->are_cdata.are_txdesc[prod]; - error = bus_dmamap_load_mbuf_sg(sc->are_cdata.are_tx_tag, txd->tx_dmamap, - *m_head, txsegs, &nsegs, BUS_DMA_NOWAIT); + error = bus_dmamap_load_mbuf_sg(sc->are_cdata.are_tx_tag, + txd->tx_dmamap, *m_head, txsegs, &nsegs, BUS_DMA_NOWAIT); if (error == EFBIG) { - panic("EFBIG"); + device_printf(sc->are_dev, "are_encap EFBIG error\n"); + m = m_defrag(*m_head, M_NOWAIT); + if (m == NULL) { + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + error = bus_dmamap_load_mbuf_sg(sc->are_cdata.are_tx_tag, + txd->tx_dmamap, *m_head, txsegs, &nsegs, BUS_DMA_NOWAIT); + if (error != 0) { + m_freem(*m_head); + *m_head = NULL; + return (error); + } + } else if (error != 0) return (error); if (nsegs == 0) { @@ -729,13 +802,12 @@ are_encap(struct are_softc *sc, struct mbuf **m_head) for (i = 0; i < nsegs; i++) { desc = &sc->are_rdata.are_tx_ring[prod]; desc->are_stat = ADSTAT_OWN; - desc->are_devcs = ARE_DMASIZE(txsegs[i].ds_len) | ADCTL_CH; - if (i == 0) - desc->are_devcs |= ADCTL_Tx_FS; + desc->are_devcs = ARE_DMASIZE(txsegs[i].ds_len); desc->are_addr = txsegs[i].ds_addr; /* link with previous descriptor */ - if (prev_desc) - prev_desc->are_link = ARE_TX_RING_ADDR(sc, prod); + /* end of descriptor */ + if (prod == ARE_TX_RING_CNT - 1) + desc->are_devcs |= ADCTL_ER; sc->are_cdata.are_tx_cnt++; prev_desc = desc; @@ -761,16 +833,16 @@ are_encap(struct are_softc *sc, struct mbuf **m_head) /* Start transmitting */ /* Check if new list is queued in NDPTR */ txstat = (CSR_READ_4(sc, CSR_STATUS) >> 20) & 7; - if (txstat == 0 || txstat == 6) { - /* Transmit Process Stat is stop or suspended */ - CSR_WRITE_4(sc, CSR_TXPOLL, TXPOLL_TPD); + if (startcount == 0 && (txstat == 0 || txstat == 6)) { + desc = &sc->are_rdata.are_tx_ring[si]; + desc->are_devcs |= ADCTL_Tx_FS; } else { link_addr = ARE_TX_RING_ADDR(sc, si); /* Get previous descriptor */ si = (si + ARE_TX_RING_CNT - 1) % ARE_TX_RING_CNT; desc = &sc->are_rdata.are_tx_ring[si]; - desc->are_link = link_addr; + desc->are_devcs &= ~(ADCTL_Tx_IC | ADCTL_Tx_LS); } return (0); @@ -782,6 +854,7 @@ are_start_locked(struct ifnet *ifp) struct are_softc *sc; struct mbuf *m_head; int enq; + int txstat; sc = ifp->if_softc; @@ -816,6 +889,14 @@ are_start_locked(struct ifnet *ifp) */ ETHER_BPF_MTAP(ifp, m_head); } + + if (enq > 0) { + txstat = (CSR_READ_4(sc, CSR_STATUS) >> 20) & 7; + if (txstat == 0 || txstat == 6) { + /* Transmit Process Stat is stop or suspended */ + CSR_WRITE_4(sc, CSR_TXPOLL, TXPOLL_TPD); + } + } } static void diff --git a/sys/mips/atheros/ar531x/if_arereg.h b/sys/mips/atheros/ar531x/if_arereg.h index f59df630d9ee5..cb84edd9e5853 100644 --- a/sys/mips/atheros/ar531x/if_arereg.h +++ b/sys/mips/atheros/ar531x/if_arereg.h @@ -44,6 +44,8 @@ struct are_desc { #define ARE_TX_RING_CNT 128 #define ARE_TX_RING_SIZE sizeof(struct are_desc) * ARE_TX_RING_CNT #define ARE_RX_RING_SIZE sizeof(struct are_desc) * ARE_RX_RING_CNT + +#define ARE_MIN_FRAMELEN 60 #define ARE_RING_ALIGN sizeof(struct are_desc) #define ARE_RX_ALIGN sizeof(uint32_t) #define ARE_MAXFRAGS 8 diff --git a/sys/mips/atheros/ar934x_chip.c b/sys/mips/atheros/ar934x_chip.c index f59d4aaa5285c..0029965a814ae 100644 --- a/sys/mips/atheros/ar934x_chip.c +++ b/sys/mips/atheros/ar934x_chip.c @@ -315,6 +315,10 @@ ar934x_chip_reset_ethernet_switch(void) DELAY(100); ar71xx_device_start(AR934X_RESET_ETH_SWITCH); DELAY(100); + ar71xx_device_stop(AR934X_RESET_ETH_SWITCH_ANALOG); + DELAY(100); + ar71xx_device_start(AR934X_RESET_ETH_SWITCH_ANALOG); + DELAY(100); } static void diff --git a/sys/mips/conf/RT2880_FDT b/sys/mips/conf/RT2880_FDT new file mode 100644 index 0000000000000..6165063d7a1df --- /dev/null +++ b/sys/mips/conf/RT2880_FDT @@ -0,0 +1,77 @@ +# +# RT2880_FDT -- Kernel configuration file for FreeBSD/MIPS RT2880 SoC +# +# This includes all the configurable parts of the kernel. +# +# $FreeBSD$ +# + +#NO_UNIVERSE + +# +# FDT_DTS_FILE should be modified to suit the target board type. +# +#makeoptions FDT_DTS_FILE=MZK-W04N-XX.dts + +# Start with a base configuration +include "../mediatek/std.rt2880" + +ident RT2880 +cpu CPU_MIPS4KC + +# Don't build any modules by default +makeoptions MODULES_OVERRIDE="" + +# Default rootfs device configuration, should be changed to suit target board +options ROOTDEVNAME=\""ufs:md0.uzip\" + +# Support geom_uzip(4) compressed disk images +device geom_map +options GEOM_UZIP + +# Support md(4) and md-based rootfs +device md +options MD_ROOT + +# Interrupt controller support +device mtk_intr_v1 + +# UART device support +nodevice uart_ns8250 +device uart_dev_mtk + +# SPI and SPI flash support +device mtk_spi_v1 +device spibus +device mx25l + +# CFI support +device cfi +device cfid + +# GPIO and gpioled support +device mtk_gpio_v1 +device gpio +device gpioled + +# USB (dwcotg) support +device usb +device mtk_usb_phy +device dwcotg + +# USB umass(4) storage and da(4) support +device umass +device da + +# CAM support, required if umass(4) is enabled above +device pass +device scbus + +# Ethernet, BPF and bridge support +device rt +device bpf +device if_bridge + +# Extres +options EXT_RESOURCES +device clk diff --git a/sys/mips/mediatek/mtk_gpio_v1.c b/sys/mips/mediatek/mtk_gpio_v1.c index e1443b177158f..632c7a744abf4 100644 --- a/sys/mips/mediatek/mtk_gpio_v1.c +++ b/sys/mips/mediatek/mtk_gpio_v1.c @@ -292,7 +292,8 @@ mtk_gpio_attach(device_t dev) for (i = 0; i < sc->num_pins; i++) { sc->pins[i].pin_caps |= GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | - GPIO_PIN_INVIN | GPIO_PIN_INVOUT; + GPIO_PIN_INVIN | GPIO_PIN_INVOUT | + GPIO_INTR_EDGE_RISING | GPIO_INTR_EDGE_FALLING; sc->pins[i].intr_polarity = INTR_POLARITY_HIGH; sc->pins[i].intr_trigger = INTR_TRIGGER_EDGE; @@ -500,22 +501,78 @@ out: } static int +mtk_gpio_pic_map_fdt(struct mtk_gpio_softc *sc, + struct intr_map_data_fdt *daf, u_int *irqp, uint32_t *modep) +{ + u_int irq; + + if (daf->ncells != 1) { + device_printf(sc->dev, "Invalid #interrupt-cells\n"); + return (EINVAL); + } + + irq = daf->cells[0]; + + if (irq >= sc->num_pins) { + device_printf(sc->dev, "Invalid interrupt number %u\n", irq); + return (EINVAL); + } + + *irqp = irq; + if (modep != NULL) + *modep = GPIO_INTR_EDGE_BOTH; + + return (0); +} + +static int +mtk_gpio_pic_map_gpio(struct mtk_gpio_softc *sc, + struct intr_map_data_gpio *dag, u_int *irqp, uint32_t *modep) +{ + u_int irq; + + irq = dag->gpio_pin_num; + if (irq >= sc->num_pins) { + device_printf(sc->dev, "Invalid interrupt number %u\n", irq); + return (EINVAL); + } + + *irqp = irq; + if (modep != NULL) + *modep = dag->gpio_intr_mode; + + return (0); +} + +static int mtk_gpio_pic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { - struct intr_map_data_fdt *daf; + int error; + u_int irq; struct mtk_gpio_softc *sc; - if (data->type != INTR_MAP_DATA_FDT) - return (ENOTSUP); - sc = device_get_softc(dev); - daf = (struct intr_map_data_fdt *)data; + switch (data->type) { + case INTR_MAP_DATA_FDT: + error = (mtk_gpio_pic_map_fdt(sc, + (struct intr_map_data_fdt *)data, &irq, NULL)); + break; + case INTR_MAP_DATA_GPIO: + error = (mtk_gpio_pic_map_gpio(sc, + (struct intr_map_data_gpio *)data, &irq, NULL)); + break; + default: + error = EINVAL; + break; + } - if (daf->ncells != 1 || daf->cells[0] >= sc->num_pins) - return (EINVAL); + if (error != 0) { + device_printf(dev, "Invalid map type\n"); + return (error); + } - *isrcp = PIC_INTR_ISRC(sc, daf->cells[0]); + *isrcp = PIC_INTR_ISRC(sc, irq); return (0); } @@ -600,6 +657,51 @@ mtk_gpio_pic_post_filter(device_t dev, struct intr_irqsrc *isrc) } static int +mtk_gpio_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + struct mtk_gpio_softc *sc; + uint32_t val; + int error; + uint32_t mode; + u_int irq; + + if (data == NULL) + return (ENOTSUP); + + sc = device_get_softc(dev); + + switch (data->type) { + case INTR_MAP_DATA_FDT: + error = mtk_gpio_pic_map_fdt(sc, + (struct intr_map_data_fdt *)data, &irq, &mode); + break; + case INTR_MAP_DATA_GPIO: + error = mtk_gpio_pic_map_gpio(sc, + (struct intr_map_data_gpio *)data, &irq, &mode); + break; + default: + error = ENOTSUP; + break; + } + + if (error != 0) + return (error); + + MTK_GPIO_LOCK(sc); + if (mode == GPIO_INTR_EDGE_BOTH || mode == GPIO_INTR_EDGE_RISING) { + val = MTK_READ_4(sc, GPIO_PIORENA) | (1u << irq); + MTK_WRITE_4(sc, GPIO_PIORENA, val); + } + if (mode == GPIO_INTR_EDGE_BOTH || mode == GPIO_INTR_EDGE_FALLING) { + val = MTK_READ_4(sc, GPIO_PIOFENA) | (1u << irq); + MTK_WRITE_4(sc, GPIO_PIOFENA, val); + } + MTK_GPIO_UNLOCK(sc); + return (0); +} + +static int mtk_gpio_intr(void *arg) { struct mtk_gpio_softc *sc; @@ -607,6 +709,7 @@ mtk_gpio_intr(void *arg) sc = arg; interrupts = MTK_READ_4(sc, GPIO_PIOINT); + MTK_WRITE_4(sc, GPIO_PIOINT, interrupts); for (i = 0; interrupts != 0; i++, interrupts >>= 1) { if ((interrupts & 0x1) == 0) @@ -649,6 +752,7 @@ static device_method_t mtk_gpio_methods[] = { DEVMETHOD(pic_disable_intr, mtk_gpio_pic_disable_intr), DEVMETHOD(pic_enable_intr, mtk_gpio_pic_enable_intr), DEVMETHOD(pic_map_intr, mtk_gpio_pic_map_intr), + DEVMETHOD(pic_setup_intr, mtk_gpio_pic_setup_intr), DEVMETHOD(pic_post_filter, mtk_gpio_pic_post_filter), DEVMETHOD(pic_post_ithread, mtk_gpio_pic_post_ithread), DEVMETHOD(pic_pre_ithread, mtk_gpio_pic_pre_ithread), diff --git a/sys/mips/mediatek/mtk_machdep.c b/sys/mips/mediatek/mtk_machdep.c index aaae78042662d..652dfa27d718d 100644 --- a/sys/mips/mediatek/mtk_machdep.c +++ b/sys/mips/mediatek/mtk_machdep.c @@ -146,7 +146,11 @@ mips_init(void) ctob(physmem) / (1024 * 1024)); } - if (ctob(physmem) < (448 * 1024 * 1024)) { + if (mtk_soc_get_socid() == MTK_SOC_RT2880) { + /* RT2880 memory start is 88000000 */ + dump_avail[1] = phys_avail[1] = ctob(physmem) + + 0x08000000; + } else if (ctob(physmem) < (448 * 1024 * 1024)) { /* * Anything up to 448MB is assumed to be directly * mappable as low memory... diff --git a/sys/mips/mediatek/mtk_soc.c b/sys/mips/mediatek/mtk_soc.c index 62e01c805633d..49db97e67610e 100644 --- a/sys/mips/mediatek/mtk_soc.c +++ b/sys/mips/mediatek/mtk_soc.c @@ -53,6 +53,7 @@ static uint32_t mtk_soc_cpuclk = MTK_CPU_CLK_880MHZ; static uint32_t mtk_soc_timerclk = MTK_CPU_CLK_880MHZ / 2; static const struct ofw_compat_data compat_data[] = { + { "ralink,rt2880-soc", MTK_SOC_RT2880 }, { "ralink,rt3050-soc", MTK_SOC_RT3050 }, { "ralink,rt3052-soc", MTK_SOC_RT3052 }, { "ralink,rt3350-soc", MTK_SOC_RT3350 }, @@ -77,6 +78,30 @@ static const struct ofw_compat_data compat_data[] = { }; static uint32_t +mtk_detect_cpuclk_rt2880(bus_space_tag_t bst, bus_space_handle_t bsh) +{ + uint32_t val; + + val = bus_space_read_4(bst, bsh, SYSCTL_SYSCFG); + val >>= RT2880_CPU_CLKSEL_OFF; + val &= RT2880_CPU_CLKSEL_MSK; + + switch (val) { + case 0: + return (MTK_CPU_CLK_250MHZ); + case 1: + return (MTK_CPU_CLK_266MHZ); + case 2: + return (MTK_CPU_CLK_280MHZ); + case 3: + return (MTK_CPU_CLK_300MHZ); + } + + /* Never reached */ + return (0); +} + +static uint32_t mtk_detect_cpuclk_rt305x(bus_space_tag_t bst, bus_space_handle_t bsh) { uint32_t val; @@ -260,7 +285,9 @@ mtk_soc_try_early_detect(void) } bst = fdtbus_bs_tag; - if (mtk_soc_socid == MTK_SOC_MT7621) + if (mtk_soc_socid == MTK_SOC_RT2880) + base = MTK_RT2880_BASE; + else if (mtk_soc_socid == MTK_SOC_MT7621) base = MTK_MT7621_BASE; else base = MTK_DEFAULT_BASE; @@ -270,6 +297,9 @@ mtk_soc_try_early_detect(void) /* First, figure out the CPU clock */ switch (mtk_soc_socid) { + case MTK_SOC_RT2880: + mtk_soc_cpuclk = mtk_detect_cpuclk_rt2880(bst, bsh); + break; case MTK_SOC_RT3050: /* fallthrough */ case MTK_SOC_RT3052: case MTK_SOC_RT3350: @@ -327,6 +357,9 @@ mtk_soc_try_early_detect(void) } switch (mtk_soc_socid) { + case MTK_SOC_RT2880: + mtk_soc_uartclk = mtk_soc_cpuclk / MTK_UARTDIV_2; + break; case MTK_SOC_RT3350: /* fallthrough */ case MTK_SOC_RT3050: /* fallthrough */ case MTK_SOC_RT3052: diff --git a/sys/mips/mediatek/mtk_soc.h b/sys/mips/mediatek/mtk_soc.h index 445675df61196..44f9df07007a0 100644 --- a/sys/mips/mediatek/mtk_soc.h +++ b/sys/mips/mediatek/mtk_soc.h @@ -32,6 +32,7 @@ enum mtk_soc_id { MTK_SOC_UNKNOWN, + MTK_SOC_RT2880, MTK_SOC_RT3050, MTK_SOC_RT3052, MTK_SOC_RT3350, @@ -47,6 +48,8 @@ enum mtk_soc_id { MTK_SOC_MAX }; +#define RT2880_CPU_CLKSEL_OFF 20 +#define RT2880_CPU_CLKSEL_MSK 0x3 #define RT305X_CPU_CLKSEL_OFF 18 #define RT305X_CPU_CLKSEL_MSK 0x1 #define RT3352_CPU_CLKSEL_OFF 8 @@ -91,7 +94,10 @@ enum mtk_soc_id { #define MTK_MHZ(x) ((x) * 1000 * 1000) #define MTK_CPU_CLK_UNKNOWN 0 +#define MTK_CPU_CLK_233MHZ 233333333 #define MTK_CPU_CLK_250MHZ 250000000 +#define MTK_CPU_CLK_266MHZ 266666666 +#define MTK_CPU_CLK_280MHZ 280000000 #define MTK_CPU_CLK_300MHZ 300000000 #define MTK_CPU_CLK_320MHZ 320000000 #define MTK_CPU_CLK_360MHZ 360000000 @@ -111,6 +117,7 @@ enum mtk_soc_id { #define MTK_UARTDIV_3 3 #define MTK_DEFAULT_BASE 0x10000000 +#define MTK_RT2880_BASE 0x00300000 #define MTK_MT7621_BASE 0x1e000000 #define MTK_DEFAULT_SIZE 0x6000 diff --git a/sys/mips/mediatek/std.rt2880 b/sys/mips/mediatek/std.rt2880 new file mode 100644 index 0000000000000..8040585e9cb94 --- /dev/null +++ b/sys/mips/mediatek/std.rt2880 @@ -0,0 +1,89 @@ +# +# std.rt2880 -- Base kernel configuration file for FreeBSD/MIPS RT2800 SoC +# +# This includes all the required drivers for the SoCs. +# +# $FreeBSD$ +# + +# Include the standard file list for Mediatek SoCs. +files "../mediatek/files.mediatek" + +# Building a mips/mipsel kernel +machine mips mipsel + +# Little-endian machine +makeoptions MIPS_LITTLE_ENDIAN=defined + +# Default kernel load address +makeoptions KERNLOADADDR=0x88001000 + +# Mediatek/Ralink SoC support depends on FDT (with static DTB for the moment) +options FDT +options FDT_DTB_STATIC + +# We rely on INTRNG code +options INTRNG +options MIPS_NIRQ=256 + +# We rely on NEW_PCIB code +options NEW_PCIB + +# Build kernel with gdb(1) debug symbols +makeoptions DEBUG=-g + +# Support for DDB and KDB +options DDB +options KDB + +# Debugging for use in -current +options INVARIANTS +options INVARIANT_SUPPORT +options WITNESS +options WITNESS_SKIPSPIN +options DEBUG_REDZONE +options DEBUG_MEMGUARD + +# For small memory footprints +options VM_KMEM_SIZE_SCALE=1 + +# General options, including scheduler, etc. +options SCHED_ULE # ULE scheduler +options INET # InterNETworking +#options INET6 # IPv6 +options PSEUDOFS # Pseude-filesystem framework +options FFS # Berkeley Fast Filesystem +#options SOFTUPDATES # Enable FFS soft updates support +#options UFS_ACL # Support for access control lists +#options UFS_DIRHASH # Improve big directory performance +#options MSDOSFS # Enable support for MSDOS filesystems +options _KPOSIX_PRIORITY_SCHEDULING # Posix P1003_1B real-time ext. + +# +# Standard drivers section +# +# The drivers in the following section are required in order to successfully +# compile the kernel. +# + +# FDT clock and pinctrl framework +device fdt_clock +device fdt_pinctrl + +# UART support +device uart + +# random support +device random + +# loop device support +device loop + +# ether device support +device ether + +# ether switch support +#device etherswitch +#device miibus +#device ip17x +#device mdio diff --git a/sys/modules/cxgbe/tom/Makefile b/sys/modules/cxgbe/tom/Makefile index 96daf2c441a87..f7abba94b96eb 100644 --- a/sys/modules/cxgbe/tom/Makefile +++ b/sys/modules/cxgbe/tom/Makefile @@ -10,6 +10,7 @@ SRCS= bus_if.h SRCS+= device_if.h SRCS+= opt_inet.h SRCS+= opt_inet6.h +SRCS+= opt_ratelimit.h SRCS+= pci_if.h SRCS+= t4_connect.c SRCS+= t4_cpl_io.c diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c index f76ddf348dbee..0f08a8de38529 100644 --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -553,6 +553,7 @@ lagg_clone_destroy(struct ifnet *ifp) LAGG_WLOCK(sc); lagg_proto_detach(sc); LAGG_UNLOCK_ASSERT(sc); + LAGG_XUNLOCK(sc); ifmedia_removeall(&sc->sc_media); ether_ifdetach(ifp); @@ -561,7 +562,6 @@ lagg_clone_destroy(struct ifnet *ifp) LAGG_LIST_LOCK(); SLIST_REMOVE(&V_lagg_list, sc, lagg_softc, sc_entries); LAGG_LIST_UNLOCK(); - LAGG_XUNLOCK(sc); LAGG_SX_DESTROY(sc); LAGG_LOCK_DESTROY(sc); diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c index b655bf69d8d2b..4416e05191a68 100644 --- a/sys/netpfil/pf/pf_ioctl.c +++ b/sys/netpfil/pf/pf_ioctl.c @@ -3712,17 +3712,8 @@ dehook_pf(void) static void pf_load_vnet(void) { - VNET_ITERATOR_DECL(vnet_iter); - - VNET_LIST_RLOCK(); - VNET_FOREACH(vnet_iter) { - CURVNET_SET(vnet_iter); - V_pf_pfil_hooked = 0; - TAILQ_INIT(&V_pf_tags); - TAILQ_INIT(&V_pf_qids); - CURVNET_RESTORE(); - } - VNET_LIST_RUNLOCK(); + TAILQ_INIT(&V_pf_tags); + TAILQ_INIT(&V_pf_qids); pfattach_vnet(); V_pf_vnet_active = 1; diff --git a/sys/rpc/clnt_vc.c b/sys/rpc/clnt_vc.c index 98bead5fd71eb..a21c1df74a438 100644 --- a/sys/rpc/clnt_vc.c +++ b/sys/rpc/clnt_vc.c @@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> +#include <sys/kernel.h> #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mbuf.h> @@ -107,6 +108,8 @@ static struct clnt_ops clnt_vc_ops = { static void clnt_vc_upcallsdone(struct ct_data *); +static int fake_wchan; + /* * Create a client handle for a connection. * Default options are set, which the user can change using clnt_control()'s. @@ -298,7 +301,7 @@ clnt_vc_call( uint32_t xid; struct mbuf *mreq = NULL, *results; struct ct_request *cr; - int error; + int error, trycnt; cr = malloc(sizeof(struct ct_request), M_RPC, M_WAITOK); @@ -328,8 +331,20 @@ clnt_vc_call( timeout = ct->ct_wait; /* use default timeout */ } + /* + * After 15sec of looping, allow it to return RPC_CANTSEND, which will + * cause the clnt_reconnect layer to create a new TCP connection. + */ + trycnt = 15 * hz; call_again: mtx_assert(&ct->ct_lock, MA_OWNED); + if (ct->ct_closing || ct->ct_closed) { + ct->ct_threads--; + wakeup(ct); + mtx_unlock(&ct->ct_lock); + free(cr, M_RPC); + return (RPC_CANTSEND); + } ct->ct_xid++; xid = ct->ct_xid; @@ -397,13 +412,16 @@ call_again: */ error = sosend(ct->ct_socket, NULL, NULL, mreq, NULL, 0, curthread); mreq = NULL; - if (error == EMSGSIZE) { + if (error == EMSGSIZE || (error == ERESTART && + (ct->ct_waitflag & PCATCH) == 0 && trycnt-- > 0)) { SOCKBUF_LOCK(&ct->ct_socket->so_snd); sbwait(&ct->ct_socket->so_snd); SOCKBUF_UNLOCK(&ct->ct_socket->so_snd); AUTH_VALIDATE(auth, xid, NULL, NULL); mtx_lock(&ct->ct_lock); TAILQ_REMOVE(&ct->ct_pending, cr, cr_link); + /* Sleep for 1 clock tick before trying the sosend() again. */ + msleep(&fake_wchan, &ct->ct_lock, 0, "rpclpsnd", 1); goto call_again; } diff --git a/sys/sys/syscallsubr.h b/sys/sys/syscallsubr.h index 6776c4c3dc332..d4cfa26045e9e 100644 --- a/sys/sys/syscallsubr.h +++ b/sys/sys/syscallsubr.h @@ -249,6 +249,8 @@ int kern_sigprocmask(struct thread *td, int how, int kern_sigsuspend(struct thread *td, sigset_t mask); int kern_sigtimedwait(struct thread *td, sigset_t waitset, struct ksiginfo *ksi, struct timespec *timeout); +int kern_sigqueue(struct thread *td, pid_t pid, int signum, + union sigval *value); int kern_socket(struct thread *td, int domain, int type, int protocol); int kern_statat(struct thread *td, int flag, int fd, char *path, enum uio_seg pathseg, struct stat *sbp, diff --git a/sys/ufs/ffs/ffs_rawread.c b/sys/ufs/ffs/ffs_rawread.c index e1fc3465525ef..0868f2ab932bc 100644 --- a/sys/ufs/ffs/ffs_rawread.c +++ b/sys/ufs/ffs/ffs_rawread.c @@ -270,7 +270,6 @@ ffs_rawread_main(struct vnode *vp, int error, nerror; struct buf *bp, *nbp, *tbp; u_int iolen; - int spl; caddr_t udata; long resid; off_t offset; @@ -330,10 +329,7 @@ ffs_rawread_main(struct vnode *vp, } } - spl = splbio(); bwait(bp, PRIBIO, "rawrd"); - splx(spl); - vunmapbuf(bp); iolen = bp->b_bcount - bp->b_resid; @@ -400,9 +396,7 @@ ffs_rawread_main(struct vnode *vp, relpbuf(bp, &ffsrawbufcnt); } if (nbp != NULL) { /* Run down readahead buffer */ - spl = splbio(); bwait(nbp, PRIBIO, "rawrd"); - splx(spl); vunmapbuf(nbp); pbrelvp(nbp); relpbuf(nbp, &ffsrawbufcnt); diff --git a/sys/ufs/ffs/fs.h b/sys/ufs/ffs/fs.h index c87c2b8f436ea..4d08210cb5229 100644 --- a/sys/ufs/ffs/fs.h +++ b/sys/ufs/ffs/fs.h @@ -608,7 +608,8 @@ struct cg { * Determining the size of a file block in the filesystem. */ #define blksize(fs, ip, lbn) \ - (((lbn) >= UFS_NDADDR || (ip)->i_size >= smalllblktosize(fs, (lbn) + 1)) \ + (((lbn) >= UFS_NDADDR || (ip)->i_size >= \ + (uint64_t)smalllblktosize(fs, (lbn) + 1)) \ ? (fs)->fs_bsize \ : (fragroundup(fs, blkoff(fs, (ip)->i_size)))) #define sblksize(fs, size, lbn) \ diff --git a/usr.bin/calendar/calendars/calendar.freebsd b/usr.bin/calendar/calendars/calendar.freebsd index b98a65bcfbb06..ab3a5a271c121 100644 --- a/usr.bin/calendar/calendars/calendar.freebsd +++ b/usr.bin/calendar/calendars/calendar.freebsd @@ -145,6 +145,7 @@ 04/29 Adam Weinberger <adamw@FreeBSD.org> born in Berkeley, California, United States, 1980 04/29 Eric Anholt <anholt@FreeBSD.org> born in Portland, Oregon, United States, 1983 05/01 Randall Stewart <rrs@FreeBSD.org> born in Spokane, Washington, United States, 1959 +05/02 Danilo G. Baio <dbaio@FreeBSD.org> born in Maringa, Parana, Brazil, 1986 05/02 Wojciech A. Koszek <wkoszek@FreeBSD.org> born in Czestochowa, Poland, 1987 05/03 Brian Dean <bsd@FreeBSD.org> born in Elkins, West Virginia, United States, 1966 05/03 Patrick Kelsey <pkelsey@FreeBSD.org> born in Freehold, New Jersey, United States, 1976 diff --git a/usr.bin/csplit/Makefile b/usr.bin/csplit/Makefile index 3f370c7c4ad31..52b2016b81390 100644 --- a/usr.bin/csplit/Makefile +++ b/usr.bin/csplit/Makefile @@ -1,5 +1,11 @@ # $FreeBSD$ +.include <src.opts.mk> + PROG= csplit +.if ${MK_TESTS} != "no" +SUBDIR+= tests +.endif + .include <bsd.prog.mk> diff --git a/usr.bin/csplit/csplit.c b/usr.bin/csplit/csplit.c index b23c85d23a6d5..0adecaf82c41f 100644 --- a/usr.bin/csplit/csplit.c +++ b/usr.bin/csplit/csplit.c @@ -399,8 +399,10 @@ do_rexp(const char *expr) first = 0; } - if (p == NULL) + if (p == NULL) { + toomuch(NULL, 0); errx(1, "%s: no match", re); + } if (ofs <= 0) { /* diff --git a/usr.bin/csplit/tests/Makefile b/usr.bin/csplit/tests/Makefile new file mode 100644 index 0000000000000..5403cc85d81ff --- /dev/null +++ b/usr.bin/csplit/tests/Makefile @@ -0,0 +1,7 @@ +# $FreeBSD$ + +PACKAGE= tests + +ATF_TESTS_SH= csplit_test + +.include <bsd.test.mk> diff --git a/usr.bin/csplit/tests/csplit_test.sh b/usr.bin/csplit/tests/csplit_test.sh new file mode 100755 index 0000000000000..a47ecd79f014f --- /dev/null +++ b/usr.bin/csplit/tests/csplit_test.sh @@ -0,0 +1,60 @@ +# Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +atf_test_case lines_lt_count +lines_lt_count_head() +{ + atf_set "descr" \ + "Test an edge case where input has fewer lines than count" +} +lines_lt_count_body() +{ + cat > expectfile00 << HERE +one +two +HERE + cat > expectfile01 << HERE +xxx 1 +three +four +HERE + cat > expectfile02 << HERE +xxx 2 +five +six +HERE + echo -e "one\ntwo\nxxx 1\nthree\nfour\nxxx 2\nfive\nsix" | \ + csplit -k - '/xxx/' '{10}' + + atf_check cmp expectfile00 xx00 + atf_check cmp expectfile01 xx01 + atf_check cmp expectfile02 xx02 +} + +atf_init_test_cases() +{ + atf_add_test_case lines_lt_count +} diff --git a/usr.bin/grep/util.c b/usr.bin/grep/util.c index bd2f8acf83a75..cfcc72d8cbd66 100644 --- a/usr.bin/grep/util.c +++ b/usr.bin/grep/util.c @@ -201,7 +201,7 @@ procfile(const char *fn) struct str *ln; mode_t s; int c, last_outed, t, tail; - bool doctx, same_file; + bool doctx, printmatch, same_file; if (strcmp(fn, "-") == 0) { fn = label != NULL ? label : getstr(1); @@ -237,12 +237,14 @@ procfile(const char *fn) last_outed = 0; same_file = false; doctx = false; - if ((!pc.binary || binbehave != BINFILE_BIN) && !cflag && !qflag && - !lflag && !Lflag && (Aflag != 0 || Bflag != 0)) + printmatch = true; + if ((pc.binary && binbehave == BINFILE_BIN) || cflag || qflag || + lflag || Lflag) + printmatch = false; + if (printmatch && (Aflag != 0 || Bflag != 0)) doctx = true; mcount = mlimit; - for (c = 0; c == 0 || !(lflag || qflag); ) { /* Reset match count for every line processed */ pc.matchidx = 0; @@ -283,7 +285,7 @@ procfile(const char *fn) tail = Aflag; } /* Print the matching line, but only if not quiet/binary */ - if (t == 0 && !qflag && !pc.binary) { + if (t == 0 && printmatch) { printline(&pc, ':'); first_match = false; same_file = true; diff --git a/usr.bin/less/defines.h b/usr.bin/less/defines.h index 9b0f5c92b25e6..6cc483a3c13fc 100644 --- a/usr.bin/less/defines.h +++ b/usr.bin/less/defines.h @@ -136,6 +136,11 @@ #define TGETENT_OK 1 /* + * HAVE_ANSI_PROTOS is 1 if your compiler supports ANSI function prototypes. + */ +#define HAVE_ANSI_PROTOS 1 + +/* * HAVE_SYS_TYPES_H is 1 if your system has <sys/types.h>. */ #define HAVE_SYS_TYPES_H 1 diff --git a/usr.bin/mt/mt.1 b/usr.bin/mt/mt.1 index d62f5e8f8a015..7d1b8785d4f90 100644 --- a/usr.bin/mt/mt.1 +++ b/usr.bin/mt/mt.1 @@ -29,7 +29,7 @@ .\" @(#)mt.1 8.1 (Berkeley) 6/6/93 .\" $FreeBSD$ .\" -.Dd May 20, 2016 +.Dd May 5, 2017 .Dt MT 1 .Os .Sh NAME @@ -284,6 +284,9 @@ One of or .Fl x must be specified to indicate which operation to perform. +See +.Xr sa 4 +for more detailed information on the parameters. .Bl -tag -width 8n .It Fl l List parameters, values and descriptions. diff --git a/usr.bin/resizewin/resizewin.1 b/usr.bin/resizewin/resizewin.1 index b7376c79026e9..ce15ce70283ed 100644 --- a/usr.bin/resizewin/resizewin.1 +++ b/usr.bin/resizewin/resizewin.1 @@ -27,17 +27,27 @@ .\" .\" $FreeBSD$ .\" -.Dd July 9, 2016 +.Dd May 8, 2017 .Dt RESIZEWIN 1 .Os .Sh NAME .Nm resizewin .Nd update the kernel window size for the current TTY +.Sh SYNOPSIS +.Nm +.Op Fl z .Sh DESCRIPTION Query the terminal emulator window size with the .Dv TIOCSWINSZ ioctl and set the window size known by the kernel to the new values. The terminal is assumed to be VT100/ANSI compatible. +.Pp +The following options are available: +.Bl -tag -width ".Fl z" +.It Fl z +Do nothing unless the current kernel terminal size is zero. +.El +.Pp .Nm is functionally similar to .Xr resize 1 , diff --git a/usr.bin/resizewin/resizewin.c b/usr.bin/resizewin/resizewin.c index 5399ad33a0e41..d86189ffe7283 100644 --- a/usr.bin/resizewin/resizewin.c +++ b/usr.bin/resizewin/resizewin.c @@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$"); #include <sys/ioctl.h> #include <sys/time.h> +#include <err.h> #include <errno.h> #include <fcntl.h> #include <stdio.h> @@ -46,20 +47,50 @@ static const char query[] = "\033[999;999H" /* Move cursor */ "\033[6n" /* Get cursor position */ "\0338"; /* Restore cursor position */ + +static void +usage(void) +{ + + fprintf(stderr, "usage: resizewin [-z]\n"); + exit(1); +} + int -main(__unused int argc, __unused char **argv) +main(int argc, char **argv) { struct termios old, new; struct winsize w; - int ret, fd, cnt, err; - char data[20]; struct timeval then, now; + char data[20]; + int ch, cnt, error, fd, ret, zflag; - err = 0; + error = 0; + zflag = 0; + while ((ch = getopt(argc, argv, "z")) != -1) { + switch (ch) { + case 'z': + zflag = 1; + break; + case '?': + default: + usage(); + } + } + argc -= optind; + if (argc != 0) + usage(); if ((fd = open("/dev/tty", O_RDWR | O_NONBLOCK)) == -1) exit(1); + if (zflag) { + if (ioctl(fd, TIOCGWINSZ, &w) == -1) + exit(1); + if (w.ws_row != 0 && w.ws_col != 0) + exit(0); + } + /* Disable echo */ if (tcgetattr(fd, &old) == -1) exit(1); @@ -70,8 +101,13 @@ main(__unused int argc, __unused char **argv) if (tcsetattr(fd, TCSANOW, &new) == -1) exit(1); + /* Discard input received so far */ + error = tcflush(fd, TCIOFLUSH); + if (error != 0) + warn("tcflush"); + if (write(fd, query, sizeof(query)) != sizeof(query)) { - err = 1; + error = 1; goto out; } @@ -87,16 +123,15 @@ main(__unused int argc, __unused char **argv) gettimeofday(&now, NULL); timersub(&now, &then, &now); if (now.tv_sec >= 2) { - fprintf(stderr, "\n\n\nTimeout reading from terminal\n"); - fprintf(stderr, "Read %d bytes, %s\n", cnt, data); - err = 1; + warnx("timeout reading from terminal"); + error = 1; goto out; } usleep(20000); continue; } - err = 1; + error = 1; goto out; } if (data[cnt] == 'R') @@ -104,26 +139,26 @@ main(__unused int argc, __unused char **argv) cnt++; if (cnt == sizeof(data) - 2) { - fprintf(stderr, "Response too long\n"); - err = 1; + warnx("response too long"); + error = 1; goto out; } } /* Parse */ if (sscanf(data, "\033[%hu;%huR", &w.ws_row, &w.ws_col) != 2) { - err = 1; - fprintf(stderr, "Unable to parse response\n"); + error = 1; + warnx("unable to parse response"); goto out; } /* Finally, what we want */ if (ioctl(fd, TIOCSWINSZ, &w) == -1) - err = 1; + error = 1; out: /* Restore echo */ tcsetattr(fd, TCSANOW, &old); close(fd); - exit(err); + exit(error); } diff --git a/usr.bin/zstd/Makefile b/usr.bin/zstd/Makefile index 224123488108b..8499b65a0dde7 100644 --- a/usr.bin/zstd/Makefile +++ b/usr.bin/zstd/Makefile @@ -12,12 +12,16 @@ CFLAGS+= -I${SRCTOP}/contrib/zstd/programs \ -I${SRCTOP}/contrib/zstd/lib/compress \ -I${SRCTOP}/contrib/zstd/lib/dictBuilder \ -I${SRCTOP}/contrib/zstd/lib \ - -DXXH_NAMESPACE=ZSTD_ + -DXXH_NAMESPACE=ZSTD_ \ + -DHAVE_THREAD=1 \ + -DZSTD_MULTITHREAD=1 SCRIPTS= zstdgrep LINKS= ${BINDIR}/zstd ${BINDIR}/unzstd \ - ${BINDIR}/zstd ${BINDIR}/zstdcat + ${BINDIR}/zstd ${BINDIR}/zstdcat \ + ${BINDIR}/zstd ${BINDIR}/zstdmt MLINKS= zstd.1 unzstd.1 \ - zstd.1 zstdcat.1 + zstd.1 zstdcat.1 \ + zstd.1 zstdmt.1 WARNS= 2 LIBADD= zstd diff --git a/usr.sbin/makefs/cd9660/cd9660_eltorito.c b/usr.sbin/makefs/cd9660/cd9660_eltorito.c index 259a91ddc7a4f..6b514a07106be 100644 --- a/usr.sbin/makefs/cd9660/cd9660_eltorito.c +++ b/usr.sbin/makefs/cd9660/cd9660_eltorito.c @@ -224,7 +224,7 @@ cd9660_boot_setup_validation_entry(char sys) boot_catalog_validation_entry *ve; int16_t checksum; unsigned char *csptr; - int i; + size_t i; entry = cd9660_init_boot_catalog_entry(); ve = &entry->entry_data.VE; diff --git a/usr.sbin/makefs/ffs.c b/usr.sbin/makefs/ffs.c index 8c6d12b4f2015..49fadd3708e47 100644 --- a/usr.sbin/makefs/ffs.c +++ b/usr.sbin/makefs/ffs.c @@ -143,7 +143,7 @@ static void *ffs_build_dinode2(struct ufs2_dinode *, dirbuf_t *, fsnode *, fsnode *, fsinfo_t *); - +int sectorsize; /* XXX: for buf.c::getblk() */ /* publicly visible functions */ void @@ -426,6 +426,8 @@ ffs_validate(const char *dir, fsnode *root, fsinfo_t *fsopts) printf("ffs_validate: dir %s; %lld bytes, %lld inodes\n", dir, (long long)fsopts->size, (long long)fsopts->inodes); } + sectorsize = fsopts->sectorsize; /* XXX - see earlier */ + /* now check calculated sizes vs requested sizes */ if (fsopts->maxsize > 0 && fsopts->size > fsopts->maxsize) { errx(1, "`%s' size of %lld is larger than the maxsize of %lld.", @@ -621,7 +623,7 @@ ffs_size_dir(fsnode *root, fsinfo_t *fsopts) if (node->type == S_IFREG) ADDSIZE(node->inode->st.st_size); if (node->type == S_IFLNK) { - int slen; + size_t slen; slen = strlen(node->symlink) + 1; if (slen >= (ffs_opts->version == 1 ? @@ -644,7 +646,7 @@ static void * ffs_build_dinode1(struct ufs1_dinode *dinp, dirbuf_t *dbufp, fsnode *cur, fsnode *root, fsinfo_t *fsopts) { - int slen; + size_t slen; void *membuf; struct stat *st = stampst.st_ino != 0 ? &stampst : &cur->inode->st; @@ -692,7 +694,7 @@ static void * ffs_build_dinode2(struct ufs2_dinode *dinp, dirbuf_t *dbufp, fsnode *cur, fsnode *root, fsinfo_t *fsopts) { - int slen; + size_t slen; void *membuf; struct stat *st = stampst.st_ino != 0 ? &stampst : &cur->inode->st; @@ -846,8 +848,8 @@ ffs_populate_dir(const char *dir, fsnode *root, fsinfo_t *fsopts) for (cur = root; cur != NULL; cur = cur->next) { if (cur->child == NULL) continue; - if (snprintf(path, sizeof(path), "%s/%s", dir, cur->name) - >= sizeof(path)) + if ((size_t)snprintf(path, sizeof(path), "%s/%s", dir, + cur->name) >= sizeof(path)) errx(1, "Pathname too long."); if (! ffs_populate_dir(path, cur->child, fsopts)) return (0); diff --git a/usr.sbin/makefs/ffs/buf.c b/usr.sbin/makefs/ffs/buf.c index 7b08d537d4b98..0ca2824545e30 100644 --- a/usr.sbin/makefs/ffs/buf.c +++ b/usr.sbin/makefs/ffs/buf.c @@ -52,6 +52,8 @@ __FBSDID("$FreeBSD$"); #include "makefs.h" #include "buf.h" +extern int sectorsize; /* XXX: from ffs.c & mkfs.c */ + static TAILQ_HEAD(buftailhead,buf) buftail; int @@ -60,7 +62,6 @@ bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *u1 __unused, { off_t offset; ssize_t rv; - fsinfo_t *fs = vp->fs; assert (bpp != NULL); @@ -68,7 +69,7 @@ bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *u1 __unused, printf("%s: blkno %lld size %d\n", __func__, (long long)blkno, size); *bpp = getblk(vp, blkno, size, 0, 0, 0); - offset = (*bpp)->b_blkno * fs->sectorsize; + offset = (*bpp)->b_blkno * sectorsize; /* XXX */ if (debug & DEBUG_BUF_BREAD) printf("%s: blkno %lld offset %lld bcount %ld\n", __func__, (long long)(*bpp)->b_blkno, (long long) offset, @@ -125,10 +126,9 @@ bwrite(struct buf *bp) { off_t offset; ssize_t rv; - fsinfo_t *fs = bp->b_fs; assert (bp != NULL); - offset = bp->b_blkno * fs->sectorsize; + offset = bp->b_blkno * sectorsize; /* XXX */ if (debug & DEBUG_BUF_BWRITE) printf("bwrite: blkno %lld offset %lld bcount %ld\n", (long long)bp->b_blkno, (long long) offset, diff --git a/usr.sbin/makefs/walk.c b/usr.sbin/makefs/walk.c index 27d5352bacb9f..6232d69f73edb 100644 --- a/usr.sbin/makefs/walk.c +++ b/usr.sbin/makefs/walk.c @@ -79,13 +79,14 @@ walk_dir(const char *root, const char *dir, fsnode *parent, fsnode *join) char path[MAXPATHLEN + 1]; struct stat stbuf; char *name, *rp; - int dot, len; + size_t len; + int dot; assert(root != NULL); assert(dir != NULL); len = snprintf(path, sizeof(path), "%s/%s", root, dir); - if (len >= (int)sizeof(path)) + if (len >= sizeof(path)) errx(1, "Pathname too long."); if (debug & DEBUG_WALK_DIR) printf("walk_dir: %s %p\n", path, parent); @@ -119,8 +120,8 @@ walk_dir(const char *root, const char *dir, fsnode *parent, fsnode *join) } if (debug & DEBUG_WALK_DIR_NODE) printf("scanning %s/%s/%s\n", root, dir, name); - if (snprintf(path + len, sizeof(path) - len, "/%s", name) >= - (int)sizeof(path) - len) + if ((size_t)snprintf(path + len, sizeof(path) - len, "/%s", + name) >= sizeof(path) - len) errx(1, "Pathname too long."); if (lstat(path, &stbuf) == -1) err(1, "Can't lstat `%s'", path); @@ -396,8 +397,8 @@ apply_specdir(const char *dir, NODE *specnode, fsnode *dirnode, int speconly) if (strcmp(curnode->name, curfsnode->name) == 0) break; } - if (snprintf(path, sizeof(path), "%s/%s", - dir, curnode->name) >= sizeof(path)) + if ((size_t)snprintf(path, sizeof(path), "%s/%s", dir, + curnode->name) >= sizeof(path)) errx(1, "Pathname too long."); if (curfsnode == NULL) { /* need new entry */ struct stat stbuf; diff --git a/usr.sbin/mpsutil/mps_show.c b/usr.sbin/mpsutil/mps_show.c index 13e8883b3ffb6..882eef069aa68 100644 --- a/usr.sbin/mpsutil/mps_show.c +++ b/usr.sbin/mpsutil/mps_show.c @@ -217,6 +217,19 @@ show_iocfacts(int ac, char **av) return (errno); } + printf(" MsgVersion: %02d.%02d\n", + facts->MsgVersion >> 8, facts->MsgVersion & 0xff); + printf(" MsgLength: %d\n", facts->MsgLength); + printf(" Function: 0x%x\n", facts->Function); + printf(" HeaderVersion: %02d,%02d\n", + facts->HeaderVersion >> 8, facts->HeaderVersion & 0xff); + printf(" IOCNumber: %d\n", facts->IOCNumber); + printf(" MsgFlags: 0x%x\n", facts->MsgFlags); + printf(" VP_ID: %d\n", facts->VP_ID); + printf(" VF_ID: %d\n", facts->VF_ID); + printf(" IOCExceptions: %d\n", facts->IOCExceptions); + printf(" IOCStatus: %d\n", facts->IOCStatus); + printf(" IOCLogInfo: 0x%x\n", facts->IOCLogInfo); printf(" MaxChainDepth: %d\n", facts->MaxChainDepth); printf(" WhoInit: 0x%x\n", facts->WhoInit); printf(" NumberOfPorts: %d\n", facts->NumberOfPorts); |
