diff options
259 files changed, 31790 insertions, 14485 deletions
@@ -21,6 +21,62 @@ b92b9da33006, 13d3bd165e22: table. One can run a multicast routing daemon within each FIB; the FIB of the routing socket determines the corresponding multicast routing table. +5c2186b99254,628d7a3270b6: + When the sector size for an NVMe namespace changes, nda(4) now + reports this as a media change rather than a disk resize + event. nda(4) also no longer reports a disk resize event in + some cases when the namespace's size had not changed. nda(4) + also reprobes NVMe namespaces each time they are opened to + check for a new sector size or namespace size. + +7a323f873662: + Remove the le(4) driver. This driver supported older 10Mbps + and 100Mbps Ethernet adapters. True hardware devices have not + been used in FreeBSD systems in many years, but the driver was + previously retained for use with MIPS QEMU virtual machines. + +e51ef8ae490f: + Add initial support for instruction-based sampling (IBS) + performance counters on AMD processors to hwpmc(4). + +d69fc3a9dc71: + The EFI boot loader now tries all ZFS pools found during + probing when looking for a boot pool. Previously it would try + only the first pool found. + +4d5c434ed16e: + Use the BSD-licensed implementation of diff3 instead of GNU diff3 + by default. + +7f54c65abc67: + Update the smartpqi(4) driver to vendor version 14.4690.0.2008 + - 15.2.0.2008. This includes support for new controllers and + renaming Microsemi to Microchip. + +350c1232a57a: + nvmecontrol(8)'s telemetry-log now works reliably. + +d14e018024bb,b125c4d13095,1322760fd127,0730a05cd383: + Permit both SCHED_ULE and SCHED_4BSD to be compiled into the + same kernel. The scheduler can be selected at boot time by + setting the kern.sched_name tunable to the scheduler name + ("4BSD" or "ULE"). The default scheduler is ULE. This is + currently enabled in amd64 kernels by default. + +90a7728cd890: + Update nuageinit to request configuration on all network + interfaces. nuageinit will now issue DHCP requests on all + network interfaces and request configuration data from the + first interface which successfully obtains a lease. + +92ba9b2fe589: + Add RoCE support for Broadcom BCM576xx controllers. + +fb4b0c911951: + Add a facility to witness(4) to provide more detailed information + about lock order reversals detailing when initial lock orders were + established. + 65f5dd42f11c: sh(1) add -l option which makes sh act as a login shell and read the profile. diff --git a/bin/sh/eval.c b/bin/sh/eval.c index 0c41c5e69eea..b4c1924f04ad 100644 --- a/bin/sh/eval.c +++ b/bin/sh/eval.c @@ -154,7 +154,7 @@ evalstring(const char *s, int flags) flags &= ~EV_EXIT; any = 0; setstackmark(&smark); - setinputstring(s, 1); + setinputstring(s); while ((n = parsecmd(0)) != NEOF) { if (n != NULL && !nflag) { if (flags_exit && preadateof()) diff --git a/bin/sh/input.c b/bin/sh/input.c index e88d31be12be..c916fb29178b 100644 --- a/bin/sh/input.c +++ b/bin/sh/input.c @@ -81,6 +81,7 @@ struct parsefile { int lleft; /* number of lines left in this buffer */ const char *nextc; /* next char in buffer */ char *buf; /* input buffer */ + size_t bufsize; /* input buffer size */ struct strpush *strpush; /* for pushing strings at this level */ struct strpush basestrpush; /* so pushing one is fast */ }; @@ -93,7 +94,8 @@ const char *parsenextc; /* copy of parsefile->nextc */ static char basebuf[BUFSIZ + 1];/* buffer for top level input file */ static struct parsefile basepf = { /* top level input file */ .nextc = basebuf, - .buf = basebuf + .buf = basebuf, + .bufsize = sizeof(basebuf), }; static struct parsefile *parsefile = &basepf; /* current input file */ int whichprompt; /* 1 == PS1, 2 == PS2 */ @@ -127,52 +129,61 @@ static int preadfd(void) { int nr; - parsenextc = parsefile->buf; retry: #ifndef NO_HISTORY if (parsefile->fd == 0 && el) { - static const char *rl_cp; - static int el_len; + const char *line; - if (rl_cp == NULL) { - el_resize(el); - rl_cp = el_gets(el, &el_len); - } - if (rl_cp == NULL) - nr = el_len == 0 ? 0 : -1; - else { - nr = el_len; - if (nr > BUFSIZ) - nr = BUFSIZ; - memcpy(parsefile->buf, rl_cp, nr); - if (nr != el_len) { - el_len -= nr; - rl_cp += nr; - } else - rl_cp = NULL; + el_resize(el); + line = el_gets(el, &nr); + if (nr > 0 && parsefile->bufsize < (size_t)nr + 1) { + size_t bufsize; + + INTOFF; + if (parsefile->buf != basebuf) { + ckfree(parsefile->buf); + parsefile->buf = NULL; + parsefile->bufsize = 0; + } + bufsize = (size_t)nr + BUFSIZ + 1; + bufsize -= bufsize % BUFSIZ; + parsefile->buf = ckmalloc(bufsize); + parsefile->bufsize = bufsize; + INTON; } + if (nr > 0 && line != NULL) + memcpy(parsefile->buf, line, nr); + else + nr = nr ? -1 : 0; } else #endif - nr = read(parsefile->fd, parsefile->buf, BUFSIZ); + nr = read(parsefile->fd, parsefile->buf, parsefile->bufsize - 1); + + if (nr < 0) + switch (errno) { + int flags; - if (nr <= 0) { - if (nr < 0) { - if (errno == EINTR) - goto retry; - if (parsefile->fd == 0 && errno == EWOULDBLOCK) { - int flags = fcntl(0, F_GETFL, 0); - if (flags >= 0 && flags & O_NONBLOCK) { - flags &=~ O_NONBLOCK; - if (fcntl(0, F_SETFL, flags) >= 0) { - out2fmt_flush("sh: turning off NDELAY mode\n"); - goto retry; - } - } - } + case EINTR: + goto retry; + case EWOULDBLOCK: + if (parsefile->fd != 0) + break; + if ((flags = fcntl(0, F_GETFL, 0)) < 0) + break; + if (!(flags & O_NONBLOCK)) + break; + if (fcntl(0, F_SETFL, flags & ~O_NONBLOCK) < 0) + break; + out2fmt_flush("sh: turning off NDELAY mode\n"); + goto retry; } - nr = -1; - } + else if (nr > 0) + parsefile->buf[nr] = '\0'; + else + nr = -1; + + parsenextc = parsefile->buf; return nr; } @@ -189,7 +200,8 @@ retry: int preadbuffer(void) { - char *p, *q, *r, *end; + const char *end; + char *q, *r; char savec; while (parsefile->strpush) { @@ -208,31 +220,22 @@ preadbuffer(void) return PEOF; again: - if (parselleft <= 0) { - if ((parselleft = preadfd()) == -1) { - parselleft = parsenleft = EOF_NLEFT; - return PEOF; - } + if (parselleft <= 0 && (parselleft = preadfd()) == -1) { + parselleft = parsenleft = EOF_NLEFT; + return (PEOF); } - - p = parsefile->buf + (parsenextc - parsefile->buf); - end = p + parselleft; - *end = '\0'; - q = strchrnul(p, '\n'); - if (q != end && *q == '\0') { + end = parsenextc + parselleft; + q = strchrnul(parsenextc, '\n'); + if (*q == '\0' && q != end) { /* delete nul characters */ - for (r = q; q != end; q++) { + for (r = q++; q != end; q++) if (*q != '\0') *r++ = *q; - } - parselleft -= end - r; - if (parselleft == 0) - goto again; - end = p + parselleft; - *end = '\0'; - q = strchrnul(p, '\n'); + *r = '\0'; + parselleft = r - parsenextc; + goto again; } - if (q == end) { + if (*q == '\0') { parsenleft = parselleft; parselleft = 0; } else /* *q == '\n' */ { @@ -307,7 +310,7 @@ pushstring(const char *s, int len, struct alias *ap) INTOFF; /*out2fmt_flush("*** calling pushstring: %s, %d\n", s, len);*/ if (parsefile->strpush) { - sp = ckmalloc(sizeof (struct strpush)); + sp = ckmalloc(sizeof(struct strpush)); sp->prev = parsefile->strpush; parsefile->strpush = sp; } else @@ -391,15 +394,15 @@ setinputfile(const char *fname, int push, int verify) void setinputfd(int fd, int push) { - if (push) { + if (push) pushfile(); - parsefile->buf = ckmalloc(BUFSIZ + 1); - } if (parsefile->fd > 0) close(parsefile->fd); parsefile->fd = fd; - if (parsefile->buf == NULL) + if (parsefile->buf == NULL) { parsefile->buf = ckmalloc(BUFSIZ + 1); + parsefile->bufsize = BUFSIZ + 1; + } parselleft = parsenleft = 0; plinno = 1; } @@ -410,14 +413,12 @@ setinputfd(int fd, int push) */ void -setinputstring(const char *string, int push) +setinputstring(const char *string) { INTOFF; - if (push) - pushfile(); + pushfile(); parsenextc = string; parselleft = parsenleft = strlen(string); - parsefile->buf = NULL; plinno = 1; INTON; } @@ -434,15 +435,12 @@ pushfile(void) { struct parsefile *pf; + pf = (struct parsefile *)ckmalloc(sizeof(struct parsefile)); + *pf = (struct parsefile){ .prev = parsefile, .fd = -1 }; parsefile->nleft = parsenleft; parsefile->lleft = parselleft; parsefile->nextc = parsenextc; parsefile->linno = plinno; - pf = (struct parsefile *)ckmalloc(sizeof (struct parsefile)); - pf->prev = parsefile; - pf->fd = -1; - pf->strpush = NULL; - pf->basestrpush.prev = NULL; parsefile = pf; } diff --git a/bin/sh/input.h b/bin/sh/input.h index 70e6b06c72da..4e8992a5bf60 100644 --- a/bin/sh/input.h +++ b/bin/sh/input.h @@ -54,7 +54,7 @@ void pungetc(void); void pushstring(const char *, int, struct alias *); void setinputfile(const char *, int, int); void setinputfd(int, int); -void setinputstring(const char *, int); +void setinputstring(const char *); void popfile(void); struct parsefile *getcurrentfile(void); void popfilesupto(struct parsefile *); diff --git a/bin/sh/parser.c b/bin/sh/parser.c index 3e42d41caec4..f6ef8d807704 100644 --- a/bin/sh/parser.c +++ b/bin/sh/parser.c @@ -1167,7 +1167,7 @@ parsebackq(char *out, struct nodelist **pbqlist, INTOFF; ostr = ckmalloc(olen); memcpy(ostr, stackblock(), olen); - setinputstring(ostr, 1); + setinputstring(ostr); INTON; } nlpp = pbqlist; @@ -2368,7 +2368,7 @@ expandstr(const char *ps) if (!setjmp(jmploc.loc)) { handler = &jmploc; parser_temp = NULL; - setinputstring(ps, 1); + setinputstring(ps); doprompt = 0; readtoken1(pgetc(), DQSYNTAX, NOEOFMARK, 0); if (backquotelist != NULL) diff --git a/bin/sh/tests/builtins/Makefile b/bin/sh/tests/builtins/Makefile index 0246009cce81..9e9ee098e90a 100644 --- a/bin/sh/tests/builtins/Makefile +++ b/bin/sh/tests/builtins/Makefile @@ -94,6 +94,7 @@ ${PACKAGE}FILES+= export1.0 ${PACKAGE}FILES+= fc1.0 ${PACKAGE}FILES+= fc2.0 ${PACKAGE}FILES+= fc3.0 fc3.0.stdout fc3.0.stderr +${PACKAGE}FILES+= fc4.0 ${PACKAGE}FILES+= for1.0 ${PACKAGE}FILES+= for2.0 ${PACKAGE}FILES+= for3.0 diff --git a/bin/sh/tests/builtins/fc4.0 b/bin/sh/tests/builtins/fc4.0 new file mode 100644 index 000000000000..5213289fcd11 --- /dev/null +++ b/bin/sh/tests/builtins/fc4.0 @@ -0,0 +1,27 @@ +v=1234 +v=$v$v$v$v +v=$v$v$v$v +v=$v$v$v$v +#v=$v$v$v$v +v=beginlong$v$v$v${v}endlong +result=$(ENV= HISTFILE=/dev/null script -q /dev/null ${SH} +m -i -o emacs <<EOF +printf '%s\n' "$v" +printf 'running %s\n' fc; fc -l +EOF +) +case $result in + *'running fc'*beginlong*endlong*) ;; + *) + set -x + : result is "$result" + exit 2 +esac +result=${result#*running fc} +result=${result#*beginlong} +result=${result%endlong*} +reflected=beginlong${result}endlong +if [ "$v" != "$reflected" ]; then + set -x + : expected "$v" reflected "$reflected" + exit 3 +fi diff --git a/bin/sh/tests/builtins/read11.0 b/bin/sh/tests/builtins/read11.0 index e57dfdbc0ea4..07bd3e70644c 100644 --- a/bin/sh/tests/builtins/read11.0 +++ b/bin/sh/tests/builtins/read11.0 @@ -27,7 +27,6 @@ ts=$(date +%s%3N) read -t 0 v <&3 || r=$? te=$(date +%s%3N) kill -TERM "$!" || : -[ "$r" -gt 128 ] -[ "$(kill -l "$r")" = ALRM ] +[ "$r" -gt 128 ] && [ "$(kill -l "$r")" = ALRM ] [ $((te-ts)) -lt 250 ] [ -z "$v" ] diff --git a/bin/sh/tests/builtins/read12.0 b/bin/sh/tests/builtins/read12.0 index 3b501a56392b..4551555adfed 100644 --- a/bin/sh/tests/builtins/read12.0 +++ b/bin/sh/tests/builtins/read12.0 @@ -1,5 +1,5 @@ -# Verify that `read -t 1 v` succeeds immediately if input is available -# and times out after 1 s if not +# Verify that `read -t 3 v` succeeds immediately if input is available +# and times out after 3 s if not set -e @@ -15,7 +15,7 @@ exec 3<fifo1 v=original_value r=0 ts=$(date +%s%3N) -read -t 1 v <&3 || r=$? +read -t 3 v <&3 || r=$? te=$(date +%s%3N) [ "$r" -eq 0 ] [ $((te-ts)) -lt 250 ] @@ -24,11 +24,9 @@ te=$(date +%s%3N) v=original_value r=0 ts=$(date +%s%3N) -read -t 1 v <&3 || r=$? +read -t 3 v <&3 || r=$? te=$(date +%s%3N) kill -TERM "$!" || : -[ "$r" -gt 128 ] -[ "$(kill -l "$r")" = ALRM ] -[ $((te-ts)) -ge 1000 ] -[ $((te-ts)) -lt 1250 ] +[ "$r" -gt 128 ] && [ "$(kill -l "$r")" = ALRM ] +[ $((te-ts)) -gt 3000 ] && [ $((te-ts)) -lt 3250 ] [ -z "$v" ] diff --git a/contrib/libarchive/libarchive/archive_string.c b/contrib/libarchive/libarchive/archive_string.c index c6ae8968d54f..4fb96a9fa178 100644 --- a/contrib/libarchive/libarchive/archive_string.c +++ b/contrib/libarchive/libarchive/archive_string.c @@ -1314,7 +1314,17 @@ create_sconv_object(const char *fc, const char *tc, else if (strcmp(fc, "CP932") == 0) sc->cd = iconv_open(tc, "SJIS"); } -#if defined(_WIN32) && !defined(__CYGWIN__) +#if defined(__FreeBSD__) && !defined(HAVE_LIBICONV) + /* + * FreeBSD's native iconv() by default returns the number of + * invalid characters in the input string, as specified by + * POSIX, but iconv_strncat_in_locale() assumes GNU iconv + * semantics. + */ + int v = 1; + + (void)iconvctl(sc->cd, ICONV_SET_ILSEQ_INVALID, &v); +#elif defined(_WIN32) && !defined(__CYGWIN__) /* * archive_mstring on Windows directly convert multi-bytes * into archive_wstring in order not to depend on locale diff --git a/contrib/sqlite3/Makefile.in b/contrib/sqlite3/Makefile.in index a77386faed7c..8e6b358befbb 100644 --- a/contrib/sqlite3/Makefile.in +++ b/contrib/sqlite3/Makefile.in @@ -222,9 +222,12 @@ install: install-lib # Flags to link the shell app either directly against sqlite3.c # (ENABLE_STATIC_SHELL==1) or libsqlite3.so (ENABLE_STATIC_SHELL==0). # +# Maintenance reminder: placement of $(LDFLAGS) is more relevant for +# some platforms than others: +# https://sqlite.org/forum/forumpost/d80ecdaddd ENABLE_STATIC_SHELL = @ENABLE_STATIC_SHELL@ -sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS.libsqlite3) -sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) +sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS) $(LDFLAGS.libsqlite3) +sqlite3-shell-link-flags.0 = $(LDFLAGS) -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) sqlite3-shell-deps.1 = $(TOP)/sqlite3.c sqlite3-shell-deps.0 = $(libsqlite3.DLL) # @@ -245,7 +248,7 @@ sqlite3$(T.exe): $(TOP)/shell.c $(sqlite3-shell-deps.$(ENABLE_STATIC_SHELL)) $(sqlite3-shell-static.flags.$(STATIC_CLI_SHELL)) \ -I. $(OPT_FEATURE_FLAGS) $(SHELL_OPT) \ $(CFLAGS) $(CFLAGS.readline) $(CFLAGS.icu) \ - $(LDFLAGS) $(LDFLAGS.readline) + $(LDFLAGS.readline) sqlite3$(T.exe)-1: sqlite3$(T.exe)-0: sqlite3$(T.exe) @@ -279,7 +282,7 @@ DIST_FILES := \ README.txt VERSION \ auto.def autosetup configure tea \ sqlite3.h sqlite3.c shell.c sqlite3ext.h \ - Makefile.in Makefile.msc Makefile.fallback \ + Makefile.in Makefile.msc Makefile.fallback make.bat \ sqlite3.rc sqlite3rc.h Replace.cs \ sqlite3.pc.in sqlite3.1 diff --git a/contrib/sqlite3/Makefile.msc b/contrib/sqlite3/Makefile.msc index dfa2dcfd5bdc..34e41d8aa0f3 100644 --- a/contrib/sqlite3/Makefile.msc +++ b/contrib/sqlite3/Makefile.msc @@ -101,13 +101,6 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF -# Set this non-0 to use the library paths and other options necessary for -# Windows Phone 8.1. -# -!IFNDEF USE_WP81_OPTS -USE_WP81_OPTS = 0 -!ENDIF - # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -156,14 +149,6 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF -# Set this non-0 to compile binaries suitable for the WinRT environment. -# This setting does not apply to any binaries that require Tcl to operate -# properly (i.e. the text fixture, etc). -# -!IFNDEF FOR_WINRT -FOR_WINRT = 0 -!ENDIF - # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -327,6 +312,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF @@ -349,6 +335,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 # Always enable math functions on Windows OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE # Should the rbu extension be enabled? If so, add compilation options # to enable it. @@ -561,10 +548,14 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + + !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + + !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -665,18 +656,6 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF -# When compiling the library for use in the WinRT environment, -# the following compile-time options must be used as well to -# disable use of Win32 APIs that are not available and to enable -# use of Win32 APIs that are specific to Windows 8 and/or WinRT. -# -!IF $(FOR_WINRT)!=0 -TCC = $(TCC) -DSQLITE_OS_WINRT=1 -RCC = $(RCC) -DSQLITE_OS_WINRT=1 -TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -!ENDIF - # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -689,35 +668,29 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 +!IF $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd +ZLIBCFLAGS = -nologo -MDd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MD BCC = $(BCC) -MD +ZLIBCFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi !ENDIF !ELSE !IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd +ZLIBCFLAGS = -nologo -MTd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT +ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF !ENDIF -# Define -DNDEBUG to compile without debugging (i.e., for production usage) -# Omitting the define will cause extra debugging code to be inserted and -# includes extra comments when "EXPLAIN stmt" is used. -# -!IF $(DEBUG)==0 -TCC = $(TCC) -DNDEBUG -BCC = $(BCC) -DNDEBUG -RCC = $(RCC) -DNDEBUG -!ENDIF - !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -907,56 +880,6 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF -# When compiling for use in the WinRT environment, the following -# linker option must be used to mark the executable as runnable -# only in the context of an application container. -# -!IF $(FOR_WINRT)!=0 -LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER -!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" -!IFNDEF STORELIBPATH -!IF "$(PLATFORM)"=="x86" -STORELIBPATH = $(CRTLIBPATH)\store -!ELSEIF "$(PLATFORM)"=="x64" -STORELIBPATH = $(CRTLIBPATH)\store\amd64 -!ELSEIF "$(PLATFORM)"=="ARM" -STORELIBPATH = $(CRTLIBPATH)\store\arm -!ELSE -STORELIBPATH = $(CRTLIBPATH)\store -!ENDIF -!ENDIF -STORELIBPATH = $(STORELIBPATH:\\=\) -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, an extra library path is -# required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFNDEF WP81LIBPATH -!IF "$(PLATFORM)"=="x86" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ELSEIF "$(PLATFORM)"=="ARM" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM -!ELSE -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ENDIF -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, some extra linker options -# are also required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFDEF WP81LIBPATH -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" -!ENDIF -LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE -LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib -LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib -!ENDIF - # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -980,9 +903,9 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /DEBUG $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) !ELSE -LDFLAGS = $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) !ENDIF @@ -1015,8 +938,10 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -1065,6 +990,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +tclsqlite-ex.c: + # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) diff --git a/contrib/sqlite3/VERSION b/contrib/sqlite3/VERSION index acecb4fcb7aa..0d767a08f23c 100644 --- a/contrib/sqlite3/VERSION +++ b/contrib/sqlite3/VERSION @@ -1 +1 @@ -3.50.4 +3.53.1 diff --git a/contrib/sqlite3/autosetup/README.md b/contrib/sqlite3/autosetup/README.md index 3301f5739599..ac013080ad02 100644 --- a/contrib/sqlite3/autosetup/README.md +++ b/contrib/sqlite3/autosetup/README.md @@ -375,18 +375,29 @@ configure process, and check it in. Patching Autosetup for Project-local Changes ------------------------------------------------------------------------ +The autosetup files require the following patches after updating +from their upstream sources: + +### `--debug` flag + Autosetup reserves the flag name **`--debug`** for its own purposes, and its own special handling of `--enable-...` flags makes `--debug` an alias for `--enable-debug`. As this project has a long history of using `--enable-debug`, we patch autosetup to use the name `--autosetup-debug` in place of `--debug`. That requires (as of this -writing) four small edits in [](/file/autosetup/autosetup), as -demonstrated in [check-in 3296c8d3](/info/3296c8d3). +writing) four small edits in +[/autosetup/autosetup](/file/autosetup/autosetup), as demonstrated in +[check-in 3296c8d3](/info/3296c8d3). If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the `debug` flag in `auto.def` - duplicated flags are not permitted. +### Fail on `malloc()` error + +See [check-in 72c8a5b94cdf5d](/info/72c8a5b94cdf5d). + + <a name="branch-customization"></a> Branch-specific Customization ======================================================================== @@ -426,7 +437,7 @@ proc sqlite-custom-flags {} { ``` That function must return either an empty string or a list in the form -used internally by `sqlite-config.tcl:sqlite-configure`. +used internally by [sqlite-config.tcl][]'s `sqlite-configure`. Next, define: @@ -450,4 +461,4 @@ all other significant processing. [sqlite-config.tcl]: /file/autosetup/sqlite-config.tcl [Makefile.in]: /file/Makefile.in [main.mk]: /file/main.mk -[JimTCL]: https://jim.tcl.tk +[JimTCL]: https://msteveb.github.io/jimtcl/ diff --git a/contrib/sqlite3/autosetup/autosetup b/contrib/sqlite3/autosetup/autosetup index 239987554ff3..c3a31bec585e 100755 --- a/contrib/sqlite3/autosetup/autosetup +++ b/contrib/sqlite3/autosetup/autosetup @@ -406,8 +406,8 @@ proc options-add {opts} { # Find the corresponding value in the user options # and set the default if necessary if {[string match "-*" $opt]} { - # This is a documentation-only option, like "-C <dir>" - set opthelp $opt + # We no longer support documentation-only options, like "-C <dir>" + autosetup-error "Option $opt is not supported" } elseif {$colon eq ""} { # Boolean option lappend autosetup(options) $name @@ -1611,7 +1611,7 @@ proc autosetup_output_block {type lines} { # Generate a command reference from inline documentation proc automf_command_reference {} { lappend files $::autosetup(prog) - lappend files {*}[lsort [glob -nocomplain $::autosetup(libdir)/*.tcl]] + lappend files {*}[lsort [glob -nocomplain $::autosetup(libdir)/{*/*.tcl,*.tcl}]] # We want to process all non-module files before module files # and then modules in alphabetical order. @@ -2124,7 +2124,7 @@ if {$autosetup(istcl)} { set frame [info frame -$i] if {[dict exists $frame file]} { # We don't need proc, so use "" - lappend stacktrace "" [dict get $frame file] [dict get $frame line] + lappend stacktrace "" [dict get $frame file] [dict get $frame line] "" } } return $stacktrace @@ -2181,8 +2181,12 @@ proc error-location {msg} { if {$::autosetup(debug)} { return -code error $msg } - # Search back through the stack trace for the first error in a .def file - foreach {p f l} [stacktrace] { + set vars {p f l cmd} + if {!$::autosetup(istcl) && ![dict exists $::tcl_platform stackFormat]} { + # Older versions of Jim had a 3 element stacktrace + set vars {p f l} + } + foreach $vars [stacktrace] { if {[string match *.def $f]} { return "[relative-path $f]:$l: Error: $msg" } @@ -2534,7 +2538,7 @@ if {[catch {main $argv} msg opts] == 1} { show-notices autosetup-full-error [error-dump $msg $opts $autosetup(debug)] if {!$autosetup(debug)} { - puts stderr "Try: '[file tail $autosetup(exe)] --autosetup-debug' for a full stack trace" + puts stderr "Try: '[file tail $autosetup(exe)] --debug' for a full stack trace" } exit 1 } diff --git a/contrib/sqlite3/autosetup/cc-shared.tcl b/contrib/sqlite3/autosetup/cc-shared.tcl index cbe568018e96..1fa200eec184 100644 --- a/contrib/sqlite3/autosetup/cc-shared.tcl +++ b/contrib/sqlite3/autosetup/cc-shared.tcl @@ -89,13 +89,15 @@ switch -glob -- [get-define host] { define SH_SOPREFIX -Wl,-h, } } - *-*-hpux { - # XXX: These haven't been tested - define SHOBJ_CFLAGS "+O3 +z" + *-*-hpux* { + define SHOBJ_CFLAGS +z define SHOBJ_LDFLAGS -b define SH_CFLAGS +z + define SH_LDFLAGS -b define SH_LINKFLAGS -Wl,+s - define LD_LIBRARY_PATH SHLIB_PATH + define SH_LINKRPATH "-Wl,+b -Wl,%s" + define SH_SOPREFIX -Wl,+h, + define STRIPLIBFLAGS -Wl,-s } *-*-haiku { define SHOBJ_CFLAGS "" diff --git a/contrib/sqlite3/autosetup/jimsh0.c b/contrib/sqlite3/autosetup/jimsh0.c index b035524c9681..0f0a890888b4 100644 --- a/contrib/sqlite3/autosetup/jimsh0.c +++ b/contrib/sqlite3/autosetup/jimsh0.c @@ -7409,11 +7409,13 @@ void *JimDefaultAllocator(void *ptr, size_t size) free(ptr); return NULL; } - else if (ptr) { - return realloc(ptr, size); - } else { - return malloc(size); + void *p = realloc(ptr, size); + if( p==0 ){ + fprintf(stderr,"Out of memory\n"); + exit(1); + } + return p; } } @@ -9132,7 +9134,7 @@ int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr) const char *sA = Jim_GetString(aObjPtr, &Alen); const char *sB = Jim_GetString(bObjPtr, &Blen); - return Alen == Blen && *sA == *sB && memcmp(sA, sB, Alen) == 0; + return Alen == Blen && memcmp(sA, sB, Alen) == 0; } } @@ -10242,7 +10244,7 @@ static int JimCommandsHT_KeyCompare(void *privdata, const void *key1, const void int len1, len2; const char *str1 = Jim_GetStringNoQualifier((Jim_Obj *)key1, &len1); const char *str2 = Jim_GetStringNoQualifier((Jim_Obj *)key2, &len2); - return len1 == len2 && *str1 == *str2 && memcmp(str1, str2, len1) == 0; + return len1 == len2 && memcmp(str1, str2, len1) == 0; } static void JimCommandsHT_ValDestructor(void *interp, void *val) @@ -13864,13 +13866,6 @@ static int JimExprOpNumUnary(Jim_Interp *interp, struct JimExprNode *node) case JIM_EXPROP_NOT: wC = !bA; break; - case JIM_EXPROP_UNARYPLUS: - case JIM_EXPROP_UNARYMINUS: - rc = JIM_ERR; - Jim_SetResultFormatted(interp, - "can't use non-numeric string as operand of \"%s\"", - node->type == JIM_EXPROP_UNARYPLUS ? "+" : "-"); - break; default: abort(); } @@ -19875,22 +19870,16 @@ wrongargs: } else if (errorCodeObj) { int len = Jim_ListLength(interp, argv[idx + 1]); + int i; - if (len > Jim_ListLength(interp, errorCodeObj)) { + ret = JIM_OK; - ret = -1; - } - else { - int i; - ret = JIM_OK; - - for (i = 0; i < len; i++) { - Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); - Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); - if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { - ret = -1; - break; - } + for (i = 0; i < len; i++) { + Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); + Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); + if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { + ret = -1; + break; } } } @@ -20266,7 +20255,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_SET: - return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG | JIM_UNSHARED); + return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG); case OPT_EXISTS:{ int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, JIM_NONE); @@ -20278,7 +20267,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_UNSET: - if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_UNSHARED) != JIM_OK) { + if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_NONE) != JIM_OK) { return JIM_ERR; } return JIM_OK; diff --git a/contrib/sqlite3/autosetup/proj.tcl b/contrib/sqlite3/autosetup/proj.tcl index 1335567064ee..caa679ad65ba 100644 --- a/contrib/sqlite3/autosetup/proj.tcl +++ b/contrib/sqlite3/autosetup/proj.tcl @@ -60,10 +60,11 @@ # $proj__Config is an internal-use-only array for storing whatever generic # internal stuff we need stored. # -array set ::proj__Config { - self-tests 1 -} - +array set ::proj__Config [subst { + self-tests [get-env proj.self-tests 0] + verbose-assert [get-env proj.assert-verbose 0] + isatty [isatty? stdout] +}] # # List of dot-in files to filter in the final stages of @@ -75,7 +76,6 @@ array set ::proj__Config { # See: proj-dot-ins-append and proj-dot-ins-process # set ::proj__Config(dot-in-files) [list] -set ::proj__Config(isatty) [isatty? stdout] # # @proj-warn msg @@ -85,28 +85,29 @@ set ::proj__Config(isatty) [isatty? stdout] # proc proj-warn {args} { show-notices - puts stderr [join [list "WARNING: \[[proj-scope 1]\]: " {*}$args] " "] + puts stderr [join [list "WARNING:" \[ [proj-scope 1] \]: {*}$args] " "] } +# # Internal impl of [proj-fatal] and [proj-error]. It must be called # using tailcall. -proc proj__faterr {failMode argv} { +# +proc proj__faterr {failMode args} { show-notices set lvl 1 - while {"-up" eq [lindex $argv 0]} { - set argv [lassign $argv -] + while {"-up" eq [lindex $args 0]} { + set args [lassign $args -] incr lvl } if {$failMode} { - puts stderr [join [list "FATAL: \[[proj-scope $lvl]]: " {*}$argv]] + puts stderr [join [list "FATAL:" \[ [proj-scope $lvl] \]: {*}$args]] exit 1 } else { - error [join [list "\[[proj-scope $lvl]]:" {*}$argv]] + error [join [list in \[ [proj-scope $lvl] \]: {*}$args]] } } - # # @proj-fatal ?-up...? msg... # @@ -118,7 +119,7 @@ proc proj__faterr {failMode argv} { # additional level. # proc proj-fatal {args} { - tailcall proj__faterr 1 $args + tailcall proj__faterr 1 {*}$args } # @@ -127,10 +128,9 @@ proc proj-fatal {args} { # Works like proj-fatal but uses [error] intead of [exit]. # proc proj-error {args} { - tailcall proj__faterr 0 $args + tailcall proj__faterr 0 {*}$args } -set ::proj__Config(verbose-assert) [get-env proj-assert-verbose 0] # # @proj-assert script ?message? # @@ -147,7 +147,7 @@ proc proj-assert {script {msg ""}} { if {"" eq $msg} { set msg $script } - proj-fatal "Assertion failed in \[[proj-scope 1]\]: $msg" + tailcall proj__faterr 1 "Assertion failed:" $msg } } @@ -378,8 +378,8 @@ proc proj-bin-define {binName {defName {}}} { # # Despite using cc-path-progs to do the search, this function clears # any define'd name that function stores for the result (because the -# caller has no sensible way of knowing which result it was unless -# they pass only a single argument). +# caller has no sensible way of knowing which [define] name it has +# unless they pass only a single argument). # proc proj-first-bin-of {args} { set rc "" @@ -451,7 +451,9 @@ proc proj-opt-set {flag {val 1}} { # @proj-opt-exists flag # # Returns 1 if the given flag has been defined as a legal configure -# option, else returns 0. +# option, else returns 0. Options set via proj-opt-set "exist" for +# this purpose even if they were not defined via autosetup's +# [options] function. # proc proj-opt-exists {flag} { expr {$flag in $::autosetup(options)}; @@ -555,7 +557,7 @@ proc proj-opt-define-bool {args} { if {$invert} { set rc [expr {!$rc}] } - msg-result $rc + msg-result [string map {0 no 1 yes} $rc] define $defName $rc return $rc } @@ -704,11 +706,20 @@ proc proj-file-write {args} { } # -# @proj-check-compile-commands ?configFlag? +# @proj-check-compile-commands ?-assume-for-clang? ?configFlag? +# +# Checks the compiler for compile_commands.json support. If +# $configFlag is not empty then it is assumed to be the name of an +# autosetup boolean config which controls whether to run/skip this +# check. # -# Checks the compiler for compile_commands.json support. If passed an -# argument it is assumed to be the name of an autosetup boolean config -# which controls whether to run/skip this check. +# If -assume-for-clang is provided and $configFlag is not empty and CC +# matches *clang* and no --$configFlag was explicitly provided to the +# configure script then behave as if --$configFlag had been provided. +# To disable that assumption, either don't pass -assume-for-clang or +# pass --$configFlag=0 to the configure script. (The reason for this +# behavior is that clang supports compile-commands but some other +# compilers report false positives with these tests.) # # Returns 1 if supported, else 0, and defines HAVE_COMPILE_COMMANDS to # that value. Defines MAKE_COMPILATION_DB to "yes" if supported, "no" @@ -716,12 +727,38 @@ proc proj-file-write {args} { # HAVE_COMPILE_COMMANDS is preferred. # # ACHTUNG: this test has a long history of false positive results -# because of compilers reacting differently to the -MJ flag. +# because of compilers reacting differently to the -MJ flag. Because +# of this, it is recommended that this support be an opt-in feature, +# rather than an on-by-default default one. That is: in the +# configure script define the option as +# {--the-flag-name=0 => {Enable ....}} # -proc proj-check-compile-commands {{configFlag {}}} { +proc proj-check-compile-commands {args} { + set i 0 + set configFlag {} + set fAssumeForClang 0 + set doAssume 0 msg-checking "compile_commands.json support... " - if {"" ne $configFlag && ![proj-opt-truthy $configFlag]} { - msg-result "explicitly disabled" + if {"-assume-for-clang" eq [lindex $args 0]} { + lassign $args - configFlag + incr fAssumeForClang + } elseif {1 == [llength $args]} { + lassign $args configFlag + } else { + proj-error "Invalid arguments" + } + if {1 == $fAssumeForClang && "" ne $configFlag} { + if {[string match *clang* [get-define CC]] + && ![proj-opt-was-provided $configFlag] + && ![proj-opt-truthy $configFlag]} { + proj-indented-notice [subst -nocommands -nobackslashes { + CC appears to be clang, so assuming that --$configFlag is likely + to work. To disable this assumption use --$configFlag=0.}] + incr doAssume + } + } + if {!$doAssume && "" ne $configFlag && ![proj-opt-truthy $configFlag]} { + msg-result "check disabled. Use --${configFlag} to enable it." define HAVE_COMPILE_COMMANDS 0 define MAKE_COMPILATION_DB no return 0 @@ -730,7 +767,7 @@ proc proj-check-compile-commands {{configFlag {}}} { # This test reportedly incorrectly succeeds on one of # Martin G.'s older systems. drh also reports a false # positive on an unspecified older Mac system. - msg-result "compiler supports compile_commands.json" + msg-result "compiler supports -MJ. Assuming it's useful for compile_commands.json" define MAKE_COMPILATION_DB yes; # deprecated define HAVE_COMPILE_COMMANDS 1 return 1 @@ -885,7 +922,9 @@ proc proj-looks-like-windows {{key host}} { # proc proj-looks-like-mac {{key host}} { switch -glob -- [get-define $key] { - *apple* { + *-*-darwin* { + # https://sqlite.org/forum/forumpost/7b218c3c9f207646 + # There's at least one Linux out there which matches *apple*. return 1 } default { @@ -927,17 +966,13 @@ proc proj-exe-extension {} { # proc proj-dll-extension {} { set inner {{key} { - switch -glob -- [get-define $key] { - *apple* { - return ".dylib" - } - *-*-ming* - *-*-cygwin - *-*-msys { - return ".dll" - } - default { - return ".so" - } + if {[proj-looks-like-mac $key]} { + return ".dylib" + } + if {[proj-looks-like-windows $key]} { + return ".dll" } + return ".so" }} define BUILD_DLLEXT [apply $inner build] define TARGET_DLLEXT [apply $inner host] @@ -1135,6 +1170,10 @@ proc proj-check-rpath {} { if {"" eq $wl} { set wl [proj-cc-check-Wl-flag -R$lp] } + if {"" eq $wl} { + # HP-UX: https://sqlite.org/forum/forumpost/d80ecdaddd + set wl [proj-cc-check-Wl-flag +b $lp] + } define LDFLAGS_RPATH $wl } } @@ -1144,7 +1183,7 @@ proc proj-check-rpath {} { # # @proj-check-soname ?libname? # -# Checks whether CC supports the -Wl,soname,lib... flag. If so, it +# Checks whether CC supports the -Wl,-soname,lib... flag. If so, it # returns 1 and defines LDFLAGS_SONAME_PREFIX to the flag's prefix, to # which the client would need to append "libwhatever.N". If not, it # returns 0 and defines LDFLAGS_SONAME_PREFIX to an empty string. @@ -1160,6 +1199,10 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { if {[cc-check-flags "-Wl,-soname,${libname}"]} { define LDFLAGS_SONAME_PREFIX "-Wl,-soname," return 1 + } elseif {[cc-check-flags "-Wl,+h,${libname}"]} { + # HP-UX: https://sqlite.org/forum/forumpost/d80ecdaddd + define LDFLAGS_SONAME_PREFIX "-Wl,+h," + return 1 } else { define LDFLAGS_SONAME_PREFIX "" return 0 @@ -1606,7 +1649,7 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { # # Similar modifications may be made for --mandir. # -# Returns 1 if it modifies the environment, else 0. +# Returns >0 if it modifies the environment, else 0. # proc proj-tweak-default-env-dirs {} { set rc 0 @@ -1645,7 +1688,11 @@ proc proj-tweak-default-env-dirs {} { # processing the file. In the context of that script, the vars # $dotInsIn and $dotInsOut will be set to the input and output file # names. This can be used, for example, to make the output file -# executable or perform validation on its contents. +# executable or perform validation on its contents: +# +## proj-dot-ins-append my.sh.in my.sh { +## catch {exec chmod u+x $dotInsOut} +## } # # See [proj-dot-ins-process], [proj-dot-ins-list] # @@ -1665,7 +1712,7 @@ proc proj-dot-ins-append {fileIn args} { proj-fatal "Too many arguments: $fileIn $args" } } - #puts "******* [proj-scope]: adding $fileIn" + #puts "******* [proj-scope]: adding [llength $fileIn]-length item: $fileIn" lappend ::proj__Config(dot-in-files) $fileIn } @@ -1703,17 +1750,18 @@ proc proj-dot-ins-list {} { # makes proj-dot-ins-append available for re-use. # proc proj-dot-ins-process {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -touch "" {return "-touch"} -clear 0 {expr 1} -validate 0 {expr 1} } + #puts "args=$args"; parray flags if {[llength $args] > 0} { error "Invalid argument to [proj-scope]: $args" } foreach f $::proj__Config(dot-in-files) { proj-assert {3==[llength $f]} \ - "Expecting proj-dot-ins-list to be stored in 3-entry lists" + "Expecting proj-dot-ins-list to be stored in 3-entry lists. Got: $f" lassign $f fIn fOut fScript #puts "DOING $fIn ==> $fOut" proj-make-from-dot-in {*}$flags(-touch) $fIn $fOut @@ -1753,7 +1801,7 @@ proc proj-validate-no-unresolved-ats {args} { set isMake [string match {*[Mm]ake*} $f] foreach line [proj-file-content-list $f] { if {!$isMake || ![string match "#*" [string trimleft $line]]} { - if {[regexp {(@[A-Za-z0-9_]+@)} $line match]} { + if {[regexp {(@[A-Za-z0-9_\.]+@)} $line match]} { error "Unresolved reference to $match at line $lnno of $f" } } @@ -1794,7 +1842,7 @@ proc proj-setup-autoreconfig {defName} { } # -# @prop-append-to defineName args... +# @prop-define-append defineName args... # # A proxy for Autosetup's [define-append]. Appends all non-empty $args # to [define-append $defineName]. @@ -1825,7 +1873,7 @@ proc proj-define-append {defineName args} { # but it is technically correct and still relevant on some # environments. # -# See: proj-append-to +# See: proj-define-append # proc proj-define-amend {args} { set defName "" @@ -1893,7 +1941,7 @@ proc proj-define-amend {args} { # proc proj-define-to-cflag {args} { set rv {} - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -list 0 {expr 1} -quote 0 {expr 1} -zero-undef 0 {expr 1} @@ -2001,7 +2049,7 @@ proc proj-cache-key {arg {addLevel 0}} { # See proj-cache-key for -key's and -level's semantics, noting that # this function adds one to -level for purposes of that call. proc proj-cache-set {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -key => 0 -level => 0 } @@ -2037,7 +2085,7 @@ proc proj-cache-remove {{key 0} {addLevel 0}} { # See proj-cache-key for $key's and $addLevel's semantics, noting that # this function adds one to $addLevel for purposes of that call. proc proj-cache-check {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -key => 0 -level => 0 } @@ -2070,147 +2118,316 @@ proc proj-coalesce {args} { } # -# @proj-parse-simple-flags ... +# @proj-parse-flags argvListName targetArrayName {prototype} # # A helper to parse flags from proc argument lists. # -# Expects a list of arguments to parse, an array name to store any -# -flag values to, and a prototype object which declares the flags. +# The first argument is the name of a var holding the args to +# parse. It will be overwritten, possibly with a smaller list. # -# The prototype must be a list in one of the following forms: +# The second argument is the name of an array variable to create in +# the caller's scope. # -# -flag defaultValue {script} +# The third argument, $prototype, is a description of how to handle +# the flags. Each entry in that list must be in one of the +# following forms: # -# -flag => defaultValue -# -----^--^ (with spaces there!) +# -flag defaultValue ?-literal|-call|-apply? +# script|number|incr|proc-name|{apply $aLambda} # -# Repeated for each flag. +# -flag* ...as above... # -# The first form represents a basic flag with no associated -# following argument. The second form extracts its value -# from the following argument in $argvName. +# -flag => defaultValue ?-call proc-name-and-args|-apply lambdaExpr? # -# The first argument to this function is the name of a var holding the -# args to parse. It will be overwritten, possibly with a smaller list. +# -flag* => ...as above... # -# The second argument the name of an array variable to create in the -# caller's scope. (Pneumonic: => points to the next argument.) +# :PRAGMA # -# For the first form of flag, $script is run in the caller's scope if -# $argv contains -flag, and the result of that script is the new value -# for $tgtArrayName(-flag). This function intercepts [return $val] -# from $script. Any empty script will result in the flag having "" -# assigned to it. +# The first two forms represents a basic flag with no associated +# following argument. The third and fourth forms, called arg-consuming +# flags, extract the value from the following argument in $argvName +# (pneumonic: => points to the next argument.). The :PRAGMA form +# offers a way to configure certain aspects of this call. # -# The args list is only inspected until the first argument which is -# not described by $prototype. i.e. the first "non-flag" (not counting -# values consumed for flags defined like --flag=>default). +# If $argv contains any given flag from $prototype, its default value +# is overridden depending on several factors: # -# If a "--" flag is encountered, no more arguments are inspected as -# flags. If "--" is the first non-flag argument, the "--" flag is -# removed from the results but all remaining arguments are passed -# through. If "--" appears after the first non-flag, it is retained. +# - If the -literal flag is used, or the flag's script is a number, +# value is used verbatim. # -# This function assumes that each flag is unique, and using a flag -# more than once behaves in a last-one-wins fashion. +# - Else if the -call flag is used, the argument must be a proc name +# and any leading arguments, e.g. {apply $myLambda}. The proc is passed +# the (flag, value) as arguments (non-consuming flags will get +# passed the flag's current/starting value and consuming flags will +# get the next argument). Its result becomes the result of the +# flag. # -# Any argvName entries not described in $prototype are not treated as -# flags. +# - Else if -apply X is used, it's effectively shorthand for -call +# {apply X}. Its argument may either be a $lambaRef or a {{f v} +# {body}} construct. # -# Returns the number of flags it processed in $argvName. +# - Else if $script is one of the following values, it is treated as +# the result of... +# +# - incr: increments the current value of the flag. +# +# - Else $script is eval'd to get its result value. That result +# becomes the new flag value for $tgtArrayName(-flag). This +# function intercepts [return $val] from eval'ing $script. Any +# empty script will result in the flag having "" assigned to it. +# +# Unless the -flag has a trailing asterisk, e.g. -flag*, this function +# assumes that each flag is unique, and using a flag more than once +# causes an error to be triggered. the -flag* forms works similarly +# except that may appear in $argv any number of times: +# +# - For non-arg-consuming flags, each invocation of -flag causes the +# result of $script to overwrite the previous value. e.g. so +# {-flag* {x} {incr foo}} has a default value of x, but passing in +# -flag twice would change it to the result of incrementing foo +# twice. This form can be used to implement, e.g., increasing +# verbosity levels by passing -verbose multiple times. +# +# - For arg-consuming flags, the given flag starts with value X, but +# if the flag is provided in $argv, the default is cleared, then +# each instance of -flag causes its value to be appended to the +# result, so {-flag* => {a b c}} defaults to {a b c}, but passing +# in -flag y -flag z would change it to {y z}, not {a b c y z}.. +# +# By default, the args list is only inspected until the first argument +# which is not described by $prototype. i.e. the first "non-flag" (not +# counting values consumed for flags defined like -flag => default). +# The :all-flags pragma (see below) can modify this behavior. +# +# If a "--" flag is encountered, no more arguments are inspected as +# flags unless the :all-flags pragma (see below) is in effect. The +# first instance of "--" is removed from the target result list but +# all remaining instances of "--" are are passed through. +# +# Any argvName entries not described in $prototype are considered to +# be "non-flags" for purposes of this function, even if they +# ostensibly look like flags. +# +# Returns the number of flags it processed in $argvName, not counting +# "--". # # Example: # -# set args [list -foo -bar {blah} 8 9 10 -theEnd] -# proj-parse-simple-flags args flags { -# -foo 0 {expr 1} -# -bar => 0 -# -no-baz 2 {return 0} -# } +## set args [list -foo -bar {blah} -z 8 9 10 -theEnd] +## proj-parse-flags args flags { +## -foo 0 {expr 1} +## -bar => 0 +## -no-baz 1 {return 0} +## -z 0 2 +## } # -# After that $flags would contain {-foo 1 -bar {blah} -no-baz 2} +# After that $flags would contain {-foo 1 -bar {blah} -no-baz 1 -z 2} # and $args would be {8 9 10 -theEnd}. # -# Potential TODOs: consider using lappend instead of set so that any -# given flag can be used more than once. Or add a syntax to indicate -# that multiples are allowed. Also consider searching the whole -# argv list, rather than stopping at the first non-flag +# Pragmas: +# +# Passing :PRAGMAS to this function may modify how it works. The +# following pragmas are supported (note the leading ":"): # -proc proj-parse-simple-flags {argvName tgtArrayName prototype} { +# :all-flags indicates that the whole input list should be scanned, +# not stopping at the first non-flag or "--". +# +proc proj-parse-flags {argvName tgtArrayName prototype} { upvar $argvName argv - upvar $tgtArrayName tgt - array set dflt {} - array set scripts {} - array set consuming {} + upvar $tgtArrayName outFlags + array set flags {}; # staging area + array set blob {}; # holds markers for various per-key state and options + set incrSkip 1; # 1 if we stop at the first non-flag, else 0 + # Parse $prototype for flag definitions... set n [llength $prototype] - # Figure out what our flags are... + set checkProtoFlag { + #puts "**** checkProtoFlag #$i of $n k=$k fv=$fv" + switch -exact -- $fv { + -literal { + proj-assert {![info exists blob(${k}.consumes)]} + set blob(${k}.script) [list expr [lindex $prototype [incr i]]] + } + -apply { + set fv [lindex $prototype [incr i]] + if {2 == [llength $fv]} { + # Treat this as a lambda literal + set fv [list $fv] + } + lappend blob(${k}.call) "apply $fv" + } + -call { + # arg is either a proc name or {apply $aLambda} + set fv [lindex $prototype [incr i]] + lappend blob(${k}.call) $fv + } + default { + proj-assert {![info exists blob(${k}.consumes)]} + set blob(${k}.script) $fv + } + } + if {$i >= $n} { + proj-error -up "[proj-scope]: Missing argument for $k flag" + } + } for {set i 0} {$i < $n} {incr i} { set k [lindex $prototype $i] #puts "**** #$i of $n k=$k" + + # Check for :PRAGMA... + switch -exact -- $k { + :all-flags { + set incrSkip 0 + continue + } + } + proj-assert {[string match -* $k]} \ - "Invalid flag value: $k" - set v "" - set s "" + "Invalid argument: $k" + + if {[string match {*\*} $k]} { + # Re-map -foo* to -foo and flag -foo as a repeatable flag + set k [string map {* ""} $k] + incr blob(${k}.multi) + } + + if {[info exists flags($k)]} { + proj-error -up "[proj-scope]: Duplicated prototype for flag $k" + } + switch -exact -- [lindex $prototype [expr {$i + 1}]] { => { + # -flag => DFLT ?-subflag arg? incr i 2 if {$i >= $n} { - proj-error "Missing argument for $k => flag" + proj-error -up "[proj-scope]: Missing argument for $k => flag" + } + incr blob(${k}.consumes) + set vi [lindex $prototype $i] + if {$vi in {-apply -call}} { + proj-error -up "[proj-scope]: Missing default value for $k flag" + } else { + set fv [lindex $prototype [expr {$i + 1}]] + if {$fv in {-apply -call}} { + incr i + eval $checkProtoFlag + } } - set consuming($k) 1 - set v [lindex $prototype $i] } default { - set v [lindex $prototype [incr i]] - set s [lindex $prototype [incr i]] - set scripts($k) $s + # -flag VALUE ?flag? SCRIPT + set vi [lindex $prototype [incr i]] + set fv [lindex $prototype [incr i]] + eval $checkProtoFlag } } - #puts "**** #$i of $n k=$k v=$v s=$s" - set dflt($k) $v + #puts "**** #$i of $n k=$k vi=$vi" + set flags($k) $vi } - # Now look for those flags in the source list - array set tgt [array get dflt] - unset dflt + #puts "-- flags"; parray flags + #puts "-- blob"; parray blob set rc 0 - set rv {} + set rv {}; # staging area for the target argv value set skipMode 0 set n [llength $argv] + # Now look for those flags in $argv... for {set i 0} {$i < $n} {incr i} { set arg [lindex $argv $i] + #puts "-- [proj-scope] arg=$arg" if {$skipMode} { lappend rv $arg } elseif {"--" eq $arg} { - incr skipMode - } elseif {[info exists tgt($arg)]} { - if {[info exists consuming($arg)]} { - if {$i + 1 >= $n} { - proj-assert 0 {Cannot happen - bounds already checked} + # "--" is the conventional way to end processing of args + if {[incr blob(--)] > 1} { + # Elide only the first one + lappend rv $arg + } + incr skipMode $incrSkip + } elseif {[info exists flags($arg)]} { + # A known flag... + set isMulti [info exists blob(${arg}.multi)] + incr blob(${arg}.seen) + if {1 < $blob(${arg}.seen) && !$isMulti} { + proj-error -up [proj-scope] "$arg flag was used multiple times" + } + set vMode 0; # 0=as-is, 1=eval, 2=call + set isConsuming [info exists blob(${arg}.consumes)] + if {$isConsuming} { + incr i + if {$i >= $n} { + proj-error -up [proj-scope] "is missing argument for $arg flag" + } + set vv [lindex $argv $i] + } elseif {[info exists blob(${arg}.script)]} { + set vMode 1 + set vv $blob(${arg}.script) + } else { + set vv $flags($arg) + } + + if {[info exists blob(${arg}.call)]} { + set vMode 2 + set vv [concat {*}$blob(${arg}.call) $arg $vv] + } elseif {$isConsuming} { + proj-assert {!$vMode} + # fall through + } elseif {"" eq $vv || [string is double -strict $vv]} { + set vMode 0 + } elseif {$vv in {incr}} { + set vMode 0 + switch -exact $vv { + incr { + set xx $flags($k); incr xx; set vv $xx; unset xx + } + default { + proj-error "Unhandled \$vv value $vv" + } } - set tgt($arg) [lindex $argv [incr i]] - } elseif {"" eq $scripts($arg)} { - set tgt($arg) "" } else { - #puts "**** running scripts($arg) $scripts($arg)" - set code [catch {uplevel 1 $scripts($arg)} xrc xopt] - #puts "**** tgt($arg)=$scripts($arg) code=$code rc=$rc" - if {$code in {0 2}} { - set tgt($arg) $xrc + set vv [list eval $vv] + set vMode 1 + } + if {$vMode} { + set code [catch [list uplevel 1 $vv] vv xopt] + if {$code ni {0 2}} { + return {*}$xopt $vv + } + } + if {$isConsuming && $isMulti} { + if {1 == $blob(${arg}.seen)} { + # On the first hit, overwrite the default with a new list. + set flags($arg) [list $vv] } else { - return {*}$xopt $xrc + # On subsequent hits, append to the list. + lappend flags($arg) $vv } + } else { + set flags($arg) $vv } incr rc } else { - incr skipMode + # Non-flag + incr skipMode $incrSkip lappend rv $arg } } set argv $rv + array set outFlags [array get flags] + #puts "-- rv=$rv argv=$argv flags="; parray flags return $rc +}; # proj-parse-flags + +# +# Older (deprecated) name of proj-parse-flags. +# +proc proj-parse-simple-flags {args} { + tailcall proj-parse-flags {*}$args } if {$::proj__Config(self-tests)} { + set __ova $::proj__Config(verbose-assert); + set ::proj__Config(verbose-assert) 1 + puts "Running [info script] self-tests..." + # proj-cache... apply {{} { #proj-warn "Test code for proj-cache" proj-assert {![proj-cache-check -key here check]} @@ -2233,4 +2450,100 @@ if {$::proj__Config(self-tests)} { proj-assert {"" eq [proj-cache-remove]} proj-assert {"" eq $check} }} -} + + # proj-parse-flags ... + apply {{} { + set foo 3 + set argv {-a "hi - world" -b -b -b -- -a {bye bye} -- -d -D c -a "" --} + proj-parse-flags argv flags { + :all-flags + -a* => "gets overwritten" + -b* 7 {incr foo} + -d 1 0 + -D 0 1 + } + + #puts "-- argv = $argv"; parray flags; + proj-assert {"-- c --" eq $argv} + proj-assert {$flags(-a) eq "{hi - world} {bye bye} {}"} + proj-assert {$foo == 6} + proj-assert {$flags(-b) eq $foo} + proj-assert {$flags(-d) == 0} + proj-assert {$flags(-D) == 1} + set foo 0 + foreach x $flags(-a) { + proj-assert {$x in {{hi - world} {bye bye} {}}} + incr foo + } + proj-assert {3 == $foo} + + set argv {-a {hi world} -b -maybe -- -a {bye bye} -- -b c --} + set foo 0 + proj-parse-flags argv flags { + -a => "aaa" + -b 0 {incr foo} + -maybe no -literal yes + } + #parray flags; puts "--- argv = $argv" + proj-assert {"-a {bye bye} -- -b c --" eq $argv} + proj-assert {$flags(-a) eq "hi world"} + proj-assert {1 == $flags(-b)} + proj-assert {"yes" eq $flags(-maybe)} + + set argv {-f -g -a aaa -M -M -M -L -H -A AAA a b c} + set foo 0 + set myLambda {{flag val} { + proj-assert {$flag in {-f -g -M}} + #puts "myLambda flag=$flag val=$val" + incr val + }} + proc myNonLambda {flag val} { + proj-assert {$flag in {-A -a}} + #puts "myNonLambda flag=$flag val=$val" + concat $val $val + } + proj-parse-flags argv flags { + -f 0 -call {apply $myLambda} + -g 2 -apply $myLambda + -h 3 -apply $myLambda + -H 30 33 + -a => aAAAa -apply {{f v} { + set v + }} + -A => AaaaA -call myNonLambda + -B => 17 -call myNonLambda + -M* 0 -apply $myLambda + -L "" -literal $myLambda + } + rename myNonLambda "" + #puts "--- argv = $argv"; parray flags + proj-assert {$flags(-f) == 1} + proj-assert {$flags(-g) == 3} + proj-assert {$flags(-h) == 3} + proj-assert {$flags(-H) == 33} + proj-assert {$flags(-a) == {aaa}} + proj-assert {$flags(-A) eq "AAA AAA"} + proj-assert {$flags(-B) == 17} + proj-assert {$flags(-M) == 3} + proj-assert {$flags(-L) eq $myLambda} + + set argv {-touch -validate} + proj-parse-flags argv flags { + -touch "" {return "-touch"} + -validate 0 1 + } + #puts "----- argv = $argv"; parray flags + proj-assert {$flags(-touch) eq "-touch"} + proj-assert {$flags(-validate) == 1} + proj-assert {$argv eq {}} + + set argv {-i -i -i} + proj-parse-flags argv flags { + -i* 0 incr + } + proj-assert {3 == $flags(-i)} + }} + set ::proj__Config(verbose-assert) $__ova + unset __ova + puts "Done running [info script] self-tests." +}; # proj- API self-tests diff --git a/contrib/sqlite3/autosetup/sqlite-config.tcl b/contrib/sqlite3/autosetup/sqlite-config.tcl index 85fe4143821c..59ecb7192a46 100644 --- a/contrib/sqlite3/autosetup/sqlite-config.tcl +++ b/contrib/sqlite3/autosetup/sqlite-config.tcl @@ -1,7 +1,7 @@ # This file holds functions for autosetup which are specific to the # sqlite build tree. They are in this file, instead of auto.def, so # that they can be reused in the autoconf sub-tree. This file requires -# functions from proj.tcl. +# functions from the project-agnostic proj.tcl. if {[string first " " $autosetup(srcdir)] != -1} { user-error "The pathname of the source tree\ @@ -11,7 +11,7 @@ if {[string first " " $autosetup(builddir)] != -1} { user-error "The pathname of the build directory\ may not contain space characters" } -#parray ::autosetup; exit 0 + use proj # # We want the package version info to be emitted early on, but doing @@ -65,11 +65,12 @@ array set sqliteConfig [subst [proj-strip-hash-comments { # The list of feature --flags which the --all flag implies. This # requires special handling in a few places. # - all-flag-enables {fts4 fts5 rtree geopoly session} + all-flag-enables {fts4 fts5 rtree geopoly session dbpage dbstat carray} # # Default value for the --all flag. Can hypothetically be modified - # by non-canonical builds. + # by non-canonical builds (it was added for a Tcl extension build + # mode which was eventually removed). # all-flag-default 0 }]] @@ -92,7 +93,7 @@ array set sqliteConfig [subst [proj-strip-hash-comments { # sqlite-configure BUILD_NAME { build-specific configure script } # # There are snippets of build-mode-specific decision-making in -# [sqlite-configure-finalize] +# [sqlite-configure-finalize], which gets run after $configScript. proc sqlite-configure {buildMode configScript} { proj-assert {$::sqliteConfig(build-mode) eq "unknown"} \ "sqlite-configure must not be called more than once" @@ -112,8 +113,10 @@ proc sqlite-configure {buildMode configScript} { # # Reference: https://msteveb.github.io/autosetup/developer/ # - # All configure flags must be described in an 'options' call. The - # general syntax is: + # All configure flags must be described in one or more calls to + # autosetup's [options] and [options-add] functions. The general + # syntax of the single argument to those functions is a list contain + # a mapping of flags to help text: # # FLAG => {Help text} # @@ -164,13 +167,18 @@ proc sqlite-configure {buildMode configScript} { ######################################################################## set allFlags { # Structure: a list of M {Z} pairs, where M is a descriptive - # option group name and Z is a list of X Y pairs. X is a list of + # option group name and Z is a list of X Y pairs. X is a list of # $buildMode name(s) to which the Y flags apply, or {*} to apply # to all builds. Y is a {block} in the form expected by - # autosetup's [options] command. Each block which is applicable - # to $buildMode is appended to a new list before that list is - # passed on to [options]. The order of each Y and sub-Y is - # retained, which is significant for rendering of --help. + # autosetup's [options] and [options-add] command. Each block + # which is applicable to $buildMode is passed on to + # [options-add]. The order of each Y and sub-Y is retained, which + # is significant for rendering of --help. + # + # Maintenance note: [options] does not support comments in + # options, but we filter this object through + # [proj-strip-hash-comments] to remove them before passing them on + # to [options]. # When writing {help text blocks}, be aware that: # @@ -180,7 +188,7 @@ proc sqlite-configure {buildMode configScript} { # pretty-printed. # # B) Vars and commands are NOT expanded, but we use a [subst] call - # below which will replace (only) var refs. + # below which will replace (only) $var refs. # Options for how to build the library build-modes { @@ -212,11 +220,19 @@ proc sqlite-configure {buildMode configScript} { geopoly => {Enable the GEOPOLY extension} rtree => {Enable the RTREE extension} session => {Enable the SESSION extension} + dbpage => {Enable the sqlite3_dbpage extension} + dbstat => {Enable the sqlite3_dbstat extension} + carray=1 => {Disable the CARRAY extension} all=$::sqliteConfig(all-flag-default) => {$allFlagHelp} largefile=1 => {This legacy flag has no effect on the library but may influence the generated sqlite_cfg.h by adding #define HAVE_LFS} } + {canonical} { + column-metadata => {Enable the column metadata APIs} + # ^^^ Affects how sqlite3.c is generated, so is not available in + # the autoconf build. + } } # Options for TCL support @@ -227,8 +243,6 @@ proc sqlite-configure {buildMode configScript} { This tree requires TCL for code generation but can use the in-tree copy of autosetup/jimsh0.c for that. The SQLite TCL extension and the test code require a canonical tclsh.} - } - {canonical} { with-tcl:DIR => {Directory containing tclConfig.sh or a directory one level up from that, from which we can derive a directory containing tclConfig.sh. @@ -236,11 +250,10 @@ proc sqlite-configure {buildMode configScript} { the --prefix flag.} with-tclsh:PATH => {Full pathname of tclsh to use. It is used for (A) trying to find - tclConfig.sh and (B) all TCL-based code generation. Warning: if - its containing dir has multiple tclsh versions, it may select the + tclConfig.sh and (B) all TCL-based code generation. Use --with-tcl + unless you have a specific need for this flag. Warning: if its + containing dir has multiple tclsh versions, it may select the wrong tclConfig.sh!} - } - {canonical} { static-tclsqlite3=0 => {Statically-link tclsqlite3. This only works if TCL support is enabled and all requisite libraries are available in @@ -319,7 +332,7 @@ proc sqlite-configure {buildMode configScript} { Needed only by ext/wasm. Default=EMSDK env var.} amalgamation-extra-src:FILES - => {Space-separated list of soure files to append as-is to the resulting + => {Space-separated list of source files to append as-is to the resulting sqlite3.c amalgamation file. May be provided multiple times.} } } @@ -334,8 +347,7 @@ proc sqlite-configure {buildMode configScript} { => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} } {canonical autoconf} { - # A potential TODO without a current use case: - #rpath=1 => {Disable use of the rpath linker flag} + rpath=1 => {Disable use of the rpath linker flag} # soname: https://sqlite.org/src/forumpost/5a3b44f510df8ded soname:=legacy => {SONAME for libsqlite3.so. "none", or not using this flag, sets no @@ -406,7 +418,6 @@ proc sqlite-configure {buildMode configScript} { # ^^^ lappend of [sqlite-custom-flags] introduces weirdness if # we delay [proj-strip-hash-comments] until after that. - ######################################################################## # sqlite-custom.tcl is intended only for vendor-branch-specific # customization. See autosetup/README.md#branch-customization for @@ -424,6 +435,8 @@ proc sqlite-configure {buildMode configScript} { } } + #lappend allFlags just-testing {{*} {soname:=duplicateEntry => {x}}} + # Filter allFlags to create the set of [options] legal for this build foreach {group XY} [subst -nobackslashes -nocommands $allFlags] { foreach {X Y} $XY { @@ -432,7 +445,7 @@ proc sqlite-configure {buildMode configScript} { } } } - #lappend opts "soname:=duplicateEntry => {x}"; #just testing + if {[catch {options {}} msg xopts]} { # Workaround for <https://github.com/msteveb/autosetup/issues/73> # where [options] behaves oddly on _some_ TCL builds when it's @@ -447,8 +460,9 @@ proc sqlite-configure {buildMode configScript} { ######################################################################## # Runs "phase 1" of the configure process: after initial --flags -# handling but before the build-specific parts are run. $buildMode -# must be the mode which was passed to [sqlite-configure]. +# handling but before sqlite-configure's $configScript argument is +# run. $buildMode must be the mode which was passed to +# [sqlite-configure]. proc sqlite-configure-phase1 {buildMode} { define PACKAGE_NAME sqlite define PACKAGE_URL {https://sqlite.org} @@ -490,6 +504,7 @@ proc sqlite-configure-phase1 {buildMode} { if {[file exists $srcdir/sqlite3.pc.in]} { proj-dot-ins-append $srcdir/sqlite3.pc.in } + sqlite-handle-hpux; # must be relatively early so that other config tests can work }; # sqlite-configure-phase1 ######################################################################## @@ -542,7 +557,25 @@ proc proc-debug {msg} { } define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. -define OPT_SHELL {} ; # Feature-related CFLAGS for the sqlite3 CLI app +# +# OPT_SHELL = feature-related CFLAGS for the sqlite3 CLI app. The +# list's initial values are defaults which are always applied and not +# affected by --feature-flags. The list is appended to by various +# --feature-flags. +define OPT_SHELL { + -DSQLITE_DQS=0 + -DSQLITE_ENABLE_FTS4 + -DSQLITE_ENABLE_RTREE + -DSQLITE_ENABLE_EXPLAIN_COMMENTS + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + -DSQLITE_ENABLE_STMTVTAB + -DSQLITE_ENABLE_DBPAGE_VTAB + -DSQLITE_ENABLE_DBSTAT_VTAB + -DSQLITE_ENABLE_BYTECODE_VTAB + -DSQLITE_ENABLE_OFFSET_SQL_FUNC + -DSQLITE_ENABLE_PERCENTILE + -DSQLITE_STRICT_SUBTYPE=1 +} ######################################################################## # Adds $args, if not empty, to OPT_FEATURE_FLAGS. If the first arg is # -shell then it strips that arg and passes the remaining args the @@ -616,7 +649,7 @@ proc sqlite-check-common-system-deps {} { # Check for needed/wanted functions cc-check-functions gmtime_r isnan localtime_r localtime_s \ - strchrnul usleep utime pread pread64 pwrite pwrite64 + usleep utime pread pread64 pwrite pwrite64 apply {{} { set ldrt "" @@ -646,6 +679,7 @@ proc sqlite-check-common-system-deps {} { define HAVE_ZLIB 1 define LDFLAGS_ZLIB -lz sqlite-add-shell-opt -DSQLITE_HAVE_ZLIB=1 + sqlite-add-feature-flag -DSQLITE_HAVE_ZLIB=1 } else { define HAVE_ZLIB 0 define LDFLAGS_ZLIB "" @@ -709,12 +743,11 @@ proc sqlite-setup-default-cflags {} { # compiling binaries for the target system (CC a.k.a. $(T.cc)). # Normally they're the same, but they will differ when # cross-compiling. - # - # When cross-compiling we default to not using the -g flag, based on a - # /chat discussion prompted by - # https://sqlite.org/forum/forumpost/9a67df63eda9925c set defaultCFlags {-O2} if {!$::sqliteConfig(is-cross-compiling)} { + # When cross-compiling we default to not using the -g flag, based + # on a /chat discussion prompted by + # https://sqlite.org/forum/forumpost/9a67df63eda9925c lappend defaultCFlags -g } define CFLAGS [proj-get-env CFLAGS $defaultCFlags] @@ -772,7 +805,12 @@ proc sqlite-handle-common-feature-flags {} { sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3 } } - scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + bytecode-vtab -DSQLITE_ENABLE_BYTECODE_VTAB {} + scanstatus {-DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB} {} + column-metadata -DSQLITE_ENABLE_COLUMN_METADATA {} + dbpage -DSQLITE_ENABLE_DBPAGE_VTAB {} + dbstat -DSQLITE_ENABLE_DBSTAT_VTAB {} + carray -DSQLITE_ENABLE_CARRAY {} }] { if {$boolFlag ni $::autosetup(options)} { # Skip flags which are in the canonical build but not @@ -1014,7 +1052,7 @@ proc sqlite-handle-emsdk {} { proc sqlite-get-readline-dir-list {} { # Historical note: the dirs list, except for the inclusion of # $prefix and some platform-specific dirs, originates from the - # legacy configure script + # legacy configure script. set dirs [list [get-define prefix]] switch -glob -- [get-define host] { *-linux-android { @@ -1032,7 +1070,7 @@ proc sqlite-get-readline-dir-list {} { if {[opt-val with-readline-ldflags] in {auto ""}} { # If the user did not supply their own --with-readline-ldflags # value, hijack that flag to inject options which are known to - # work on a default Haiku installation. + # work on Haiku OS installations. if {"" ne [glob -nocomplain /boot/system/lib/libreadline*]} { proj-opt-set with-readline-ldflags {-L/boot/system/lib -lreadline} } @@ -1082,8 +1120,8 @@ proc sqlite-get-readline-dir-list {} { # 4) Default to automatic search for optional readline # # 5) Try to find readline or editline. If it's not found AND the -# corresponding --FEATURE flag was explicitly given, fail fatally, -# else fail silently. +# corresponding --FEATURE flag was explicitly given then fail +# fatally, else fail non-fatally. proc sqlite-check-line-editing {} { msg-result "Checking for line-editing capability..." define HAVE_READLINE 0 @@ -1096,9 +1134,30 @@ proc sqlite-check-line-editing {} { # if the library is not found. set libsForReadline {readline edit} ; # -l<LIB> names to check for readline(). # The libedit check changes this. - set editLibName "readline" ; # "readline" or "editline" + set editLibName "readline" ; # "readline" or "editline" set editLibDef "HAVE_READLINE" ; # "HAVE_READLINE" or "HAVE_EDITLINE" set dirLn [opt-val with-linenoise] + + # If none of --with-linenoise, --enable-readline, or --enable-editline + # are provided, but there exists a directory "linenoise" at $HOME or + # a sibling of the build or source directory, then try to use that linenoise + # direcctory. + # + if {"" eq $dirLn + && ![proj-opt-was-provided readline] + && ![proj-opt-was-provided editline] + } { + set dirlist ../linenoise + catch {lappend dirlist [file-normalize $::autosetup(srcdir)/../linenoise]} + catch {lappend dirlist $::env(HOME)/linenoise} + foreach d $dirlist { + if {[file exists $d/linenoise.c] && [file exists $d/linenoise.h]} { + set dirLn $d + break + } + } + } + if {"" ne $dirLn} { # Use linenoise from a copy of its sources (not a library)... if {![file isdir $dirLn]} { @@ -1113,7 +1172,7 @@ proc sqlite-check-line-editing {} { foreach f $lnCOpts { if {[file exists $dirLn/$f]} { set lnC $dirLn/$f - break; + break } } if {"" eq $lnC} { @@ -1134,6 +1193,8 @@ proc sqlite-check-line-editing {} { if {$::sqliteConfig(use-jim-for-codegen) && 2 == $lnVal} { define-append CFLAGS_JIMSH -DUSE_LINENOISE [get-define CFLAGS_READLINE] user-notice "Adding linenoise support to jimsh." + } else { + msg-result "Using linenoise at [file-normalize $dirLn]" } return "linenoise ($flavor)" } elseif {[opt-bool editline]} { @@ -1223,16 +1284,18 @@ proc sqlite-check-line-editing {} { set rlLib [opt-val with-readline-ldflags] #proc-debug "rlLib=$rlLib" if {$rlLib in {auto ""}} { - set rlLib "" - set libTerm "" - if {[proj-check-function-in-lib tgetent "$editLibName ncurses curses termcap"]} { + set rlLib "" ; # make sure it's not "auto", as we may append to it below + set libTerm ""; # lib with tgetent(3) + if {[proj-check-function-in-lib tgetent [list $editLibName ncurses curses termcap]]} { # ^^^ that libs list comes from the legacy configure script ^^^ set libTerm [get-define lib_tgetent] undefine lib_tgetent } if {$editLibName eq $libTerm} { + # tgetent(3) was found in the editing library set rlLib $libTerm } elseif {[proj-check-function-in-lib readline $libsForReadline $libTerm]} { + # tgetent(3) was found in an external lib set rlLib [get-define lib_readline] lappend rlLib $libTerm undefine lib_readline @@ -1262,8 +1325,8 @@ proc sqlite-check-line-editing {} { msg-result "Using $editLibName flags: $rlInc $rlLib" # Check whether rl_completion_matches() has a signature we can use # and disable that sub-feature if it doesn't. - if {![cctest \ - -cflags "$rlInc -D${editLibDef}" -libs $rlLib -nooutput 1 -source { + if {![cctest -cflags "$rlInc -D${editLibDef}" -libs $rlLib -nooutput 1 \ + -source { #include <stdio.h> #ifdef HAVE_EDITLINE #include <editline/readline.h> @@ -1316,7 +1379,7 @@ proc sqlite-handle-line-editing {} { # - pkg-config: use only pkg-config to determine flags # - /path/to/icu-config: use that to determine flags # -# If --with-icu-config is used as neither pkg-config nor icu-config +# If --with-icu-config is used and neither pkg-config nor icu-config # are found, fail fatally. # # If both --with-icu-ldflags and --with-icu-config are provided, they @@ -1409,39 +1472,50 @@ proc sqlite-handle-icu {} { # Makes the following environment changes: # # - defines LDFLAGS_DLOPEN to any linker flags needed for this -# feature. It may legally be empty on some systems where dlopen() -# is in libc. +# feature. It may legally be empty on (A) some systems where +# dlopen() is in libc and (B) certain Unix-esque Windows +# environments which identify as Windows for SQLite's purposes so +# use LoadLibrary(). # # - If the feature is not available, adds # -DSQLITE_OMIT_LOAD_EXTENSION=1 to the feature flags list. proc sqlite-handle-load-extension {} { define LDFLAGS_DLOPEN "" set found 0 + set suffix "" proj-if-opt-truthy load-extension { - set found [proj-check-function-in-lib dlopen dl] - if {$found} { - define LDFLAGS_DLOPEN [get-define lib_dlopen] - undefine lib_dlopen - } else { - if {[proj-opt-was-provided load-extension]} { - # Explicit --enable-load-extension: fail if not found - proj-indented-notice -error { - --enable-load-extension was provided but dlopen() - not found. Use --disable-load-extension to bypass this - check. - } - } else { - # It was implicitly enabled: warn if not found - proj-indented-notice { - WARNING: dlopen() not found, so loadable module support will - be disabled. Use --disable-load-extension to bypass this - check. + switch -glob -- [get-define host] { + *-*-mingw* - *windows* { + incr found + set suffix "Using LoadLibrary()" + } + default { + set found [proj-check-function-in-lib dlopen dl] + if {$found} { + set suffix [define LDFLAGS_DLOPEN [get-define lib_dlopen]] + undefine lib_dlopen + } else { + if {[proj-opt-was-provided load-extension]} { + # Explicit --enable-load-extension: fail if not found + proj-indented-notice -error { + --enable-load-extension was provided but dlopen() + not found. Use --disable-load-extension to bypass this + check. + } + } else { + # It was implicitly enabled: warn if not found + proj-indented-notice { + WARNING: dlopen() not found, so loadable module support will + be disabled. Use --disable-load-extension to bypass this + check. + } + } } } } } if {$found} { - msg-result "Loadable extension support enabled." + msg-result "Loadable extension support enabled. $suffix" } else { msg-result "Disabling loadable extension support. Use --enable-load-extension to enable them." sqlite-add-feature-flag -DSQLITE_OMIT_LOAD_EXTENSION=1 @@ -1458,7 +1532,7 @@ proc sqlite-handle-math {} { } define LDFLAGS_MATH [get-define lib_ceil] undefine lib_ceil - sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS + sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS -DSQLITE_ENABLE_PERCENTILE msg-result "Enabling math SQL functions" } { define LDFLAGS_MATH "" @@ -1520,6 +1594,19 @@ proc sqlite-handle-mac-install-name {} { return $rc } +# +# Checks specific to HP-UX. +# +proc sqlite-handle-hpux {} { + switch -glob -- [get-define host] { + *hpux* { + if {[cc-check-flags "-Ae"]} { + define-append CFLAGS -Ae + } + } + } +} + ######################################################################## # Handles the --dll-basename configure flag. [define]'s # SQLITE_DLL_BASENAME to the DLL's preferred base name (minus @@ -1969,13 +2056,14 @@ proc sqlite-check-tcl {} { # TCLLIBDIR from here, which will cause the canonical makefile to # use this one rather than to re-calculate it at make-time. set tcllibdir [get-env TCLLIBDIR ""] + set sq3Ver [get-define PACKAGE_VERSION] if {"" eq $tcllibdir} { # Attempt to extract TCLLIBDIR from TCL's $auto_path if {"" ne $with_tclsh && [catch {exec echo "puts stdout \$auto_path" | "$with_tclsh"} result] == 0} { foreach i $result { if {[file isdir $i]} { - set tcllibdir $i/sqlite3 + set tcllibdir $i/sqlite${sq3Ver} break } } @@ -2111,15 +2199,31 @@ proc sqlite-determine-codegen-tcl {} { # sqlite-determine-codegen-tcl. proc sqlite-handle-tcl {} { sqlite-check-tcl - if {"canonical" eq $::sqliteConfig(build-mode)} { - msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + if {"canonical" ne $::sqliteConfig(build-mode)} return + msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + + # Determine the base name of the Tcl extension's DLL + # + if {[get-define HAVE_TCL]} { + if {[string match *-cygwin [get-define host]]} { + set libname cyg + } else { + set libname lib + } + if {[get-define TCL_MAJOR_VERSION] > 8} { + append libname tcl9 + } + append libname sqlite + } else { + set libname "" } + define TCL_EXT_DLL_BASENAME $libname + # The extension is added in the makefile } ######################################################################## # Handle the --enable/disable-rpath flag. proc sqlite-handle-rpath {} { - proj-check-rpath # autosetup/cc-shared.tcl sets the rpath flag definition in # [get-define SH_LINKRPATH], but it does so on a per-platform basis # rather than as a compiler check. Though we should do a proper @@ -2128,12 +2232,13 @@ proc sqlite-handle-rpath {} { # for which sqlite-env-is-unix-on-windows returns a non-empty # string. -# if {[proj-opt-truthy rpath]} { -# proj-check-rpath -# } else { -# msg-result "Disabling use of rpath." -# define LDFLAGS_RPATH "" -# } + # https://sqlite.org/forum/forumpost/13cac3b56516f849 + if {[proj-opt-truthy rpath]} { + proj-check-rpath + } else { + msg-result "Disabling use of rpath." + define LDFLAGS_RPATH "" + } } ######################################################################## diff --git a/contrib/sqlite3/autosetup/teaish/core.tcl b/contrib/sqlite3/autosetup/teaish/core.tcl index 09017029d77f..c9abfa0626c0 100644 --- a/contrib/sqlite3/autosetup/teaish/core.tcl +++ b/contrib/sqlite3/autosetup/teaish/core.tcl @@ -92,6 +92,7 @@ array set teaish__Config [proj-strip-hash-comments { -tm.tcl.in TEAISH_TM_TCL_IN -options {} -pragmas {} + -src {} } # @@ -219,7 +220,9 @@ proc teaish-configure-core {} { => {Full pathname of tclsh to use. It is used for trying to find tclConfig.sh. Warning: if its containing dir has multiple tclsh versions, it may select the wrong tclConfig.sh! - Defaults to the $TCLSH environment variable.} + Defaults to the $TCLSH environment variable.} + + tcl-stubs=0 => {Enable use of Tcl stubs library.} # TEA has --with-tclinclude but it appears to only be useful for # building an extension against an uninstalled copy of TCL's own @@ -331,29 +334,33 @@ proc teaish-configure-core {} { -url - -v "" -tm.tcl - -v "" -tm.tcl.in - -v "" + -src - -v "" } { + #proj-assert 0 {Just testing} set isPIFlag [expr {"-" ne $pflag}] if {$isPIFlag} { if {[info exists ::teaish__PkgInfo($pflag)]} { # Was already set - skip it. continue; } - proj-assert {{-} eq $key} + proj-assert {{-} eq $key};# "Unexpected pflag=$pflag key=$key type=$type val=$val" set key $f2d($pflag) } - proj-assert {"" ne $key} - set got [get-define $key "<nope>"] - if {"<nope>" ne $got} { - # Was already set - skip it. - continue + if {"" ne $key} { + if {"<nope>" ne [get-define $key "<nope>"]} { + # Was already set - skip it. + continue + } } switch -exact -- $type { -v {} -e { set val [eval $val] } default { proj-error "Invalid type flag: $type" } } - #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag got=$got" - define $key $val + #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag" + if {$key ne ""} { + define $key $val + } if {$isPIFlag} { set ::teaish__PkgInfo($pflag) $val } @@ -493,6 +500,8 @@ proc teaish__configure_phase1 {} { } teaish-checks-run -post + define TEAISH_USE_STUBS [opt-bool tcl-stubs] + apply {{} { # Set up "vsatisfies" code for pkgIndex.tcl.in, # _teaish.tester.tcl.in, and for a configure-time check. We would @@ -541,10 +550,10 @@ proc teaish__configure_phase1 {} { define TEAISH_VSATISFIES_CODE [join $code "\n"] }}; # vsatisfies - if {[proj-looks-like-windows] || [proj-looks-like-mac]} { + if {[proj-looks-like-windows]} { # Without this, linking of an extension will not work on Cygwin or # Msys2. - msg-result "Using USE_TCL_STUBS for this environment" + msg-result "Using USE_TCL_STUBS for Unix(ish)-on-Windows environment" teaish-cflags-add -DUSE_TCL_STUBS=1 } @@ -585,7 +594,8 @@ proc teaish__configure_phase1 {} { # if {0x0f & $::teaish__Config(pkginit-policy)} { file delete -force -- [get-define TEAISH_PKGINIT_TCL] - proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] + proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] \ + [get-define TEAISH_PKGINIT_TCL] } if {0x0f & $::teaish__Config(tm-policy)} { file delete -force -- [get-define TEAISH_TM_TCL] @@ -595,17 +605,20 @@ proc teaish__configure_phase1 {} { apply {{} { # Queue up any remaining dot-in files set dotIns [list] - foreach d { - TEAISH_TESTER_TCL_IN - TEAISH_TEST_TCL_IN - TEAISH_MAKEFILE_IN + foreach {dIn => dOut} { + TEAISH_TESTER_TCL_IN => TEAISH_TESTER_TCL + TEAISH_TEST_TCL_IN => TEAISH_TEST_TCL + TEAISH_MAKEFILE_IN => TEAISH_MAKEFILE } { - lappend dotIns [get-define $d ""] + lappend dotIns [get-define $dIn ""] [get-define $dOut ""] } - lappend dotIns $::autosetup(srcdir)/Makefile.in; # must be after TEAISH_MAKEFILE_IN - foreach f $dotIns { - if {"" ne $f} { - proj-dot-ins-append $f + lappend dotIns $::autosetup(srcdir)/Makefile.in Makefile; # must be after TEAISH_MAKEFILE_IN. + # Much later: probably because of timestamps for deps purposes :-? + #puts "dotIns=$dotIns" + foreach {i o} $dotIns { + if {"" ne $i && "" ne $o} { + #puts " pre-dot-ins-append: \[$i\] -> \[$o\]" + proj-dot-ins-append $i $o } } }} @@ -640,10 +653,10 @@ proc teaish__configure_phase1 {} { # # NO [define]s after this point! # - proj-dot-ins-process -validate proj-if-opt-truthy teaish-dump-defines { proj-file-write config.defines.txt $tdefs } + proj-dot-ins-process -validate }; # teaish__configure_phase1 @@ -1068,7 +1081,7 @@ If you are attempting an out-of-tree build, use ]]} { if {[string match *.in $extM]} { define TEAISH_MAKEFILE_IN $extM - define TEAISH_MAKEFILE [file rootname [file tail $extM]] + define TEAISH_MAKEFILE _[file rootname [file tail $extM]] } else { define TEAISH_MAKEFILE_IN "" define TEAISH_MAKEFILE $extM @@ -1136,8 +1149,8 @@ If you are attempting an out-of-tree build, use set flist [list $dirExt/teaish.test.tcl.in $dirExt/teaish.test.tcl] if {[proj-first-file-found ttt $flist]} { if {[string match *.in $ttt]} { - # Generate teaish.test.tcl from $ttt - set xt [file rootname [file tail $ttt]] + # Generate _teaish.test.tcl from $ttt + set xt _[file rootname [file tail $ttt]] file delete -force -- $xt; # ensure no stale copy is used define TEAISH_TEST_TCL $xt define TEAISH_TEST_TCL_IN $ttt @@ -1304,7 +1317,6 @@ proc teaish-ldflags-prepend {args} { # object files (which are typically in the build tree)). # proc teaish-src-add {args} { - set i 0 proj-parse-simple-flags args flags { -dist 0 {expr 1} -dir 0 {expr 1} @@ -1389,7 +1401,7 @@ proc teaish__cleanup_rule {{tgt clean}} { return ${tgt}-_${x}_ } -# @teaish-make-obj objfile srcfile ?...args? +# @teaish-make-obj ?flags? ?...args? # # Uses teaish-make-add to inject makefile rules for $objfile from # $srcfile, which is assumed to be C code which uses libtcl. Unless @@ -1403,43 +1415,45 @@ proc teaish__cleanup_rule {{tgt clean}} { # Any arguments after the 2nd may be flags described below or, if no # -recipe is provided, flags for the compiler call. # +# -obj obj-filename.o +# +# -src src-filename.c +# # -recipe {...} # Uses the trimmed value of {...} as the recipe, prefixing it with # a single hard-tab character. # # -deps {...} -# List of extra files to list as dependencies of $o. Good luck -# escaping non-trivial cases properly. +# List of extra files to list as dependencies of $o. # # -clean # Generate cleanup rules as well. -proc teaish-make-obj {o src args} { - set consume 0 - set clean 0 - set flag "" - array set flags {} - set xargs {} - foreach arg $args { - if {$consume} { - set consume 0 - set flags($flag) $arg - continue - } - switch -exact -- $arg { - -clean {incr clean} - -recipe - - -deps { - set flag $arg - incr consume - } - default { - lappend xargs $arg - } +proc teaish-make-obj {args} { + proj-parse-simple-flags args flags { + -clean 0 {expr 1} + -recipe => {} + -deps => {} + -obj => {} + -src => {} + } + #parray flags + if {"" eq $flags(-obj)} { + set args [lassign $args flags(-obj)] + if {"" eq $flags(-obj)} { + proj-error "Missing -obj flag." } } + foreach f {-deps -src} { + set flags($f) [string trim [string map {\n " "} $flags($f)]] + } + foreach f {-deps -src} { + set flags($f) [string trim $flags($f)] + } + #parray flags + #puts "-- args=$args" teaish-make-add \ - "# [proj-scope 1] -> [proj-scope] $o $src" -nl \ - "$o: $src $::teaish__Config(teaish.tcl)" + "# [proj-scope 1] -> [proj-scope] $flags(-obj) $flags(-src)" -nl \ + "$flags(-obj): $flags(-src) $::teaish__Config(teaish.tcl)" if {[info exists flags(-deps)]} { teaish-make-add " " [join $flags(-deps)] } @@ -1447,12 +1461,12 @@ proc teaish-make-obj {o src args} { if {[info exists flags(-recipe)]} { teaish-make-add [string trim $flags(-recipe)] -nl } else { - teaish-make-add [join [list \$(CC.tcl) -c $src {*}$xargs]] -nl + teaish-make-add [join [list \$(CC.tcl) -c $flags(-src) {*}$args]] -nl } - if {$clean} { + if {$flags(-clean)} { set rule [teaish__cleanup_rule] teaish-make-add \ - "clean: $rule\n$rule:\n\trm -f \"$o\"\n" + "clean: $rule\n$rule:\n\trm -f \"$flags(-obj)\"\n" } } @@ -2080,6 +2094,17 @@ proc teaish-pkginfo-set {args} { set v $x } + -src { + set d $::teaish__Config(extension-dir) + foreach f $v { + lappend ::teaish__Config(dist-files) $f + lappend ::teaish__Config(extension-src) $d/$f + lappend ::teaish__PkgInfo(-src) $f + # ^^^ so that default-value initialization in + # teaish-configure-core recognizes that it's been set. + } + } + -tm.tcl - -tm.tcl.in { if {0x30 & $::teaish__Config(pkgindex-policy)} { @@ -2517,7 +2542,7 @@ proc teaish__install {{dDest ""}} { ] { teaish__verbose 1 msg-result "Copying files to $destDir..." file mkdir $destDir - foreach f [glob -directory $srcDir *] { + foreach f [glob -nocomplain -directory $srcDir *] { if {[string match {*~} $f] || [string match "#*#" [file tail $f]]} { # Editor-generated backups and emacs lock files continue diff --git a/contrib/sqlite3/autosetup/teaish/tester.tcl b/contrib/sqlite3/autosetup/teaish/tester.tcl index d8b5f7a0e8c2..a25b366e8d76 100644 --- a/contrib/sqlite3/autosetup/teaish/tester.tcl +++ b/contrib/sqlite3/autosetup/teaish/tester.tcl @@ -99,7 +99,7 @@ proc test__affert {failMode args} { lassign $args script msg } incr ::test__Counters($what) - if {![uplevel 1 [concat expr [list $script]]]} { + if {![uplevel 1 expr [list $script]]} { if {"" eq $msg} { set msg $script } @@ -137,6 +137,40 @@ proc assert {args} { } # +# @assert-matches ?-e? pattern ?-e? rhs ?msg? +# +# Equivalent to assert {[string match $pattern $rhs]} except that +# if either of those are prefixed with an -e flag, they are eval'd +# and their results are used. +# +proc assert-matches {args} { + set evalLhs 0 + set evalRhs 0 + if {"-e" eq [lindex $args 0]} { + incr evalLhs + set args [lassign $args -] + } + set args [lassign $args pattern] + if {"-e" eq [lindex $args 0]} { + incr evalRhs + set args [lassign $args -] + } + set args [lassign $args rhs msg] + + if {$evalLhs} { + set pattern [uplevel 1 $pattern] + } + if {$evalRhs} { + set rhs [uplevel 1 $rhs] + } + #puts "***pattern=$pattern\n***rhs=$rhs" + tailcall test__affert 1 \ + [join [list \[ string match [list $pattern] [list $rhs] \]]] $msg + # why does this not work? [list \[ string match [list $pattern] [list $rhs] \]] $msg + # "\[string match [list $pattern] [list $rhs]\]" +} + +# # @test-assert testId script ?msg? # # Works like [assert] but emits $testId to stdout first. @@ -157,7 +191,7 @@ proc test-expect {testId script result} { puts "test $testId" set x [string trim [uplevel 1 $script]] set result [string trim $result] - tailcall test__affert 0 [list $x eq $result] \ + tailcall test__affert 0 [list "{$x}" eq "{$result}"] \ "\nEXPECTED: <<$result>>\nGOT: <<$x>>" } @@ -169,7 +203,7 @@ proc test-expect {testId script result} { # proc test-catch {cmd args} { if {[catch { - $cmd {*}$args + uplevel 1 $cmd {*}$args } rc xopts]} { puts "[test-current-scope] ignoring failure of: $cmd [lindex $args 0]: $rc" return 1 @@ -177,6 +211,37 @@ proc test-catch {cmd args} { return 0 } +# +# @test-catch-matching pattern (script|cmd args...) +# +# Works like test-catch, but it expects its argument(s) to to throw an +# error matching the given string (checked with [string match]). If +# they do not throw, or the error does not match $pattern, this +# function throws, else it returns 1. +# +# If there is no second argument, the $cmd is assumed to be a script, +# and will be eval'd in the caller's scope. +# +# TODO: add -glob and -regex flags to control matching flavor. +# +proc test-catch-matching {pattern cmd args} { + if {[catch { + #puts "**** catch-matching cmd=$cmd args=$args" + if {0 == [llength $args]} { + uplevel 1 $cmd {*}$args + } else { + $cmd {*}$args + } + } rc xopts]} { + if {[string match $pattern $rc]} { + return 1 + } else { + error "[test-current-scope] exception does not match {$pattern}: {$rc}" + } + } + error "[test-current-scope] expecting to see an error matching {$pattern}" +} + if {![array exists ::teaish__BuildFlags]} { array set ::teaish__BuildFlags {} } diff --git a/contrib/sqlite3/make.bat b/contrib/sqlite3/make.bat new file mode 100644 index 000000000000..2dc9b61c0e18 --- /dev/null +++ b/contrib/sqlite3/make.bat @@ -0,0 +1,2 @@ +@echo off +nmake /f Makefile.msc %* diff --git a/contrib/sqlite3/shell.c b/contrib/sqlite3/shell.c index 0d5bf19e74d5..57faceb4691f 100644 --- a/contrib/sqlite3/shell.c +++ b/contrib/sqlite3/shell.c @@ -1,21 +1,45 @@ -/* DO NOT EDIT! -** This file is automatically generated by the script in the canonical -** SQLite source tree at tool/mkshellc.tcl. That script combines source -** code from various constituent source files of SQLite into this single -** "shell.c" file used to implement the SQLite command-line shell. -** -** Most of the code found below comes from the "src/shell.c.in" file in -** the canonical SQLite source tree. That main file contains "INCLUDE" -** lines that specify other files in the canonical source tree that are -** inserted to getnerate this complete program source file. +/* +** This is the amalgamated source code to the "sqlite3" or "sqlite3.exe" +** command-line shell (CLI) for SQLite. This file is automatically +** generated by the tool/mkshellc.tcl script from the following sources: ** -** The code from multiple files is combined into this single "shell.c" -** source file to help make the command-line program easier to compile. +** ext/expert/sqlite3expert.c +** ext/expert/sqlite3expert.h +** ext/intck/sqlite3intck.c +** ext/intck/sqlite3intck.h +** ext/misc/appendvfs.c +** ext/misc/base64.c +** ext/misc/base85.c +** ext/misc/completion.c +** ext/misc/decimal.c +** ext/misc/fileio.c +** ext/misc/ieee754.c +** ext/misc/memtrace.c +** ext/misc/pcachetrace.c +** ext/misc/regexp.c +** ext/misc/series.c +** ext/misc/sha1.c +** ext/misc/shathree.c +** ext/misc/sqlar.c +** ext/misc/sqlite3_stdio.c +** ext/misc/sqlite3_stdio.h +** ext/misc/stmtrand.c +** ext/misc/uint.c +** ext/misc/vfstrace.c +** ext/misc/windirent.h +** ext/misc/zipfile.c +** ext/qrf/qrf.c +** ext/qrf/qrf.h +** ext/recover/dbdata.c +** ext/recover/sqlite3recover.c +** ext/recover/sqlite3recover.h +** src/shell.c.in ** ** To modify this program, get a copy of the canonical SQLite source tree, -** edit the src/shell.c.in" and/or some of the other files that are included -** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. +** edit the src/shell.c.in file and/or some of the other files that are +** listed above, then rerun the command "make shell.c". */ +/************************* Begin src/shell.c.in ******************/ /* ** 2001 September 15 ** @@ -27,7 +51,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains code to implement the "sqlite" command line +** This file contains code to implement the "sqlite3" command line ** utility for accessing SQLite databases. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) @@ -38,6 +62,22 @@ typedef unsigned int u32; typedef unsigned short int u16; /* +** Limit input nesting via .read or any other input redirect. +** It's not too expensive, so a generous allowance can be made. +*/ +#define MAX_INPUT_NESTING 25 + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + +/* ** Optionally #include a user-defined header, whereby compilation options ** may be set prior to where they take effect, but after platform setup. ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include @@ -50,14 +90,6 @@ typedef unsigned short int u16; #endif /* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - -/* ** If SQLITE_SHELL_FIDDLE is defined then the shell is modified ** somewhat for use as a WASM module in a web browser. This flag ** should only be used when building the "fiddle" web application, as @@ -65,6 +97,10 @@ typedef unsigned short int u16; ** and this build mode rewires the user input subsystem to account for ** that. */ +#if defined(SQLITE_SHELL_FIDDLE) +# undef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif /* ** Warning pragmas copied from msvc.h in the core. @@ -118,12 +154,17 @@ typedef unsigned short int u16; #include <stdio.h> #include <assert.h> #include <math.h> +#include <stdint.h> #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; typedef unsigned char u8; #include <ctype.h> #include <stdarg.h> +#ifndef _WIN32 +# include <sys/time.h> +# include <sys/ioctl.h> +#endif #if !defined(_WIN32) && !defined(WIN32) # include <signal.h> @@ -192,9 +233,6 @@ typedef unsigned char u8; #endif #if defined(_WIN32) || defined(WIN32) -# if SQLITE_OS_WINRT -# define SQLITE_OMIT_POPEN 1 -# else # include <io.h> # include <fcntl.h> # define isatty(h) _isatty(h) @@ -209,7 +247,6 @@ typedef unsigned char u8; # endif # undef pclose # define pclose _pclose -# endif #else /* Make sure isatty() has a prototype. */ extern int isatty(int); @@ -240,9 +277,6 @@ typedef unsigned char u8; #define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) -#if SQLITE_OS_WINRT -#include <intrin.h> -#endif #undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include <windows.h> @@ -252,7 +286,7 @@ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif -/************************* Begin ../ext/misc/sqlite3_stdio.h ******************/ +/************************* Begin ext/misc/sqlite3_stdio.h ******************/ /* ** 2024-09-24 ** @@ -286,6 +320,7 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #ifdef _WIN32 /**** Definitions For Windows ****/ #include <stdio.h> +#include <stdarg.h> #include <windows.h> FILE *sqlite3_fopen(const char *zFilename, const char *zMode); @@ -293,6 +328,7 @@ FILE *sqlite3_popen(const char *zCommand, const char *type); char *sqlite3_fgets(char *s, int size, FILE *stream); int sqlite3_fputs(const char *s, FILE *stream); int sqlite3_fprintf(FILE *stream, const char *format, ...); +int sqlite3_vfprintf(FILE *stream, const char *format, va_list); void sqlite3_fsetmode(FILE *stream, int mode); @@ -304,13 +340,14 @@ void sqlite3_fsetmode(FILE *stream, int mode); #define sqlite3_fgets fgets #define sqlite3_fputs fputs #define sqlite3_fprintf fprintf +#define sqlite3_vfprintf vfprintf #define sqlite3_fsetmode(F,X) /*no-op*/ #endif #endif /* _SQLITE3_STDIO_H_ */ -/************************* End ../ext/misc/sqlite3_stdio.h ********************/ -/************************* Begin ../ext/misc/sqlite3_stdio.c ******************/ +/************************* End ext/misc/sqlite3_stdio.h ********************/ +/************************* Begin ext/misc/sqlite3_stdio.c ******************/ /* ** 2024-09-24 ** @@ -414,8 +451,8 @@ FILE *sqlite3_fopen(const char *zFilename, const char *zMode){ sz1 = (int)strlen(zFilename); sz2 = (int)strlen(zMode); - b1 = sqlite3_malloc( (sz1+1)*sizeof(b1[0]) ); - b2 = sqlite3_malloc( (sz2+1)*sizeof(b1[0]) ); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); if( b1 && b2 ){ sz1 = MultiByteToWideChar(CP_UTF8, 0, zFilename, sz1, b1, sz1); b1[sz1] = 0; @@ -440,8 +477,8 @@ FILE *sqlite3_popen(const char *zCommand, const char *zMode){ sz1 = (int)strlen(zCommand); sz2 = (int)strlen(zMode); - b1 = sqlite3_malloc( (sz1+1)*sizeof(b1[0]) ); - b2 = sqlite3_malloc( (sz2+1)*sizeof(b1[0]) ); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); if( b1 && b2 ){ sz1 = MultiByteToWideChar(CP_UTF8, 0, zCommand, sz1, b1, sz1); b1[sz1] = 0; @@ -464,7 +501,7 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){ ** that into UTF-8. Otherwise, non-ASCII characters all get translated ** into '?'. */ - wchar_t *b1 = sqlite3_malloc( sz*sizeof(wchar_t) ); + wchar_t *b1 = sqlite3_malloc64( sz*sizeof(wchar_t) ); if( b1==0 ) return 0; #ifdef SQLITE_USE_W32_FOR_CONSOLE_IO DWORD nRead = 0; @@ -539,7 +576,7 @@ int sqlite3_fputs(const char *z, FILE *out){ ** to the console on Windows. */ int sz = (int)strlen(z); - wchar_t *b1 = sqlite3_malloc( (sz+1)*sizeof(wchar_t) ); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(wchar_t) ); if( b1==0 ) return 0; sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz); b1[sz] = 0; @@ -571,7 +608,7 @@ int sqlite3_fputs(const char *z, FILE *out){ /* -** Work-alike for fprintf() from the standard C library. +** Work-alikes for fprintf() and vfprintf() from the standard C library. */ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ int rc; @@ -598,6 +635,24 @@ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ } return rc; } +int sqlite3_vfprintf(FILE *out, const char *zFormat, va_list ap){ + int rc; + if( UseWtextForOutput(out) ){ + /* When writing to the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode and write UTF-16 characters. + */ + char *z; + z = sqlite3_vmprintf(zFormat, ap); + sqlite3_fputs(z, out); + rc = (int)strlen(z); + sqlite3_free(z); + }else{ + /* Writing to a file or other destination, just write bytes without + ** any translation. */ + rc = vfprintf(out, zFormat, ap); + } + return rc; +} /* ** Set the mode for an output stream. mode argument is typically _O_BINARY or @@ -616,393 +671,694 @@ void sqlite3_fsetmode(FILE *fp, int mode){ #endif /* defined(_WIN32) */ -/************************* End ../ext/misc/sqlite3_stdio.c ********************/ - -/* Use console I/O package as a direct INCLUDE. */ -#define SQLITE_INTERNAL_LINKAGE static - -#ifdef SQLITE_SHELL_FIDDLE -/* Deselect most features from the console I/O package for Fiddle. */ -# define SQLITE_CIO_NO_REDIRECT -# define SQLITE_CIO_NO_CLASSIFY -# define SQLITE_CIO_NO_TRANSLATE -# define SQLITE_CIO_NO_SETMODE -# define SQLITE_CIO_NO_FLUSH +/************************* End ext/misc/sqlite3_stdio.c ********************/ +/************************* Begin ext/qrf/qrf.h ******************/ +/* +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. +*/ +#ifndef SQLITE_QRF_H +#define SQLITE_QRF_H +#ifdef __cplusplus +extern "C" { #endif +#include <stdlib.h> +/* #include "sqlite3.h" */ -#define eputz(z) sqlite3_fputs(z,stderr) -#define sputz(fp,z) sqlite3_fputs(z,fp) - -/* True if the timer is enabled */ -static int enableTimer = 0; - -/* A version of strcmp() that works with NULL values */ -static int cli_strcmp(const char *a, const char *b){ - if( a==0 ) a = ""; - if( b==0 ) b = ""; - return strcmp(a,b); -} -static int cli_strncmp(const char *a, const char *b, size_t n){ - if( a==0 ) a = ""; - if( b==0 ) b = ""; - return strncmp(a,b,n); -} - -/* Return the current wall-clock time */ -static sqlite3_int64 timeOfDay(void){ - static sqlite3_vfs *clockVfs = 0; - sqlite3_int64 t; - if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); - if( clockVfs==0 ) return 0; /* Never actually happens */ - if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ - clockVfs->xCurrentTimeInt64(clockVfs, &t); - }else{ - double r; - clockVfs->xCurrentTime(clockVfs, &r); - t = (sqlite3_int64)(r*86400000.0); - } - return t; -} - -#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) -#include <sys/time.h> -#include <sys/resource.h> - -/* VxWorks does not support getrusage() as far as we can determine */ -#if defined(_WRS_KERNEL) || defined(__RTP__) -struct rusage { - struct timeval ru_utime; /* user CPU time used */ - struct timeval ru_stime; /* system CPU time used */ +/* +** Specification used by clients to define the output format they want +*/ +typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; +struct sqlite3_qrf_spec { + unsigned char iVersion; /* Version number of this structure */ + unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ + unsigned char eEsc; /* How to escape control characters in text */ + unsigned char eText; /* Quoting style for text */ + unsigned char eTitle; /* Quating style for the text of column names */ + unsigned char eBlob; /* Quoting style for BLOBs */ + unsigned char bTitles; /* True to show column names */ + unsigned char bWordWrap; /* Try to wrap on word boundaries */ + unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ + unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ + unsigned char eTitleAlign; /* Alignment for column headers */ + unsigned char bSplitColumn; /* Wrap single-column output into many columns */ + unsigned char bBorder; /* Show outer border in Box and Table styles */ + short int nWrap; /* Wrap columns wider than this */ + short int nScreenWidth; /* Maximum overall table width */ + short int nLineLimit; /* Maximum number of lines for any row */ + short int nTitleLimit; /* Maximum number of characters in a title */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ + int nCharLimit; /* Maximum number of characters in a cell */ + int nWidth; /* Number of entries in aWidth[] */ + int nAlign; /* Number of entries in aAlignment[] */ + short int *aWidth; /* Column widths */ + unsigned char *aAlign; /* Column alignments */ + char *zColumnSep; /* Alternative column separator */ + char *zRowSep; /* Alternative row separator */ + char *zTableName; /* Output table name */ + char *zNull; /* Rendering of NULL */ + char *(*xRender)(void*,sqlite3_value*); /* Render a value */ + int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ + void *pRenderArg; /* First argument to the xRender callback */ + void *pWriteArg; /* First argument to the xWrite callback */ + char **pzOutput; /* Storage location for output string */ + /* Additional fields may be added in the future */ }; -#define getrusage(A,B) memset(B,0,sizeof(*B)) -#endif +/* +** Interfaces +*/ +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* SQL statement to run */ + const sqlite3_qrf_spec *pSpec, /* Result format specification */ + char **pzErr /* OUT: Write error message here */ +); -/* Saved resource information for the beginning of an operation */ -static struct rusage sBegin; /* CPU time at start */ -static sqlite3_int64 iBegin; /* Wall-clock time at start */ +/* +** Range of values for sqlite3_qrf_spec.aWidth[] entries and for +** sqlite3_qrf_spec.mxColWidth and .nScreenWidth +*/ +#define QRF_MAX_WIDTH 10000 +#define QRF_MIN_WIDTH 0 /* -** Begin timing an operation +** Output styles: */ -static void beginTimer(void){ - if( enableTimer ){ - getrusage(RUSAGE_SELF, &sBegin); - iBegin = timeOfDay(); - } -} +#define QRF_STYLE_Auto 0 /* Choose a style automatically */ +#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ +#define QRF_STYLE_Column 2 /* One record per line in neat columns */ +#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ +#define QRF_STYLE_Csv 4 /* Comma-separated-value */ +#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ +#define QRF_STYLE_Explain 6 /* EXPLAIN output */ +#define QRF_STYLE_Html 7 /* Generate an XHTML table */ +#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ +#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ +#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ +#define QRF_STYLE_Line 11 /* One column per line. */ +#define QRF_STYLE_List 12 /* One record per line with a separator */ +#define QRF_STYLE_Markdown 13 /* Markdown formatting */ +#define QRF_STYLE_Off 14 /* No query output shown */ +#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ +#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ +#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ +#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ +#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ -/* Return the difference of two time_structs in seconds */ -static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + - (double)(pEnd->tv_sec - pStart->tv_sec); -} +/* +** Quoting styles for text. +** Allowed values for sqlite3_qrf_spec.eText +*/ +#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ +#define QRF_TEXT_Plain 1 /* Literal text */ +#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ +#define QRF_TEXT_Csv 3 /* CSV-style quoting */ +#define QRF_TEXT_Html 4 /* HTML-style quoting */ +#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ +#define QRF_TEXT_Json 6 /* JSON quoting */ +#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ /* -** Print the timing results. +** Quoting styles for BLOBs +** Allowed values for sqlite3_qrf_spec.eBlob */ -static void endTimer(FILE *out){ - if( enableTimer ){ - sqlite3_int64 iEnd = timeOfDay(); - struct rusage sEnd; - getrusage(RUSAGE_SELF, &sEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); - } -} +#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ +#define QRF_BLOB_Text 1 /* Display content exactly as it is */ +#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ +#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ +#define QRF_BLOB_Tcl 4 /* "\000" notation */ +#define QRF_BLOB_Json 5 /* A JSON string */ +#define QRF_BLOB_Size 6 /* Display the blob size only */ -#define BEGIN_TIMER beginTimer() -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER 1 +/* +** Control-character escape modes. +** Allowed values for sqlite3_qrf_spec.eEsc +*/ +#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ +#define QRF_ESC_Off 1 /* Do not escape control characters */ +#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ +#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ -#elif (defined(_WIN32) || defined(WIN32)) +/* +** Allowed values for "boolean" fields, such as "bColumnNames", "bWordWrap", +** and "bTextJsonb". There is an extra "auto" variants so these are actually +** tri-state settings, not booleans. +*/ +#define QRF_SW_Auto 0 /* Let QRF choose the best value */ +#define QRF_SW_Off 1 /* This setting is forced off */ +#define QRF_SW_On 2 /* This setting is forced on */ +#define QRF_Auto 0 /* Alternate spelling for QRF_*_Auto */ +#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ +#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ -/* Saved resource information for the beginning of an operation */ -static HANDLE hProcess; -static FILETIME ftKernelBegin; -static FILETIME ftUserBegin; -static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, - LPFILETIME, LPFILETIME); -static GETPROCTIMES getProcessTimesAddr = NULL; +/* +** Possible alignment values alignment settings +** +** Horizontal Vertial +** ---------- -------- */ +#define QRF_ALIGN_Auto 0 /* auto auto */ +#define QRF_ALIGN_Left 1 /* left auto */ +#define QRF_ALIGN_Center 2 /* center auto */ +#define QRF_ALIGN_Right 3 /* right auto */ +#define QRF_ALIGN_Top 4 /* auto top */ +#define QRF_ALIGN_NW 5 /* left top */ +#define QRF_ALIGN_N 6 /* center top */ +#define QRF_ALIGN_NE 7 /* right top */ +#define QRF_ALIGN_Middle 8 /* auto middle */ +#define QRF_ALIGN_W 9 /* left middle */ +#define QRF_ALIGN_C 10 /* center middle */ +#define QRF_ALIGN_E 11 /* right middle */ +#define QRF_ALIGN_Bottom 12 /* auto bottom */ +#define QRF_ALIGN_SW 13 /* left bottom */ +#define QRF_ALIGN_S 14 /* center bottom */ +#define QRF_ALIGN_SE 15 /* right bottom */ +#define QRF_ALIGN_HMASK 3 /* Horizontal alignment mask */ +#define QRF_ALIGN_VMASK 12 /* Vertical alignment mask */ /* -** Check to see if we have timer support. Return 1 if necessary -** support found (or found previously). +** Auxiliary routines contined within this module that might be useful +** in other contexts, and which are therefore exported. */ -static int hasTimer(void){ - if( getProcessTimesAddr ){ - return 1; - } else { -#if !SQLITE_OS_WINRT - /* GetProcessTimes() isn't supported in WIN95 and some other Windows - ** versions. See if the version we are running on has it, and if it - ** does, save off a pointer to it and the current process handle. - */ - hProcess = GetCurrentProcess(); - if( hProcess ){ - HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); - if( NULL != hinstLib ){ - getProcessTimesAddr = - (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); - if( NULL != getProcessTimesAddr ){ - return 1; - } - FreeLibrary(hinstLib); - } - } -#endif - } - return 0; -} +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int sqlite3_qrf_wcwidth(int c); /* -** Begin timing an operation +** Return an estimate of the number of display columns used by the +** string in the argument. The width of individual characters is +** determined as for sqlite3_qrf_wcwidth(). VT100 escape code sequences +** are assigned a width of zero. */ -static void beginTimer(void){ - if( enableTimer && getProcessTimesAddr ){ - FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess,&ftCreation,&ftExit, - &ftKernelBegin,&ftUserBegin); - ftWallBegin = timeOfDay(); - } -} +size_t sqlite3_qrf_wcswidth(const char*); -/* Return the difference of two FILETIME structs in seconds */ -static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ - sqlite_int64 i64Start = *((sqlite_int64 *) pStart); - sqlite_int64 i64End = *((sqlite_int64 *) pEnd); - return (double) ((i64End - i64Start) / 10000000.0); + +#ifdef __cplusplus } +#endif +#endif /* !defined(SQLITE_QRF_H) */ +/************************* End ext/qrf/qrf.h ********************/ +/************************* Begin ext/qrf/qrf.c ******************/ /* -** Print the timing results. +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. */ -static void endTimer(FILE *out){ - if( enableTimer && getProcessTimesAddr){ - FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; - sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - sqlite3_fprintf(out, "Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); - } -} - -#define BEGIN_TIMER beginTimer() -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER hasTimer() +#ifndef SQLITE_QRF_H +#include "qrf.h" +#endif +#include <string.h> +#include <assert.h> +#include <stdint.h> -#else -#define BEGIN_TIMER -#define END_TIMER(X) /*no-op*/ -#define HAS_TIMER 0 +#ifndef SQLITE_AMALGAMATION +/* typedef sqlite3_int64 i64; */ #endif +/* A single line in the EQP output */ +typedef struct qrfEQPGraphRow qrfEQPGraphRow; +struct qrfEQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + qrfEQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct qrfEQPGraph qrfEQPGraph; +struct qrfEQPGraph { + qrfEQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + qrfEQPGraphRow *pLast; /* Last element of the pRow list */ + int nWidth; /* Width of the graph */ + char zPrefix[400]; /* Graph prefix */ +}; + /* -** Used to prevent warnings about unused parameters +** Private state information. Subject to change from one release to the +** next. */ -#define UNUSED_PARAMETER(x) (void)(x) +typedef struct Qrf Qrf; +struct Qrf { + sqlite3_stmt *pStmt; /* The statement whose output is to be rendered */ + sqlite3 *db; /* The corresponding database connection */ + sqlite3_stmt *pJTrans; /* JSONB to JSON translator statement */ + char **pzErr; /* Write error message here, if not NULL */ + sqlite3_str *pOut; /* Accumulated output */ + int iErr; /* Error code */ + int nCol; /* Number of output columns */ + int expMode; /* Original sqlite3_stmt_isexplain() plus 1 */ + int mxWidth; /* Screen width */ + int mxHeight; /* nLineLimit */ + union { + struct { /* Content for QRF_STYLE_Line */ + int mxColWth; /* Maximum display width of any column */ + char **azCol; /* Names of output columns (MODE_Line) */ + } sLine; + qrfEQPGraph *pGraph; /* EQP graph (Eqp, Stats, and StatsEst) */ + struct { /* Content for QRF_STYLE_Explain */ + int nIndent; /* Slots allocated for aiIndent */ + int iIndent; /* Current slot */ + int *aiIndent; /* Indentation for each opcode */ + } sExpln; + unsigned int nIns; /* Bytes used for current INSERT stmt */ + } u; + sqlite3_int64 nRow; /* Number of rows handled so far */ + int *actualWidth; /* Actual width of each column */ + sqlite3_qrf_spec spec; /* Copy of the original spec */ +}; /* -** Number of elements in an array +** Data for substitute ctype.h functions. Used for x-platform +** consistency and so that '_' is counted as an alphabetic +** character. +** +** 0x01 - space +** 0x02 - digit +** 0x04 - alphabetic, including '_' */ -#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) +static const char qrfCType[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +#define qrfSpace(x) ((qrfCType[(unsigned char)x]&1)!=0) +#define qrfDigit(x) ((qrfCType[(unsigned char)x]&2)!=0) +#define qrfAlpha(x) ((qrfCType[(unsigned char)x]&4)!=0) +#define qrfAlnum(x) ((qrfCType[(unsigned char)x]&6)!=0) + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif /* -** If the following flag is set, then command execution stops -** at an error if we are not interactive. +** Set an error code and error message. */ -static int bail_on_error = 0; +static void qrfError( + Qrf *p, /* Query result state */ + int iCode, /* Error code */ + const char *zFormat, /* Message format (or NULL) */ + ... +){ + p->iErr = iCode; + if( p->pzErr!=0 ){ + sqlite3_free(*p->pzErr); + *p->pzErr = 0; + if( zFormat ){ + va_list ap; + va_start(ap, zFormat); + *p->pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + } + } +} /* -** Treat stdin as an interactive input if the following variable -** is true. Otherwise, assume stdin is connected to a file or pipe. +** Out-of-memory error. */ -static int stdin_is_interactive = 1; +static void qrfOom(Qrf *p){ + qrfError(p, SQLITE_NOMEM, "out of memory"); +} /* -** On Windows systems we need to know if standard output is a console -** in order to show that UTF-16 translation is done in the sign-on -** banner. The following variable is true if it is the console. +** Transfer any error in pStr over into p. */ -static int stdout_is_console = 1; +static void qrfStrErr(Qrf *p, sqlite3_str *pStr){ + int rc = pStr ? sqlite3_str_errcode(pStr) : 0; + if( rc ){ + qrfError(p, rc, sqlite3_errstr(rc)); + } +} + /* -** The following is the open SQLite database. We make a pointer -** to this database a static variable so that it can be accessed -** by the SIGINT handler to interrupt database processing. +** Add a new entry to the EXPLAIN QUERY PLAN data */ -static sqlite3 *globalDb = 0; +static void qrfEqpAppend(Qrf *p, int iEqpId, int p2, const char *zText){ + qrfEQPGraphRow *pNew; + sqlite3_int64 nText; + if( zText==0 ) return; + if( p->u.pGraph==0 ){ + p->u.pGraph = sqlite3_malloc64( sizeof(qrfEQPGraph) ); + if( p->u.pGraph==0 ){ + qrfOom(p); + return; + } + memset(p->u.pGraph, 0, sizeof(qrfEQPGraph) ); + } + nText = strlen(zText); + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + if( pNew==0 ){ + qrfOom(p); + return; + } + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->u.pGraph->pLast ){ + p->u.pGraph->pLast->pNext = pNew; + }else{ + p->u.pGraph->pRow = pNew; + } + p->u.pGraph->pLast = pNew; +} /* -** True if an interrupt (Control-C) has been received. +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->u.pGraph. */ -static volatile int seenInterrupt = 0; +static void qrfEqpReset(Qrf *p){ + qrfEQPGraphRow *pRow, *pNext; + if( p->u.pGraph ){ + for(pRow = p->u.pGraph->pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + sqlite3_free(p->u.pGraph); + p->u.pGraph = 0; + } +} -/* -** This is the name of our program. It is set in main(), used -** in a number of other places, mostly for error messages. +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL */ -static char *Argv0; +static qrfEQPGraphRow *qrfEqpNextRow(Qrf *p, int iEqpId, qrfEQPGraphRow *pOld){ + qrfEQPGraphRow *pRow = pOld ? pOld->pNext : p->u.pGraph->pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void qrfEqpRenderLevel(Qrf *p, int iEqpId){ + qrfEQPGraphRow *pRow, *pNext; + i64 n = strlen(p->u.pGraph->zPrefix); + char *z; + for(pRow = qrfEqpNextRow(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = qrfEqpNextRow(p, iEqpId, pRow); + z = pRow->zText; + sqlite3_str_appendf(p->pOut, "%s%s%s\n", p->u.pGraph->zPrefix, + pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->u.pGraph->zPrefix)-7 ){ + memcpy(&p->u.pGraph->zPrefix[n], pNext ? "| " : " ", 4); + qrfEqpRenderLevel(p, pRow->iEqpId); + p->u.pGraph->zPrefix[n] = 0; + } + } +} /* -** Prompt strings. Initialized in main. Settable with -** .prompt main continue +** Render the 64-bit value N in a more human-readable format into +** pOut. +** +** + Only show the first three significant digits. +** + Append suffixes K, M, G, T, P, and E for 1e3, 1e6, ... 1e18 */ -#define PROMPT_LEN_MAX 128 -/* First line prompt. default: "sqlite> " */ -static char mainPrompt[PROMPT_LEN_MAX]; -/* Continuation prompt. default: " ...> " */ -static char continuePrompt[PROMPT_LEN_MAX]; +static void qrfApproxInt64(sqlite3_str *pOut, i64 N){ + static const char aSuffix[] = { 'K', 'M', 'G', 'T', 'P', 'E' }; + int i; + if( N<0 ){ + N = N==INT64_MIN ? INT64_MAX : -N; + sqlite3_str_append(pOut, "-", 1); + } + if( N<10000 ){ + sqlite3_str_appendf(pOut, "%4lld ", N); + return; + } + for(i=1; i<=18; i++){ + N = (N+5)/10; + if( N<10000 ){ + int n = (int)N; + switch( i%3 ){ + case 0: + sqlite3_str_appendf(pOut, "%d.%02d", n/1000, (n%1000)/10); + break; + case 1: + sqlite3_str_appendf(pOut, "%2d.%d", n/100, (n%100)/10); + break; + case 2: + sqlite3_str_appendf(pOut, "%4d", n/10); + break; + } + sqlite3_str_append(pOut, &aSuffix[i/3], 1); + break; + } + } +} -/* This is variant of the standard-library strncpy() routine with the -** one change that the destination string is always zero-terminated, even -** if there is no zero-terminator in the first n-1 characters of the source -** string. +/* +** Display and reset the EXPLAIN QUERY PLAN data */ -static char *shell_strncpy(char *dest, const char *src, size_t n){ - size_t i; - for(i=0; i<n-1 && src[i]!=0; i++) dest[i] = src[i]; - dest[i] = 0; - return dest; +static void qrfEqpRender(Qrf *p, i64 nCycle){ + qrfEQPGraphRow *pRow; + if( p->u.pGraph!=0 && (pRow = p->u.pGraph->pRow)!=0 ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + qrfEqpReset(p); + return; + } + sqlite3_str_appendf(p->pOut, "%s\n", pRow->zText+3); + p->u.pGraph->pRow = pRow->pNext; + sqlite3_free(pRow); + }else if( nCycle>0 ){ + int nSp = p->u.pGraph->nWidth - 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops (est) Rows (est)\n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ------------ ------------\n"); + }else{ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops Rows \n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ----- -----\n"); + } + sqlite3_str_appendall(p->pOut, "QUERY PLAN"); + sqlite3_str_appendchar(p->pOut, nSp - 10, ' '); + qrfApproxInt64(p->pOut, nCycle); + sqlite3_str_appendall(p->pOut, " 100%\n"); + }else{ + sqlite3_str_appendall(p->pOut, "QUERY PLAN\n"); + } + p->u.pGraph->zPrefix[0] = 0; + qrfEqpRenderLevel(p, 0); + qrfEqpReset(p); + } } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** strcpy() workalike to squelch an unwarranted link-time warning -** from OpenBSD. +** Helper function for qrfExpStats(). +** */ -static void shell_strcpy(char *dest, const char *src){ - while( (*(dest++) = *(src++))!=0 ){} +static int qrfStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; } +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + /* -** Optionally disable dynamic continuation prompt. -** Unless disabled, the continuation prompt shows open SQL lexemes if any, -** or open parentheses level if non-zero, or continuation prompt as set. -** This facility interacts with the scanner and process_input() where the -** below 5 macros are used. +** Generate ".scanstatus est" style of EQP output. */ -#ifdef SQLITE_OMIT_DYNAPROMPT -# define CONTINUATION_PROMPT continuePrompt -# define CONTINUE_PROMPT_RESET -# define CONTINUE_PROMPT_AWAITS(p,s) -# define CONTINUE_PROMPT_AWAITC(p,c) -# define CONTINUE_PAREN_INCR(p,n) -# define CONTINUE_PROMPT_PSTATE 0 -typedef void *t_NoDynaPrompt; -# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt +static void qrfEqpStats(Qrf *p){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + qrfError(p, SQLITE_ERROR, "not available in this build"); #else -# define CONTINUATION_PROMPT dynamicContinuePrompt() -# define CONTINUE_PROMPT_RESET \ - do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) -# define CONTINUE_PROMPT_AWAITS(p,s) \ - if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) -# define CONTINUE_PROMPT_AWAITC(p,c) \ - if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) -# define CONTINUE_PAREN_INCR(p,n) \ - if(p && stdin_is_interactive) (trackParenLevel(p,n)) -# define CONTINUE_PROMPT_PSTATE (&dynPrompt) -typedef struct DynaPrompt *t_DynaPromptRef; -# define SCAN_TRACKER_REFTYPE t_DynaPromptRef + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *pS = p->pStmt; + int i = 0; + i64 nTotal = 0; + int nWidth = 0; + int prevPid = -1; /* Previous iPid */ + double rEstCum = 1.0; /* Cumulative row estimate */ + sqlite3_str *pLine = sqlite3_str_new(p->db); + sqlite3_str *pStats = sqlite3_str_new(p->db); + qrfEqpReset(p); -static struct DynaPrompt { - char dynamicPrompt[PROMPT_LEN_MAX]; - char acAwait[2]; - int inParenLevel; - char *zScannerAwaits; -} dynPrompt = { {0}, {0}, 0, 0 }; + for(i=0; 1; i++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = (int)strlen(z) + qrfStatsHeight(pS,i)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 2; -/* Record parenthesis nesting level change, or force level to 0. */ -static void trackParenLevel(struct DynaPrompt *p, int ni){ - p->inParenLevel += ni; - if( ni==0 ) p->inParenLevel = 0; - p->zScannerAwaits = 0; -} + sqlite3_stmt_scanstatus_v2(pS,-1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(i=0; 1; i++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + double rEst = 0.0; -/* Record that a lexeme is opened, or closed with args==0. */ -static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ - if( s!=0 || c==0 ){ - p->zScannerAwaits = s; - p->acAwait[0] = 0; - }else{ - p->acAwait[0] = c; - p->zScannerAwaits = p->acAwait; - } -} + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + if( iPid!=prevPid ){ + prevPid = iPid; + rEstCum = 1.0; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + rEstCum *= rEst; + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NAME,f,(void*)&zName); -/* Upon demand, derive the continuation prompt to display. */ -static char *dynamicContinuePrompt(void){ - if( continuePrompt[0]==0 - || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ - return continuePrompt; - }else{ - if( dynPrompt.zScannerAwaits ){ - size_t ncp = strlen(continuePrompt); - size_t ndp = strlen(dynPrompt.zScannerAwaits); - if( ndp > ncp-3 ) return continuePrompt; - shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); - while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; - shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, - PROMPT_LEN_MAX-4); - }else{ - if( dynPrompt.inParenLevel>9 ){ - shell_strncpy(dynPrompt.dynamicPrompt, "(..", 4); - }else if( dynPrompt.inParenLevel<0 ){ - shell_strncpy(dynPrompt.dynamicPrompt, ")x!", 4); - }else{ - shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); - dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + int nSp = 0; + sqlite3_str_reset(pStats); + if( nCycle>=0 && nTotal>0 ){ + qrfApproxInt64(pStats, nCycle); + sqlite3_str_appendf(pStats, " %3d%%", + ((nCycle*100)+nTotal/2) / nTotal + ); + nSp = 2; } - shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, - PROMPT_LEN_MAX-4); + if( nLoop>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nLoop); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)(rEstCum/rEst)); + } + } + if( nRow>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nRow); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)rEstCum); + } + } + sqlite3_str_appendf(pLine, + "% *s %s", -1*(nWidth-qrfStatsHeight(pS,i)*3), zo, + sqlite3_str_value(pStats) + ); + sqlite3_str_reset(pStats); + qrfEqpAppend(p, iId, iPid, sqlite3_str_value(pLine)); + sqlite3_str_reset(pLine); + }else{ + qrfEqpAppend(p, iId, iPid, zo); } } - return dynPrompt.dynamicPrompt; -} -#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ - -/* Indicate out-of-memory and exit. */ -static void shell_out_of_memory(void){ - eputz("Error: out of memory\n"); - exit(1); + if( p->u.pGraph ) p->u.pGraph->nWidth = nWidth; + qrfStrErr(p, pLine); + sqlite3_free(sqlite3_str_finish(pLine)); + qrfStrErr(p, pStats); + sqlite3_free(sqlite3_str_finish(pStats)); +#endif } -/* Check a pointer to see if it is NULL. If it is NULL, exit with an -** out-of-memory error. -*/ -static void shell_check_oom(const void *p){ - if( p==0 ) shell_out_of_memory(); -} /* -** Write I/O traces to the following stream. +** Reset the prepared statement. */ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif +static void qrfResetStmt(Qrf *p){ + int rc = sqlite3_reset(p->pStmt); + if( rc!=SQLITE_OK && p->iErr==SQLITE_OK ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + } +} /* -** This routine works like printf in that its first argument is a -** format string and subsequent arguments are values to be substituted -** in place of % fields. The result of formatting this string -** is written to iotrace. +** If xWrite is defined, send all content of pOut to xWrite and +** reset pOut. */ -#ifdef SQLITE_ENABLE_IOTRACE -static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ - va_list ap; - char *z; - if( iotrace==0 ) return; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - sqlite3_fprintf(iotrace, "%s", z); - sqlite3_free(z); +static void qrfWrite(Qrf *p){ + int n; + if( p->spec.xWrite && (n = sqlite3_str_length(p->pOut))>0 ){ + int rc = p->spec.xWrite(p->spec.pWriteArg, + sqlite3_str_value(p->pOut), + (sqlite3_int64)n); + sqlite3_str_reset(p->pOut); + if( rc ){ + qrfError(p, rc, "Failed to write %d bytes of output", n); + } + } } -#endif /* Lookup table to estimate the number of columns consumed by a Unicode ** character. @@ -1010,7 +1366,7 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ static const struct { unsigned char w; /* Width of the character in columns */ int iFirst; /* First character in a span having this width */ -} aUWidth[] = { +} aQrfUWidth[] = { /* {1, 0x00000}, */ {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, @@ -1046,7 +1402,7 @@ static const struct { {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, - {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, + {0, 0x01036}, {1, 0x0103b}, {0, 0x01058}, {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, @@ -1085,37 +1441,38 @@ static const struct { ** Inaccuracies in the width estimates might cause columns to be misaligned. ** Unfortunately, there is nothing we can do about that. */ -int cli_wcwidth(int c){ +int sqlite3_qrf_wcwidth(int c){ int iFirst, iLast; /* Fast path for common characters */ - if( c<=0x300 ) return 1; + if( c<0x300 ) return 1; /* The general case */ iFirst = 0; - iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; + iLast = sizeof(aQrfUWidth)/sizeof(aQrfUWidth[0]) - 1; while( iFirst<iLast-1 ){ int iMid = (iFirst+iLast)/2; - int cMid = aUWidth[iMid].iFirst; + int cMid = aQrfUWidth[iMid].iFirst; if( cMid < c ){ iFirst = iMid; }else if( cMid > c ){ iLast = iMid - 1; }else{ - return aUWidth[iMid].w; + return aQrfUWidth[iMid].w; } } - if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; - return aUWidth[iLast].w; + if( aQrfUWidth[iLast].iFirst > c ) return aQrfUWidth[iFirst].w; + return aQrfUWidth[iLast].w; } /* ** Compute the value and length of a multi-byte UTF-8 character that -** begins at z[0]. Return the length. Write the Unicode value into *pU. +** begins at z[0]. Return the length. Write the Unicode value into *pU. ** -** This routine only works for *multi-byte* UTF-8 characters. +** This routine only works for *multi-byte* UTF-8 characters. It does +** not attempt to detect illegal characters. */ -static int decodeUtf8(const unsigned char *z, int *pU){ +int sqlite3_qrf_decode_utf8(const unsigned char *z, int *pU){ if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); return 2; @@ -1128,89 +1485,686 @@ static int decodeUtf8(const unsigned char *z, int *pU){ && (z[3] & 0xc0)==0x80 ){ *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 - | (z[4] & 0x3f); + | (z[3] & 0x3f); return 4; } *pU = 0; return 1; } +/* +** Check to see if z[] is a valid VT100 escape. If it is, then +** return the number of bytes in the escape sequence. Return 0 if +** z[] is not a VT100 escape. +** +** This routine assumes that z[0] is \033 (ESC). +*/ +static int qrfIsVt100(const unsigned char *z){ + int i; + if( z[1]!='[' ) return 0; + i = 2; + while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} + +/* +** Return the length of a string in display characters. +** +** Most characters of the input string count as 1, including +** multi-byte UTF8 characters. However, zero-width unicode +** characters and VT100 escape sequences count as zero, and +** double-width characters count as two. +** +** The definition of "zero-width" and "double-width" characters +** is not precise. It depends on the output device, to some extent, +** and it varies according to the Unicode version. This routine +** makes the best guess that it can. +*/ +size_t sqlite3_qrf_wcswidth(const char *zIn){ + const unsigned char *z = (const unsigned char*)zIn; + size_t n = 0; + while( *z ){ + if( z[0]<' ' ){ + int k; + if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + return n; +} -#if 0 /* NOT USED */ /* -** Return the width, in display columns, of a UTF-8 string. +** Return the display width of the longest line of text +** in the (possibly) multi-line input string zIn[0..nByte]. +** zIn[] is not necessarily zero-terminated. Take +** into account tab characters, zero- and double-width +** characters, CR and NL, and VT100 escape codes. ** -** Each normal character counts as 1. Zero-width characters count -** as zero, and double-width characters count as 2. +** Write the number of newlines into *pnNL. So, *pnNL will +** return 0 if everything fits on one line, or positive it +** it will need to be split. */ -int cli_wcswidth(const char *z){ - const unsigned char *a = (const unsigned char*)z; +static int qrfDisplayWidth(const char *zIn, sqlite3_int64 nByte, int *pnNL){ + const unsigned char *z; + const unsigned char *zEnd; + int mx = 0; int n = 0; - int i = 0; - unsigned char c; - while( (c = a[i])!=0 ){ - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&a[i], &u); - i += len; - n += cli_wcwidth(u); - }else if( c>=' ' ){ + int nNL = 0; + if( zIn==0 ) zIn = ""; + z = (const unsigned char*)zIn; + zEnd = &z[nByte]; + while( z<zEnd ){ + if( z[0]<' ' ){ + int k; + if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else{ + if( z[0]=='\t' ){ + n = (n+8)&~7; + }else if( z[0]=='\n' || z[0]=='\r' ){ + nNL++; + if( n>mx ) mx = n; + n = 0; + } + z++; + } + }else if( (0x80&z[0])==0 ){ n++; - i++; + z++; }else{ - i++; + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); } } + if( mx>n ) n = mx; + if( pnNL ) *pnNL = nNL; return n; } -#endif /* -** Check to see if z[] is a valid VT100 escape. If it is, then -** return the number of bytes in the escape sequence. Return 0 if -** z[] is not a VT100 escape. +** Escape the input string if it is needed and in accordance with +** eEsc, which is either QRF_ESC_Ascii or QRF_ESC_Symbol. ** -** This routine assumes that z[0] is \033 (ESC). +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppOut to NULL +** and return 0. If escaping is needed, write the escaped string into +** memory obtained from sqlite3_malloc64() and make *ppOut point to that +** memory and return 0. If an error occurs, return non-zero. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. */ -static int isVt100(const unsigned char *z){ - int i; - if( z[1]!='[' ) return 0; - i = 2; - while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } - while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } - if( z[i]<0x40 || z[i]>0x7e ) return 0; - return i+1; +static void qrfEscape( + int eEsc, /* QRF_ESC_Ascii or QRF_ESC_Symbol */ + sqlite3_str *pStr, /* String to be escaped */ + int iStart /* Begin escapding on this byte of pStr */ +){ + sqlite3_int64 i, j; /* Loop counters */ + sqlite3_int64 sz; /* Size of the string prior to escaping */ + sqlite3_int64 nCtrl = 0;/* Number of control characters to escape */ + unsigned char *zIn; /* Text to be escaped */ + unsigned char c; /* A single character of the text */ + unsigned char *zOut; /* Where to write the results */ + + /* Find the text to be escaped */ + zIn = (unsigned char*)sqlite3_str_value(pStr); + if( zIn==0 ) return; + zIn += iStart; + + /* Count the control characters */ + for(i=0; (c = zIn[i])!=0; i++){ + if( c<=0x1f + && c!='\t' + && c!='\n' + && (c!='\r' || zIn[i+1]!='\n') + ){ + nCtrl++; + } + } + if( nCtrl==0 ) return; /* Early out if no control characters */ + + /* Make space to hold the escapes. Copy the original text to the end + ** of the available space. */ + sz = sqlite3_str_length(pStr) - iStart; + if( eEsc==QRF_ESC_Symbol ) nCtrl *= 2; + sqlite3_str_appendchar(pStr, nCtrl, ' '); + zOut = (unsigned char*)sqlite3_str_value(pStr); + if( zOut==0 ) return; + zOut += iStart; + zIn = zOut + nCtrl; + memmove(zIn,zOut,sz); + + /* Convert the control characters */ + for(i=j=0; (c = zIn[i])!=0; i++){ + if( c>0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') + ){ + continue; + } + if( i>0 ){ + memmove(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + if( eEsc==QRF_ESC_Symbol ){ + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + }else{ + zOut[j++] = '^'; + zOut[j++] = 0x40+c; + } + } } /* -** Output string zUtf to stdout as w characters. If w is negative, -** then right-justify the text. W is the width in UTF-8 characters, not -** in bytes. This is different from the %*.*s specification in printf -** since with %*.*s the width is measured in bytes, not characters. +** Determine if the string z[] can be shown as plain text. Return true +** if z[] is unambiguously text. Return false if z[] needs to be +** quoted. +** +** All of the following must be true in order for z[] to be relaxable: ** -** Take into account zero-width and double-width Unicode characters. -** In other words, a zero-width character does not count toward the -** the w limit. A double-width character counts as two. +** (1) z[] does not begin or end with ' or whitespace +** (2) z[] is not the same as the NULL rendering +** (3) z[] does not looks like a numeric literal */ -static void utf8_width_print(FILE *out, int w, const char *zUtf){ +static int qrfRelaxable(Qrf *p, const char *z){ + size_t i, n; + if( z[0]=='\'' || qrfSpace(z[0]) ) return 0; + if( z[0]==0 ){ + return (p->spec.zNull!=0 && p->spec.zNull[0]!=0); + } + n = strlen(z); + if( n==0 || z[n-1]=='\'' || qrfSpace(z[n-1]) ) return 0; + if( p->spec.zNull && strcmp(p->spec.zNull,z)==0 ) return 0; + i = (z[0]=='-' || z[0]=='+'); + if( strcmp(z+i,"Inf")==0 ) return 0; + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + if( z[i]=='.' ){ + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + } + if( z[i]=='e' || z[i]=='E' ){ + i++; + if( z[i]=='+' || z[i]=='-' ){ i++; } + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + } + return z[i]!=0; +} + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char qrfCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Encode text appropriately and append it to pOut. +*/ +static void qrfEncodeText(Qrf *p, sqlite3_str *pOut, const char *zTxt){ + int iStart = sqlite3_str_length(pOut); + switch( p->spec.eText ){ + case QRF_TEXT_Relaxed: + if( qrfRelaxable(p, zTxt) ){ + sqlite3_str_appendall(pOut, zTxt); + break; + } + deliberate_fall_through; /* FALLTHRU */ + case QRF_TEXT_Sql: { + if( p->spec.eEsc==QRF_ESC_Off ){ + sqlite3_str_appendf(pOut, "%Q", zTxt); + }else{ + sqlite3_str_appendf(pOut, "%#Q", zTxt); + } + break; + } + case QRF_TEXT_Csv: { + unsigned int i; + for(i=0; zTxt[i]; i++){ + if( qrfCsvQuote[((const unsigned char*)zTxt)[i]] ){ + i = 0; + break; + } + } + if( i==0 || strstr(zTxt, p->spec.zColumnSep)!=0 ){ + sqlite3_str_appendf(pOut, "\"%w\"", zTxt); + }else{ + sqlite3_str_appendall(pOut, zTxt); + } + break; + } + case QRF_TEXT_Html: { + const unsigned char *z = (const unsigned char*)zTxt; + while( *z ){ + unsigned int i = 0; + unsigned char c; + while( (c=z[i])>'>' + || (c && c!='<' && c!='>' && c!='&' && c!='\"' && c!='\'') + ){ + i++; + } + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + switch( z[i] ){ + case '>': sqlite3_str_append(pOut, "<", 4); break; + case '&': sqlite3_str_append(pOut, "&", 5); break; + case '<': sqlite3_str_append(pOut, "<", 4); break; + case '"': sqlite3_str_append(pOut, """, 6); break; + case '\'': sqlite3_str_append(pOut, "'", 5); break; + default: i--; + } + z += i + 1; + } + break; + } + case QRF_TEXT_Tcl: + case QRF_TEXT_Json: { + const unsigned char *z = (const unsigned char*)zTxt; + sqlite3_str_append(pOut, "\"", 1); + while( *z ){ + unsigned int i; + for(i=0; z[i]>=0x20 && z[i]!='\\' && z[i]!='"'; i++){} + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + if( z[i]==0 ) break; + switch( z[i] ){ + case '"': sqlite3_str_append(pOut, "\\\"", 2); break; + case '\\': sqlite3_str_append(pOut, "\\\\", 2); break; + case '\b': sqlite3_str_append(pOut, "\\b", 2); break; + case '\f': sqlite3_str_append(pOut, "\\f", 2); break; + case '\n': sqlite3_str_append(pOut, "\\n", 2); break; + case '\r': sqlite3_str_append(pOut, "\\r", 2); break; + case '\t': sqlite3_str_append(pOut, "\\t", 2); break; + default: { + if( p->spec.eText==QRF_TEXT_Json ){ + sqlite3_str_appendf(pOut, "\\u%04x", z[i]); + }else{ + sqlite3_str_appendf(pOut, "\\%03o", z[i]); + } + break; + } + } + z += i + 1; + } + sqlite3_str_append(pOut, "\"", 1); + break; + } + default: { + sqlite3_str_appendall(pOut, zTxt); + break; + } + } + if( p->spec.eEsc!=QRF_ESC_Off ){ + qrfEscape(p->spec.eEsc, pOut, iStart); + } +} + +/* +** Do a quick sanity check to see aBlob[0..nBlob-1] is valid JSONB +** return true if it is and false if it is not. +** +** False positives are possible, but not false negatives. +*/ +static int qrfJsonbQuickCheck(unsigned char *aBlob, int nBlob){ + unsigned char x; /* Payload size half-byte */ + int i; /* Loop counter */ + int n; /* Bytes in the payload size integer */ + sqlite3_uint64 sz; /* value of the payload size integer */ + + if( nBlob==0 ) return 0; + x = aBlob[0]>>4; + if( x<=11 ) return nBlob==(1+x); + n = x<14 ? x-11 : 4*(x-13); + if( nBlob<1+n ) return 0; + sz = aBlob[1]; + for(i=1; i<n; i++) sz = (sz<<8) + aBlob[i+1]; + return sz+n+1==(sqlite3_uint64)nBlob; +} + +/* +** The current iCol-th column of p->pStmt is known to be a BLOB. Check +** to see if that BLOB is really a JSONB blob. If it is, then translate +** it into a text JSON representation and return a pointer to that text JSON. +** If the BLOB is not JSONB, then return a NULL pointer. +** +** The memory used to hold the JSON text is managed internally by the +** "p" object and is overwritten and/or deallocated upon the next call +** to this routine (with the same p argument) or when the p object is +** finailized. +*/ +static const char *qrfJsonbToJson(Qrf *p, int iCol){ + int nByte; + const void *pBlob; + int rc; + nByte = sqlite3_column_bytes(p->pStmt, iCol); + pBlob = sqlite3_column_blob(p->pStmt, iCol); + if( qrfJsonbQuickCheck((unsigned char*)pBlob, nByte)==0 ){ + return 0; + } + if( p->pJTrans==0 ){ + sqlite3 *db; + rc = sqlite3_open(":memory:",&db); + if( rc ){ + sqlite3_close(db); + return 0; + } + rc = sqlite3_prepare_v2(db, "SELECT json(?1)", -1, &p->pJTrans, 0); + if( rc ){ + sqlite3_finalize(p->pJTrans); + p->pJTrans = 0; + sqlite3_close(db); + return 0; + } + }else{ + sqlite3_reset(p->pJTrans); + } + sqlite3_bind_blob(p->pJTrans, 1, (void*)pBlob, nByte, SQLITE_STATIC); + rc = sqlite3_step(p->pJTrans); + if( rc==SQLITE_ROW ){ + return (const char*)sqlite3_column_text(p->pJTrans, 0); + }else{ + return 0; + } +} + +/* +** Adjust the input string zIn[] such that it is no more than N display +** characters wide. If it is wider than that, then truncate and add +** ellipsis. Or if zIn[] contains a \r or \n, truncate at that point, +** adding ellipsis. Embedded tabs in zIn[] are converted into ordinary +** spaces. +** +** Return this display width of the modified title string. +*/ +static int qrfTitleLimit(char *zIn, int N){ + unsigned char *z = (unsigned char*)zIn; + int n = 0; + unsigned char *zEllipsis = 0; + while( 1 /*exit-by-break*/ ){ + if( z[0]<' ' ){ + int k; + if( z[0]==0 ){ + zEllipsis = 0; + break; + }else if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else if( z[0]=='\t' ){ + z[0] = ' '; + }else if( z[0]=='\n' || z[0]=='\r' ){ + z[0] = ' '; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + if( n>=(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n==N ){ z[0] = 0; break; } + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + if( n+len>(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n+len>N ){ z[0] = 0; break; } + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + if( zEllipsis && N>=3 ) memcpy(zEllipsis,"...",4); + return n; +} + + +/* +** Render value pVal into pOut +*/ +static void qrfRenderValue(Qrf *p, sqlite3_str *pOut, int iCol){ +#if SQLITE_VERSION_NUMBER>=3052000 + int iStartLen = sqlite3_str_length(pOut); +#endif + if( p->spec.xRender ){ + sqlite3_value *pVal; + char *z; + pVal = sqlite3_value_dup(sqlite3_column_value(p->pStmt,iCol)); + z = p->spec.xRender(p->spec.pRenderArg, pVal); + sqlite3_value_free(pVal); + if( z ){ + sqlite3_str_appendall(pOut, z); + sqlite3_free(z); + return; + } + } + switch( sqlite3_column_type(p->pStmt,iCol) ){ + case SQLITE_INTEGER: { + sqlite3_str_appendf(pOut, "%lld", sqlite3_column_int64(p->pStmt,iCol)); + break; + } + case SQLITE_FLOAT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + sqlite3_str_appendall(pOut, zTxt); + break; + } + case SQLITE_BLOB: { + if( p->spec.bTextJsonb==QRF_Yes ){ + const char *zJson = qrfJsonbToJson(p, iCol); + if( zJson ){ + if( p->spec.eText==QRF_TEXT_Sql ){ + sqlite3_str_append(pOut,"jsonb(",6); + qrfEncodeText(p, pOut, zJson); + sqlite3_str_append(pOut,")",1); + }else{ + qrfEncodeText(p, pOut, zJson); + } + break; + } + } + switch( p->spec.eBlob ){ + case QRF_BLOB_Hex: + case QRF_BLOB_Sql: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_append(pOut, "x'", 2); + } + iStart = sqlite3_str_length(pOut); + sqlite3_str_appendchar(pOut, nBlob, ' '); + sqlite3_str_appendchar(pOut, nBlob, ' '); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_appendchar(pOut, 1, '\''); + } + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i<nBlob; i++, j+=2){ + unsigned char c = a[i]; + zVal[j] = "0123456789abcdef"[(c>>4)&0xf]; + zVal[j+1] = "0123456789abcdef"[(c)&0xf]; + } + break; + } + case QRF_BLOB_Tcl: + case QRF_BLOB_Json: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + int szC = p->spec.eBlob==QRF_BLOB_Json ? 6 : 4; + sqlite3_str_append(pOut, "\"", 1); + iStart = sqlite3_str_length(pOut); + for(i=szC; i>0; i--){ + sqlite3_str_appendchar(pOut, nBlob, ' '); + } + sqlite3_str_appendchar(pOut, 1, '"'); + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i<nBlob; i++, j+=szC){ + unsigned char c = a[i]; + zVal[j] = '\\'; + if( szC==4 ){ + zVal[j+1] = '0' + ((c>>6)&3); + zVal[j+2] = '0' + ((c>>3)&7); + zVal[j+3] = '0' + (c&7); + }else{ + zVal[j+1] = 'u'; + zVal[j+2] = '0'; + zVal[j+3] = '0'; + zVal[j+4] = "0123456789abcdef"[(c>>4)&0xf]; + zVal[j+5] = "0123456789abcdef"[(c)&0xf]; + } + } + break; + } + case QRF_BLOB_Size: { + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + sqlite3_str_appendf(pOut, "(%d-byte blob)", nBlob); + break; + } + default: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + } + } + break; + } + case SQLITE_NULL: { + sqlite3_str_appendall(pOut, p->spec.zNull); + break; + } + case SQLITE_TEXT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + break; + } + } +#if SQLITE_VERSION_NUMBER>=3052000 + if( p->spec.nCharLimit>0 + && (sqlite3_str_length(pOut) - iStartLen) > p->spec.nCharLimit + ){ + const unsigned char *z; + int ii = 0, w = 0, limit = p->spec.nCharLimit; + z = (const unsigned char*)sqlite3_str_value(pOut) + iStartLen; + if( limit<4 ) limit = 4; + while( 1 ){ + if( z[ii]<' ' ){ + int k; + if( z[ii]=='\033' && (k = qrfIsVt100(z+ii))>0 ){ + ii += k; + }else if( z[ii]==0 ){ + break; + }else{ + ii++; + } + }else if( (0x80&z[ii])==0 ){ + w++; + if( w>limit ) break; + ii++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[ii], &u); + w += sqlite3_qrf_wcwidth(u); + if( w>limit ) break; + ii += len; + } + } + if( w>limit ){ + sqlite3_str_truncate(pOut, iStartLen+ii); + sqlite3_str_append(pOut, "...", 3); + } + } +#endif +} + +/* Trim spaces of the end if pOut +*/ +static void qrfRTrim(sqlite3_str *pOut){ +#if SQLITE_VERSION_NUMBER>=3052000 + int nByte = sqlite3_str_length(pOut); + const char *zOut = sqlite3_str_value(pOut); + while( nByte>0 && zOut[nByte-1]==' ' ){ nByte--; } + sqlite3_str_truncate(pOut, nByte); +#endif +} + +/* +** Store string zUtf to pOut as w characters. If w is negative, +** then right-justify the text. W is the width in display characters, not +** in bytes. Double-width unicode characters count as two characters. +** VT100 escape sequences count as zero. And so forth. +*/ +static void qrfWidthPrint(Qrf *p, sqlite3_str *pOut, int w, const char *zUtf){ const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; unsigned char c; int i = 0; int n = 0; int k; - int aw = w<0 ? -w : w; - if( zUtf==0 ) zUtf = ""; + int aw; + (void)p; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; + if( a==0 ) a = (const unsigned char*)""; while( (c = a[i])!=0 ){ if( (c&0xc0)==0xc0 ){ int u; - int len = decodeUtf8(a+i, &u); - int x = cli_wcwidth(u); + int len = sqlite3_qrf_decode_utf8(a+i, &u); + int x = sqlite3_qrf_wcwidth(u); if( x+n>aw ){ break; } i += len; n += x; - }else if( c==0x1b && (k = isVt100(&a[i]))>0 ){ + }else if( c==0x1b && (k = qrfIsVt100(&a[i]))>0 ){ i += k; }else if( n>=aw ){ break; @@ -1220,303 +2174,1185 @@ static void utf8_width_print(FILE *out, int w, const char *zUtf){ } } if( n>=aw ){ - sqlite3_fprintf(out, "%.*s", i, zUtf); + sqlite3_str_append(pOut, zUtf, i); }else if( w<0 ){ - sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + sqlite3_str_append(pOut, zUtf, i); }else{ - sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); + sqlite3_str_append(pOut, zUtf, i); + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); } } - /* -** Determines if a string is a number of not. +** (*pz)[] is a line of text that is to be displayed the box or table or +** similar tabular formats. z[] contain newlines or might be too wide +** to fit in the columns so will need to be split into multiple line. +** +** This routine determines: +** +** * How many bytes of z[] should be shown on the current line. +** * How many character positions those bytes will cover. +** * The byte offset to the start of the next line. */ -static int isNumber(const char *z, int *realnum){ - if( *z=='-' || *z=='+' ) z++; - if( !IsDigit(*z) ){ - return 0; +static void qrfWrapLine( + const char *zIn, /* Input text to be displayed */ + int w, /* Column width in characters (not bytes) */ + int bWrap, /* True if we should do word-wrapping */ + int *pnThis, /* OUT: How many bytes of z[] for the current line */ + int *pnWide, /* OUT: How wide is the text of this line */ + int *piNext /* OUT: Offset into z[] to start of the next line */ +){ + int i; /* Input bytes consumed */ + int k; /* Bytes in a VT100 code */ + int n; /* Output column number */ + const unsigned char *z = (const unsigned char*)zIn; + unsigned char c = 0; + + if( z[0]==0 ){ + *pnThis = 0; + *pnWide = 0; + *piNext = 0; + return; + } + n = 0; + for(i=0; n<=w; i++){ + c = z[i]; + if( c>=0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + int wcw = sqlite3_qrf_wcwidth(u); + if( wcw+n>w ) break; + i += len-1; + n += wcw; + continue; + } + if( c>=' ' ){ + if( n==w ) break; + n++; + continue; + } + if( c==0 || c=='\n' ) break; + if( c=='\r' && z[i+1]=='\n' ){ c = z[++i]; break; } + if( c=='\t' ){ + int wcw = 8 - (n&7); + if( n+wcw>w ) break; + n += wcw; + continue; + } + if( c==0x1b && (k = qrfIsVt100(&z[i]))>0 ){ + i += k-1; + }else if( n==w ){ + break; + }else{ + n++; + } } - z++; - if( realnum ) *realnum = 0; - while( IsDigit(*z) ){ z++; } - if( *z=='.' ){ - z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; + if( c==0 ){ + *pnThis = i; + *pnWide = n; + *piNext = i; + return; } - if( *z=='e' || *z=='E' ){ - z++; - if( *z=='+' || *z=='-' ) z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; + if( c=='\n' ){ + *pnThis = i; + *pnWide = n; + *piNext = i+1; + return; } - return *z==0; + + /* If we get this far, that means the current line will end at some + ** point that is neither a "\n" or a 0x00. Figure out where that + ** split should occur + */ + if( bWrap && z[i]!=0 && !qrfSpace(z[i]) && qrfAlnum(c)==qrfAlnum(z[i]) ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i-1; k>=i/2; k--){ + if( qrfSpace(z[k]) ) break; + } + if( k<i/2 ){ + for(k=i; k>=i/2; k--){ + if( qrfAlnum(z[k-1])!=qrfAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k>=i/2 ){ + i = k; + n = qrfDisplayWidth((const char*)z, k, 0); + } + } + *pnThis = i; + *pnWide = n; + while( zIn[i]==' ' || zIn[i]=='\t' || zIn[i]=='\r' ){ i++; } + *piNext = i; } /* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. +** Append nVal bytes of text from zVal onto the end of pOut. +** Convert tab characters in zVal to the appropriate number of +** spaces. */ -static int strlen30(const char *z){ - const char *z2 = z; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); -} +static void qrfAppendWithTabs( + sqlite3_str *pOut, /* Append text here */ + const char *zVal, /* Text to append */ + int nVal /* Use only the first nVal bytes of zVal[] */ +){ + int i = 0; + unsigned int col = 0; + unsigned char *z = (unsigned char *)zVal; + while( i<nVal ){ + unsigned char c = z[i]; + if( c<' ' ){ + int k; + sqlite3_str_append(pOut, (const char*)z, i); + nVal -= i; + z += i; + i = 0; + if( c=='\033' && (k = qrfIsVt100(z))>0 ){ + sqlite3_str_append(pOut, (const char*)z, k); + z += k; + nVal -= k; + }else if( c=='\t' ){ + k = 8 - (col&7); + sqlite3_str_appendchar(pOut, k, ' '); + col += k; + z++; + nVal--; + }else if( c=='\r' && nVal==1 ){ + z++; + nVal--; + }else{ + char zCtrlPik[4]; + col++; + zCtrlPik[0] = 0xe2; + zCtrlPik[1] = 0x90; + zCtrlPik[2] = 0x80+c; + sqlite3_str_append(pOut, zCtrlPik, 3); + z++; + nVal--; + } + }else if( (0x80&c)==0 ){ + i++; + col++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + i += len; + col += sqlite3_qrf_wcwidth(u); + } + } + sqlite3_str_append(pOut, (const char*)z, i); +} /* -** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +** GCC does not define the offsetof() macro so we'll have to do it +** ourselves. */ -static int strlenChar(const char *z){ - int n = 0; - while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif + +/* +** Data for columnar layout, collected into a single object so +** that it can be more easily passed into subroutines. +*/ +typedef struct qrfColData qrfColData; +struct qrfColData { + Qrf *p; /* The QRF instance */ + int nCol; /* Number of columns in the table */ + unsigned char bMultiRow; /* One or more cells will span multiple lines */ + unsigned char nMargin; /* Width of column margins */ + sqlite3_int64 nRow; /* Number of rows */ + sqlite3_int64 nAlloc; /* Number of cells allocated */ + sqlite3_int64 n; /* Number of cells. nCol*nRow */ + char **az; /* Content of all cells */ + int *aiWth; /* Width of each cell */ + unsigned char *abNum; /* True for each numeric cell */ + struct qrfPerCol { /* Per-column data */ + char *z; /* Cache of text for current row */ + int w; /* Computed width of this column */ + int mxW; /* Maximum natural (unwrapped) width */ + unsigned char e; /* Alignment */ + unsigned char fx; /* Width is fixed */ + unsigned char bNum; /* True if is numeric */ + } *a; /* One per column */ +}; + +/* +** Output horizontally justified text into pOut. The text is the +** first nVal bytes of zVal. Include nWS bytes of whitespace, either +** split between both sides, or on the left, or on the right, depending +** on eAlign. +*/ +static void qrfPrintAligned( + sqlite3_str *pOut, /* Append text here */ + struct qrfPerCol *pCol, /* Information about the text to print */ + int nVal, /* Use only the first nVal bytes of zVal[] */ + int nWS /* Whitespace for horizonal alignment */ +){ + unsigned char eAlign = pCol->e & QRF_ALIGN_HMASK; + if( eAlign==QRF_Auto && pCol->bNum ) eAlign = QRF_ALIGN_Right; + if( eAlign==QRF_ALIGN_Center ){ + /* Center the text */ + sqlite3_str_appendchar(pOut, nWS/2, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS - nWS/2, ' '); + }else if( eAlign==QRF_ALIGN_Right ){ + /* Right justify the text */ + sqlite3_str_appendchar(pOut, nWS, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + }else{ + /* Left justify the text */ + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS, ' '); } - return n; } /* -** Return open FILE * if zFile exists, can be opened for read -** and is an ordinary file or a character stream source. -** Otherwise return 0. +** Free all the memory allocates in the qrfColData object */ -static FILE * openChrSource(const char *zFile){ -#if defined(_WIN32) || defined(WIN32) - struct __stat64 x = {0}; -# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) - /* On Windows, open first, then check the stream nature. This order - ** is necessary because _stat() and sibs, when checking a named pipe, - ** effectively break the pipe as its supplier sees it. */ - FILE *rv = sqlite3_fopen(zFile, "rb"); - if( rv==0 ) return 0; - if( _fstat64(_fileno(rv), &x) != 0 - || !STAT_CHR_SRC(x.st_mode)){ - fclose(rv); - rv = 0; +static void qrfColDataFree(qrfColData *p){ + sqlite3_int64 i; + for(i=0; i<p->n; i++) sqlite3_free(p->az[i]); + sqlite3_free(p->az); + sqlite3_free(p->aiWth); + sqlite3_free(p->abNum); + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); +} + +/* +** Allocate space for more cells in the qrfColData object. +** Return non-zero if a memory allocation fails. +*/ +static int qrfColDataEnlarge(qrfColData *p){ + char **azData; + int *aiWth; + unsigned char *abNum; + p->nAlloc = 2*p->nAlloc + 10*p->nCol; + azData = sqlite3_realloc64(p->az, p->nAlloc*sizeof(char*)); + if( azData==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; } - return rv; -#else - struct stat x = {0}; - int rc = stat(zFile, &x); -# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) - if( rc!=0 ) return 0; - if( STAT_CHR_SRC(x.st_mode) ){ - return sqlite3_fopen(zFile, "rb"); - }else{ - return 0; + p->az = azData; + aiWth = sqlite3_realloc64(p->aiWth, p->nAlloc*sizeof(int)); + if( aiWth==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; } -#endif -#undef STAT_CHR_SRC + p->aiWth = aiWth; + abNum = sqlite3_realloc64(p->abNum, p->nAlloc); + if( abNum==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->abNum = abNum; + return 0; } /* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails. +** Print a markdown or table-style row separator using ascii-art +*/ +static void qrfRowSeparator(sqlite3_str *pOut, qrfColData *p, char cSep){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + sqlite3_str_appendchar(pOut, p->a[0].w+p->nMargin, '-'); + for(i=1; i<p->nCol; i++){ + sqlite3_str_append(pOut, &cSep, 1); + sqlite3_str_appendchar(pOut, p->a[i].w+p->nMargin, '-'); + } + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + } + sqlite3_str_append(pOut, "\n", 1); +} + +/* +** UTF8 box-drawing characters. Imagine box lines like this: ** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. */ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; - int n = 0; +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ - while( 1 ){ - if( n+100>nLine ){ - nLine = nLine*2 + 100; - zLine = realloc(zLine, nLine); - shell_check_oom(zLine); +/* Rounded corners: */ +#define BOX_R12 "\342\225\260" /* U+2570 '- */ +#define BOX_R23 "\342\225\255" /* U+256d ,- */ +#define BOX_R34 "\342\225\256" /* U+256e -, */ +#define BOX_R14 "\342\225\257" /* U+256f -' */ + +/* Doubled horizontal lines: */ +#define DBL_24 "\342\225\220" /* U+2550 === */ +#define DBL_123 "\342\225\236" /* U+255e |= */ +#define DBL_134 "\342\225\241" /* U+2561 =| */ +#define DBL_1234 "\342\225\252" /* U+256a =|= */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void qrfBoxLine(sqlite3_str *pOut, int N, int bDbl){ + const char *azDash[2] = { + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24, + DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 + };/* 0 1 2 3 4 5 6 7 8 9 */ + const int nDash = 30; + N *= 3; + while( N>nDash ){ + sqlite3_str_append(pOut, azDash[bDbl], nDash); + N -= nDash; + } + sqlite3_str_append(pOut, azDash[bDbl], N); +} + +/* +** Draw a horizontal separator for a QRF_STYLE_Box table. +*/ +static void qrfBoxSeparator( + sqlite3_str *pOut, + qrfColData *p, + const char *zSep1, + const char *zSep2, + const char *zSep3, + int bDbl +){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep1); } - if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ - if( n==0 ){ - free(zLine); - return 0; - } - zLine[n] = 0; - break; + qrfBoxLine(pOut, p->a[0].w+p->nMargin, bDbl); + for(i=1; i<p->nCol; i++){ + sqlite3_str_appendall(pOut, zSep2); + qrfBoxLine(pOut, p->a[i].w+p->nMargin, bDbl); } - while( zLine[n] ) n++; - if( n>0 && zLine[n-1]=='\n' ){ - n--; - if( n>0 && zLine[n-1]=='\r' ) n--; - zLine[n] = 0; - break; + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep3); } } - return zLine; + sqlite3_str_append(pOut, "\n", 1); } /* -** Retrieve a single line of input text. -** -** If in==0 then read from standard input and prompt before each line. -** If isContinuation is true, then a continuation prompt is appropriate. -** If isContinuation is zero, then the main prompt should be used. +** Load into pData the default alignment for the body of a table. +*/ +static void qrfLoadAlignment(qrfColData *pData, Qrf *p){ + sqlite3_int64 i; + for(i=0; i<pData->nCol; i++){ + pData->a[i].e = p->spec.eDfltAlign; + if( i<p->spec.nAlign ){ + unsigned char ax = p->spec.aAlign[i]; + if( (ax & QRF_ALIGN_HMASK)!=0 ){ + pData->a[i].e = (ax & QRF_ALIGN_HMASK) | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + }else if( i<p->spec.nWidth ){ + if( p->spec.aWidth[i]<0 ){ + pData->a[i].e = QRF_ALIGN_Right | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + } + } +} + +/* +** If the single column in pData->a[] with pData->n entries can be +** laid out as nCol columns with a 2-space gap between each such +** that all columns fit within nSW, then return a pointer to an array +** of integers which is the width of each column from left to right. ** -** If zPrior is not NULL then it is a buffer from a prior call to this -** routine that can be reused. +** If the layout is not possible, return a NULL pointer. ** -** The result is stored in space obtained from malloc() and must either -** be freed by the caller or else passed back into this routine via the -** zPrior argument for reuse. +** Space to hold the returned array is from sqlite_malloc64(). */ -#ifndef SQLITE_SHELL_FIDDLE -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ - char *zPrompt; - char *zResult; - if( in!=0 ){ - zResult = local_getline(zPrior, in); - }else{ - zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; -#if SHELL_USE_LOCAL_GETLINE - sputz(stdout, zPrompt); - fflush(stdout); - do{ - zResult = local_getline(zPrior, stdin); - zPrior = 0; - /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ - if( zResult==0 ) sqlite3_sleep(50); - }while( zResult==0 && seenInterrupt>0 ); -#else - free(zPrior); - zResult = shell_readline(zPrompt); - while( zResult==0 ){ - /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ - sqlite3_sleep(50); - if( seenInterrupt==0 ) break; - zResult = shell_readline(""); +static int *qrfValidLayout( + qrfColData *pData, /* Collected query results */ + Qrf *p, /* On which to report an OOM */ + int nCol, /* Attempt this many columns */ + int nSW /* Screen width */ +){ + int i; /* Loop counter */ + int nr; /* Number of rows */ + int w = 0; /* Width of the current column */ + int t; /* Total width of all columns */ + int *aw; /* Array of individual column widths */ + + aw = sqlite3_malloc64( sizeof(int)*nCol ); + if( aw==0 ){ + qrfOom(p); + return 0; + } + nr = (pData->n + nCol - 1)/nCol; + for(i=0; i<pData->n; i++){ + if( (i%nr)==0 ){ + if( i>0 ) aw[i/nr-1] = w; + w = pData->aiWth[i]; + }else if( pData->aiWth[i]>w ){ + w = pData->aiWth[i]; } - if( zResult && *zResult ) shell_add_history(zResult); -#endif } - return zResult; + aw[nCol-1] = w; + for(t=i=0; i<nCol; i++) t += aw[i]; + t += 2*(nCol-1); + if( t>nSW ){ + sqlite3_free(aw); + return 0; + } + return aw; } -#endif /* !SQLITE_SHELL_FIDDLE */ /* -** Return the value of a hexadecimal digit. Return -1 if the input -** is not a hex digit. +** The output is single-column and the bSplitColumn flag is set. +** Check to see if the single-column output can be split into multiple +** columns that appear side-by-side. Adjust pData appropriately. */ -static int hexDigitValue(char c){ - if( c>='0' && c<='9' ) return c - '0'; - if( c>='a' && c<='f' ) return c - 'a' + 10; - if( c>='A' && c<='F' ) return c - 'A' + 10; - return -1; +static void qrfSplitColumn(qrfColData *pData, Qrf *p){ + int nCol = 1; + int *aw = 0; + char **az = 0; + int *aiWth = 0; + unsigned char *abNum = 0; + int nColNext = 2; + int w; + struct qrfPerCol *a = 0; + sqlite3_int64 nRow = 1; + sqlite3_int64 i; + while( 1/*exit-by-break*/ ){ + int *awNew = qrfValidLayout(pData, p, nColNext, p->spec.nScreenWidth); + if( awNew==0 ) break; + sqlite3_free(aw); + aw = awNew; + nCol = nColNext; + nRow = (pData->n + nCol - 1)/nCol; + if( nRow==1 ) break; + nColNext++; + while( (pData->n + nColNext - 1)/nColNext == nRow ) nColNext++; + } + if( nCol==1 ){ + sqlite3_free(aw); + return; /* Cannot do better than 1 column */ + } + az = sqlite3_malloc64( nRow*nCol*sizeof(char*) ); + if( az==0 ){ + qrfOom(p); + return; + } + aiWth = sqlite3_malloc64( nRow*nCol*sizeof(int) ); + if( aiWth==0 ){ + sqlite3_free(az); + qrfOom(p); + return; + } + a = sqlite3_malloc64( nCol*sizeof(struct qrfPerCol) ); + if( a==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + qrfOom(p); + return; + } + abNum = sqlite3_malloc64( nRow*nCol ); + if( abNum==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + sqlite3_free(a); + qrfOom(p); + return; + } + for(i=0; i<pData->n; i++){ + sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); + az[j] = pData->az[i]; + abNum[j]= pData->abNum[i]; + pData->az[i] = 0; + aiWth[j] = pData->aiWth[i]; + } + while( i<nRow*nCol ){ + sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); + az[j] = sqlite3_mprintf(""); + if( az[j]==0 ) qrfOom(p); + aiWth[j] = 0; + abNum[j] = 0; + i++; + } + for(i=0; i<nCol; i++){ + a[i].fx = a[i].mxW = a[i].w = aw[i]; + a[i].e = pData->a[0].e; + } + sqlite3_free(pData->az); + sqlite3_free(pData->aiWth); + sqlite3_free(pData->a); + sqlite3_free(pData->abNum); + sqlite3_free(aw); + pData->az = az; + pData->aiWth = aiWth; + pData->a = a; + pData->abNum = abNum; + pData->nCol = nCol; + pData->n = pData->nAlloc = nRow*nCol; + for(i=w=0; i<nCol; i++) w += a[i].w; + pData->nMargin = (p->spec.nScreenWidth - w)/(nCol - 1); + if( pData->nMargin>5 ) pData->nMargin = 5; } /* -** Interpret zArg as an integer value, possibly with suffixes. +** Adjust the layout for the screen width restriction */ -static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_int64 v = 0; - static const struct { char *zSuffix; int iMult; } aMult[] = { - { "KiB", 1024 }, - { "MiB", 1024*1024 }, - { "GiB", 1024*1024*1024 }, - { "KB", 1000 }, - { "MB", 1000000 }, - { "GB", 1000000000 }, - { "K", 1000 }, - { "M", 1000000 }, - { "G", 1000000000 }, - }; - int i; - int isNeg = 0; - if( zArg[0]=='-' ){ - isNeg = 1; - zArg++; - }else if( zArg[0]=='+' ){ - zArg++; +static void qrfRestrictScreenWidth(qrfColData *pData, Qrf *p){ + int sepW; /* Width of all box separators and margins */ + int sumW; /* Total width of data area over all columns */ + int targetW; /* Desired total data area */ + int i; /* Loop counters */ + int nCol; /* Number of columns */ + + pData->nMargin = 2; /* Default to normal margins */ + if( p->spec.nScreenWidth==0 ) return; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol*2 - 2; + }else{ + sepW = pData->nCol*3 + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; } - if( zArg[0]=='0' && zArg[1]=='x' ){ - int x; - zArg += 2; - while( (x = hexDigitValue(zArg[0]))>=0 ){ - v = (v<<4) + x; - zArg++; - } + nCol = pData->nCol; + for(i=sumW=0; i<nCol; i++) sumW += pData->a[i].w; + if( p->spec.nScreenWidth >= sumW+sepW ) return; + + /* First thing to do is reduce the separation between columns */ + pData->nMargin = 0; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol - 1; }else{ - while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; - zArg++; - } + sepW = pData->nCol + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; } - for(i=0; i<ArraySize(aMult); i++){ - if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){ - v *= aMult[i].iMult; - break; + targetW = p->spec.nScreenWidth - sepW; + +#define MIN_SQUOZE 8 +#define MIN_EX_SQUOZE 16 + /* Reduce the width of the widest eligible column. A column is + ** eligible for narrowing if: + ** + ** * It is not a fixed-width column (a[0].fx is false) + ** * The current width is more than MIN_SQUOZE + ** * Either: + ** + The current width is more then MIN_EX_SQUOZE, or + ** + The current width is more than half the max width (a[].mxW) + ** + ** Keep making reductions until either no more reductions are + ** possible or until the size target is reached. + */ + while( sumW > targetW ){ + int gain, w; + int ix = -1; + int mx = 0; + for(i=0; i<nCol; i++){ + if( pData->a[i].fx==0 + && (w = pData->a[i].w)>mx + && w>MIN_SQUOZE + && (w>MIN_EX_SQUOZE || w*2>pData->a[i].mxW) + ){ + ix = i; + mx = w; + } } + if( ix<0 ) break; + if( mx>=MIN_SQUOZE*2 ){ + gain = mx/2; + }else{ + gain = mx - MIN_SQUOZE; + } + if( sumW - gain < targetW ){ + gain = sumW - targetW; + } + sumW -= gain; + pData->a[ix].w -= gain; + pData->bMultiRow = 1; } - return isNeg? -v : v; } /* -** A variable length string to which one can append text. +** Columnar modes require that the entire query be evaluated first, with +** results written into memory, so that we can compute appropriate column +** widths. */ -typedef struct ShellText ShellText; -struct ShellText { - char *z; - int n; - int nAlloc; -}; +static void qrfColumnar(Qrf *p){ + sqlite3_int64 i, j; /* Loop counters */ + const char *colSep = 0; /* Column separator text */ + const char *rowSep = 0; /* Row terminator text */ + const char *rowStart = 0; /* Row start text */ + int szColSep, szRowSep, szRowStart; /* Size in bytes of previous 3 */ + int rc; /* Result code */ + int nColumn = p->nCol; /* Number of columns */ + int bWW; /* True to do word-wrap */ + sqlite3_str *pStr; /* Temporary rendering */ + qrfColData data; /* Columnar layout data */ + int bRTrim; /* Trim trailing space */ + + rc = sqlite3_step(p->pStmt); + if( rc!=SQLITE_ROW || nColumn==0 ){ + return; /* No output */ + } + + /* Initialize the data container */ + memset(&data, 0, sizeof(data)); + data.nCol = p->nCol; + data.p = p; + data.a = sqlite3_malloc64( nColumn*sizeof(struct qrfPerCol) ); + if( data.a==0 ){ + qrfOom(p); + return; + } + memset(data.a, 0, nColumn*sizeof(struct qrfPerCol) ); + if( qrfColDataEnlarge(&data) ) return; + assert( data.az!=0 ); + + /* Load the column header names and all cell content into data */ + if( p->spec.bTitles==QRF_Yes ){ + unsigned char saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + memset(data.abNum, 0, nColumn); + for(i=0; i<nColumn; i++){ + const char *z = (const char*)sqlite3_column_name(p->pStmt,i); + int nNL = 0; + int n, w; + pStr = sqlite3_str_new(p->db); + qrfEncodeText(p, pStr, z ? z : ""); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + if( p->spec.nTitleLimit ){ + nNL = 0; + data.aiWth[data.n] = w = qrfTitleLimit(data.az[data.n], + p->spec.nTitleLimit ); + }else{ + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + } + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->spec.eText = saved_eText; + p->nRow++; + } + do{ + if( data.n+nColumn > data.nAlloc ){ + if( qrfColDataEnlarge(&data) ) return; + } + for(i=0; i<nColumn; i++){ + char *z; + int nNL = 0; + int n, w; + int eType = sqlite3_column_type(p->pStmt,i); + pStr = sqlite3_str_new(p->db); + qrfRenderValue(p, pStr, i); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + data.abNum[data.n] = eType==SQLITE_INTEGER || eType==SQLITE_FLOAT; + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->nRow++; + }while( sqlite3_step(p->pStmt)==SQLITE_ROW && p->iErr==SQLITE_OK ); + if( p->iErr ){ + qrfColDataFree(&data); + return; + } + + /* Compute the width and alignment of every column */ + if( p->spec.bTitles==QRF_No ){ + qrfLoadAlignment(&data, p); + }else{ + unsigned char e; + if( p->spec.eTitleAlign==QRF_Auto ){ + e = QRF_ALIGN_Center; + }else{ + e = p->spec.eTitleAlign; + } + for(i=0; i<nColumn; i++) data.a[i].e = e; + } + + for(i=0; i<nColumn; i++){ + int w = 0; + if( i<p->spec.nWidth ){ + w = p->spec.aWidth[i]; + if( w==(-32768) ){ + w = 0; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + }else if( w<0 ){ + w = -w; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + } + if( w ) data.a[i].fx = 1; + } + if( w==0 ){ + w = data.a[i].mxW; + if( p->spec.nWrap>0 && w>p->spec.nWrap ){ + w = p->spec.nWrap; + data.bMultiRow = 1; + } + }else if( (data.bMultiRow==0 || w==1) && data.a[i].mxW>w ){ + data.bMultiRow = 1; + if( w==1 ){ + /* If aiWth[j] is 2 or more, then there might be a double-wide + ** character somewhere. So make the column width at least 2. */ + w = 2; + } + } + data.a[i].w = w; + } + + if( nColumn==1 + && data.n>1 + && p->spec.bSplitColumn==QRF_Yes + && p->spec.eStyle==QRF_STYLE_Column + && p->spec.bTitles==QRF_No + && p->spec.nScreenWidth>data.a[0].w+3 + ){ + /* Attempt to convert single-column tables into multi-column by + ** verticle wrapping, if the screen is wide enough and if the + ** bSplitColumn flag is set. */ + qrfSplitColumn(&data, p); + nColumn = data.nCol; + }else{ + /* Adjust the column widths due to screen width restrictions */ + qrfRestrictScreenWidth(&data, p); + } + + /* Draw the line across the top of the table. Also initialize + ** the row boundary and column separator texts. */ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + if( data.nMargin ){ + rowStart = BOX_13 " "; + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + }else{ + rowStart = BOX_13; + colSep = BOX_13; + rowSep = BOX_13 "\n"; + } + if( p->spec.bBorder==QRF_No){ + rowStart += 3; + rowSep = "\n"; + }else{ + qrfBoxSeparator(p->pOut, &data, BOX_R23, BOX_234, BOX_R34, 0); + } + break; + case QRF_STYLE_Table: + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + if( p->spec.bBorder==QRF_No ){ + rowStart += 1; + rowSep = "\n"; + }else{ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + case QRF_STYLE_Column: { + static const char zSpace[] = " "; + rowStart = ""; + if( data.nMargin<2 ){ + colSep = " "; + }else if( data.nMargin<=5 ){ + colSep = &zSpace[5-data.nMargin]; + }else{ + colSep = zSpace; + } + rowSep = "\n"; + break; + } + default: /*case QRF_STYLE_Markdown:*/ + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + break; + } + szRowStart = (int)strlen(rowStart); + szRowSep = (int)strlen(rowSep); + szColSep = (int)strlen(colSep); + + bWW = (p->spec.bWordWrap==QRF_Yes && data.bMultiRow); + if( p->spec.eStyle==QRF_STYLE_Column + || (p->spec.bBorder==QRF_No + && (p->spec.eStyle==QRF_STYLE_Box || p->spec.eStyle==QRF_STYLE_Table) + ) + ){ + bRTrim = 1; + }else{ + bRTrim = 0; + } + for(i=0; i<data.n && sqlite3_str_errcode(p->pOut)==SQLITE_OK; i+=nColumn){ + int bMore; + int nRow = 0; + + /* Draw a single row of the table. This might be the title line + ** (if there is a title line) or a row in the body of the table. + ** The column number will be j. The row number is i/nColumn. + */ + for(j=0; j<nColumn; j++){ + data.a[j].z = data.az[i+j]; + if( data.a[j].z==0 ) data.a[j].z = ""; + data.a[j].bNum = data.abNum[i+j]; + } + do{ + sqlite3_str_append(p->pOut, rowStart, szRowStart); + bMore = 0; + for(j=0; j<nColumn; j++){ + int nThis = 0; + int nWide = 0; + int iNext = 0; + int nWS; + qrfWrapLine(data.a[j].z, data.a[j].w, bWW, &nThis, &nWide, &iNext); + nWS = data.a[j].w - nWide; + qrfPrintAligned(p->pOut, &data.a[j], nThis, nWS); + data.a[j].z += iNext; + if( data.a[j].z[0]!=0 ){ + bMore = 1; + } + if( j<nColumn-1 ){ + sqlite3_str_append(p->pOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }while( bMore && ++nRow < p->mxHeight ); + if( bMore ){ + /* This row was terminated by nLineLimit. Show ellipsis. */ + sqlite3_str_append(p->pOut, rowStart, szRowStart); + for(j=0; j<nColumn; j++){ + if( data.a[j].z[0]==0 ){ + sqlite3_str_appendchar(p->pOut, data.a[j].w, ' '); + }else{ + int nE = 3; + if( nE>data.a[j].w ) nE = data.a[j].w; + data.a[j].z = "..."; + qrfPrintAligned(p->pOut, &data.a[j], nE, data.a[j].w-nE); + } + if( j<nColumn-1 ){ + sqlite3_str_append(p->pOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + } + + /* Draw either (1) the separator between the title line and the body + ** of the table, or (2) separators between individual rows of the table + ** body. isTitleDataSeparator will be true if we are doing (1). + */ + if( (i==0 || data.bMultiRow) && i+nColumn<data.n ){ + int isTitleDataSeparator = (i==0 && p->spec.bTitles==QRF_Yes); + if( isTitleDataSeparator ){ + qrfLoadAlignment(&data, p); + } + switch( p->spec.eStyle ){ + case QRF_STYLE_Table: { + if( isTitleDataSeparator || data.bMultiRow ){ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + } + case QRF_STYLE_Box: { + if( isTitleDataSeparator ){ + qrfBoxSeparator(p->pOut, &data, DBL_123, DBL_1234, DBL_134, 1); + }else if( data.bMultiRow ){ + qrfBoxSeparator(p->pOut, &data, BOX_123, BOX_1234, BOX_134, 0); + } + break; + } + case QRF_STYLE_Markdown: { + if( isTitleDataSeparator ){ + qrfRowSeparator(p->pOut, &data, '|'); + } + break; + } + case QRF_STYLE_Column: { + if( isTitleDataSeparator ){ + for(j=0; j<nColumn; j++){ + sqlite3_str_appendchar(p->pOut, data.a[j].w, '-'); + if( j<nColumn-1 ){ + sqlite3_str_append(p->pOut, colSep, szColSep); + }else{ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }else if( data.bMultiRow ){ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, "\n", 1); + } + break; + } + } + } + } + + /* Draw the line across the bottom of the table */ + if( p->spec.bBorder!=QRF_No ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + qrfBoxSeparator(p->pOut, &data, BOX_R12, BOX_124, BOX_R14, 0); + break; + case QRF_STYLE_Table: + qrfRowSeparator(p->pOut, &data, '+'); + break; + } + } + qrfWrite(p); + + qrfColDataFree(&data); + return; +} /* -** Initialize and destroy a ShellText object +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. */ -static void initText(ShellText *p){ - memset(p, 0, sizeof(*p)); -} -static void freeText(ShellText *p){ - free(p->z); - initText(p); +static int qrfStringInArray(const char *zStr, const char **azArray){ + int i; + if( zStr==0 ) return 0; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; + } + return 0; } -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. +/* +** Print out an EXPLAIN with indentation. This is a two-pass algorithm. ** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. +** On the first pass, we compute aiIndent[iOp] which is the amount of +** indentation to apply to the iOp-th opcode. The output actually occurs +** on the second pass. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. */ -static void appendText(ShellText *p, const char *zAppend, char quote){ - i64 len; - i64 i; - i64 nAppend = strlen30(zAppend); +static void qrfExplain(Qrf *p){ + int *abYield = 0; /* abYield[iOp] is rue if opcode iOp is an OP_Yield */ + int *aiIndent = 0; /* Indent the iOp-th opcode by aiIndent[iOp] */ + i64 nAlloc = 0; /* Allocated size of aiIndent[], abYield */ + int nIndent = 0; /* Number of entries in aiIndent[] */ + int iOp; /* Opcode number */ + int i; /* Column loop counter */ - len = nAppend+p->n+1; - if( quote ){ - len += 2; - for(i=0; i<nAppend; i++){ - if( zAppend[i]==quote ) len++; + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(p->pStmt)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 3), "p2" ) ); + + for(iOp=0; SQLITE_ROW==sqlite3_step(p->pStmt) && !p->iErr; iOp++){ + int iAddr = sqlite3_column_int(p->pStmt, 0); + const char *zOp = (const char*)sqlite3_column_text(p->pStmt, 1); + int p1 = sqlite3_column_int(p->pStmt, 2); + int p2 = sqlite3_column_int(p->pStmt, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + aiIndent = (int*)sqlite3_realloc64(aiIndent, nAlloc*sizeof(int)); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( aiIndent==0 || abYield==0 ){ + qrfOom(p); + sqlite3_free(aiIndent); + sqlite3_free(abYield); + return; + } } - } - if( p->z==0 || p->n+len>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - shell_check_oom(p->z); + abYield[iOp] = qrfStringInArray(zOp, azYield); + aiIndent[iOp] = 0; + nIndent = iOp+1; + if( qrfStringInArray(zOp, azNext) && p2op>0 ){ + for(i=p2op; i<iOp; i++) aiIndent[i] += 2; + } + if( qrfStringInArray(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){ + for(i=p2op; i<iOp; i++) aiIndent[i] += 2; + } } + sqlite3_free(abYield); - if( quote ){ - char *zCsr = p->z+p->n; - *zCsr++ = quote; - for(i=0; i<nAppend; i++){ - *zCsr++ = zAppend[i]; - if( zAppend[i]==quote ) *zCsr++ = quote; + /* Second pass. Actually generate output */ + sqlite3_reset(p->pStmt); + if( p->iErr==SQLITE_OK ){ + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4,15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = sizeof(aExplainWidth)/sizeof(int); + int iIndent = 1; + int nArg = p->nCol; + if( p->spec.eStyle==QRF_STYLE_StatsVm ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = sizeof(aScanExpWidth)/sizeof(int); + iIndent = 3; } - *zCsr++ = quote; - p->n = (int)(zCsr - p->z); - *zCsr = '\0'; - }else{ - memcpy(p->z+p->n, zAppend, nAppend); - p->n += nAppend; - p->z[p->n] = '\0'; + if( nArg>nWidth ) nArg = nWidth; + + for(iOp=0; sqlite3_step(p->pStmt)==SQLITE_ROW && !p->iErr; iOp++){ + /* If this is the first row seen, print out the headers */ + if( iOp==0 ){ + for(i=0; i<nArg; i++){ + const char *zCol = sqlite3_column_name(p->pStmt, aMap[i]); + qrfWidthPrint(p,p->pOut, aWidth[i], zCol); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + for(i=0; i<nArg; i++){ + sqlite3_str_appendf(p->pOut, "%.*c", aWidth[i], '-'); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + } + + for(i=0; i<nArg; i++){ + const char *zSep = " "; + int w = aWidth[i]; + const char *zVal = (const char*)sqlite3_column_text(p->pStmt, aMap[i]); + int len; + if( i==nArg-1 ) w = 0; + if( zVal==0 ) zVal = ""; + len = (int)sqlite3_qrf_wcswidth(zVal); + if( len>w ){ + w = len; + zSep = " "; + } + if( i==iIndent && aiIndent && iOp<nIndent ){ + sqlite3_str_appendchar(p->pOut, aiIndent[iOp], ' '); + } + qrfWidthPrint(p, p->pOut, w, zVal); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_appendall(p->pOut, zSep); + } + } + p->nRow++; + } + qrfWrite(p); + } + sqlite3_free(aiIndent); +} + +/* +** Do a "scanstatus vm" style EXPLAIN listing on p->pStmt. +** +** p->pStmt is probably not an EXPLAIN query. Instead, construct a +** new query that is a bytecode() rendering of p->pStmt with extra +** columns for the "scanstatus vm" outputs, and run the results of +** that new query through the normal EXPLAIN formatting. +*/ +static void qrfScanStatusVm(Qrf *p){ + sqlite3_stmt *pOrigStmt = p->pStmt; + sqlite3_stmt *pExplain; + int rc; + static const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?1)"; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pExplain, 0); + if( rc ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + sqlite3_finalize(pExplain); + return; } + sqlite3_bind_pointer(pExplain, 1, pOrigStmt, "stmt-pointer", 0); + p->pStmt = pExplain; + p->nCol = 10; + qrfExplain(p); + sqlite3_finalize(pExplain); + p->pStmt = pOrigStmt; } /* @@ -1525,178 +3361,544 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ ** SQLite keyword. Be conservative in this estimate: When in doubt assume ** that quoting is required. ** -** Return '"' if quoting is required. Return 0 if no quoting is required. +** Return 1 if quoting is required. Return 0 if no quoting is required. */ -static char quoteChar(const char *zName){ + +static int qrf_need_quote(const char *zName){ int i; - if( zName==0 ) return '"'; - if( !IsAlpha(zName[0]) && zName[0]!='_' ) return '"'; - for(i=0; zName[i]; i++){ - if( !IsAlnum(zName[i]) && zName[i]!='_' ) return '"'; + const unsigned char *z = (const unsigned char*)zName; + if( z==0 ) return 1; + if( !qrfAlpha(z[0]) ) return 1; + for(i=0; z[i]; i++){ + if( !qrfAlnum(z[i]) ) return 1; } - return sqlite3_keyword_check(zName, i) ? '"' : 0; + return sqlite3_keyword_check(zName, i)!=0; } /* -** Construct a fake object name and column list to describe the structure -** of the view, virtual table, or table valued function zSchema.zName. +** Helper function for QRF_STYLE_Json and QRF_STYLE_JObject. +** The initial "{" for a JSON object that will contain row content +** has been output. Now output all the content. */ -static char *shellFakeSchema( - sqlite3 *db, /* The database connection containing the vtab */ - const char *zSchema, /* Schema of the database holding the vtab */ - const char *zName /* The name of the virtual table */ -){ - sqlite3_stmt *pStmt = 0; - char *zSql; - ShellText s; - char cQuote; - char *zDiv = "("; - int nRow = 0; - - zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", - zSchema ? zSchema : "main", zName); - shell_check_oom(zSql); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - initText(&s); - if( zSchema ){ - cQuote = quoteChar(zSchema); - if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; - appendText(&s, zSchema, cQuote); - appendText(&s, ".", 0); - } - cQuote = quoteChar(zName); - appendText(&s, zName, cQuote); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); - nRow++; - appendText(&s, zDiv, 0); - zDiv = ","; - if( zCol==0 ) zCol = ""; - cQuote = quoteChar(zCol); - appendText(&s, zCol, cQuote); +static void qrfOneJsonRow(Qrf *p){ + int i, nItem; + for(nItem=i=0; i<p->nCol; i++){ + const char *zCName; + zCName = sqlite3_column_name(p->pStmt, i); + if( nItem>0 ) sqlite3_str_append(p->pOut, ",", 1); + nItem++; + qrfEncodeText(p, p->pOut, zCName); + sqlite3_str_append(p->pOut, ":", 1); + qrfRenderValue(p, p->pOut, i); } - appendText(&s, ")", 0); - sqlite3_finalize(pStmt); - if( nRow==0 ){ - freeText(&s); - s.z = 0; + qrfWrite(p); +} + +/* +** Render a single row of output for non-columnar styles - any +** style that lets us render row by row as the content is received +** from the query. +*/ +static void qrfOneSimpleRow(Qrf *p){ + int i; + switch( p->spec.eStyle ){ + case QRF_STYLE_Off: + case QRF_STYLE_Count: { + /* No-op */ + break; + } + case QRF_STYLE_Json: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "[{", 2); + }else{ + sqlite3_str_append(p->pOut, "},\n{", 4); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_JObject: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "{", 1); + }else{ + sqlite3_str_append(p->pOut, "}\n{", 3); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_Html: { + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + sqlite3_str_append(p->pOut, "<TR>", 4); + for(i=0; i<p->nCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + sqlite3_str_append(p->pOut, "\n<TH>", 5); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_append(p->pOut, "\n</TR>\n", 7); + } + sqlite3_str_append(p->pOut, "<TR>", 4); + for(i=0; i<p->nCol; i++){ + sqlite3_str_append(p->pOut, "\n<TD>", 5); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_append(p->pOut, "\n</TR>\n", 7); + qrfWrite(p); + break; + } + case QRF_STYLE_Insert: { + unsigned int mxIns = p->spec.nMultiInsert; + int szStart = sqlite3_str_length(p->pOut); + if( p->u.nIns==0 || p->u.nIns>=mxIns ){ + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + p->u.nIns = 0; + } + if( qrf_need_quote(p->spec.zTableName) ){ + sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); + }else{ + sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); + } + if( p->spec.bTitles==QRF_Yes ){ + for(i=0; i<p->nCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( qrf_need_quote(zCName) ){ + sqlite3_str_appendf(p->pOut, "%c\"%w\"", + i==0 ? '(' : ',', zCName); + }else{ + sqlite3_str_appendf(p->pOut, "%c%s", + i==0 ? '(' : ',', zCName); + } + } + sqlite3_str_append(p->pOut, ")", 1); + } + sqlite3_str_append(p->pOut," VALUES(", 8); + }else{ + sqlite3_str_append(p->pOut,",\n (", 5); + } + for(i=0; i<p->nCol; i++){ + if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); + qrfRenderValue(p, p->pOut, i); + } + p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart; + if( p->u.nIns>=mxIns ){ + sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns = 0; + }else{ + sqlite3_str_append(p->pOut, ")", 1); + } + qrfWrite(p); + break; + } + case QRF_STYLE_Line: { + sqlite3_str *pVal; + int mxW; + int bWW; + int nSep; + if( p->u.sLine.azCol==0 ){ + p->u.sLine.azCol = sqlite3_malloc64( p->nCol*sizeof(char*) ); + if( p->u.sLine.azCol==0 ){ + qrfOom(p); + break; + } + p->u.sLine.mxColWth = 0; + for(i=0; i<p->nCol; i++){ + int sz; + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( zCName==0 ) zCName = "unknown"; + p->u.sLine.azCol[i] = sqlite3_mprintf("%s", zCName); + if( p->spec.nTitleLimit>0 ){ + (void)qrfTitleLimit(p->u.sLine.azCol[i], p->spec.nTitleLimit); + } + sz = (int)sqlite3_qrf_wcswidth(p->u.sLine.azCol[i]); + if( sz > p->u.sLine.mxColWth ) p->u.sLine.mxColWth = sz; + } + } + if( p->nRow ) sqlite3_str_append(p->pOut, "\n", 1); + pVal = sqlite3_str_new(p->db); + nSep = (int)strlen(p->spec.zColumnSep); + mxW = p->mxWidth - (nSep + p->u.sLine.mxColWth); + bWW = p->spec.bWordWrap==QRF_Yes; + for(i=0; i<p->nCol; i++){ + const char *zVal; + int cnt = 0; + qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth, p->u.sLine.azCol[i]); + sqlite3_str_append(p->pOut, p->spec.zColumnSep, nSep); + qrfRenderValue(p, pVal, i); + zVal = sqlite3_str_value(pVal); + if( zVal==0 ) zVal = ""; + do{ + int nThis, nWide, iNext; + qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); + if( cnt ){ + sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+nSep,' '); + } + cnt++; + if( cnt>p->mxHeight ){ + zVal = "..."; + nThis = iNext = 3; + } + sqlite3_str_append(p->pOut, zVal, nThis); + sqlite3_str_append(p->pOut, "\n", 1); + zVal += iNext; + }while( zVal[0] ); + sqlite3_str_reset(pVal); + } + qrfStrErr(p, pVal); + sqlite3_free(sqlite3_str_finish(pVal)); + qrfWrite(p); + break; + } + case QRF_STYLE_Eqp: { + const char *zEqpLine = (const char*)sqlite3_column_text(p->pStmt,3); + int iEqpId = sqlite3_column_int(p->pStmt, 0); + int iParentId = sqlite3_column_int(p->pStmt, 1); + if( zEqpLine==0 ) zEqpLine = ""; + if( zEqpLine[0]=='-' ) qrfEqpRender(p, 0); + qrfEqpAppend(p, iEqpId, iParentId, zEqpLine); + break; + } + default: { /* QRF_STYLE_List */ + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + int saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + for(i=0; i<p->nCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + p->spec.eText = saved_eText; + } + for(i=0; i<p->nCol; i++){ + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + break; + } } - return s.z; + p->nRow++; } /* -** SQL function: strtod(X) -** -** Use the C-library strtod() function to convert string X into a double. -** Used for comparing the accuracy of SQLite's internal text-to-float conversion -** routines against the C-library. +** Initialize the internal Qrf object. */ -static void shellStrtod( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal +static void qrfInitialize( + Qrf *p, /* State object to be initialized */ + sqlite3_stmt *pStmt, /* Query whose output to be formatted */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write errors here */ ){ - char *z = (char*)sqlite3_value_text(apVal[0]); - UNUSED_PARAMETER(nVal); - if( z==0 ) return; - sqlite3_result_double(pCtx, strtod(z,0)); + size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ + memset(p, 0, sizeof(*p)); + p->pzErr = pzErr; + if( pSpec->iVersion>1 ){ + qrfError(p, SQLITE_ERROR, + "unusable sqlite3_qrf_spec.iVersion (%d)", + pSpec->iVersion); + return; + } + p->pStmt = pStmt; + p->db = sqlite3_db_handle(pStmt); + p->pOut = sqlite3_str_new(p->db); + if( p->pOut==0 ){ + qrfOom(p); + return; + } + p->iErr = SQLITE_OK; + p->nCol = sqlite3_column_count(p->pStmt); + p->nRow = 0; + sz = sizeof(sqlite3_qrf_spec); + memcpy(&p->spec, pSpec, sz); + if( p->spec.zNull==0 ) p->spec.zNull = ""; + p->mxWidth = p->spec.nScreenWidth; + if( p->mxWidth<=0 ) p->mxWidth = QRF_MAX_WIDTH; + p->mxHeight = p->spec.nLineLimit; + if( p->mxHeight<=0 ) p->mxHeight = 2147483647; + if( p->spec.eStyle>QRF_STYLE_Table ) p->spec.eStyle = QRF_Auto; + if( p->spec.eEsc>QRF_ESC_Symbol ) p->spec.eEsc = QRF_Auto; + if( p->spec.eText>QRF_TEXT_Relaxed ) p->spec.eText = QRF_Auto; + if( p->spec.eTitle>QRF_TEXT_Relaxed ) p->spec.eTitle = QRF_Auto; + if( p->spec.eBlob>QRF_BLOB_Size ) p->spec.eBlob = QRF_Auto; +qrf_reinit: + switch( p->spec.eStyle ){ + case QRF_Auto: { + switch( sqlite3_stmt_isexplain(pStmt) ){ + case 0: p->spec.eStyle = QRF_STYLE_Box; break; + case 1: p->spec.eStyle = QRF_STYLE_Explain; break; + default: p->spec.eStyle = QRF_STYLE_Eqp; break; + } + goto qrf_reinit; + } + case QRF_STYLE_List: { + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = "|"; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_JObject: + case QRF_STYLE_Json: { + p->spec.eText = QRF_TEXT_Json; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Html: { + p->spec.eText = QRF_TEXT_Html; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Insert: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ + p->spec.zTableName = "tab"; + } + p->u.nIns = 0; + break; + } + case QRF_STYLE_Line: { + if( p->spec.zColumnSep==0 ){ + p->spec.zColumnSep = ": "; + } + break; + } + case QRF_STYLE_Csv: { + p->spec.eStyle = QRF_STYLE_List; + p->spec.eText = QRF_TEXT_Csv; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\r\n"; + p->spec.zNull = ""; + break; + } + case QRF_STYLE_Quote: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_Eqp: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=2 ){ + sqlite3_stmt_explain(p->pStmt, 2); + p->expMode = expMode+1; + } + break; + } + case QRF_STYLE_Explain: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=1 ){ + sqlite3_stmt_explain(p->pStmt, 1); + p->expMode = expMode+1; + } + break; + } + } + if( p->spec.eEsc==QRF_Auto ){ + p->spec.eEsc = QRF_ESC_Ascii; + } + if( p->spec.eText==QRF_Auto ){ + p->spec.eText = QRF_TEXT_Plain; + } + if( p->spec.eTitle==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + p->spec.eTitle = QRF_TEXT_Plain; + break; + default: + p->spec.eTitle = p->spec.eText; + break; + } + } + if( p->spec.eBlob==QRF_Auto ){ + switch( p->spec.eText ){ + case QRF_TEXT_Sql: p->spec.eBlob = QRF_BLOB_Sql; break; + case QRF_TEXT_Csv: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Tcl: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Json: p->spec.eBlob = QRF_BLOB_Json; break; + default: p->spec.eBlob = QRF_BLOB_Text; break; + } + } + if( p->spec.bTitles==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Csv: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + case QRF_STYLE_Markdown: + p->spec.bTitles = QRF_Yes; + break; + default: + p->spec.bTitles = QRF_No; + break; + } + } + if( p->spec.bWordWrap==QRF_Auto ){ + p->spec.bWordWrap = QRF_Yes; + } + if( p->spec.bTextJsonb==QRF_Auto ){ + p->spec.bTextJsonb = QRF_No; + } + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = ","; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; } /* -** SQL function: dtostr(X) -** -** Use the C-library printf() function to convert real value X into a string. -** Used for comparing the accuracy of SQLite's internal float-to-text conversion -** routines against the C-library. +** Finish rendering the results */ -static void shellDtostr( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - double r = sqlite3_value_double(apVal[0]); - int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; - char z[400]; - if( n<1 ) n = 1; - if( n>350 ) n = 350; - sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); - sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); +static void qrfFinalize(Qrf *p){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Count: { + sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); + break; + } + case QRF_STYLE_Json: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}]\n", 3); + } + break; + } + case QRF_STYLE_JObject: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}\n", 2); + } + break; + } + case QRF_STYLE_Insert: { + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + } + break; + } + case QRF_STYLE_Line: { + if( p->u.sLine.azCol ){ + int i; + for(i=0; i<p->nCol; i++) sqlite3_free(p->u.sLine.azCol[i]); + sqlite3_free(p->u.sLine.azCol); + } + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + i64 nCycle = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + sqlite3_stmt_scanstatus_v2(p->pStmt, -1, SQLITE_SCANSTAT_NCYCLE, + SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); +#endif + qrfEqpRender(p, nCycle); + break; + } + case QRF_STYLE_Eqp: { + qrfEqpRender(p, 0); + break; + } + } + qrfWrite(p); + qrfStrErr(p, p->pOut); + if( p->spec.pzOutput ){ + if( p->spec.pzOutput[0] ){ + sqlite3_int64 n, sz; + char *zCombined; + sz = strlen(p->spec.pzOutput[0]); + n = sqlite3_str_length(p->pOut); + zCombined = sqlite3_realloc64(p->spec.pzOutput[0], sz+n+1); + if( zCombined==0 ){ + sqlite3_free(p->spec.pzOutput[0]); + p->spec.pzOutput[0] = 0; + qrfOom(p); + }else{ + p->spec.pzOutput[0] = zCombined; + memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n+1); + } + sqlite3_free(sqlite3_str_finish(p->pOut)); + }else{ + p->spec.pzOutput[0] = sqlite3_str_finish(p->pOut); + } + }else if( p->pOut ){ + sqlite3_free(sqlite3_str_finish(p->pOut)); + } + if( p->expMode>0 ){ + sqlite3_stmt_explain(p->pStmt, p->expMode-1); + } + if( p->actualWidth ){ + sqlite3_free(p->actualWidth); + } + if( p->pJTrans ){ + sqlite3 *db = sqlite3_db_handle(p->pJTrans); + sqlite3_finalize(p->pJTrans); + sqlite3_close(db); + } } /* -** SQL function: shell_add_schema(S,X) -** -** Add the schema name X to the CREATE statement in S and return the result. -** Examples: -** -** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); -** -** Also works on -** -** CREATE INDEX -** CREATE UNIQUE INDEX -** CREATE VIEW -** CREATE TRIGGER -** CREATE VIRTUAL TABLE -** -** This UDF is used by the .schema command to insert the schema name of -** attached databases into the middle of the sqlite_schema.sql field. +** Run the prepared statement pStmt and format the results according +** to the specification provided in pSpec. Return an error code. +** If pzErr is not NULL and if an error occurs, write an error message +** into *pzErr. */ -static void shellAddSchemaName( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* Statement to evaluate */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write error message here */ ){ - static const char *aPrefix[] = { - "TABLE", - "INDEX", - "UNIQUE INDEX", - "VIEW", - "TRIGGER", - "VIRTUAL TABLE" - }; - int i = 0; - const char *zIn = (const char*)sqlite3_value_text(apVal[0]); - const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); - const char *zName = (const char*)sqlite3_value_text(apVal[2]); - sqlite3 *db = sqlite3_context_db_handle(pCtx); - UNUSED_PARAMETER(nVal); - if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ - for(i=0; i<ArraySize(aPrefix); i++){ - int n = strlen30(aPrefix[i]); - if( cli_strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){ - char *z = 0; - char *zFake = 0; - if( zSchema ){ - char cQuote = quoteChar(zSchema); - if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){ - z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8); - }else{ - z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8); - } - } - if( zName - && aPrefix[i][0]=='V' - && (zFake = shellFakeSchema(db, zSchema, zName))!=0 - ){ - if( z==0 ){ - z = sqlite3_mprintf("%s\n/* %s */", zIn, zFake); - }else{ - z = sqlite3_mprintf("%z\n/* %s */", z, zFake); - } - free(zFake); - } - if( z ){ - sqlite3_result_text(pCtx, z, -1, sqlite3_free); - return; - } + Qrf qrf; /* The new Qrf being created */ + + if( pStmt==0 ) return SQLITE_OK; /* No-op */ + if( pSpec==0 ) return SQLITE_MISUSE; + qrfInitialize(&qrf, pStmt, pSpec, pzErr); + switch( qrf.spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Markdown: + case QRF_STYLE_Table: { + /* Columnar modes require that the entire query be evaluated and the + ** results stored in memory, so that we can compute column widths */ + qrfColumnar(&qrf); + break; + } + case QRF_STYLE_Explain: { + qrfExplain(&qrf); + break; + } + case QRF_STYLE_StatsVm: { + qrfScanStatusVm(&qrf); + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + qrfEqpStats(&qrf); + break; + } + default: { + /* Non-columnar modes where the output can occur after each row + ** of result is received */ + while( qrf.iErr==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + qrfOneSimpleRow(&qrf); } + break; } } - sqlite3_result_value(pCtx, apVal[0]); + qrfResetStmt(&qrf); + qrfFinalize(&qrf); + return qrf.iErr; } +/************************* End ext/qrf/qrf.c ********************/ + +/* Use console I/O package as a direct INCLUDE. */ +#define SQLITE_INTERNAL_LINKAGE static + +#ifdef SQLITE_SHELL_FIDDLE +/* Deselect most features from the console I/O package for Fiddle. */ +# define SQLITE_CIO_NO_REDIRECT +# define SQLITE_CIO_NO_CLASSIFY +# define SQLITE_CIO_NO_TRANSLATE +# define SQLITE_CIO_NO_SETMODE +# define SQLITE_CIO_NO_FLUSH +#endif + /* ** The source code for several run-time loadable extensions is inserted ** below by the ../tool/mkshellc.tcl script. Before processing that included @@ -1706,10 +3908,9 @@ static void shellAddSchemaName( #define SQLITE_EXTENSION_INIT1 #define SQLITE_EXTENSION_INIT2(X) (void)(X) -#if defined(_WIN32) && defined(_MSC_VER) -/************************* Begin test_windirent.h ******************/ +/************************* Begin ext/misc/windirent.h ******************/ /* -** 2015 November 30 +** 2025-06-05 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -1719,323 +3920,161 @@ static void shellAddSchemaName( ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains declarations for most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. +** +** An implementation of opendir(), readdir(), and closedir() for Windows, +** based on the FindFirstFile(), FindNextFile(), and FindClose() APIs +** of Win32. +** +** #include this file inside any C-code module that needs to use +** opendir()/readdir()/closedir(). This file is a no-op on non-Windows +** machines. On Windows, static functions are defined that implement +** those standard interfaces. */ - #if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) #define SQLITE_WINDIRENT_H -/* -** We need several data types from the Windows SDK header. -*/ - #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif - -#include "windows.h" - -/* -** We need several support functions from the SQLite core. -*/ - -/* #include "sqlite3.h" */ - -/* -** We need several things from the ANSI and MSVCRT headers. -*/ - +#include <windows.h> +#include <io.h> #include <stdio.h> #include <stdlib.h> #include <errno.h> -#include <io.h> #include <limits.h> #include <sys/types.h> #include <sys/stat.h> - -/* -** We may need several defines that should have been in "sys/stat.h". -*/ - +#include <string.h> +#ifndef FILENAME_MAX +# define FILENAME_MAX (260) +#endif #ifndef S_ISREG -#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) #endif - #ifndef S_ISDIR -#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) #endif - #ifndef S_ISLNK -#define S_ISLNK(mode) (0) -#endif - -/* -** We may need to provide the "mode_t" type. -*/ - -#ifndef MODE_T_DEFINED - #define MODE_T_DEFINED - typedef unsigned short mode_t; +#define S_ISLNK(m) (0) #endif +typedef unsigned short mode_t; -/* -** We may need to provide the "ino_t" type. +/* The dirent object for Windows is abbreviated. The only field really +** usable by applications is d_name[]. */ - -#ifndef INO_T_DEFINED - #define INO_T_DEFINED - typedef unsigned short ino_t; -#endif - -/* -** We need to define "NAME_MAX" if it was not present in "limits.h". -*/ - -#ifndef NAME_MAX -# ifdef FILENAME_MAX -# define NAME_MAX (FILENAME_MAX) -# else -# define NAME_MAX (260) -# endif -# define DIRENT_NAME_MAX (NAME_MAX) -#endif - -/* -** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". -*/ - -#ifndef NULL_INTPTR_T -# define NULL_INTPTR_T ((intptr_t)(0)) -#endif - -#ifndef BAD_INTPTR_T -# define BAD_INTPTR_T ((intptr_t)(-1)) -#endif - -/* -** We need to provide the necessary structures and related types. -*/ - -#ifndef DIRENT_DEFINED -#define DIRENT_DEFINED -typedef struct DIRENT DIRENT; -typedef DIRENT *LPDIRENT; -struct DIRENT { - ino_t d_ino; /* Sequence number, do not use. */ - unsigned d_attributes; /* Win32 file attributes. */ - char d_name[NAME_MAX + 1]; /* Name within the directory. */ +struct dirent { + int d_ino; /* Inode number (synthesized) */ + unsigned d_attributes; /* File attributes */ + char d_name[FILENAME_MAX]; /* Null-terminated filename */ }; -#endif -#ifndef DIR_DEFINED -#define DIR_DEFINED +/* The internals of DIR are opaque according to standards. So it +** does not matter what we put here. */ typedef struct DIR DIR; -typedef DIR *LPDIR; struct DIR { - intptr_t d_handle; /* Value returned by "_findfirst". */ - DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ - DIRENT d_next; /* DIRENT constructed based on "_findnext". */ + intptr_t d_handle; /* Handle for findfirst()/findnext() */ + struct dirent cur; /* Current entry */ }; -#endif - -/* -** Provide a macro, for use by the implementation, to determine if a -** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir(). -*/ - -#ifndef is_filtered -# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) -#endif - -/* -** Provide the function prototype for the POSIX compatible getenv() -** function. This function is not thread-safe. -*/ - -extern const char *windirent_getenv(const char *name); - -/* -** Finally, we can provide the function prototypes for the opendir(), -** readdir(), and closedir() POSIX functions. -*/ - -extern LPDIR opendir(const char *dirname); -extern LPDIRENT readdir(LPDIR dirp); -extern INT closedir(LPDIR dirp); - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.h ********************/ -/************************* Begin test_windirent.c ******************/ -/* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code to implement most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ -#if defined(_WIN32) && defined(_MSC_VER) -/* #include "test_windirent.h" */ +/* Ignore hidden and system files */ +#define WindowsFileToIgnore(a) \ + ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) /* -** Implementation of the POSIX getenv() function using the Win32 API. -** This function is not thread-safe. +** Close a previously opened directory */ -const char *windirent_getenv( - const char *name -){ - static char value[32768]; /* Maximum length, per MSDN */ - DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ - DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ - - memset(value, 0, sizeof(value)); - dwRet = GetEnvironmentVariableA(name, value, dwSize); - if( dwRet==0 || dwRet>dwSize ){ - /* - ** The function call to GetEnvironmentVariableA() failed -OR- - ** the buffer is not large enough. Either way, return NULL. - */ - return 0; - }else{ - /* - ** The function call to GetEnvironmentVariableA() succeeded - ** -AND- the buffer contains the entire value. - */ - return value; +static int closedir(DIR *pDir){ + int rc = 0; + if( pDir==0 ){ + return EINVAL; + } + if( pDir->d_handle!=0 && pDir->d_handle!=(-1) ){ + rc = _findclose(pDir->d_handle); } + sqlite3_free(pDir); + return rc; } /* -** Implementation of the POSIX opendir() function using the MSVCRT. +** Open a new directory. The directory name should be UTF-8 encoded. +** appropriate translations happen automatically. */ -LPDIR opendir( - const char *dirname /* Directory name, UTF8 encoding */ -){ - struct _wfinddata_t data; - LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); - SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); +static DIR *opendir(const char *zDirName){ + DIR *pDir; wchar_t *b1; sqlite3_int64 sz; + struct _wfinddata_t data; - if( dirp==NULL ) return NULL; - memset(dirp, 0, sizeof(DIR)); - - /* TODO: Remove this if Unix-style root paths are not used. */ - if( sqlite3_stricmp(dirname, "/")==0 ){ - dirname = windirent_getenv("SystemDrive"); - } - + pDir = sqlite3_malloc64( sizeof(DIR) ); + if( pDir==0 ) return 0; + memset(pDir, 0, sizeof(DIR)); memset(&data, 0, sizeof(data)); - sz = strlen(dirname); + sz = strlen(zDirName); b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); if( b1==0 ){ - closedir(dirp); + closedir(pDir); return NULL; } - sz = MultiByteToWideChar(CP_UTF8, 0, dirname, sz, b1, sz); + sz = MultiByteToWideChar(CP_UTF8, 0, zDirName, sz, b1, sz); b1[sz++] = '\\'; b1[sz++] = '*'; b1[sz] = 0; - if( sz+1>(sqlite3_int64)namesize ){ - closedir(dirp); + if( sz+1>sizeof(data.name)/sizeof(data.name[0]) ){ + closedir(pDir); sqlite3_free(b1); return NULL; } memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); sqlite3_free(b1); - dirp->d_handle = _wfindfirst(data.name, &data); - - if( dirp->d_handle==BAD_INTPTR_T ){ - closedir(dirp); + pDir->d_handle = _wfindfirst(data.name, &data); + if( pDir->d_handle<0 ){ + closedir(pDir); return NULL; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ){ -next: - + while( WindowsFileToIgnore(data) ){ memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ){ - closedir(dirp); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + closedir(pDir); return NULL; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; } - - dirp->d_first.d_attributes = data.attrib; + pDir->cur.d_ino = 0; + pDir->cur.d_attributes = data.attrib; WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_first.d_name, DIRENT_NAME_MAX, 0, 0); - return dirp; + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return pDir; } /* -** Implementation of the POSIX readdir() function using the MSVCRT. +** Read the next entry from a directory. +** +** The returned struct-dirent object is managed by DIR. It is only +** valid until the next readdir() or closedir() call. Only the +** d_name[] field is meaningful. The d_name[] value has been +** translated into UTF8. */ -LPDIRENT readdir( - LPDIR dirp -){ +static struct dirent *readdir(DIR *pDir){ struct _wfinddata_t data; - - if( dirp==NULL ) return NULL; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - return &dirp->d_first; + if( pDir==0 ) return 0; + if( (pDir->cur.d_ino++)==0 ){ + return &pDir->cur; } - -next: - - memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ) return NULL; - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - dirp->d_next.d_ino++; - dirp->d_next.d_attributes = data.attrib; + do{ + memset(&data, 0, sizeof(data)); + if( _wfindnext(pDir->d_handle, &data)==-1 ){ + return NULL; + } + }while( WindowsFileToIgnore(data) ); + pDir->cur.d_attributes = data.attrib; WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_next.d_name, DIRENT_NAME_MAX, 0, 0); - return &dirp->d_next; -} - -/* -** Implementation of the POSIX closedir() function using the MSVCRT. -*/ -INT closedir( - LPDIR dirp -){ - INT result = 0; - - if( dirp==NULL ) return EINVAL; - - if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ - result = _findclose(dirp->d_handle); - } - - sqlite3_free(dirp); - return result; + pDir->cur.d_name, FILENAME_MAX, 0, 0); + return &pDir->cur; } -#endif /* defined(WIN32) && defined(_MSC_VER) */ +#endif /* defined(_WIN32) && defined(_MSC_VER) */ -/************************* End test_windirent.c ********************/ -#define dirent DIRENT -#endif -/************************* Begin ../ext/misc/memtrace.c ******************/ +/************************* End ext/misc/windirent.h ********************/ +/************************* Begin ext/misc/memtrace.c ******************/ /* ** 2019-01-21 ** @@ -2145,8 +4184,8 @@ int sqlite3MemTraceDeactivate(void){ return rc; } -/************************* End ../ext/misc/memtrace.c ********************/ -/************************* Begin ../ext/misc/pcachetrace.c ******************/ +/************************* End ext/misc/memtrace.c ********************/ +/************************* Begin ext/misc/pcachetrace.c ******************/ /* ** 2023-06-21 ** @@ -2327,8 +4366,8 @@ int sqlite3PcacheTraceDeactivate(void){ return rc; } -/************************* End ../ext/misc/pcachetrace.c ********************/ -/************************* Begin ../ext/misc/shathree.c ******************/ +/************************* End ext/misc/pcachetrace.c ********************/ +/************************* Begin ext/misc/shathree.c ******************/ /* ** 2017-03-08 ** @@ -3184,8 +5223,8 @@ int sqlite3_shathree_init( return rc; } -/************************* End ../ext/misc/shathree.c ********************/ -/************************* Begin ../ext/misc/sha1.c ******************/ +/************************* End ext/misc/shathree.c ********************/ +/************************* Begin ext/misc/sha1.c ******************/ /* ** 2017-01-27 ** @@ -3418,13 +5457,16 @@ static void hash_finish( *****************************************************************************/ /* -** Implementation of the sha1(X) function. +** Two SQL functions: sha1(X) and sha1b(X). ** -** Return a lower-case hexadecimal rendering of the SHA1 hash of the -** argument X. If X is a BLOB, it is hashed as is. For all other +** sha1(X) returns a lower-case hexadecimal rendering of the SHA1 hash +** of the argument X. If X is a BLOB, it is hashed as is. For all other ** types of input, X is converted into a UTF-8 string and the string -** is hash without the trailing 0x00 terminator. The hash of a NULL +** is hashed without the trailing 0x00 terminator. The hash of a NULL ** value is NULL. +** +** sha1b(X) is the same except that it returns a 20-byte BLOB containing +** the binary hash instead of a hexadecimal string. */ static void sha1Func( sqlite3_context *context, @@ -3434,22 +5476,27 @@ static void sha1Func( SHA1Context cx; int eType = sqlite3_value_type(argv[0]); int nByte = sqlite3_value_bytes(argv[0]); + const unsigned char *pData; char zOut[44]; assert( argc==1 ); if( eType==SQLITE_NULL ) return; hash_init(&cx); if( eType==SQLITE_BLOB ){ - hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_blob(argv[0]); }else{ - hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_text(argv[0]); } + if( pData==0 ) return; + hash_step(&cx, pData, nByte); if( sqlite3_user_data(context)!=0 ){ + /* sha1b() - binary result */ hash_finish(&cx, zOut, 1); sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); }else{ + /* sha1() - hexadecimal text result */ hash_finish(&cx, zOut, 0); - sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } } @@ -3503,6 +5550,7 @@ static void sha1QueryFunc( } nCol = sqlite3_column_count(pStmt); z = sqlite3_sql(pStmt); + if( z==0 ) z = ""; n = (int)strlen(z); hash_step_vformat(&cx,"S%d:",n); hash_step(&cx,(unsigned char*)z,n); @@ -3596,8 +5644,8 @@ int sqlite3_sha_init( return rc; } -/************************* End ../ext/misc/sha1.c ********************/ -/************************* Begin ../ext/misc/uint.c ******************/ +/************************* End ext/misc/sha1.c ********************/ +/************************* Begin ext/misc/uint.c ******************/ /* ** 2020-04-14 ** @@ -3691,8 +5739,8 @@ int sqlite3_uint_init( return sqlite3_create_collation(db, "uint", SQLITE_UTF8, 0, uintCollFunc); } -/************************* End ../ext/misc/uint.c ********************/ -/************************* Begin ../ext/misc/decimal.c ******************/ +/************************* End ext/misc/uint.c ********************/ +/************************* Begin ext/misc/decimal.c ******************/ /* ** 2020-06-22 ** @@ -3726,6 +5774,10 @@ SQLITE_EXTENSION_INIT1 #define IsSpace(X) isspace((unsigned char)X) #endif +#ifndef SQLITE_DECIMAL_MAX_DIGIT +# define SQLITE_DECIMAL_MAX_DIGIT 10000000 +#endif + /* A decimal object */ typedef struct Decimal Decimal; struct Decimal { @@ -3764,7 +5816,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ int i; int iExp = 0; - p = sqlite3_malloc( sizeof(*p) ); + if( zIn==0 ) goto new_from_text_failed; + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) goto new_from_text_failed; p->sign = 0; p->oom = 0; @@ -3823,8 +5876,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); + if( a==0 ) goto new_from_text_failed; + p->a = a; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; } @@ -3842,14 +5897,21 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); + if( a==0 ) goto new_from_text_failed; + p->a = a; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); p->nDigit += iExp; p->nFrac += iExp; } } + if( p->sign ){ + for(i=0; i<p->nDigit && p->a[i]==0; i++){} + if( i>=p->nDigit ) p->sign = 0; + } + if( p->nDigit>SQLITE_DECIMAL_MAX_DIGIT ) goto new_from_text_failed; return p; new_from_text_failed: @@ -3942,7 +6004,7 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - z = sqlite3_malloc( p->nDigit+4 ); + z = sqlite3_malloc64( (sqlite3_int64)p->nDigit+4 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -3981,11 +6043,37 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ } /* +** Round a decimal value to N significant digits. N must be positive. +*/ +static void decimal_round(Decimal *p, int N){ + int i; + int nZero; + if( N<1 ) return; + if( p==0 ) return; + if( p->nDigit<=N ) return; + for(nZero=0; nZero<p->nDigit && p->a[nZero]==0; nZero++){} + N += nZero; + if( p->nDigit<=N ) return; + if( p->a[N]>4 ){ + p->a[N-1]++; + for(i=N-1; i>0 && p->a[i]>9; i--){ + p->a[i] = 0; + p->a[i-1]++; + } + if( p->a[0]>9 ){ + p->a[0] = 1; + p->nFrac--; + } + } + memset(&p->a[N], 0, p->nDigit - N); +} + +/* ** Make the given Decimal the result in an format similar to '%+#e'. ** In other words, show exponential notation with leading and trailing ** zeros omitted. */ -static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ char *z; /* The output buffer */ int i; /* Loop counter */ int nZero; /* Number of leading zeros */ @@ -4003,11 +6091,12 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + if( N<1 ) N = 0; + for(nDigit=p->nDigit; nDigit>N && p->a[nDigit-1]==0; nDigit--){} for(nZero=0; nZero<nDigit && p->a[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; - z = sqlite3_malloc( nDigit+20 ); + z = sqlite3_malloc64( (sqlite3_int64)nDigit+20 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -4052,13 +6141,21 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ ** pB!=0 ** pB->isNull==0 */ -static int decimal_cmp(const Decimal *pA, const Decimal *pB){ +static int decimal_cmp(Decimal *pA, Decimal *pB){ int nASig, nBSig, rc, n; + while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ + pA->nDigit--; + pA->nFrac--; + } + while( pB->nFrac>0 && pB->a[pB->nDigit-1]==0 ){ + pB->nDigit--; + pB->nFrac--; + } if( pA->sign!=pB->sign ){ return pA->sign ? -1 : +1; } if( pA->sign ){ - const Decimal *pTemp = pA; + Decimal *pTemp = pA; pA = pB; pB = pTemp; } @@ -4111,15 +6208,18 @@ cmp_done: static void decimal_expand(Decimal *p, int nDigit, int nFrac){ int nAddSig; int nAddFrac; + signed char *a; if( p==0 ) return; nAddFrac = nFrac - p->nFrac; nAddSig = (nDigit - p->nDigit) - nAddFrac; if( nAddFrac==0 && nAddSig==0 ) return; - p->a = sqlite3_realloc64(p->a, nDigit+1); - if( p->a==0 ){ + if( nDigit+1>SQLITE_DECIMAL_MAX_DIGIT ){ p->oom = 1; return; } + a = sqlite3_realloc64(p->a, nDigit+1); + if( a==0 ){ p->oom = 1; return; } + p->a = a; if( nAddSig ){ memmove(p->a+nAddSig, p->a, p->nDigit); memset(p->a, 0, nAddSig); @@ -4214,13 +6314,18 @@ static void decimalMul(Decimal *pA, Decimal *pB){ signed char *acc = 0; int i, j, k; int minFrac; + sqlite3_int64 sumDigit; if( pA==0 || pA->oom || pA->isNull || pB==0 || pB->oom || pB->isNull ){ goto mul_end; } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + sumDigit = pA->nDigit; + sumDigit += pB->nDigit; + sumDigit += 2; + if( sumDigit>SQLITE_DECIMAL_MAX_DIGIT ){ pA->oom = 1; return; } + acc = sqlite3_malloc64( sumDigit ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -4307,7 +6412,7 @@ static Decimal *decimalFromDouble(double r){ isNeg = 0; } memcpy(&a,&r,sizeof(a)); - if( a==0 ){ + if( a==0 || a==(sqlite3_int64)0x8000000000000000LL){ e = 0; m = 0; }else{ @@ -4357,10 +6462,16 @@ static void decimalFunc( sqlite3_value **argv ){ Decimal *p = decimal_new(context, argv[0], 0); - UNUSED_PARAMETER(argc); + int N; + if( argc==2 ){ + N = sqlite3_value_int(argv[1]); + if( N>0 ) decimal_round(p, N); + }else{ + N = 0; + } if( p ){ if( sqlite3_user_data(context)!=0 ){ - decimal_result_sci(context, p); + decimal_result_sci(context, p, N); }else{ decimal_result(context, p); } @@ -4446,7 +6557,7 @@ static void decimalSumStep( if( p==0 ) return; if( !p->isInit ){ p->isInit = 1; - p->a = sqlite3_malloc(2); + p->a = sqlite3_malloc64(2); if( p->a==0 ){ p->oom = 1; }else{ @@ -4530,7 +6641,7 @@ static void decimalPow2Func( UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); - decimal_result_sci(context, pA); + decimal_result_sci(context, pA, 0); decimal_free(pA); } } @@ -4551,7 +6662,9 @@ int sqlite3_decimal_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "decimal", 1, 0, decimalFunc }, + { "decimal", 2, 0, decimalFunc }, { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_exp", 2, 1, decimalFunc }, { "decimal_cmp", 2, 0, decimalCmpFunc }, { "decimal_add", 2, 0, decimalAddFunc }, { "decimal_sub", 2, 0, decimalSubFunc }, @@ -4581,516 +6694,8 @@ int sqlite3_decimal_init( return rc; } -/************************* End ../ext/misc/decimal.c ********************/ -/************************* Begin ../ext/misc/percentile.c ******************/ -/* -** 2013-05-28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains code to implement the percentile(Y,P) SQL function -** and similar as described below: -** -** (1) The percentile(Y,P) function is an aggregate function taking -** exactly two arguments. -** -** (2) If the P argument to percentile(Y,P) is not the same for every -** row in the aggregate then an error is thrown. The word "same" -** in the previous sentence means that the value differ by less -** than 0.001. -** -** (3) If the P argument to percentile(Y,P) evaluates to anything other -** than a number in the range of 0.0 to 100.0 inclusive then an -** error is thrown. -** -** (4) If any Y argument to percentile(Y,P) evaluates to a value that -** is not NULL and is not numeric then an error is thrown. -** -** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus -** infinity then an error is thrown. (SQLite always interprets NaN -** values as NULL.) -** -** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, -** including CASE WHEN expressions. -** -** (7) The percentile(Y,P) aggregate is able to handle inputs of at least -** one million (1,000,000) rows. -** -** (8) If there are no non-NULL values for Y, then percentile(Y,P) -** returns NULL. -** -** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) -** returns the one Y value. -** -** (10) If there N non-NULL values of Y where N is two or more and -** the Y values are ordered from least to greatest and a graph is -** drawn from 0 to N-1 such that the height of the graph at J is -** the J-th Y value and such that straight lines are drawn between -** adjacent Y values, then the percentile(Y,P) function returns -** the height of the graph at P*(N-1)/100. -** -** (11) The percentile(Y,P) function always returns either a floating -** point number or NULL. -** -** (12) The percentile(Y,P) is implemented as a single C99 source-code -** file that compiles into a shared-library or DLL that can be loaded -** into SQLite using the sqlite3_load_extension() interface. -** -** (13) A separate median(Y) function is the equivalent percentile(Y,50). -** -** (14) A separate percentile_cont(Y,P) function is equivalent to -** percentile(Y,P/100.0). In other words, the fraction value in -** the second argument is in the range of 0 to 1 instead of 0 to 100. -** -** (15) A separate percentile_disc(Y,P) function is like -** percentile_cont(Y,P) except that instead of returning the weighted -** average of the nearest two input values, it returns the next lower -** value. So the percentile_disc(Y,P) will always return a value -** that was one of the inputs. -** -** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and -** percentile_disc(Y,P) can be used as window functions. -** -** Differences from standard SQL: -** -** * The percentile_cont(X,P) function is equivalent to the following in -** standard SQL: -** -** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) -** -** The SQLite syntax is much more compact. The standard SQL syntax -** is also supported if SQLite is compiled with the -** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. -** -** * No median(X) function exists in the SQL standard. App developers -** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". -** -** * No percentile(Y,P) function exists in the SQL standard. Instead of -** percential(Y,P), developers must write this: -** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that -** the fraction parameter to percentile() goes from 0 to 100 whereas -** the fraction parameter in SQL standard percentile_cont() goes from -** 0 to 1. -** -** Implementation notes as of 2024-08-31: -** -** * The regular aggregate-function versions of these routines work -** by accumulating all values in an array of doubles, then sorting -** that array using quicksort before computing the answer. Thus -** the runtime is O(NlogN) where N is the number of rows of input. -** -** * For the window-function versions of these routines, the array of -** inputs is sorted as soon as the first value is computed. Thereafter, -** the array is kept in sorted order using an insert-sort. This -** results in O(N*K) performance where K is the size of the window. -** One can imagine alternative implementations that give O(N*logN*logK) -** performance, but they require more complex logic and data structures. -** The developers have elected to keep the asymptotically slower -** algorithm for now, for simplicity, under the theory that window -** functions are seldom used and when they are, the window size K is -** often small. The developers might revisit that decision later, -** should the need arise. -*/ -#if defined(SQLITE3_H) - /* no-op */ -#elif defined(SQLITE_STATIC_PERCENTILE) -/* # include "sqlite3.h" */ -#else -/* # include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#endif -#include <assert.h> -#include <string.h> -#include <stdlib.h> - -/* The following object is the group context for a single percentile() -** aggregate. Remember all input Y values until the very end. -** Those values are accumulated in the Percentile.a[] array. -*/ -typedef struct Percentile Percentile; -struct Percentile { - unsigned nAlloc; /* Number of slots allocated for a[] */ - unsigned nUsed; /* Number of slots actually used in a[] */ - char bSorted; /* True if a[] is already in sorted order */ - char bKeepSorted; /* True if advantageous to keep a[] sorted */ - char bPctValid; /* True if rPct is valid */ - double rPct; /* Fraction. 0.0 to 1.0 */ - double *a; /* Array of Y values */ -}; - -/* Details of each function in the percentile family */ -typedef struct PercentileFunc PercentileFunc; -struct PercentileFunc { - const char *zName; /* Function name */ - char nArg; /* Number of arguments */ - char mxFrac; /* Maximum value of the "fraction" input */ - char bDiscrete; /* True for percentile_disc() */ -}; -static const PercentileFunc aPercentFunc[] = { - { "median", 1, 1, 0 }, - { "percentile", 2, 100, 0 }, - { "percentile_cont", 2, 1, 0 }, - { "percentile_disc", 2, 1, 1 }, -}; - -/* -** Return TRUE if the input floating-point number is an infinity. -*/ -static int percentIsInfinity(double r){ - sqlite3_uint64 u; - assert( sizeof(u)==sizeof(r) ); - memcpy(&u, &r, sizeof(u)); - return ((u>>52)&0x7ff)==0x7ff; -} - -/* -** Return TRUE if two doubles differ by 0.001 or less. -*/ -static int percentSameValue(double a, double b){ - a -= b; - return a>=-0.001 && a<=0.001; -} - -/* -** Search p (which must have p->bSorted) looking for an entry with -** value y. Return the index of that entry. -** -** If bExact is true, return -1 if the entry is not found. -** -** If bExact is false, return the index at which a new entry with -** value y should be insert in order to keep the values in sorted -** order. The smallest return value in this case will be 0, and -** the largest return value will be p->nUsed. -*/ -static int percentBinarySearch(Percentile *p, double y, int bExact){ - int iFirst = 0; /* First element of search range */ - int iLast = p->nUsed - 1; /* Last element of search range */ - while( iLast>=iFirst ){ - int iMid = (iFirst+iLast)/2; - double x = p->a[iMid]; - if( x<y ){ - iFirst = iMid + 1; - }else if( x>y ){ - iLast = iMid - 1; - }else{ - return iMid; - } - } - if( bExact ) return -1; - return iFirst; -} - -/* -** Generate an error for a percentile function. -** -** The error format string must have exactly one occurrence of "%%s()" -** (with two '%' characters). That substring will be replaced by the name -** of the function. -*/ -static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - char *zMsg1; - char *zMsg2; - va_list ap; - - va_start(ap, zFormat); - zMsg1 = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0; - sqlite3_result_error(pCtx, zMsg2, -1); - sqlite3_free(zMsg1); - sqlite3_free(zMsg2); -} - -/* -** The "step" function for percentile(Y,P) is called once for each -** input row. -*/ -static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ - Percentile *p; - double rPct; - int eType; - double y; - assert( argc==2 || argc==1 ); - - if( argc==1 ){ - /* Requirement 13: median(Y) is the same as percentile(Y,50). */ - rPct = 0.5; - }else{ - /* Requirement 3: P must be a number between 0 and 100 */ - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - eType = sqlite3_value_numeric_type(argv[1]); - rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac; - if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) - || rPct<0.0 || rPct>1.0 - ){ - percentError(pCtx, "the fraction argument to %%s()" - " is not between 0.0 and %.1f", - (double)pFunc->mxFrac); - return; - } - } - - /* Allocate the session context. */ - p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); - if( p==0 ) return; - - /* Remember the P value. Throw an error if the P value is different - ** from any prior row, per Requirement (2). */ - if( !p->bPctValid ){ - p->rPct = rPct; - p->bPctValid = 1; - }else if( !percentSameValue(p->rPct,rPct) ){ - percentError(pCtx, "the fraction argument to %%s()" - " is not the same for all input rows"); - return; - } - - /* Ignore rows for which Y is NULL */ - eType = sqlite3_value_type(argv[0]); - if( eType==SQLITE_NULL ) return; - - /* If not NULL, then Y must be numeric. Otherwise throw an error. - ** Requirement 4 */ - if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ - percentError(pCtx, "input to %%s() is not numeric"); - return; - } - - /* Throw an error if the Y value is infinity or NaN */ - y = sqlite3_value_double(argv[0]); - if( percentIsInfinity(y) ){ - percentError(pCtx, "Inf input to %%s()"); - return; - } - - /* Allocate and store the Y */ - if( p->nUsed>=p->nAlloc ){ - unsigned n = p->nAlloc*2 + 250; - double *a = sqlite3_realloc64(p->a, sizeof(double)*n); - if( a==0 ){ - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); - sqlite3_result_error_nomem(pCtx); - return; - } - p->nAlloc = n; - p->a = a; - } - if( p->nUsed==0 ){ - p->a[p->nUsed++] = y; - p->bSorted = 1; - }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ - p->a[p->nUsed++] = y; - }else if( p->bKeepSorted ){ - int i; - i = percentBinarySearch(p, y, 0); - if( i<(int)p->nUsed ){ - memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); - } - p->a[i] = y; - p->nUsed++; - }else{ - p->a[p->nUsed++] = y; - p->bSorted = 0; - } -} - -/* -** Interchange two doubles. -*/ -#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} - -/* -** Sort an array of doubles. -** -** Algorithm: quicksort -** -** This is implemented separately rather than using the qsort() routine -** from the standard library because: -** -** (1) To avoid a dependency on qsort() -** (2) To avoid the function call to the comparison routine for each -** comparison. -*/ -static void percentSort(double *a, unsigned int n){ - int iLt; /* Entries before a[iLt] are less than rPivot */ - int iGt; /* Entries at or after a[iGt] are greater than rPivot */ - int i; /* Loop counter */ - double rPivot; /* The pivot value */ - - assert( n>=2 ); - if( a[0]>a[n-1] ){ - SWAP_DOUBLE(a[0],a[n-1]) - } - if( n==2 ) return; - iGt = n-1; - i = n/2; - if( a[0]>a[i] ){ - SWAP_DOUBLE(a[0],a[i]) - }else if( a[i]>a[iGt] ){ - SWAP_DOUBLE(a[i],a[iGt]) - } - if( n==3 ) return; - rPivot = a[i]; - iLt = i = 1; - do{ - if( a[i]<rPivot ){ - if( i>iLt ) SWAP_DOUBLE(a[i],a[iLt]) - iLt++; - i++; - }else if( a[i]>rPivot ){ - do{ - iGt--; - }while( iGt>i && a[iGt]>rPivot ); - SWAP_DOUBLE(a[i],a[iGt]) - }else{ - i++; - } - }while( i<iGt ); - if( iLt>=2 ) percentSort(a, iLt); - if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); - -/* Uncomment for testing */ -#if 0 - for(i=0; i<n-1; i++){ - assert( a[i]<=a[i+1] ); - } -#endif -} - - -/* -** The "inverse" function for percentile(Y,P) is called to remove a -** row that was previously inserted by "step". -*/ -static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){ - Percentile *p; - int eType; - double y; - int i; - assert( argc==2 || argc==1 ); - - /* Allocate the session context. */ - p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); - assert( p!=0 ); - - /* Ignore rows for which Y is NULL */ - eType = sqlite3_value_type(argv[0]); - if( eType==SQLITE_NULL ) return; - - /* If not NULL, then Y must be numeric. Otherwise throw an error. - ** Requirement 4 */ - if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ - return; - } - - /* Ignore the Y value if it is infinity or NaN */ - y = sqlite3_value_double(argv[0]); - if( percentIsInfinity(y) ){ - return; - } - if( p->bSorted==0 ){ - assert( p->nUsed>1 ); - percentSort(p->a, p->nUsed); - p->bSorted = 1; - } - p->bKeepSorted = 1; - - /* Find and remove the row */ - i = percentBinarySearch(p, y, 1); - if( i>=0 ){ - p->nUsed--; - if( i<(int)p->nUsed ){ - memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); - } - } -} - -/* -** Compute the final output of percentile(). Clean up all allocated -** memory if and only if bIsFinal is true. -*/ -static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ - Percentile *p; - PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); - unsigned i1, i2; - double v1, v2; - double ix, vx; - p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); - if( p==0 ) return; - if( p->a==0 ) return; - if( p->nUsed ){ - if( p->bSorted==0 ){ - assert( p->nUsed>1 ); - percentSort(p->a, p->nUsed); - p->bSorted = 1; - } - ix = p->rPct*(p->nUsed-1); - i1 = (unsigned)ix; - if( pFunc->bDiscrete ){ - vx = p->a[i1]; - }else{ - i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; - v1 = p->a[i1]; - v2 = p->a[i2]; - vx = v1 + (v2-v1)*(ix-i1); - } - sqlite3_result_double(pCtx, vx); - } - if( bIsFinal ){ - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); - }else{ - p->bKeepSorted = 1; - } -} -static void percentFinal(sqlite3_context *pCtx){ - percentCompute(pCtx, 1); -} -static void percentValue(sqlite3_context *pCtx){ - percentCompute(pCtx, 0); -} - -#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE) - -#endif -int sqlite3_percentile_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - unsigned int i; -#ifdef SQLITE3EXT_H - SQLITE_EXTENSION_INIT2(pApi); -#else - (void)pApi; /* Unused parameter */ -#endif - (void)pzErrMsg; /* Unused parameter */ - for(i=0; i<sizeof(aPercentFunc)/sizeof(aPercentFunc[0]); i++){ - rc = sqlite3_create_window_function(db, - aPercentFunc[i].zName, - aPercentFunc[i].nArg, - SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_SELFORDER1, - (void*)&aPercentFunc[i], - percentStep, percentFinal, percentValue, percentInverse, 0); - if( rc ) break; - } - return rc; -} - -/************************* End ../ext/misc/percentile.c ********************/ -#undef sqlite3_base_init -#define sqlite3_base_init sqlite3_base64_init -/************************* Begin ../ext/misc/base64.c ******************/ +/************************* End ext/misc/decimal.c ********************/ +/************************* Begin ext/misc/base64.c ******************/ /* ** 2022-11-18 ** @@ -5300,7 +6905,9 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ /* This function does the work for the SQLite base64(x) UDF. */ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ - int nb, nc, nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nb; + sqlite3_int64 nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nc; int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), SQLITE_LIMIT_LENGTH, -1); char *cBuf; @@ -5309,7 +6916,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ switch( sqlite3_value_type(av[0]) ){ case SQLITE_BLOB: nb = nv; - nc = 4*(nv+2/3); /* quads needed */ + nc = 4*((nv+2)/3); /* quads needed */ nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big", -1); @@ -5323,7 +6930,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_text(context,"",-1,SQLITE_STATIC); break; } - cBuf = sqlite3_malloc(nc); + cBuf = sqlite3_malloc64(nc); if( !cBuf ) goto memFail; nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); @@ -5345,7 +6952,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_zeroblob(context, 0); break; } - bBuf = sqlite3_malloc(nb); + bBuf = sqlite3_malloc64(nb); if( !bBuf ) goto memFail; nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); @@ -5366,7 +6973,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 #endif -int sqlite3_base_init +int sqlite3_base64_init #else static int sqlite3_base64_init #endif @@ -5387,11 +6994,8 @@ static int sqlite3_base64_init #define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) #define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ -/************************* End ../ext/misc/base64.c ********************/ -#undef sqlite3_base_init -#define sqlite3_base_init sqlite3_base85_init -#define OMIT_BASE85_CHECKER -/************************* Begin ../ext/misc/base85.c ******************/ +/************************* End ext/misc/base64.c ********************/ +/************************* Begin ext/misc/base85.c ******************/ /* ** 2022-11-16 ** @@ -5656,7 +7260,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER /* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); @@ -5676,11 +7280,11 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ return; } } -# endif +#endif /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ - int nb, nc, nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), SQLITE_LIMIT_LENGTH, -1); char *cBuf; @@ -5703,7 +7307,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_text(context,"",-1,SQLITE_STATIC); break; } - cBuf = sqlite3_malloc(nc); + cBuf = sqlite3_malloc64(nc); if( !cBuf ) goto memFail; nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); @@ -5725,7 +7329,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_zeroblob(context, 0); break; } - bBuf = sqlite3_malloc(nb); + bBuf = sqlite3_malloc64(nb); if( !bBuf ) goto memFail; nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); @@ -5746,14 +7350,14 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 #endif -int sqlite3_base_init +int sqlite3_base85_init #else static int sqlite3_base85_init #endif (sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER { int rc = sqlite3_create_function (db, "is_base85", 1, @@ -5761,7 +7365,7 @@ static int sqlite3_base85_init 0, is_base85, 0, 0); if( rc!=SQLITE_OK ) return rc; } -# endif +#endif return sqlite3_create_function (db, "base85", 1, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, @@ -5826,9 +7430,9 @@ int main(int na, char *av[]){ int nc = strlen(cBuf); size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER b85Clean &= allBase85( cBuf, nc ); -# endif +#endif } break; default: @@ -5847,8 +7451,8 @@ int main(int na, char *av[]){ #endif -/************************* End ../ext/misc/base85.c ********************/ -/************************* Begin ../ext/misc/ieee754.c ******************/ +/************************* End ext/misc/base85.c ********************/ +/************************* Begin ext/misc/ieee754.c ******************/ /* ** 2013-04-17 ** @@ -5985,6 +7589,9 @@ static void ieee754func( if( a==0 ){ e = 0; m = 0; + }else if( a==(sqlite3_int64)0x8000000000000000LL ){ + e = -1996; + m = -1; }else{ e = a>>52; m = a & ((((sqlite3_int64)1)<<52)-1); @@ -6027,9 +7634,9 @@ static void ieee754func( } if( m<0 ){ + if( m<(-9223372036854775807LL) ) return; isNeg = 1; m = -m; - if( m<0 ) return; }else if( m==0 && e>-1000 && e<1000 ){ sqlite3_result_double(context, 0.0); return; @@ -6108,6 +7715,38 @@ static void ieee754func_to_blob( } /* +** Functions to convert between 64-bit integers and floats. +** +** The bit patterns are copied. The numeric values are different. +*/ +static void ieee754func_from_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + double r; + sqlite3_int64 v = sqlite3_value_int64(argv[0]); + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(context, r); + } +} +static void ieee754func_to_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[0]); + sqlite3_uint64 v; + memcpy(&v, &r, sizeof(v)); + sqlite3_result_int64(context, v); + } +} + +/* ** SQL Function: ieee754_inc(r,N) ** ** Move the floating point value r by N quantums and return the new @@ -6159,6 +7798,8 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_to_int", 1, 0, ieee754func_to_int }, + { "ieee754_from_int", 1, 0, ieee754func_from_int }, { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; @@ -6174,8 +7815,8 @@ int sqlite3_ieee_init( return rc; } -/************************* End ../ext/misc/ieee754.c ********************/ -/************************* Begin ../ext/misc/series.c ******************/ +/************************* End ext/misc/ieee754.c ********************/ +/************************* Begin ext/misc/series.c ******************/ /* ** 2015-08-18, 2023-04-28 ** @@ -6208,19 +7849,20 @@ int sqlite3_ieee_init( ** SELECT * FROM generate_series(0,100,5); ** ** The query above returns integers from 0 through 100 counting by steps -** of 5. +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. ** ** SELECT * FROM generate_series(0,100); ** -** Integers from 0 through 100 with a step size of 1. +** Integers from 0 through 100 with a step size of 1. 101 rows. ** ** SELECT * FROM generate_series(20) LIMIT 10; ** -** Integers 20 through 29. +** Integers 20 through 29. 10 rows. ** ** SELECT * FROM generate_series(0,-100,-5); ** -** Integers 0 -5 -10 ... -100. +** Integers 0 -5 -10 ... -100. 21 rows. ** ** SELECT * FROM generate_series(0,-1); ** @@ -6296,139 +7938,88 @@ SQLITE_EXTENSION_INIT1 #include <math.h> #ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Return that member of a generate_series(...) sequence whose 0-based -** index is ix. The 0th member is given by smBase. The sequence members -** progress per ix increment by smStep. -*/ -static sqlite3_int64 genSeqMember( - sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix -){ - static const sqlite3_uint64 mxI64 = - ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; - if( ix>=mxI64 ){ - /* Get ix into signed i64 range. */ - ix -= mxI64; - /* With 2's complement ALU, this next can be 1 step, but is split into - * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (mxI64/2) * smStep; - smBase += (mxI64 - mxI64/2) * smStep; - } - /* Under UBSAN (or on 1's complement machines), must do this last term - * in steps to avoid the dreaded (and harmless) signed multiply overflow. */ - if( ix>=2 ){ - sqlite3_int64 ix2 = (sqlite3_int64)ix/2; - smBase += ix2*smStep; - ix -= ix2; - } - return smBase + ((sqlite3_int64)ix)*smStep; -} +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 +*/ /* typedef unsigned char u8; */ - -typedef struct SequenceSpec { - sqlite3_int64 iOBase; /* Original starting value ("start") */ - sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ - sqlite3_int64 iBase; /* Starting value to actually use */ - sqlite3_int64 iTerm; /* Terminal value to actually use */ - sqlite3_int64 iStep; /* Increment ("step") */ - sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ - sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ - sqlite3_int64 iValueNow; /* Current value during generation */ - u8 isNotEOF; /* Sequence generation not exhausted */ - u8 isReversing; /* Sequence is being reverse generated */ -} SequenceSpec; +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ +}; /* -** Prepare a SequenceSpec for use in generating an integer series -** given initialized iBase, iTerm and iStep values. Sequence is -** initialized per given isReversing. Other members are computed. +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. */ -static void setupSequence( SequenceSpec *pss ){ - int bSameSigns; - pss->uSeqIndexMax = 0; - pss->isNotEOF = 0; - bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); - if( pss->iTerm < pss->iBase ){ - sqlite3_uint64 nuspan = 0; - if( bSameSigns ){ - nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iBase>=0 and iTerm<0 . */ - nuspan = 1; - nuspan += pss->iBase; - nuspan += -(pss->iTerm+1); - } - if( pss->iStep<0 ){ - pss->isNotEOF = 1; - if( nuspan==ULONG_MAX ){ - pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; - }else if( pss->iStep>LLONG_MIN ){ - pss->uSeqIndexMax = nuspan/-pss->iStep; - } - } - }else if( pss->iTerm > pss->iBase ){ - sqlite3_uint64 puspan = 0; - if( bSameSigns ){ - puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iTerm>=0 and iBase<0 . */ - puspan = 1; - puspan += pss->iTerm; - puspan += -(pss->iBase+1); - } - if( pss->iStep>0 ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = puspan/pss->iStep; - } - }else if( pss->iTerm == pss->iBase ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = 0; - } - pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; - pss->iValueNow = (pss->isReversing) - ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) - : pss->iBase; -} +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} /* -** Progress sequence generator to yield next value, if any. -** Leave its state to either yield next value or be at EOF. -** Return whether there is a next value, or 0 at EOF. +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. */ -static int progressSequence( SequenceSpec *pss ){ - if( !pss->isNotEOF ) return 0; - if( pss->isReversing ){ - if( pss->uSeqIndexNow > 0 ){ - pss->uSeqIndexNow--; - pss->iValueNow -= pss->iStep; - }else{ - pss->isNotEOF = 0; - } - }else{ - if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ - pss->uSeqIndexNow++; - pss->iValueNow += pss->iStep; - }else{ - pss->isNotEOF = 0; - } - } - return pss->isNotEOF; +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; } - -/* series_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -typedef struct series_cursor series_cursor; -struct series_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - SequenceSpec ss; /* (this) Derived class data */ -}; /* ** The seriesConnect() method is invoked to create a new @@ -6467,7 +8058,7 @@ static int seriesConnect( rc = sqlite3_declare_vtab(db, "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = *ppVtab = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); @@ -6489,7 +8080,7 @@ static int seriesDisconnect(sqlite3_vtab *pVtab){ static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ series_cursor *pCur; (void)pUnused; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; @@ -6510,7 +8101,15 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - progressSequence( & pCur->ss ); + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } return SQLITE_OK; } @@ -6526,19 +8125,19 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; - default: x = pCur->ss.iValueNow; break; + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; } #ifndef LARGEST_UINT64 -#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) -#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) #endif /* @@ -6546,7 +8145,7 @@ static int seriesColumn( */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->ss.iValueNow; + *pRowid = pCur->iValue; return SQLITE_OK; } @@ -6556,7 +8155,7 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - return !pCur->ss.isNotEOF; + return pCur->bDone; } /* True to cause run-time checking of the start=, stop=, and/or step= @@ -6568,6 +8167,59 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ #endif /* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + +/* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least ** once prior to any call to seriesColumn() or seriesRowid() or @@ -6600,33 +8252,42 @@ static int seriesFilter( int argc, sqlite3_value **argv ){ series_cursor *pCur = (series_cursor *)pVtabCursor; - int i = 0; - int returnNoRows = 0; - sqlite3_int64 iMin = SMALLEST_INT64; - sqlite3_int64 iMax = LARGEST_INT64; - sqlite3_int64 iLimit = 0; - sqlite3_int64 iOffset = 0; + int iArg = 0; /* Arguments used so far */ + int i; /* Loop counter */ + sqlite3_int64 iMin = SMALLEST_INT64; /* Smallest allowed output value */ + sqlite3_int64 iMax = LARGEST_INT64; /* Largest allowed output value */ + sqlite3_int64 iLimit = 0; /* if >0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ (void)idxStrUnused; + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i<argc; i++){ + if( sqlite3_value_type(argv[i])==SQLITE_NULL ){ + goto series_no_rows; + } + } + + /* Capture the three HIDDEN parameters to the virtual table and insert + ** default values for any parameters that are omitted. + */ if( idxNum & 0x01 ){ - pCur->ss.iBase = sqlite3_value_int64(argv[i++]); + pCur->iOBase = sqlite3_value_int64(argv[iArg++]); }else{ - pCur->ss.iBase = 0; + pCur->iOBase = 0; } if( idxNum & 0x02 ){ - pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); + pCur->iOTerm = sqlite3_value_int64(argv[iArg++]); }else{ - pCur->ss.iTerm = 0xffffffff; + pCur->iOTerm = 0xffffffff; } if( idxNum & 0x04 ){ - pCur->ss.iStep = sqlite3_value_int64(argv[i++]); - if( pCur->ss.iStep==0 ){ - pCur->ss.iStep = 1; - }else if( pCur->ss.iStep<0 ){ - if( (idxNum & 0x10)==0 ) idxNum |= 0x08; - } + pCur->iOStep = sqlite3_value_int64(argv[iArg++]); + if( pCur->iOStep==0 ) pCur->iOStep = 1; }else{ - pCur->ss.iStep = 1; + pCur->iOStep = 1; } /* If there are constraints on the value column but there are @@ -6636,72 +8297,94 @@ static int seriesFilter( ** further below. */ if( (idxNum & 0x05)==0 && (idxNum & 0x0380)!=0 ){ - pCur->ss.iBase = SMALLEST_INT64; + pCur->iOBase = SMALLEST_INT64; } if( (idxNum & 0x06)==0 && (idxNum & 0x3080)!=0 ){ - pCur->ss.iTerm = LARGEST_INT64; + pCur->iOTerm = LARGEST_INT64; + } + pCur->iBase = pCur->iOBase; + pCur->iTerm = pCur->iOTerm; + if( pCur->iOStep>0 ){ + pCur->iStep = pCur->iOStep; + }else if( pCur->iOStep>SMALLEST_INT64 ){ + pCur->iStep = -pCur->iOStep; + }else{ + pCur->iStep = LARGEST_INT64; + pCur->iStep++; + } + pCur->bDesc = pCur->iOStep<0; + if( pCur->bDesc==0 && pCur->iBase>pCur->iTerm ){ + goto series_no_rows; + } + if( pCur->bDesc!=0 && pCur->iBase<pCur->iTerm ){ + goto series_no_rows; } - pCur->ss.iOBase = pCur->ss.iBase; - pCur->ss.iOTerm = pCur->ss.iTerm; /* Extract the LIMIT and OFFSET values, but do not apply them yet. ** The range must first be constrained by the limits on value. */ if( idxNum & 0x20 ){ - iLimit = sqlite3_value_int64(argv[i++]); + iLimit = sqlite3_value_int64(argv[iArg++]); if( idxNum & 0x40 ){ - iOffset = sqlite3_value_int64(argv[i++]); + iOffset = sqlite3_value_int64(argv[iArg++]); } } + /* Narrow the range of iMin and iMax (the minimum and maximum outputs) + ** based on equality and inequality constraints on the "value" column. + */ if( idxNum & 0x3380 ){ - /* Extract the maximum range of output values determined by - ** constraints on the "value" column. - */ - if( idxNum & 0x0080 ){ - if( sqlite3_value_numeric_type(argv[i])==SQLITE_FLOAT ){ - double r = sqlite3_value_double(argv[i++]); - if( r==ceil(r) ){ + if( idxNum & 0x0080 ){ /* value=X */ + if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[iArg++]); + if( r==seriesCeil(r) + && r>=(double)SMALLEST_INT64 + && r<=(double)LARGEST_INT64 + ){ iMin = iMax = (sqlite3_int64)r; }else{ - returnNoRows = 1; + goto series_no_rows; } }else{ - iMin = iMax = sqlite3_value_int64(argv[i++]); + iMin = iMax = sqlite3_value_int64(argv[iArg++]); } }else{ - if( idxNum & 0x0300 ){ - if( sqlite3_value_numeric_type(argv[i])==SQLITE_FLOAT ){ - double r = sqlite3_value_double(argv[i++]); - if( idxNum & 0x0200 && r==ceil(r) ){ - iMin = (sqlite3_int64)ceil(r+1.0); + if( idxNum & 0x0300 ){ /* value>X or value>=X */ + if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[iArg++]); + if( r<(double)SMALLEST_INT64 ){ + iMin = SMALLEST_INT64; + }else if( (idxNum & 0x0200)!=0 && r==seriesCeil(r) ){ + iMin = (sqlite3_int64)seriesCeil(r+1.0); }else{ - iMin = (sqlite3_int64)ceil(r); + iMin = (sqlite3_int64)seriesCeil(r); } }else{ - iMin = sqlite3_value_int64(argv[i++]); - if( idxNum & 0x0200 ){ + iMin = sqlite3_value_int64(argv[iArg++]); + if( (idxNum & 0x0200)!=0 ){ if( iMin==LARGEST_INT64 ){ - returnNoRows = 1; + goto series_no_rows; }else{ iMin++; } } } } - if( idxNum & 0x3000 ){ - if( sqlite3_value_numeric_type(argv[i])==SQLITE_FLOAT ){ - double r = sqlite3_value_double(argv[i++]); - if( (idxNum & 0x2000)!=0 && r==floor(r) ){ + if( idxNum & 0x3000 ){ /* value<X or value<=X */ + if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[iArg++]); + if( r>(double)LARGEST_INT64 ){ + iMax = LARGEST_INT64; + }else if( (idxNum & 0x2000)!=0 && r==seriesFloor(r) ){ iMax = (sqlite3_int64)(r-1.0); }else{ - iMax = (sqlite3_int64)floor(r); + iMax = (sqlite3_int64)seriesFloor(r); } }else{ - iMax = sqlite3_value_int64(argv[i++]); + iMax = sqlite3_value_int64(argv[iArg++]); if( idxNum & 0x2000 ){ if( iMax==SMALLEST_INT64 ){ - returnNoRows = 1; + goto series_no_rows; }else{ iMax--; } @@ -6709,72 +8392,99 @@ static int seriesFilter( } } if( iMin>iMax ){ - returnNoRows = 1; + goto series_no_rows; } } /* Try to reduce the range of values to be generated based on ** constraints on the "value" column. */ - if( pCur->ss.iStep>0 ){ - sqlite3_int64 szStep = pCur->ss.iStep; - if( pCur->ss.iBase<iMin ){ - sqlite3_uint64 d = iMin - pCur->ss.iBase; - pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep; + if( pCur->bDesc==0 ){ + if( pCur->iBase<iMin ){ + sqlite3_uint64 span = span64(iMin,pCur->iBase); + pCur->iBase = add64(pCur->iBase, (span/pCur->iStep)*pCur->iStep); + if( pCur->iBase<iMin ){ + if( pCur->iBase > sub64(LARGEST_INT64, pCur->iStep) ){ + goto series_no_rows; + } + pCur->iBase = add64(pCur->iBase, pCur->iStep); + } } - if( pCur->ss.iTerm>iMax ){ - pCur->ss.iTerm = iMax; + if( pCur->iTerm>iMax ){ + pCur->iTerm = iMax; } }else{ - sqlite3_int64 szStep = -pCur->ss.iStep; - assert( szStep>0 ); - if( pCur->ss.iBase>iMax ){ - sqlite3_uint64 d = pCur->ss.iBase - iMax; - pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep; + if( pCur->iBase>iMax ){ + sqlite3_uint64 span = span64(pCur->iBase,iMax); + pCur->iBase = sub64(pCur->iBase, (span/pCur->iStep)*pCur->iStep); + if( pCur->iBase>iMax ){ + if( pCur->iBase < add64(SMALLEST_INT64, pCur->iStep) ){ + goto series_no_rows; + } + pCur->iBase = sub64(pCur->iBase, pCur->iStep); + } } - if( pCur->ss.iTerm<iMin ){ - pCur->ss.iTerm = iMin; + if( pCur->iTerm<iMin ){ + pCur->iTerm = iMin; } } } + /* Adjust iTerm so that it is exactly the last value of the series. + */ + if( pCur->bDesc==0 ){ + if( pCur->iBase>pCur->iTerm ){ + goto series_no_rows; + } + pCur->iTerm = sub64(pCur->iTerm, + span64(pCur->iTerm,pCur->iBase) % pCur->iStep); + }else{ + if( pCur->iBase<pCur->iTerm ){ + goto series_no_rows; + } + pCur->iTerm = add64(pCur->iTerm, + span64(pCur->iBase,pCur->iTerm) % pCur->iStep); + } + + /* Transform the series generator to output values in the requested + ** order. + */ + if( ((idxNum & 0x0008)!=0 && pCur->bDesc==0) + || ((idxNum & 0x0010)!=0 && pCur->bDesc!=0) + ){ + sqlite3_int64 tmp = pCur->iBase; + pCur->iBase = pCur->iTerm; + pCur->iTerm = tmp; + pCur->bDesc = !pCur->bDesc; + } + /* Apply LIMIT and OFFSET constraints, if any */ + assert( pCur->iStep!=0 ); if( idxNum & 0x20 ){ if( iOffset>0 ){ - pCur->ss.iBase += pCur->ss.iStep*iOffset; - } - if( iLimit>=0 ){ - sqlite3_int64 iTerm; - iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; - if( pCur->ss.iStep<0 ){ - if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + if( seriesSteps(pCur) < (sqlite3_uint64)iOffset ){ + goto series_no_rows; + }else if( pCur->bDesc ){ + pCur->iBase = sub64(pCur->iBase, pCur->iStep*iOffset); }else{ - if( iTerm<pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + pCur->iBase = add64(pCur->iBase, pCur->iStep*iOffset); } } - } - - - for(i=0; i<argc; i++){ - if( sqlite3_value_type(argv[i])==SQLITE_NULL ){ - /* If any of the constraints have a NULL value, then return no rows. - ** See ticket https://sqlite.org/src/info/fac496b61722daf2 */ - returnNoRows = 1; - break; + if( iLimit>=0 && seriesSteps(pCur) > (sqlite3_uint64)iLimit ){ + pCur->iTerm = add64(pCur->iBase, (iLimit - 1)*pCur->iStep); } } - if( returnNoRows ){ - pCur->ss.iBase = 1; - pCur->ss.iTerm = 0; - pCur->ss.iStep = 1; - } - if( idxNum & 0x08 ){ - pCur->ss.isReversing = pCur->ss.iStep > 0; - }else{ - pCur->ss.isReversing = pCur->ss.iStep < 0; - } - setupSequence( &pCur->ss ); + pCur->iValue = pCur->iBase; + pCur->bDone = 0; return SQLITE_OK; + +series_no_rows: + pCur->iBase = 0; + pCur->iTerm = 0; + pCur->iStep = 1; + pCur->bDesc = 0; + pCur->bDone = 1; + return SQLITE_OK; } /* @@ -7045,8 +8755,8 @@ int sqlite3_series_init( return rc; } -/************************* End ../ext/misc/series.c ********************/ -/************************* Begin ../ext/misc/regexp.c ******************/ +/************************* End ext/misc/series.c ********************/ +/************************* Begin ext/misc/regexp.c ******************/ /* ** 2012-11-13 ** @@ -7081,7 +8791,7 @@ int sqlite3_series_init( ** ^X X occurring at the beginning of the string ** X$ X occurring at the end of the string ** . Match any single character -** \c Character c where c is one of \{}()[]|*+?. +** \c Character c where c is one of \{}()[]|*+?-. ** \c C-language escapes for c in afnrtv. ex: \t or \n ** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX ** \xXX Where XX is exactly 2 hex digits, unicode value XX @@ -7104,6 +8814,8 @@ int sqlite3_series_init( ** to p copies of X following by q-p copies of X? and that the size of the ** regular expression in the O(N*M) performance bound is computed after ** this expansion. +** +** To help prevent DoS attacks, the maximum size of the NFA is restricted. */ #include <string.h> #include <stdlib.h> @@ -7145,32 +8857,6 @@ SQLITE_EXTENSION_INIT1 #define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ #define RE_OP_ATSTART 18 /* Currently at the start of the string */ -#if defined(SQLITE_DEBUG) -/* Opcode names used for symbolic debugging */ -static const char *ReOpName[] = { - "EOF", - "MATCH", - "ANY", - "ANYSTAR", - "FORK", - "GOTO", - "ACCEPT", - "CC_INC", - "CC_EXC", - "CC_VALUE", - "CC_RANGE", - "WORD", - "NOTWORD", - "DIGIT", - "NOTDIGIT", - "SPACE", - "NOTSPACE", - "BOUNDARY", - "ATSTART", -}; -#endif /* SQLITE_DEBUG */ - - /* Each opcode is a "state" in the NFA */ typedef unsigned short ReStateNumber; @@ -7207,6 +8893,7 @@ struct ReCompiled { int nInit; /* Number of bytes in zInit */ unsigned nState; /* Number of entries in aOp[] and aArg[] */ unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ + unsigned mxAlloc; /* Complexity limit */ }; /* Add a state to the given state set if it is not already there */ @@ -7421,14 +9108,15 @@ re_match_end: /* Resize the opcode and argument arrays for an RE under construction. */ -static int re_resize(ReCompiled *p, int N){ +static int re_resize(ReCompiled *p, unsigned int N){ char *aOp; int *aArg; + if( N>p->mxAlloc ){ p->zErr = "REGEXP pattern too big"; return 1; } aOp = sqlite3_realloc64(p->aOp, N*sizeof(p->aOp[0])); - if( aOp==0 ) return 1; + if( aOp==0 ){ p->zErr = "out of memory"; return 1; } p->aOp = aOp; aArg = sqlite3_realloc64(p->aArg, N*sizeof(p->aArg[0])); - if( aArg==0 ) return 1; + if( aArg==0 ){ p->zErr = "out of memory"; return 1; } p->aArg = aArg; p->nAlloc = N; return 0; @@ -7459,7 +9147,7 @@ static int re_append(ReCompiled *p, int op, int arg){ /* Make a copy of N opcodes starting at iStart onto the end of the RE ** under construction. */ -static void re_copy(ReCompiled *p, int iStart, int N){ +static void re_copy(ReCompiled *p, int iStart, unsigned int N){ if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); @@ -7488,7 +9176,7 @@ static int re_hex(int c, int *pV){ ** return its interpretation. */ static unsigned re_esc_char(ReCompiled *p){ - static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; + static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]-"; static const char zTrans[] = "\a\f\n\r\t\v"; int i, v = 0; char c; @@ -7612,18 +9300,26 @@ static const char *re_subcompile_string(ReCompiled *p){ break; } case '{': { - int m = 0, n = 0; - int sz, j; + unsigned int m = 0, n = 0; + unsigned int sz, j; if( iPrev<0 ) return "'{m,n}' without operand"; - while( (c=rePeek(p))>='0' && c<='9' ){ m = m*10 + c - '0'; p->sIn.i++; } + while( (c=rePeek(p))>='0' && c<='9' ){ + m = m*10 + c - '0'; + if( m*2>p->mxAlloc ) return "REGEXP pattern too big"; + p->sIn.i++; + } n = m; if( c==',' ){ p->sIn.i++; n = 0; - while( (c=rePeek(p))>='0' && c<='9' ){ n = n*10 + c-'0'; p->sIn.i++; } + while( (c=rePeek(p))>='0' && c<='9' ){ + n = n*10 + c-'0'; + if( n*2>p->mxAlloc ) return "REGEXP pattern too big"; + p->sIn.i++; + } } if( c!='}' ) return "unmatched '{'"; - if( n>0 && n<m ) return "n less than m in '{m,n}'"; + if( n<m ) return "n less than m in '{m,n}'"; p->sIn.i++; sz = p->nState - iPrev; if( m==0 ){ @@ -7639,7 +9335,7 @@ static const char *re_subcompile_string(ReCompiled *p){ re_copy(p, iPrev, sz); } if( n==0 && m>0 ){ - re_append(p, RE_OP_FORK, -sz); + re_append(p, RE_OP_FORK, -(int)sz); } break; } @@ -7705,8 +9401,7 @@ static const char *re_subcompile_string(ReCompiled *p){ ** regular expression. Applications should invoke this routine once ** for every call to re_compile() to avoid memory leaks. */ -static void re_free(void *p){ - ReCompiled *pRe = (ReCompiled*)p; +static void re_free(ReCompiled *pRe){ if( pRe ){ sqlite3_free(pRe->aOp); sqlite3_free(pRe->aArg); @@ -7715,26 +9410,42 @@ static void re_free(void *p){ } /* +** Version of re_free() that accepts a pointer of type (void*). Required +** to satisfy sanitizers when the re_free() function is called via a +** function pointer. +*/ +static void re_free_voidptr(void *p){ + re_free((ReCompiled*)p); +} + +/* ** Compile a textual regular expression in zIn[] into a compiled regular ** expression suitable for us by re_match() and return a pointer to the ** compiled regular expression in *ppRe. Return NULL on success or an ** error message if something goes wrong. */ -static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ +static const char *re_compile( + ReCompiled **ppRe, /* OUT: write compiled NFA here */ + const char *zIn, /* Input regular expression */ + int mxRe, /* Complexity limit */ + int noCase /* True for caseless comparisons */ +){ ReCompiled *pRe; const char *zErr; int i, j; *ppRe = 0; - pRe = sqlite3_malloc( sizeof(*pRe) ); + pRe = sqlite3_malloc64( sizeof(*pRe) ); if( pRe==0 ){ return "out of memory"; } memset(pRe, 0, sizeof(*pRe)); pRe->xNextChar = noCase ? re_next_char_nocase : re_next_char; + pRe->mxAlloc = mxRe; if( re_resize(pRe, 30) ){ + zErr = pRe->zErr; re_free(pRe); - return "out of memory"; + return zErr; } if( zIn[0]=='^' ){ zIn++; @@ -7788,6 +9499,21 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ } /* +** The value of LIMIT_MAX_PATTERN_LENGTH. +*/ +static int re_maxlen(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + return sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1); +} + +/* +** Maximum NFA size given a maximum pattern length. +*/ +static int re_maxnfa(int mxlen){ + return 75+mxlen/2; +} + +/* ** Implementation of the regexp() SQL function. This function implements ** the build-in REGEXP operator. The first argument to the function is the ** pattern and the second argument is the string. So, the SQL statements: @@ -7810,9 +9536,17 @@ static void re_sql_func( (void)argc; /* Unused */ pRe = sqlite3_get_auxdata(context, 0); if( pRe==0 ){ + int mxLen = re_maxlen(context); + int nPattern; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + nPattern = sqlite3_value_bytes(argv[0]); + if( nPattern>mxLen ){ + zErr = "REGEXP pattern too big"; + }else{ + zErr = re_compile(&pRe, zPattern, re_maxnfa(mxLen), + sqlite3_user_data(context)!=0); + } if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -7829,7 +9563,7 @@ static void re_sql_func( sqlite3_result_int(context, re_match(pRe, zStr, -1)); } if( setAux ){ - sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + sqlite3_set_auxdata(context, 0, pRe, re_free_voidptr); } } @@ -7853,11 +9587,33 @@ static void re_bytecode_func( int i; int n; char *z; - (void)argc; + static const char *ReOpName[] = { + "EOF", + "MATCH", + "ANY", + "ANYSTAR", + "FORK", + "GOTO", + "ACCEPT", + "CC_INC", + "CC_EXC", + "CC_VALUE", + "CC_RANGE", + "WORD", + "NOTWORD", + "DIGIT", + "NOTDIGIT", + "SPACE", + "NOTSPACE", + "BOUNDARY", + "ATSTART", + }; + (void)argc; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + zErr = re_compile(&pRe, zPattern, re_maxnfa(re_maxlen(context)), + sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -7930,9 +9686,9 @@ int sqlite3_regexp_init( return rc; } -/************************* End ../ext/misc/regexp.c ********************/ +/************************* End ext/misc/regexp.c ********************/ #ifndef SQLITE_SHELL_FIDDLE -/************************* Begin ../ext/misc/fileio.c ******************/ +/************************* Begin ext/misc/fileio.c ******************/ /* ** 2014-06-13 ** @@ -8002,16 +9758,12 @@ int sqlite3_regexp_init( ** data: For a regular file, a blob containing the file data. For a ** symlink, a text value containing the text of the link. For a ** directory, NULL. +** level: Directory hierarchy level. Topmost is 1. ** ** If a non-NULL value is specified for the optional $dir parameter and ** $path is a relative path, then $path is interpreted relative to $dir. ** And the paths returned in the "name" column of the table are also ** relative to directory $dir. -** -** Notes on building this extension for Windows: -** Unless linked statically with the SQLite library, a preprocessor -** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone -** DLL form of this extension for WIN32. See its use below for details. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 @@ -8028,15 +9780,16 @@ SQLITE_EXTENSION_INIT1 # include <utime.h> # include <sys/time.h> # define STRUCT_STAT struct stat +# include <limits.h> +# include <stdlib.h> #else -# include "windows.h" -# include <io.h> +/* # include "windirent.h" */ # include <direct.h> -/* # include "test_windirent.h" */ -# define dirent DIRENT # define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); #endif #include <time.h> #include <errno.h> @@ -8052,26 +9805,25 @@ SQLITE_EXTENSION_INIT1 /* ** Structure of the fsdir() table-valued function */ - /* 0 1 2 3 4 5 */ -#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" + /* 0 1 2 3 4 5 6 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,level,path HIDDEN,dir HIDDEN)" + #define FSDIR_COLUMN_NAME 0 /* Name of the file */ #define FSDIR_COLUMN_MODE 1 /* Access mode */ #define FSDIR_COLUMN_MTIME 2 /* Last modification time */ #define FSDIR_COLUMN_DATA 3 /* File content */ -#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ -#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ +#define FSDIR_COLUMN_LEVEL 4 /* Level. Topmost is 1 */ +#define FSDIR_COLUMN_PATH 5 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 6 /* Path is relative to this directory */ /* ** UTF8 chmod() function for Windows */ #if defined(_WIN32) || defined(WIN32) static int fileio_chmod(const char *zPath, int pmode){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wchmod(b1, pmode); sqlite3_free(b1); return rc; @@ -8083,12 +9835,9 @@ static int fileio_chmod(const char *zPath, int pmode){ */ #if defined(_WIN32) || defined(WIN32) static int fileio_mkdir(const char *zPath){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wmkdir(b1); sqlite3_free(b1); return rc; @@ -8201,50 +9950,7 @@ static sqlite3_uint64 fileTimeToUnixTime( return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; } - - -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -# /* To allow a standalone DLL, use this next replacement function: */ -# undef sqlite3_win32_utf8_to_unicode -# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 -# -LPWSTR utf8_to_utf16(const char *z){ - int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); - LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); - if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) - return rv; - sqlite3_free(rv); - return 0; -} -#endif - -/* -** This function attempts to normalize the time values found in the stat() -** buffer to UTC. This is necessary on Win32, where the runtime library -** appears to return these values as local times. -*/ -static void statTimesToUtc( - const char *zPath, - STRUCT_STAT *pStatBuf -){ - HANDLE hFindFile; - WIN32_FIND_DATAW fd; - LPWSTR zUnicodeName; - extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); - zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); - if( zUnicodeName ){ - memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); - hFindFile = FindFirstFileW(zUnicodeName, &fd); - if( hFindFile!=NULL ){ - pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); - pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); - pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); - FindClose(hFindFile); - } - sqlite3_free(zUnicodeName); - } -} -#endif +#endif /* _WIN32 */ /* ** This function is used in place of stat(). On Windows, special handling @@ -8256,14 +9962,23 @@ static int fileStat( STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return 1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wstat(b1, pStatBuf); - if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + if( rc==0 ){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(b1, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + } + sqlite3_free(b1); return rc; #else return stat(zPath, pStatBuf); @@ -8394,7 +10109,6 @@ static int writeFile( if( mtime>=0 ){ #if defined(_WIN32) -#if !SQLITE_OS_WINRT /* Windows */ FILETIME lastAccess; FILETIME lastWrite; @@ -8425,7 +10139,6 @@ static int writeFile( }else{ return 1; } -#endif #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; @@ -8556,6 +10269,7 @@ struct fsdir_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ int nLvl; /* Number of entries in aLvl[] array */ + int mxLvl; /* Maximum level */ int iLvl; /* Index of current entry */ FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ @@ -8590,7 +10304,7 @@ static int fsdirConnect( (void)pzErr; rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + pNew = (fsdir_tab*)sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); @@ -8613,7 +10327,7 @@ static int fsdirDisconnect(sqlite3_vtab *pVtab){ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ fsdir_cursor *pCur; (void)p; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->iLvl = -1; @@ -8674,7 +10388,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ mode_t m = pCur->sStat.st_mode; pCur->iRowid++; - if( S_ISDIR(m) ){ + if( S_ISDIR(m) && pCur->iLvl+3<pCur->mxLvl ){ /* Descend into this directory */ int iNew = pCur->iLvl + 1; FsdirLevel *pLvl; @@ -8694,7 +10408,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ pCur->zPath = 0; pLvl->pDir = opendir(pLvl->zDir); if( pLvl->pDir==0 ){ - fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + fsdirSetErrmsg(pCur, "cannot read directory: %s", pLvl->zDir); return SQLITE_ERROR; } } @@ -8782,7 +10496,11 @@ static int fsdirColumn( }else{ readFileContents(ctx, pCur->zPath); } + break; } + case FSDIR_COLUMN_LEVEL: + sqlite3_result_int(ctx, pCur->iLvl+2); + break; case FSDIR_COLUMN_PATH: default: { /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. @@ -8816,8 +10534,11 @@ static int fsdirEof(sqlite3_vtab_cursor *cur){ /* ** xFilter callback. ** -** idxNum==1 PATH parameter only -** idxNum==2 Both PATH and DIR supplied +** idxNum bit Meaning +** 0x01 PATH=N +** 0x02 DIR=N +** 0x04 LEVEL<N +** 0x08 LEVEL<=N */ static int fsdirFilter( sqlite3_vtab_cursor *cur, @@ -8826,6 +10547,7 @@ static int fsdirFilter( ){ const char *zDir = 0; fsdir_cursor *pCur = (fsdir_cursor*)cur; + int i; (void)idxStr; fsdirResetCursor(pCur); @@ -8834,14 +10556,24 @@ static int fsdirFilter( return SQLITE_ERROR; } - assert( argc==idxNum && (argc==1 || argc==2) ); + assert( (idxNum & 0x01)!=0 && argc>0 ); zDir = (const char*)sqlite3_value_text(argv[0]); if( zDir==0 ){ fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); return SQLITE_ERROR; } - if( argc==2 ){ - pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + i = 1; + if( (idxNum & 0x02)!=0 ){ + assert( argc>i ); + pCur->zBase = (const char*)sqlite3_value_text(argv[i++]); + } + if( (idxNum & 0x0c)!=0 ){ + assert( argc>i ); + pCur->mxLvl = sqlite3_value_int(argv[i++]); + if( idxNum & 0x08 ) pCur->mxLvl++; + if( pCur->mxLvl<=0 ) pCur->mxLvl = 1000000000; + }else{ + pCur->mxLvl = 1000000000; } if( pCur->zBase ){ pCur->nBase = (int)strlen(pCur->zBase)+1; @@ -8870,10 +10602,11 @@ static int fsdirFilter( ** In this implementation idxNum is used to represent the ** query plan. idxStr is unused. ** -** The query plan is represented by values of idxNum: +** The query plan is represented by bits in idxNum: ** -** (1) The path value is supplied by argv[0] -** (2) Path is in argv[0] and dir is in argv[1] +** 0x01 The path value is supplied by argv[0] +** 0x02 dir is in argv[1] +** 0x04 maxdepth is in argv[1] or [2] */ static int fsdirBestIndex( sqlite3_vtab *tab, @@ -8882,6 +10615,9 @@ static int fsdirBestIndex( int i; /* Loop over constraints */ int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int idxLevel = -1; /* Index in pIdxInfo->aConstraint of LEVEL< or <= */ + int idxLevelEQ = 0; /* 0x08 for LEVEL<= or LEVEL=. 0x04 for LEVEL< */ + int omitLevel = 0; /* omit the LEVEL constraint */ int seenPath = 0; /* True if an unusable PATH= constraint is seen */ int seenDir = 0; /* True if an unusable DIR= constraint is seen */ const struct sqlite3_index_constraint *pConstraint; @@ -8889,25 +10625,48 @@ static int fsdirBestIndex( (void)tab; pConstraint = pIdxInfo->aConstraint; for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - switch( pConstraint->iColumn ){ - case FSDIR_COLUMN_PATH: { - if( pConstraint->usable ){ - idxPath = i; - seenPath = 0; - }else if( idxPath<0 ){ - seenPath = 1; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; } - break; - } - case FSDIR_COLUMN_DIR: { - if( pConstraint->usable ){ - idxDir = i; - seenDir = 0; - }else if( idxDir<0 ){ - seenDir = 1; + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; } - break; + case FSDIR_COLUMN_LEVEL: { + if( pConstraint->usable && idxLevel<0 ){ + idxLevel = i; + idxLevelEQ = 0x08; + omitLevel = 0; + } + break; + } + } + }else + if( pConstraint->iColumn==FSDIR_COLUMN_LEVEL + && pConstraint->usable + && idxLevel<0 + ){ + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE ){ + idxLevel = i; + idxLevelEQ = 0x08; + omitLevel = 1; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ + idxLevel = i; + idxLevelEQ = 0x04; + omitLevel = 1; } } } @@ -8924,14 +10683,20 @@ static int fsdirBestIndex( }else{ pIdxInfo->aConstraintUsage[idxPath].omit = 1; pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + pIdxInfo->idxNum = 0x01; + pIdxInfo->estimatedCost = 1.0e9; + i = 2; if( idxDir>=0 ){ pIdxInfo->aConstraintUsage[idxDir].omit = 1; - pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; - pIdxInfo->idxNum = 2; - pIdxInfo->estimatedCost = 10.0; - }else{ - pIdxInfo->idxNum = 1; - pIdxInfo->estimatedCost = 100.0; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = i++; + pIdxInfo->idxNum |= 0x02; + pIdxInfo->estimatedCost /= 1.0e4; + } + if( idxLevel>=0 ){ + pIdxInfo->aConstraintUsage[idxLevel].omit = omitLevel; + pIdxInfo->aConstraintUsage[idxLevel].argvIndex = i++; + pIdxInfo->idxNum |= idxLevelEQ; + pIdxInfo->estimatedCost /= 1.0e4; } } @@ -8977,6 +10742,154 @@ static int fsdirRegister(sqlite3 *db){ # define fsdirRegister(x) SQLITE_OK #endif +/* +** This version of realpath() works on any system. The string +** returned is held in memory allocated using sqlite3_malloc64(). +** The caller is responsible for calling sqlite3_free(). +*/ +static char *portable_realpath(const char *zPath){ +#if !defined(_WIN32) /* BEGIN unix */ + + char *zOut = 0; /* Result */ + char *z; /* Temporary buffer */ +#if defined(PATH_MAX) + char zBuf[PATH_MAX+1]; /* Space for the temporary buffer */ +#endif + + if( zPath==0 ) return 0; +#if defined(PATH_MAX) + z = realpath(zPath, zBuf); + if( z ){ + zOut = sqlite3_mprintf("%s", zBuf); + } +#endif /* defined(PATH_MAX) */ + if( zOut==0 ){ + /* Try POSIX.1-2008 malloc behavior */ + z = realpath(zPath, NULL); + if( z ){ + zOut = sqlite3_mprintf("%s", z); + free(z); + } + } + return zOut; + +#else /* End UNIX, Begin WINDOWS */ + + wchar_t *zPath16; /* UTF16 translation of zPath */ + char *zOut = 0; /* Result */ + wchar_t *z = 0; /* Temporary buffer */ + + if( zPath==0 ) return 0; + + zPath16 = sqlite3_win32_utf8_to_unicode(zPath); + if( zPath16==0 ) return 0; + z = _wfullpath(NULL, zPath16, 0); + sqlite3_free(zPath16); + if( z ){ + zOut = sqlite3_win32_unicode_to_utf8(z); + free(z); + } + return zOut; + +#endif /* End WINDOWS, Begin common code */ +} + +/* +** SQL function: realpath(X) +** +** Try to convert file or pathname X into its real, absolute pathname. +** Return NULL if unable. +** +** The file or directory X is not required to exist. The answer is formed +** by calling system realpath() on the prefix of X that does exist and +** appending the tail of X that does not (yet) exist. +*/ +static void realpathFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPath; /* Original input path */ + char *zCopy; /* An editable copy of zPath */ + char *zOut; /* The result */ + char cSep = 0; /* Separator turned into \000 */ + size_t len; /* Prefix length before cSep */ +#ifdef _WIN32 + const int isWin = 1; +#else + const int isWin = 0; +#endif + + (void)argc; + zPath = (const char*)sqlite3_value_text(argv[0]); + if( zPath==0 ) return; + if( zPath[0]==0 ) zPath = "."; + zCopy = sqlite3_mprintf("%s",zPath); + len = strlen(zCopy); + while( len>1 && (zCopy[len-1]=='/' || (isWin && zCopy[len-1]=='\\')) ){ + len--; + } + zCopy[len] = 0; + while( 1 /*exit-by-break*/ ){ + zOut = portable_realpath(zCopy); + zCopy[len] = cSep; + if( zOut ){ + if( cSep ){ + zOut = sqlite3_mprintf("%z%s",zOut,&zCopy[len]); + } + break; + }else{ + size_t i = len-1; + while( i>0 ){ + if( zCopy[i]=='/' || (isWin && zCopy[i]=='\\') ) break; + i--; + } + if( i<=0 ){ + if( zCopy[0]=='/' ){ + zOut = zCopy; + zCopy = 0; + }else if( (zOut = portable_realpath("."))!=0 ){ + zOut = sqlite3_mprintf("%z/%s", zOut, zCopy); + } + break; + } + cSep = zCopy[i]; + zCopy[i] = 0; + len = i; + } + } + sqlite3_free(zCopy); + if( zOut ){ + /* Simplify any "/./" or "/../" that might have snuck into the + ** pathname due to appending of zCopy. We only have to consider + ** unix "/" separators, because the _wfilepath() system call on + ** Windows will have already done this simplification for us. */ + size_t i, j, n; + n = strlen(zOut); + for(i=j=0; i<n; i++){ + if( zOut[i]=='/' ){ + if( zOut[i+1]=='/' ) continue; + if( zOut[i+1]=='.' && i+2<n && zOut[i+2]=='/' ){ + i += 1; + continue; + } + if( zOut[i+1]=='.' && i+3<n && zOut[i+2]=='.' && zOut[i+3]=='/' ){ + while( j>0 && zOut[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + zOut[j++] = zOut[i]; + } + zOut[j] = 0; + + /* Return the result */ + sqlite3_result_text(context, zOut, -1, sqlite3_free); + } +} + + #ifdef _WIN32 #endif @@ -9003,19 +10916,16 @@ int sqlite3_fileio_init( if( rc==SQLITE_OK ){ rc = fsdirRegister(db); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "realpath", 1, + SQLITE_UTF8, 0, + realpathFunc, 0, 0); + } return rc; } -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -/* To allow a standalone DLL, make test_windirent.c use the same - * redefined SQLite API calls as the above extension code does. - * Just pull in this .c to accomplish this. As a beneficial side - * effect, this extension becomes a single translation unit. */ -# include "test_windirent.c" -#endif - -/************************* End ../ext/misc/fileio.c ********************/ -/************************* Begin ../ext/misc/completion.c ******************/ +/************************* End ext/misc/fileio.c ********************/ +/************************* Begin ext/misc/completion.c ******************/ /* ** 2017-07-10 ** @@ -9150,7 +11060,7 @@ static int completionConnect( " phase INT HIDDEN" /* Used for debugging only */ ")"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -9172,7 +11082,7 @@ static int completionDisconnect(sqlite3_vtab *pVtab){ */ static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ completion_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((completion_vtab*)p)->db; @@ -9217,6 +11127,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ completion_cursor *pCur = (completion_cursor*)cur; int eNextPhase = 0; /* Next phase to try if current phase reaches end */ int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + int rc; pCur->iRowid++; while( pCur->ePhase!=COMPLETION_EOF ){ switch( pCur->ePhase ){ @@ -9242,22 +11153,27 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_TABLES: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str* pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT name FROM \"%w\".sqlite_schema", - zSql, zSep, zDb + zSep, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_COLUMNS; @@ -9266,24 +11182,29 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_COLUMNS: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str *pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" " WHERE sm.type='table'", - zSql, zSep, zDb, zDb + zSep, zDb, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_EOF; @@ -9300,9 +11221,10 @@ static int completionNext(sqlite3_vtab_cursor *cur){ pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); }else{ /* When all rows are finished, advance to the next phase */ - sqlite3_finalize(pCur->pStmt); + rc = sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; pCur->ePhase = eNextPhase; + if( rc ) return rc; continue; } } @@ -9388,6 +11310,7 @@ static int completionFilter( if( pCur->nPrefix>0 ){ pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + pCur->nPrefix = (int)strlen(pCur->zPrefix); } iArg = 1; } @@ -9396,6 +11319,7 @@ static int completionFilter( if( pCur->nLine>0 ){ pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zLine==0 ) return SQLITE_NOMEM; + pCur->nLine = (int)strlen(pCur->zLine); } } if( pCur->zLine!=0 && pCur->zPrefix==0 ){ @@ -9407,6 +11331,7 @@ static int completionFilter( if( pCur->nPrefix>0 ){ pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + pCur->nPrefix = (int)strlen(pCur->zPrefix); } } pCur->iRowid = 0; @@ -9524,8 +11449,8 @@ int sqlite3_completion_init( return rc; } -/************************* End ../ext/misc/completion.c ********************/ -/************************* Begin ../ext/misc/appendvfs.c ******************/ +/************************* End ext/misc/completion.c ********************/ +/************************* Begin ext/misc/appendvfs.c ******************/ /* ** 2017-10-20 ** @@ -10199,10 +12124,10 @@ int sqlite3_appendvfs_init( return rc; } -/************************* End ../ext/misc/appendvfs.c ********************/ +/************************* End ext/misc/appendvfs.c ********************/ #endif #ifdef SQLITE_HAVE_ZLIB -/************************* Begin ../ext/misc/zipfile.c ******************/ +/************************* Begin ext/misc/zipfile.c ******************/ /* ** 2017-12-26 ** @@ -10320,7 +12245,13 @@ static const char ZIPFILE_SCHEMA[] = ") WITHOUT ROWID;"; #define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "file" in the above */ -#define ZIPFILE_BUFFER_SIZE (64*1024) +#define ZIPFILE_MX_NAME (250) /* Windows limitation on filename size */ + +/* +** The buffer should be large enough to contain 3 65536 byte strings - the +** filename, the extra field and the file comment. +*/ +#define ZIPFILE_BUFFER_SIZE (200*1024) /* @@ -10592,7 +12523,7 @@ static int zipfileConnect( rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); + pNew = (ZipfileTab*)sqlite3_malloc64((i64)nByte+nFile); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, nByte+nFile); pNew->db = db; @@ -10655,7 +12586,7 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ZipfileTab *pTab = (ZipfileTab*)p; ZipfileCsr *pCsr; - pCsr = sqlite3_malloc(sizeof(*pCsr)); + pCsr = sqlite3_malloc64(sizeof(*pCsr)); *ppCsr = (sqlite3_vtab_cursor*)pCsr; if( pCsr==0 ){ return SQLITE_NOMEM; @@ -10738,14 +12669,15 @@ static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ static int zipfileReadData( FILE *pFile, /* Read from this file */ u8 *aRead, /* Read into this buffer */ - int nRead, /* Number of bytes to read */ + i64 nRead, /* Number of bytes to read */ i64 iOff, /* Offset to read from */ char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ ){ size_t n; fseek(pFile, (long)iOff, SEEK_SET); - n = fread(aRead, 1, nRead, pFile); - if( (int)n!=nRead ){ + n = fread(aRead, 1, (long)nRead, pFile); + if( n!=(size_t)nRead ){ + sqlite3_free(*pzErrmsg); *pzErrmsg = sqlite3_mprintf("error in fread()"); return SQLITE_ERROR; } @@ -10762,7 +12694,7 @@ static int zipfileAppendData( fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET); n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); if( (int)n!=nWrite ){ - pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); + zipfileTableErr(pTab,"error in fwrite()"); return SQLITE_ERROR; } pTab->szCurrent += nWrite; @@ -10877,6 +12809,7 @@ static int zipfileReadLFH( pLFH->szUncompressed = zipfileRead32(aRead); pLFH->nFile = zipfileRead16(aRead); pLFH->nExtra = zipfileRead16(aRead); + if( pLFH->nFile>ZIPFILE_MX_NAME ) rc = SQLITE_ERROR; } return rc; } @@ -10902,7 +12835,12 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ u8 *p = aExtra; u8 *pEnd = &aExtra[nExtra]; - while( p<pEnd ){ + /* Stop when there are less than 9 bytes left to scan in the buffer. This + ** is because the timestamp field requires exactly 9 bytes - 4 bytes of + ** header fields and 5 bytes of data. If there are less than 9 bytes + ** remaining, either it is some other field or else the extra data + ** is corrupt. Either way, do not process it. */ + while( p+(2*sizeof(u16) + 1 + sizeof(u32))<=pEnd ){ u16 id = zipfileRead16(p); u16 nByte = zipfileRead16(p); @@ -11004,6 +12942,16 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ } /* +** Set (*pzErr) to point to a buffer from sqlite3_malloc() containing a +** generic corruption message and return SQLITE_CORRUPT; +*/ +static int zipfileCorrupt(char **pzErr){ + sqlite3_free(*pzErr); + *pzErr = sqlite3_mprintf("zip archive is corrupt"); + return SQLITE_CORRUPT; +} + +/* ** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in ** size) containing an entire zip archive image. Or, if aBlob is NULL, ** then pFile is a file-handle open on a zip file. In either case, this @@ -11017,7 +12965,7 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ static int zipfileGetEntry( ZipfileTab *pTab, /* Store any error message here */ const u8 *aBlob, /* Pointer to in-memory file image */ - int nBlob, /* Size of aBlob[] in bytes */ + i64 nBlob, /* Size of aBlob[] in bytes */ FILE *pFile, /* If aBlob==0, read from this file */ i64 iOff, /* Offset of CDS record */ ZipfileEntry **ppEntry /* OUT: Pointer to new object */ @@ -11025,12 +12973,15 @@ static int zipfileGetEntry( u8 *aRead; char **pzErr = &pTab->base.zErrMsg; int rc = SQLITE_OK; - (void)nBlob; if( aBlob==0 ){ aRead = pTab->aBuffer; rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); }else{ + if( (iOff+ZIPFILE_CDS_FIXED_SZ)>nBlob ){ + /* Not enough data for the CDS structure. Corruption. */ + return zipfileCorrupt(pzErr); + } aRead = (u8*)&aBlob[iOff]; } @@ -11054,13 +13005,16 @@ static int zipfileGetEntry( memset(pNew, 0, sizeof(ZipfileEntry)); rc = zipfileReadCDS(aRead, &pNew->cds); if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + zipfileTableErr(pTab, "failed to read CDS at offset %lld", iOff); }else if( aBlob==0 ){ rc = zipfileReadData( pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr ); }else{ aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; + if( (iOff + ZIPFILE_CDS_FIXED_SZ + nFile + nExtra)>nBlob ){ + rc = zipfileCorrupt(pzErr); + } } } @@ -11083,18 +13037,26 @@ static int zipfileGetEntry( rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); }else{ aRead = (u8*)&aBlob[pNew->cds.iOffset]; + if( ((i64)pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ)>nBlob ){ + rc = zipfileCorrupt(pzErr); + } } + memset(&lfh, 0, sizeof(lfh)); if( rc==SQLITE_OK ) rc = zipfileReadLFH(aRead, &lfh); if( rc==SQLITE_OK ){ - pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff = (i64)pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; pNew->iDataOff += lfh.nFile + lfh.nExtra; if( aBlob && pNew->cds.szCompressed ){ - pNew->aData = &pNew->aExtra[nExtra]; - memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + if( pNew->iDataOff + pNew->cds.szCompressed > nBlob ){ + rc = zipfileCorrupt(pzErr); + }else{ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } } }else{ - *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + zipfileTableErr(pTab, "failed to read LFH at offset %d", (int)pNew->cds.iOffset ); } @@ -11118,7 +13080,7 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){ int rc = SQLITE_OK; if( pCsr->pFile ){ - i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + i64 iEof = (i64)pCsr->eocd.iOffset + (i64)pCsr->eocd.nSize; zipfileEntryFree(pCsr->pCurrent); pCsr->pCurrent = 0; if( pCsr->iNextOff>=iEof ){ @@ -11163,7 +13125,7 @@ static void zipfileInflate( int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ ){ - u8 *aRes = sqlite3_malloc(nOut); + u8 *aRes = sqlite3_malloc64(nOut); if( aRes==0 ){ sqlite3_result_error_nomem(pCtx); }else{ @@ -11184,7 +13146,7 @@ static void zipfileInflate( if( err!=Z_STREAM_END ){ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); }else{ - sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + sqlite3_result_blob(pCtx, aRes, (int)str.total_out, zipfileFree); aRes = 0; } } @@ -11356,12 +13318,12 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ const u8 *aBlob, /* Pointer to in-memory file image */ - int nBlob, /* Size of aBlob[] in bytes */ + i64 nBlob, /* Size of aBlob[] in bytes */ FILE *pFile, /* Read from this file if aBlob==0 */ ZipfileEOCD *pEOCD /* Object to populate */ ){ u8 *aRead = pTab->aBuffer; /* Temporary buffer */ - int nRead; /* Bytes to read from file */ + i64 nRead; /* Bytes to read from file */ int rc = SQLITE_OK; memset(pEOCD, 0, sizeof(ZipfileEOCD)); @@ -11382,7 +13344,7 @@ static int zipfileReadEOCD( } if( rc==SQLITE_OK ){ - int i; + i64 i; /* Scan backwards looking for the signature bytes */ for(i=nRead-20; i>=0; i--){ @@ -11393,9 +13355,7 @@ static int zipfileReadEOCD( } } if( i<0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "cannot find end of central directory record" - ); + zipfileTableErr(pTab, "cannot find end of central directory record"); return SQLITE_ERROR; } @@ -11440,7 +13400,7 @@ static void zipfileAddEntry( } } -static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, i64 nBlob){ ZipfileEOCD eocd; int rc; int i; @@ -11488,7 +13448,7 @@ static int zipfileFilter( }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ static const u8 aEmptyBlob = 0; const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); - int nBlob = sqlite3_value_bytes(argv[0]); + i64 nBlob = sqlite3_value_bytes(argv[0]); assert( pTab->pFirstEntry==0 ); if( aBlob==0 ){ aBlob = &aEmptyBlob; @@ -11562,7 +13522,7 @@ static int zipfileBestIndex( static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + pNew = sqlite3_malloc64(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); pNew->cds.zFile = sqlite3_mprintf("%s", zPath); @@ -11686,7 +13646,7 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ assert( pTab->pWriteFd==0 ); if( pTab->zFile==0 || pTab->zFile[0]==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); + zipfileTableErr(pTab, "zipfile: missing filename"); return SQLITE_ERROR; } @@ -11696,9 +13656,9 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ ** in main-memory until the transaction is committed. */ pTab->pWriteFd = sqlite3_fopen(pTab->zFile, "ab+"); if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); + zipfileTableErr(pTab, + "zipfile: failed to open file %s for writing", pTab->zFile + ); rc = SQLITE_ERROR; }else{ fseek(pTab->pWriteFd, 0, SEEK_END); @@ -11878,6 +13838,11 @@ static int zipfileUpdate( zPath = (const char*)sqlite3_value_text(apVal[2]); if( zPath==0 ) zPath = ""; nPath = (int)strlen(zPath); + if( nPath>ZIPFILE_MX_NAME ){ + zipfileTableErr(pTab, "filename too long; max: %d bytes", + ZIPFILE_MX_NAME); + rc = SQLITE_CONSTRAINT; + } mTime = zipfileGetTime(apVal[4]); } @@ -12158,7 +14123,7 @@ struct ZipfileCtx { ZipfileBuffer cds; }; -static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ +static int zipfileBufferGrow(ZipfileBuffer *pBuf, i64 nByte){ if( pBuf->n+nByte>pBuf->nAlloc ){ u8 *aNew; sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; @@ -12207,7 +14172,7 @@ static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ char *zName = 0; /* Path (name) of new entry */ int nName = 0; /* Size of zName in bytes */ char *zFree = 0; /* Free this before returning */ - int nByte; + i64 nByte; memset(&e, 0, sizeof(e)); p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); @@ -12239,6 +14204,13 @@ static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ rc = SQLITE_ERROR; goto zipfile_step_out; } + if( nName>ZIPFILE_MX_NAME ){ + zErr = sqlite3_mprintf( + "filename argument to zipfile() too big; max: %d bytes", + ZIPFILE_MX_NAME); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use ** deflate compression) or NULL (choose automatically). */ @@ -12450,8 +14422,8 @@ int sqlite3_zipfile_init( return zipfileRegister(db); } -/************************* End ../ext/misc/zipfile.c ********************/ -/************************* Begin ../ext/misc/sqlar.c ******************/ +/************************* End ext/misc/zipfile.c ********************/ +/************************* Begin ext/misc/sqlar.c ******************/ /* ** 2017-12-17 ** @@ -12500,7 +14472,7 @@ static void sqlarCompressFunc( uLongf nOut = compressBound(nData); Bytef *pOut; - pOut = (Bytef*)sqlite3_malloc(nOut); + pOut = (Bytef*)sqlite3_malloc64(nOut); if( pOut==0 ){ sqlite3_result_error_nomem(context); return; @@ -12538,14 +14510,14 @@ static void sqlarUncompressFunc( sqlite3_int64 sz; assert( argc==2 ); - sz = sqlite3_value_int(argv[1]); + sz = sqlite3_value_int64(argv[1]); if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); - Bytef *pOut = sqlite3_malloc(sz); + Bytef *pOut = sqlite3_malloc64(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ @@ -12579,9 +14551,10 @@ int sqlite3_sqlar_init( return rc; } -/************************* End ../ext/misc/sqlar.c ********************/ +/************************* End ext/misc/sqlar.c ********************/ #endif -/************************* Begin ../ext/expert/sqlite3expert.h ******************/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) +/************************* Begin ext/expert/sqlite3expert.h ******************/ /* ** 2017 April 07 ** @@ -12751,8 +14724,8 @@ void sqlite3_expert_destroy(sqlite3expert*); #endif /* !defined(SQLITEEXPERT_H) */ -/************************* End ../ext/expert/sqlite3expert.h ********************/ -/************************* Begin ../ext/expert/sqlite3expert.c ******************/ +/************************* End ext/expert/sqlite3expert.h ********************/ +/************************* Begin ext/expert/sqlite3expert.c ******************/ /* ** 2017 April 09 ** @@ -12927,11 +14900,11 @@ struct sqlite3expert { ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. */ -static void *idxMalloc(int *pRc, int nByte){ +static void *idxMalloc(int *pRc, i64 nByte){ void *pRet; assert( *pRc==SQLITE_OK ); assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); + pRet = sqlite3_malloc64(nByte); if( pRet ){ memset(pRet, 0, nByte); }else{ @@ -12998,7 +14971,7 @@ static int idxHashAdd( return 1; } } - pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); + pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + (i64)nKey+1 + (i64)nVal+1); if( pEntry ){ pEntry->zKey = (char*)&pEntry[1]; memcpy(pEntry->zKey, zKey, nKey); @@ -13133,15 +15106,15 @@ struct ExpertCsr { }; static char *expertDequote(const char *zIn){ - int n = STRLEN(zIn); - char *zRet = sqlite3_malloc(n); + i64 n = STRLEN(zIn); + char *zRet = sqlite3_malloc64(n); assert( zIn[0]=='\'' ); assert( zIn[n-1]=='\'' ); if( zRet ){ - int iOut = 0; - int iIn = 0; + i64 iOut = 0; + i64 iIn = 0; for(iIn=1; iIn<(n-1); iIn++){ if( zIn[iIn]=='\'' ){ assert( zIn[iIn+1]=='\'' ); @@ -13454,7 +15427,7 @@ static int idxGetTableInfo( sqlite3_stmt *p1 = 0; int nCol = 0; int nTab; - int nByte; + i64 nByte; IdxTable *pNew = 0; int rc, rc2; char *pCsr = 0; @@ -13546,14 +15519,14 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ va_list ap; char *zAppend = 0; char *zRet = 0; - int nIn = zIn ? STRLEN(zIn) : 0; - int nAppend = 0; + i64 nIn = zIn ? STRLEN(zIn) : 0; + i64 nAppend = 0; va_start(ap, zFmt); if( *pRc==SQLITE_OK ){ zAppend = sqlite3_vmprintf(zFmt, ap); if( zAppend ){ nAppend = STRLEN(zAppend); - zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); + zRet = (char*)sqlite3_malloc64(nIn + nAppend + 1); } if( zAppend && zRet ){ if( nIn ) memcpy(zRet, zIn, nIn); @@ -14317,8 +16290,8 @@ struct IdxRemCtx { int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ i64 iVal; /* SQLITE_INTEGER value */ double rVal; /* SQLITE_FLOAT value */ - int nByte; /* Bytes of space allocated at z */ - int n; /* Size of buffer z */ + i64 nByte; /* Bytes of space allocated at z */ + i64 n; /* Size of buffer z */ char *z; /* SQLITE_TEXT/BLOB value */ } aSlot[1]; }; @@ -14354,11 +16327,13 @@ static void idxRemFunc( break; case SQLITE_BLOB: - sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + assert( pSlot->n <= 0x7fffffff ); + sqlite3_result_blob(pCtx, pSlot->z, (int)pSlot->n, SQLITE_TRANSIENT); break; case SQLITE_TEXT: - sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + assert( pSlot->n <= 0x7fffffff ); + sqlite3_result_text(pCtx, pSlot->z, (int)pSlot->n, SQLITE_TRANSIENT); break; } @@ -14378,10 +16353,10 @@ static void idxRemFunc( case SQLITE_BLOB: case SQLITE_TEXT: { - int nByte = sqlite3_value_bytes(argv[1]); + i64 nByte = sqlite3_value_bytes(argv[1]); const void *pData = 0; if( nByte>pSlot->nByte ){ - char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); + char *zNew = (char*)sqlite3_realloc64(pSlot->z, nByte*2); if( zNew==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -14436,7 +16411,7 @@ static int idxPopulateOneStat1( int nCol = 0; int i; sqlite3_stmt *pQuery = 0; - int *aStat = 0; + i64 *aStat = 0; int rc = SQLITE_OK; assert( p->iSample>0 ); @@ -14482,7 +16457,7 @@ static int idxPopulateOneStat1( sqlite3_free(zQuery); if( rc==SQLITE_OK ){ - aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); + aStat = (i64*)idxMalloc(&rc, sizeof(i64)*(nCol+1)); } if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ IdxHashEntry *pEntry; @@ -14499,11 +16474,11 @@ static int idxPopulateOneStat1( } if( rc==SQLITE_OK ){ - int s0 = aStat[0]; - zStat = sqlite3_mprintf("%d", s0); + i64 s0 = aStat[0]; + zStat = sqlite3_mprintf("%lld", s0); if( zStat==0 ) rc = SQLITE_NOMEM; for(i=1; rc==SQLITE_OK && i<=nCol; i++){ - zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]); + zStat = idxAppendText(&rc, zStat, " %lld", (s0+aStat[i]/2) / aStat[i]); } } @@ -14582,7 +16557,7 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); if( rc==SQLITE_OK ){ - int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); + i64 nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); } @@ -14599,7 +16574,7 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ } if( rc==SQLITE_OK ){ - pCtx->nSlot = nMax+1; + pCtx->nSlot = (i64)nMax+1; rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); } if( rc==SQLITE_OK ){ @@ -14866,7 +16841,7 @@ int sqlite3_expert_sql( if( pStmt ){ IdxStatement *pNew; const char *z = sqlite3_sql(pStmt); - int n = STRLEN(z); + i64 n = STRLEN(z); pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); if( rc==SQLITE_OK ){ pNew->zSql = (char*)&pNew[1]; @@ -14988,8 +16963,9 @@ void sqlite3_expert_destroy(sqlite3expert *p){ #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ -/************************* End ../ext/expert/sqlite3expert.c ********************/ -/************************* Begin ../ext/intck/sqlite3intck.h ******************/ +/************************* End ext/expert/sqlite3expert.c ********************/ +#endif +/************************* Begin ext/intck/sqlite3intck.h ******************/ /* ** 2024-02-08 ** @@ -15162,8 +17138,8 @@ const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); #endif /* ifndef _SQLITE_INTCK_H */ -/************************* End ../ext/intck/sqlite3intck.h ********************/ -/************************* Begin ../ext/intck/sqlite3intck.c ******************/ +/************************* End ext/intck/sqlite3intck.h ********************/ +/************************* Begin ext/intck/sqlite3intck.c ******************/ /* ** 2024-02-08 ** @@ -15326,6 +17302,7 @@ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ sqlite3_free(zRet); zRet = 0; } + va_end(ap); return zRet; } @@ -15484,7 +17461,7 @@ static int intckGetToken(const char *z){ char c = z[0]; int iRet = 1; if( c=='\'' || c=='"' || c=='`' ){ - while( 1 ){ + while( z[iRet] ){ if( z[iRet]==c ){ iRet++; if( z[iRet]!=c ) break; @@ -16105,8 +18082,8 @@ const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ return p->zTestSql; } -/************************* End ../ext/intck/sqlite3intck.c ********************/ -/************************* Begin ../ext/misc/stmtrand.c ******************/ +/************************* End ext/intck/sqlite3intck.c ********************/ +/************************* Begin ext/misc/stmtrand.c ******************/ /* ** 2024-05-24 ** @@ -16161,7 +18138,7 @@ static void stmtrandFunc( p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); if( p==0 ){ unsigned int seed; - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ sqlite3_result_error_nomem(context); return; @@ -16205,8 +18182,8 @@ int sqlite3_stmtrand_init( return rc; } -/************************* End ../ext/misc/stmtrand.c ********************/ -/************************* Begin ../ext/misc/vfstrace.c ******************/ +/************************* End ext/misc/stmtrand.c ********************/ +/************************* Begin ext/misc/vfstrace.c ******************/ /* ** 2011 March 16 ** @@ -17101,7 +19078,7 @@ static int vfstraceOpen( vfstrace_printf(pInfo, "%s.xOpen(%s,flags=0x%x)", pInfo->zVfsName, p->zFName, flags); if( p->pReal->pMethods ){ - sqlite3_io_methods *pNew = sqlite3_malloc( sizeof(*pNew) ); + sqlite3_io_methods *pNew = sqlite3_malloc64( sizeof(*pNew) ); const sqlite3_io_methods *pSub = p->pReal->pMethods; memset(pNew, 0, sizeof(*pNew)); pNew->iVersion = pSub->iVersion; @@ -17419,7 +19396,7 @@ void vfstrace_unregister(const char *zTraceName){ sqlite3_free(pVfs); } -/************************* End ../ext/misc/vfstrace.c ********************/ +/************************* End ext/misc/vfstrace.c ********************/ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 @@ -17427,7 +19404,7 @@ void vfstrace_unregister(const char *zTraceName){ #define SQLITE_SHELL_HAVE_RECOVER 0 #endif #if SQLITE_SHELL_HAVE_RECOVER -/************************* Begin ../ext/recover/sqlite3recover.h ******************/ +/************************* Begin ext/recover/sqlite3recover.h ******************/ /* ** 2022-08-27 ** @@ -17678,9 +19655,9 @@ int sqlite3_recover_finish(sqlite3_recover*); #endif /* ifndef _SQLITE_RECOVER_H */ -/************************* End ../ext/recover/sqlite3recover.h ********************/ +/************************* End ext/recover/sqlite3recover.h ********************/ # ifndef SQLITE_HAVE_SQLITE3R -/************************* Begin ../ext/recover/dbdata.c ******************/ +/************************* Begin ext/recover/dbdata.c ******************/ /* ** 2019-04-17 ** @@ -18705,8 +20682,8 @@ int sqlite3_dbdata_init( #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ -/************************* End ../ext/recover/dbdata.c ********************/ -/************************* Begin ../ext/recover/sqlite3recover.c ******************/ +/************************* End ext/recover/dbdata.c ********************/ +/************************* Begin ext/recover/sqlite3recover.c ******************/ /* ** 2022-08-27 ** @@ -19242,17 +21219,38 @@ static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ } /* +** Run a single SQL statement in zSql. If zSql contains two or more +** SQL statements separated by ';', only the first is run. +** +** Return the sqlite3_finalizer() or sqlite3_prepare() result code +** from running the zSql statement. +*/ +static int recoverOneStmt(sqlite3 *db, const char *zSql){ + sqlite3_stmt *pStmt = 0; + int rc; + if( zSql==0 ) return SQLITE_OK; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc ){ + sqlite3_finalize(pStmt); + return rc; + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){} + return sqlite3_finalize(pStmt); +} + +/* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this ** case. ** -** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. -** Or, if an error occurs, leave an error code and message in the recover -** handle and return a copy of the error code. +** Otherwise, execute a single SQL statment in zSql. Even if zSql contains +** two or more SQL statements separated by ';', only execute the first one. +** If successful, return SQLITE_OK. Or, if an error occurs, leave an error +** code and message in the recover handle and return a copy of the error code. */ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ if( p->errCode==SQLITE_OK ){ - int rc = sqlite3_exec(db, zSql, 0, 0, 0); + int rc = recoverOneStmt(db, zSql); if( rc ){ recoverDbError(p, db); } @@ -19652,7 +21650,8 @@ static void recoverTransferSettings(sqlite3_recover *p){ } recoverFinalize(p, p1); } - recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); + recoverExec(p, db2, "CREATE TABLE t1(a)"); + recoverExec(p, db2, "DROP TABLE t1"); if( p->errCode==SQLITE_OK ){ sqlite3 *db = p->dbOut; @@ -19734,12 +21733,12 @@ static int recoverOpenOutput(sqlite3_recover *p){ static void recoverOpenRecovery(sqlite3_recover *p){ char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); recoverExec(p, p->dbOut, zSql); - recoverExec(p, p->dbOut, - "PRAGMA writable_schema = 1;" - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - ); sqlite3_free(zSql); + recoverExec(p, p->dbOut, "PRAGMA writable_schema = 1"); + recoverExec(p, p->dbOut, + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT)"); + recoverExec(p, p->dbOut, + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql)"); } @@ -19879,7 +21878,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ ")" "SELECT rootpage, tbl, isVirtual, name, sql" " FROM dbschema " - " WHERE tbl OR isIndex" + " WHERE (tbl OR isIndex) AND sql GLOB 'CREATE *'" " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" ); @@ -19905,7 +21904,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ zName, zName, zSql )); } - rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + rc = recoverOneStmt(p->dbOut, zSql); if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); if( bTable && !bVirtual ){ @@ -19947,15 +21946,17 @@ static int recoverWriteSchema2(sqlite3_recover *p){ p->bSlowIndexes ? "SELECT rootpage, sql FROM recovery.schema " " WHERE type!='table' AND type!='index'" + " AND sql GLOB 'CREATE *'" : "SELECT rootpage, sql FROM recovery.schema " " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + " AND sql GLOB 'CREATE *'" ); if( pSelect ){ while( sqlite3_step(pSelect)==SQLITE_ROW ){ const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); - int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + int rc = recoverOneStmt(p->dbOut, zSql); if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); }else if( rc!=SQLITE_ERROR ){ @@ -21333,7 +23334,7 @@ static void recoverStep(sqlite3_recover *p){ if( bUseWrapper ) recoverUninstallWrapper(p); }while( p->errCode==SQLITE_NOTADB && (bUseWrapper--) - && SQLITE_OK==sqlite3_exec(p->dbIn, "ROLLBACK", 0, 0, 0) + && SQLITE_OK==recoverOneStmt(p->dbIn, "ROLLBACK") ); } @@ -21398,7 +23399,7 @@ static void recoverStep(sqlite3_recover *p){ ** database. Regardless of whether or not an error has occurred, make ** an attempt to end the read transaction on the input database. */ recoverExec(p, p->dbOut, "COMMIT"); - rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + rc = recoverOneStmt(p->dbIn, "END"); if( p->errCode==SQLITE_OK ) p->errCode = rc; recoverSqlCallback(p, "PRAGMA writable_schema = off"); @@ -21594,7 +23595,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ }else{ recoverFinalCleanup(p); if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ - rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + rc = recoverOneStmt(p->dbIn, "END"); if( p->errCode==SQLITE_OK ) p->errCode = rc; } rc = p->errCode; @@ -21609,7 +23610,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ -/************************* End ../ext/recover/sqlite3recover.c ********************/ +/************************* End ext/recover/sqlite3recover.c ********************/ # endif /* SQLITE_HAVE_SQLITE3R */ #endif #ifdef SQLITE_SHELL_EXTSRC @@ -21629,37 +23630,32 @@ struct OpenSession { }; #endif +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) typedef struct ExpertInfo ExpertInfo; struct ExpertInfo { sqlite3expert *pExpert; int bVerbose; }; +#endif -/* A single line in the EQP output */ -typedef struct EQPGraphRow EQPGraphRow; -struct EQPGraphRow { - int iEqpId; /* ID for this row */ - int iParentId; /* ID of the parent row */ - EQPGraphRow *pNext; /* Next row in sequence */ - char zText[1]; /* Text to display for this row */ -}; +/* All the parameters that determine how to render query results. +*/ +typedef struct Mode { + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ + u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ + u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ + u8 eMode; /* One of the MODE_ values */ + sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ +} Mode; -/* All EQP output is collected into an instance of the following */ -typedef struct EQPGraph EQPGraph; -struct EQPGraph { - EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ - EQPGraphRow *pLast; /* Last element of the pRow list */ - char zPrefix[100]; /* Graph prefix */ -}; +/* Flags for Mode.mFlags */ +#define MFLG_ECHO 0x01 /* Echo inputs to output */ +#define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ +#define MFLG_HDR 0x04 /* .header used to change headers on/off */ -/* Parameters affecting columnar mode result display (defaulting together) */ -typedef struct ColModeOpts { - int iWrap; /* In columnar modes, wrap lines reaching this limit */ - u8 bQuote; /* Quote results for .mode box and table */ - u8 bWordWrap; /* In columnar modes, wrap at word boundaries */ -} ColModeOpts; -#define ColModeOpts_default { 60, 0, 0 } -#define ColModeOpts_default_qbox { 60, 1, 0 } /* ** State information about the database connection is contained in an @@ -21668,11 +23664,6 @@ typedef struct ColModeOpts { typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ - u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ - u8 autoEQPtest; /* autoEQP is in test mode */ - u8 autoEQPtrace; /* autoEQP is in trace mode */ - u8 scanstatsOn; /* True to display scan stats before each finalize */ u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ @@ -21680,48 +23671,44 @@ struct ShellState { u8 bSafeMode; /* True to prohibit unsafe operations */ u8 bSafeModePersist; /* The long-term value of bSafeMode */ u8 eRestoreState; /* See comments above doAutoDetectRestore() */ - u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ - u8 eEscMode; /* Escape mode for text output */ - ColModeOpts cmOpts; /* Option values affecting columnar mode output */ unsigned statsOn; /* True to display memory stats before each finalize */ unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ + u8 nPopOutput; /* Revert .output settings when reaching zero */ + u8 nPopMode; /* Revert .mode settings when reaching zero */ + u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ int inputNesting; /* Track nesting level of .read and other redirects */ - int outCount; /* Revert to stdout when reaching zero */ - int cnt; /* Number of records displayed so far */ - int lineno; /* Line number of last line read from in */ + double prevTimer; /* Last reported timer value */ + double tmProgress; /* --timeout option for .progress */ + i64 lineno; /* Line number of last line read from in */ + const char *zInFile; /* Name of the input file */ int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ FILE *in; /* Read commands from this stream */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ int nErr; /* Number of errors seen */ - int mode; /* An output mode setting */ - int modePrior; /* Saved mode */ - int cMode; /* temporary output mode for the current query */ - int normalMode; /* Output mode before ".explain on" */ int writableSchema; /* True if PRAGMA writable_schema=ON */ - int showHeader; /* True to show column names in List or Column mode */ int nCheck; /* Number of ".check" commands run */ unsigned nProgress; /* Number of progress callbacks encountered */ unsigned mxProgress; /* Maximum progress callbacks before failing */ unsigned flgProgress; /* Flags for the progress callback */ unsigned shellFlgs; /* Various flags */ - unsigned priorShFlgs; /* Saved copy of flags */ + unsigned nTestRun; /* Number of test cases run */ + unsigned nTestErr; /* Number of test cases that failed */ sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ + char *zErrPrefix; /* Alternative error message prefix */ char zTestcase[30]; /* Name of current test case */ - char colSeparator[20]; /* Column separator character for several modes */ - char rowSeparator[20]; /* Row separator character for MODE_Ascii */ - char colSepPrior[20]; /* Saved column separator */ - char rowSepPrior[20]; /* Saved row separator */ - int *colWidth; /* Requested width of each column in columnar modes */ - int *actualWidth; /* Actual width of each column */ - int nWidth; /* Number of slots in colWidth[] and actualWidth[] */ - char nullValue[20]; /* The text to print when a NULL comes back from - ** the database */ char outfile[FILENAME_MAX]; /* Filename for *out */ sqlite3_stmt *pStmt; /* Current statement if any. */ FILE *pLog; /* Write log output here */ + Mode mode; /* Current display mode */ + Mode modePrior; /* Backup */ + struct SavedMode { /* Ability to define custom mode configurations */ + char *zTag; /* Name of this saved mode */ + Mode mode; /* The saved mode */ + } *aSavedModes; /* Array of saved .mode settings. system malloc() */ + int nSavedModes; /* Number of saved .mode settings */ struct AuxDb { /* Storage space for auxiliary database connections */ sqlite3 *db; /* Connection pointer */ const char *zDbFilename; /* Filename used to open the connection */ @@ -21732,12 +23719,19 @@ struct ShellState { #endif } aAuxDb[5], /* Array of all database connections */ *pAuxDb; /* Currently active database connection */ - int *aiIndent; /* Array of indents used in MODE_Explain */ - int nIndent; /* Size of array aiIndent[] */ - int iIndent; /* Index of current op in aiIndent[] */ char *zNonce; /* Nonce for temporary safe-mode escapes */ - EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +#endif + struct DotCmdLine { /* Info about arguments to a dot-command */ + const char *zOrig; /* Original text of the dot-command */ + char *zCopy; /* Copy of zOrig, from malloc() */ + int nAlloc; /* Size of allocates for arrays below */ + int nArg; /* Number of argument slots actually used */ + char **azArg; /* Pointer to each argument, dequoted */ + int *aiOfst; /* Offset into zOrig[] for start of each arg */ + char *abQuot; /* True if the argment was originally quoted */ + } dot; #ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ @@ -21752,7 +23746,7 @@ static ShellState shellState; #endif -/* Allowed values for ShellState.autoEQP +/* Allowed values for ShellState.mode.autoEQP */ #define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ #define AUTOEQP_on 1 /* Automatic EQP is on */ @@ -21765,9 +23759,8 @@ static ShellState shellState; #define SHELL_OPEN_NORMAL 1 /* Normal database file */ #define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ #define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ -#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ -#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ -#define SHELL_OPEN_HEXDB 6 /* Use "dbtotxt" output as data source */ +#define SHELL_OPEN_DESERIALIZE 4 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 5 /* Use "dbtotxt" output as data source */ /* Allowed values for ShellState.eTraceType */ @@ -21781,15 +23774,13 @@ static ShellState shellState; ** callback limit is reached, and for each ** top-level SQL statement */ #define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ +#define SHELL_PROGRESS_TMOUT 0x08 /* Stop after tmProgress seconds */ -/* Allowed values for ShellState.eEscMode. The default value should -** be 0, so to change the default, reorder the names. +/* Names of values for Mode.spec.eEsc and Mode.spec.eText */ -#define SHELL_ESC_ASCII 0 /* Substitute ^Y for X where Y=X+0x40 */ -#define SHELL_ESC_SYMBOL 1 /* Substitute U+2400 graphics */ -#define SHELL_ESC_OFF 2 /* Send characters verbatim */ - -static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; +static const char *qrfEscNames[] = { "auto", "off", "ascii", "symbol" }; +static const char *qrfQuoteNames[] = + { "off","off","sql","hex","csv","tcl","json","relaxed"}; /* ** These are the allowed shellFlgs values @@ -21798,10 +23789,8 @@ static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; #define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ #define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ #define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ -#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ +#define SHFLG_NoErrLineno 0x00000010 /* Omit line numbers from error msgs */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ -#define SHFLG_Echo 0x00000040 /* .echo on/off, or --echo setting */ -#define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ #define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ #define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ #define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ @@ -21814,54 +23803,107 @@ static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; #define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* -** These are the allowed modes. +** These are the allowed values for Mode.eMode. There is a lot of overlap +** between these values and the Mode.spec.eStyle values, but they are not +** one-to-one, and thus need to be tracked separately. */ -#define MODE_Line 0 /* One column per line. Blank line between records */ -#define MODE_Column 1 /* One record per line in neat columns */ -#define MODE_List 2 /* One record per line with a separator */ -#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ -#define MODE_Html 4 /* Generate an XHTML table */ -#define MODE_Insert 5 /* Generate SQL "insert" statements */ -#define MODE_Quote 6 /* Quote values as for SQL */ -#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ -#define MODE_Csv 8 /* Quote strings, numbers are plain */ -#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ -#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ -#define MODE_Pretty 11 /* Pretty-print schemas */ -#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ -#define MODE_Json 13 /* Output JSON */ -#define MODE_Markdown 14 /* Markdown formatting */ -#define MODE_Table 15 /* MySQL-style table formatting */ -#define MODE_Box 16 /* Unicode box-drawing characters */ -#define MODE_Count 17 /* Output only a count of the rows of output */ -#define MODE_Off 18 /* No query output shown */ -#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ -#define MODE_Www 20 /* Full web-page output */ +#define MODE_Ascii 0 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Box 1 /* Unicode box-drawing characters */ +#define MODE_C 2 /* Comma-separated list of C-strings */ +#define MODE_Column 3 /* One record per line in neat columns */ +#define MODE_Count 4 /* Output only a count of the rows of output */ +#define MODE_Csv 5 /* Quote strings, numbers are plain */ +#define MODE_Html 6 /* Generate an XHTML table */ +#define MODE_Insert 7 /* Generate SQL "insert" statements */ +#define MODE_JAtom 8 /* Comma-separated list of JSON atoms */ +#define MODE_JObject 9 /* One JSON object per row */ +#define MODE_Json 10 /* Output JSON */ +#define MODE_Line 11 /* One column per line. Blank line between records */ +#define MODE_List 12 /* One record per line with a separator */ +#define MODE_Markdown 13 /* Markdown formatting */ +#define MODE_Off 14 /* No query output shown */ +#define MODE_Psql 15 /* Similar to psql */ +#define MODE_QBox 16 /* BOX with SQL-quoted content */ +#define MODE_Quote 17 /* Quote values as for SQL */ +#define MODE_Split 18 /* Split-column mode */ +#define MODE_Table 19 /* MySQL-style table formatting */ +#define MODE_Tabs 20 /* Tab-separated values */ +#define MODE_Tcl 21 /* Space-separated list of TCL strings */ +#define MODE_Www 22 /* Full web-page output */ + +#define MODE_BUILTIN 22 /* Maximum built-in mode */ +#define MODE_BATCH 50 /* Default mode for batch processing */ +#define MODE_TTY 51 /* Default mode for interactive processing */ +#define MODE_USER 75 /* First user-defined mode */ +#define MODE_N_USER 25 /* Maximum number of user-defined modes */ -static const char *modeDescr[] = { - "line", - "column", - "list", - "semi", - "html", - "insert", - "quote", - "tcl", - "csv", - "explain", - "ascii", - "prettyprint", - "eqp", - "json", - "markdown", - "table", - "box", - "count", - "off", - "scanexp", - "www", +/* +** Information about built-in display modes +*/ +typedef struct ModeInfo ModeInfo; +struct ModeInfo { + char zName[9]; /* Symbolic name of the mode */ + unsigned char eCSep; /* Column separator */ + unsigned char eRSep; /* Row separator */ + unsigned char eNull; /* Null representation */ + unsigned char eText; /* Default text encoding */ + unsigned char eHdr; /* Default header encoding. */ + unsigned char eBlob; /* Default blob encoding. */ + unsigned char bHdr; /* Show headers by default. 0: n/a, 1: no 2: yes */ + unsigned char eStyle; /* Underlying QRF style */ + unsigned char eCx; /* 0: other, 1: line, 2: columnar */ + unsigned char mFlg; /* Flags. 1=border-off 2=split-column */ }; +/* String constants used by built-in modes */ +static const char *aModeStr[] = + /* 0 1 2 3 4 5 6 7 8 */ + { 0, "\n", "|", " ", ",", "\r\n", "\036", "\037", "\t", + "", "NULL", "null", "\"\"", ": ", }; + /* 9 10 11 12 13 */ + +static const ModeInfo aModeInfo[] = { +/* zName eCSep eRSep eNull eText eHdr eBlob bHdr eStyle eCx mFlg */ + { "ascii", 7, 6, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "box", 0, 0, 9, 1, 1, 0, 2, 1, 2, 0 }, + { "c", 4, 1, 10, 5, 5, 4, 1, 12, 0, 0 }, + { "column", 0, 0, 9, 1, 1, 0, 2, 2, 2, 0 }, + { "count", 0, 0, 0, 0, 0, 0, 0, 3, 0, 0 }, + { "csv", 4, 5, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "html", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 }, + { "insert", 0, 0, 10, 2, 2, 0, 1, 8, 0, 0 }, + { "jatom", 4, 1, 11, 6, 6, 0, 1, 12, 0, 0 }, + { "jobject", 0, 1, 11, 6, 6, 0, 0, 10, 0, 0 }, + { "json", 0, 0, 11, 6, 6, 0, 0, 9, 0, 0 }, + { "line", 13, 1, 9, 1, 1, 0, 0, 11, 1, 0 }, + { "list", 2, 1, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "markdown", 0, 0, 9, 1, 1, 0, 2, 13, 2, 0 }, + { "off", 0, 0, 0, 0, 0, 0, 0, 14, 0, 0 }, + { "psql", 0, 0, 9, 1, 1, 0, 2, 19, 2, 1 }, + { "qbox", 0, 0, 10, 2, 1, 0, 2, 1, 2, 0 }, + { "quote", 4, 1, 10, 2, 2, 0, 1, 12, 0, 0 }, + { "split", 0, 0, 9, 1, 1, 0, 1, 2, 2, 2 }, + { "table", 0, 0, 9, 1, 1, 0, 2, 19, 2, 0 }, + { "tabs", 8, 1, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "tcl", 3, 1, 12, 5, 5, 4, 1, 12, 0, 0 }, + { "www", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 } +}; /* | / / | / / | | \ + ** | / / | / / | | \_ 2: columnar + ** Index into aModeStr[] | / / | | 1: line + ** | / / | | 0: other + ** | / / | \ + ** text encoding |/ | show | \ + ** v-------------------' | hdrs? | The QRF style + ** 0: n/a blob | v-----' + ** 1: plain v_---------' 0: n/a + ** 2: sql 0: auto 1: no + ** 3: csv 1: as-text 2: yes + ** 4: html 2: sql + ** 5: c 3: hex + ** 6: json 4: c + ** 5: json + ** 6: size + ******************************************************************/ /* ** These are the column/row/line separators used by the various ** import/export modes. @@ -21876,10 +23918,1133 @@ static const char *modeDescr[] = { #define SEP_Record "\x1E" /* -** Limit input nesting via .read or any other input redirect. -** It's not too expensive, so a generous allowance can be made. +** Default values for the various QRF limits */ -#define MAX_INPUT_NESTING 25 +#ifndef DFLT_CHAR_LIMIT +# define DFLT_CHAR_LIMIT 300 +#endif +#ifndef DFLT_LINE_LIMIT +# define DFLT_LINE_LIMIT 5 +#endif +#ifndef DFLT_TITLE_LIMIT +# define DFLT_TITLE_LIMIT 20 +#endif +#ifndef DFLT_MULTI_INSERT +# define DFLT_MULTI_INSERT 3000 +#endif + +/* +** If the following flag is set, then command execution stops +** at an error if we are not interactive. +*/ +static int bail_on_error = 0; + +/* +** Treat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. +*/ +static int stdin_is_interactive = 1; + +/* +** Treat stdout like a TTY if true. +*/ +static int stdout_is_console = 1; + +/* +** Use this value as the width of the output device. Or, figure it +** out at runtime if the value is negative. Or use a default width +** if this value is zero. +*/ +static int stdout_tty_width = -1; + +/* +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. +*/ +static sqlite3 *globalDb = 0; + +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; + +/* +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. +*/ +static char *Argv0; + +/* +** Prompt strings. Initialized in main. Settable with +** .prompt main continue +*/ +#define PROMPT_LEN_MAX 128 +/* First line prompt. default: "sqlite> " */ +static char mainPrompt[PROMPT_LEN_MAX]; +/* Continuation prompt. default: " ...> " */ +static char continuePrompt[PROMPT_LEN_MAX]; + +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + +/* +** Output routines that are able to redirect to memory rather than +** doing actually I/O. +** Works like. +** -------------- +** cli_printf(FILE*, const char*, ...); fprintf() +** cli_puts(const char*, FILE*); fputs() +** cli_vprintf(FILE*, const char*, va_list); vfprintf() +** +** These are just thin wrappers with the following added semantics: +** If the file-scope variable cli_output_capture is not NULL, and +** if the FILE* argument is stdout or stderr, then rather than +** writing to stdout/stdout, append the text to the cli_output_capture +** variable. +** +** The cli_exit(int) routine works like exit() except that it +** first dumps any capture output to stdout. +*/ +static sqlite3_str *cli_output_capture = 0; +static int cli_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + int rc; + va_start(ap,zFormat); + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + rc = 1; + }else{ + rc = sqlite3_vfprintf(out, zFormat, ap); + } + va_end(ap); + return rc; +} +static int cli_puts(const char *zText, FILE *out){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_appendall(cli_output_capture, zText); + return 1; + } + return sqlite3_fputs(zText, out); +} +#if 0 /* Not currently used - available if we need it later */ +static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + return 1; + }else{ + return sqlite3_vfprintf(out, zFormat, ap); + } +} +#endif +static void cli_exit(int rc){ + if( cli_output_capture ){ + char *z = sqlite3_str_finish(cli_output_capture); + sqlite3_fputs(z, stdout); + fflush(stdout); + } + exit(rc); +} + + +#define eputz(z) cli_puts(z,stderr) +#define sputz(fp,z) cli_puts(z,fp) + +/* A version of strcmp() that works with NULL values */ +static int cli_strcmp(const char *a, const char *b){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strcmp(a,b); +} +static int cli_strncmp(const char *a, const char *b, size_t n){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strncmp(a,b,n); +} + +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ +static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; /* Never actually happens */ + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif +} + + + +/* This is variant of the standard-library strncpy() routine with the +** one change that the destination string is always zero-terminated, even +** if there is no zero-terminator in the first n-1 characters of the source +** string. +*/ +static char *shell_strncpy(char *dest, const char *src, size_t n){ + size_t i; + for(i=0; i<n-1 && src[i]!=0; i++) dest[i] = src[i]; + dest[i] = 0; + return dest; +} + +/* +** strcpy() workalike to squelch an unwarranted link-time warning +** from OpenBSD. +*/ +static void shell_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} + +/* +** Optionally disable dynamic continuation prompt. +** Unless disabled, the continuation prompt shows open SQL lexemes if any, +** or open parentheses level if non-zero, or continuation prompt as set. +** This facility interacts with the scanner and process_input() where the +** below 5 macros are used. +*/ +#ifdef SQLITE_OMIT_DYNAPROMPT +# define CONTINUATION_PROMPT continuePrompt +# define CONTINUE_PROMPT_RESET +# define CONTINUE_PROMPT_AWAITS(p,s) +# define CONTINUE_PROMPT_AWAITC(p,c) +# define CONTINUE_PAREN_INCR(p,n) +# define CONTINUE_PROMPT_PSTATE 0 +typedef void *t_NoDynaPrompt; +# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt +#else +# define CONTINUATION_PROMPT dynamicContinuePrompt() +# define CONTINUE_PROMPT_RESET \ + do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0) +# define CONTINUE_PROMPT_AWAITS(p,s) \ + if(p && stdin_is_interactive) setLexemeOpen(p, s, 0) +# define CONTINUE_PROMPT_AWAITC(p,c) \ + if(p && stdin_is_interactive) setLexemeOpen(p, 0, c) +# define CONTINUE_PAREN_INCR(p,n) \ + if(p && stdin_is_interactive) (trackParenLevel(p,n)) +# define CONTINUE_PROMPT_PSTATE (&dynPrompt) +typedef struct DynaPrompt *t_DynaPromptRef; +# define SCAN_TRACKER_REFTYPE t_DynaPromptRef + +static struct DynaPrompt { + char dynamicPrompt[PROMPT_LEN_MAX]; + char acAwait[2]; + int inParenLevel; + char *zScannerAwaits; +} dynPrompt = { {0}, {0}, 0, 0 }; + +/* Record parenthesis nesting level change, or force level to 0. */ +static void trackParenLevel(struct DynaPrompt *p, int ni){ + p->inParenLevel += ni; + if( ni==0 ) p->inParenLevel = 0; + p->zScannerAwaits = 0; +} + +/* Record that a lexeme is opened, or closed with args==0. */ +static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ + if( s!=0 || c==0 ){ + p->zScannerAwaits = s; + p->acAwait[0] = 0; + }else{ + p->acAwait[0] = c; + p->zScannerAwaits = p->acAwait; + } +} + +/* Upon demand, derive the continuation prompt to display. */ +static char *dynamicContinuePrompt(void){ + if( continuePrompt[0]==0 + || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ + return continuePrompt; + }else{ + if( dynPrompt.zScannerAwaits ){ + size_t ncp = strlen(continuePrompt); + size_t ndp = strlen(dynPrompt.zScannerAwaits); + if( ndp > ncp-3 ) return continuePrompt; + shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + }else{ + if( dynPrompt.inParenLevel>9 ){ + shell_strncpy(dynPrompt.dynamicPrompt, "(..", 4); + }else if( dynPrompt.inParenLevel<0 ){ + shell_strncpy(dynPrompt.dynamicPrompt, ")x!", 4); + }else{ + shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); + dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + } + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + } + } + return dynPrompt.dynamicPrompt; +} +#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ + +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + eputz("Error: out of memory\n"); + cli_exit(1); +} + +/* Check a pointer to see if it is NULL. If it is NULL, exit with an +** out-of-memory error. +*/ +static void shell_check_oom(const void *p){ + if( p==0 ) shell_out_of_memory(); +} + +/* +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + cli_printf(iotrace, "%s", z); + sqlite3_free(z); +} +#endif + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + size_t n; + if( z==0 ) return 0; + n = strlen(z); + return n>0x3fffffff ? 0x3fffffff : (int)n; +} + +/* +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. +*/ +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = sqlite3_fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; + } + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return sqlite3_fopen(zFile, "rb"); + }else{ + return 0; + } +#endif +#undef STAT_CHR_SRC +} + +/* +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails, or if the length of the line is longer than about a gigabyte. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. +*/ +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; + + while( 1 ){ + if( n+100>nLine ){ + if( nLine>=1073741773 ){ + free(zLine); + return 0; + } + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + shell_check_oom(zLine); + } + if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; + zLine[n] = 0; + break; + } + } + return zLine; +} + +/* +** Retrieve a single line of input text. +** +** If in==0 then read from standard input and prompt before each line. +** If isContinuation is true, then a continuation prompt is appropriate. +** If isContinuation is zero, then the main prompt should be used. +** +** If zPrior is not NULL then it is a buffer from a prior call to this +** routine that can be reused. +** +** The result is stored in space obtained from malloc() and must either +** be freed by the caller or else passed back into this routine via the +** zPrior argument for reuse. +*/ +#ifndef SQLITE_SHELL_FIDDLE +static char *one_input_line(ShellState *p, char *zPrior, int isContinuation){ + char *zPrompt; + char *zResult; + FILE *in = p->in; + if( in!=0 ){ + zResult = local_getline(zPrior, in); + }else{ + zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; +#if SHELL_USE_LOCAL_GETLINE + sputz(stdout, zPrompt); + fflush(stdout); + do{ + zResult = local_getline(zPrior, stdin); + zPrior = 0; + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + if( zResult==0 ) sqlite3_sleep(50); + }while( zResult==0 && seenInterrupt>0 ); +#else + free(zPrior); + zResult = shell_readline(zPrompt); + while( zResult==0 ){ + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + sqlite3_sleep(50); + if( seenInterrupt==0 ) break; + zResult = shell_readline(""); + } + if( zResult && *zResult ) shell_add_history(zResult); +#endif + } + return zResult; +} +#endif /* !SQLITE_SHELL_FIDDLE */ + +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +** +** If the value specified by zArg is outside the range of values that +** can be represented using a 64-bit twos-complement integer, then return +** the nearest representable value. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_uint64 v = 0; + static const struct { char *zSuffix; unsigned int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + if( v > 0x0fffffffffffffffULL ) goto integer_overflow; + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + if( v>=922337203685477580LL ){ + if( v>922337203685477580LL || zArg[0]>='8' ) goto integer_overflow; + } + v = v*10 + (zArg[0] - '0'); + zArg++; + } + } + for(i=0; i<ArraySize(aMult); i++){ + if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){ + if( 0x7fffffffffffffffULL/aMult[i].iMult < v ) goto integer_overflow; + v *= aMult[i].iMult; + break; + } + } + if( isNeg && v>0x7fffffffffffffffULL ) goto integer_overflow; + return isNeg? -(sqlite3_int64)v : (sqlite3_int64)v; +integer_overflow: + return isNeg ? (i64)0x8000000000000000LL : 0x7fffffffffffffffLL; +} + +/* +** A variable length string to which one can append text. +*/ +typedef struct ShellText ShellText; +struct ShellText { + char *zTxt; /* The text */ + i64 n; /* Number of bytes of zTxt[] actually used */ + i64 nAlloc; /* Number of bytes allocated for zTxt[] */ +}; + +/* +** Initialize and destroy a ShellText object +*/ +static void initText(ShellText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(ShellText *p){ + sqlite3_free(p->zTxt); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, const char *zAppend, char quote){ + i64 len; + i64 i; + i64 nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; i<nAppend; i++){ + if( zAppend[i]==quote ) len++; + } + } + + if( p->zTxt==0 || p->n+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->zTxt = sqlite3_realloc64(p->zTxt, p->nAlloc); + shell_check_oom(p->zTxt); + } + + if( quote ){ + char *zCsr = p->zTxt+p->n; + *zCsr++ = quote; + for(i=0; i<nAppend; i++){ + *zCsr++ = zAppend[i]; + if( zAppend[i]==quote ) *zCsr++ = quote; + } + *zCsr++ = quote; + p->n = (i64)(zCsr - p->zTxt); + *zCsr = '\0'; + }else{ + memcpy(p->zTxt+p->n, zAppend, nAppend); + p->n += nAppend; + p->zTxt[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( zName==0 ) return '"'; + if( !IsAlpha(zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !IsAlnum(zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} + +/* +** Construct a fake object name and column list to describe the structure +** of the view, virtual table, or table valued function zSchema.zName. +** +** The returned string comes from sqlite3_mprintf() and should be freed +** by the caller using sqlite3_free(). +*/ +static char *shellFakeSchema( + sqlite3 *db, /* The database connection containing the vtab */ + const char *zSchema, /* Schema of the database holding the vtab */ + const char *zName /* The name of the virtual table */ +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + ShellText s; + char cQuote; + char *zDiv = "("; + int nRow = 0; + + zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", + zSchema ? zSchema : "main", zName); + shell_check_oom(zSql); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + initText(&s); + if( zSchema ){ + cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; + appendText(&s, zSchema, cQuote); + appendText(&s, ".", 0); + } + cQuote = quoteChar(zName); + appendText(&s, zName, cQuote); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); + nRow++; + appendText(&s, zDiv, 0); + zDiv = ","; + if( zCol==0 ) zCol = ""; + cQuote = quoteChar(zCol); + appendText(&s, zCol, cQuote); + } + appendText(&s, ")", 0); + sqlite3_finalize(pStmt); + if( nRow==0 ){ + freeText(&s); + s.zTxt = 0; + } + return s.zTxt; +} + +/* +** SQL function: strtod(X) +** +** Use the C-library strtod() function to convert string X into a double. +** Used for comparing the accuracy of SQLite's internal text-to-float conversion +** routines against the C-library. +*/ +static void shellStrtod( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + char *z = (char*)sqlite3_value_text(apVal[0]); + UNUSED_PARAMETER(nVal); + if( z==0 ) return; + sqlite3_result_double(pCtx, strtod(z,0)); +} + +/* +** SQL function: dtostr(X) +** +** Use the C-library printf() function to convert real value X into a string. +** Used for comparing the accuracy of SQLite's internal float-to-text conversion +** routines against the C-library. +*/ +static void shellDtostr( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + double r = sqlite3_value_double(apVal[0]); + int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + char z[400]; + if( n<1 ) n = 1; + if( n>350 ) n = 350; + sprintf(z, "%#+.*e", n, r); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); +} + +/* +** SQL function: shell_add_schema(S,X) +** +** Add the schema name X to the CREATE statement in S and return the result. +** Examples: +** +** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); +** +** Also works on +** +** CREATE INDEX +** CREATE UNIQUE INDEX +** CREATE VIEW +** CREATE TRIGGER +** CREATE VIRTUAL TABLE +** +** This UDF is used by the .schema command to insert the schema name of +** attached databases into the middle of the sqlite_schema.sql field. +*/ +static void shellAddSchemaName( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + static const char *aPrefix[] = { + "TABLE", + "INDEX", + "UNIQUE INDEX", + "VIEW", + "TRIGGER", + "VIRTUAL TABLE" + }; + int i = 0; + const char *zIn = (const char*)sqlite3_value_text(apVal[0]); + const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); + const char *zName = (const char*)sqlite3_value_text(apVal[2]); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + UNUSED_PARAMETER(nVal); + if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ + for(i=0; i<ArraySize(aPrefix); i++){ + int n = strlen30(aPrefix[i]); + if( cli_strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){ + char *z = 0; + char *zFake = 0; + if( zSchema ){ + char cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")!=0 ){ + z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8); + }else{ + z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8); + } + } + if( zName + && aPrefix[i][0]=='V' + && (zFake = shellFakeSchema(db, zSchema, zName))!=0 + ){ + if( z==0 ){ + z = sqlite3_mprintf("%s\n/* %s */", zIn, zFake); + }else{ + z = sqlite3_mprintf("%z\n/* %s */", z, zFake); + } + sqlite3_free(zFake); + } + if( z ){ + sqlite3_result_text(pCtx, z, -1, sqlite3_free); + return; + } + } + } + } + sqlite3_result_value(pCtx, apVal[0]); +} + + +/************************* BEGIN PERFORMANCE TIMER *****************************/ +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include <sys/time.h> +#include <sys/resource.h> +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + +/* +** Begin timing an operation +*/ +static void beginTimer(ShellState *p){ + if( p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* Return the time since the start of the timer in +** seconds. */ +static double elapseTime(ShellState *NotUsed){ + (void)NotUsed; + if( iBegin==0 ) return 0.0; + return (timeOfDay() - iBegin)*0.000001; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + +/* +** Print the timing results. +*/ +static void endTimer(ShellState *p){ + if( p->enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + p->prevTimer = (iEnd - iBegin)*0.000001; + cli_printf(p->out, "Run Time: real %.6f user %.6f sys %.6f\n", + p->prevTimer, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + if( p->enableTimer==1 ) p->enableTimer = 0; + iBegin = 0; + } +} + +#define BEGIN_TIMER(X) beginTimer(X) +#define END_TIMER(X) endTimer(X) +#define ELAPSE_TIME(X) elapseTime(X) +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + +/* +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). +*/ +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } + } + return 0; +} + +/* +** Begin timing an operation +*/ +static void beginTimer(ShellState *p){ + if( (p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0) + && getProcessTimesAddr + ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); + } +} + +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* Return the time since the start of the timer in +** seconds. */ +static double elapseTime(ShellState *NotUsed){ + (void)NotUsed; + if( ftWallBegin==0 ) return 0.0; + return (timeOfDay() - ftWallBegin)*0.000001; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + +/* +** Print the timing results. +*/ +static void endTimer(ShellState *p){ + if( p->enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); + p->prevTimer = (ftWallEnd - ftWallBegin)*0.000001; +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + cli_printf(p->out, "Run Time: real %.6f user %f sys %f\n", + p->prevTimer, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + cli_printf(p->out, "Run Time: real %.3f user %.3f sys %.3f\n", + p->prevTimer, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif + if( p->enableTimer==1 ) p->enableTimer = 0; + ftWallBegin = 0; + } +} + +#define BEGIN_TIMER(X) beginTimer(X) +#define ELAPSE_TIME(X) elapseTime(X) +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER hasTimer() + +#else +#define BEGIN_TIMER(X) /* no-op */ +#define ELAPSE_TIME(X) 0.0 +#define END_TIMER(X) /*no-op*/ +#define HAS_TIMER 0 +#endif +/************************* END PERFORMANCE TIMER ******************************/ + +/* +** Clear a display mode, freeing any allocated memory that it +** contains. +*/ +static void modeFree(Mode *p){ + u8 autoExplain = p->autoExplain; + free(p->spec.aWidth); + free(p->spec.aAlign); + free(p->spec.zColumnSep); + free(p->spec.zRowSep); + free(p->spec.zTableName); + free(p->spec.zNull); + memset(p, 0, sizeof(*p)); + p->spec.iVersion = 1; + p->autoExplain = autoExplain; +} + +/* +** Duplicate Mode pSrc into pDest. pDest is assumed to be +** uninitialized prior to invoking this routine. +*/ +static void modeDup(Mode *pDest, Mode *pSrc){ + memcpy(pDest, pSrc, sizeof(*pDest)); + if( pDest->spec.aWidth ){ + size_t sz = sizeof(pSrc->spec.aWidth[0]) * pSrc->spec.nWidth; + pDest->spec.aWidth = malloc( sz ); + if( pDest->spec.aWidth ){ + memcpy(pDest->spec.aWidth, pSrc->spec.aWidth, sz); + }else{ + pDest->spec.nWidth = 0; + } + } + if( pDest->spec.aAlign ){ + size_t sz = sizeof(pSrc->spec.aAlign[0]) * pSrc->spec.nAlign; + pDest->spec.aAlign = malloc( sz ); + if( pDest->spec.aAlign ){ + memcpy(pDest->spec.aAlign, pSrc->spec.aAlign, sz); + }else{ + pDest->spec.nAlign = 0; + } + } + if( pDest->spec.zColumnSep ){ + pDest->spec.zColumnSep = strdup(pSrc->spec.zColumnSep); + } + if( pDest->spec.zRowSep ){ + pDest->spec.zRowSep = strdup(pSrc->spec.zRowSep); + } + if( pDest->spec.zTableName ){ + pDest->spec.zTableName = strdup(pSrc->spec.zTableName); + } + if( pDest->spec.zNull ){ + pDest->spec.zNull = strdup(pSrc->spec.zNull); + } +} + +/* +** Set a string value to a copy of the zNew string in memory +** obtained from system malloc(). +*/ +static void modeSetStr(char **az, const char *zNew){ + free(*az); + if( zNew==0 ){ + *az = 0; + }else{ + size_t n = strlen(zNew); + *az = malloc( n+1 ); + if( *az ){ + memcpy(*az, zNew, n+1 ); + } + } +} + +/* +** Change the mode to eMode +*/ +static void modeChange(ShellState *p, unsigned char eMode){ + const ModeInfo *pI; + if( eMode<ArraySize(aModeInfo) ){ + Mode *pM = &p->mode; + pI = &aModeInfo[eMode]; + pM->eMode = eMode; + if( pI->eCSep ) modeSetStr(&pM->spec.zColumnSep, aModeStr[pI->eCSep]); + if( pI->eRSep ) modeSetStr(&pM->spec.zRowSep, aModeStr[pI->eRSep]); + if( pI->eNull ) modeSetStr(&pM->spec.zNull, aModeStr[pI->eNull]); + pM->spec.eText = pI->eText; + pM->spec.eBlob = pI->eBlob; + if( (pM->mFlags & MFLG_HDR)==0 ){ + pM->spec.bTitles = pI->bHdr; + } + pM->spec.eTitle = pI->eHdr; + if( pI->mFlg & 0x01 ){ + pM->spec.bBorder = QRF_No; + }else{ + pM->spec.bBorder = QRF_Auto; + } + if( pI->mFlg & 0x02 ){ + pM->spec.bSplitColumn = QRF_Yes; + pM->bAutoScreenWidth = 1; + }else{ + pM->spec.bSplitColumn = QRF_No; + } + }else if( eMode>=MODE_USER && eMode-MODE_USER<p->nSavedModes ){ + modeFree(&p->mode); + modeDup(&p->mode, &p->aSavedModes[eMode-MODE_USER].mode); + }else if( eMode==MODE_BATCH ){ + u8 mFlags = p->mode.mFlags; + modeFree(&p->mode); + modeChange(p, MODE_List); + p->mode.mFlags = mFlags; + }else if( eMode==MODE_TTY ){ + u8 mFlags = p->mode.mFlags; + modeFree(&p->mode); + modeChange(p, MODE_QBox); + p->mode.bAutoScreenWidth = 1; + p->mode.spec.eText = QRF_TEXT_Relaxed; + p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; + p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; + p->mode.spec.bTextJsonb = QRF_Yes; + p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + p->mode.spec.nMultiInsert = DFLT_MULTI_INSERT; + p->mode.mFlags = mFlags; + } +} + +/* +** Set the mode to the default. It assumed that the mode has +** already been freed and zeroed prior to calling this routine. +*/ +static void modeDefault(ShellState *p){ + p->mode.spec.iVersion = 1; + p->mode.autoExplain = 1; + if( stdin_is_interactive || stdout_is_console ){ + modeChange(p, MODE_TTY); + }else{ + modeChange(p, MODE_BATCH); + } +} + +/* +** Find the number of a display mode given its name. Return -1 if +** the name does not match any mode. +** +** Saved modes are also searched if p!=NULL. The number returned +** for a saved mode is the index into the p->aSavedModes[] array +** plus MODE_USER. +** +** Two special mode names are also available: "batch" and "tty". +** evaluate to the default mode for batch operation and interactive +** operation on a TTY, respectively. +*/ +static int modeFind(ShellState *p, const char *zName){ + int i; + for(i=0; i<ArraySize(aModeInfo); i++){ + if( cli_strcmp(aModeInfo[i].zName,zName)==0 ) return i; + } + for(i=0; i<p->nSavedModes; i++){ + if( cli_strcmp(p->aSavedModes[i].zTag,zName)==0 ) return i+MODE_USER; + } + if( strcmp(zName,"batch")==0 ) return MODE_BATCH; + if( strcmp(zName,"tty")==0 ) return MODE_TTY; + return -1; +} + +/* +** Save or restore the current output mode +*/ +static void modePush(ShellState *p){ + if( p->nPopMode==0 ){ + modeFree(&p->modePrior); + modeDup(&p->modePrior,&p->mode); + } +} +static void modePop(ShellState *p){ + if( p->modePrior.spec.iVersion>0 ){ + modeFree(&p->mode); + p->mode = p->modePrior; + memset(&p->modePrior, 0, sizeof(p->modePrior)); + } +} + /* ** A callback for the sqlite3_log() interface. @@ -21887,7 +25052,7 @@ static const char *modeDescr[] = { static void shellLog(void *pArg, int iErrCode, const char *zMsg){ ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; - sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + cli_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); } @@ -21904,11 +25069,27 @@ static void shellPutsFunc( ){ ShellState *p = (ShellState*)sqlite3_user_data(pCtx); (void)nVal; - sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + cli_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); sqlite3_result_value(pCtx, apVal[0]); } /* +** Compute the name of the location of an input error in memory +** obtained from sqlite3_malloc(). +*/ +static char *shellErrorLocation(ShellState *p){ + char *zLoc; + if( p->zErrPrefix ){ + zLoc = sqlite3_mprintf("%s", p->zErrPrefix); + }else if( p->zInFile==0 || strcmp(p->zInFile,"<stdin>")==0){ + zLoc = sqlite3_mprintf("line %lld:", p->lineno); + }else{ + zLoc = sqlite3_mprintf("%s:%lld:", p->zInFile, p->lineno); + } + return zLoc; +} + +/* ** If in safe mode, print an error message described by the arguments ** and exit immediately. */ @@ -21920,15 +25101,51 @@ static void failIfSafeMode( if( p->bSafeMode ){ va_list ap; char *zMsg; + char *zLoc = shellErrorLocation(p); va_start(ap, zErrMsg); zMsg = sqlite3_vmprintf(zErrMsg, ap); va_end(ap); - sqlite3_fprintf(stderr, "line %d: %s\n", p->lineno, zMsg); - exit(1); + cli_printf(stderr, "%s %s\n", zLoc, zMsg); + cli_exit(1); } } /* +** Issue an error message from a dot-command. +*/ +static void dotCmdError( + ShellState *p, /* Shell state */ + int iArg, /* Index of argument on which error occurred */ + const char *zBrief, /* Brief (<20 character) error description */ + const char *zDetail, /* Error details */ + ... +){ + FILE *out = stderr; + char *zLoc = shellErrorLocation(p); + if( zBrief!=0 && iArg>=0 && iArg<p->dot.nArg ){ + int i = p->dot.aiOfst[iArg]; + int nPrompt = strlen30(zBrief) + 5; + cli_printf(out, "%s %s\n", zLoc, p->dot.zOrig); + if( i > nPrompt ){ + cli_printf(out, "%s %*s%s ---^\n", zLoc, 1+i-nPrompt, "", zBrief); + }else{ + cli_printf(out, "%s %*s^--- %s\n", zLoc, i, "", zBrief); + } + } + if( zDetail ){ + char *zMsg; + va_list ap; + va_start(ap, zDetail); + zMsg = sqlite3_vmprintf(zDetail,ap); + va_end(ap); + cli_printf(out,"%s %s\n", zLoc, zMsg); + sqlite3_free(zMsg); + } + sqlite3_free(zLoc); +} + + +/* ** SQL function: edit(VALUE) ** edit(VALUE,EDITOR) ** @@ -22073,27 +25290,11 @@ edit_func_end: #endif /* SQLITE_NOHAVE_SYSTEM */ /* -** Save or restore the current output mode -*/ -static void outputModePush(ShellState *p){ - p->modePrior = p->mode; - p->priorShFlgs = p->shellFlgs; - memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); - memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); -} -static void outputModePop(ShellState *p){ - p->mode = p->modePrior; - p->shellFlgs = p->priorShFlgs; - memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); - memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); -} - -/* ** Set output mode to text or binary for Windows. */ static void setCrlfMode(ShellState *p){ #ifdef _WIN32 - if( p->crlfMode ){ + if( p->mode.mFlags & MFLG_CRLF ){ sqlite3_fsetmode(p->out, _O_TEXT); }else{ sqlite3_fsetmode(p->out, _O_BINARY); @@ -22104,126 +25305,6 @@ static void setCrlfMode(ShellState *p){ } /* -** Output the given string as a hex-encoded blob (eg. X'1234' ) -*/ -static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ - int i; - unsigned char *aBlob = (unsigned char*)pBlob; - - char *zStr = sqlite3_malloc(nBlob*2 + 1); - shell_check_oom(zStr); - - for(i=0; i<nBlob; i++){ - static const char aHex[] = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - zStr[i*2] = aHex[ (aBlob[i] >> 4) ]; - zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; - } - zStr[i*2] = '\0'; - - sqlite3_fprintf(out, "X'%s'", zStr); - sqlite3_free(zStr); -} - -/* -** Output the given string as a quoted string using SQL quoting conventions: -** -** (1) Single quotes (') within the string are doubled -** (2) The whle string is enclosed in '...' -** (3) Control characters other than \n, \t, and \r\n are escaped -** using \u00XX notation and if such substitutions occur, -** the whole string is enclosed in unistr('...') instead of '...'. -** -** Step (3) is omitted if the control-character escape mode is OFF. -** -** See also: output_quoted_escaped_string() which does the same except -** that it does not make exceptions for \n, \t, and \r\n in step (3). -*/ -static void output_quoted_string(ShellState *p, const char *zInX){ - int i; - int needUnistr = 0; - int needDblQuote = 0; - const unsigned char *z = (const unsigned char*)zInX; - unsigned char c; - FILE *out = p->out; - sqlite3_fsetmode(out, _O_BINARY); - if( z==0 ) return; - for(i=0; (c = z[i])!=0; i++){ - if( c=='\'' ){ needDblQuote = 1; } - if( c>0x1f ) continue; - if( c=='\t' || c=='\n' ) continue; - if( c=='\r' && z[i+1]=='\n' ) continue; - needUnistr = 1; - break; - } - if( (needDblQuote==0 && needUnistr==0) - || (needDblQuote==0 && p->eEscMode==SHELL_ESC_OFF) - ){ - sqlite3_fprintf(out, "'%s'",z); - }else if( p->eEscMode==SHELL_ESC_OFF ){ - char *zEncoded = sqlite3_mprintf("%Q", z); - sqlite3_fputs(zEncoded, out); - sqlite3_free(zEncoded); - }else{ - if( needUnistr ){ - sqlite3_fputs("unistr('", out); - }else{ - sqlite3_fputs("'", out); - } - while( *z ){ - for(i=0; (c = z[i])!=0; i++){ - if( c=='\'' ) break; - if( c>0x1f ) continue; - if( c=='\t' || c=='\n' ) continue; - if( c=='\r' && z[i+1]=='\n' ) continue; - break; - } - if( i ){ - sqlite3_fprintf(out, "%.*s", i, z); - z += i; - } - if( c==0 ) break; - if( c=='\'' ){ - sqlite3_fputs("''", out); - }else{ - sqlite3_fprintf(out, "\\u%04x", c); - } - z++; - } - if( needUnistr ){ - sqlite3_fputs("')", out); - }else{ - sqlite3_fputs("'", out); - } - } - setCrlfMode(p); -} - -/* -** Output the given string as a quoted string using SQL quoting conventions. -** Additionallly , escape the "\n" and "\r" characters so that they do not -** get corrupted by end-of-line translation facilities in some operating -** systems. -** -** This is like output_quoted_string() but with the addition of the \r\n -** escape mechanism. -*/ -static void output_quoted_escaped_string(ShellState *p, const char *z){ - char *zEscaped; - sqlite3_fsetmode(p->out, _O_BINARY); - if( p->eEscMode==SHELL_ESC_OFF ){ - zEscaped = sqlite3_mprintf("%Q", z); - }else{ - zEscaped = sqlite3_mprintf("%#Q", z); - } - sqlite3_fputs(zEscaped, p->out); - sqlite3_free(zEscaped); - setCrlfMode(p); -} - -/* ** Find earliest of chars within s specified in zAny. ** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. */ @@ -22286,13 +25367,14 @@ static void output_c_string(FILE *out, const char *z){ static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - sqlite3_fputs(zq, out); + cli_puts(zq, out); + if( z==0 ) z = ""; while( *z!=0 ){ const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + cli_printf(out, "%.*s", (int)(pcEnd-z), z); } if( (c = *pcEnd)==0 ) break; ++pcEnd; @@ -22308,250 +25390,106 @@ static void output_c_string(FILE *out, const char *z){ } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_fputs(ace, out); + cli_puts(ace, out); }else if( !isprint(c&0xff) ){ - sqlite3_fprintf(out, "\\%03o", c&0xff); + cli_printf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - sqlite3_fputs(ace+1, out); + cli_puts(ace+1, out); } z = pcEnd; } - sqlite3_fputs(zq, out); + cli_puts(zq, out); } -/* -** Output the given string as quoted according to JSON quoting rules. +/* Encode input string z[] as a C-language string literal and +** append it to the sqlite3_str. If z is NULL render and empty string. */ -static void output_json_string(FILE *out, const char *z, i64 n){ - unsigned char c; +static void append_c_string(sqlite3_str *out, const char *z){ + char c; static const char *zq = "\""; static long ctrlMask = ~0L; - static const char *zDQBS = "\"\\"; - const char *pcLimit; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - if( z==0 ) z = ""; - pcLimit = z + ((n<0)? strlen(z) : (size_t)n); - sqlite3_fputs(zq, out); - while( z < pcLimit ){ - const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); - const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); - const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + sqlite3_str_appendall(out,zq); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); - z = pcEnd; + sqlite3_str_appendf(out, "%.*s", (int)(pcEnd-z), z); } - if( z >= pcLimit ) break; - c = (unsigned char)*(z++); + if( (c = *pcEnd)==0 ) break; + ++pcEnd; switch( c ){ - case '"': case '\\': + case '\\': case '"': cbsSay = (char)c; break; - case '\b': cbsSay = 'b'; break; - case '\f': cbsSay = 'f'; break; + case '\t': cbsSay = 't'; break; case '\n': cbsSay = 'n'; break; case '\r': cbsSay = 'r'; break; - case '\t': cbsSay = 't'; break; + case '\f': cbsSay = 'f'; break; default: cbsSay = 0; break; } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_fputs(ace, out); - }else if( c<=0x1f || c>=0x7f ){ - sqlite3_fprintf(out, "\\u%04x", c); + sqlite3_str_appendall(out,ace); + }else if( !isprint(c&0xff) ){ + sqlite3_str_appendf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - sqlite3_fputs(ace+1, out); - } - } - sqlite3_fputs(zq, out); -} - -/* -** Escape the input string if it is needed and in accordance with -** eEscMode. -** -** Escaping is needed if the string contains any control characters -** other than \t, \n, and \r\n -** -** If no escaping is needed (the common case) then set *ppFree to NULL -** and return the original string. If escapingn is needed, write the -** escaped string into memory obtained from sqlite3_malloc64() or the -** equivalent, and return the new string and set *ppFree to the new string -** as well. -** -** The caller is responsible for freeing *ppFree if it is non-NULL in order -** to reclaim memory. -*/ -static const char *escapeOutput( - ShellState *p, - const char *zInX, - char **ppFree -){ - i64 i, j; - i64 nCtrl = 0; - unsigned char *zIn; - unsigned char c; - unsigned char *zOut; - - - /* No escaping if disabled */ - if( p->eEscMode==SHELL_ESC_OFF ){ - *ppFree = 0; - return zInX; - } - - /* Count the number of control characters in the string. */ - zIn = (unsigned char*)zInX; - for(i=0; (c = zIn[i])!=0; i++){ - if( c<=0x1f - && c!='\t' - && c!='\n' - && (c!='\r' || zIn[i+1]!='\n') - ){ - nCtrl++; - } - } - if( nCtrl==0 ){ - *ppFree = 0; - return zInX; - } - if( p->eEscMode==SHELL_ESC_SYMBOL ) nCtrl *= 2; - zOut = sqlite3_malloc64( i + nCtrl + 1 ); - shell_check_oom(zOut); - for(i=j=0; (c = zIn[i])!=0; i++){ - if( c>0x1f - || c=='\t' - || c=='\n' - || (c=='\r' && zIn[i+1]=='\n') - ){ - continue; - } - if( i>0 ){ - memcpy(&zOut[j], zIn, i); - j += i; + sqlite3_str_appendall(out, ace+1); } - zIn += i+1; - i = -1; - switch( p->eEscMode ){ - case SHELL_ESC_SYMBOL: - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80+c; - break; - case SHELL_ESC_ASCII: - zOut[j++] = '^'; - zOut[j++] = 0x40+c; - break; - } - } - if( i>0 ){ - memcpy(&zOut[j], zIn, i); - j += i; + z = pcEnd; } - zOut[j] = 0; - *ppFree = (char*)zOut; - return (char*)zOut; + sqlite3_str_appendall(out, zq); } /* -** Output the given string with characters that are special to -** HTML escaped. +** This routine runs when the user presses Ctrl-C */ -static void output_html_string(FILE *out, const char *z){ - int i; - if( z==0 ) z = ""; - while( *z ){ - for(i=0; z[i] - && z[i]!='<' - && z[i]!='&' - && z[i]!='>' - && z[i]!='\"' - && z[i]!='\''; - i++){} - if( i>0 ){ - sqlite3_fprintf(out, "%.*s",i,z); - } - if( z[i]=='<' ){ - sqlite3_fputs("<", out); - }else if( z[i]=='&' ){ - sqlite3_fputs("&", out); - }else if( z[i]=='>' ){ - sqlite3_fputs(">", out); - }else if( z[i]=='\"' ){ - sqlite3_fputs(""", out); - }else if( z[i]=='\'' ){ - sqlite3_fputs("'", out); - }else{ - break; - } - z += i + 1; - } +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + if( ++seenInterrupt>1 ) cli_exit(1); + if( globalDb ) sqlite3_interrupt(globalDb); } -/* -** If a field contains any character identified by a 1 in the following -** array, then the string must be quoted for CSV. +/* Try to determine the screen width. Use the default if unable. */ -static const char needCsvQuote[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -}; - -/* -** Output a single term of CSV. Actually, p->colSeparator is used for -** the separator, which may or may not be a comma. p->nullValue is -** the null value. Strings are quoted if necessary. The separator -** is only issued if bSep is true. -*/ -static void output_csv(ShellState *p, const char *z, int bSep){ - if( z==0 ){ - sqlite3_fprintf(p->out, "%s",p->nullValue); +int shellScreenWidth(void){ + if( stdout_tty_width>0 ){ + return stdout_tty_width; }else{ - unsigned i; - for(i=0; z[i]; i++){ - if( needCsvQuote[((unsigned char*)z)[i]] ){ - i = 0; - break; - } +#if defined(TIOCGSIZE) + struct ttysize ts; + if( ioctl(STDIN_FILENO, TIOCGSIZE, &ts)>=0 + || ioctl(STDOUT_FILENO, TIOCGSIZE, &ts)>=0 + || ioctl(STDERR_FILENO, TIOCGSIZE, &ts)>=0 + ){ + return ts.ts_cols; } - if( i==0 || strstr(z, p->colSeparator)!=0 ){ - char *zQuoted = sqlite3_mprintf("\"%w\"", z); - shell_check_oom(zQuoted); - sqlite3_fputs(zQuoted, p->out); - sqlite3_free(zQuoted); - }else{ - sqlite3_fputs(z, p->out); +#elif defined(TIOCGWINSZ) + struct winsize ws; + if( ioctl(STDIN_FILENO, TIOCGWINSZ, &ws)>=0 + || ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)>=0 + || ioctl(STDERR_FILENO, TIOCGWINSZ, &ws)>=0 + ){ + return ws.ws_col; } +#elif defined(_WIN32) + CONSOLE_SCREEN_BUFFER_INFO csbi; + if( GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi) + || GetConsoleScreenBufferInfo(GetStdHandle(STD_ERROR_HANDLE), &csbi) + || GetConsoleScreenBufferInfo(GetStdHandle(STD_INPUT_HANDLE), &csbi) + ){ + return csbi.srWindow.Right - csbi.srWindow.Left + 1; + } +#endif +#define DEFAULT_SCREEN_WIDTH 80 + return DEFAULT_SCREEN_WIDTH; } - if( bSep ){ - sqlite3_fputs(p->colSeparator, p->out); - } -} - -/* -** This routine runs when the user presses Ctrl-C -*/ -static void interrupt_handler(int NotUsed){ - UNUSED_PARAMETER(NotUsed); - if( ++seenInterrupt>1 ) exit(1); - if( globalDb ) sqlite3_interrupt(globalDb); } #if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) @@ -22587,6 +25525,7 @@ static int safeModeAuth( "fts3_tokenizer", "load_extension", "readfile", + "realpath", "writefile", "zipfile", "zipfile_cds", @@ -22649,23 +25588,23 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - sqlite3_fprintf(p->out, "authorizer: %s", azAction[op]); + cli_printf(p->out, "authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - sqlite3_fputs(" ", p->out); + cli_puts(" ", p->out); if( az[i] ){ output_c_string(p->out, az[i]); }else{ - sqlite3_fputs("NULL", p->out); + cli_puts("NULL", p->out); } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } #endif /* -** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. +** Print a schema statement. This is helper routine to dump_callbac(). ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. @@ -22696,18 +25635,12 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + cli_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - sqlite3_fprintf(out, "%s%s", z, zTail); + cli_printf(out, "%s%s", z, zTail); } sqlite3_free(zToFree); } -static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ - char c = z[n]; - z[n] = 0; - printSchemaLine(out, z, zTail); - z[n] = c; -} /* ** Return true if string z[] has nothing but whitespace and comments to the @@ -22725,95 +25658,130 @@ static int wsToEol(const char *z){ } /* -** Add a new entry to the EXPLAIN QUERY PLAN data +** SQL Function: shell_format_schema(SQL,FLAGS) +** +** This function is internally by the CLI to assist with the +** ".schema", ".fullschema", and ".dump" commands. The first +** argument is the value from sqlite_schema.sql. The value returned +** is a modification of the input that can actually be run as SQL +** to recreate the schema object. +** +** When FLAGS is zero, the only changes is to append ";". If the +** 0x01 bit of FLAGS is set, then transformations are made to implement +** ".schema --indent". */ -static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ - EQPGraphRow *pNew; - i64 nText; - if( zText==0 ) return; - nText = strlen(zText); - if( p->autoEQPtest ){ - sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); +static void shellFormatSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + int flags; /* Value of 2nd parameter */ + const char *zSql; /* Value of 1st parameter */ + int nSql; /* Bytes of text in zSql[] */ + sqlite3_str *pOut; /* Output buffer */ + char *z; /* Writable copy of zSql */ + int i, j; /* Loop counters */ + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + int isIndex; + int isWhere = 0; + + assert( nVal==2 ); + pOut = sqlite3_str_new(sqlite3_context_db_handle(pCtx)); + nSql = sqlite3_value_bytes(apVal[0]); + zSql = (const char*)sqlite3_value_text(apVal[0]); + if( zSql==0 || zSql[0]==0 ) goto shellFormatSchema_finish; + flags = sqlite3_value_int(apVal[1]); + if( (flags & 0x01)==0 ){ + sqlite3_str_append(pOut, zSql, nSql); + sqlite3_str_append(pOut, ";", 1); + goto shellFormatSchema_finish; } - pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); - shell_check_oom(pNew); - pNew->iEqpId = iEqpId; - pNew->iParentId = p2; - memcpy(pNew->zText, zText, nText+1); - pNew->pNext = 0; - if( p->sGraph.pLast ){ - p->sGraph.pLast->pNext = pNew; - }else{ - p->sGraph.pRow = pNew; + if( sqlite3_strlike("CREATE VIEW%", zSql, 0)==0 + || sqlite3_strlike("CREATE TRIG%", zSql, 0)==0 + ){ + sqlite3_str_append(pOut, zSql, nSql); + sqlite3_str_append(pOut, ";", 1); + goto shellFormatSchema_finish; } - p->sGraph.pLast = pNew; -} - -/* -** Free and reset the EXPLAIN QUERY PLAN data that has been collected -** in p->sGraph. -*/ -static void eqp_reset(ShellState *p){ - EQPGraphRow *pRow, *pNext; - for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ - pNext = pRow->pNext; - sqlite3_free(pRow); + isIndex = sqlite3_strlike("CREATE INDEX%", zSql, 0)==0 + || sqlite3_strlike("CREATE UNIQUE INDEX%", zSql, 0)==0; + z = sqlite3_mprintf("%s", zSql); + if( z==0 ){ + sqlite3_str_free(pOut); + sqlite3_result_error_nomem(pCtx); + return; } - memset(&p->sGraph, 0, sizeof(p->sGraph)); -} - -/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after -** pOld, or return the first such line if pOld is NULL -*/ -static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ - EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; - while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; - return pRow; -} - -/* Render a single level of the graph that has iEqpId as its parent. Called -** recursively to render sublevels. -*/ -static void eqp_render_level(ShellState *p, int iEqpId){ - EQPGraphRow *pRow, *pNext; - i64 n = strlen(p->sGraph.zPrefix); - char *z; - for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ - pNext = eqp_next_row(p, iEqpId, pRow); - z = pRow->zText; - sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, - pNext ? "|--" : "`--", z); - if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ - memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); - eqp_render_level(p, pRow->iEqpId); - p->sGraph.zPrefix[n] = 0; + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( z[j-1]=='\r' ) z[j-1] = '\n'; + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; } + z[j++] = c; } -} - -/* -** Display and reset the EXPLAIN QUERY PLAN data -*/ -static void eqp_render(ShellState *p, i64 nCycle){ - EQPGraphRow *pRow = p->sGraph.pRow; - if( pRow ){ - if( pRow->zText[0]=='-' ){ - if( pRow->pNext==0 ){ - eqp_reset(p); - return; + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( cEnd!=0){ + /* No-op */ + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='-' && z[i+1]=='-' ){ + cEnd = '\n'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 && !isWhere ){ + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n", 1); + j = 0; + } + }else if( (c=='w' || c=='W') + && nParen==0 && isIndex + && sqlite3_strnicmp("WHERE",&z[i],5)==0 + && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ + isWhere = 1; + }else if( isWhere && (c=='A' || c=='a') + && nParen==0 + && sqlite3_strnicmp("AND",&z[i],3)==0 + && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n ", 5); + j = 0; + } + z[j++] = c; + if( nParen==1 && cEnd==0 + && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + && !isWhere + ){ + if( c=='\n' ) j--; + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n ", 3); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } } - sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); - p->sGraph.pRow = pRow->pNext; - sqlite3_free(pRow); - }else if( nCycle>0 ){ - sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); - }else{ - sqlite3_fputs("QUERY PLAN\n", p->out); } - p->sGraph.zPrefix[0] = 0; - eqp_render_level(p, 0); - eqp_reset(p); + z[j] = 0; } + sqlite3_str_appendall(pOut, z); + sqlite3_str_append(pOut, ";", 1); + sqlite3_free(z); + +shellFormatSchema_finish: + sqlite3_result_text(pCtx, sqlite3_str_finish(pOut), -1, sqlite3_free); } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -22823,494 +25791,27 @@ static void eqp_render(ShellState *p, i64 nCycle){ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; + if( (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 + && ELAPSE_TIME(p)>=p->tmProgress + ){ + cli_printf(p->out, "Progress timeout after %.6f seconds\n", + ELAPSE_TIME(p)); + return 1; + } if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); + cli_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); + cli_printf(p->out, "Progress %u\n", p->nProgress); } return 0; } #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ /* -** Print N dashes -*/ -static void print_dashes(FILE *out, int N){ - const char zDash[] = "--------------------------------------------------"; - const int nDash = sizeof(zDash) - 1; - while( N>nDash ){ - sqlite3_fputs(zDash, out); - N -= nDash; - } - sqlite3_fprintf(out, "%.*s", N, zDash); -} - -/* -** Print a markdown or table-style row separator using ascii-art -*/ -static void print_row_separator( - ShellState *p, - int nArg, - const char *zSep -){ - int i; - if( nArg>0 ){ - sqlite3_fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[0]+2); - for(i=1; i<nArg; i++){ - sqlite3_fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[i]+2); - } - sqlite3_fputs(zSep, p->out); - } - sqlite3_fputs("\n", p->out); -} - -/* -** This is the callback routine that the shell -** invokes for each row of a query result. -*/ -static int shell_callback( - void *pArg, - int nArg, /* Number of result columns */ - char **azArg, /* Text of each result column */ - char **azCol, /* Column names */ - int *aiType /* Column types. Might be NULL */ -){ - int i; - ShellState *p = (ShellState*)pArg; - - if( azArg==0 ) return 0; - switch( p->cMode ){ - case MODE_Count: - case MODE_Off: { - break; - } - case MODE_Line: { - int w = 5; - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - int len = strlen30(azCol[i] ? azCol[i] : ""); - if( len>w ) w = len; - } - if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); - for(i=0; i<nArg; i++){ - char *pFree = 0; - const char *pDisplay; - pDisplay = escapeOutput(p, azArg[i] ? azArg[i] : p->nullValue, &pFree); - sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i], - pDisplay, p->rowSeparator); - if( pFree ) sqlite3_free(pFree); - } - break; - } - case MODE_ScanExp: - case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; - static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; - static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; - - const int *aWidth = aExplainWidth; - const int *aMap = aExplainMap; - int nWidth = ArraySize(aExplainWidth); - int iIndent = 1; - - if( p->cMode==MODE_ScanExp ){ - aWidth = aScanExpWidth; - aMap = aScanExpMap; - nWidth = ArraySize(aScanExpWidth); - iIndent = 3; - } - if( nArg>nWidth ) nArg = nWidth; - - /* If this is the first row seen, print out the headers */ - if( p->cnt++==0 ){ - for(i=0; i<nArg; i++){ - utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); - sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); - } - for(i=0; i<nArg; i++){ - print_dashes(p->out, aWidth[i]); - sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); - } - } - - /* If there is no data, exit early. */ - if( azArg==0 ) break; - - for(i=0; i<nArg; i++){ - const char *zSep = " "; - int w = aWidth[i]; - const char *zVal = azArg[ aMap[i] ]; - if( i==nArg-1 ) w = 0; - if( zVal && strlenChar(zVal)>w ){ - w = strlenChar(zVal); - zSep = " "; - } - if( i==iIndent && p->aiIndent && p->pStmt ){ - if( p->iIndent<p->nIndent ){ - sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); - } - p->iIndent++; - } - utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); - sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); - } - break; - } - case MODE_Semi: { /* .schema and .fullschema output */ - printSchemaLine(p->out, azArg[0], ";\n"); - break; - } - case MODE_Pretty: { /* .schema and .fullschema with --indent */ - char *z; - int j; - int nParen = 0; - char cEnd = 0; - char c; - int nLine = 0; - int isIndex; - int isWhere = 0; - assert( nArg==1 ); - if( azArg[0]==0 ) break; - if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 - || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 - ){ - sqlite3_fprintf(p->out, "%s;\n", azArg[0]); - break; - } - isIndex = sqlite3_strlike("CREATE INDEX%", azArg[0], 0)==0 - || sqlite3_strlike("CREATE UNIQUE INDEX%", azArg[0], 0)==0; - z = sqlite3_mprintf("%s", azArg[0]); - shell_check_oom(z); - j = 0; - for(i=0; IsSpace(z[i]); i++){} - for(; (c = z[i])!=0; i++){ - if( IsSpace(c) ){ - if( z[j-1]=='\r' ) z[j-1] = '\n'; - if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; - }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ - j--; - } - z[j++] = c; - } - while( j>0 && IsSpace(z[j-1]) ){ j--; } - z[j] = 0; - if( strlen30(z)>=79 ){ - for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ - if( c==cEnd ){ - cEnd = 0; - }else if( c=='"' || c=='\'' || c=='`' ){ - cEnd = c; - }else if( c=='[' ){ - cEnd = ']'; - }else if( c=='-' && z[i+1]=='-' ){ - cEnd = '\n'; - }else if( c=='(' ){ - nParen++; - }else if( c==')' ){ - nParen--; - if( nLine>0 && nParen==0 && j>0 && !isWhere ){ - printSchemaLineN(p->out, z, j, "\n"); - j = 0; - } - }else if( (c=='w' || c=='W') - && nParen==0 && isIndex - && sqlite3_strnicmp("WHERE",&z[i],5)==0 - && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ - isWhere = 1; - }else if( isWhere && (c=='A' || c=='a') - && nParen==0 - && sqlite3_strnicmp("AND",&z[i],3)==0 - && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ - printSchemaLineN(p->out, z, j, "\n "); - j = 0; - } - z[j++] = c; - if( nParen==1 && cEnd==0 - && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) - && !isWhere - ){ - if( c=='\n' ) j--; - printSchemaLineN(p->out, z, j, "\n "); - j = 0; - nLine++; - while( IsSpace(z[i+1]) ){ i++; } - } - } - z[j] = 0; - } - printSchemaLine(p->out, z, ";\n"); - sqlite3_free(z); - break; - } - case MODE_List: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - char *z = azCol[i]; - char *pFree; - const char *zOut = escapeOutput(p, z, &pFree); - sqlite3_fprintf(p->out, "%s%s", zOut, - i==nArg-1 ? p->rowSeparator : p->colSeparator); - if( pFree ) sqlite3_free(pFree); - } - } - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - char *z = azArg[i]; - char *pFree; - const char *zOut; - if( z==0 ) z = p->nullValue; - zOut = escapeOutput(p, z, &pFree); - sqlite3_fputs(zOut, p->out); - if( pFree ) sqlite3_free(pFree); - sqlite3_fputs((i<nArg-1)? p->colSeparator : p->rowSeparator, p->out); - } - break; - } - case MODE_Www: - case MODE_Html: { - if( p->cnt==0 && p->cMode==MODE_Www ){ - sqlite3_fputs( - "</PRE>\n" - "<TABLE border='1' cellspacing='0' cellpadding='2'>\n" - ,p->out - ); - } - if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ - sqlite3_fputs("<TR>", p->out); - for(i=0; i<nArg; i++){ - sqlite3_fputs("<TH>", p->out); - output_html_string(p->out, azCol[i]); - sqlite3_fputs("</TH>\n", p->out); - } - sqlite3_fputs("</TR>\n", p->out); - } - p->cnt++; - if( azArg==0 ) break; - sqlite3_fputs("<TR>", p->out); - for(i=0; i<nArg; i++){ - sqlite3_fputs("<TD>", p->out); - output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - sqlite3_fputs("</TD>\n", p->out); - } - sqlite3_fputs("</TR>\n", p->out); - break; - } - case MODE_Tcl: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - output_c_string(p->out, azCol[i] ? azCol[i] : ""); - if(i<nArg-1) sqlite3_fputs(p->colSeparator, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - if(i<nArg-1) sqlite3_fputs(p->colSeparator, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_Csv: { - sqlite3_fsetmode(p->out, _O_BINARY); - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( nArg>0 ){ - for(i=0; i<nArg; i++){ - output_csv(p, azArg[i], i<nArg-1); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - setCrlfMode(p); - break; - } - case MODE_Insert: { - if( azArg==0 ) break; - sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); - if( p->showHeader ){ - sqlite3_fputs("(", p->out); - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(",", p->out); - if( quoteChar(azCol[i]) ){ - char *z = sqlite3_mprintf("\"%w\"", azCol[i]); - shell_check_oom(z); - sqlite3_fputs(z, p->out); - sqlite3_free(z); - }else{ - sqlite3_fprintf(p->out, "%s", azCol[i]); - } - } - sqlite3_fputs(")", p->out); - } - p->cnt++; - for(i=0; i<nArg; i++){ - sqlite3_fputs(i>0 ? "," : " VALUES(", p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("NULL", p->out); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p, azArg[i]); - }else{ - output_quoted_escaped_string(p, azArg[i]); - } - }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - sqlite3_fputs(azArg[i], p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_uint64 ur; - memcpy(&ur,&r,sizeof(r)); - if( ur==0x7ff0000000000000LL ){ - sqlite3_fputs("9.0e+999", p->out); - }else if( ur==0xfff0000000000000LL ){ - sqlite3_fputs("-9.0e+999", p->out); - }else{ - sqlite3_int64 ir = (sqlite3_int64)r; - if( r==(double)ir ){ - sqlite3_snprintf(50,z,"%lld.0", ir); - }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - } - sqlite3_fputs(z, p->out); - } - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); - }else if( isNumber(azArg[i], 0) ){ - sqlite3_fputs(azArg[i], p->out); - }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p, azArg[i]); - }else{ - output_quoted_escaped_string(p, azArg[i]); - } - } - sqlite3_fputs(");\n", p->out); - break; - } - case MODE_Json: { - if( azArg==0 ) break; - if( p->cnt==0 ){ - sqlite3_fputs("[{", p->out); - }else{ - sqlite3_fputs(",\n{", p->out); - } - p->cnt++; - for(i=0; i<nArg; i++){ - output_json_string(p->out, azCol[i], -1); - sqlite3_fputs(":", p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("null", p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_uint64 ur; - memcpy(&ur,&r,sizeof(r)); - if( ur==0x7ff0000000000000LL ){ - sqlite3_fputs("9.0e+999", p->out); - }else if( ur==0xfff0000000000000LL ){ - sqlite3_fputs("-9.0e+999", p->out); - }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - sqlite3_fputs(z, p->out); - } - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_json_string(p->out, pBlob, nBlob); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_json_string(p->out, azArg[i], -1); - }else{ - sqlite3_fputs(azArg[i], p->out); - } - if( i<nArg-1 ){ - sqlite3_fputs(",", p->out); - } - } - sqlite3_fputs("}", p->out); - break; - } - case MODE_Quote: { - if( azArg==0 ) break; - if( p->cnt==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - output_quoted_string(p, azCol[i]); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - p->cnt++; - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("NULL", p->out); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(p, azArg[i]); - }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - sqlite3_fputs(azArg[i], p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_snprintf(50,z,"%!.20g", r); - sqlite3_fputs(z, p->out); - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); - }else if( isNumber(azArg[i], 0) ){ - sqlite3_fputs(azArg[i], p->out); - }else{ - output_quoted_string(p, azArg[i]); - } - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_Ascii: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_EQP: { - eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); - break; - } - } - return 0; -} - -/* -** This is the callback routine that the SQLite library -** invokes for each row of a query result. -*/ -static int callback(void *pArg, int nArg, char **azArg, char **azCol){ - /* since we don't have type info, call the shell_callback with a NULL value */ - return shell_callback(pArg, nArg, azArg, azCol, NULL); -} - -/* ** This is the callback routine from sqlite3_exec() that appends all ** output onto the end of a ShellText object. */ @@ -23369,7 +25870,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + cli_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -23382,28 +25883,13 @@ static void createSelftestTable(ShellState *p){ ** table name. */ static void set_table_name(ShellState *p, const char *zName){ - int i, n; - char cQuote; - char *z; - if( p->zDestTable ){ - free(p->zDestTable); + sqlite3_free(p->zDestTable); p->zDestTable = 0; } if( zName==0 ) return; - cQuote = quoteChar(zName); - n = strlen30(zName); - if( cQuote ) n += n+2; - z = p->zDestTable = malloc( n+1 ); - shell_check_oom(z); - n = 0; - if( cQuote ) z[n++] = cQuote; - for(i=0; zName[i]; i++){ - z[n++] = zName[i]; - if( zName[i]==cQuote ) z[n++] = cQuote; - } - if( cQuote ) z[n++] = cQuote; - z[n] = 0; + p->zDestTable = sqlite3_mprintf("%s", zName); + shell_check_oom(p->zDestTable); } /* @@ -23472,7 +25958,7 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", + cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; @@ -23482,22 +25968,22 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - sqlite3_fprintf(p->out, "%s", z); + cli_printf(p->out, "%s", z); for(i=1; i<nResult; i++){ - sqlite3_fprintf(p->out, ",%s", sqlite3_column_text(pSelect, i)); + cli_printf(p->out, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - sqlite3_fputs("\n;\n", p->out); + cli_puts("\n;\n", p->out); }else{ - sqlite3_fputs(";\n", p->out); + cli_puts(";\n", p->out); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", + cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } @@ -23557,7 +26043,7 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; i<ArraySize(aTrans); i++){ int n = strlen30(aTrans[i].zPattern); if( cli_strncmp(aTrans[i].zPattern, z, n)==0 ){ - sqlite3_fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); + cli_printf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); break; } } @@ -23589,7 +26075,7 @@ static void displayStatLine( }else{ sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); } - sqlite3_fprintf(out, "%-36s %s\n", zLabel, zLine); + cli_printf(out, "%-36s %s\n", zLabel, zLine); } /* @@ -23600,8 +26086,8 @@ static int display_stats( ShellState *pArg, /* Pointer to ShellState */ int bReset /* True to reset the stats */ ){ - int iCur; - int iHiwtr; + int iCur, iHiwtr; + sqlite3_int64 iCur64, iHiwtr64; FILE *out; if( pArg==0 || pArg->out==0 ) return 0; out = pArg->out; @@ -23611,22 +26097,22 @@ static int display_stats( sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); + cli_printf(out, "%-36s %d\n", "Number of output columns:", nCol); for(i=0; i<nCol; i++){ sqlite3_snprintf(sizeof(z),z,"Column %d %nname:", i, &x); - sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); + cli_printf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); #ifndef SQLITE_OMIT_DECLTYPE sqlite3_snprintf(30, z+x, "declared type:"); - sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); + cli_printf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA sqlite3_snprintf(30, z+x, "database name:"); - sqlite3_fprintf(out, "%-36s %s\n", z, + cli_printf(out, "%-36s %s\n", z, sqlite3_column_database_name(pStmt,i)); sqlite3_snprintf(30, z+x, "table name:"); - sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); + cli_printf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); sqlite3_snprintf(30, z+x, "origin name:"); - sqlite3_fprintf(out, "%-36s %s\n", z,sqlite3_column_origin_name(pStmt,i)); + cli_printf(out, "%-36s %s\n", z,sqlite3_column_origin_name(pStmt,i)); #endif } } @@ -23634,7 +26120,7 @@ static int display_stats( if( pArg->statsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - sqlite3_fprintf(out, "VM-steps: %d\n", iCur); + cli_printf(out, "VM-steps: %d\n", iCur); } return 0; } @@ -23663,48 +26149,55 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache misses: %d\n", iCur); + iHiwtr64 = iCur64 = -1; + sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, + 0); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache spills: %d\n", iCur); + cli_printf(out, + "Temporary data spilled to disk: %lld\n", iCur64); + sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, + 1); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } @@ -23712,33 +26205,33 @@ static int display_stats( int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - sqlite3_fprintf(out, + cli_printf(out, "Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - sqlite3_fprintf(out, + cli_printf(out, "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - sqlite3_fprintf(out, + cli_printf(out, "Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Memory used by prepared stmt: %d\n", iCur); } @@ -23751,267 +26244,6 @@ static int display_stats( return 0; } - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ - int iPid = 0; - int ret = 1; - sqlite3_stmt_scanstatus_v2(p, iEntry, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - while( iPid!=0 ){ - int ii; - for(ii=0; 1; ii++){ - int iId; - int res; - res = sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId - ); - if( res ) break; - if( iId==iPid ){ - sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - } - } - ret++; - } - return ret; -} -#endif - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -static void display_explain_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ - static const int f = SQLITE_SCANSTAT_COMPLEX; - sqlite3_stmt *p = pArg->pStmt; - int ii = 0; - i64 nTotal = 0; - int nWidth = 0; - eqp_reset(pArg); - - for(ii=0; 1; ii++){ - const char *z = 0; - int n = 0; - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ - break; - } - n = (int)strlen(z) + scanStatsHeight(p, ii)*3; - if( n>nWidth ) nWidth = n; - } - nWidth += 4; - - sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); - for(ii=0; 1; ii++){ - i64 nLoop = 0; - i64 nRow = 0; - i64 nCycle = 0; - int iId = 0; - int iPid = 0; - const char *zo = 0; - const char *zName = 0; - char *zText = 0; - double rEst = 0.0; - - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ - break; - } - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); - - zText = sqlite3_mprintf("%s", zo); - if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ - char *z = 0; - if( nCycle>=0 && nTotal>0 ){ - z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, - nCycle, ((nCycle*100)+nTotal/2) / nTotal - ); - } - if( nLoop>=0 ){ - z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); - } - if( nRow>=0 ){ - z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); - } - - if( zName && pArg->scanstatsOn>1 ){ - double rpl = (double)nRow / (double)nLoop; - z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); - } - - zText = sqlite3_mprintf( - "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z - ); - } - - eqp_append(pArg, iId, iPid, zText); - sqlite3_free(zText); - } - - eqp_render(pArg, nTotal); -} -#endif - - -/* -** Parameter azArray points to a zero-terminated array of strings. zStr -** points to a single nul-terminated string. Return non-zero if zStr -** is equal, according to strcmp(), to any of the strings in the array. -** Otherwise, return zero. -*/ -static int str_in_array(const char *zStr, const char **azArray){ - int i; - for(i=0; azArray[i]; i++){ - if( 0==cli_strcmp(zStr, azArray[i]) ) return 1; - } - return 0; -} - -/* -** If compiled statement pSql appears to be an EXPLAIN statement, allocate -** and populate the ShellState.aiIndent[] array with the number of -** spaces each opcode should be indented before it is output. -** -** The indenting rules are: -** -** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent -** all opcodes that occur between the p2 jump destination and the opcode -** itself by 2 spaces. -** -** * Do the previous for "Return" instructions for when P2 is positive. -** See tag-20220407a in wherecode.c and vdbe.c. -** -** * For each "Goto", if the jump destination is earlier in the program -** and ends on one of: -** Yield SeekGt SeekLt RowSetRead Rewind -** or if the P1 parameter is one instead of zero, -** then indent all opcodes between the earlier instruction -** and "Goto" by 2 spaces. -*/ -static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - int *abYield = 0; /* True if op is an OP_Yield */ - int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ - int iOp; /* Index of operation in p->aiIndent[] */ - - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", - "Return", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", - "Rewind", 0 }; - const char *azGoto[] = { "Goto", 0 }; - - /* The caller guarantees that the leftmost 4 columns of the statement - ** passed to this function are equivalent to the leftmost 4 columns - ** of EXPLAIN statement output. In practice the statement may be - ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ - assert( sqlite3_column_count(pSql)>=4 ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); - - for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ - int i; - int iAddr = sqlite3_column_int(pSql, 0); - const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - int p1 = sqlite3_column_int(pSql, 2); - int p2 = sqlite3_column_int(pSql, 3); - - /* Assuming that p2 is an instruction address, set variable p2op to the - ** index of that instruction in the aiIndent[] array. p2 and p2op may be - ** different if the current instruction is part of a sub-program generated - ** by an SQL trigger or foreign key. */ - int p2op = (p2 + (iOp-iAddr)); - - /* Grow the p->aiIndent array as required */ - if( iOp>=nAlloc ){ - nAlloc += 100; - p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); - shell_check_oom(p->aiIndent); - abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); - shell_check_oom(abYield); - } - - abYield[iOp] = str_in_array(zOp, azYield); - p->aiIndent[iOp] = 0; - p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ - for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2; - } - if( str_in_array(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){ - for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2; - } - } - - p->iIndent = 0; - sqlite3_free(abYield); - sqlite3_reset(pSql); -} - -/* -** Free the array allocated by explain_data_prepare(). -*/ -static void explain_data_delete(ShellState *p){ - sqlite3_free(p->aiIndent); - p->aiIndent = 0; - p->nIndent = 0; - p->iIndent = 0; -} - -static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); - -/* -** Display scan stats. -*/ -static void display_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else - if( pArg->scanstatsOn==3 ){ - const char *zSql = - " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," - " format('% 6s (%.2f%%)'," - " CASE WHEN ncycle<100_000 THEN ncycle || ' '" - " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" - " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" - " ELSE (ncycle/1000_000_000) || 'G' END," - " ncycle*100.0/(sum(ncycle) OVER ())" - " ) AS cycles" - " FROM bytecode(?)"; - - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSave = pArg->pStmt; - pArg->pStmt = pStmt; - sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); - - pArg->cnt = 0; - pArg->cMode = MODE_ScanExp; - explain_data_prepare(pArg, pStmt); - exec_prepared_stmt(pArg, pStmt); - explain_data_delete(pArg); - - sqlite3_finalize(pStmt); - pArg->pStmt = pSave; - } - }else{ - display_explain_scanstats(db, pArg); - } -#endif -} - /* ** Disable and restore .wheretrace and .treetrace/.selecttrace settings. */ @@ -24103,6 +26335,38 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ memcpy(zBuf, &zVar[6], szVar-5); sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } + }else if( strcmp(zVar, "$TIMER")==0 ){ + sqlite3_bind_double(pStmt, i, pArg->prevTimer); +#ifdef SQLITE_ENABLE_CARRAY + }else if( strncmp(zVar, "$carray_", 8)==0 ){ + static char *azColorNames[] = { + "azure", "black", "blue", "brown", "cyan", "fuchsia", "gold", + "gray", "green", "indigo", "khaki", "lime", "magenta", "maroon", + "navy", "olive", "orange", "pink", "purple", "red", "silver", + "tan", "teal", "violet", "white", "yellow" + }; + static int aPrimes[] = { + 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, + 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 + }; + /* Special bindings: carray($carray_clr), carray($carray_primes) + ** with --unsafe-testing: carray($carray_clr_p,26,'char*'), + ** carray($carray_primes_p,26,'int32') + */ + if( strcmp(zVar+8,"clr")==0 ){ + sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0); + }else if( strcmp(zVar+8,"primes")==0 ){ + sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0); + }else if( strcmp(zVar+8,"clr_p")==0 + && ShellHasFlag(pArg,SHFLG_TestingMode) ){ + sqlite3_bind_pointer(pStmt,i,azColorNames,"carray",0); + }else if( strcmp(zVar+8,"primes_p")==0 + && ShellHasFlag(pArg,SHFLG_TestingMode) ){ + sqlite3_bind_pointer(pStmt,i,aPrimes,"carray",0); + }else{ + sqlite3_bind_null(pStmt, i); + } +#endif }else{ sqlite3_bind_null(pStmt, i); } @@ -24111,581 +26375,7 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ sqlite3_finalize(pQ); } -/* -** UTF8 box-drawing characters. Imagine box lines like this: -** -** 1 -** | -** 4 --+-- 2 -** | -** 3 -** -** Each box characters has between 2 and 4 of the lines leading from -** the center. The characters are here identified by the numbers of -** their corresponding lines. -*/ -#define BOX_24 "\342\224\200" /* U+2500 --- */ -#define BOX_13 "\342\224\202" /* U+2502 | */ -#define BOX_23 "\342\224\214" /* U+250c ,- */ -#define BOX_34 "\342\224\220" /* U+2510 -, */ -#define BOX_12 "\342\224\224" /* U+2514 '- */ -#define BOX_14 "\342\224\230" /* U+2518 -' */ -#define BOX_123 "\342\224\234" /* U+251c |- */ -#define BOX_134 "\342\224\244" /* U+2524 -| */ -#define BOX_234 "\342\224\254" /* U+252c -,- */ -#define BOX_124 "\342\224\264" /* U+2534 -'- */ -#define BOX_1234 "\342\224\274" /* U+253c -|- */ - -/* Draw horizontal line N characters long using unicode box -** characters -*/ -static void print_box_line(FILE *out, int N){ - const char zDash[] = - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; - const int nDash = sizeof(zDash) - 1; - N *= 3; - while( N>nDash ){ - sqlite3_fputs(zDash, out); - N -= nDash; - } - sqlite3_fprintf(out, "%.*s", N, zDash); -} - -/* -** Draw a horizontal separator for a MODE_Box table. -*/ -static void print_box_row_separator( - ShellState *p, - int nArg, - const char *zSep1, - const char *zSep2, - const char *zSep3 -){ - int i; - if( nArg>0 ){ - sqlite3_fputs(zSep1, p->out); - print_box_line(p->out, p->actualWidth[0]+2); - for(i=1; i<nArg; i++){ - sqlite3_fputs(zSep2, p->out); - print_box_line(p->out, p->actualWidth[i]+2); - } - sqlite3_fputs(zSep3, p->out); - } - sqlite3_fputs("\n", p->out); -} - -/* -** z[] is a line of text that is to be displayed the .mode box or table or -** similar tabular formats. z[] might contain control characters such -** as \n, \t, \f, or \r. -** -** Compute characters to display on the first line of z[]. Stop at the -** first \r, \n, or \f. Expand \t into spaces. Return a copy (obtained -** from malloc()) of that first line, which caller should free sometime. -** Write anything to display on the next line into *pzTail. If this is -** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) -*/ -static char *translateForDisplayAndDup( - ShellState *p, /* To access current settings */ - const unsigned char *z, /* Input text to be transformed */ - const unsigned char **pzTail, /* OUT: Tail of the input for next line */ - int mxWidth, /* Max width. 0 means no limit */ - u8 bWordWrap /* If true, avoid breaking mid-word */ -){ - int i; /* Input bytes consumed */ - int j; /* Output bytes generated */ - int k; /* Input bytes to be displayed */ - int n; /* Output column number */ - unsigned char *zOut; /* Output text */ - - if( z==0 ){ - *pzTail = 0; - return 0; - } - if( mxWidth<0 ) mxWidth = -mxWidth; - if( mxWidth==0 ) mxWidth = 1000000; - i = j = n = 0; - while( n<mxWidth ){ - unsigned char c = z[i]; - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&z[i], &u); - i += len; - j += len; - n += cli_wcwidth(u); - continue; - } - if( c>=' ' ){ - n++; - i++; - j++; - continue; - } - if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break; - if( c=='\t' ){ - do{ - n++; - j++; - }while( (n&7)!=0 && n<mxWidth ); - i++; - continue; - } - if( c==0x1b && p->eEscMode==SHELL_ESC_OFF && (k = isVt100(&z[i]))>0 ){ - i += k; - j += k; - }else{ - n++; - j += 3; - i++; - } - } - if( n>=mxWidth && bWordWrap ){ - /* Perhaps try to back up to a better place to break the line */ - for(k=i; k>i/2; k--){ - if( IsSpace(z[k-1]) ) break; - } - if( k<=i/2 ){ - for(k=i; k>i/2; k--){ - if( IsAlnum(z[k-1])!=IsAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; - } - } - if( k<=i/2 ){ - k = i; - }else{ - i = k; - while( z[i]==' ' ) i++; - } - }else{ - k = i; - } - if( n>=mxWidth && z[i]>=' ' ){ - *pzTail = &z[i]; - }else if( z[i]=='\r' && z[i+1]=='\n' ){ - *pzTail = z[i+2] ? &z[i+2] : 0; - }else if( z[i]==0 || z[i+1]==0 ){ - *pzTail = 0; - }else{ - *pzTail = &z[i+1]; - } - zOut = malloc( j+1 ); - shell_check_oom(zOut); - i = j = n = 0; - while( i<k ){ - unsigned char c = z[i]; - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&z[i], &u); - do{ zOut[j++] = z[i++]; }while( (--len)>0 ); - n += cli_wcwidth(u); - continue; - } - if( c>=' ' ){ - n++; - zOut[j++] = z[i++]; - continue; - } - if( c==0 ) break; - if( z[i]=='\t' ){ - do{ - n++; - zOut[j++] = ' '; - }while( (n&7)!=0 && n<mxWidth ); - i++; - continue; - } - switch( p->eEscMode ){ - case SHELL_ESC_SYMBOL: - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80 + c; - break; - case SHELL_ESC_ASCII: - zOut[j++] = '^'; - zOut[j++] = 0x40 + c; - break; - case SHELL_ESC_OFF: { - int nn; - if( c==0x1b && (nn = isVt100(&z[i]))>0 ){ - memcpy(&zOut[j], &z[i], nn); - j += nn; - i += nn - 1; - }else{ - zOut[j++] = c; - } - break; - } - } - i++; - } - zOut[j] = 0; - return (char*)zOut; -} - -/* Return true if the text string z[] contains characters that need -** unistr() escaping. -*/ -static int needUnistr(const unsigned char *z){ - unsigned char c; - if( z==0 ) return 0; - while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; } - return c!=0; -} - -/* Extract the value of the i-th current column for pStmt as an SQL literal -** value. Memory is obtained from sqlite3_malloc64() and must be freed by -** the caller. -*/ -static char *quoted_column(sqlite3_stmt *pStmt, int i){ - switch( sqlite3_column_type(pStmt, i) ){ - case SQLITE_NULL: { - return sqlite3_mprintf("NULL"); - } - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); - } - case SQLITE_TEXT: { - const unsigned char *zText = sqlite3_column_text(pStmt,i); - return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText); - } - case SQLITE_BLOB: { - int j; - sqlite3_str *pStr = sqlite3_str_new(0); - const unsigned char *a = sqlite3_column_blob(pStmt,i); - int n = sqlite3_column_bytes(pStmt,i); - sqlite3_str_append(pStr, "x'", 2); - for(j=0; j<n; j++){ - sqlite3_str_appendf(pStr, "%02x", a[j]); - } - sqlite3_str_append(pStr, "'", 1); - return sqlite3_str_finish(pStr); - } - } - return 0; /* Not reached */ -} - -/* -** Run a prepared statement and output the result in one of the -** table-oriented formats: MODE_Column, MODE_Markdown, MODE_Table, -** or MODE_Box. -** -** This is different from ordinary exec_prepared_stmt() in that -** it has to run the entire query and gather the results into memory -** first, in order to determine column widths, before providing -** any output. -*/ -static void exec_prepared_stmt_columnar( - ShellState *p, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statement to run */ -){ - sqlite3_int64 nRow = 0; - int nColumn = 0; - char **azData = 0; - sqlite3_int64 nAlloc = 0; - char *abRowDiv = 0; - const unsigned char *uz; - const char *z; - char **azQuoted = 0; - int rc; - sqlite3_int64 i, nData; - int j, nTotal, w, n; - const char *colSep = 0; - const char *rowSep = 0; - const unsigned char **azNextLine = 0; - int bNextLine = 0; - int bMultiLineRowExists = 0; - int bw = p->cmOpts.bWordWrap; - const char *zEmpty = ""; - const char *zShowNull = p->nullValue; - - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ) return; - nColumn = sqlite3_column_count(pStmt); - if( nColumn==0 ) goto columnar_end; - nAlloc = nColumn*4; - if( nAlloc<=0 ) nAlloc = 1; - azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); - shell_check_oom(azData); - azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom(azNextLine); - memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); - if( p->cmOpts.bQuote ){ - azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom(azQuoted); - memset(azQuoted, 0, nColumn*sizeof(char*) ); - } - abRowDiv = sqlite3_malloc64( nAlloc/nColumn ); - shell_check_oom(abRowDiv); - if( nColumn>p->nWidth ){ - p->colWidth = realloc(p->colWidth, (nColumn+1)*2*sizeof(int)); - shell_check_oom(p->colWidth); - for(i=p->nWidth; i<nColumn; i++) p->colWidth[i] = 0; - p->nWidth = nColumn; - p->actualWidth = &p->colWidth[nColumn]; - } - memset(p->actualWidth, 0, nColumn*sizeof(int)); - for(i=0; i<nColumn; i++){ - w = p->colWidth[i]; - if( w<0 ) w = -w; - p->actualWidth[i] = w; - } - for(i=0; i<nColumn; i++){ - const unsigned char *zNotUsed; - int wx = p->colWidth[i]; - if( wx==0 ){ - wx = p->cmOpts.iWrap; - } - if( wx<0 ) wx = -wx; - uz = (const unsigned char*)sqlite3_column_name(pStmt,i); - if( uz==0 ) uz = (u8*)""; - azData[i] = translateForDisplayAndDup(p, uz, &zNotUsed, wx, bw); - } - do{ - int useNextLine = bNextLine; - bNextLine = 0; - if( (nRow+2)*nColumn >= nAlloc ){ - nAlloc *= 2; - azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); - shell_check_oom(azData); - abRowDiv = sqlite3_realloc64(abRowDiv, nAlloc/nColumn); - shell_check_oom(abRowDiv); - } - abRowDiv[nRow] = 1; - nRow++; - for(i=0; i<nColumn; i++){ - int wx = p->colWidth[i]; - if( wx==0 ){ - wx = p->cmOpts.iWrap; - } - if( wx<0 ) wx = -wx; - if( useNextLine ){ - uz = azNextLine[i]; - if( uz==0 ) uz = (u8*)zEmpty; - }else if( p->cmOpts.bQuote ){ - assert( azQuoted!=0 ); - sqlite3_free(azQuoted[i]); - azQuoted[i] = quoted_column(pStmt,i); - uz = (const unsigned char*)azQuoted[i]; - }else{ - uz = (const unsigned char*)sqlite3_column_text(pStmt,i); - if( uz==0 ) uz = (u8*)zShowNull; - } - azData[nRow*nColumn + i] - = translateForDisplayAndDup(p, uz, &azNextLine[i], wx, bw); - if( azNextLine[i] ){ - bNextLine = 1; - abRowDiv[nRow-1] = 0; - bMultiLineRowExists = 1; - } - } - }while( bNextLine || sqlite3_step(pStmt)==SQLITE_ROW ); - nTotal = nColumn*(nRow+1); - for(i=0; i<nTotal; i++){ - z = azData[i]; - if( z==0 ) z = (char*)zEmpty; - n = strlenChar(z); - j = i%nColumn; - if( n>p->actualWidth[j] ) p->actualWidth[j] = n; - } - if( seenInterrupt ) goto columnar_end; - switch( p->cMode ){ - case MODE_Column: { - colSep = " "; - rowSep = "\n"; - if( p->showHeader ){ - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - if( p->colWidth[i]<0 ) w = -w; - utf8_width_print(p->out, w, azData[i]); - sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); - } - for(i=0; i<nColumn; i++){ - print_dashes(p->out, p->actualWidth[i]); - sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); - } - } - break; - } - case MODE_Table: { - colSep = " | "; - rowSep = " |\n"; - print_row_separator(p, nColumn, "+"); - sqlite3_fputs("| ", p->out); - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", - azData[i], (w-n+1)/2, ""); - sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); - } - print_row_separator(p, nColumn, "+"); - break; - } - case MODE_Markdown: { - colSep = " | "; - rowSep = " |\n"; - sqlite3_fputs("| ", p->out); - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", - azData[i], (w-n+1)/2, ""); - sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); - } - print_row_separator(p, nColumn, "|"); - break; - } - case MODE_Box: { - colSep = " " BOX_13 " "; - rowSep = " " BOX_13 "\n"; - print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - sqlite3_fputs(BOX_13 " ", p->out); - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s%s", - (w-n)/2, "", azData[i], (w-n+1)/2, "", - i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); - } - print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); - break; - } - } - for(i=nColumn, j=0; i<nTotal; i++, j++){ - if( j==0 && p->cMode!=MODE_Column ){ - sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); - } - z = azData[i]; - if( z==0 ) z = p->nullValue; - w = p->actualWidth[j]; - if( p->colWidth[j]<0 ) w = -w; - utf8_width_print(p->out, w, z); - if( j==nColumn-1 ){ - sqlite3_fputs(rowSep, p->out); - if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1<nTotal ){ - if( p->cMode==MODE_Table ){ - print_row_separator(p, nColumn, "+"); - }else if( p->cMode==MODE_Box ){ - print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); - }else if( p->cMode==MODE_Column ){ - sqlite3_fputs("\n", p->out); - } - } - j = -1; - if( seenInterrupt ) goto columnar_end; - }else{ - sqlite3_fputs(colSep, p->out); - } - } - if( p->cMode==MODE_Table ){ - print_row_separator(p, nColumn, "+"); - }else if( p->cMode==MODE_Box ){ - print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); - } -columnar_end: - if( seenInterrupt ){ - sqlite3_fputs("Interrupt\n", p->out); - } - nData = (nRow+1)*nColumn; - for(i=0; i<nData; i++){ - z = azData[i]; - if( z!=zEmpty && z!=zShowNull ) free(azData[i]); - } - sqlite3_free(azData); - sqlite3_free((void*)azNextLine); - sqlite3_free(abRowDiv); - if( azQuoted ){ - for(i=0; i<nColumn; i++) sqlite3_free(azQuoted[i]); - sqlite3_free(azQuoted); - } -} - -/* -** Run a prepared statement -*/ -static void exec_prepared_stmt( - ShellState *pArg, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statement to run */ -){ - int rc; - sqlite3_uint64 nRow = 0; - - if( pArg->cMode==MODE_Column - || pArg->cMode==MODE_Table - || pArg->cMode==MODE_Box - || pArg->cMode==MODE_Markdown - ){ - exec_prepared_stmt_columnar(pArg, pStmt); - return; - } - - /* perform the first step. this will tell us if we - ** have a result set or not and how wide it is. - */ - rc = sqlite3_step(pStmt); - /* if we have a result set... */ - if( SQLITE_ROW == rc ){ - /* allocate space for col name ptr, value ptr, and type */ - int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); - if( !pData ){ - shell_out_of_memory(); - }else{ - char **azCols = (char **)pData; /* Names of result columns */ - char **azVals = &azCols[nCol]; /* Results */ - int *aiTypes = (int *)&azVals[nCol]; /* Result types */ - int i, x; - assert(sizeof(int) <= sizeof(char *)); - /* save off ptrs to column names */ - for(i=0; i<nCol; i++){ - azCols[i] = (char *)sqlite3_column_name(pStmt, i); - } - do{ - nRow++; - /* extract the data and data types */ - for(i=0; i<nCol; i++){ - aiTypes[i] = x = sqlite3_column_type(pStmt, i); - if( x==SQLITE_BLOB - && pArg - && (pArg->cMode==MODE_Insert || pArg->cMode==MODE_Quote) - ){ - azVals[i] = ""; - }else{ - azVals[i] = (char*)sqlite3_column_text(pStmt, i); - } - if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ - rc = SQLITE_NOMEM; - break; /* from for */ - } - } /* end for */ - - /* if data and types extracted successfully... */ - if( SQLITE_ROW == rc ){ - /* call the supplied callback with the result row data */ - if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ - rc = SQLITE_ABORT; - }else{ - rc = sqlite3_step(pStmt); - } - } - } while( SQLITE_ROW == rc ); - sqlite3_free(pData); - if( pArg->cMode==MODE_Json ){ - sqlite3_fputs("]\n", pArg->out); - }else if( pArg->cMode==MODE_Www ){ - sqlite3_fputs("</TABLE>\n<PRE>\n", pArg->out); - }else if( pArg->cMode==MODE_Count ){ - char zBuf[200]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n", - nRow, nRow!=1 ? "s" : ""); - printf("%s", zBuf); - } - } - } -} - -#ifndef SQLITE_OMIT_VIRTUALTABLE +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) /* ** This function is called to process SQL if the previous shell command ** was ".expert". It passes the SQL in the second argument directly to @@ -24736,8 +26426,8 @@ static int expertFinish( if( bVerbose ){ const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); - sqlite3_fputs("-- Candidates -----------------------------\n", out); - sqlite3_fprintf(out, "%s\n", zCand); + cli_puts("-- Candidates -----------------------------\n", out); + cli_printf(out, "%s\n", zCand); } for(i=0; i<nQuery; i++){ const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL); @@ -24745,12 +26435,12 @@ static int expertFinish( const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN); if( zIdx==0 ) zIdx = "(no new indexes)\n"; if( bVerbose ){ - sqlite3_fprintf(out, + cli_printf(out, "-- Query %d --------------------------------\n" "%s\n\n" ,i+1, zSql); } - sqlite3_fprintf(out, "%s\n%s\n", zIdx, zEQP); + cli_printf(out, "%s\n%s\n", zIdx, zEQP); } } } @@ -24785,18 +26475,18 @@ static int expertDotCommand( } else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ - sqlite3_fprintf(stderr, "option requires an argument: %s\n", z); + cli_printf(stderr, "option requires an argument: %s\n", z); rc = SQLITE_ERROR; }else{ iSample = (int)integerValue(azArg[++i]); if( iSample<0 || iSample>100 ){ - sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]); + cli_printf(stderr,"value out of range: %s\n", azArg[i]); rc = SQLITE_ERROR; } } } else{ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); + cli_printf(stderr,"unknown option: %s\n", z); rc = SQLITE_ERROR; } } @@ -24804,7 +26494,7 @@ static int expertDotCommand( if( rc==SQLITE_OK ){ pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); if( pState->expert.pExpert==0 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory"); rc = SQLITE_ERROR; }else{ @@ -24817,7 +26507,16 @@ static int expertDotCommand( return rc; } -#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ +#endif /* !SQLITE_OMIT_VIRTUALTABLE && !SQLITE_OMIT_AUTHORIZATION */ + +/* +** QRF write callback +*/ +static int shellWriteQR(void *pX, const char *z, sqlite3_int64 n){ + ShellState *pArg = (ShellState*)pX; + cli_printf(pArg->out, "%.*s", (int)n, z); + return SQLITE_OK; +} /* ** Execute a statement or set of statements. Print @@ -24838,19 +26537,42 @@ static int shell_exec( int rc2; const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3 *db = pArg->db; + unsigned char eStyle; + sqlite3_qrf_spec spec; if( pzErrMsg ){ *pzErrMsg = NULL; } + memcpy(&spec, &pArg->mode.spec, sizeof(spec)); + spec.xWrite = shellWriteQR; + spec.pWriteArg = (void*)pArg; + if( pArg->mode.eMode==MODE_Insert && ShellHasFlag(pArg,SHFLG_PreserveRowid) ){ + spec.bTitles = QRF_SW_On; + } + /* ,- This is true, but it is omitted + ** vvvvvvvvvvvvvvvvvvv ----- to avoid compiler warnings. */ + assert( /*pArg->mode.eMode>=0 &&*/ pArg->mode.eMode<ArraySize(aModeInfo) ); + eStyle = aModeInfo[pArg->mode.eMode].eStyle; + if( pArg->mode.bAutoScreenWidth ){ + spec.nScreenWidth = shellScreenWidth(); + } + if( spec.eBlob==QRF_BLOB_Auto ){ + switch( spec.eText ){ + case QRF_TEXT_Relaxed: /* fall through */ + case QRF_TEXT_Sql: spec.eBlob = QRF_BLOB_Sql; break; + case QRF_TEXT_Json: spec.eBlob = QRF_BLOB_Json; break; + default: spec.eBlob = QRF_BLOB_Text; break; + } + } -#ifndef SQLITE_OMIT_VIRTUALTABLE +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( pArg->expert.pExpert ){ rc = expertHandleSQL(pArg, zSql, pzErrMsg); return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg); } #endif - while( zSql[0] && (SQLITE_OK == rc) ){ + while( zSql && zSql[0] && (SQLITE_OK == rc) ){ static const char *zStmtSql; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); if( SQLITE_OK != rc ){ @@ -24858,6 +26580,7 @@ static int shell_exec( *pzErrMsg = save_err_msg(db, "in prepare", rc, zSql); } }else{ + int isExplain; if( !pStmt ){ /* this happens for a comment or white-space */ zSql = zLeftover; @@ -24868,80 +26591,58 @@ static int shell_exec( if( zStmtSql==0 ) zStmtSql = ""; while( IsSpace(zStmtSql[0]) ) zStmtSql++; - /* save off the prepared statement handle and reset row count */ + /* save off the prepared statement handle */ if( pArg ){ pArg->pStmt = pStmt; - pArg->cnt = 0; } - + /* Show the EXPLAIN QUERY PLAN if .eqp is on */ - if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ - sqlite3_stmt *pExplain; + isExplain = sqlite3_stmt_isexplain(pStmt); + if( pArg && pArg->mode.autoEQP && isExplain==0 && pArg->dot.nArg==0 ){ int triggerEQP = 0; + u8 savedEnableTimer = pArg->enableTimer; + pArg->enableTimer = 0; disable_debug_trace_modes(); sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); - if( pArg->autoEQP>=AUTOEQP_trigger ){ + if( pArg->mode.autoEQP>=AUTOEQP_trigger ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); } - pExplain = pStmt; - sqlite3_reset(pExplain); - rc = sqlite3_stmt_explain(pExplain, 2); - if( rc==SQLITE_OK ){ - bind_prepared_stmt(pArg, pExplain); - while( sqlite3_step(pExplain)==SQLITE_ROW ){ - const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); - int iEqpId = sqlite3_column_int(pExplain, 0); - int iParentId = sqlite3_column_int(pExplain, 1); - if( zEQPLine==0 ) zEQPLine = ""; - if( zEQPLine[0]=='-' ) eqp_render(pArg, 0); - eqp_append(pArg, iEqpId, iParentId, zEQPLine); - } - eqp_render(pArg, 0); - } - if( pArg->autoEQP>=AUTOEQP_full ){ - /* Also do an EXPLAIN for ".eqp full" mode */ - sqlite3_reset(pExplain); - rc = sqlite3_stmt_explain(pExplain, 1); - if( rc==SQLITE_OK ){ - pArg->cMode = MODE_Explain; - assert( sqlite3_stmt_isexplain(pExplain)==1 ); - bind_prepared_stmt(pArg, pExplain); - explain_data_prepare(pArg, pExplain); - exec_prepared_stmt(pArg, pExplain); - explain_data_delete(pArg); - } + sqlite3_reset(pStmt); + spec.eStyle = QRF_STYLE_Auto; + sqlite3_stmt_explain(pStmt, 2); + sqlite3_format_query_result(pStmt, &spec, 0); + if( pArg->mode.autoEQP>=AUTOEQP_full ){ + sqlite3_reset(pStmt); + sqlite3_stmt_explain(pStmt, 1); + sqlite3_format_query_result(pStmt, &spec, 0); } - if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ + + if( pArg->mode.autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); } sqlite3_reset(pStmt); sqlite3_stmt_explain(pStmt, 0); restore_debug_trace_modes(); - } - - if( pArg ){ - int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); - pArg->cMode = pArg->mode; - if( pArg->autoExplain ){ - if( bIsExplain ){ - pArg->cMode = MODE_Explain; - } - if( sqlite3_stmt_isexplain(pStmt)==2 ){ - pArg->cMode = MODE_EQP; - } - } - - /* If the shell is currently in ".explain" mode, gather the extra - ** data required to add indents to the output.*/ - if( pArg->cMode==MODE_Explain && bIsExplain ){ - explain_data_prepare(pArg, pStmt); - } + pArg->enableTimer = savedEnableTimer; } bind_prepared_stmt(pArg, pStmt); - exec_prepared_stmt(pArg, pStmt); - explain_data_delete(pArg); - eqp_render(pArg, 0); + if( isExplain && pArg->mode.autoExplain ){ + spec.eStyle = isExplain==1 ? QRF_STYLE_Explain : QRF_STYLE_Eqp; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + }else if( pArg->mode.eMode==MODE_Www ){ + cli_printf(pArg->out, + "</PRE>\n" + "<TABLE border='1' cellspacing='0' cellpadding='2'>\n"); + spec.eStyle = QRF_STYLE_Html; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + cli_printf(pArg->out, + "</TABLE>\n" + "<PRE>"); + }else{ + spec.eStyle = eStyle; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + } /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ @@ -24949,8 +26650,19 @@ static int shell_exec( } /* print loop-counters if required */ - if( pArg && pArg->scanstatsOn ){ - display_scanstats(db, pArg); + if( pArg && pArg->mode.scanstatsOn ){ + char *zErr = 0; + switch( pArg->mode.scanstatsOn ){ + case 1: spec.eStyle = QRF_STYLE_Stats; break; + case 2: spec.eStyle = QRF_STYLE_StatsEst; break; + default: spec.eStyle = QRF_STYLE_StatsVm; break; + } + sqlite3_reset(pStmt); + rc = sqlite3_format_query_result(pStmt, &spec, &zErr); + if( rc ){ + cli_printf(stderr, "Stats query failed: %s\n", zErr); + sqlite3_free(zErr); + } } /* Finalize the statement just executed. If this fails, save a @@ -25005,7 +26717,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ sqlite3_stmt *pStmt; char *zSql; int nCol = 0; - int nAlloc = 0; + i64 nAlloc = 0; int nPK = 0; /* Number of PRIMARY KEY columns seen */ int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid); @@ -25019,7 +26731,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ while( sqlite3_step(pStmt)==SQLITE_ROW ){ if( nCol>=nAlloc-2 ){ nAlloc = nAlloc*2 + nCol + 10; - azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + azCol = sqlite3_realloc64(azCol, nAlloc*sizeof(azCol[0])); shell_check_oom(azCol); } azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); @@ -25108,6 +26820,9 @@ static void toggleSelectOrder(sqlite3 *db){ sqlite3_exec(db, zStmt, 0, 0, 0); } +/* Forward reference */ +static int db_int(sqlite3 *db, const char *zSql, ...); + /* ** This is a different callback routine used for dumping the database. ** Each row received by this callback consists of a table name, @@ -25134,9 +26849,23 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ - /* no-op */ + /* The sqlite_sequence table is repopulated last. Delete content + ** in the sqlite_sequence table added by prior repopulations prior to + ** repopulating sqlite_sequence itself. But only do this if the + ** table is non-empty, because if it is empty the table might not + ** have been recreated by prior repopulations. See forum posts: + ** 2024-10-13T17:10:01z and 2025-10-29T19:38:43z + */ + if( db_int(p->db, "SELECT count(*) FROM sqlite_sequence")>0 ){ + if( !p->writableSchema ){ + cli_puts("PRAGMA writable_schema=ON;\n", p->out); + p->writableSchema = 1; + } + cli_puts("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n" + "DELETE FROM sqlite_sequence;\n", p->out); + } }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ - if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); + if( !dataOnly ) cli_puts("ANALYZE sqlite_schema;\n", p->out); }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ @@ -25144,7 +26873,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ - sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out); + cli_puts("PRAGMA writable_schema=ON;\n", p->out); p->writableSchema = 1; } zIns = sqlite3_mprintf( @@ -25152,7 +26881,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ "VALUES('table','%q','%q',0,'%q');", zTable, zTable, zSql); shell_check_oom(zIns); - sqlite3_fprintf(p->out, "%s\n", zIns); + cli_printf(p->out, "%s\n", zIns); sqlite3_free(zIns); return 0; }else{ @@ -25164,8 +26893,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ ShellText sTable; char **azCol; int i; - char *savedDestTable; - int savedMode; + Mode savedMode; azCol = tableColumnList(p, zTable); if( azCol==0 ){ @@ -25208,18 +26936,21 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ appendText(&sSelect, " FROM ", 0); appendText(&sSelect, zTable, quoteChar(zTable)); - savedDestTable = p->zDestTable; + savedMode = p->mode; - p->zDestTable = sTable.z; - p->mode = p->cMode = MODE_Insert; - rc = shell_exec(p, sSelect.z, 0); + p->mode.spec.zTableName = (char*)zTable; + p->mode.eMode = MODE_Insert; + p->mode.spec.eText = QRF_TEXT_Sql; + p->mode.spec.eBlob = QRF_BLOB_Sql; + p->mode.spec.bTitles = QRF_No; + p->mode.spec.nCharLimit = 0; + rc = shell_exec(p, sSelect.zTxt, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ - sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out); + cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); toggleSelectOrder(p->db); - shell_exec(p, sSelect.z, 0); + shell_exec(p, sSelect.zTxt, 0); toggleSelectOrder(p->db); } - p->zDestTable = savedDestTable; p->mode = savedMode; freeText(&sTable); freeText(&sSelect); @@ -25245,9 +26976,9 @@ static int run_schema_dump_query( if( rc==SQLITE_CORRUPT ){ char *zQ2; int len = strlen30(zQuery); - sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out); + cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); if( zErr ){ - sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr); + cli_printf(p->out, "/****** %s ******/\n", zErr); sqlite3_free(zErr); zErr = 0; } @@ -25256,7 +26987,7 @@ static int run_schema_dump_query( sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); if( rc ){ - sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr); + cli_printf(p->out, "/****** ERROR: %s ******/\n", zErr); }else{ rc = SQLITE_CORRUPT; } @@ -25314,8 +27045,8 @@ static const char *(azHelp[]) = { ".cd DIRECTORY Change the working directory to DIRECTORY", #endif ".changes on|off Show number of rows changed by SQL", + ".check OPTIONS ... Verify the results of a .testcase", #ifndef SQLITE_SHELL_FIDDLE - ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", #endif ".connection [close] [#] Open or close an auxiliary database connection", @@ -25349,7 +27080,9 @@ static const char *(azHelp[]) = { #ifndef SQLITE_SHELL_FIDDLE ".exit ?CODE? Exit this program with return-code CODE", #endif +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) ".expert EXPERIMENTAL. Suggest indexes for queries", +#endif ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", ".filectrl CMD ... Run various sqlite3_file_control() operations", " --schema SCHEMA Use SCHEMA instead of \"main\"", @@ -25359,26 +27092,14 @@ static const char *(azHelp[]) = { ".help ?-all? ?PATTERN? Show help text for PATTERN", #ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", - " Options:", - " --ascii Use \\037 and \\036 as column and row separators", - " --csv Use , and \\n as column and row separators", - " --skip N Skip the first N rows of input", - " --schema S Target table to be S.TABLE", - " -v \"Verbose\" - increase auxiliary output", - " Notes:", - " * If TABLE does not exist, it is created. The first row of input", - " determines the column names.", - " * If neither --csv or --ascii are used, the input mode is derived", - " from the \".mode\" output mode", - " * If FILE begins with \"|\" then it is a command that generates the", - " input text.", #endif #ifndef SQLITE_OMIT_TEST_CONTROL - ",imposter INDEX TABLE Create imposter table TABLE on index INDEX", + ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif - ".indexes ?TABLE? Show names of indexes", - " If TABLE is specified, only show indexes for", - " tables matching TABLE using the LIKE operator.", + ".indexes ?PATTERN? Show names of indexes matching PATTERN", + " -a|--all Also show system-generated indexes", + " --expr Show only expression indexes", + " --sys Show only system-generated indexes", ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", @@ -25396,42 +27117,12 @@ static const char *(azHelp[]) = { ".log on|off Turn logging on or off.", #endif ".mode ?MODE? ?OPTIONS? Set output mode", - " MODE is one of:", - " ascii Columns/rows delimited by 0x1F and 0x1E", - " box Tables using unicode box-drawing characters", - " csv Comma-separated values", - " column Output in columns. (See .width)", - " html HTML <table> code", - " insert SQL insert statements for TABLE", - " json Results in a JSON array", - " line One value per line", - " list Values delimited by \"|\"", - " markdown Markdown table format", - " qbox Shorthand for \"box --wrap 60 --quote\"", - " quote Escape answers as for SQL", - " table ASCII-art table", - " tabs Tab-separated values", - " tcl TCL list elements", - " OPTIONS: (for columnar modes or insert mode):", - " --escape T ctrl-char escape; T is one of: symbol, ascii, off", - " --wrap N Wrap output lines to no longer than N characters", - " --wordwrap B Wrap or not at word boundaries per B (on/off)", - " --ww Shorthand for \"--wordwrap 1\"", - " --quote Quote output text as SQL literals", - " --noquote Do not quote output text", - " TABLE The name of SQL table used for \"insert\" mode", #ifndef SQLITE_SHELL_FIDDLE ".nonce STRING Suspend safe mode for one command if nonce matches", #endif ".nullvalue STRING Use STRING in place of NULL values", #ifndef SQLITE_SHELL_FIDDLE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", - " If FILE begins with '|' then open as a pipe", - " --bom Put a UTF8 byte-order mark at the beginning", - " -e Send output to the system text editor", - " --plain Use text/plain output instead of HTML for -w option", - " -w Send output as HTML to a web browser (same as \".www\")", - " -x Send output as CSV to a spreadsheet (same as \".excel\")", /* Note that .open is (partially) available in WASM builds but is ** currently only intended to be used by the fiddle tool, not ** end users, so is "undocumented." */ @@ -25441,23 +27132,22 @@ static const char *(azHelp[]) = { #endif #ifndef SQLITE_OMIT_DESERIALIZE " --deserialize Load into memory using sqlite3_deserialize()", +#endif +/*" --exclusive Set the SQLITE_OPEN_EXCLUSIVE flag", UNDOCUMENTED */ +#ifndef SQLITE_OMIT_DESERIALIZE " --hexdb Load the output of \"dbtotxt\" as an in-memory db", +#endif + " --ifexist Only open if FILE already exists", +#ifndef SQLITE_OMIT_DESERIALIZE " --maxsize N Maximum size for --hexdb or --deserialized database", #endif " --new Initialize FILE to an empty database", + " --normal FILE is an ordinary SQLite database", " --nofollow Do not follow symbolic links", " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", #ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", - " If FILE begins with '|' then open it as a pipe.", - " If FILE is 'off' then output is disabled.", - " Options:", - " --bom Prefix output with a UTF8 byte-order mark", - " -e Send output to the system text editor", - " --plain Use text/plain for -w option", - " -w Send output to a web browser", - " -x Send output as CSV to a spreadsheet", #endif ".parameter CMD ... Manage SQL parameter bindings", " clear Erase all bindings", @@ -25473,6 +27163,7 @@ static const char *(azHelp[]) = { " --once Do no more than one progress interrupt", " --quiet|-q No output except at interrupts", " --reset Reset the count for each input and interrupt", + " --timeout S Halt after running for S seconds", #endif ".prompt MAIN CONTINUE Replace the standard prompts", #ifndef SQLITE_SHELL_FIDDLE @@ -25500,7 +27191,7 @@ static const char *(azHelp[]) = { " Options:", " --init Create a new SELFTEST table", " -v Verbose output", - ".separator COL ?ROW? Change the column and row separators", + ",separator COL ?ROW? Change the column and row separators", #if defined(SQLITE_ENABLE_SESSION) ".session ?NAME? CMD ... Create or control sessions", " Subcommands:", @@ -25527,7 +27218,7 @@ static const char *(azHelp[]) = { #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif - ".show Show the current values for various settings", + ",show Show the current values for various settings", ".stats ?ARG? Show stats or turn stats on or off", " off Turn off automatic stat display", " on Turn on automatic stat display", @@ -25537,13 +27228,11 @@ static const char *(azHelp[]) = { ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", -#ifndef SQLITE_SHELL_FIDDLE - ",testcase NAME Begin redirecting output to 'testcase-out.txt'", -#endif + ".testcase NAME Begin a test case.", ",testctrl CMD ... Run various sqlite3_test_control() operations", " Run \".testctrl\" with no arguments for details", ".timeout MS Try opening locked tables for MS milliseconds", - ".timer on|off Turn SQL timer on or off", + ".timer on|off|once Turn SQL timer on or off.", #ifndef SQLITE_OMIT_TRACE ".trace ?OPTIONS? Output each SQL statement as it is run", " FILE Send output to FILE", @@ -25568,7 +27257,7 @@ static const char *(azHelp[]) = { ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", - ".width NUM1 NUM2 ... Set minimum column widths for columnar output", + ",width NUM1 NUM2 ... Set minimum column widths for columnar output", " Negative values right-justify", #ifndef SQLITE_SHELL_FIDDLE ".www Display output of the next command in web browser", @@ -25576,6 +27265,204 @@ static const char *(azHelp[]) = { #endif }; +/************************************************************** +** "Usage" help text automatically generated from comments */ +static const struct { + const char *zCmd; /* Name of the dot-command */ + const char *zUsage; /* Documentation */ +} aUsage[] = { + { ".import", +"USAGE: .import [OPTIONS] FILE TABLE\n" +"\n" +"Import CSV or similar text from FILE into TABLE. If TABLE does\n" +"not exist, it is created using the first row of FILE as the column\n" +"names. If FILE begins with \"|\" then it is a command that is run\n" +"and the output from the command is used as the input data. If\n" +"FILE begins with \"<<\" followed by a label, then content is read from\n" +"the script until the first line that matches the label.\n" +"\n" +"The content of FILE is interpreted using RFC-4180 (\"CSV\") quoting\n" +"rules unless the current mode is \"ascii\" or \"tabs\" or unless one\n" +"the --ascii option is used.\n" +"\n" +"The column and row separators must be single ASCII characters. If\n" +"multiple characters or a Unicode character are specified for the\n" +"separators, then only the first byte of the separator is used. Except,\n" +"if the row separator is \\n and the mode is not --ascii, then \\r\\n is\n" +"understood as a row separator too.\n" +"\n" +"Options:\n" +" --ascii Do not use RFC-4180 quoting. Use \\037 and \\036\n" +" as column and row separators on input, unless other\n" +" delimiters are specified using --colsep and/or --rowsep\n" +" --colsep CHAR Use CHAR as the column separator.\n" +" --csv Input is standard RFC-4180 CSV.\n" +" --esc CHAR Use CHAR as an escape character in unquoted CSV inputs.\n" +" --qesc CHAR Use CHAR as an escape character in quoted CSV inputs.\n" +" --rowsep CHAR Use CHAR as the row separator.\n" +" --schema S When creating TABLE, put it in schema S\n" +" --skip N Ignore the first N rows of input\n" +" -v Verbose mode\n" + }, + { ".mode", +"USAGE: .mode [MODE] [OPTIONS]\n" +"\n" +"Change the output mode to MODE and/or apply OPTIONS to the output mode.\n" +"Arguments are processed from left to right. If no arguments, show the\n" +"current output mode and relevant options.\n" +"\n" +"Options:\n" +" --align STRING Set the alignment of text in columnar modes\n" +" String consists of characters 'L', 'C', 'R'\n" +" meaning \"left\", \"centered\", and \"right\", with\n" +" one letter per column starting from the left.\n" +" Unspecified alignment defaults to 'L'.\n" +" --blob-quote ARG ARG can be \"auto\", \"text\", \"sql\", \"hex\", \"tcl\",\n" +" \"json\", or \"size\". Default is \"auto\".\n" +" --border on|off Show outer border on \"box\" and \"table\" modes.\n" +" --charlimit N Set the maximum number of output characters to\n" +" show for any single SQL value to N. Longer values\n" +" truncated. Zero means \"no limit\".\n" +" --colsep STRING Use STRING as the column separator\n" +" --escape ESC Enable/disable escaping of control characters\n" +" found in the output. ESC can be \"off\", \"ascii\",\n" +" or \"symbol\".\n" +" --linelimit N Set the maximum number of output lines to show for\n" +" any single SQL value to N. Longer values are\n" +" truncated. Zero means \"no limit\". Only works\n" +" in \"line\" mode and in columnar modes.\n" +" --limits L,C,T Shorthand for \"--linelimit L --charlimit C\n" +" --titlelimit T\". The \",T\" can be omitted in which\n" +" case the --titlelimit is unchanged. The argument\n" +" can also be \"off\" to mean \"0,0,0\" or \"on\" to\n" +" mean \"5,300,20\".\n" +" --list List available modes\n" +" --multiinsert N In \"insert\" mode, put multiple rows on a single\n" +" INSERT statement until the size exceeds N bytes.\n" +" --null STRING Render SQL NULL values as the given string\n" +" --once Setting changes to the right are reverted after\n" +" the next SQL command.\n" +" --quote ARG Enable/disable quoting of text. ARG can be\n" +" \"off\", \"on\", \"sql\", \"relaxed\", \"csv\", \"html\",\n" +" \"tcl\", or \"json\". \"off\" means show the text as-is.\n" +" \"on\" is an alias for \"sql\".\n" +" --reset Changes all mode settings back to their default.\n" +" --rowsep STRING Use STRING as the row separator\n" +" --sw|--screenwidth N Declare the screen width of the output device\n" +" to be N characters. An attempt may be made to\n" +" wrap output text to fit within this limit. Zero\n" +" means \"no limit\". Or N can be \"auto\" to set the\n" +" width automatically.\n" +" --tablename NAME Set the name of the table for \"insert\" mode.\n" +" --tag NAME Save mode to the left as NAME.\n" +" --textjsonb BOOLEAN If enabled, JSONB text is displayed as text JSON.\n" +" --title ARG Whether or not to show column headers, and if so\n" +" how to encode them. ARG can be \"off\", \"on\",\n" +" \"sql\", \"csv\", \"html\", \"tcl\", or \"json\".\n" +" --titlelimit N Limit the length of column titles to N characters.\n" +" -v|--verbose Verbose output\n" +" --widths LIST Set the columns widths for columnar modes. The\n" +" argument is a list of integers, one for each\n" +" column. A \"0\" width means use a dynamic width\n" +" based on the actual width of data. If there are\n" +" fewer entries in LIST than columns, \"0\" is used\n" +" for the unspecified widths.\n" +" --wordwrap BOOLEAN Enable/disable word wrapping\n" +" --wrap N Wrap columns wider than N characters\n" +" --ww Shorthand for \"--wordwrap on\"\n" + }, + { ".output", +"USAGE: .output [OPTIONS] [FILE]\n" +"\n" +"Begin redirecting output to FILE. Or if FILE is omitted, revert\n" +"to sending output to the console. If FILE begins with \"|\" then\n" +"the remainder of file is taken as a pipe and output is directed\n" +"into that pipe. If FILE is \"memory\" then output is captured in an\n" +"internal memory buffer. If FILE is \"off\" then output is redirected\n" +"into /dev/null or the equivalent.\n" +"\n" +"Options:\n" +" --bom Prepend a byte-order mark to the output\n" +" -e Accumulate output in a temporary text file then\n" +" launch a text editor when the redirection ends.\n" +" --error-prefix X Use X as the left-margin prefix for error messages.\n" +" Set to an empty string to restore the default.\n" +" --keep Keep redirecting output to its current destination.\n" +" Use this option in combination with --show or\n" +" with --error-prefix when you do not want to stop\n" +" a current redirection.\n" +" --plain Use plain text rather than HTML tables with -w\n" +" --show Show output text captured by .testcase or by\n" +" redirecting to \"memory\".\n" +" -w Show the output in a web browser. Output is\n" +" written into a temporary HTML file until the\n" +" redirect ends, then the web browser is launched.\n" +" Query results are shown as HTML tables, unless\n" +" the --plain is used too.\n" +" -x Show the output in a spreadsheet. Output is\n" +" written to a temp file as CSV then the spreadsheet\n" +" is launched when\n" + }, + { ".once", +"USAGE: .once [OPTIONS] FILE ...\n" +"\n" +"Write the output for the next line of SQL or the next dot-command into\n" +"FILE. If FILE begins with \"|\" then it is a program into which output\n" +"is written. The FILE argument should be omitted if one of the -e, -w,\n" +"or -x options is used.\n" +"\n" +"Options:\n" +" -e Capture output into a temporary file then bring up\n" +" a text editor on that temporary file.\n" +" --plain Use plain text rather than HTML tables with -w\n" +" -w Capture output into an HTML file then bring up that\n" +" file in a web browser\n" +" -x Show the output in a spreadsheet. Output is\n" +" written to a temp file as CSV then the spreadsheet\n" +" is launched when\n" + }, + { ".check", +"USAGE: .check [OPTIONS] PATTERN\n" +"\n" +"Verify results of commands since the most recent .testcase command.\n" +"Restore output to the console, unless --keep is used.\n" +"\n" +"If PATTERN starts with \"<<ENDMARK\" then the actual pattern is taken from\n" +"subsequent lines of text up to the first line that begins with ENDMARK.\n" +"All pattern lines and the ENDMARK are discarded.\n" +"\n" +"Options:\n" +" --exact Do an exact comparison including leading and\n" +" trailing whitespace.\n" +" --glob Treat PATTERN as a GLOB\n" +" --keep Do not reset the testcase. More .check commands\n" +" will follow.\n" +" --notglob Output should not match PATTERN\n" +" --show Write testcase output to the screen, for debugging.\n" + }, + { ".testcase", +"USAGE: .testcase [OPTIONS] NAME\n" +"\n" +"Start a new test case identified by NAME. All output\n" +"through the next \".check\" command is captured for comparison. See the\n" +"\".check\" commandn for additional informatioon.\n" +"\n" +"Options:\n" +" --error-prefix TEXT Change error message prefix text to TEXT\n" + }, +}; + +/* +** Return a pointer to usage text for zCmd, or NULL if none exists. +*/ +static const char *findUsage(const char *zCmd){ + int i; + for(i=0; i<ArraySize(aUsage); i++){ + if( sqlite3_strglob(zCmd, aUsage[i].zCmd)==0 ) return aUsage[i].zUsage; + } + return 0; +} + /* ** Output help text for commands that match zPattern. ** @@ -25605,6 +27492,7 @@ static int showHelp(FILE *out, const char *zPattern){ int j = 0; int n = 0; char *zPat; + const char *zHit = 0; if( zPattern==0 ){ /* Show just the first line for all help topics */ zPattern = "[a-z]"; @@ -25622,37 +27510,46 @@ static int showHelp(FILE *out, const char *zPattern){ show = 0; }else if( azHelp[i][0]==',' ){ show = 1; - sqlite3_fprintf(out, ".%s\n", &azHelp[i][1]); + cli_printf(out, ".%s\n", &azHelp[i][1]); n++; }else if( show ){ - sqlite3_fprintf(out, "%s\n", azHelp[i]); + cli_printf(out, "%s\n", azHelp[i]); } } return n; } /* Seek documented commands for which zPattern is an exact prefix */ - zPat = sqlite3_mprintf(".%s*", zPattern); + zPat = sqlite3_mprintf(".%s*", zPattern[0]=='.' ? &zPattern[1] : zPattern); shell_check_oom(zPat); for(i=0; i<ArraySize(azHelp); i++){ if( sqlite3_strglob(zPat, azHelp[i])==0 ){ - sqlite3_fprintf(out, "%s\n", azHelp[i]); + if( zHit ) cli_printf(out, "%s\n", zHit); + zHit = azHelp[i]; j = i+1; n++; } } - sqlite3_free(zPat); if( n ){ if( n==1 ){ - /* when zPattern is a prefix of exactly one command, then include - ** the details of that command, which should begin at offset j */ - while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){ - sqlite3_fprintf(out, "%s\n", azHelp[j]); - j++; + const char *zUsage = findUsage(zPat); + if( zUsage ){ + cli_puts(zUsage, out); + }else{ + /* when zPattern is a prefix of exactly one command, then include + ** the details of that command, which should begin at offset j */ + cli_printf(out, "%s\n", zHit); + while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){ + cli_printf(out, "%s\n", azHelp[j]); + j++; + } } + }else{ + cli_printf(out, "%s\n", zHit); } - return n; } + sqlite3_free(zPat); + if( n ) return n; /* Look for documented commands that contain zPattern anywhere. ** Show complete text of all documented commands that match. */ @@ -25665,10 +27562,10 @@ static int showHelp(FILE *out, const char *zPattern){ } if( azHelp[i][0]=='.' ) j = i; if( sqlite3_strlike(zPat, azHelp[i], 0)==0 ){ - sqlite3_fprintf(out, "%s\n", azHelp[j]); + cli_printf(out, "%s\n", azHelp[j]); while( j<ArraySize(azHelp)-1 && azHelp[j+1][0]==' ' ){ j++; - sqlite3_fprintf(out, "%s\n", azHelp[j]); + cli_printf(out, "%s\n", azHelp[j]); } i = j; n++; @@ -25679,7 +27576,7 @@ static int showHelp(FILE *out, const char *zPattern){ } /* Forward reference */ -static int process_input(ShellState *p); +static int process_input(ShellState *p, const char*); /* ** Read the content of file zName into memory obtained from sqlite3_malloc64() @@ -25705,7 +27602,7 @@ static char *readFile(const char *zName, int *pnByte){ if( in==0 ) return 0; rc = fseek(in, 0, SEEK_END); if( rc!=0 ){ - sqlite3_fprintf(stderr,"Error: '%s' not seekable\n", zName); + cli_printf(stderr,"Error: '%s' not seekable\n", zName); fclose(in); return 0; } @@ -25713,7 +27610,7 @@ static char *readFile(const char *zName, int *pnByte){ rewind(in); pBuf = sqlite3_malloc64( nIn+1 ); if( pBuf==0 ){ - sqlite3_fputs("Error: out of memory\n", stderr); + cli_puts("Error: out of memory\n", stderr); fclose(in); return 0; } @@ -25721,7 +27618,7 @@ static char *readFile(const char *zName, int *pnByte){ fclose(in); if( nRead!=1 ){ sqlite3_free(pBuf); - sqlite3_fprintf(stderr,"Error: cannot read '%s'\n", zName); + cli_printf(stderr,"Error: cannot read '%s'\n", zName); return 0; } pBuf[nIn] = 0; @@ -25778,6 +27675,52 @@ static int session_filter(void *pCtx, const char *zTab){ #endif /* +** Return the size of the named file in bytes. Or return a negative +** number if the file does not exist. +*/ +static sqlite3_int64 fileSize(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct _stat64 x; + if( _stat64(zFile, &x)!=0 ) return -1; + return (sqlite3_int64)x.st_size; +#else + struct stat x; + if( stat(zFile, &x)!=0 ) return -1; + return (sqlite3_int64)x.st_size; +#endif +} + +/* +** Return true if zFile is an SQLite database. +** +** Algorithm: +** * If the file does not exist -> return false +** * If the size of the file is not a multiple of 512 -> return false +** * If sqlite3_open() fails -> return false +** * if sqlite3_prepare() or sqlite3_step() fails -> return false +** * Otherwise -> return true +*/ +static int isDatabaseFile(const char *zFile, int openFlags){ + sqlite3 *db = 0; + sqlite3_stmt *pStmt = 0; + int rc; + sqlite3_int64 sz = fileSize(zFile); + if( sz<512 || (sz%512)!=0 ) return 0; + if( sqlite3_open_v2(zFile, &db, openFlags, 0)==SQLITE_OK + && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0) + ==SQLITE_OK + && sqlite3_step(pStmt)==SQLITE_ROW + ){ + rc = 1; + }else{ + rc = 0; + } + sqlite3_finalize(pStmt); + sqlite3_close(db); + return rc; +} + +/* ** Try to deduce the type of file for zName based on its content. Return ** one of the SHELL_OPEN_* constants. ** @@ -25786,18 +27729,18 @@ static int session_filter(void *pCtx, const char *zTab){ ** Otherwise, assume an ordinary database regardless of the filename if ** the type cannot be determined from content. */ -int deduceDatabaseType(const char *zName, int dfltZip){ - FILE *f = sqlite3_fopen(zName, "rb"); +int deduceDatabaseType(const char *zName, int dfltZip, int openFlags){ + FILE *f; size_t n; int rc = SHELL_OPEN_UNSPEC; char zBuf[100]; - if( f==0 ){ - if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ - return SHELL_OPEN_ZIPFILE; - }else{ - return SHELL_OPEN_NORMAL; - } + if( access(zName,0)!=0 ) goto database_type_by_name; + if( isDatabaseFile(zName, openFlags) ){ + rc = SHELL_OPEN_NORMAL; } + if( rc==SHELL_OPEN_NORMAL ) return SHELL_OPEN_NORMAL; + f = sqlite3_fopen(zName, "rb"); + if( f==0 ) goto database_type_by_name; n = fread(zBuf, 16, 1, f); if( n==1 && memcmp(zBuf, "SQLite format 3", 16)==0 ){ fclose(f); @@ -25819,6 +27762,43 @@ int deduceDatabaseType(const char *zName, int dfltZip){ } fclose(f); return rc; + +database_type_by_name: + if( dfltZip && sqlite3_strlike("%.zip",zName,0)==0 ){ + rc = SHELL_OPEN_ZIPFILE; + }else{ + rc = SHELL_OPEN_NORMAL; + } + return rc; +} + +/* +** If the text in z[] is the name of a readable file and that file appears +** to contain SQL text and/or dot-commands, then return true. If z[] is +** not a file, or if the file is unreadable, or if the file is a database +** or anything else that is not SQL text and dot-commands, then return false. +** +** If the bLeaveUninit flag is set, then be sure to leave SQLite in an +** uninitialized state. This means invoking sqlite3_shutdown() after any +** SQLite API is used. +** +** Some amount of guesswork is involved in this decision. +*/ +static int isScriptFile(const char *z, int bLeaveUninit){ + sqlite3_int64 sz = fileSize(z); + if( sz<=0 ) return 0; + if( (sz%512)==0 ){ + int rc; + sqlite3_initialize(); + rc = isDatabaseFile(z, SQLITE_OPEN_READONLY); + if( bLeaveUninit ){ + sqlite3_shutdown(); + } + if( rc ) return 0; /* Is a database */ + } + if( sqlite3_strlike("%.sql",z,0)==0 ) return 1; + if( sqlite3_strlike("%.txt",z,0)==0 ) return 1; + return 0; } #ifndef SQLITE_OMIT_DESERIALIZE @@ -25829,11 +27809,11 @@ int deduceDatabaseType(const char *zName, int dfltZip){ */ static unsigned char *readHexDb(ShellState *p, int *pnData){ unsigned char *a = 0; - int nLine; - int n = 0; + i64 nLine; + int n = 0; /* Size of db per first line of hex dump */ + i64 sz = 0; /* n rounded up to nearest page boundary */ int pgsz = 0; - int iOffset = 0; - int j, k; + i64 iOffset = 0; int rc; FILE *in; const char *zDbFilename = p->pAuxDb->zDbFilename; @@ -25842,7 +27822,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ if( zDbFilename ){ in = sqlite3_fopen(zDbFilename, "r"); if( in==0 ){ - sqlite3_fprintf(stderr,"cannot open \"%s\" for reading\n", zDbFilename); + cli_printf(stderr,"cannot open \"%s\" for reading\n", zDbFilename); return 0; } nLine = 0; @@ -25857,16 +27837,21 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); if( rc!=2 ) goto readHexDb_error; if( n<0 ) goto readHexDb_error; - if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error; - n = (n+pgsz-1)&~(pgsz-1); /* Round n up to the next multiple of pgsz */ - a = sqlite3_malloc( n ? n : 1 ); - shell_check_oom(a); - memset(a, 0, n); if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ - sqlite3_fputs("invalid pagesize\n", stderr); + cli_puts("invalid pagesize\n", stderr); goto readHexDb_error; } + sz = ((i64)n+pgsz-1)&~(pgsz-1); /* Round up to nearest multiple of pgsz */ + a = sqlite3_malloc64( sz ? sz : 1 ); + shell_check_oom(a); + memset(a, 0, sz); for(nLine++; sqlite3_fgets(zLine, sizeof(zLine), in)!=0; nLine++){ + int j = 0; /* Page number from "| page" line */ + int k = 0; /* Offset from "| page" line */ + if( nLine>=2000000000 ){ + cli_printf(stderr, "input too big\n"); + goto readHexDb_error; + } rc = sscanf(zLine, "| page %d offset %d", &j, &k); if( rc==2 ){ iOffset = k; @@ -25879,14 +27864,14 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7], &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); if( rc==17 ){ - k = iOffset+j; - if( k+16<=n && k>=0 ){ + i64 iOff = iOffset+j; + if( iOff+16<=sz && iOff>=0 ){ int ii; - for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff; + for(ii=0; ii<16; ii++) a[iOff+ii] = x[ii]&0xff; } } } - *pnData = n; + *pnData = sz; if( in!=p->in ){ fclose(in); }else{ @@ -25905,7 +27890,7 @@ readHexDb_error: p->lineno = nLine; } sqlite3_free(a); - sqlite3_fprintf(stderr,"Error on line %d of --hexdb input\n", nLine); + cli_printf(stderr,"Error on line %lld of --hexdb input\n", nLine); return 0; } #endif /* SQLITE_OMIT_DESERIALIZE */ @@ -25953,7 +27938,7 @@ static void shellModuleSchema( if( zFake ){ sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), -1, sqlite3_free); - free(zFake); + sqlite3_free(zFake); } } @@ -25982,13 +27967,16 @@ static void open_db(ShellState *p, int openFlags){ p->openMode = SHELL_OPEN_NORMAL; }else{ p->openMode = (u8)deduceDatabaseType(zDbFilename, - (openFlags & OPEN_DB_ZIPFILE)!=0); + (openFlags & OPEN_DB_ZIPFILE)!=0, p->openFlags); } } + if( (p->openFlags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE))==0 ){ + if( p->openFlags==0 ) p->openFlags = SQLITE_OPEN_CREATE; + p->openFlags |= SQLITE_OPEN_READWRITE; + } switch( p->openMode ){ case SHELL_OPEN_APPENDVFS: { - sqlite3_open_v2(zDbFilename, &p->db, - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs"); + sqlite3_open_v2(zDbFilename, &p->db, p->openFlags, "apndvfs"); break; } case SHELL_OPEN_HEXDB: @@ -26000,32 +27988,26 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_open(":memory:", &p->db); break; } - case SHELL_OPEN_READONLY: { - sqlite3_open_v2(zDbFilename, &p->db, - SQLITE_OPEN_READONLY|p->openFlags, 0); - break; - } case SHELL_OPEN_UNSPEC: case SHELL_OPEN_NORMAL: { - sqlite3_open_v2(zDbFilename, &p->db, - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, 0); + sqlite3_open_v2(zDbFilename, &p->db, p->openFlags, 0); break; } } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n", + cli_printf(stderr,"Error: unable to open database \"%s\": %s\n", zDbFilename, sqlite3_errmsg(p->db)); if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){ - exit(1); + cli_exit(1); } sqlite3_close(p->db); sqlite3_open(":memory:", &p->db); if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - sqlite3_fputs("Also: unable to open substitute in-memory database.\n", + cli_puts("Also: unable to open substitute in-memory database.\n", stderr); - exit(1); + cli_exit(1); }else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Notice: using substitute in-memory database instead of \"%s\"\n", zDbFilename); } @@ -26048,7 +28030,6 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_uint_init(p->db, 0, 0); sqlite3_stmtrand_init(p->db, 0, 0); sqlite3_decimal_init(p->db, 0, 0); - sqlite3_percentile_init(p->db, 0, 0); sqlite3_base64_init(p->db, 0, 0); sqlite3_base85_init(p->db, 0, 0); sqlite3_regexp_init(p->db, 0, 0); @@ -26104,6 +28085,8 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_format_schema", 2, SQLITE_UTF8, p, + shellFormatSchema, 0, 0); sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, shellUSleepFunc, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM @@ -26138,7 +28121,7 @@ static void open_db(ShellState *p, int openFlags){ SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); if( rc ){ - sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc); + cli_printf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc); } if( p->szMax>0 ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); @@ -26147,11 +28130,13 @@ static void open_db(ShellState *p, int openFlags){ #endif } if( p->db!=0 ){ +#ifndef SQLITE_OMIT_AUTHORIZATION if( p->bSafeModePersist ){ sqlite3_set_authorizer(p->db, safeModeAuth, p); } +#endif sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 ); } } @@ -26162,7 +28147,7 @@ static void open_db(ShellState *p, int openFlags){ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db)); } } @@ -26335,7 +28320,7 @@ static int booleanValue(const char *zArg){ if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ return 0; } - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg); return 0; } @@ -26363,18 +28348,18 @@ static void output_file_close(FILE *f){ ** recognized and do the right thing. NULL is returned if the output ** filename is "off". */ -static FILE *output_file_open(const char *zFile){ +static FILE *output_file_open(ShellState *p, const char *zFile){ FILE *f; if( cli_strcmp(zFile,"stdout")==0 ){ f = stdout; }else if( cli_strcmp(zFile, "stderr")==0 ){ f = stderr; - }else if( cli_strcmp(zFile, "off")==0 ){ + }else if( cli_strcmp(zFile, "off")==0 || p->bSafeMode ){ f = 0; }else{ f = sqlite3_fopen(zFile, "w"); if( f==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zFile); } } return f; @@ -26427,12 +28412,12 @@ static int sql_trace_callback( switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql); + cli_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; - sqlite3_fprintf(p->traceOut, + cli_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } @@ -26461,9 +28446,11 @@ struct ImportCtx { const char *zFile; /* Name of the input file */ FILE *in; /* Read the CSV text from this input stream */ int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close in */ + char *zIn; /* Input text */ char *z; /* Accumulated text for a field */ - int n; /* Number of bytes in z */ - int nAlloc; /* Space allocated for z[] */ + i64 nUsed; /* Bytes of zIn[] used so far */ + i64 n; /* Number of bytes in z */ + i64 nAlloc; /* Space allocated for z[] */ int nLine; /* Current line number */ int nRow; /* Number of rows imported */ int nErr; /* Number of errors encountered */ @@ -26471,6 +28458,8 @@ struct ImportCtx { int cTerm; /* Character that terminated the most recent field */ int cColSep; /* The column separator character. (Usually ",") */ int cRowSep; /* The row separator character. (Usually "\n") */ + int cQEscape; /* Escape character with "...". 0 for none */ + int cUQEscape; /* Escape character not with "...". 0 for none */ }; /* Clean up resourced used by an ImportCtx */ @@ -26481,9 +28470,28 @@ static void import_cleanup(ImportCtx *p){ } sqlite3_free(p->z); p->z = 0; + if( p->zIn ){ + sqlite3_free(p->zIn); + p->zIn = 0; + } +} + +/* Read a single character of the .import input text. Return EOF +** at end-of-file. +*/ +static int import_getc(ImportCtx *p){ + if( p->in ){ + return fgetc(p->in); + }else if( p->zIn && p->zIn[p->nUsed]!=0 ){ + return p->zIn[p->nUsed++]; + }else{ + return EOF; + } } -/* Append a single byte to z[] */ +/* Append a single byte to the field value begin constructed +** in the p->z[] buffer +*/ static void import_append_char(ImportCtx *p, int c){ if( p->n+1>=p->nAlloc ){ p->nAlloc += p->nAlloc + 100; @@ -26499,8 +28507,8 @@ static void import_append_char(ImportCtx *p, int c){ ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes ** from sqlite3_malloc64(). -** + Use p->cSep as the column separator. The default is ",". -** + Use p->rSep as the row separator. The default is "\n". +** + Use p->cColSep as the column separator. The default is ",". +** + Use p->cRowSep as the row separator. The default is "\n". ** + Keep track of the line number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -26511,7 +28519,7 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int cSep = (u8)p->cColSep; int rSep = (u8)p->cRowSep; p->n = 0; - c = fgetc(p->in); + c = import_getc(p); if( c==EOF || seenInterrupt ){ p->cTerm = EOF; return 0; @@ -26520,10 +28528,17 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int pc, ppc; int startLine = p->nLine; int cQuote = c; + int cEsc = (u8)p->cQEscape; pc = ppc = 0; while( 1 ){ - c = fgetc(p->in); + c = import_getc(p); if( c==rSep ) p->nLine++; + if( c==cEsc && cEsc!=0 ){ + c = import_getc(p); + import_append_char(p, c); + ppc = pc = 0; + continue; + } if( c==cQuote ){ if( pc==cQuote ){ pc = 0; @@ -26540,11 +28555,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ break; } if( pc==cQuote && c!='\r' ){ - sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", - p->zFile, p->nLine, cQuote); + cli_printf(stderr,"%s:%d: unescaped %c character\n", + p->zFile, p->nLine, cQuote); } if( c==EOF ){ - sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n", + cli_printf(stderr,"%s:%d: unterminated %c-quoted field\n", p->zFile, startLine, cQuote); p->cTerm = c; break; @@ -26556,12 +28571,13 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ }else{ /* If this is the first field being parsed and it begins with the ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ + int cEsc = p->cUQEscape; if( (c&0xff)==0xef && p->bNotFirst==0 ){ import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); if( (c&0xff)==0xbb ){ import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); if( (c&0xff)==0xbf ){ p->bNotFirst = 1; p->n = 0; @@ -26570,8 +28586,9 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ } } while( c!=EOF && c!=cSep && c!=rSep ){ + if( c==cEsc && cEsc!=0 ) c = import_getc(p); import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); } if( c==rSep ){ p->nLine++; @@ -26589,8 +28606,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes ** from sqlite3_malloc64(). -** + Use p->cSep as the column separator. The default is "\x1F". -** + Use p->rSep as the row separator. The default is "\x1E". +** + Use p->cColSep as the column separator. The default is "\x1F". +** + Use p->cRowSep as the row separator. The default is "\x1E". ** + Keep track of the row number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -26601,14 +28618,14 @@ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ int cSep = (u8)p->cColSep; int rSep = (u8)p->cRowSep; p->n = 0; - c = fgetc(p->in); + c = import_getc(p); if( c==EOF || seenInterrupt ){ p->cTerm = EOF; return 0; } while( c!=EOF && c!=cSep && c!=rSep ){ import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); } if( c==rSep ){ p->nLine++; @@ -26643,7 +28660,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n", + cli_printf(stderr,"Error %d: %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_data_xfer; } @@ -26660,7 +28677,7 @@ static void tryToCloneData( memcpy(zInsert+i, ");", 3); rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0); if( rc ){ - sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n", + cli_printf(stderr,"Error %d: %s on [%s]\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert); goto end_data_xfer; } @@ -26696,7 +28713,7 @@ static void tryToCloneData( } /* End for */ rc = sqlite3_step(pInsert); if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - sqlite3_fprintf(stderr,"Error %d: %s\n", + cli_printf(stderr,"Error %d: %s\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb)); } sqlite3_reset(pInsert); @@ -26714,7 +28731,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable); + cli_printf(stderr,"Warning: cannot step \"%s\" backwards", zTable); break; } } /* End for(k=0...) */ @@ -26751,7 +28768,7 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; @@ -26761,10 +28778,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){ - sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout); + cli_printf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -26782,7 +28799,7 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n", + cli_printf(stderr,"Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } @@ -26791,10 +28808,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue; - sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout); + cli_printf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -26818,12 +28835,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ - sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb); + cli_printf(stderr,"File \"%s\" already exists.\n", zNewDb); return; } rc = sqlite3_open(zNewDb, &newDb); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Cannot create output database: %s\n", sqlite3_errmsg(newDb)); }else{ sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); @@ -26842,12 +28859,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ */ static void output_redir(ShellState *p, FILE *pfNew){ if( p->out != stdout ){ - sqlite3_fputs("Output already redirected.\n", stderr); + cli_puts("Output already redirected.\n", stderr); }else{ p->out = pfNew; setCrlfMode(p); - if( p->mode==MODE_Www ){ - sqlite3_fputs( + if( p->mode.eMode==MODE_Www ){ + cli_puts( "<!DOCTYPE html>\n" "<HTML><BODY><PRE>\n", p->out @@ -26869,8 +28886,8 @@ static void output_reset(ShellState *p){ pclose(p->out); #endif }else{ - if( p->mode==MODE_Www ){ - sqlite3_fputs("</PRE></BODY></HTML>\n", p->out); + if( p->mode.eMode==MODE_Www ){ + cli_puts("</PRE></BODY></HTML>\n", p->out); } output_file_close(p->out); #ifndef SQLITE_NOHAVE_SYSTEM @@ -26886,7 +28903,7 @@ static void output_reset(ShellState *p){ char *zCmd; zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ - sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd); + cli_printf(stderr,"Failed: [%s]\n", zCmd); }else{ /* Give the start/open/xdg-open command some time to get ** going before we continue, and potential delete the @@ -26894,7 +28911,7 @@ static void output_reset(ShellState *p){ sqlite3_sleep(2000); } sqlite3_free(zCmd); - outputModePop(p); + modePop(p); p->doXdgOpen = 0; } #endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ @@ -26902,6 +28919,10 @@ static void output_reset(ShellState *p){ p->outfile[0] = 0; p->out = stdout; setCrlfMode(p); + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + cli_output_capture = 0; + } } #else # define output_redir(SS,pfO) @@ -26933,10 +28954,13 @@ static int db_int(sqlite3 *db, const char *zSql, ...){ ** Convert a 2-byte or 4-byte big-endian integer into a native integer */ static unsigned int get2byteInt(unsigned char *a){ - return (a[0]<<8) + a[1]; + return ((unsigned int)a[0]<<8) + (unsigned int)a[1]; } static unsigned int get4byteInt(unsigned char *a){ - return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; + return ((unsigned int)a[0]<<24) + + ((unsigned int)a[1]<<16) + + ((unsigned int)a[2]<<8) + + (unsigned int)a[3]; } /* @@ -26983,7 +29007,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); + cli_printf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -26996,28 +29020,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ - sqlite3_fputs("unable to read database header\n", stderr); + cli_puts("unable to read database header\n", stderr); sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); if( i==1 ) i = 65536; - sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i); - sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]); - sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]); - sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + cli_printf(p->out, "%-20s %d\n", "database page size:", i); + cli_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); + cli_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); + cli_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); for(i=0; i<ArraySize(aField); i++){ int ofst = aField[i].ofst; unsigned int val = get4byteInt(aHdr + ofst); - sqlite3_fprintf(p->out, "%-20s %u", aField[i].zName, val); + cli_printf(p->out, "%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) sqlite3_fputs(" (utf8)", p->out); - if( val==2 ) sqlite3_fputs(" (utf16le)", p->out); - if( val==3 ) sqlite3_fputs(" (utf16be)", p->out); + if( val==1 ) cli_puts(" (utf8)", p->out); + if( val==2 ) cli_puts(" (utf16le)", p->out); + if( val==3 ) cli_puts(" (utf16be)", p->out); } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); @@ -27028,11 +29052,11 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } for(i=0; i<ArraySize(aQuery); i++){ int val = db_int(p->db, aQuery[i].zSql, zSchemaTab); - sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val); + cli_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); - sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion); + cli_printf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } #endif /* SQLITE_SHELL_HAVE_RECOVER */ @@ -27073,7 +29097,7 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ sqlite3_finalize(pStmt); pStmt = 0; if( nPage<1 ) goto dbtotxt_error; - rc = sqlite3_prepare_v2(p->db, "PRAGMA databases", -1, &pStmt, 0); + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ) goto dbtotxt_error; if( sqlite3_step(pStmt)!=SQLITE_ROW ){ zTail = "unk.db"; @@ -27084,10 +29108,15 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ #if defined(_WIN32) if( zTail==0 ) zTail = strrchr(zFilename, '\\'); #endif + if( zTail==0 ){ + zTail = zFilename; + }else if( zTail[1]!=0 ){ + zTail++; + } } zName = strdup(zTail); shell_check_oom(zName); - sqlite3_fprintf(p->out, "| size %lld pagesize %d filename %s\n", + cli_printf(p->out, "| size %lld pagesize %d filename %s\n", nPage*pgSz, pgSz, zName); sqlite3_finalize(pStmt); pStmt = 0; @@ -27103,27 +29132,27 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ for(j=0; j<16 && aLine[j]==0; j++){} if( j==16 ) continue; if( !seenPageLabel ){ - sqlite3_fprintf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); + cli_printf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); seenPageLabel = 1; } - sqlite3_fprintf(p->out, "| %5d:", i); - for(j=0; j<16; j++) sqlite3_fprintf(p->out, " %02x", aLine[j]); - sqlite3_fprintf(p->out, " "); + cli_printf(p->out, "| %5d:", i); + for(j=0; j<16; j++) cli_printf(p->out, " %02x", aLine[j]); + cli_printf(p->out, " "); for(j=0; j<16; j++){ unsigned char c = (unsigned char)aLine[j]; - sqlite3_fprintf(p->out, "%c", bShow[c]); + cli_printf(p->out, "%c", bShow[c]); } - sqlite3_fprintf(p->out, "\n"); + cli_printf(p->out, "\n"); } } sqlite3_finalize(pStmt); - sqlite3_fprintf(p->out, "| end %s\n", zName); + cli_printf(p->out, "| end %s\n", zName); free(zName); return 0; dbtotxt_error: if( rc ){ - sqlite3_fprintf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); + cli_printf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); } sqlite3_finalize(pStmt); free(zName); @@ -27134,7 +29163,7 @@ dbtotxt_error: ** Print the given string as an error message. */ static void shellEmitError(const char *zErr){ - sqlite3_fprintf(stderr,"Error: %s\n", zErr); + cli_printf(stderr,"Error: %s\n", zErr); } /* ** Print the current sqlite3_errmsg() value to stderr and return 1. @@ -27254,6 +29283,43 @@ static int optionMatch(const char *zStr, const char *zOpt){ } /* +** The input zFN is guaranteed to start with "file:" and is thus a URI +** filename. Extract the actual filename and return a pointer to that +** filename in spaced obtained from sqlite3_malloc(). +** +** The caller is responsible for freeing space using sqlite3_free() when +** it has finished with the filename. +*/ +static char *shellFilenameFromUri(const char *zFN){ + char *zOut; + int i, j, d1, d2; + + assert( cli_strncmp(zFN,"file:",5)==0 ); + zOut = sqlite3_mprintf("%s", zFN+5); + shell_check_oom(zOut); + for(i=j=0; zOut[i]!=0 && zOut[i]!='?'; i++){ + if( zOut[i]!='%' ){ + zOut[j++] = zOut[i]; + continue; + } + d1 = hexDigitValue(zOut[i+1]); + if( d1<0 ){ + zOut[j] = 0; + break; + } + d2 = hexDigitValue(zOut[i+2]); + if( d2<0 ){ + zOut[j] = 0; + break; + } + zOut[j++] = d1*16 + d2; + i += 2; + } + zOut[j] = 0; + return zOut; +} + +/* ** Delete a file. */ int shellDeleteFile(const char *zFilename){ @@ -27280,39 +29346,42 @@ static void clearTempFile(ShellState *p){ p->zTempFile = 0; } +/* Forward reference */ +static char *find_home_dir(int clearFlag); + /* ** Create a new temp file name with the given suffix. +** +** Because the classic temp folders like /tmp are no longer +** accessible to web browsers, for security reasons, create the +** temp file in the user's home directory. */ static void newTempFile(ShellState *p, const char *zSuffix){ - clearTempFile(p); - sqlite3_free(p->zTempFile); - p->zTempFile = 0; - if( p->db ){ - sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); - } - if( p->zTempFile==0 ){ - /* If p->db is an in-memory database then the TEMPFILENAME file-control - ** will not work and we will need to fallback to guessing */ - char *zTemp; - sqlite3_uint64 r; - sqlite3_randomness(sizeof(r), &r); - zTemp = getenv("TEMP"); - if( zTemp==0 ) zTemp = getenv("TMP"); - if( zTemp==0 ){ + char *zHome; /* Home directory */ + int i; /* Loop counter */ + sqlite3_uint64 r = 0; /* Integer with 64 bits of randomness */ + char zRand[32]; /* Text string with 160 bits of randomness */ #ifdef _WIN32 - zTemp = "\\tmp"; + const char cDirSep = '\\'; #else - zTemp = "/tmp"; + const char cDirSep = '/'; #endif - } - p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); - }else{ - p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); + + for(i=0; i<31; i++){ + if( (i%12)==0 ) sqlite3_randomness(sizeof(r),&r); + zRand[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[r%36]; + r /= 36; } + zRand[i] = 0; + clearTempFile(p); + sqlite3_free(p->zTempFile); + p->zTempFile = 0; + zHome = find_home_dir(0); + p->zTempFile = sqlite3_mprintf("%s%ctemp-%s.%s", + zHome,cDirSep,zRand,zSuffix); shell_check_oom(p->zTempFile); } - /* ** The implementation of SQL scalar function fkey_collate_clause(), used ** by the ".lint fkey-indexes" command. This scalar function is always @@ -27457,7 +29526,7 @@ static int lintFkeyIndexes( zIndent = " "; } else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]); return SQLITE_ERROR; } @@ -27502,22 +29571,22 @@ static int lintFkeyIndexes( if( rc!=SQLITE_OK ) break; if( res<0 ){ - sqlite3_fputs("Error: internal error", stderr); + cli_puts("Error: internal error", stderr); break; }else{ if( bGroupByParent && (bVerbose || res==0) && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) ){ - sqlite3_fprintf(out, "-- Parent table %s\n", zParent); + cli_printf(out, "-- Parent table %s\n", zParent); sqlite3_free(zPrev); zPrev = sqlite3_mprintf("%s", zParent); } if( res==0 ){ - sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + cli_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); }else if( bVerbose ){ - sqlite3_fprintf(out, + cli_printf(out, "%s/* no extra indexes required for %s -> %s */\n", zIndent, zFrom, zTarget ); @@ -27527,16 +29596,16 @@ static int lintFkeyIndexes( sqlite3_free(zPrev); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } rc2 = sqlite3_finalize(pSql); if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ rc = rc2; - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } }else{ - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } return rc; @@ -27556,9 +29625,9 @@ static int lintDotCommand( return lintFkeyIndexes(pState, azArg, nArg); usage: - sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); - sqlite3_fprintf(stderr, "Where sub-commands are:\n"); - sqlite3_fprintf(stderr, " fkey-indexes\n"); + cli_printf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); + cli_printf(stderr, "Where sub-commands are:\n"); + cli_printf(stderr, " fkey-indexes\n"); return SQLITE_ERROR; } @@ -27572,7 +29641,7 @@ static void shellPrepare( if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db)); *pRc = rc; } @@ -27617,7 +29686,7 @@ static void shellFinalize( int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -27639,7 +29708,7 @@ void shellReset( if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ sqlite3 *db = sqlite3_db_handle(pStmt); - sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -27692,9 +29761,9 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_end(ap); shellEmitError(z); if( pAr->fromCmdLine ){ - sqlite3_fputs("Use \"-A\" for more help\n", stderr); + cli_puts("Use \"-A\" for more help\n", stderr); }else{ - sqlite3_fputs("Use \".archive --help\" for more help\n", stderr); + cli_puts("Use \".archive --help\" for more help\n", stderr); } sqlite3_free(z); return SQLITE_ERROR; @@ -27794,7 +29863,7 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ - sqlite3_fprintf(stderr, "Wrong number of arguments. Usage:\n"); + cli_printf(stderr, "Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; @@ -27900,7 +29969,7 @@ static int arParseCommand( } } if( pAr->eCmd==0 ){ - sqlite3_fprintf(stderr, "Required argument missing. Usage:\n"); + cli_printf(stderr, "Required argument missing. Usage:\n"); return arUsage(stderr); } return SQLITE_OK; @@ -27943,7 +30012,7 @@ static int arCheckEntries(ArCommand *pAr){ } shellReset(&rc, pTest); if( rc==SQLITE_OK && bOk==0 ){ - sqlite3_fprintf(stderr,"not found in archive: %s\n", z); + cli_printf(stderr,"not found in archive: %s\n", z); rc = SQLITE_ERROR; } } @@ -27966,25 +30035,41 @@ static void arWhereClause( char **pzWhere /* OUT: New WHERE clause */ ){ char *zWhere = 0; - const char *zSameOp = (pAr->bGlob)? "GLOB" : "="; if( *pRc==SQLITE_OK ){ if( pAr->nArg==0 ){ zWhere = sqlite3_mprintf("1"); }else{ + char *z1 = sqlite3_mprintf(pAr->bGlob ? "" : "name IN("); + char *z2 = sqlite3_mprintf(""); + const char *zSep1 = ""; + const char *zSep2 = ""; + int i; - const char *zSep = ""; - for(i=0; i<pAr->nArg; i++){ + for(i=0; i<pAr->nArg && z1 && z2; i++){ const char *z = pAr->azArg[i]; - zWhere = sqlite3_mprintf( - "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'", - zWhere, zSep, zSameOp, z, strlen30(z)+1, zSameOp, z - ); - if( zWhere==0 ){ - *pRc = SQLITE_NOMEM; - break; + int n = strlen30(z); + + if( pAr->bGlob ){ + z1 = sqlite3_mprintf("%z%sname GLOB '%q'", z1, zSep2, z); + z2 = sqlite3_mprintf( + "%z%ssubstr(name,1,%d) GLOB '%q/'", z2, zSep2, n+1,z + ); + }else{ + z1 = sqlite3_mprintf("%z%s'%q'", z1, zSep1, z); + z2 = sqlite3_mprintf("%z%ssubstr(name,1,%d) = '%q/'",z2,zSep2,n+1,z); } - zSep = " OR "; + zSep1 = ", "; + zSep2 = " OR "; + } + if( z1==0 || z2==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + zWhere = sqlite3_mprintf("(%s%s OR (name GLOB '*/*' AND (%s))) ", + z1, pAr->bGlob==0 ? ")" : "", z2 + ); } + sqlite3_free(z1); + sqlite3_free(z2); } } *pzWhere = zWhere; @@ -28010,15 +30095,15 @@ static int arListCommand(ArCommand *pAr){ shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); + cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ - sqlite3_fprintf(pAr->out, "%s % 10d %s %s\n", + cli_printf(pAr->out, "%s % 10d %s %s\n", sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1), sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3)); }else{ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); + cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -28045,7 +30130,7 @@ static int arRemoveCommand(ArCommand *pAr){ zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;", pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", zSql); + cli_printf(pAr->out, "%s\n", zSql); }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0); @@ -28058,7 +30143,7 @@ static int arRemoveCommand(ArCommand *pAr){ } } if( zErr ){ - sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */ + cli_printf(stdout, "ERROR: %s\n", zErr); /* stdout? */ sqlite3_free(zErr); } } @@ -28073,11 +30158,15 @@ static int arRemoveCommand(ArCommand *pAr){ */ static int arExtractCommand(ArCommand *pAr){ const char *zSql1 = - "SELECT " - " ($dir || name)," - " writefile(($dir || name), %s, mode, mtime) " - "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" - " AND name NOT GLOB '*..[/\\]*'"; + "WITH dest(dpath,dlen) AS (SELECT realpath($dir),length(realpath($dir)))\n" + "SELECT ($dir || name),\n" + " CASE WHEN $dryrun THEN 0\n" + " ELSE writefile($dir||name, %s, mode, mtime) END\n" + " FROM dest CROSS JOIN %s\n" + " WHERE (%s)\n" + " AND (data IS NULL OR $pass==0)\n" /* Dirs both passes */ + " AND dpath=substr(realpath($dir||name),1,dlen)\n" /* No escapes */ + " AND name NOT GLOB '*..[/\\]*'\n"; /* No /../ in paths */ const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", @@ -28112,24 +30201,28 @@ static int arExtractCommand(ArCommand *pAr){ if( rc==SQLITE_OK ){ j = sqlite3_bind_parameter_index(pSql, "$dir"); sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC); + j = sqlite3_bind_parameter_index(pSql, "$dryrun"); + sqlite3_bind_int(pSql, j, pAr->bDryRun); - /* Run the SELECT statement twice. The first time, writefile() is called - ** for all archive members that should be extracted. The second time, - ** only for the directories. This is because the timestamps for - ** extracted directories must be reset after they are populated (as - ** populating them changes the timestamp). */ + /* Run the SELECT statement twice + ** (0) writefile() all files and directories + ** (1) writefile() for directory again + ** The second pass is so that the timestamps for extracted directories + ** will be reset to the value in the archive, since populating them + ** in the first pass will have changed the timestamp. */ for(i=0; i<2; i++){ - j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); + j = sqlite3_bind_parameter_index(pSql, "$pass"); sqlite3_bind_int(pSql, j, i); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); - }else{ - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - if( i==0 && pAr->bVerbose ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); - } + cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); + if( pAr->bVerbose==0 ) break; + } + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } + if( pAr->bDryRun ) break; shellReset(&rc, pSql); } shellFinalize(&rc, pSql); @@ -28146,13 +30239,13 @@ static int arExtractCommand(ArCommand *pAr){ static int arExecSql(ArCommand *pAr, const char *zSql){ int rc; if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", zSql); + cli_printf(pAr->out, "%s\n", zSql); rc = SQLITE_OK; }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); if( zErr ){ - sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); + cli_printf(stdout, "ERROR: %s\n", zErr); sqlite3_free(zErr); } } @@ -28304,7 +30397,7 @@ static int arDotCommand( cmd.out = pState->out; cmd.db = pState->db; if( cmd.zFile ){ - eDbType = deduceDatabaseType(cmd.zFile, 1); + eDbType = deduceDatabaseType(cmd.zFile, 1, 0); }else{ eDbType = pState->openMode; } @@ -28328,13 +30421,13 @@ static int arDotCommand( } cmd.db = 0; if( cmd.bDryRun ){ - sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, + cli_printf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n", + cli_printf(stderr, "cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db)); goto end_ar_command; } @@ -28348,7 +30441,7 @@ static int arDotCommand( if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ - sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n"); + cli_printf(stderr, "database does not contain an 'sqlar' table\n"); rc = SQLITE_ERROR; goto end_ar_command; } @@ -28406,7 +30499,7 @@ end_ar_command: */ static int recoverSqlCb(void *pCtx, const char *zSql){ ShellState *pState = (ShellState*)pCtx; - sqlite3_fprintf(pState->out, "%s;\n", zSql); + cli_printf(pState->out, "%s;\n", zSql); return SQLITE_OK; } @@ -28449,7 +30542,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ bRowids = 0; } else{ - sqlite3_fprintf(stderr,"unexpected option: %s\n", azArg[i]); + cli_printf(stderr,"unexpected option: %s\n", azArg[i]); showHelp(pState->out, azArg[0]); return 1; } @@ -28459,17 +30552,19 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ pState->db, "main", recoverSqlCb, (void*)pState ); - sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + if( !pState->bSafeMode ){ + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + } sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); - sqlite3_fprintf(pState->out, ".dbconfig defensive off\n"); + cli_printf(pState->out, ".dbconfig defensive off\n"); sqlite3_recover_run(p); if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); int errCode = sqlite3_recover_errcode(p); - sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode); + cli_printf(stderr,"sql error: %s (%d)\n", zErr, errCode); } rc = sqlite3_recover_finish(p); return rc; @@ -28491,7 +30586,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ while( SQLITE_OK==sqlite3_intck_step(p) ){ const char *zMsg = sqlite3_intck_message(p); if( zMsg ){ - sqlite3_fprintf(pState->out, "%s\n", zMsg); + cli_printf(pState->out, "%s\n", zMsg); nError++; } nStep++; @@ -28501,11 +30596,11 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ } rc = sqlite3_intck_error(p, &zErr); if( zErr ){ - sqlite3_fprintf(stderr,"%s\n", zErr); + cli_printf(stderr,"%s\n", zErr); } sqlite3_intck_close(p); - sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError); + cli_printf(pState->out, "%lld steps, %lld errors\n", nStep, nError); } return rc; @@ -28528,7 +30623,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ #define rc_err_oom_die(rc) \ if( rc==SQLITE_NOMEM ) shell_check_oom(0); \ else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \ - sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0) + cli_printf(stderr,"E:%d\n",rc), assert(0) #else static void rc_err_oom_die(int rc){ if( rc==SQLITE_NOMEM ) shell_check_oom(0); @@ -28643,7 +30738,7 @@ SELECT CASE WHEN (nc < 10) THEN 1 WHEN (nc < 100) THEN 2 \ SELECT\ '('||x'0a'\ || group_concat(\ - cname||' TEXT',\ + cname||' ANY',\ ','||iif((cpos-1)%4>0, ' ', x'0a'||' '))\ ||')' AS ColsSpec \ FROM (\ @@ -28742,10 +30837,10 @@ static int outputDumpWarning(ShellState *p, const char *zLike){ sqlite3_stmt *pStmt = 0; shellPreparePrintf(p->db, &rc, &pStmt, "SELECT 1 FROM sqlite_schema o WHERE " - "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" + "sql LIKE 'CREATE VIRTUAL TABLE%%' AND (%s)", zLike ? zLike : "true" ); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_fputs("/* WARNING: " + cli_puts("/* WARNING: " "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", p->out ); @@ -28778,13 +30873,13 @@ static int faultsim_callback(int iArg){ if( faultsim_state.iCnt ){ if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; if( faultsim_state.eVerbose>=2 ){ - sqlite3_fprintf(stdout, + cli_printf(stdout, "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); } return SQLITE_OK; } if( faultsim_state.eVerbose>=1 ){ - sqlite3_fprintf(stdout, + cli_printf(stdout, "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); } faultsim_state.iCnt = faultsim_state.iInterval; @@ -28796,47 +30891,1595 @@ static int faultsim_callback(int iArg){ } /* +** pickStr(zArg, &zErr, zS1, zS2, ..., ""); +** +** Try to match zArg against zS1, zS2, and so forth until the first +** emptry string. Return the index of the match or -1 if none is found. +** If no match is found, and &zErr is not NULL, then write into +** zErr a message describing the valid choices. +*/ +static int pickStr(const char *zArg, char **pzErr, ...){ + int i, n; + const char *z; + sqlite3_str *pMsg; + va_list ap; + va_start(ap, pzErr); + i = 0; + while( (z = va_arg(ap,const char*))!=0 && z[0]!=0 ){ + if( cli_strcmp(zArg, z)==0 ) return i; + i++; + } + va_end(ap); + if( pzErr==0 ) return -1; + n = i; + pMsg = sqlite3_str_new(0); + va_start(ap, pzErr); + sqlite3_str_appendall(pMsg, "should be"); + i = 0; + while( (z = va_arg(ap, const char*))!=0 && z[0]!=0 ){ + if( i==n-1 ){ + sqlite3_str_append(pMsg,", or",4); + }else if( i>0 ){ + sqlite3_str_append(pMsg, ",", 1); + } + sqlite3_str_appendf(pMsg, " %s", z); + i++; + } + va_end(ap); + *pzErr = sqlite3_str_finish(pMsg); + return -1; +} + +/* +** DOT-COMMAND: .import +** +** USAGE: .import [OPTIONS] FILE TABLE +** +** Import CSV or similar text from FILE into TABLE. If TABLE does +** not exist, it is created using the first row of FILE as the column +** names. If FILE begins with "|" then it is a command that is run +** and the output from the command is used as the input data. If +** FILE begins with "<<" followed by a label, then content is read from +** the script until the first line that matches the label. +** +** The content of FILE is interpreted using RFC-4180 ("CSV") quoting +** rules unless the current mode is "ascii" or "tabs" or unless one +** the --ascii option is used. +** +** The column and row separators must be single ASCII characters. If +** multiple characters or a Unicode character are specified for the +** separators, then only the first byte of the separator is used. Except, +** if the row separator is \n and the mode is not --ascii, then \r\n is +** understood as a row separator too. +** +** Options: +** --ascii Do not use RFC-4180 quoting. Use \037 and \036 +** as column and row separators on input, unless other +** delimiters are specified using --colsep and/or --rowsep +** --colsep CHAR Use CHAR as the column separator. +** --csv Input is standard RFC-4180 CSV. +** --esc CHAR Use CHAR as an escape character in unquoted CSV inputs. +** --qesc CHAR Use CHAR as an escape character in quoted CSV inputs. +** --rowsep CHAR Use CHAR as the row separator. +** --schema S When creating TABLE, put it in schema S +** --skip N Ignore the first N rows of input +** -v Verbose mode +*/ +static int dotCmdImport(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg;/* Argument list */ + char *zTable = 0; /* Insert data into this table */ + char *zSchema = 0; /* Schema of zTable */ + char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + i64 nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int needCommit; /* True to COMMIT or ROLLBACK at end */ + char *zSql = 0; /* An SQL statement */ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int eVerbose = 0; /* Larger for more console output */ + i64 nSkip = 0; /* Initial lines to skip */ + i64 iLineOffset = 0; /* Offset to the first line of input */ + char *zCreate = 0; /* CREATE TABLE statement text */ + int rc; /* Result code */ + + failIfSafeMode(p, "cannot run .import in safe mode"); + memset(&sCtx, 0, sizeof(sCtx)); + if( p->mode.eMode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' ) z++; + if( z[0]!='-' ){ + if( zFile==0 ){ + zFile = z; + }else if( zTable==0 ){ + zTable = z; + }else{ + dotCmdError(p, i, "unknown argument", 0); + return 1; + } + }else if( cli_strcmp(z,"-v")==0 ){ + eVerbose++; + }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ + zSchema = azArg[++i]; + }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ + nSkip = integerValue(azArg[++i]); + }else if( cli_strcmp(z,"-ascii")==0 ){ + if( sCtx.cColSep==0 ) sCtx.cColSep = SEP_Unit[0]; + if( sCtx.cRowSep==0 ) sCtx.cRowSep = SEP_Record[0]; + xRead = ascii_read_one_field; + }else if( cli_strcmp(z,"-csv")==0 ){ + if( sCtx.cColSep==0 ) sCtx.cColSep = ','; + if( sCtx.cRowSep==0 ) sCtx.cRowSep = '\n'; + xRead = csv_read_one_field; + }else if( cli_strcmp(z,"-esc")==0 ){ + sCtx.cUQEscape = azArg[++i][0]; + }else if( cli_strcmp(z,"-qesc")==0 ){ + sCtx.cQEscape = azArg[++i][0]; + }else if( cli_strcmp(z,"-colsep")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + sCtx.cColSep = azArg[i][0]; + }else if( cli_strcmp(z,"-rowsep")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + sCtx.cRowSep = azArg[i][0]; + }else{ + dotCmdError(p, i, "unknown option", 0); + return 1; + } + } + if( zTable==0 ){ + dotCmdError(p, nArg, 0, "Missing %s argument\n", + zFile==0 ? "FILE" : "TABLE"); + return 1; + } + seenInterrupt = 0; + open_db(p, 0); + if( sCtx.cColSep==0 ){ + if( p->mode.spec.zColumnSep && p->mode.spec.zColumnSep[0]!=0 ){ + sCtx.cColSep = p->mode.spec.zColumnSep[0]; + }else{ + sCtx.cColSep = ','; + } + } + if( (sCtx.cColSep & 0x80)!=0 ){ + eputz("Error: .import column separator must be ASCII\n"); + return 1; + } + if( sCtx.cRowSep==0 ){ + if( p->mode.spec.zRowSep && p->mode.spec.zRowSep[0]!=0 ){ + sCtx.cRowSep = p->mode.spec.zRowSep[0]; + }else{ + sCtx.cRowSep = '\n'; + } + } + if( sCtx.cRowSep=='\r' && xRead!=ascii_read_one_field ){ + sCtx.cRowSep = '\n'; + } + if( (sCtx.cRowSep & 0x80)!=0 ){ + eputz("Error: .import row separator must be ASCII\n"); + return 1; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + return 1; +#else + sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); + sCtx.zFile = "<pipe>"; + sCtx.xCloser = pclose; +#endif + }else if( sCtx.zFile[0]=='<' && sCtx.zFile[1]=='<' && sCtx.zFile[2]!=0 ){ + /* Input text comes from subsequent lines of script until the zFile + ** delimiter */ + int nEndMark = strlen30(zFile)-2; + char *zEndMark = &zFile[2]; + sqlite3_str *pContent = sqlite3_str_new(p->db); + int ckEnd = 1; + i64 iStart = p->lineno; + char zLine[2000]; + sCtx.zFile = p->zInFile; + sCtx.nLine = p->lineno+1; + iLineOffset = p->lineno; + while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ + if( ckEnd && cli_strncmp(zLine,zEndMark,nEndMark)==0 ){ + ckEnd = 2; + if( strchr(zLine,'\n') ) p->lineno++; + break; + } + if( strchr(zLine,'\n') ){ + p->lineno++; + ckEnd = 1; + }else{ + ckEnd = 0; + } + sqlite3_str_appendall(pContent, zLine); + } + sCtx.zIn = sqlite3_str_finish(pContent); + if( sCtx.zIn==0 ){ + sCtx.zIn = sqlite3_mprintf(""); + } + if( ckEnd<2 ){ + i64 savedLn = p->lineno; + p->lineno = iStart; + dotCmdError(p, 0, 0,"Content terminator \"%s\" not found.",zEndMark); + p->lineno = savedLn; + import_cleanup(&sCtx); + return 1; + } + }else{ + sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); + sCtx.xCloser = fclose; + } + if( sCtx.in==0 && sCtx.zIn==0 ){ + dotCmdError(p, 0, 0, "cannot open \"%s\"", zFile); + import_cleanup(&sCtx); + return 1; + } + if( eVerbose>=1 ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + cli_puts("Column separator ", p->out); + output_c_string(p->out, zSep); + cli_puts(", row separator ", p->out); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + cli_puts("\n", p->out); + } + sCtx.z = sqlite3_malloc64(120); + if( sCtx.z==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + /* Below, resources must be freed before exit. */ + while( nSkip>0 ){ + nSkip--; + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } + import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) + && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" + " WHERE name=%Q AND type='view'", + zSchema ? zSchema : "main", zTable) + ){ + /* Table does not exist. Create it. */ + sqlite3 *dbCols = 0; + char *zRenames = 0; + char *zColDefs; + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); + while( xRead(&sCtx) ){ + zAutoColumn(sCtx.z, &dbCols, 0); + if( sCtx.cTerm!=sCtx.cColSep ) break; + } + zColDefs = zAutoColumn(0, &dbCols, &zRenames); + if( zRenames!=0 ){ + cli_printf((stdin_is_interactive && p->in==stdin)? p->out : stderr, + "Columns renamed during .import %s due to duplicates:\n" + "%s\n", sCtx.zFile, zRenames); + sqlite3_free(zRenames); + } + assert(dbCols==0); + if( zColDefs==0 ){ + cli_printf(stderr,"%s: empty file\n", sCtx.zFile); + import_cleanup(&sCtx); + sqlite3_free(zCreate); + return 1; + } + zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( eVerbose>=1 ){ + cli_printf(p->out, "%s\n", zCreate); + } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + if( rc ){ + cli_printf(stderr, + "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + } + sqlite3_free(zCreate); + zCreate = 0; + if( rc ){ + import_cleanup(&sCtx); + return 1; + } + } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + shellDatabaseError(p->db); + import_cleanup(&sCtx); + return 1; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } + j = strlen30(zSql); + for(i=1; i<nCol; i++){ + zSql[j++] = ','; + zSql[j++] = '?'; + } + zSql[j++] = ')'; + zSql[j] = 0; + assert( j<nByte ); + if( eVerbose>=2 ){ + cli_printf(p->out, "Insert using: %s\n", zSql); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + shellDatabaseError(p->db); + if (pStmt) sqlite3_finalize(pStmt); + import_cleanup(&sCtx); + return 1; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + do{ + int startLine = sCtx.nLine; + for(i=0; i<nCol; i++){ + char *z = xRead(&sCtx); + /* + ** Did we reach end-of-file before finding any columns? + ** If so, stop instead of NULL filling the remaining columns. + */ + if( z==0 && i==0 ) break; + /* + ** Did we reach end-of-file OR end-of-line before finding any + ** columns in ASCII mode? If so, stop instead of NULL filling + ** the remaining columns. + */ + if( p->mode.eMode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + /* + ** For CSV mode, per RFC 4180, accept EOF in lieu of final + ** record terminator but only for last field of multi-field row. + ** (If there are too few fields, it's not valid CSV anyway.) + */ + if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ + z = ""; + } + sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); + if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ + if( i==0 && (strcmp(z,"\n")==0 || strcmp(z,"\r\n")==0) ){ + /* Ignore trailing \n or \r\n when some other row separator */ + break; + } + cli_printf(stderr,"%s:%d: expected %d columns but found %d" + " - filling the rest with NULL\n", + sCtx.zFile, startLine, nCol, i+1); + i += 2; + while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } + } + } + if( sCtx.cTerm==sCtx.cColSep ){ + do{ + xRead(&sCtx); + i++; + }while( sCtx.cTerm==sCtx.cColSep ); + cli_printf(stderr, + "%s:%d: expected %d columns but found %d - extras ignored\n", + sCtx.zFile, startLine, nCol, i); + } + if( i>=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + cli_printf(stderr,"%s:%d: INSERT failed: %s\n", + sCtx.zFile, startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + if( bail_on_error ) break; + }else{ + sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); + + import_cleanup(&sCtx); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + cli_printf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1-iLineOffset); + } + return sCtx.nErr ? 1 : 0; +} + + +/* +** This function computes what to show the user about the configured +** titles (or column-names). Output is an integer between 0 and 3: +** +** 0: The titles do not matter. Never show anything. +** 1: Show "--titles off" +** 2: Show "--titles on" +** 3: Show "--title VALUE" where VALUE is an encoding method +** to use, one of: plain sql csv html tcl json +** +** Inputs are: +** +** spec.bTitles (bT) Whether or not to show the titles +** spec.eTitle (eT) The actual encoding to be used for titles +** ModeInfo.bHdr (bH) Default value for spec.bTitles +** ModeInfo.eHdr (eH) Default value for spec.eTitle +** bAll Whether the -v option is used +*/ +static int modeTitleDsply(ShellState *p, int bAll){ + int eMode = p->mode.eMode; + const ModeInfo *pI = &aModeInfo[eMode]; + int bT = p->mode.spec.bTitles; + int eT = p->mode.spec.eTitle; + int bH = pI->bHdr; + int eH = pI->eHdr; + + /* Variable "v" is the truth table that will determine the answer + ** + ** Actual encoding is different from default + ** vvvvvvvv */ + sqlite3_uint64 v = UINT64_C(0x0133013311220102); + /* ^^^^ ^^^^ + ** Upper 2-byte groups for when ON/OFF disagrees with + ** the default. */ + + if( bH==0 ) return 0; /* Header not appliable. Ex: off, count */ + + if( eT==0 ) eT = eH; /* Fill in missing spec.eTitle */ + if( bT==0 ) bT = bH; /* Fill in missing spec.bTitles */ + + if( eT!=eH ) v >>= 32; /* Encoding disagree in upper 4-bytes */ + if( bT!=bH ) v >>= 16; /* ON/OFF disagree in upper 2-byte pairs */ + if( bT<2 ) v >>= 8; /* ON in even bytes, OFF in odd bytes (1st byte 0) */ + if( !bAll ) v >>= 4; /* bAll values are in the lower half-byte */ + + return v & 3; /* Return the selected truth-table entry */ +} + +/* +** DOT-COMMAND: .mode +** +** USAGE: .mode [MODE] [OPTIONS] +** +** Change the output mode to MODE and/or apply OPTIONS to the output mode. +** Arguments are processed from left to right. If no arguments, show the +** current output mode and relevant options. +** +** Options: +** --align STRING Set the alignment of text in columnar modes +** String consists of characters 'L', 'C', 'R' +** meaning "left", "centered", and "right", with +** one letter per column starting from the left. +** Unspecified alignment defaults to 'L'. +** --blob-quote ARG ARG can be "auto", "text", "sql", "hex", "tcl", +** "json", or "size". Default is "auto". +** --border on|off Show outer border on "box" and "table" modes. +** --charlimit N Set the maximum number of output characters to +** show for any single SQL value to N. Longer values +** truncated. Zero means "no limit". +** --colsep STRING Use STRING as the column separator +** --escape ESC Enable/disable escaping of control characters +** found in the output. ESC can be "off", "ascii", +** or "symbol". +** --linelimit N Set the maximum number of output lines to show for +** any single SQL value to N. Longer values are +** truncated. Zero means "no limit". Only works +** in "line" mode and in columnar modes. +** --limits L,C,T Shorthand for "--linelimit L --charlimit C +** --titlelimit T". The ",T" can be omitted in which +** case the --titlelimit is unchanged. The argument +** can also be "off" to mean "0,0,0" or "on" to +** mean "5,300,20". +** --list List available modes +** --multiinsert N In "insert" mode, put multiple rows on a single +** INSERT statement until the size exceeds N bytes. +** --null STRING Render SQL NULL values as the given string +** --once Setting changes to the right are reverted after +** the next SQL command. +** --quote ARG Enable/disable quoting of text. ARG can be +** "off", "on", "sql", "relaxed", "csv", "html", +** "tcl", or "json". "off" means show the text as-is. +** "on" is an alias for "sql". +** --reset Changes all mode settings back to their default. +** --rowsep STRING Use STRING as the row separator +** --sw|--screenwidth N Declare the screen width of the output device +** to be N characters. An attempt may be made to +** wrap output text to fit within this limit. Zero +** means "no limit". Or N can be "auto" to set the +** width automatically. +** --tablename NAME Set the name of the table for "insert" mode. +** --tag NAME Save mode to the left as NAME. +** --textjsonb BOOLEAN If enabled, JSONB text is displayed as text JSON. +** --title ARG Whether or not to show column headers, and if so +** how to encode them. ARG can be "off", "on", +** "sql", "csv", "html", "tcl", or "json". +** --titlelimit N Limit the length of column titles to N characters. +** -v|--verbose Verbose output +** --widths LIST Set the columns widths for columnar modes. The +** argument is a list of integers, one for each +** column. A "0" width means use a dynamic width +** based on the actual width of data. If there are +** fewer entries in LIST than columns, "0" is used +** for the unspecified widths. +** --wordwrap BOOLEAN Enable/disable word wrapping +** --wrap N Wrap columns wider than N characters +** --ww Shorthand for "--wordwrap on" +*/ +static int dotCmdMode(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg;/* Argument list */ + int eMode = -1; /* New mode value, or -1 for none */ + int iMode = -1; /* Index of the argument that is the mode name */ + int i; /* Loop counter */ + int k; /* Misc index variable */ + int chng = 0; /* True if anything has changed */ + int bAll = 0; /* Show all details of the mode */ + + for(i=1; i<nArg; i++){ + const char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' ) z++; + if( z[0]!='-' + && iMode<0 + && (eMode = modeFind(p, azArg[i]))>=0 + && eMode!=MODE_Www + ){ + iMode = i; + modeChange(p, eMode); + /* (Legacy) If the mode is MODE_Insert and the next argument + ** is not an option, then the next argument must be the table + ** name. + */ + if( i+1<nArg && azArg[i+1][0]!='-' ){ + i++; + modeSetStr(&p->mode.spec.zTableName, azArg[i]); + } + chng = 1; + }else if( optionMatch(z,"align") ){ + char *zAlign; + int nAlign; + int nErr = 0; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + zAlign = azArg[i]; + nAlign = 0x3fff & strlen(zAlign); + free(p->mode.spec.aAlign); + p->mode.spec.aAlign = malloc(nAlign); + shell_check_oom(p->mode.spec.aAlign); + for(k=0; k<nAlign; k++){ + unsigned char c = 0; + switch( zAlign[k] ){ + case 'l': case 'L': c = QRF_ALIGN_Left; break; + case 'c': case 'C': c = QRF_ALIGN_Center; break; + case 'r': case 'R': c = QRF_ALIGN_Right; break; + default: nErr++; break; + } + p->mode.spec.aAlign[k] = c; + } + p->mode.spec.nAlign = nAlign; + chng = 1; + if( nErr ){ + dotCmdError(p, i, "bad alignment string", + "Should contain only characters L, C, and R."); + return 1; + } + }else if( pickStr(z,0,"-blob","-blob-quote","")>=0 ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i], 0, + "auto", "text", "sql", "hex", "tcl", "json", "size", ""); + /* 0 1 2 3 4 5 6 + ** Must match QRF_BLOB_xxxx values. See also tag-20251124a */ + if( k>=0 ){ + p->mode.spec.eBlob = k & 0xff; + } + chng = 1; + }else if( optionMatch(z,"border") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i], 0, "auto", "off", "on", ""); + if( k>=0 ){ + p->mode.spec.bBorder = k & 0x3; + } + chng = 1; + }else if( 0<=(k=pickStr(z,0, + "-charlimit","-linelimit","-titlelimit","-multiinsert","")) ){ + int w; /* 0 1 2 3 */ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + w = integerValue(azArg[++i]); + switch( k ){ + case 0: p->mode.spec.nCharLimit = w; break; + case 1: p->mode.spec.nLineLimit = w; break; + case 2: p->mode.spec.nTitleLimit = w; break; + default: p->mode.spec.nMultiInsert = w; break; + } + chng = 1; + }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){ + /* 0 1 2 3 */ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + switch( k ){ + case 0: modeSetStr(&p->mode.spec.zTableName, azArg[i]); break; + case 1: modeSetStr(&p->mode.spec.zRowSep, azArg[i]); break; + case 2: modeSetStr(&p->mode.spec.zColumnSep, azArg[i]); break; + case 3: modeSetStr(&p->mode.spec.zNull, azArg[i]); break; + } + chng = 1; + }else if( optionMatch(z,"escape") ){ + /* See similar code at tag-20250224-1 */ + char *zErr = 0; + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } /* 0 1 2 <-- One less than QRF_ESC_ */ + k = pickStr(azArg[i],&zErr,"off","ascii","symbol",""); + if( k<0 ){ + dotCmdError(p, i, "unknown escape type", "%s", zErr); + sqlite3_free(zErr); + return 1; + } + p->mode.spec.eEsc = k+1; + chng = 1; + }else if( optionMatch(z,"limits") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i],0,"on","off",""); + if( k==0 ){ + p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; + p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; + p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + }else if( k==1 ){ + p->mode.spec.nLineLimit = 0; + p->mode.spec.nCharLimit = 0; + p->mode.spec.nTitleLimit = 0; + }else{ + int L, C, T = 0; + int nNum = sscanf(azArg[i], "%d,%d,%d", &L, &C, &T); + if( nNum<2 || L<0 || C<0 || T<0){ + dotCmdError(p, i, "bad argument", "Should be \"L,C,T\" where L, C" + " and T are unsigned integers"); + return 1; + } + p->mode.spec.nLineLimit = L; + p->mode.spec.nCharLimit = C; + if( nNum==3 ) p->mode.spec.nTitleLimit = T; + } + chng = 1; + }else if( optionMatch(z,"list") ){ + int ii; + cli_puts("available modes:", p->out); + for(ii=0; ii<ArraySize(aModeInfo); ii++){ + if( ii==MODE_Www ) continue; + cli_printf(p->out, " %s", aModeInfo[ii].zName); + } + for(ii=0; ii<p->nSavedModes; ii++){ + cli_printf(p->out, " %s", p->aSavedModes[ii].zTag); + } + cli_puts(" batch tty\n", p->out); + chng = 1; /* Not really a change, but we still want to suppress the + ** "current mode" output */ + }else if( optionMatch(z,"once") ){ + p->nPopMode = 0; + modePush(p); + p->nPopMode = 1; + }else if( optionMatch(z,"noquote") ){ + /* (undocumented legacy) --noquote always turns quoting off */ + p->mode.spec.eText = QRF_TEXT_Plain; + p->mode.spec.eBlob = QRF_BLOB_Auto; + chng = 1; + }else if( optionMatch(z,"quote") ){ + if( i+1<nArg + && azArg[i+1][0]!='-' + && (iMode>0 || strcmp(azArg[i+1],"off")==0 || modeFind(p, azArg[i+1])<0) + ){ + /* --quote is followed by an argument other that is not an option + ** or a mode name. See it must be a boolean or a keyword to describe + ** how to set quoting. */ + i++; + if( (k = pickStr(azArg[i],0,"no","yes","0","1",""))>=0 ){ + k &= 1; /* 0 for "off". 1 for "on". */ + }else{ + char *zErr = 0; + k = pickStr(azArg[i],&zErr, + "off","on","sql","csv","html","tcl","json","relaxed",""); + /* 0 1 2 3 4 5 6 7 */ + if( k<0 ){ + dotCmdError(p, i, "unknown", "%z", zErr); + return 1; + } + } + }else{ + /* (Legacy) no following boolean argument. Turn quoting on */ + k = 1; + } + switch( k ){ + case 1: /* on */ + modeSetStr(&p->mode.spec.zNull, "NULL"); + /* Fall through */ + case 2: /* sql */ + p->mode.spec.eText = QRF_TEXT_Sql; + break; + case 3: /* csv */ + p->mode.spec.eText = QRF_TEXT_Csv; + break; + case 4: /* html */ + p->mode.spec.eText = QRF_TEXT_Html; + break; + case 5: /* tcl */ + p->mode.spec.eText = QRF_TEXT_Tcl; + break; + case 6: /* json */ + p->mode.spec.eText = QRF_TEXT_Json; + break; + case 7: /* relaxed */ + p->mode.spec.eText = QRF_TEXT_Relaxed; + break; + default: /* off */ + p->mode.spec.eText = QRF_TEXT_Plain; + break; + } + chng = 1; + }else if( optionMatch(z,"reset") ){ + int saved_eMode = p->mode.eMode; + modeFree(&p->mode); + modeChange(p, saved_eMode); + }else if( optionMatch(z,"screenwidth") || optionMatch(z,"sw") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i],0,"off","auto",""); + if( k==0 ){ + p->mode.bAutoScreenWidth = 0; + p->mode.spec.nScreenWidth = 0; + }else if( k==1 ){ + p->mode.bAutoScreenWidth = 1; + }else{ + i64 w = integerValue(azArg[i]); + p->mode.bAutoScreenWidth = 0; + if( w<0 ) w = 0; + if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.nScreenWidth = w; + } + chng = 1; + }else if( optionMatch(z,"tag") ){ + size_t nByte; + int n; + const char *zTag; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + zTag = azArg[++i]; + if( modeFind(p, zTag)>=0 ){ + dotCmdError(p, i, "mode already exists", 0); + return 1; + } + if( p->nSavedModes > MODE_N_USER ){ + dotCmdError(p, i-1, "cannot add more modes", 0); + return 1; + } + n = p->nSavedModes++; + nByte = sizeof(p->aSavedModes[0]); + nByte *= n+1; + p->aSavedModes = realloc( p->aSavedModes, nByte ); + shell_check_oom(p->aSavedModes); + p->aSavedModes[n].zTag = strdup(zTag); + shell_check_oom(p->aSavedModes[n].zTag); + modeDup(&p->aSavedModes[n].mode, &p->mode); + chng = 1; + }else if( optionMatch(z,"textjsonb") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + p->mode.spec.bTextJsonb = booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; + chng = 1; + }else if( optionMatch(z,"titles") || optionMatch(z,"title") ){ + char *zErr = 0; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + k = pickStr(azArg[++i],&zErr, + "off","on","plain","sql","csv","html","tcl","json",""); + /* 0 1 2 3 4 5 6 7 */ + if( k<0 ){ + dotCmdError(p, i, "bad --titles value","%z", zErr); + return 1; + } + p->mode.spec.bTitles = k>=1 ? QRF_Yes : QRF_No; + p->mode.mFlags &= ~MFLG_HDR; + p->mode.spec.eTitle = k>1 ? k-1 : aModeInfo[p->mode.eMode].eHdr; + chng = 1; + }else if( optionMatch(z,"widths") || optionMatch(z,"width") ){ + int nWidth = 0; + short int *aWidth; + const char *zW; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + zW = azArg[++i]; + /* Every width value takes at least 2 bytes in the input string to + ** specify, so strlen(zW) bytes should be plenty of space to hold the + ** result. */ + aWidth = malloc( strlen(zW) ); + while( IsSpace(zW[0]) ) zW++; + while( zW[0] ){ + int w = 0; + int nDigit = 0; + k = zW[0]=='-' && IsDigit(zW[1]); + while( IsDigit(zW[k]) ){ + w = w*10 + zW[k] - '0'; + if( w>QRF_MAX_WIDTH ){ + dotCmdError(p,i+1,"width too big", + "Maximum column width is %d", QRF_MAX_WIDTH); + free(aWidth); + return 1; + } + nDigit++; + k++; + } + if( nDigit==0 ){ + dotCmdError(p,i+1,"syntax error", + "should be a comma-separated list if integers"); + free(aWidth); + return 1; + } + if( zW[0]=='-' ) w = -w; + aWidth[nWidth++] = w; + zW += k; + if( zW[0]==',' ) zW++; + while( IsSpace(zW[0]) ) zW++; + } + free(p->mode.spec.aWidth); + p->mode.spec.aWidth = aWidth; + p->mode.spec.nWidth = nWidth; + chng = 1; + }else if( optionMatch(z,"wrap") ){ + int w; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + w = integerValue(azArg[++i]); + if( w<(-QRF_MAX_WIDTH) ) w = -QRF_MAX_WIDTH; + if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.nWrap = w; + chng = 1; + }else if( optionMatch(z,"ww") ){ + p->mode.spec.bWordWrap = QRF_Yes; + chng = 1; + }else if( optionMatch(z,"wordwrap") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + p->mode.spec.bWordWrap = (u8)booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; + chng = 1; + }else if( optionMatch(z,"v") || optionMatch(z,"verbose") ){ + bAll = 1; + }else if( z[0]=='-' ){ + dotCmdError(p, i, "bad option", "Use \".help .mode\" for more info"); + return 1; + }else{ + dotCmdError(p, i, iMode>0?"bad argument":"unknown mode", + "Use \".help .mode\" for more info"); + return 1; + } + } + if( !chng || bAll ){ + const ModeInfo *pI = aModeInfo + p->mode.eMode; + sqlite3_str *pDesc = sqlite3_str_new(p->db); + char *zDesc; + const char *zSetting; + + if( p->nPopMode ) sqlite3_str_appendall(pDesc, "--once "); + sqlite3_str_appendall(pDesc,pI->zName); + if( bAll || (p->mode.spec.nAlign && pI->eCx==2) ){ + int ii; + sqlite3_str_appendall(pDesc, " --align \""); + for(ii=0; ii<p->mode.spec.nAlign; ii++){ + unsigned char a = p->mode.spec.aAlign[ii]; + sqlite3_str_appendchar(pDesc, 1, "LLCR"[a&3]); + } + sqlite3_str_append(pDesc, "\"", 1); + } + if( bAll + || (p->mode.spec.bBorder==QRF_No) != ((pI->mFlg&1)!=0) + ){ + sqlite3_str_appendf(pDesc," --border %s", + p->mode.spec.bBorder==QRF_No ? "off" : "on"); + } + if( bAll || p->mode.spec.eBlob!=QRF_BLOB_Auto ){ + const char *azBQuote[] = + { "auto", "text", "sql", "hex", "tcl", "json", "size" }; + /* 0 1 2 3 4 5 6 + ** Must match QRF_BLOB_xxxx values. See all instances of tag-20251124a */ + u8 e = p->mode.spec.eBlob; + sqlite3_str_appendf(pDesc, " --blob-quote %s", azBQuote[e]); + } + zSetting = aModeStr[pI->eCSep]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zColumnSep)!=0) ){ + sqlite3_str_appendf(pDesc, " --colsep "); + append_c_string(pDesc, p->mode.spec.zColumnSep); + } + if( bAll || p->mode.spec.eEsc!=QRF_Auto ){ + sqlite3_str_appendf(pDesc, " --escape %s",qrfEscNames[p->mode.spec.eEsc]); + } + if( bAll + || (p->mode.spec.nLineLimit>0 && pI->eCx>0) + || p->mode.spec.nCharLimit>0 + || (p->mode.spec.nTitleLimit>0 && pI->eCx>0) + ){ + if( p->mode.spec.nLineLimit==0 + && p->mode.spec.nCharLimit==0 + && p->mode.spec.nTitleLimit==0 + ){ + sqlite3_str_appendf(pDesc, " --limits off"); + }else if( p->mode.spec.nLineLimit==DFLT_LINE_LIMIT + && p->mode.spec.nCharLimit==DFLT_CHAR_LIMIT + && p->mode.spec.nTitleLimit==DFLT_TITLE_LIMIT + ){ + sqlite3_str_appendf(pDesc, " --limits on"); + }else{ + sqlite3_str_appendf(pDesc, " --limits %d,%d,%d", + p->mode.spec.nLineLimit, p->mode.spec.nCharLimit, + p->mode.spec.nTitleLimit); + } + } + if( bAll + || (p->mode.spec.nMultiInsert && p->mode.spec.eStyle==QRF_STYLE_Insert) + ){ + sqlite3_str_appendf(pDesc, " --multiinsert %u", + p->mode.spec.nMultiInsert); + } + zSetting = aModeStr[pI->eNull]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){ + sqlite3_str_appendf(pDesc, " --null "); + append_c_string(pDesc, p->mode.spec.zNull); + } + if( bAll + || (pI->eText!=p->mode.spec.eText && (pI->eText>1 || p->mode.spec.eText>1)) + ){ + sqlite3_str_appendf(pDesc," --quote %s",qrfQuoteNames[p->mode.spec.eText]); + } + zSetting = aModeStr[pI->eRSep]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zRowSep)!=0) ){ + sqlite3_str_appendf(pDesc, " --rowsep "); + append_c_string(pDesc, p->mode.spec.zRowSep); + } + if( bAll + || (pI->eCx && (p->mode.spec.nScreenWidth>0 || p->mode.bAutoScreenWidth)) + ){ + if( p->mode.bAutoScreenWidth ){ + sqlite3_str_appendall(pDesc, " --sw auto"); + }else{ + sqlite3_str_appendf(pDesc," --sw %d", + p->mode.spec.nScreenWidth); + } + } + if( bAll || p->mode.eMode==MODE_Insert ){ + sqlite3_str_appendf(pDesc," --tablename "); + append_c_string(pDesc, p->mode.spec.zTableName); + } + if( bAll || p->mode.spec.bTextJsonb ){ + sqlite3_str_appendf(pDesc," --textjsonb %s", + p->mode.spec.bTextJsonb==QRF_Yes ? "on" : "off"); + } + k = modeTitleDsply(p, bAll); + if( k==1 ){ + sqlite3_str_appendall(pDesc, " --titles off"); + }else if( k==2 ){ + sqlite3_str_appendall(pDesc, " --titles on"); + }else if( k==3 ){ + static const char *azTitle[] = + { "plain", "sql", "csv", "html", "tcl", "json"}; + sqlite3_str_appendf(pDesc, " --titles %s", + azTitle[p->mode.spec.eTitle-1]); + } + if( p->mode.spec.nWidth>0 && (bAll || pI->eCx==2) ){ + int ii; + const char *zSep = " --widths "; + for(ii=0; ii<p->mode.spec.nWidth; ii++){ + sqlite3_str_appendf(pDesc, "%s%d", zSep, (int)p->mode.spec.aWidth[ii]); + zSep = ","; + } + }else if( bAll ){ + sqlite3_str_appendall(pDesc, " --widths \"\""); + } + if( bAll || (pI->eCx>0 && p->mode.spec.bWordWrap) ){ + if( bAll ){ + sqlite3_str_appendf(pDesc, " --wordwrap %s", + p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off"); + } + if( p->mode.spec.nWrap ){ + sqlite3_str_appendf(pDesc, " --wrap %d", p->mode.spec.nWrap); + } + if( !bAll ) sqlite3_str_append(pDesc, " --ww", 5); + } + zDesc = sqlite3_str_finish(pDesc); + cli_printf(p->out, ".mode %s\n", zDesc); + fflush(p->out); + sqlite3_free(zDesc); + } + return 0; +} + +/* +** DOT-COMMAND: .output +** USAGE: .output [OPTIONS] [FILE] +** +** Begin redirecting output to FILE. Or if FILE is omitted, revert +** to sending output to the console. If FILE begins with "|" then +** the remainder of file is taken as a pipe and output is directed +** into that pipe. If FILE is "memory" then output is captured in an +** internal memory buffer. If FILE is "off" then output is redirected +** into /dev/null or the equivalent. +** +** Options: +** --bom Prepend a byte-order mark to the output +** -e Accumulate output in a temporary text file then +** launch a text editor when the redirection ends. +** --error-prefix X Use X as the left-margin prefix for error messages. +** Set to an empty string to restore the default. +** --keep Keep redirecting output to its current destination. +** Use this option in combination with --show or +** with --error-prefix when you do not want to stop +** a current redirection. +** --plain Use plain text rather than HTML tables with -w +** --show Show output text captured by .testcase or by +** redirecting to "memory". +** -w Show the output in a web browser. Output is +** written into a temporary HTML file until the +** redirect ends, then the web browser is launched. +** Query results are shown as HTML tables, unless +** the --plain is used too. +** -x Show the output in a spreadsheet. Output is +** written to a temp file as CSV then the spreadsheet +** is launched when +** +** DOT-COMMAND: .once +** USAGE: .once [OPTIONS] FILE ... +** +** Write the output for the next line of SQL or the next dot-command into +** FILE. If FILE begins with "|" then it is a program into which output +** is written. The FILE argument should be omitted if one of the -e, -w, +** or -x options is used. +** +** Options: +** -e Capture output into a temporary file then bring up +** a text editor on that temporary file. +** --plain Use plain text rather than HTML tables with -w +** -w Capture output into an HTML file then bring up that +** file in a web browser +** -x Show the output in a spreadsheet. Output is +** written to a temp file as CSV then the spreadsheet +** is launched when +** +** DOT-COMMAND: .excel +** Shorthand for ".once -x" +** +** DOT-COMMAND: .www [--plain] +** Shorthand for ".once -w" or ".once --plain -w" +*/ +static int dotCmdOutput(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + char *zFile = 0; /* The FILE argument */ + int i; /* Loop counter */ + int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ + int bPlain = 0; /* --plain option */ + int bKeep = 0; /* Keep redirecting */ + static const char *zBomUtf8 = "\357\273\277"; + const char *zBom = 0; + char c = azArg[0][0]; + int n = strlen30(azArg[0]); + + failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( c=='w' ){ + eMode = 'w'; + bOnce = 2; + }else if( n>=2 && cli_strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' ){ + if( z[1]=='-' ) z++; + if( cli_strcmp(z,"-bom")==0 ){ + zBom = zBomUtf8; + }else if( cli_strcmp(z,"-plain")==0 ){ + bPlain = 1; + }else if( c=='o' && sqlite3_strglob("-[ewx]",z)==0 ){ + if( bKeep || eMode ){ + dotCmdError(p, i, "incompatible with prior options",0); + goto dotCmdOutput_error; + } + eMode = z[1]; + }else if( cli_strcmp(z,"-show")==0 ){ + if( cli_output_capture ){ + sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); + } + }else if( cli_strcmp(z,"-keep")==0 ){ + bKeep = 1; + }else if( optionMatch(z,"error-prefix") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + free(p->zErrPrefix); + i++; + p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); + }else{ + dotCmdError(p, i, "unknown option", 0); + sqlite3_free(zFile); + return 1; + } + }else if( zFile==0 && eMode==0 ){ + if( bKeep ){ + dotCmdError(p, i, "incompatible with prior options",0); + goto dotCmdOutput_error; + } + if( cli_strcmp(z, "memory")==0 && bOnce ){ + dotCmdError(p, 0, "cannot redirect to \"memory\"", 0); + goto dotCmdOutput_error; + } + if( cli_strcmp(z, "off")==0 ){ +#ifdef _WIN32 + zFile = sqlite3_mprintf("nul"); +#else + zFile = sqlite3_mprintf("/dev/null"); +#endif + }else{ + zFile = sqlite3_mprintf("%s", z); + } + if( zFile && zFile[0]=='|' ){ + while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); + break; + } + }else{ + dotCmdError(p, i, "surplus argument", 0); + sqlite3_free(zFile); + return 1; + } + } + if( zFile==0 && !bKeep ){ + zFile = sqlite3_mprintf("stdout"); + shell_check_oom(zFile); + } + if( bOnce ){ + p->nPopOutput = 2; + }else{ + p->nPopOutput = 0; + } + if( !bKeep ) output_reset(p); +#ifndef SQLITE_NOHAVE_SYSTEM + if( eMode=='e' || eMode=='x' || eMode=='w' ){ + p->doXdgOpen = 1; + modePush(p); + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); + p->mode.mFlags &= ~MFLG_ECHO; + p->mode.eMode = MODE_Csv; + modeSetStr(&p->mode.spec.zColumnSep, SEP_Comma); + modeSetStr(&p->mode.spec.zRowSep, SEP_CrLf); +#ifdef _WIN32 + zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does + ** not work without it. */ +#endif + }else if( eMode=='w' ){ + /* web-browser mode. */ + newTempFile(p, "html"); + if( !bPlain ) p->mode.eMode = MODE_Www; + }else{ + /* text editor mode */ + newTempFile(p, "txt"); + } + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); + } +#endif /* SQLITE_NOHAVE_SYSTEM */ + if( !bKeep ) shell_check_oom(zFile); + if( bKeep ){ + /* no-op */ + }else if( cli_strcmp(zFile,"memory")==0 ){ + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + } + cli_output_capture = sqlite3_str_new(0); + }else if( zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + output_redir(p, stdout); + goto dotCmdOutput_error; +#else + FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); + if( pfPipe==0 ){ + assert( stderr!=NULL ); + cli_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); + goto dotCmdOutput_error; + }else{ + output_redir(p, pfPipe); + if( zBom ) cli_puts(zBom, pfPipe); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } +#endif + }else{ + FILE *pfFile = output_file_open(p, zFile); + if( pfFile==0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ + assert( stderr!=NULL ); + cli_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + } + goto dotCmdOutput_error; + } else { + output_redir(p, pfFile); + if( zBom ) cli_puts(zBom, pfFile); + if( bPlain && eMode=='w' ){ + cli_puts( + "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", + pfFile + ); + } + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } + sqlite3_free(zFile); + return 0; + +dotCmdOutput_error: + sqlite3_free(zFile); + return 1; +} + +/* +** DOT-COMMAND: .check +** USAGE: .check [OPTIONS] PATTERN +** +** Verify results of commands since the most recent .testcase command. +** Restore output to the console, unless --keep is used. +** +** If PATTERN starts with "<<ENDMARK" then the actual pattern is taken from +** subsequent lines of text up to the first line that begins with ENDMARK. +** All pattern lines and the ENDMARK are discarded. +** +** Options: +** --exact Do an exact comparison including leading and +** trailing whitespace. +** --glob Treat PATTERN as a GLOB +** --keep Do not reset the testcase. More .check commands +** will follow. +** --notglob Output should not match PATTERN +** --show Write testcase output to the screen, for debugging. +*/ +static int dotCmdCheck(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + int i; /* Loop counter */ + int k; /* Result of pickStr() */ + char *zTest; /* Textcase result */ + int bKeep = 0; /* --keep option */ + char *zCheck = 0; /* PATTERN argument */ + char *zPattern = 0; /* Actual test pattern */ + int eCheck = 0; /* 1: --glob, 2: --notglob, 3: --exact */ + int isOk; /* True if results are OK */ + sqlite3_int64 iStart = p->lineno; /* Line number of .check statement */ + + if( p->zTestcase[0]==0 ){ + dotCmdError(p, 0, "no .testcase is active", 0); + return 1; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( cli_strcmp(z,"-keep")==0 ){ + bKeep = 1; + }else if( cli_strcmp(z,"-show")==0 ){ + if( cli_output_capture ){ + sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); + } + bKeep = 1; + }else if( z[0]=='-' + && (k = pickStr(&z[1],0,"glob","notglob","exact",""))>=0 + ){ + if( eCheck && eCheck!=k+1 ){ + dotCmdError(p, i, "incompatible with prior options",0); + return 1; + } + eCheck = k+1; + }else if( zCheck ){ + dotCmdError(p, i, "unknown option", 0); + return 1; + }else{ + zCheck = azArg[i]; + } + } + if( zCheck==0 ){ + dotCmdError(p, 0, "no PATTERN specified", 0); + return 1; + } + if( cli_output_capture && sqlite3_str_length(cli_output_capture) ){ + zTest = sqlite3_str_value(cli_output_capture); + shell_check_oom(zTest); + }else{ + zTest = ""; + } + p->nTestRun++; + if( zCheck[0]=='<' && zCheck[1]=='<' && zCheck[2]!=0 ){ + int nCheck = strlen30(zCheck); + sqlite3_str *pPattern = sqlite3_str_new(p->db); + char zLine[2000]; + while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ + if( strchr(zLine,'\n') ) p->lineno++; + if( cli_strncmp(&zCheck[2],zLine,nCheck-2)==0 ) break; + sqlite3_str_appendall(pPattern, zLine); + } + zPattern = sqlite3_str_finish(pPattern); + if( zPattern==0 ){ + zPattern = sqlite3_mprintf(""); + } + }else{ + zPattern = zCheck; + } + shell_check_oom(zPattern); + switch( eCheck ){ + case 1: { + char *zGlob = sqlite3_mprintf("*%s*", zPattern); + isOk = testcase_glob(zGlob, zTest)!=0; + sqlite3_free(zGlob); + break; + } + case 2: { + char *zGlob = sqlite3_mprintf("*%s*", zPattern); + isOk = testcase_glob(zGlob, zTest)==0; + sqlite3_free(zGlob); + break; + } + case 3: { + isOk = cli_strcmp(zTest,zPattern)==0; + break; + } + default: { + /* Skip leading and trailing \n and \r on both pattern and test output */ + const char *z1 = zPattern; + const char *z2 = zTest; + size_t n1, n2; + while( z1[0]=='\n' || z1[0]=='\r' ) z1++; + n1 = strlen(z1); + while( n1>0 && (z1[n1-1]=='\n' || z1[n1-1]=='\r') ) n1--; + while( z2[0]=='\n' || z2[0]=='\r' ) z2++; + n2 = strlen(z2); + while( n2>0 && (z2[n2-1]=='\n' || z2[n2-1]=='\r') ) n2--; + isOk = n1==n2 && memcmp(z1,z2,n1)==0; + break; + } + } + if( !isOk ){ + sqlite3_fprintf(stderr, + "%s:%lld: .check failed for testcase %s\n", + p->zInFile, iStart, p->zTestcase); + p->nTestErr++; + sqlite3_fprintf(stderr, "Expected: [%s]\n", zPattern); + sqlite3_fprintf(stderr, "Got: [%s]\n", zTest); + } + if( zPattern!=zCheck ){ + sqlite3_free(zPattern); + } + if( !bKeep ){ + output_reset(p); + p->zTestcase[0] = 0; + } + return 0; +} + +/* +** DOT-COMMAND: .testcase +** USAGE: .testcase [OPTIONS] NAME +** +** Start a new test case identified by NAME. All output +** through the next ".check" command is captured for comparison. See the +** ".check" commandn for additional informatioon. +** +** Options: +** --error-prefix TEXT Change error message prefix text to TEXT +*/ +static int dotCmdTestcase(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + int i; /* Loop counter */ + const char *zName = 0; /* Testcase name */ + + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( optionMatch(z,"error-prefix") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + free(p->zErrPrefix); + i++; + p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); + }else if( zName ){ + dotCmdError(p, i, "unknown option", 0); + return 1; + }else{ + zName = azArg[i]; + } + } + output_reset(p); + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + } + cli_output_capture = sqlite3_str_new(0); + if( zName ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", zName); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s:%lld", + p->zInFile, p->lineno); + } + return 0; +} + +/* +** Enlarge the space allocated in p->dot so that it can hold more +** than nArg parsed command-line arguments. +*/ +static void parseDotRealloc(ShellState *p, int nArg){ + p->dot.nAlloc = nArg+22; + p->dot.azArg = realloc(p->dot.azArg,p->dot.nAlloc*sizeof(char*)); + shell_check_oom(p->dot.azArg); + p->dot.aiOfst = realloc(p->dot.aiOfst,p->dot.nAlloc*sizeof(int)); + shell_check_oom(p->dot.aiOfst); + p->dot.abQuot = realloc(p->dot.abQuot,p->dot.nAlloc); + shell_check_oom(p->dot.abQuot); +} + + +/* +** Parse input line zLine up into individual arguments. Retain the +** parse in the p->dot substructure. +*/ +static void parseDotCmdArgs(const char *zLine, ShellState *p){ + char *z; + int h = 1; + int nArg = 0; + size_t szLine; + + p->dot.zOrig = zLine; + free(p->dot.zCopy); + z = p->dot.zCopy = strdup(zLine); + shell_check_oom(z); + szLine = strlen(z); + while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; + if( szLine>0 && z[szLine-1]==';' ){ + szLine--; + while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; + } + z[szLine] = 0; + parseDotRealloc(p, 2); + while( z[h] ){ + while( IsSpace(z[h]) ){ h++; } + if( z[h]==0 ) break; + if( nArg+2>p->dot.nAlloc ){ + parseDotRealloc(p, nArg); + } + if( z[h]=='\'' || z[h]=='"' ){ + int delim = z[h++]; + p->dot.abQuot[nArg] = 1; + p->dot.azArg[nArg] = &z[h]; + p->dot.aiOfst[nArg] = h; + while( z[h] && z[h]!=delim ){ + if( z[h]=='\\' && delim=='"' && z[h+1]!=0 ) h++; + h++; + } + if( z[h]==delim ){ + z[h++] = 0; + } + if( delim=='"' ) resolve_backslashes(p->dot.azArg[nArg]); + }else{ + p->dot.abQuot[nArg] = 0; + p->dot.azArg[nArg] = &z[h]; + p->dot.aiOfst[nArg] = h; + while( z[h] && !IsSpace(z[h]) ){ h++; } + if( z[h] ) z[h++] = 0; + } + nArg++; + } + p->dot.nArg = nArg; + p->dot.azArg[nArg] = 0; +} + +/* ** If an input line begins with "." then invoke this routine to ** process that line. ** ** Return 1 on error, 2 to exit, and 0 otherwise. */ -static int do_meta_command(char *zLine, ShellState *p){ - int h = 1; - int nArg = 0; +static int do_meta_command(const char *zLine, ShellState *p){ + int nArg; int n, c; int rc = 0; - char *azArg[52]; + char **azArg; -#ifndef SQLITE_OMIT_VIRTUALTABLE +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( p->expert.pExpert ){ expertFinish(p, 1, 0); } #endif - /* Parse the input line into tokens. + /* Parse the input line into tokens stored in p->dot. */ - while( zLine[h] && nArg<ArraySize(azArg)-1 ){ - while( IsSpace(zLine[h]) ){ h++; } - if( zLine[h]==0 ) break; - if( zLine[h]=='\'' || zLine[h]=='"' ){ - int delim = zLine[h++]; - azArg[nArg++] = &zLine[h]; - while( zLine[h] && zLine[h]!=delim ){ - if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++; - h++; - } - if( zLine[h]==delim ){ - zLine[h++] = 0; - } - if( delim=='"' ) resolve_backslashes(azArg[nArg-1]); - }else{ - azArg[nArg++] = &zLine[h]; - while( zLine[h] && !IsSpace(zLine[h]) ){ h++; } - if( zLine[h] ) zLine[h++] = 0; - } - } - azArg[nArg] = 0; + parseDotCmdArgs(zLine, p); + nArg = p->dot.nArg; + azArg = p->dot.azArg; /* Process the input line. */ @@ -28848,7 +32491,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_OMIT_AUTHORIZATION if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ - sqlite3_fprintf(stderr, "Usage: .auth ON|OFF\n"); + cli_printf(stderr, "Usage: .auth ON|OFF\n"); rc = 1; goto meta_command_exit; } @@ -28895,7 +32538,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bAsync = 1; }else { - sqlite3_fprintf(stderr,"unknown option: %s\n", azArg[j]); + dotCmdError(p, j, "unknown option", "should be -append or -async"); return 1; } }else if( zDestFile==0 ){ @@ -28904,19 +32547,19 @@ static int do_meta_command(char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - sqlite3_fprintf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); + cli_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } if( zDestFile==0 ){ - sqlite3_fprintf(stderr, "missing FILENAME argument on .backup\n"); + cli_printf(stderr, "missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zDestFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; } @@ -28977,7 +32620,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = chdir(azArg[1]); #endif if( rc ){ - sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); + cli_printf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); rc = 1; } }else{ @@ -28996,31 +32639,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ - char *zRes = 0; - output_reset(p); - if( nArg!=2 ){ - eputz("Usage: .check GLOB-PATTERN\n"); - rc = 2; - }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ - rc = 2; - }else if( testcase_glob(azArg[1],zRes)==0 ){ - sqlite3_fprintf(stderr, - "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", - p->zTestcase, azArg[1], zRes); - rc = 1; - }else{ - sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase); - p->nCheck++; - } - sqlite3_free(zRes); + rc = dotCmdCheck(p); }else -#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_SHELL_FIDDLE if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ @@ -29048,9 +32673,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zFile = "(temporary-file)"; } if( p->pAuxDb == &p->aAuxDb[i] ){ - sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile); + cli_printf(stdout, "ACTIVE %d: %s\n", i, zFile); }else if( p->aAuxDb[i].db!=0 ){ - sqlite3_fprintf(stdout, " %d: %s\n", i, zFile); + cli_printf(stdout, " %d: %s\n", i, zFile); } } }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){ @@ -29086,12 +32711,17 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ if( nArg==2 ){ #ifdef _WIN32 - p->crlfMode = booleanValue(azArg[1]); + if( booleanValue(azArg[1]) ){ + p->mode.mFlags |= MFLG_CRLF; + }else{ + p->mode.mFlags &= ~MFLG_CRLF; + } #else - p->crlfMode = 0; + p->mode.mFlags &= ~MFLG_CRLF; #endif } - sqlite3_fprintf(stderr, "crlf is %s\n", p->crlfMode ? "ON" : "OFF"); + cli_printf(stderr, "crlf is %s\n", + (p->mode.mFlags & MFLG_CRLF)!=0 ? "ON" : "OFF"); }else if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ @@ -29109,7 +32739,7 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zSchema = (const char *)sqlite3_column_text(pStmt,1); const char *zFile = (const char*)sqlite3_column_text(pStmt,2); if( zSchema==0 || zFile==0 ) continue; - azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*)); + azName = sqlite3_realloc64(azName, (nName+1)*2*sizeof(char*)); shell_check_oom(azName); azName[nName*2] = strdup(zSchema); azName[nName*2+1] = strdup(zFile); @@ -29121,7 +32751,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int eTxn = sqlite3_txn_state(p->db, azName[i*2]); int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); const char *z = azName[i*2+1]; - sqlite3_fprintf(p->out, "%s: %s %s%s\n", + cli_printf(p->out, "%s: %s %s%s\n", azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w", eTxn==SQLITE_TXN_NONE ? "" : eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); @@ -29147,6 +32777,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "fp_digits", SQLITE_DBCONFIG_FP_DIGITS }, { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, @@ -29163,16 +32794,24 @@ static int do_meta_command(char *zLine, ShellState *p){ for(ii=0; ii<ArraySize(aDbConfig); ii++){ if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ - sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ + sqlite3_db_config(p->db, aDbConfig[ii].op, atoi(azArg[2]), 0); + }else{ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } } sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); - sqlite3_fprintf(p->out, "%19s %s\n", - aDbConfig[ii].zName, v ? "on" : "off"); + if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ + cli_printf(p->out, "%19s %d\n", aDbConfig[ii].zName, v); + }else{ + cli_printf(p->out, "%19s %s\n", + aDbConfig[ii].zName, v ? "on" : "off"); + } if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ - sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]); - eputz("Enter \".dbconfig\" with no arguments for a list\n"); + dotCmdError(p, 1, "unknown dbconfig", + "Enter \".dbconfig\" with no arguments for a list"); } }else @@ -29191,19 +32830,19 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zLike = 0; char *zSql; int i; - int savedShowHeader = p->showHeader; int savedShellFlags = p->shellFlgs; + Mode saved_mode; ShellClearFlag(p, - SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo - |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); + SHFLG_PreserveRowid|SHFLG_DumpDataOnly|SHFLG_DumpNoSys); for(i=1; i<nArg; i++){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; if( z[0]=='-' ) z++; if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE - eputz("The --preserve-rowids option is not compatible" - " with SQLITE_OMIT_VIRTUALTABLE\n"); + dotCmdError(p, i, "unable", + "The --preserve-rowids option is not compatible" + " with SQLITE_OMIT_VIRTUALTABLE"); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -29212,7 +32851,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else if( cli_strcmp(z,"newlines")==0 ){ - ShellSetFlag(p, SHFLG_Newlines); + /*ShellSetFlag(p, SHFLG_Newlines);*/ }else if( cli_strcmp(z,"data-only")==0 ){ ShellSetFlag(p, SHFLG_DumpDataOnly); @@ -29221,8 +32860,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { - sqlite3_fprintf(stderr, - "Unknown option \"%s\" on \".dump\"\n", azArg[i]); + dotCmdError(p, i, "unknown option", 0); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -29252,16 +32890,17 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); + modeDup(&saved_mode, &p->mode); outputDumpWarning(p, zLike); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out); - sqlite3_fputs("BEGIN TRANSACTION;\n", p->out); + cli_puts("PRAGMA foreign_keys=OFF;\n", p->out); + cli_puts("BEGIN TRANSACTION;\n", p->out); } p->writableSchema = 0; - p->showHeader = 0; + p->mode.spec.bTitles = QRF_No; /* Set writable_schema=ON since doing so forces SQLite to initialize ** as much of the schema as it can even if the sqlite_schema table is ** corrupt. */ @@ -29290,21 +32929,27 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zLike); if( p->writableSchema ){ - sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out); + cli_puts("PRAGMA writable_schema=OFF;\n", p->out); p->writableSchema = 0; } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ - sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); + cli_puts(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); } - p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; + modeFree(&p->mode); + p->mode = saved_mode; + rc = p->nErr>0; }else if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - setOrClearFlag(p, SHFLG_Echo, azArg[1]); + if( booleanValue(azArg[1]) ){ + p->mode.mFlags |= MFLG_ECHO; + }else{ + p->mode.mFlags &= ~MFLG_ECHO; + } }else{ eputz("Usage: .echo on|off\n"); rc = 1; @@ -29312,33 +32957,30 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbtotxt", n)==0 ){ + open_db(p, 0); rc = shell_dbtotxt_command(p, nArg, azArg); }else if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ - p->autoEQPtest = 0; - if( p->autoEQPtrace ){ + if( p->mode.autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); - p->autoEQPtrace = 0; + p->mode.autoEQPtrace = 0; } if( cli_strcmp(azArg[1],"full")==0 ){ - p->autoEQP = AUTOEQP_full; + p->mode.autoEQP = AUTOEQP_full; }else if( cli_strcmp(azArg[1],"trigger")==0 ){ - p->autoEQP = AUTOEQP_trigger; + p->mode.autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG - }else if( cli_strcmp(azArg[1],"test")==0 ){ - p->autoEQP = AUTOEQP_on; - p->autoEQPtest = 1; }else if( cli_strcmp(azArg[1],"trace")==0 ){ - p->autoEQP = AUTOEQP_full; - p->autoEQPtrace = 1; + p->mode.autoEQP = AUTOEQP_full; + p->mode.autoEQPtrace = 1; open_db(p, 0); sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0); sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); #endif }else{ - p->autoEQP = (u8)booleanValue(azArg[1]); + p->mode.autoEQP = (u8)booleanValue(azArg[1]); } }else{ eputz("Usage: .eqp off|on|trace|trigger|full\n"); @@ -29348,7 +32990,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ - if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) cli_exit(rc); rc = 2; }else #endif @@ -29356,31 +32998,19 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ - int val = 1; if( nArg>=2 ){ if( cli_strcmp(azArg[1],"auto")==0 ){ - val = 99; + p->mode.autoExplain = 1; }else{ - val = booleanValue(azArg[1]); + p->mode.autoExplain = booleanValue(azArg[1]); } } - if( val==1 && p->mode!=MODE_Explain ){ - p->normalMode = p->mode; - p->mode = MODE_Explain; - p->autoExplain = 0; - }else if( val==0 ){ - if( p->mode==MODE_Explain ) p->mode = p->normalMode; - p->autoExplain = 0; - }else if( val==99 ){ - if( p->mode==MODE_Explain ) p->mode = p->normalMode; - p->autoExplain = 1; - } }else -#ifndef SQLITE_OMIT_VIRTUALTABLE +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", azArg[0]); rc = 1; @@ -29438,9 +33068,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all file-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - sqlite3_fputs("Available file-controls:\n", p->out); + cli_puts("Available file-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ - sqlite3_fprintf(p->out, + cli_printf(p->out, " .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -29456,7 +33086,7 @@ static int do_meta_command(char *zLine, ShellState *p){ filectrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - sqlite3_fprintf(stderr,"Error: ambiguous file-control: \"%s\"\n" + cli_printf(stderr,"Error: ambiguous file-control: \"%s\"\n" "Use \".filectrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -29464,7 +33094,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( filectrl<0 ){ - sqlite3_fprintf(stderr,"Error: unknown file-control: %s\n" + cli_printf(stderr,"Error: unknown file-control: %s\n" "Use \".filectrl --help\" for help\n", zCmd); }else{ switch(filectrl){ @@ -29508,7 +33138,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg!=2 ) break; sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ - sqlite3_fprintf(p->out, "%s\n", z); + cli_printf(p->out, "%s\n", z); sqlite3_free(z); } isOk = 2; @@ -29522,31 +33152,30 @@ static int do_meta_command(char *zLine, ShellState *p){ } x = -1; sqlite3_file_control(p->db, zSchema, filectrl, &x); - sqlite3_fprintf(p->out, "%d\n", x); + cli_printf(p->out, "%d\n", x); isOk = 2; break; } } } if( isOk==0 && iCtrl>=0 ){ - sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n", + cli_printf(p->out, "Usage: .filectrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ char zBuf[100]; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); - sqlite3_fprintf(p->out, "%s\n", zBuf); + cli_printf(p->out, "%s\n", zBuf); } }else if( c=='f' && cli_strncmp(azArg[0], "fullschema", n)==0 ){ ShellState data; int doStats = 0; - memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; + int hasStat[5]; + int flgs = 0; + char *zSql; if( nArg==2 && optionMatch(azArg[1], "indent") ){ - data.cMode = data.mode = MODE_Pretty; nArg = 1; } if( nArg!=1 ){ @@ -29555,43 +33184,61 @@ static int do_meta_command(char *zLine, ShellState *p){ goto meta_command_exit; } open_db(p, 0); - rc = sqlite3_exec(p->db, - "SELECT sql FROM" + zSql = sqlite3_mprintf( + "SELECT shell_format_schema(sql,%d) FROM" " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_schema UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " - "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " - "ORDER BY x", - callback, &data, 0 - ); + "WHERE type!='meta' AND sql NOTNULL" + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_' " + "ORDER BY x", flgs); + memcpy(&data, p, sizeof(data)); + data.mode.spec.bTitles = QRF_No; + data.mode.eMode = MODE_List; + data.mode.spec.eText = QRF_TEXT_Plain; + data.mode.spec.nCharLimit = 0; + data.mode.spec.zRowSep = "\n"; + rc = shell_exec(&data,zSql,0); + sqlite3_free(zSql); if( rc==SQLITE_OK ){ sqlite3_stmt *pStmt; + memset(hasStat, 0, sizeof(hasStat)); rc = sqlite3_prepare_v2(p->db, - "SELECT rowid FROM sqlite_schema" + "SELECT substr(name,12,1) FROM sqlite_schema" " WHERE name GLOB 'sqlite_stat[134]'", -1, &pStmt, 0); if( rc==SQLITE_OK ){ - doStats = sqlite3_step(pStmt)==SQLITE_ROW; - sqlite3_finalize(pStmt); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int k = sqlite3_column_int(pStmt,0); + assert( k==1 || k==3 || k==4 ); + hasStat[k] = 1; + doStats = 1; + } } + sqlite3_finalize(pStmt); } if( doStats==0 ){ - sqlite3_fputs("/* No STAT tables available */\n", p->out); + cli_puts("/* No STAT tables available */\n", p->out); }else{ - sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); - data.cMode = data.mode = MODE_Insert; - data.zDestTable = "sqlite_stat1"; - shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); - data.zDestTable = "sqlite_stat4"; - shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); - sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); + cli_puts("ANALYZE sqlite_schema;\n", p->out); + data.mode.eMode = MODE_Insert; + if( hasStat[1] ){ + data.mode.spec.zTableName = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); + } + if( hasStat[4] ){ + data.mode.spec.zTableName = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); + } + cli_puts("ANALYZE sqlite_schema;\n", p->out); } }else if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){ if( nArg==2 ){ - p->showHeader = booleanValue(azArg[1]); - p->shellFlgs |= SHFLG_HeaderSet; + p->mode.spec.bTitles = booleanValue(azArg[1]) ? QRF_Yes : QRF_No; + p->mode.mFlags |= MFLG_HDR; + p->mode.spec.eTitle = aModeInfo[p->mode.eMode].eHdr; }else{ eputz("Usage: .headers on|off\n"); rc = 1; @@ -29602,7 +33249,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg>=2 ){ n = showHelp(p->out, azArg[1]); if( n==0 ){ - sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]); + cli_printf(p->out, "Nothing matches '%s'\n", azArg[1]); } }else{ showHelp(p->out, 0); @@ -29611,326 +33258,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ - char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* Schema of zTable */ - char *zFile = 0; /* Name of file to extra content from */ - sqlite3_stmt *pStmt = NULL; /* A statement */ - int nCol; /* Number of columns in the table */ - i64 nByte; /* Number of bytes in an SQL string */ - int i, j; /* Loop counters */ - int needCommit; /* True to COMMIT or ROLLBACK at end */ - int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql = 0; /* An SQL statement */ - ImportCtx sCtx; /* Reader context */ - char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ - int eVerbose = 0; /* Larger for more console output */ - int nSkip = 0; /* Initial lines to skip */ - int useOutputMode = 1; /* Use output mode to determine separators */ - char *zCreate = 0; /* CREATE TABLE statement text */ - - failIfSafeMode(p, "cannot run .import in safe mode"); - memset(&sCtx, 0, sizeof(sCtx)); - if( p->mode==MODE_Ascii ){ - xRead = ascii_read_one_field; - }else{ - xRead = csv_read_one_field; - } - rc = 1; - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' ) z++; - if( z[0]!='-' ){ - if( zFile==0 ){ - zFile = z; - }else if( zTable==0 ){ - zTable = z; - }else{ - sqlite3_fprintf(p->out, "ERROR: extra argument: \"%s\". Usage:\n",z); - showHelp(p->out, "import"); - goto meta_command_exit; - } - }else if( cli_strcmp(z,"-v")==0 ){ - eVerbose++; - }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ - zSchema = azArg[++i]; - }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ - nSkip = integerValue(azArg[++i]); - }else if( cli_strcmp(z,"-ascii")==0 ){ - sCtx.cColSep = SEP_Unit[0]; - sCtx.cRowSep = SEP_Record[0]; - xRead = ascii_read_one_field; - useOutputMode = 0; - }else if( cli_strcmp(z,"-csv")==0 ){ - sCtx.cColSep = ','; - sCtx.cRowSep = '\n'; - xRead = csv_read_one_field; - useOutputMode = 0; - }else{ - sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z); - showHelp(p->out, "import"); - goto meta_command_exit; - } - } - if( zTable==0 ){ - sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n", - zFile==0 ? "FILE" : "TABLE"); - showHelp(p->out, "import"); - goto meta_command_exit; - } - seenInterrupt = 0; - open_db(p, 0); - if( useOutputMode ){ - /* If neither the --csv or --ascii options are specified, then set - ** the column and row separator characters from the output mode. */ - nSep = strlen30(p->colSeparator); - if( nSep==0 ){ - eputz("Error: non-null column separator required for import\n"); - goto meta_command_exit; - } - if( nSep>1 ){ - eputz("Error: multi-character column separators not allowed" - " for import\n"); - goto meta_command_exit; - } - nSep = strlen30(p->rowSeparator); - if( nSep==0 ){ - eputz("Error: non-null row separator required for import\n"); - goto meta_command_exit; - } - if( nSep==2 && p->mode==MODE_Csv - && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 - ){ - /* When importing CSV (only), if the row separator is set to the - ** default output row separator, change it to the default input - ** row separator. This avoids having to maintain different input - ** and output row separators. */ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - nSep = strlen30(p->rowSeparator); - } - if( nSep>1 ){ - eputz("Error: multi-character row separators not allowed" - " for import\n"); - goto meta_command_exit; - } - sCtx.cColSep = (u8)p->colSeparator[0]; - sCtx.cRowSep = (u8)p->rowSeparator[0]; - } - sCtx.zFile = zFile; - sCtx.nLine = 1; - if( sCtx.zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - goto meta_command_exit; -#else - sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); - sCtx.zFile = "<pipe>"; - sCtx.xCloser = pclose; -#endif - }else{ - sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); - sCtx.xCloser = fclose; - } - if( sCtx.in==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); - goto meta_command_exit; - } - if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ - char zSep[2]; - zSep[1] = 0; - zSep[0] = sCtx.cColSep; - sqlite3_fputs("Column separator ", p->out); - output_c_string(p->out, zSep); - sqlite3_fputs(", row separator ", p->out); - zSep[0] = sCtx.cRowSep; - output_c_string(p->out, zSep); - sqlite3_fputs("\n", p->out); - } - sCtx.z = sqlite3_malloc64(120); - if( sCtx.z==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - /* Below, resources must be freed before exit. */ - while( (nSkip--)>0 ){ - while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} - } - import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) - && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" - " WHERE name=%Q AND type='view'", - zSchema ? zSchema : "main", zTable) - ){ - /* Table does not exist. Create it. */ - sqlite3 *dbCols = 0; - char *zRenames = 0; - char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", - zSchema ? zSchema : "main", zTable); - while( xRead(&sCtx) ){ - zAutoColumn(sCtx.z, &dbCols, 0); - if( sCtx.cTerm!=sCtx.cColSep ) break; - } - zColDefs = zAutoColumn(0, &dbCols, &zRenames); - if( zRenames!=0 ){ - sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr, - "Columns renamed during .import %s due to duplicates:\n" - "%s\n", sCtx.zFile, zRenames); - sqlite3_free(zRenames); - } - assert(dbCols==0); - if( zColDefs==0 ){ - sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile); - import_cleanup(&sCtx); - rc = 1; - sqlite3_free(zCreate); - goto meta_command_exit; - } - zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); - if( zCreate==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( eVerbose>=1 ){ - sqlite3_fprintf(p->out, "%s\n", zCreate); - } - rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); - if( rc ){ - sqlite3_fprintf(stderr, - "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - } - sqlite3_free(zCreate); - zCreate = 0; - if( rc ){ - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - } - zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", - zTable, zSchema); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - if (pStmt) sqlite3_finalize(pStmt); - shellDatabaseError(p->db); - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nCol = sqlite3_column_int(pStmt, 0); - }else{ - nCol = 0; - } - sqlite3_finalize(pStmt); - pStmt = 0; - if( nCol==0 ) return 0; /* no columns, no error */ - - nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ - + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ - + strlen(zTable)*2 + 2 /* Quoted table name */ - + nCol*2; /* Space for ",?" for each column */ - zSql = sqlite3_malloc64( nByte ); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( zSchema ){ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", - zSchema, zTable); - }else{ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); - } - j = strlen30(zSql); - for(i=1; i<nCol; i++){ - zSql[j++] = ','; - zSql[j++] = '?'; - } - zSql[j++] = ')'; - zSql[j] = 0; - assert( j<nByte ); - if( eVerbose>=2 ){ - sqlite3_fprintf(p->out, "Insert using: %s\n", zSql); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - shellDatabaseError(p->db); - if (pStmt) sqlite3_finalize(pStmt); - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - needCommit = sqlite3_get_autocommit(p->db); - if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); - do{ - int startLine = sCtx.nLine; - for(i=0; i<nCol; i++){ - char *z = xRead(&sCtx); - /* - ** Did we reach end-of-file before finding any columns? - ** If so, stop instead of NULL filling the remaining columns. - */ - if( z==0 && i==0 ) break; - /* - ** Did we reach end-of-file OR end-of-line before finding any - ** columns in ASCII mode? If so, stop instead of NULL filling - ** the remaining columns. - */ - if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; - /* - ** For CSV mode, per RFC 4180, accept EOF in lieu of final - ** record terminator but only for last field of multi-field row. - ** (If there are too few fields, it's not valid CSV anyway.) - */ - if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ - z = ""; - } - sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); - if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ - sqlite3_fprintf(stderr,"%s:%d: expected %d columns but found %d" - " - filling the rest with NULL\n", - sCtx.zFile, startLine, nCol, i+1); - i += 2; - while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } - } - } - if( sCtx.cTerm==sCtx.cColSep ){ - do{ - xRead(&sCtx); - i++; - }while( sCtx.cTerm==sCtx.cColSep ); - sqlite3_fprintf(stderr, - "%s:%d: expected %d columns but found %d - extras ignored\n", - sCtx.zFile, startLine, nCol, i); - } - if( i>=nCol ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n", - sCtx.zFile, startLine, sqlite3_errmsg(p->db)); - sCtx.nErr++; - }else{ - sCtx.nRow++; - } - } - }while( sCtx.cTerm!=EOF ); - - import_cleanup(&sCtx); - sqlite3_finalize(pStmt); - if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); - if( eVerbose>0 ){ - sqlite3_fprintf(p->out, - "Added %d rows with %d errors using %d lines of input\n", - sCtx.nRow, sCtx.nErr, sCtx.nLine-1); - } + rc = dotCmdImport(p); }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -29943,12 +33271,6 @@ static int do_meta_command(char *zLine, ShellState *p){ int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ int i; - if( !ShellHasFlag(p,SHFLG_TestingMode) ){ - sqlite3_fprintf(stderr,".%s unavailable without --unsafe-testing\n", - "imposter"); - rc = 1; - goto meta_command_exit; - } if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ eputz("Usage: .imposter INDEX IMPOSTER\n" " .imposter off\n"); @@ -29969,10 +33291,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } zSql = sqlite3_mprintf( "SELECT rootpage, 0 FROM sqlite_schema" - " WHERE name='%q' AND type='index'" + " WHERE type='index' AND lower(name)=lower('%q')" "UNION ALL " "SELECT rootpage, 1 FROM sqlite_schema" - " WHERE name='%q' AND type='table'" + " WHERE type='table' AND lower(name)=lower('%q')" " AND sql LIKE '%%without%%rowid%%'", azArg[1], azArg[1] ); @@ -30010,7 +33332,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_finalize(pStmt); if( i==0 || tnum==0 ){ - sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]); + cli_printf(stderr,"no such index: \"%s\"\n", azArg[1]); rc = 1; sqlite3_free(zCollist); goto meta_command_exit; @@ -30020,27 +33342,110 @@ static int do_meta_command(char *zLine, ShellState *p){ "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%.*s))WITHOUT ROWID", azArg[2], zCollist, lenPK, zCollist); sqlite3_free(zCollist); - rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 2, tnum); if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); }else{ - sqlite3_fprintf(stdout, "%s;\n", zSql); - sqlite3_fprintf(stdout, - "WARNING: writing to an imposter table will corrupt" - " the \"%s\" %s!\n", azArg[1], isWO ? "table" : "index"); + cli_printf(stdout, "%s;\n", zSql); } }else{ - sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + cli_printf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); rc = 1; } sqlite3_free(zSql); }else #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + if( c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 + || cli_strncmp(azArg[0], "indexes", n)==0) + ){ + sqlite3_str *pSql; + int i; + int allFlag = 0; + int sysFlag = 0; + int exprFlag = 0; + int debugFlag = 0; /* Undocument --debug flag */ + const char *zPattern = 0; + const char *zSep = "WHERE"; + + for(i=1; i<nArg; i++){ + if( azArg[i][0]=='-' ){ + const char *z = azArg[i]+1; + if( z[0]=='-' ) z++; + if( cli_strcmp(z,"all")==0 || cli_strcmp(z,"a")==0 ){ + allFlag = 1; + }else + if( cli_strcmp(z,"sys")==0 ){ + sysFlag = 1; + allFlag = 0; + }else + if( cli_strcmp(z,"expr")==0 ){ + exprFlag = 1; + allFlag = 0; + }else + if( cli_strcmp(z,"debug")==0 ){ + debugFlag = 1; + }else + { + dotCmdError(p, i, "unknown option", 0); + rc = 1; + goto meta_command_exit; + } + }else if( zPattern==0 ){ + zPattern = azArg[i]; + }else{ + dotCmdError(p, i, "unknown argument", 0); + rc = 1; + goto meta_command_exit; + } + } + + open_db(p, 0); + pSql = sqlite3_str_new(p->db); + sqlite3_str_appendf(pSql, + "SELECT if(t.schema='main',i.name,t.schema||'.'||i.name)\n" + "FROM pragma_table_list t, pragma_index_list(t.name,t.schema) i\n" + ); + if( exprFlag ){ + allFlag = 0; + sqlite3_str_appendf(pSql, + "%s (EXISTS(SELECT 1 FROM pragma_index_xinfo(i.name) WHERE cid=-2)\n" + " OR\n" + " EXISTS(SELECT cid FROM pragma_table_xinfo(t.name) WHERE hidden=2" + " INTERSECT " + " SELECT cid FROM pragma_index_info(i.name)))\n", zSep); + zSep = "AND"; + } + if( sysFlag ){ + sqlite3_str_appendf(pSql, + "%s i.name LIKE 'sqlite__autoindex__%%' ESCAPE '_'\n", zSep); + zSep = "AND"; + }else if( !allFlag ){ + sqlite3_str_appendf(pSql, + "%s i.name NOT LIKE 'sqlite__%%' ESCAPE '_'\n", zSep); + zSep = "AND"; + } + if( zPattern ){ + sqlite3_str_appendf(pSql, "%s i.name LIKE '%%%q%%'\n", zSep, zPattern); + } + sqlite3_str_appendf(pSql, "ORDER BY 1"); + + /* Run the SQL statement in "split" mode. */ + if( debugFlag ){ + cli_printf(stdout,"%s;\n", sqlite3_str_value(pSql)); + }else{ + modePush(p); + modeChange(p, MODE_Split); + shell_exec(p, sqlite3_str_value(pSql), 0); + modePop(p); + } + sqlite3_str_free(pSql); + }else + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ i64 iArg = 0; if( nArg==2 ){ @@ -30048,7 +33453,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iArg==0 ) iArg = -1; } if( (nArg!=1 && nArg!=2) || iArg<0 ){ - sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); + cli_printf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); rc = 1; goto meta_command_exit; } @@ -30069,7 +33474,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ iotrace = sqlite3_fopen(azArg[1], "w"); if( iotrace==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); sqlite3IoTrace = 0; rc = 1; }else{ @@ -30088,6 +33493,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, { "column", SQLITE_LIMIT_COLUMN }, { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, + { "parser_depth", SQLITE_LIMIT_PARSER_DEPTH }, { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, @@ -30101,7 +33507,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; i<ArraySize(aLimit); i++){ - sqlite3_fprintf(stdout, "%20s %d\n", aLimit[i].zLimitName, + cli_printf(stdout, "%20s %d\n", aLimit[i].zLimitName, sqlite3_limit(p->db, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ @@ -30116,14 +33522,14 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iLimit<0 ){ iLimit = i; }else{ - sqlite3_fprintf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); + cli_printf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); rc = 1; goto meta_command_exit; } } } if( iLimit<0 ){ - sqlite3_fprintf(stderr,"unknown limit: \"%s\"\n" + cli_printf(stderr,"unknown limit: \"%s\"\n" "enter \".limits\" with no arguments for a list.\n", azArg[1]); rc = 1; @@ -30132,9 +33538,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 ){ sqlite3_limit(p->db, aLimit[iLimit].limitCode, (int)integerValue(azArg[2])); + }else{ + cli_printf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } - sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, - sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } }else @@ -30182,174 +33589,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } output_file_close(p->pLog); if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; - p->pLog = output_file_open(zFile); + p->pLog = output_file_open(p, zFile); } }else if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ - const char *zMode = 0; - const char *zTabname = 0; - int i, n2; - int chng = 0; /* 0x01: change to cmopts. 0x02: Any other change */ - ColModeOpts cmOpts = ColModeOpts_default; - for(i=1; i<nArg; i++){ - const char *z = azArg[i]; - if( optionMatch(z,"wrap") && i+1<nArg ){ - cmOpts.iWrap = integerValue(azArg[++i]); - chng |= 1; - }else if( optionMatch(z,"ww") ){ - cmOpts.bWordWrap = 1; - chng |= 1; - }else if( optionMatch(z,"wordwrap") && i+1<nArg ){ - cmOpts.bWordWrap = (u8)booleanValue(azArg[++i]); - chng |= 1; - }else if( optionMatch(z,"quote") ){ - cmOpts.bQuote = 1; - chng |= 1; - }else if( optionMatch(z,"noquote") ){ - cmOpts.bQuote = 0; - chng |= 1; - }else if( optionMatch(z,"escape") && i+1<nArg ){ - /* See similar code at tag-20250224-1 */ - const char *zEsc = azArg[++i]; - int k; - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ - p->eEscMode = k; - chng |= 2; - break; - } - } - if( k>=ArraySize(shell_EscModeNames) ){ - sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" - " - choices:", zEsc); - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); - } - sqlite3_fprintf(stderr, "\n"); - rc = 1; - goto meta_command_exit; - } - }else if( zMode==0 ){ - zMode = z; - /* Apply defaults for qbox pseudo-mode. If that - * overwrites already-set values, user was informed of this. - */ - chng |= 1; - if( cli_strcmp(z, "qbox")==0 ){ - ColModeOpts cmo = ColModeOpts_default_qbox; - zMode = "box"; - cmOpts = cmo; - } - }else if( zTabname==0 ){ - zTabname = z; - }else if( z[0]=='-' ){ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); - eputz("options:\n" - " --escape MODE\n" - " --noquote\n" - " --quote\n" - " --wordwrap on/off\n" - " --wrap N\n" - " --ww\n"); - rc = 1; - goto meta_command_exit; - }else{ - sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); - rc = 1; - goto meta_command_exit; - } - } - if( !chng ){ - if( p->mode==MODE_Column - || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) - ){ - sqlite3_fprintf(p->out, - "current output mode: %s --wrap %d --wordwrap %s " - "--%squote --escape %s\n", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no", - shell_EscModeNames[p->eEscMode] - ); - }else{ - sqlite3_fprintf(p->out, - "current output mode: %s --escape %s\n", - modeDescr[p->mode], - shell_EscModeNames[p->eEscMode] - ); - } - } - if( zMode==0 ){ - zMode = modeDescr[p->mode]; - if( (chng&1)==0 ) cmOpts = p->cmOpts; - } - n2 = strlen30(zMode); - if( cli_strncmp(zMode,"lines",n2)==0 ){ - p->mode = MODE_Line; - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"columns",n2)==0 ){ - p->mode = MODE_Column; - if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ - p->showHeader = 1; - } - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"list",n2)==0 ){ - p->mode = MODE_List; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"html",n2)==0 ){ - p->mode = MODE_Html; - }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ - p->mode = MODE_Tcl; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"csv",n2)==0 ){ - p->mode = MODE_Csv; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); - }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ - p->mode = MODE_List; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); - }else if( cli_strncmp(zMode,"insert",n2)==0 ){ - p->mode = MODE_Insert; - set_table_name(p, zTabname ? zTabname : "table"); - if( p->eEscMode==SHELL_ESC_OFF ){ - ShellSetFlag(p, SHFLG_Newlines); - }else{ - ShellClearFlag(p, SHFLG_Newlines); - } - }else if( cli_strncmp(zMode,"quote",n2)==0 ){ - p->mode = MODE_Quote; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ - p->mode = MODE_Ascii; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); - }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ - p->mode = MODE_Markdown; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"table",n2)==0 ){ - p->mode = MODE_Table; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"box",n2)==0 ){ - p->mode = MODE_Box; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"count",n2)==0 ){ - p->mode = MODE_Count; - }else if( cli_strncmp(zMode,"off",n2)==0 ){ - p->mode = MODE_Off; - }else if( cli_strncmp(zMode,"json",n2)==0 ){ - p->mode = MODE_Json; - }else{ - eputz("Error: mode should be one of: " - "ascii box column csv html insert json line list markdown " - "qbox quote table tabs tcl\n"); - rc = 1; - } - p->cMode = p->mode; + rc = dotCmdMode(p); }else #ifndef SQLITE_SHELL_FIDDLE @@ -30358,9 +33603,9 @@ static int do_meta_command(char *zLine, ShellState *p){ eputz("Usage: .nonce NONCE\n"); rc = 1; }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ - sqlite3_fprintf(stderr,"line %d: incorrect nonce: \"%s\"\n", + cli_printf(stderr,"line %lld: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); - exit(1); + cli_exit(1); }else{ p->bSafeMode = 0; return 0; /* Return immediately to bypass the safe mode reset @@ -30371,8 +33616,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ - sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, - "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); + modeSetStr(&p->mode.spec.zNull, azArg[1]); }else{ eputz("Usage: .nullvalue STRING\n"); rc = 1; @@ -30385,6 +33629,9 @@ static int do_meta_command(char *zLine, ShellState *p){ int iName = 1; /* Index in azArg[] of the filename */ int newFlag = 0; /* True to delete file before opening */ int openMode = SHELL_OPEN_UNSPEC; + int openFlags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + + if( p->bSafeMode ) openFlags = SQLITE_OPEN_READONLY; /* Check for command-line arguments */ for(iName=1; iName<nArg; iName++){ @@ -30393,31 +33640,38 @@ static int do_meta_command(char *zLine, ShellState *p){ if( optionMatch(z,"new") ){ newFlag = 1; #ifdef SQLITE_HAVE_ZLIB - }else if( optionMatch(z, "zip") ){ + }else if( optionMatch(z, "zip") && !p->bSafeMode ){ openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( optionMatch(z, "append") ){ + }else if( optionMatch(z, "append") && !p->bSafeMode ){ openMode = SHELL_OPEN_APPENDVFS; }else if( optionMatch(z, "readonly") ){ - openMode = SHELL_OPEN_READONLY; + openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + openFlags |= SQLITE_OPEN_READONLY; + }else if( optionMatch(z, "exclusive") ){ + openFlags |= SQLITE_OPEN_EXCLUSIVE; + }else if( optionMatch(z, "ifexists") ){ + openFlags &= ~(SQLITE_OPEN_CREATE); }else if( optionMatch(z, "nofollow") ){ - p->openFlags |= SQLITE_OPEN_NOFOLLOW; + openFlags |= SQLITE_OPEN_NOFOLLOW; #ifndef SQLITE_OMIT_DESERIALIZE }else if( optionMatch(z, "deserialize") ){ openMode = SHELL_OPEN_DESERIALIZE; }else if( optionMatch(z, "hexdb") ){ openMode = SHELL_OPEN_HEXDB; + }else if( optionMatch(z, "normal") ){ + openMode = SHELL_OPEN_NORMAL; }else if( optionMatch(z, "maxsize") && iName+1<nArg ){ p->szMax = integerValue(azArg[++iName]); #endif /* SQLITE_OMIT_DESERIALIZE */ }else #endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); + cli_printf(stderr,"unknown option: %s\n", z); rc = 1; goto meta_command_exit; }else if( zFN ){ - sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); + cli_printf(stderr,"extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; }else{ @@ -30433,12 +33687,21 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(p->pAuxDb->zFreeOnClose); p->pAuxDb->zFreeOnClose = 0; p->openMode = openMode; - p->openFlags = 0; + p->openFlags = openFlags; p->szMax = 0; /* If a filename is specified, try to open it first */ if( zFN || p->openMode==SHELL_OPEN_HEXDB ){ - if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN); + if( newFlag && zFN && !p->bSafeMode ){ + if( cli_strncmp(zFN,"file:",5)==0 ){ + char *zDel = shellFilenameFromUri(zFN); + shell_check_oom(zDel); + shellDeleteFile(zDel); + sqlite3_free(zDel); + }else{ + shellDeleteFile(zFN); + } + } #ifndef SQLITE_SHELL_FIDDLE if( p->bSafeMode && p->openMode!=SHELL_OPEN_HEXDB @@ -30459,7 +33722,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->pAuxDb->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename); + cli_printf(stderr,"Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); }else{ p->pAuxDb->zFreeOnClose = zNewFilename; @@ -30479,144 +33742,7 @@ static int do_meta_command(char *zLine, ShellState *p){ || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0) ){ - char *zFile = 0; - int i; - int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ - int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ - int bPlain = 0; /* --plain option */ - static const char *zBomUtf8 = "\357\273\277"; - const char *zBom = 0; - - failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); - if( c=='e' ){ - eMode = 'x'; - bOnce = 2; - }else if( c=='w' ){ - eMode = 'w'; - bOnce = 2; - }else if( cli_strncmp(azArg[0],"once",n)==0 ){ - bOnce = 1; - } - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' ){ - if( z[1]=='-' ) z++; - if( cli_strcmp(z,"-bom")==0 ){ - zBom = zBomUtf8; - }else if( cli_strcmp(z,"-plain")==0 ){ - bPlain = 1; - }else if( c=='o' && cli_strcmp(z,"-x")==0 ){ - eMode = 'x'; /* spreadsheet */ - }else if( c=='o' && cli_strcmp(z,"-e")==0 ){ - eMode = 'e'; /* text editor */ - }else if( c=='o' && cli_strcmp(z,"-w")==0 ){ - eMode = 'w'; /* Web browser */ - }else{ - sqlite3_fprintf(p->out, - "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); - showHelp(p->out, azArg[0]); - rc = 1; - goto meta_command_exit; - } - }else if( zFile==0 && eMode==0 ){ - if( cli_strcmp(z, "off")==0 ){ -#ifdef _WIN32 - zFile = sqlite3_mprintf("nul"); -#else - zFile = sqlite3_mprintf("/dev/null"); -#endif - }else{ - zFile = sqlite3_mprintf("%s", z); - } - if( zFile && zFile[0]=='|' ){ - while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); - break; - } - }else{ - sqlite3_fprintf(p->out, - "ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); - showHelp(p->out, azArg[0]); - rc = 1; - sqlite3_free(zFile); - goto meta_command_exit; - } - } - if( zFile==0 ){ - zFile = sqlite3_mprintf("stdout"); - } - shell_check_oom(zFile); - if( bOnce ){ - p->outCount = 2; - }else{ - p->outCount = 0; - } - output_reset(p); -#ifndef SQLITE_NOHAVE_SYSTEM - if( eMode=='e' || eMode=='x' || eMode=='w' ){ - p->doXdgOpen = 1; - outputModePush(p); - if( eMode=='x' ){ - /* spreadsheet mode. Output as CSV. */ - newTempFile(p, "csv"); - ShellClearFlag(p, SHFLG_Echo); - p->mode = MODE_Csv; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); -#ifdef _WIN32 - zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does - ** not work without it. */ -#endif - }else if( eMode=='w' ){ - /* web-browser mode. */ - newTempFile(p, "html"); - if( !bPlain ) p->mode = MODE_Www; - }else{ - /* text editor mode */ - newTempFile(p, "txt"); - } - sqlite3_free(zFile); - zFile = sqlite3_mprintf("%s", p->zTempFile); - } -#endif /* SQLITE_NOHAVE_SYSTEM */ - shell_check_oom(zFile); - if( zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - rc = 1; - output_redir(p, stdout); -#else - FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); - if( pfPipe==0 ){ - assert( stderr!=NULL ); - sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); - rc = 1; - }else{ - output_redir(p, pfPipe); - if( zBom ) sqlite3_fputs(zBom, pfPipe); - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } -#endif - }else{ - FILE *pfFile = output_file_open(zFile); - if( pfFile==0 ){ - if( cli_strcmp(zFile,"off")!=0 ){ - assert( stderr!=NULL ); - sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile); - } - rc = 1; - } else { - output_redir(p, pfFile); - if( zBom ) sqlite3_fputs(zBom, pfFile); - if( bPlain && eMode=='w' ){ - sqlite3_fputs( - "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", - pfFile - ); - } - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } - } - sqlite3_free(zFile); + rc = dotCmdOutput(p); }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -30653,7 +33779,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT key, quote(value) " "FROM temp.sqlite_parameters;", -1, &pStmt, 0); while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), sqlite3_column_text(pStmt,1)); } @@ -30699,7 +33825,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rx!=SQLITE_OK ){ - sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + cli_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); pStmt = 0; rc = 1; @@ -30729,10 +33855,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i<nArg; i++){ - if( i>1 ) sqlite3_fputs(" ", p->out); - sqlite3_fputs(azArg[i], p->out); + if( i>1 ) cli_puts(" ", p->out); + cli_puts(azArg[i], p->out); } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -30759,6 +33885,19 @@ static int do_meta_command(char *zLine, ShellState *p){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } + if( cli_strcmp(z,"timeout")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + p->tmProgress = atof(azArg[i]); + if( p->tmProgress>0.0 ){ + p->flgProgress = SHELL_PROGRESS_QUIET|SHELL_PROGRESS_TMOUT; + if( nn==0 ) nn = 100; + } + continue; + } if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ eputz("Error: missing argument on --limit\n"); @@ -30769,7 +33908,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } continue; } - sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); + cli_printf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); rc = 1; goto meta_command_exit; }else{ @@ -30799,7 +33938,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){ FILE *inSaved = p->in; - int savedLineno = p->lineno; + i64 savedLineno = p->lineno; failIfSafeMode(p, "cannot run .read in safe mode"); if( nArg!=2 ){ eputz("Usage: .read FILE\n"); @@ -30813,18 +33952,20 @@ static int do_meta_command(char *zLine, ShellState *p){ #else p->in = sqlite3_popen(azArg[1]+1, "r"); if( p->in==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ - rc = process_input(p); + rc = process_input(p, "<pipe>"); pclose(p->in); } #endif }else if( (p->in = openChrSource(azArg[1]))==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ - rc = process_input(p); + char *zFilename = strdup(azArg[1]); + rc = process_input(p, zFilename); + free(zFilename); fclose(p->in); } p->in = inSaved; @@ -30854,7 +33995,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); close_db(pSrc); return 1; } @@ -30892,21 +34033,21 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ if( nArg==2 ){ if( cli_strcmp(azArg[1], "vm")==0 ){ - p->scanstatsOn = 3; + p->mode.scanstatsOn = 3; }else if( cli_strcmp(azArg[1], "est")==0 ){ - p->scanstatsOn = 2; + p->mode.scanstatsOn = 2; }else{ - p->scanstatsOn = (u8)booleanValue(azArg[1]); + p->mode.scanstatsOn = (u8)booleanValue(azArg[1]); } open_db(p, 0); sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 ); #if !defined(SQLITE_ENABLE_STMT_SCANSTATUS) eputz("Warning: .scanstats not available in this build.\n"); #elif !defined(SQLITE_ENABLE_BYTECODE_VTAB) - if( p->scanstatsOn==3 ){ + if( p->mode.scanstatsOn==3 ){ eputz("Warning: \".scanstats vm\" not available in this build.\n"); } #endif @@ -30917,7 +34058,6 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ - ShellText sSelect; ShellState data; char *zErrMsg = 0; const char *zDiv = "("; @@ -30925,22 +34065,27 @@ static int do_meta_command(char *zLine, ShellState *p){ int iSchema = 0; int bDebug = 0; int bNoSystemTabs = 0; + int bIndent = 0; int ii; - + sqlite3_str *pSql; + sqlite3_stmt *pStmt = 0; + open_db(p, 0); memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; - initText(&sSelect); + data.mode.spec.bTitles = QRF_No; + data.mode.eMode = MODE_List; + data.mode.spec.eText = QRF_TEXT_Plain; + data.mode.spec.nCharLimit = 0; + data.mode.spec.zRowSep = "\n"; for(ii=1; ii<nArg; ii++){ if( optionMatch(azArg[ii],"indent") ){ - data.cMode = data.mode = MODE_Pretty; + bIndent = 1; }else if( optionMatch(azArg[ii],"debug") ){ bDebug = 1; }else if( optionMatch(azArg[ii],"nosys") ){ bNoSystemTabs = 1; }else if( azArg[ii][0]=='-' ){ - sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); + cli_printf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); rc = 1; goto meta_command_exit; }else if( zName==0 ){ @@ -30957,96 +34102,85 @@ static int do_meta_command(char *zLine, ShellState *p){ || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0 || sqlite3_strlike(zName,"sqlite_temp_schema", '\\')==0; if( isSchema ){ - char *new_argv[2], *new_colv[2]; - new_argv[0] = sqlite3_mprintf( - "CREATE TABLE %s (\n" + cli_printf(p->out, + "CREATE TABLE %ssqlite_schema (\n" " type text,\n" " name text,\n" " tbl_name text,\n" " rootpage integer,\n" " sql text\n" - ")", zName); - shell_check_oom(new_argv[0]); - new_argv[1] = 0; - new_colv[0] = "sql"; - new_colv[1] = 0; - callback(&data, 1, new_argv, new_colv); - sqlite3_free(new_argv[0]); + ");\n", + sqlite3_strlike("sqlite_t%",zName,0)==0 ? "temp." : "" + ); } } - if( zDiv ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", - -1, &pStmt, 0); - if( rc ){ - shellDatabaseError(p->db); - sqlite3_finalize(pStmt); - rc = 1; - goto meta_command_exit; - } - appendText(&sSelect, "SELECT sql FROM", 0); - iSchema = 0; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); - char zScNum[30]; - sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); - appendText(&sSelect, zDiv, 0); - zDiv = " UNION ALL "; - appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); - if( sqlite3_stricmp(zDb, "main")!=0 ){ - appendText(&sSelect, zDb, '\''); - }else{ - appendText(&sSelect, "NULL", 0); - } - appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0); - appendText(&sSelect, zScNum, 0); - appendText(&sSelect, " AS snum, ", 0); - appendText(&sSelect, zDb, '\''); - appendText(&sSelect, " AS sname FROM ", 0); - appendText(&sSelect, zDb, quoteChar(zDb)); - appendText(&sSelect, ".sqlite_schema", 0); - } + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + shellDatabaseError(p->db); sqlite3_finalize(pStmt); -#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS - if( zName ){ - appendText(&sSelect, - " UNION ALL SELECT shell_module_schema(name)," - " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", - 0); + + rc = 1; + goto meta_command_exit; + } + pSql = sqlite3_str_new(p->db); + sqlite3_str_appendf(pSql, "SELECT sql FROM", 0); + iSchema = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 1); + char zScNum[30]; + sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); + sqlite3_str_appendall(pSql, zDiv); + zDiv = " UNION ALL "; + if( sqlite3_stricmp(zDb, "main")==0 ){ + sqlite3_str_appendf(pSql, + "SELECT shell_format_schema(shell_add_schema(sql,NULL,name),%d)", + bIndent); + }else{ + sqlite3_str_appendf(pSql, + "SELECT shell_format_schema(shell_add_schema(sql,%Q,name),%d)", + zDb, bIndent); } + sqlite3_str_appendf(pSql, + " AS sql, type, tbl_name, name, rowid, %d AS snum, %Q as sname", + ++iSchema, zDb); + sqlite3_str_appendf(pSql," FROM \"%w\".sqlite_schema", zDb); + } + sqlite3_finalize(pStmt); +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + if( zName ){ + sqlite3_str_appendall(pSql, + " UNION ALL SELECT shell_module_schema(name)," + " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list"); + } #endif - appendText(&sSelect, ") WHERE ", 0); - if( zName ){ - char *zQarg = sqlite3_mprintf("%Q", zName); - int bGlob; - shell_check_oom(zQarg); - bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || - strchr(zName, '[') != 0; - if( strchr(zName, '.') ){ - appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); - }else{ - appendText(&sSelect, "lower(tbl_name)", 0); - } - appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); - appendText(&sSelect, zQarg, 0); - if( !bGlob ){ - appendText(&sSelect, " ESCAPE '\\' ", 0); - } - appendText(&sSelect, " AND ", 0); - sqlite3_free(zQarg); - } - if( bNoSystemTabs ){ - appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0); + sqlite3_str_appendf(pSql, ") WHERE ", 0); + if( zName ){ + int bGlob; + bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; + if( strchr(zName, '.') ){ + sqlite3_str_appendall(pSql, "lower(format('%%s.%%s',sname,tbl_name))"); + }else{ + sqlite3_str_appendall(pSql, "lower(tbl_name)"); } - appendText(&sSelect, "sql IS NOT NULL" - " ORDER BY snum, rowid", 0); - if( bDebug ){ - sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.z); + if( bGlob ){ + sqlite3_str_appendf(pSql, " GLOB %Q AND ", zName); }else{ - rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg); + sqlite3_str_appendf(pSql, " LIKE %Q ESCAPE '\\' AND ", zName); } - freeText(&sSelect); } + if( bNoSystemTabs ){ + sqlite3_str_appendf(pSql, " name NOT LIKE 'sqlite__%%' ESCAPE '_' AND "); + } + sqlite3_str_appendf(pSql, "sql IS NOT NULL ORDER BY snum, rowid"); + if( bDebug ){ + cli_printf(p->out, "SQL: %s;\n", sqlite3_str_value(pSql)); + }else{ + rc = shell_exec(&data, sqlite3_str_value(pSql), &zErrMsg); + } + sqlite3_str_free(pSql); + if( zErrMsg ){ shellEmitError(zErrMsg); sqlite3_free(zErrMsg); @@ -31102,7 +34236,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ rc = sqlite3session_attach(pSession->p, azCmd[1]); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n",rc); rc = 0; } @@ -31122,7 +34256,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( pSession->p==0 ) goto session_not_open; out = sqlite3_fopen(azCmd[1], "wb"); if( out==0 ){ - sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n", + cli_printf(stderr,"ERROR: cannot open \"%s\" for writing\n", azCmd[1]); }else{ int szChng; @@ -31133,12 +34267,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); } if( rc ){ - sqlite3_fprintf(stdout, "Error: error code %d\n", rc); + cli_printf(stdout, "Error: error code %d\n", rc); rc = 0; } if( pChng && fwrite(pChng, szChng, 1, out)!=1 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", szChng); } sqlite3_free(pChng); @@ -31166,7 +34300,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_enable(pSession->p, ii); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s enable flag = %d\n", pSession->zName, ii); } }else @@ -31175,7 +34309,8 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Set a list of GLOB patterns of table names to be excluded. */ if( cli_strcmp(azCmd[0], "filter")==0 ){ - int ii, nByte; + int ii; + i64 nByte; if( nCmd<2 ) goto session_syntax_error; if( pAuxDb->nSession ){ for(ii=0; ii<pSession->nFilter; ii++){ @@ -31183,7 +34318,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(pSession->azFilter); nByte = sizeof(pSession->azFilter[0])*(nCmd-1); - pSession->azFilter = sqlite3_malloc( nByte ); + pSession->azFilter = sqlite3_malloc64( nByte ); shell_check_oom( pSession->azFilter ); for(ii=1; ii<nCmd; ii++){ char *x = pSession->azFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); @@ -31202,7 +34337,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_indirect(pSession->p, ii); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s indirect flag = %d\n", pSession->zName, ii); } }else @@ -31215,7 +34350,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ ii = sqlite3session_isempty(pSession->p); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s isempty flag = %d\n", pSession->zName, ii); } }else @@ -31225,7 +34360,7 @@ static int do_meta_command(char *zLine, ShellState *p){ */ if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; i<pAuxDb->nSession; i++){ - sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); + cli_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); } }else @@ -31240,19 +34375,19 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zName[0]==0 ) goto session_syntax_error; for(i=0; i<pAuxDb->nSession; i++){ if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ - sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName); + cli_printf(stderr,"Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ - sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc); + cli_printf(stderr,"Cannot open session: error code=%d\n", rc); rc = 0; goto meta_command_exit; } @@ -31276,7 +34411,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, v; for(i=1; i<nArg; i++){ v = booleanValue(azArg[i]); - sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); + cli_printf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); } } if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ @@ -31285,7 +34420,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char zBuf[200]; v = integerValue(azArg[i]); sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v); - sqlite3_fputs(zBuf, p->out); + cli_puts(zBuf, p->out); } } }else @@ -31312,9 +34447,9 @@ static int do_meta_command(char *zLine, ShellState *p){ bVerbose++; }else { - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); - sqlite3_fputs("Should be one of: --init -v\n", stderr); + cli_puts("Should be one of: --init -v\n", stderr); rc = 1; goto meta_command_exit; } @@ -31359,34 +34494,34 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zAns==0 ) continue; k = 0; if( bVerbose>0 ){ - sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql); + cli_printf(stdout, "%d: %s %s\n", tno, zOp, zSql); } if( cli_strcmp(zOp,"memo")==0 ){ - sqlite3_fprintf(p->out, "%s\n", zSql); + cli_printf(p->out, "%s\n", zSql); }else if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; str.n = 0; - str.z[0] = 0; + str.zTxt[0] = 0; rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); nTest++; if( bVerbose ){ - sqlite3_fprintf(p->out, "Result: %s\n", str.z); + cli_printf(p->out, "Result: %s\n", str.zTxt); } if( rc || zErrMsg ){ nErr++; rc = 1; - sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); + cli_printf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); sqlite3_free(zErrMsg); - }else if( cli_strcmp(zAns,str.z)!=0 ){ + }else if( cli_strcmp(zAns,str.zTxt)!=0 ){ nErr++; rc = 1; - sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns); - sqlite3_fprintf(p->out, "%d: Got: [%s]\n", tno, str.z); + cli_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + cli_printf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); } } else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); rc = 1; break; @@ -31395,7 +34530,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } /* End loop over k */ freeText(&str); - sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest); + cli_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); }else if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ @@ -31404,12 +34539,10 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } if( nArg>=2 ){ - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, - "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]); + modeSetStr(&p->mode.spec.zColumnSep, azArg[1]); } if( nArg>=3 ){ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, - "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]); + modeSetStr(&p->mode.spec.zRowSep,azArg[2]); } }else @@ -31443,7 +34576,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bDebug = 1; }else { - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); showHelp(p->out, azArg[0]); rc = 1; @@ -31467,7 +34600,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" - " AND name NOT LIKE 'sqlite_%'" + " AND name NOT LIKE 'sqlite__%' ESCAPE '_'" " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); @@ -31498,7 +34631,7 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); } appendText(&sSql, zSep, 0); - appendText(&sSql, sQuery.z, '\''); + appendText(&sSql, sQuery.zTxt, '\''); sQuery.n = 0; appendText(&sSql, ",", 0); appendText(&sSql, zTab, '\''); @@ -31510,19 +34643,19 @@ static int do_meta_command(char *zLine, ShellState *p){ "%s))" " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label" " FROM [sha3sum$query]", - sSql.z, iSize); + sSql.zTxt, iSize); }else{ zSql = sqlite3_mprintf( "%s))" " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash" " FROM [sha3sum$query]", - sSql.z, iSize); + sSql.zTxt, iSize); } shell_check_oom(zSql); freeText(&sQuery); freeText(&sSql); if( bDebug ){ - sqlite3_fprintf(p->out, "%s\n", zSql); + cli_printf(p->out, "%s\n", zSql); }else{ shell_exec(p, zSql, 0); } @@ -31532,7 +34665,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zRevText = /* Query for reversible to-blob-to-text check */ "SELECT lower(name) as tname FROM sqlite_schema\n" "WHERE type='table' AND coalesce(rootpage,0)>1\n" - "AND name NOT LIKE 'sqlite_%%'%s\n" + "AND name NOT LIKE 'sqlite__%%' ESCAPE '_'%s\n" "ORDER BY 1 collate nocase"; zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); zRevText = sqlite3_mprintf( @@ -31552,7 +34685,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "' OR ') as query, tname from tabcols group by tname)" , zRevText); shell_check_oom(zRevText); - if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText); + if( bDebug ) cli_printf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); if( lrc!=SQLITE_OK ){ /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the @@ -31565,7 +34698,7 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery); + if( bDebug ) cli_printf(p->out, "%s\n", zGenQuery); if( lrc!=SQLITE_OK ){ rc = 1; }else{ @@ -31573,7 +34706,7 @@ static int do_meta_command(char *zLine, ShellState *p){ double countIrreversible = sqlite3_column_double(pCheckStmt, 0); if( countIrreversible>0 ){ int sz = (int)(countIrreversible + 0.5); - sqlite3_fprintf(stderr, + cli_printf(stderr, "Digest includes %d invalidly encoded text field%s.\n", sz, (sz>1)? "s": ""); } @@ -31612,7 +34745,7 @@ static int do_meta_command(char *zLine, ShellState *p){ x = zCmd!=0 ? system(zCmd) : 1; /*consoleRenewSetup();*/ sqlite3_free(zCmd); - if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x); + if( x ) cli_printf(stderr,"System command returns %d\n", x); }else #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ @@ -31625,48 +34758,51 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } - sqlite3_fprintf(p->out, "%12.12s: %s\n","echo", - azBool[ShellHasFlag(p, SHFLG_Echo)]); - sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); - sqlite3_fprintf(p->out, "%12.12s: %s\n","explain", - p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - sqlite3_fprintf(p->out, "%12.12s: %s\n","headers", - azBool[p->showHeader!=0]); - if( p->mode==MODE_Column - || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) + cli_printf(p->out, "%12.12s: %s\n","echo", + azBool[(p->mode.mFlags & MFLG_ECHO)!=0]); + cli_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->mode.autoEQP&3]); + cli_printf(p->out, "%12.12s: %s\n","explain", + p->mode.autoExplain ? "auto" : "off"); + cli_printf(p->out, "%12.12s: %s\n","headers", + azBool[p->mode.spec.bTitles==QRF_Yes]); + if( p->mode.spec.eStyle==QRF_STYLE_Column + || p->mode.spec.eStyle==QRF_STYLE_Box + || p->mode.spec.eStyle==QRF_STYLE_Table + || p->mode.spec.eStyle==QRF_STYLE_Markdown ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no"); + aModeInfo[p->mode.eMode].zName, p->mode.spec.nWrap, + p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off", + p->mode.spec.eText==QRF_TEXT_Sql ? "" : "no"); }else{ - sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); + cli_printf(p->out, "%12.12s: %s\n","mode", + aModeInfo[p->mode.eMode].zName); } - sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue"); - output_c_string(p->out, p->nullValue); - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: %s\n","output", + cli_printf(p->out, "%12.12s: ", "nullvalue"); + output_c_string(p->out, p->mode.spec.zNull); + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: %s\n","output", strlen30(p->outfile) ? p->outfile : "stdout"); - sqlite3_fprintf(p->out, "%12.12s: ", "colseparator"); - output_c_string(p->out, p->colSeparator); - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator"); - output_c_string(p->out, p->rowSeparator); - sqlite3_fputs("\n", p->out); + cli_printf(p->out, "%12.12s: ", "colseparator"); + output_c_string(p->out, p->mode.spec.zColumnSep); + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: ", "rowseparator"); + output_c_string(p->out, p->mode.spec.zRowSep); + cli_puts("\n", p->out); switch( p->statsOn ){ case 0: zOut = "off"; break; default: zOut = "on"; break; case 2: zOut = "stmt"; break; case 3: zOut = "vmstep"; break; } - sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut); - sqlite3_fprintf(p->out, "%12.12s: ", "width"); - for (i=0;i<p->nWidth;i++) { - sqlite3_fprintf(p->out, "%d ", p->colWidth[i]); + cli_printf(p->out, "%12.12s: %s\n","stats", zOut); + cli_printf(p->out, "%12.12s: ", "width"); + for(i=0; i<p->mode.spec.nWidth; i++){ + cli_printf(p->out, "%d ", (int)p->mode.spec.aWidth[i]); } - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename", + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: %s\n", "filename", p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else @@ -31687,16 +34823,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) - || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 - || cli_strncmp(azArg[0], "indexes", n)==0) ) - ){ + if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) ){ sqlite3_stmt *pStmt; - char **azResult; - int nRow, nAlloc; - int ii; - ShellText s; - initText(&s); + sqlite3_str *pSql; + const char *zPattern = nArg>1 ? azArg[1] : 0; + open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ @@ -31704,112 +34835,46 @@ static int do_meta_command(char *zLine, ShellState *p){ return shellDatabaseError(p->db); } - if( nArg>2 && c=='i' ){ - /* It is an historical accident that the .indexes command shows an error - ** when called with the wrong number of arguments whereas the .tables - ** command does not. */ - eputz("Usage: .indexes ?LIKE-PATTERN?\n"); - rc = 1; - sqlite3_finalize(pStmt); - goto meta_command_exit; - } - for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ + pSql = sqlite3_str_new(p->db); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); if( zDbName==0 ) continue; - if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0); + if( sqlite3_str_length(pSql) ){ + sqlite3_str_appendall(pSql, " UNION ALL "); + } if( sqlite3_stricmp(zDbName, "main")==0 ){ - appendText(&s, "SELECT name FROM ", 0); + sqlite3_str_appendall(pSql, "SELECT name FROM "); }else{ - appendText(&s, "SELECT ", 0); - appendText(&s, zDbName, '\''); - appendText(&s, "||'.'||name FROM ", 0); + sqlite3_str_appendf(pSql, "SELECT %Q||'.'||name FROM ", zDbName); } - appendText(&s, zDbName, '"'); - appendText(&s, ".sqlite_schema ", 0); - if( c=='t' ){ - appendText(&s," WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite_%'" - " AND name LIKE ?1", 0); - }else{ - appendText(&s," WHERE type='index'" - " AND tbl_name LIKE ?1", 0); + sqlite3_str_appendf(pSql, "\"%w\".sqlite_schema", zDbName); + sqlite3_str_appendf(pSql, + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_'" + ); + if( zPattern ){ + sqlite3_str_appendf(pSql," AND name LIKE %Q", zPattern); } } rc = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ){ - appendText(&s, " ORDER BY 1", 0); - rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0); - } - freeText(&s); - if( rc ) return shellDatabaseError(p->db); - - /* Run the SQL statement prepared by the above block. Store the results - ** as an array of nul-terminated strings in azResult[]. */ - nRow = nAlloc = 0; - azResult = 0; - if( nArg>1 ){ - sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); - }else{ - sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); - } - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - if( nRow>=nAlloc ){ - char **azNew; - int n2 = nAlloc*2 + 10; - azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); - shell_check_oom(azNew); - nAlloc = n2; - azResult = azNew; - } - azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - shell_check_oom(azResult[nRow]); - nRow++; - } - if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ - rc = shellDatabaseError(p->db); - } - - /* Pretty-print the contents of array azResult[] to the output */ - if( rc==0 && nRow>0 ){ - int len, maxlen = 0; - int i, j; - int nPrintCol, nPrintRow; - for(i=0; i<nRow; i++){ - len = strlen30(azResult[i]); - if( len>maxlen ) maxlen = len; - } - nPrintCol = 80/(maxlen+2); - if( nPrintCol<1 ) nPrintCol = 1; - nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; - for(i=0; i<nPrintRow; i++){ - for(j=i; j<nRow; j+=nPrintRow){ - char *zSp = j<nPrintRow ? "" : " "; - sqlite3_fprintf(p->out, - "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); - } - sqlite3_fputs("\n", p->out); - } + sqlite3_str_appendall(pSql, " ORDER BY 1"); } - for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]); - sqlite3_free(azResult); + /* Run the SQL statement in "split" mode. */ + modePush(p); + modeChange(p, MODE_Split); + shell_exec(p, sqlite3_str_value(pSql), 0); + sqlite3_str_free(pSql); + modePop(p); + if( rc ) return shellDatabaseError(p->db); }else -#ifndef SQLITE_SHELL_FIDDLE - /* Begin redirecting output to the file "testcase-out.txt" */ + /* Set the p->zTestcase name and begin redirecting output into + ** the cli_output_capture sqlite3_str */ if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ - output_reset(p); - p->out = output_file_open("testcase-out.txt"); - if( p->out==0 ){ - eputz("Error: cannot open 'testcase-out.txt'\n"); - } - if( nArg>=2 ){ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); - }else{ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); - } + rc = dotCmdTestcase(p); }else -#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ @@ -31822,7 +34887,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"always", SQLITE_TESTCTRL_ALWAYS, 1, "BOOLEAN" }, {"assert", SQLITE_TESTCTRL_ASSERT, 1, "BOOLEAN" }, /*{"benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, "" },*/ - /*{"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ + {"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "SIZE INT-ARRAY"}, {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, {"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..." }, @@ -31862,10 +34927,10 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all test-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - sqlite3_fputs("Available test-controls:\n", p->out); + cli_puts("Available test-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue; - sqlite3_fprintf(p->out, " .testctrl %s %s\n", + cli_printf(p->out, " .testctrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -31882,7 +34947,7 @@ static int do_meta_command(char *zLine, ShellState *p){ testctrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n" + cli_printf(stderr,"Error: ambiguous test-control: \"%s\"\n" "Use \".testctrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -31890,7 +34955,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( testctrl<0 ){ - sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n" + cli_printf(stderr,"Error: unknown test-control: %s\n" "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ @@ -31921,7 +34986,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { 0x00000080, 1, "Transitive" }, { 0x00000100, 1, "OmitNoopJoin" }, { 0x00000200, 1, "CountOfView" }, - { 0x00000400, 1, "CurosrHints" }, + { 0x00000400, 1, "CursorHints" }, { 0x00000800, 1, "Stat4" }, { 0x00001000, 1, "PushDown" }, { 0x00002000, 1, "SimplifyJoin" }, @@ -31941,6 +35006,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { 0x08000000, 1, "OnePass" }, { 0x10000000, 1, "OrderBySubq" }, { 0x20000000, 1, "StarQuery" }, + { 0x40000000, 1, "ExistsToJoin" }, { 0xffffffff, 0, "All" }, }; unsigned int curOpt; @@ -31969,13 +35035,13 @@ static int do_meta_command(char *zLine, ShellState *p){ if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break; } if( jj>=ArraySize(aLabel) ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: no such optimization: \"%s\"\n", zLabel); - sqlite3_fputs("Should be one of:", stderr); + cli_puts("Should be one of:", stderr); for(jj=0; jj<ArraySize(aLabel); jj++){ - sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel); + cli_printf(stderr," %s", aLabel[jj].zLabel); } - sqlite3_fputs("\n", stderr); + cli_puts("\n", stderr); rc = 1; goto meta_command_exit; } @@ -31993,23 +35059,23 @@ static int do_meta_command(char *zLine, ShellState *p){ if( m & newOpt ) nOff++; } if( nOff<12 ){ - sqlite3_fputs("+All", p->out); + cli_puts("+All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)!=0 ){ - sqlite3_fprintf(p->out, " -%s", aLabel[ii].zLabel); + cli_printf(p->out, " -%s", aLabel[ii].zLabel); } } }else{ - sqlite3_fputs("-All", p->out); + cli_puts("-All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)==0 ){ - sqlite3_fprintf(p->out, " +%s", aLabel[ii].zLabel); + cli_printf(p->out, " +%s", aLabel[ii].zLabel); } } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); rc2 = isOk = 3; break; } @@ -32049,7 +35115,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3 *db; if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){ sqlite3_randomness(sizeof(ii),&ii); - sqlite3_fprintf(stdout, "-- random seed: %d\n", ii); + cli_printf(stdout, "-- random seed: %d\n", ii); } if( nArg==3 ){ db = 0; @@ -32102,7 +35168,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_SEEK_COUNT: { u64 x = 0; rc2 = sqlite3_test_control(testctrl, p->db, &x); - sqlite3_fprintf(p->out, "%llu\n", x); + cli_printf(p->out, "%llu\n", x); isOk = 3; break; } @@ -32133,11 +35199,11 @@ static int do_meta_command(char *zLine, ShellState *p){ int val = 0; rc2 = sqlite3_test_control(testctrl, -id, &val); if( rc2!=SQLITE_OK ) break; - if( id>1 ) sqlite3_fputs(" ", p->out); - sqlite3_fprintf(p->out, "%d: %d", id, val); + if( id>1 ) cli_puts(" ", p->out); + cli_printf(p->out, "%d: %d", id, val); id++; } - if( id>1 ) sqlite3_fputs("\n", p->out); + if( id>1 ) cli_puts("\n", p->out); isOk = 3; } break; @@ -32160,6 +35226,49 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_test_control(testctrl, &rc2); break; + case SQLITE_TESTCTRL_BITVEC_TEST: { + /* Examples: + ** .testctrl bitvec_test 100 6,1 -- Show BITVEC constants + ** .testctrl bitvec_test 1000 1,12,7,3 -- Simple test + ** ---- -------- + ** size of Bitvec -----^ ^--- aOp array. 0 added at end. + ** + ** See comments on sqlite3BitvecBuiltinTest() for more information + ** about the aOp[] array. + */ + int iSize; + const char *zTestArg; + int nOp; + int ii, jj, x; + int *aOp; + if( nArg!=4 ){ + cli_printf(stderr, + "ERROR - should be: \".testctrl bitvec_test SIZE INT-ARRAY\"\n" + ); + rc = 1; + goto meta_command_exit; + } + isOk = 3; + iSize = (int)integerValue(azArg[2]); + zTestArg = azArg[3]; + nOp = (int)strlen(zTestArg)+1; + aOp = malloc( sizeof(int)*(nOp+1) ); + shell_check_oom(aOp); + memset(aOp, 0, sizeof(int)*(nOp+1) ); + for(ii = jj = x = 0; zTestArg[ii]!=0; ii++){ + if( IsDigit(zTestArg[ii]) ){ + x = x*10 + zTestArg[ii] - '0'; + }else{ + aOp[jj++] = x; + x = 0; + } + } + aOp[jj] = x; + x = sqlite3_test_control(testctrl, iSize, aOp); + cli_printf(p->out, "result: %d\n", x); + free(aOp); + break; + } case SQLITE_TESTCTRL_FAULT_INSTALL: { int kk; int bShowHelp = nArg<=2; @@ -32179,21 +35288,21 @@ static int do_meta_command(char *zLine, ShellState *p){ faultsim_state.nHit = 0; sqlite3_test_control(testctrl, faultsim_callback); }else if( cli_strcmp(z,"status")==0 ){ - sqlite3_fprintf(p->out, "faultsim.iId: %d\n", + cli_printf(p->out, "faultsim.iId: %d\n", faultsim_state.iId); - sqlite3_fprintf(p->out, "faultsim.iErr: %d\n", + cli_printf(p->out, "faultsim.iErr: %d\n", faultsim_state.iErr); - sqlite3_fprintf(p->out, "faultsim.iCnt: %d\n", + cli_printf(p->out, "faultsim.iCnt: %d\n", faultsim_state.iCnt); - sqlite3_fprintf(p->out, "faultsim.nHit: %d\n", + cli_printf(p->out, "faultsim.nHit: %d\n", faultsim_state.nHit); - sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n", + cli_printf(p->out, "faultsim.iInterval: %d\n", faultsim_state.iInterval); - sqlite3_fprintf(p->out, "faultsim.eVerbose: %d\n", + cli_printf(p->out, "faultsim.eVerbose: %d\n", faultsim_state.eVerbose); - sqlite3_fprintf(p->out, "faultsim.nRepeat: %d\n", + cli_printf(p->out, "faultsim.nRepeat: %d\n", faultsim_state.nRepeat); - sqlite3_fprintf(p->out, "faultsim.nSkip: %d\n", + cli_printf(p->out, "faultsim.nSkip: %d\n", faultsim_state.nSkip); }else if( cli_strcmp(z,"-v")==0 ){ if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++; @@ -32212,7 +35321,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){ bShowHelp = 1; }else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unrecognized fault_install argument: \"%s\"\n", azArg[kk]); rc = 1; @@ -32221,7 +35330,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( bShowHelp ){ - sqlite3_fputs( + cli_puts( "Usage: .testctrl fault_install ARGS\n" "Possible arguments:\n" " off Disable faultsim\n" @@ -32243,13 +35352,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( isOk==0 && iCtrl>=0 ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ - sqlite3_fprintf(p->out, "%d\n", rc2); + cli_printf(p->out, "%d\n", rc2); }else if( isOk==2 ){ - sqlite3_fprintf(p->out, "0x%08x\n", rc2); + cli_printf(p->out, "0x%08x\n", rc2); } }else #endif /* !defined(SQLITE_UNTESTABLE) */ @@ -32261,13 +35370,17 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ - enableTimer = booleanValue(azArg[1]); - if( enableTimer && !HAS_TIMER ){ + if( cli_strcmp(azArg[1],"once")==0 ){ + p->enableTimer = 1; + }else{ + p->enableTimer = 2*booleanValue(azArg[1]); + } + if( p->enableTimer && !HAS_TIMER ){ eputz("Error: timer not available on this system.\n"); - enableTimer = 0; + p->enableTimer = 0; } }else{ - eputz("Usage: .timer on|off\n"); + eputz("Usage: .timer on|off|once\n"); rc = 1; } }else @@ -32304,13 +35417,13 @@ static int do_meta_command(char *zLine, ShellState *p){ mType |= SQLITE_TRACE_CLOSE; } else { - sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); + cli_printf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); rc = 1; goto meta_command_exit; } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(z); + p->traceOut = output_file_open(p, z); } } if( p->traceOut==0 ){ @@ -32349,21 +35462,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; - sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, + cli_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB - sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion()); + cli_printf(p->out, "zlib version %s\n", zlibVersion()); #endif #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) #if defined(__clang__) && defined(__clang_major__) - sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." + cli_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." CTIMEOPT_VAL(__clang_minor__) "." CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); + cli_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); + cli_printf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else @@ -32373,10 +35486,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ - sqlite3_fprintf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); - sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + cli_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); + cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else @@ -32388,13 +35501,13 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ - sqlite3_fprintf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + cli_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, pVfs==pCurrent ? " <--- CURRENT" : ""); - sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ - sqlite3_fputs("-----------------------------------\n", p->out); + cli_puts("-----------------------------------\n", p->out); } } }else @@ -32405,7 +35518,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - sqlite3_fprintf(p->out, "%s\n", zVfsName); + cli_printf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } @@ -32418,28 +35531,31 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; - assert( nArg<=ArraySize(azArg) ); - p->nWidth = nArg-1; - p->colWidth = realloc(p->colWidth, (p->nWidth+1)*sizeof(int)*2); - if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory(); - if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth]; + p->mode.spec.nWidth = nArg-1; + p->mode.spec.aWidth = realloc(p->mode.spec.aWidth, + (p->mode.spec.nWidth+1)*sizeof(short int)); + shell_check_oom(p->mode.spec.aWidth); for(j=1; j<nArg; j++){ - p->colWidth[j-1] = (int)integerValue(azArg[j]); + i64 w = integerValue(azArg[j]); + if( w < -QRF_MAX_WIDTH ) w = -QRF_MAX_WIDTH; + if( w > QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.aWidth[j-1] = (short int)w; } }else { - sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: " + cli_printf(stderr,"Error: unknown command or invalid arguments: " " \"%s\". Enter \".help\" for help\n", azArg[0]); rc = 1; } meta_command_exit: - if( p->outCount ){ - p->outCount--; - if( p->outCount==0 ) output_reset(p); + if( p->nPopOutput ){ + p->nPopOutput--; + if( p->nPopOutput==0 ) output_reset(p); } p->bSafeMode = p->bSafeModePersist; + p->dot.nArg = 0; return rc; } @@ -32669,18 +35785,25 @@ static int doAutoDetectRestore(ShellState *p, const char *zSql){ /* ** Run a single line of SQL. Return the number of errors. */ -static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ +static int runOneSqlLine( + ShellState *p, /* Execution context */ + char *zSql, /* SQL to be run */ + const char *zFilename, /* Source file of the sql */ + int startline /* linenumber */ +){ int rc; char *zErrMsg = 0; open_db(p, 0); if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; - BEGIN_TIMER; + BEGIN_TIMER(p); rc = shell_exec(p, zSql, &zErrMsg); - END_TIMER(p->out); + END_TIMER(p); if( rc || zErrMsg ){ - char zPrefix[100]; + char zPrefix[2048 + /* must be relatively large: + ** https://sqlite.org/forum/forumpost/205f73db1b2806f5 */]; const char *zErrorTail; const char *zErrorType; if( zErrMsg==0 ){ @@ -32696,13 +35819,21 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ zErrorType = "Error"; zErrorTail = zErrMsg; } - if( in!=0 || !stdin_is_interactive ){ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, - "%s near line %d:", zErrorType, startline); + if( zFilename || !stdin_is_interactive ){ + if( cli_strcmp(zFilename,"cmdline")==0 ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s in %r command line argument:", zErrorType, startline); + }else if( cli_strcmp(zFilename,"<stdin>")==0 ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d:", zErrorType, startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d of %s:", zErrorType, startline, zFilename); + } }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } - sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail); + cli_printf(stderr,"%s %s\n", zPrefix, zErrorTail); sqlite3_free(zErrMsg); zErrMsg = 0; return 1; @@ -32711,7 +35842,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ sqlite3_snprintf(sizeof(zLineBuf), zLineBuf, "changes: %lld total_changes: %lld", sqlite3_changes64(p->db), sqlite3_total_changes64(p->db)); - sqlite3_fprintf(p->out, "%s\n", zLineBuf); + cli_printf(p->out, "%s\n", zLineBuf); } if( doAutoDetectRestore(p, zSql) ) return 1; @@ -32719,8 +35850,8 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ } static void echo_group_input(ShellState *p, const char *zDo){ - if( ShellHasFlag(p, SHFLG_Echo) ){ - sqlite3_fprintf(p->out, "%s\n", zDo); + if( p->mode.mFlags & MFLG_ECHO ){ + cli_printf(p->out, "%s\n", zDo); fflush(p->out); } } @@ -32731,12 +35862,13 @@ static void echo_group_input(ShellState *p, const char *zDo){ ** impl because we need the global shellState and cannot access it from that ** function without moving lots of code around (creating a larger/messier diff). */ -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ +static char *one_input_line(ShellState *p, char *zPrior, int isContinuation){ /* Parse the next line from shellState.wasm.zInput. */ const char *zBegin = shellState.wasm.zPos; const char *z = zBegin; char *zLine = 0; i64 nZ = 0; + FILE *in = p->in; UNUSED_PARAMETER(in); UNUSED_PARAMETER(isContinuation); @@ -32767,7 +35899,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ ** ** Return the number of errors. */ -static int process_input(ShellState *p){ +static int process_input(ShellState *p, const char *zSrc){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ i64 nLine; /* Length of current line */ @@ -32777,22 +35909,27 @@ static int process_input(ShellState *p){ int errCnt = 0; /* Number of errors seen */ i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ + const char *saved_zInFile; /* Prior value of p->zInFile */ + i64 saved_lineno; /* Prior value of p->lineno */ if( p->inputNesting==MAX_INPUT_NESTING ){ /* This will be more informative in a later version. */ - sqlite3_fprintf(stderr,"Input nesting limit (%d) reached at line %d." - " Check recursion.\n", MAX_INPUT_NESTING, p->lineno); + cli_printf(stderr,"%s: Input nesting limit (%d) reached at line %lld." + " Check recursion.\n", zSrc, MAX_INPUT_NESTING, p->lineno); return 1; } ++p->inputNesting; + saved_zInFile = p->zInFile; + p->zInFile = zSrc; + saved_lineno = p->lineno; p->lineno = 0; CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); - zLine = one_input_line(p->in, zLine, nSql>0); + zLine = one_input_line(p, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out); + if( p->in==0 && stdin_is_interactive ) cli_puts("\n", p->out); break; } if( seenInterrupt ){ @@ -32846,17 +35983,29 @@ static int process_input(ShellState *p){ memcpy(zSql+nSql, zLine, nLine+1); nSql += nLine; } - if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ + if( nSql>0x7fff0000 ){ + char zSize[100]; + sqlite3_snprintf(sizeof(zSize),zSize,"%,lld",nSql); + cli_printf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", + zSrc, startline, zSize); + nSql = 0; + errCnt++; + break; + }else if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); - errCnt += runOneSqlLine(p, zSql, p->in, startline); + errCnt += runOneSqlLine(p, zSql, p->zInFile, startline); CONTINUE_PROMPT_RESET; nSql = 0; - if( p->outCount ){ + if( p->nPopOutput ){ output_reset(p); - p->outCount = 0; + p->nPopOutput = 0; }else{ clearTempFile(p); } + if( p->nPopMode ){ + modePop(p); + p->nPopMode = 0; + } p->bSafeMode = p->bSafeModePersist; qss = QSS_Start; }else if( nSql && QSS_PLAINWHITE(qss) ){ @@ -32868,12 +36017,14 @@ static int process_input(ShellState *p){ if( nSql ){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); - errCnt += runOneSqlLine(p, zSql, p->in, startline); + errCnt += runOneSqlLine(p, zSql, p->zInFile, startline); CONTINUE_PROMPT_RESET; } free(zSql); free(zLine); --p->inputNesting; + p->zInFile = saved_zInFile; + p->lineno = saved_lineno; return errCnt>0; } @@ -32947,59 +36098,79 @@ static char *find_home_dir(int clearFlag){ } /* -** On non-Windows platforms, look for $XDG_CONFIG_HOME. -** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return -** the path to it. If there is no $(XDG_CONFIG_HOME) then -** look for $(HOME)/.config/sqlite3/sqliterc and if found -** return that. If none of these are found, return 0. +** On non-Windows platforms, look for: +** +** - ${zEnvVar}/${zBaseName} +** - ${HOME}/${zSubdir}/${zBaseName} +** +** $zEnvVar is intended to be the name of an XDG_... environment +** variable, e.g. XDG_CONFIG_HOME or XDG_STATE_HOME. If zEnvVar is +** NULL or getenv(zEnvVar) is NULL then fall back to the second +** option. If the selected option is not found in the filesystem, +** return 0. +** +** zSubdir may be NULL or empty, in which case ${HOME}/${zBaseName} +** becomes the fallback. ** -** The string returned is obtained from sqlite3_malloc() and -** should be freed by the caller. +** Both zSubdir and zBaseName may contain subdirectory parts. zSubdir +** will conventionally be ".config" or ".local/state", which, not +** coincidentally, is the typical subdir of the corresponding XDG_... +** var with the XDG var's $HOME prefix. +** +** The returned string is obtained from sqlite3_malloc() and should be +** sqlite3_free()'d by the caller. */ -static char *find_xdg_config(void){ +static char *find_xdg_file(const char *zEnvVar, const char *zSubdir, + const char *zBaseName){ #if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \ || defined(__RTP__) || defined(_WRS_KERNEL) return 0; #else - char *zConfig = 0; - const char *zXdgHome; + char *zConfigFile = 0; + const char *zXdgDir; - zXdgHome = getenv("XDG_CONFIG_HOME"); - if( zXdgHome==0 ){ - const char *zHome = getenv("HOME"); - if( zHome==0 ) return 0; - zConfig = sqlite3_mprintf("%s/.config/sqlite3/sqliterc", zHome); + zXdgDir = zEnvVar ? getenv(zEnvVar) : 0; + if( zXdgDir ){ + zConfigFile = sqlite3_mprintf("%s/%s", zXdgDir, zBaseName); }else{ - zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); + const char * zHome = find_home_dir(0); + if( zHome==0 ) return 0; + zConfigFile = (zSubdir && *zSubdir) + ? sqlite3_mprintf("%s/%s/%s", zHome, zSubdir, zBaseName) + : sqlite3_mprintf("%s/%s", zHome, zBaseName); } - shell_check_oom(zConfig); - if( access(zConfig,0)!=0 ){ - sqlite3_free(zConfig); - zConfig = 0; + shell_check_oom(zConfigFile); + if( access(zConfigFile,0)!=0 ){ + sqlite3_free(zConfigFile); + zConfigFile = 0; } - return zConfig; + return zConfigFile; #endif } /* -** Read input from the file given by sqliterc_override. Or if that -** parameter is NULL, take input from the first of find_xdg_config() -** or ~/.sqliterc which is found. +** Read input from the file sqliterc_override. If that parameter is +** NULL, take it from find_xdg_file(), if found, or fall back to +** ~/.sqliterc. ** -** Returns the number of errors. +** Failure to read the config is only considered a failure if +** sqliterc_override is not NULL, in which case this function may emit +** a warning or, if ::bail_on_error is true, fail fatally if the file +** named by sqliterc_override is not found. */ static void process_sqliterc( ShellState *p, /* Configuration data */ const char *sqliterc_override /* Name of config file. NULL to use default */ ){ char *home_dir = NULL; - const char *sqliterc = sqliterc_override; - char *zBuf = 0; + char *sqliterc = (char*)sqliterc_override; FILE *inSaved = p->in; - int savedLineno = p->lineno; + i64 savedLineno = p->lineno; if( sqliterc == NULL ){ - sqliterc = zBuf = find_xdg_config(); + sqliterc = find_xdg_file("XDG_CONFIG_HOME", + ".config", + "sqlite3/sqliterc"); } if( sqliterc == NULL ){ home_dir = find_home_dir(0); @@ -33008,24 +36179,25 @@ static void process_sqliterc( " cannot read ~/.sqliterc\n"); return; } - zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); - shell_check_oom(zBuf); - sqliterc = zBuf; + sqliterc = sqlite3_mprintf("%s/.sqliterc",home_dir); + shell_check_oom(sqliterc); } - p->in = sqlite3_fopen(sqliterc,"rb"); + p->in = sqliterc ? sqlite3_fopen(sqliterc,"rb") : 0; if( p->in ){ if( stdin_is_interactive ){ - sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc); + cli_printf(stderr,"-- Loading resources from %s\n", sqliterc); } - if( process_input(p) && bail_on_error ) exit(1); + if( process_input(p, sqliterc) && bail_on_error ) cli_exit(1); fclose(p->in); }else if( sqliterc_override!=0 ){ - sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc); - if( bail_on_error ) exit(1); + cli_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + if( bail_on_error ) cli_exit(1); } p->in = inSaved; p->lineno = savedLineno; - sqlite3_free(zBuf); + if( sqliterc != sqliterc_override ){ + sqlite3_free(sqliterc); + } } /* @@ -33041,8 +36213,8 @@ static const char zOptions[] = " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -box set output mode to 'box'\n" - " -column set output mode to 'column'\n" " -cmd COMMAND run \"COMMAND\" before reading stdin\n" + " -column set output mode to 'column'\n" " -csv set output mode to 'csv'\n" #if !defined(SQLITE_OMIT_DESERIALIZE) " -deserialize open the database using sqlite3_deserialize()\n" @@ -33056,6 +36228,7 @@ static const char zOptions[] = #endif " -help show this message\n" " -html set output mode to HTML\n" + " -ifexists only open if database already exists\n" " -interactive force interactive I/O\n" " -json set output mode to 'json'\n" " -line set output mode to 'line'\n" @@ -33072,6 +36245,7 @@ static const char zOptions[] = #endif " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" + " -noinit Do not read the ~/.sqliterc file at startup\n" " -nonce STRING set the safe-mode escape nonce\n" " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" @@ -33080,6 +36254,7 @@ static const char zOptions[] = " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -safe enable safe-mode\n" + " -screenwidth N use N as the default screenwidth \n" " -separator SEP set output column separator. Default: '|'\n" #ifdef SQLITE_ENABLE_SORTER_REFERENCES " -sorterref SIZE sorter references threshold size\n" @@ -33096,11 +36271,11 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" + cli_printf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ - sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions); + cli_printf(stderr,"OPTIONS include:\n%s", zOptions); }else{ eputz("Use the -help option for additional information\n"); } @@ -33121,19 +36296,11 @@ static void verify_uninitialized(void){ /* ** Initialize the state information in data */ -static void main_init(ShellState *data) { - memset(data, 0, sizeof(*data)); - data->normalMode = data->cMode = data->mode = MODE_List; - data->autoExplain = 1; -#ifdef _WIN32 - data->crlfMode = 1; -#endif - data->pAuxDb = &data->aAuxDb[0]; - memcpy(data->colSeparator,SEP_Column, 2); - memcpy(data->rowSeparator,SEP_Row, 2); - data->showHeader = 0; - data->shellFlgs = SHFLG_Lookaside; - sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); +static void main_init(ShellState *p) { + memset(p, 0, sizeof(*p)); + p->pAuxDb = &p->aAuxDb[0]; + p->shellFlgs = SHFLG_Lookaside; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, p); #if !defined(SQLITE_SHELL_FIDDLE) verify_uninitialized(); #endif @@ -33148,22 +36315,18 @@ static void main_init(ShellState *data) { */ #if defined(_WIN32) || defined(WIN32) static void printBold(const char *zText){ -#if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; GetConsoleScreenBufferInfo(out, &defaultScreenInfo); SetConsoleTextAttribute(out, FOREGROUND_RED|FOREGROUND_INTENSITY ); -#endif sputz(stdout, zText); -#if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); -#endif } #else static void printBold(const char *zText){ - sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText); + cli_printf(stdout, "\033[1m%s\033[0m", zText); } #endif @@ -33173,9 +36336,9 @@ static void printBold(const char *zText){ */ static char *cmdline_option_value(int argc, char **argv, int i){ if( i==argc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]); - exit(1); + cli_exit(1); } return argv[i]; } @@ -33188,34 +36351,85 @@ static void sayAbnormalExit(void){ */ static int vfstraceOut(const char *z, void *pArg){ ShellState *p = (ShellState*)pArg; - sqlite3_fputs(z, p->out); + cli_puts(z, p->out); fflush(p->out); return 1; } -#ifndef SQLITE_SHELL_IS_UTF8 -# if (defined(_WIN32) || defined(WIN32)) \ - && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) -# define SQLITE_SHELL_IS_UTF8 (0) -# else -# define SQLITE_SHELL_IS_UTF8 (1) -# endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_SHELL_FIDDLE) +/* Ensure that sqlite3_reset_auto_extension() clears auto-extension +** memory. https://sqlite.org/forum/forumpost/310cb231b07c80d1. +** Testing this: if we (inadvertently) remove the +** sqlite3_reset_auto_extension() call from main(), most shell tests +** will fail because of a leak message in their output. */ +static int auto_ext_leak_tester( + sqlite3 *db, + char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + (void)db; (void)pzErrMsg; (void)pThunk; + return SQLITE_OK; +} #endif +/* Alternative name to the entry point for Fiddle */ #ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main #endif -#if SQLITE_SHELL_IS_UTF8 -int SQLITE_CDECL main(int argc, char **argv){ -#else +/* Use the wmain() entry point on Windows. Translate arguments to +** UTF8, then invoke the traditional main() entry point which is +** renamed using a #define to utf8_main() . +*/ +#if defined(_WIN32) && !defined(__MINGW32__) && !defined(main) +# define main utf8_main /* Rename entry point to utf_main() */ +int SQLITE_CDECL utf8_main(int,char**); /* Forward declaration */ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ - char **argv; -#endif + int rc, i; + char **argv = malloc( sizeof(char*) * (argc+1) ); + char **orig = argv; + if( argv==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + for(i=0; i<argc; i++){ + int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); + if( nByte==0 ){ + argv[i] = 0; + }else{ + argv[i] = malloc( nByte ); + if( argv[i]==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); + if( nByte==0 ){ + free(argv[i]); + argv[i] = 0; + } + } + } + argv[argc] = 0; + rc = utf8_main(argc, argv); + for(i=0; i<argc; i++) free(orig[i]); + free(argv); + return rc; +} +#endif /* WIN32 */ + +/* +** This is the main entry point for the process. Everything starts here. +** +** The "main" identifier may have been #defined to something else: +** +** utf8_main On Windows +** fiddle_main In Fiddle +** sqlite3_shell Other projects that use shell.c as a subroutine +*/ +int SQLITE_CDECL main(int argc, char **argv){ #ifdef SQLITE_DEBUG sqlite3_int64 mem_main_enter = 0; #endif - char *zErrMsg = 0; #ifdef SQLITE_SHELL_FIDDLE # define data shellState #else @@ -33226,15 +36440,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int rc = 0; int warnInmemoryDb = 0; int readStdin = 1; + int noInit = 0; /* Do not read ~/.sqliterc if true */ int nCmd = 0; int nOptsEnd = argc; int bEnableVfstrace = 0; char **azCmd = 0; + int *aiCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ -#if !SQLITE_SHELL_IS_UTF8 - char **argvToFree = 0; - int argcToFree = 0; -#endif setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ #ifdef SQLITE_SHELL_FIDDLE @@ -33253,7 +36465,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ char zLine[100]; - sqlite3_fprintf(stderr, + cli_printf(stderr, "attach debugger to process %d and press ENTER to continue...", GETPID()); if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0 @@ -33263,11 +36475,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } }else{ #if defined(_WIN32) || defined(WIN32) -#if SQLITE_OS_WINRT - __debugbreak(); -#else DebugBreak(); -#endif #elif defined(SIGTRAP) raise(SIGTRAP); #endif @@ -33285,7 +36493,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); @@ -33293,32 +36501,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif main_init(&data); - /* On Windows, we must translate command-line arguments into UTF-8. - ** The SQLite memory allocator subsystem has to be enabled in order to - ** do this. But we want to run an sqlite3_shutdown() afterwards so that - ** subsequent sqlite3_config() calls will work. So copy all results into - ** memory that does not come from the SQLite memory allocator. - */ -#if !SQLITE_SHELL_IS_UTF8 - sqlite3_initialize(); - argvToFree = malloc(sizeof(argv[0])*argc*2); - shell_check_oom(argvToFree); - argcToFree = argc; - argv = argvToFree + argc; - for(i=0; i<argc; i++){ - char *z = sqlite3_win32_unicode_to_utf8(wargv[i]); - i64 n; - shell_check_oom(z); - n = strlen(z); - argv[i] = malloc( n+1 ); - shell_check_oom(argv[i]); - memcpy(argv[i], z, n+1); - argvToFree[i] = argv[i]; - sqlite3_free(z); - } - sqlite3_shutdown(); -#endif - assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; @@ -33334,7 +36516,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif - /* Do an initial pass through the command-line argument to locate + /* Do an initial pass through the command-line arguments to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, options affecting commands ** or SQL run from the command line, and the first command to execute. @@ -33346,16 +36528,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *z; z = argv[i]; if( z[0]!='-' || i>nOptsEnd ){ - if( data.aAuxDb->zDbFilename==0 ){ + if( data.aAuxDb->zDbFilename==0 && !isScriptFile(z,1) ){ data.aAuxDb->zDbFilename = z; }else{ /* Excess arguments are interpreted as SQL (or dot-commands) and ** mean that nothing is read from stdin */ readStdin = 0; + stdin_is_interactive = 0; nCmd++; azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); shell_check_oom(azCmd); + aiCmd = realloc(aiCmd, sizeof(aiCmd[0])*nCmd); + shell_check_oom(azCmd); azCmd[nCmd-1] = z; + aiCmd[nCmd-1] = i; } continue; } @@ -33378,6 +36564,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; + stdout_is_console = 0; + modeChange(&data, MODE_BATCH); + }else if( cli_strcmp(z,"-screenwidth")==0 ){ + int n = atoi(cmdline_option_value(argc, argv, ++i)); + if( n<2 ){ + sqlite3_fprintf(stderr,"minimum --screenwidth is 2\n"); + exit(1); + } + stdout_tty_width = n; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ @@ -33400,12 +36595,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-pagecache")==0 ){ sqlite3_int64 n, sz; sz = integerValue(cmdline_option_value(argc,argv,++i)); - if( sz>70000 ) sz = 70000; + if( sz>65536 ) sz = 65536; if( sz<0 ) sz = 0; n = integerValue(cmdline_option_value(argc,argv,++i)); if( sz>0 && n>0 && 0xffffffffffffLL/sz<n ){ n = 0xffffffffffffLL/sz; } + if( sz>0 && (sz & (sz-1))==0 ){ + /* If SIZE is a power of two, round it up by the PCACHE_HDRSZ */ + int szHdr = 0; + sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &szHdr); + sz += szHdr; + cli_printf(stdout, "Page cache size increased to %d to accommodate" + " the %d-byte headers\n", (int)sz, szHdr); + } verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_PAGECACHE, (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n); @@ -33418,7 +36621,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( n<0 ) n = 0; verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); - if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; + if( (i64)sz*(i64)n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; }else if( cli_strcmp(z,"-threadsafe")==0 ){ int n; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); @@ -33460,9 +36663,17 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.szMax = integerValue(argv[++i]); #endif }else if( cli_strcmp(z,"-readonly")==0 ){ - data.openMode = SHELL_OPEN_READONLY; + data.openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ - data.openFlags = SQLITE_OPEN_NOFOLLOW; + data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( cli_strcmp(z,"-noinit")==0 ){ + noInit = 1; + }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ + data.openFlags |= SQLITE_OPEN_EXCLUSIVE; + }else if( cli_strcmp(z,"-ifexists")==0 ){ + data.openFlags &= ~(SQLITE_OPEN_CREATE); + if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE; #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( cli_strncmp(z, "-A",2)==0 ){ /* All remaining command-line arguments are passed to the ".archive" @@ -33485,6 +36696,16 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* skip over the argument */ i++; + }else if( cli_strcmp(z,"-test-argv")==0 ){ + /* Undocumented test option. Print the values in argv[] and exit. + ** Use this to verify that any translation of the argv[], for example + ** on Windows that receives wargv[] from the OS and must convert + ** to UTF8 prior to calling this routine. */ + int kk; + for(kk=0; kk<argc; kk++){ + sqlite3_fprintf(stdout,"argv[%d] = \"%s\"\n", kk, argv[kk]); + } + return 0; } } #ifndef SQLITE_SHELL_FIDDLE @@ -33511,8 +36732,27 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); - }else{ - sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs); + } +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + else if( access(zVfs,0)==0 ){ + /* If the VFS name is not the name of an existing VFS, but it is + ** the name of a file, then try to load that file as an extension. + ** Presumably the extension implements the desired VFS. */ + sqlite3 *db = 0; + char *zErr = 0; + sqlite3_open(":memory:", &db); + sqlite3_enable_load_extension(db, 1); + rc = sqlite3_load_extension(db, zVfs, 0, &zErr); + sqlite3_close(db); + if( (rc&0xff)!=SQLITE_OK ){ + cli_printf(stderr, "could not load extension VFS \"%s\": %s\n", + zVfs, zErr); + exit(1); + } + } +#endif + else{ + cli_printf(stderr,"no such VFS: \"%s\"\n", zVfs); exit(1); } } @@ -33522,9 +36762,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.pAuxDb->zDbFilename = ":memory:"; warnInmemoryDb = argc==1; #else - sqlite3_fprintf(stderr, + cli_printf(stderr, "%s: Error: no database filename specified\n", Argv0); - return 1; + rc = 1; + goto shell_main_exit; #endif } data.out = stdout; @@ -33533,7 +36774,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); +#ifdef SQLITE_DEBUG + sqlite3_auto_extension( (void (*)(void))auto_ext_leak_tester ); +#endif #endif + modeDefault(&data); /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -33548,9 +36793,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** is given on the command line, look for a file named ~/.sqliterc and ** try to process it. */ - process_sqliterc(&data,zInitFile); + if( !noInit ) process_sqliterc(&data,zInitFile); - /* Make a second pass through the command-line argument and set + /* Make a second pass through the command-line arguments and set ** options. This second pass is delayed until after the initialization ** file is processed so that the command-line arguments will override ** settings in the initialization file. @@ -33562,45 +36807,44 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( cli_strcmp(z,"-init")==0 ){ i++; }else if( cli_strcmp(z,"-html")==0 ){ - data.mode = MODE_Html; + modeChange(&data, MODE_Html); }else if( cli_strcmp(z,"-list")==0 ){ - data.mode = MODE_List; + modeChange(&data, MODE_List); }else if( cli_strcmp(z,"-quote")==0 ){ - data.mode = MODE_Quote; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row); + modeChange(&data, MODE_Quote); }else if( cli_strcmp(z,"-line")==0 ){ - data.mode = MODE_Line; + modeChange(&data, MODE_Line); }else if( cli_strcmp(z,"-column")==0 ){ - data.mode = MODE_Column; + modeChange(&data, MODE_Column); }else if( cli_strcmp(z,"-json")==0 ){ - data.mode = MODE_Json; + modeChange(&data, MODE_Json); }else if( cli_strcmp(z,"-markdown")==0 ){ - data.mode = MODE_Markdown; + modeChange(&data, MODE_Markdown); }else if( cli_strcmp(z,"-table")==0 ){ - data.mode = MODE_Table; + modeChange(&data, MODE_Table); + }else if( cli_strcmp(z,"-psql")==0 ){ + modeChange(&data, MODE_Psql); }else if( cli_strcmp(z,"-box")==0 ){ - data.mode = MODE_Box; + modeChange(&data, MODE_Box); }else if( cli_strcmp(z,"-csv")==0 ){ - data.mode = MODE_Csv; - memcpy(data.colSeparator,",",2); + modeChange(&data, MODE_Csv); }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* See similar code at tag-20250224-1 */ const char *zEsc = argv[++i]; int k; - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ - data.eEscMode = k; + for(k=0; k<ArraySize(qrfEscNames); k++){ + if( sqlite3_stricmp(zEsc,qrfEscNames[k])==0 ){ + data.mode.spec.eEsc = k; break; } } - if( k>=ArraySize(shell_EscModeNames) ){ - sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" + if( k>=ArraySize(qrfEscNames) ){ + cli_printf(stderr, "unknown control character escape mode \"%s\"" " - choices:", zEsc); - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); + for(k=0; k<ArraySize(qrfEscNames); k++){ + cli_printf(stderr, " %s", qrfEscNames[k]); } - sqlite3_fprintf(stderr, "\n"); + cli_printf(stderr, "\n"); exit(1); } #ifdef SQLITE_HAVE_ZLIB @@ -33616,42 +36860,44 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.szMax = integerValue(argv[++i]); #endif }else if( cli_strcmp(z,"-readonly")==0 ){ - data.openMode = SHELL_OPEN_READONLY; + data.openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( cli_strcmp(z,"-noinit")==0 ){ + /* No-op */ + }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ + data.openFlags |= SQLITE_OPEN_EXCLUSIVE; + }else if( cli_strcmp(z,"-ifexists")==0 ){ + data.openFlags &= ~(SQLITE_OPEN_CREATE); + if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE; }else if( cli_strcmp(z,"-ascii")==0 ){ - data.mode = MODE_Ascii; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Record); + modeChange(&data, MODE_Ascii); }else if( cli_strcmp(z,"-tabs")==0 ){ - data.mode = MODE_List; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Tab); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Row); + modeChange(&data, MODE_Tabs); }else if( cli_strcmp(z,"-separator")==0 ){ - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zColumnSep, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-newline")==0 ){ - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zRowSep, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-nullvalue")==0 ){ - sqlite3_snprintf(sizeof(data.nullValue), data.nullValue, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zNull, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-header")==0 ){ - data.showHeader = 1; - ShellSetFlag(&data, SHFLG_HeaderSet); + data.mode.spec.bTitles = QRF_Yes; }else if( cli_strcmp(z,"-noheader")==0 ){ - data.showHeader = 0; - ShellSetFlag(&data, SHFLG_HeaderSet); + data.mode.spec.bTitles = QRF_No; }else if( cli_strcmp(z,"-echo")==0 ){ - ShellSetFlag(&data, SHFLG_Echo); + data.mode.mFlags |= MFLG_ECHO; }else if( cli_strcmp(z,"-eqp")==0 ){ - data.autoEQP = AUTOEQP_on; + data.mode.autoEQP = AUTOEQP_on; }else if( cli_strcmp(z,"-eqpfull")==0 ){ - data.autoEQP = AUTOEQP_full; + data.mode.autoEQP = AUTOEQP_full; }else if( cli_strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( cli_strcmp(z,"-scanstats")==0 ){ - data.scanstatsOn = 1; + data.mode.scanstatsOn = 1; }else if( cli_strcmp(z,"-backslash")==0 ){ /* Undocumented command-line option: -backslash ** Causes C-style backslash escapes to be evaluated in SQL statements @@ -33662,9 +36908,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-bail")==0 ){ /* No-op. The bail_on_error flag should already be set. */ }else if( cli_strcmp(z,"-version")==0 ){ - sqlite3_fprintf(stdout, "%s %s (%d-bit)\n", + cli_printf(stdout, "%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*)); - return 0; + rc = 0; + goto shell_main_exit; }else if( cli_strcmp(z,"-interactive")==0 ){ /* Need to check for interactive override here to so that it can ** affect console setup (for Windows only) and testing thereof. @@ -33672,6 +36919,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* already handled */ + }else if( cli_strcmp(z,"-screenwidth")==0 ){ + i++; }else if( cli_strcmp(z,"-utf8")==0 ){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ @@ -33717,24 +36966,21 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc==2 ? 0 : rc; - }else{ - open_db(&data, 0); - rc = shell_exec(&data, z, &zErrMsg); - if( zErrMsg!=0 ){ - shellEmitError(zErrMsg); - if( bail_on_error ) return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); - if( bail_on_error ) return rc; + if( rc && (bail_on_error || rc==2) ){ + if( rc==2 ) rc = 0; + goto shell_main_exit; } + }else{ + rc = runOneSqlLine(&data, z, "cmdline", i); + if( bail_on_error ) goto shell_main_exit; } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( cli_strncmp(z, "-A", 2)==0 ){ if( nCmd>0 ){ - sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands" + cli_printf(stderr,"Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); - return 1; + rc = 1; + goto shell_main_exit; } open_db(&data, OPEN_DB_ZIPFILE); if( z[2] ){ @@ -33744,6 +36990,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ arDotCommand(&data, 1, argv+i, argc-i); } readStdin = 0; + stdin_is_interactive = 0; break; #endif }else if( cli_strcmp(z,"-safe")==0 ){ @@ -33751,11 +36998,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ /* Acted upon in first pass. */ }else{ - sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); + cli_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); eputz("Use -help for a list of options.\n"); - return 1; + rc = 1; + goto shell_main_exit; } - data.cMode = data.mode; } if( !readStdin ){ @@ -33765,26 +37012,46 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ for(i=0; i<nCmd; i++){ echo_group_input(&data, azCmd[i]); - if( azCmd[i][0]=='.' ){ + if( isScriptFile(azCmd[i],0) ){ + FILE *inSaved = data.in; + i64 savedLineno = data.lineno; + int res = 1; + if( (data.in = openChrSource(azCmd[i]))!=0 ){ + res = process_input(&data, azCmd[i]); + fclose(data.in); + } + data.in = inSaved; + data.lineno = savedLineno; + if( res ) i = nCmd; + }else if( azCmd[i][0]=='.' ){ + char *zErrCtx = malloc( 64 ); + shell_check_oom(zErrCtx); + sqlite3_snprintf(64,zErrCtx,"argv[%i]:",aiCmd[i]); + data.zInFile = "<cmdline>"; + data.zErrPrefix = zErrCtx; rc = do_meta_command(azCmd[i], &data); + free(data.zErrPrefix); + data.zErrPrefix = 0; if( rc ){ if( rc==2 ) rc = 0; goto shell_main_exit; } }else{ - open_db(&data, 0); - rc = shell_exec(&data, azCmd[i], &zErrMsg); - if( zErrMsg || rc ){ - if( zErrMsg!=0 ){ - shellEmitError(zErrMsg); - }else{ - sqlite3_fprintf(stderr, - "Error: unable to process SQL: %s\n", azCmd[i]); - } - sqlite3_free(zErrMsg); - if( rc==0 ) rc = 1; + rc = runOneSqlLine(&data, azCmd[i], "cmdline", aiCmd[i]); + if( data.nPopMode ){ + modePop(&data); + data.nPopMode = 0; + } + if( rc ){ goto shell_main_exit; } + + } + if( data.nPopOutput && azCmd[i][0]!='.' ){ + output_reset(&data); + data.nPopOutput = 0; + }else{ + clearTempFile(&data); } } }else{ @@ -33793,8 +37060,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; - int nHistory; - sqlite3_fprintf(stdout, + cli_printf(stdout, "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", sqlite3_libversion(), sqlite3_sourceid()); @@ -33806,11 +37072,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } zHistory = getenv("SQLITE_HISTORY"); if( zHistory ){ - zHistory = strdup(zHistory); - }else if( (zHome = find_home_dir(0))!=0 ){ - nHistory = strlen30(zHome) + 20; - if( (zHistory = malloc(nHistory))!=0 ){ - sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome); + zHistory = sqlite3_mprintf("%s", zHistory); + shell_check_oom(zHistory); + }else{ + zHistory = find_xdg_file("XDG_STATE_HOME", + ".local/state", + "sqlite_history"); + if( 0==zHistory && (zHome = find_home_dir(0))!=0 ){ + zHistory = sqlite3_mprintf("%s/.sqlite_history", zHome); + shell_check_oom(zHistory); } } if( zHistory ){ shell_read_history(zHistory); } @@ -33822,27 +37092,28 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ linenoiseSetCompletionCallback(linenoise_completion, NULL); #endif data.in = 0; - rc = process_input(&data); + rc = process_input(&data, "<stdin>"); if( zHistory ){ shell_stifle_history(2000); shell_write_history(zHistory); - free(zHistory); + sqlite3_free(zHistory); } }else{ data.in = stdin; - rc = process_input(&data); + rc = process_input(&data, "<stdin>"); } } #ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ -#ifndef SQLITE_OMIT_VIRTUALTABLE +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( data.expert.pExpert ){ expertFinish(&data, 1, 0); } #endif shell_main_exit: free(azCmd); + free(aiCmd); set_table_name(&data, 0); if( data.db ){ session_close_all(&data, -1); @@ -33859,21 +37130,38 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ output_reset(&data); data.doXdgOpen = 0; clearTempFile(&data); -#if !SQLITE_SHELL_IS_UTF8 - for(i=0; i<argcToFree; i++) free(argvToFree[i]); - free(argvToFree); -#endif - free(data.colWidth); + modeFree(&data.mode); + if( data.nSavedModes ){ + int ii; + for(ii=0; ii<data.nSavedModes; ii++){ + modeFree(&data.aSavedModes[ii].mode); + free(data.aSavedModes[ii].zTag); + } + free(data.aSavedModes); + } + free(data.zErrPrefix); free(data.zNonce); + free(data.dot.zCopy); + free(data.dot.azArg); + free(data.dot.aiOfst); + free(data.dot.abQuot); + if( data.nTestRun ){ + sqlite3_fprintf(stdout, "%d test%s run with %d error%s\n", + data.nTestRun, data.nTestRun==1 ? "" : "s", + data.nTestErr, data.nTestErr==1 ? "" : "s"); + fflush(stdout); + rc = data.nTestErr>0; + } /* Clear the global data structure so that valgrind will detect memory ** leaks */ memset(&data, 0, sizeof(data)); if( bEnableVfstrace ){ vfstrace_unregister("trace"); } + sqlite3_reset_auto_extension(); #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ - sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n", + cli_printf(stderr,"Memory leaked: %u bytes\n", (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif @@ -33913,7 +37201,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); + cli_printf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -33950,7 +37238,7 @@ void fiddle_reset_db(void){ ** Resolve problem reported in ** https://sqlite.org/forum/forumpost/0b41a25d65 */ - sqlite3_fputs("Rolling back in-progress transaction.\n", stdout); + cli_puts("Rolling back in-progress transaction.\n", stdout); sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); } rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); @@ -34012,8 +37300,9 @@ void fiddle_exec(const char * zSql){ if('.'==*zSql) puts(zSql); shellState.wasm.zInput = zSql; shellState.wasm.zPos = zSql; - process_input(&shellState); + process_input(&shellState, "<stdin>"); shellState.wasm.zInput = shellState.wasm.zPos = 0; } } #endif /* SQLITE_SHELL_FIDDLE */ +/************************* End src/shell.c.in ******************/ diff --git a/contrib/sqlite3/sqlite3.1 b/contrib/sqlite3/sqlite3.1 index 08b1ff262b66..e4ba5a0cb88d 100644 --- a/contrib/sqlite3/sqlite3.1 +++ b/contrib/sqlite3/sqlite3.1 @@ -137,17 +137,37 @@ continue prompt = " ...> " .sp .fi -o If the file +o If the environment variable XDG_CONFIG_HOME is set then .B ${XDG_CONFIG_HOME}/sqlite3/sqliterc -or +is checked, else +.B ~/.config/sqlite3/sqliterc +is checked. If the selected file does not exist then the fallback of .B ~/.sqliterc -exists, the first of those to be found is processed during startup. -It should generally only contain meta-commands. +is used. It should generally only contain meta-commands. o If the -init option is present, the specified file is processed. o All other command line options are processed. +.SH HISTORY FILE +.B sqlite3 +may be configured to use a history file to save SQL statements and +meta-commands entered interactively. These statements and commands can be +retrieved, edited and, reused at the main and continue prompts. If the +environment variable +.B SQLITE_HISTORY +is set, it will be used as the name of the history file, whether it +already exists or not. If it is not set but the XDG_STATE_HOME +environment variable is then +.B ${XDG_STATE_HOME}/sqlite_history +is used. If XDG_STATE_HOME is not set then +.B ~/.local/state/sqlite_history +is used. If the selected file does not exist then +.B ~/.sqlite_history +will be used as the history file. If any history file is found, it +will be written if the shell exits interactive mode normally, +regardless of whether it existed previously, though saving will +silently fail if the history file's directory does not exist. .SH SEE ALSO https://sqlite.org/cli.html .br diff --git a/contrib/sqlite3/sqlite3.c b/contrib/sqlite3/sqlite3.c index 26a7a43d8658..dfd557adeda5 100644 --- a/contrib/sqlite3/sqlite3.c +++ b/contrib/sqlite3/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.50.4. By combining all the individual C code files into this +** version 3.53.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** 4d8adfb30e03f9cf27f800a2c1ba3c48fb4c with changes in files: +** c88b22011a54b4f6fbd149e9f8e4de77658c with changes in files: ** ** */ @@ -170,7 +170,9 @@ #define HAVE_UTIME 1 #else /* This is not VxWorks. */ -#define OS_VXWORKS 0 +#ifndef OS_VXWORKS +# define OS_VXWORKS 0 +#endif #define HAVE_FCHOWN 1 #define HAVE_READLINK 1 #define HAVE_LSTAT 1 @@ -465,9 +467,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.50.4" -#define SQLITE_VERSION_NUMBER 3050004 -#define SQLITE_SOURCE_ID "2025-07-30 19:33:53 4d8adfb30e03f9cf27f800a2c1ba3c48fb4ca1b08b0f5ed59a4d5ecbf45e20a3" +#define SQLITE_VERSION "3.53.1" +#define SQLITE_VERSION_NUMBER 3053001 +#define SQLITE_SOURCE_ID "2026-05-05 10:34:17 c88b22011a54b4f6fbd149e9f8e4de77658ce58143a1af0e3785e4e6475127e9" +#define SQLITE_SCM_BRANCH "branch-3.53" +#define SQLITE_SCM_TAGS "release version-3.53.1" +#define SQLITE_SCM_DATETIME "2026-05-05T10:34:17.344Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -487,9 +492,9 @@ extern "C" { ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 ); ** </pre></blockquote>)^ ** -** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] -** macro. ^The sqlite3_libversion() function returns a pointer to the -** to the sqlite3_version[] string constant. The sqlite3_libversion() +** ^The sqlite3_version[] string constant contains the text of the +** [SQLITE_VERSION] macro. ^The sqlite3_libversion() function returns a +** pointer to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to @@ -689,7 +694,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, -** semicolon-separate SQL statements passed into its 2nd argument, +** semicolon-separated SQL statements passed into its 2nd argument, ** in the context of the [database connection] passed in as its 1st ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row @@ -722,7 +727,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** result row is NULL then the corresponding string pointer for the ** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the ** sqlite3_exec() callback is an array of pointers to strings where each -** entry represents the name of corresponding result column as obtained +** entry represents the name of a corresponding result column as obtained ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer @@ -816,6 +821,9 @@ SQLITE_API int sqlite3_exec( #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_ERROR_RESERVESIZE (SQLITE_ERROR | (4<<8)) +#define SQLITE_ERROR_KEY (SQLITE_ERROR | (5<<8)) +#define SQLITE_ERROR_UNABLE (SQLITE_ERROR | (6<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -850,6 +858,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) #define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_IOERR_BADKEY (SQLITE_IOERR | (35<<8)) +#define SQLITE_IOERR_CODEC (SQLITE_IOERR | (36<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -889,7 +899,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal only */ /* ** CAPI3REF: Flags For File Open Operations @@ -908,7 +918,7 @@ SQLITE_API int sqlite3_exec( ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into ** [sqlite3_open_v2()] does *not* cause the underlying database file ** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into -** [sqlite3_open_v2()] has historically be a no-op and might become an +** [sqlite3_open_v2()] has historically been a no-op and might become an ** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ @@ -1002,7 +1012,7 @@ SQLITE_API int sqlite3_exec( ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods ** of an [sqlite3_io_methods] object. These values are ordered from -** lest restrictive to most restrictive. +** least restrictive to most restrictive. ** ** The argument to xLock() is always SHARED or higher. The argument to ** xUnlock is either SHARED or NONE. @@ -1243,7 +1253,7 @@ struct sqlite3_io_methods { ** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] -** No longer in use. +** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used. ** ** <li>[[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and @@ -1318,7 +1328,7 @@ struct sqlite3_io_methods { ** ** <li>[[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of -** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** all [VFSes] in the VFS stack. The names of all VFS shims and the ** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. @@ -1332,7 +1342,7 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level ** [VFSes] currently in use. ^(The argument X in ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X +** of type "[sqlite3_vfs] **". This opcode will set *X ** to a pointer to the top-level VFS.)^ ** ^When there are multiple VFS shims in the stack, this opcode finds the ** upper-most shim only. @@ -1522,7 +1532,7 @@ struct sqlite3_io_methods { ** <li>[[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect ** whether or not there is a database client in another process with a wal-mode -** transaction open on the database or not. It is only available on unix.The +** transaction open on the database or not. It is only available on unix. The ** (void*) argument passed with this file-control should be a pointer to a ** value of type (int). The integer value is set to 1 if the database is a wal ** mode database and there exists at least one client in another process that @@ -1540,6 +1550,15 @@ struct sqlite3_io_methods { ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control ** purges the contents of the in-memory page cache. If there is an open ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. +** +** <li>[[SQLITE_FCNTL_FILESTAT]] +** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information +** about the [sqlite3_file] objects used access the database and journal files +** for the given schema. The fourth parameter to [sqlite3_file_control()] +** should be an initialized [sqlite3_str] pointer. JSON text describing +** various aspects of the sqlite3_file object is appended to the sqlite3_str. +** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time +** options are used to enable it. ** </ul> */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1585,12 +1604,19 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_RESET_CACHE 42 #define SQLITE_FCNTL_NULL_IO 43 #define SQLITE_FCNTL_BLOCK_ON_CONNECT 44 +#define SQLITE_FCNTL_FILESTAT 45 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO +/* reserved file-control numbers: +** 101 +** 102 +** 103 +*/ + /* ** CAPI3REF: Mutex Handle @@ -1791,7 +1817,7 @@ typedef const char *sqlite3_filename; ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. ** -** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** ^The xSetSystemCall(), xGetSystemCall(), and xNextSystemCall() interfaces ** are not used by the SQLite core. These optional interfaces are provided ** by some VFSes to facilitate testing of the VFS code. By overriding ** system calls with functions under its control, a test program can @@ -1947,7 +1973,7 @@ struct sqlite3_vfs { ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically -** initialized when [sqlite3_open()] is called if it has not be initialized +** initialized when [sqlite3_open()] is called if it has not been initialized ** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly @@ -2012,7 +2038,8 @@ SQLITE_API int sqlite3_os_end(void); ** are called "anytime configuration options". ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before ** [sqlite3_shutdown()] with a first argument that is not an anytime -** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** configuration option, then the sqlite3_config() call will +** return SQLITE_MISUSE. ** Note, however, that ^sqlite3_config() can be called as part of the ** implementation of an application-defined [sqlite3_os_init()]. ** @@ -2204,21 +2231,21 @@ struct sqlite3_mem_methods { ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation -** routines with a wrapper that simulations memory allocation failure or +** routines with a wrapper that simulates memory allocation failure or ** tracks memory usage, for example. </dd> ** ** [[SQLITE_CONFIG_SMALL_MALLOC]] <dt>SQLITE_CONFIG_SMALL_MALLOC</dt> -** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of ** type int, interpreted as a boolean, which if true provides a hint to ** SQLite that it should avoid large memory allocations if possible. ** SQLite will run faster if it is free to make large memory allocations, -** but some application might prefer to run slower in exchange for +** but some applications might prefer to run slower in exchange for ** guarantees about memory fragmentation that are possible if large ** allocations are avoided. This hint is normally off. ** </dd> ** ** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt> -** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: @@ -2263,7 +2290,7 @@ struct sqlite3_mem_methods { ** ^If pMem is NULL and N is non-zero, then each database connection ** does an initial bulk allocation for page cache memory ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional +** of -1024*N bytes if N is negative. ^If additional ** page cache memory is needed beyond what is provided by the initial ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each ** additional cache line. </dd> @@ -2292,7 +2319,7 @@ struct sqlite3_mem_methods { ** <dd> ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. ** The argument specifies alternative low-level mutex routines to be used -** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** in place of the mutex routines built into SQLite.)^ ^SQLite makes a copy of ** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then @@ -2334,7 +2361,7 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]] <dt>SQLITE_CONFIG_GETPCACHE2</dt> ** <dd> ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies off ** the current page cache implementation into that object.)^ </dd> ** ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt> @@ -2351,7 +2378,7 @@ struct sqlite3_mem_methods { ** the logger function is a copy of the first parameter to the corresponding ** [sqlite3_log()] call and is intended to be a [result code] or an ** [extended result code]. ^The third parameter passed to the logger is -** log message after formatting via [sqlite3_snprintf()]. +** a log message after formatting via [sqlite3_snprintf()]. ** The SQLite logging interface is not reentrant; the logger function ** supplied by the application must not invoke any SQLite interface. ** In a multi-threaded application, the application-defined logger @@ -2542,7 +2569,7 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the second parameter to the [sqlite3_db_config()] interface. ** -** The [sqlite3_db_config()] interface is a var-args functions. It takes a +** The [sqlite3_db_config()] interface is a var-args function. It takes a ** variable number of parameters, though always at least two. The number of ** parameters passed into sqlite3_db_config() depends on which of these ** constants is given as the second parameter. This documentation page @@ -2578,9 +2605,10 @@ struct sqlite3_mem_methods { ** is less than 8. The "sz" argument should be a multiple of 8 less than ** 65536. If "sz" does not meet this constraint, it is reduced in size until ** it does. -** <li><p>The third argument ("cnt") is the number of slots. Lookaside is disabled -** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so -** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +** <li><p>The third argument ("cnt") is the number of slots. +** Lookaside is disabled if "cnt"is less than 1. +* The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" ** parameter is usually chosen so that the product of "sz" and "cnt" is less ** than 1,000,000. ** </ol> @@ -2654,17 +2682,20 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt> -** <dd> ^This option is used to enable or disable the -** [fts3_tokenizer()] function which is part of the -** [FTS3] full-text search engine extension. -** There must be two additional arguments. -** The first argument is an integer which is 0 to disable fts3_tokenizer() or -** positive to enable fts3_tokenizer() or negative to leave the setting -** unchanged. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled -** following this call. The second parameter may be a NULL pointer, in -** which case the new setting is not reported back. </dd> +** <dd> ^This option is used to enable or disable using the +** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine +** extension - without using bound parameters as the parameters. Doing so +** is disabled by default. There must be two additional arguments. The first +** argument is an integer. If it is passed 0, then using fts3_tokenizer() +** without bound parameters is disabled. If it is passed a positive value, +** then calling fts3_tokenizer without bound parameters is enabled. If it +** is passed a negative value, this setting is not modified - this can be +** used to query for the current setting. The second parameter is a pointer +** to an integer into which is written 0 or 1 to indicate the current value +** of this setting (after it is modified, if applicable). The second +** parameter may be a NULL pointer, in which case the value of the setting +** is not reported back. Refer to [FTS3] documentation for further details. +** </dd> ** ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] ** <dt>SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION</dt> @@ -2676,8 +2707,8 @@ struct sqlite3_mem_methods { ** When the first argument to this interface is 1, then only the C-API is ** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. +** If the first argument is -1, then no changes are made to the state of either +** the C-API or the SQL function. ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface ** is disabled or enabled following this call. The second parameter may @@ -2795,7 +2826,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] ** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates -** the legacy behavior of the [ALTER TABLE RENAME] command such it +** the legacy behavior of the [ALTER TABLE RENAME] command such that it ** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ** additional information. This feature can also be turned on and off @@ -2844,7 +2875,7 @@ struct sqlite3_mem_methods { ** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly -** created database file to have a schema format version number (the 4-byte +** created database files to have a schema format version number (the 4-byte ** integer found at offset 44 into the database header) of 1. This in turn ** means that the resulting database file will be readable and writable by ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, @@ -2865,13 +2896,16 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] ** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt> ** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in -** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears -** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() -** statistics. For statistics to be collected, the flag must be set on -** the database handle both when the SQL statement is prepared and when it -** is stepped. The flag is set (collection of statistics is enabled) -** by default. <p>This option takes two arguments: an integer and a pointer to -** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** [SQLITE_ENABLE_STMT_SCANSTATUS] builds. In this case, it sets or clears +** a flag that enables collection of run-time performance statistics +** used by [sqlite3_stmt_scanstatus_v2()] and the [nexec and ncycle] +** columns of the [bytecode virtual table]. +** For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is +** [sqlite3_prepare|prepared] and when it is [sqlite3_step|stepped]. +** The flag is set (collection of statistics is enabled) by default. +** <p>This option takes two arguments: an integer and a pointer to +** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after ** processing the first argument is written into the integer that the second @@ -2914,8 +2948,8 @@ struct sqlite3_mem_methods { ** <dd>The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the ** ability of the [ATTACH DATABASE] SQL command to open a database for writing. ** This capability is enabled by default. Applications can disable or -** reenable this capability using the current DBCONFIG option. If the -** the this capability is disabled, the [ATTACH] command will still work, +** reenable this capability using the current DBCONFIG option. If +** this capability is disabled, the [ATTACH] command will still work, ** but the database will be opened read-only. If this option is disabled, ** then the ability to create a new database using [ATTACH] is also disabled, ** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE] @@ -2943,16 +2977,34 @@ struct sqlite3_mem_methods { ** comments are allowed in SQL text after processing the first argument. ** </dd> ** +** [[SQLITE_DBCONFIG_FP_DIGITS]] +** <dt>SQLITE_DBCONFIG_FP_DIGITS</dt> +** <dd>The SQLITE_DBCONFIG_FP_DIGITS setting is a small integer that determines +** the number of significant digits that SQLite will attempt to preserve when +** converting floating point numbers (IEEE 754 "doubles") into text. The +** default value 17, as of SQLite version 3.52.0. The value was 15 in all +** prior versions.<p> +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is a small integer, between 3 and 23, or +** zero. The FP_DIGITS setting is changed to that small integer, or left +** unaltered if the first argument is zero or out of range. The second argument +** is a pointer to an integer. If the pointer is not NULL, then the value of +** the FP_DIGITS setting, after possibly being modified by the first +** arguments, is written into the integer to which the second argument points. +** </dd> +** ** </dl> ** ** [[DBCONFIG arguments]] <h3>Arguments To SQLITE_DBCONFIG Options</h3> ** ** <p>Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is a integer. -** The second argument is a pointer to an integer. If the first argument is 1, -** then the option becomes enabled. If the first integer argument is 0, then the -** option is disabled. If the first argument is -1, then the option setting +** The first argument (the third parameter to sqlite3_db_config()) is +** an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, +** then the option is disabled. +** If the first argument is -1, then the option setting ** is unchanged. The second argument, the pointer to an integer, may be NULL. ** If the second argument is not NULL, then a value of 0 or 1 is written into ** the integer to which the second argument points, depending on whether the @@ -2960,9 +3012,10 @@ struct sqlite3_mem_methods { ** the first argument. ** ** <p>While most SQLITE_DBCONFIG options use the argument format -** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] -** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the -** documentation of those exceptional options for details. +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME], +** [SQLITE_DBCONFIG_LOOKASIDE], and [SQLITE_DBCONFIG_FP_DIGITS] options +** are different. See the documentation of those exceptional options for +** details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2987,7 +3040,8 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_FP_DIGITS 1023 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1023 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -3239,7 +3293,7 @@ SQLITE_API int sqlite3_is_interrupted(sqlite3*); ** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** ^These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements and thus ** will not detect syntactically incorrect SQL. ** ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior @@ -3356,7 +3410,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** indefinitely if possible. The results of passing any other negative value ** are undefined. ** -** Internally, each SQLite database handle store two timeout values - the +** Internally, each SQLite database handle stores two timeout values - the ** busy-timeout (used for rollback mode databases, or if the VFS does not ** support blocking locks) and the setlk-timeout (used for blocking locks ** on wal-mode databases). The sqlite3_busy_timeout() method sets both @@ -3386,7 +3440,7 @@ SQLITE_API int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. ** -** Definition: A <b>result table</b> is memory data structure created by the +** Definition: A <b>result table</b> is a memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -3529,7 +3583,7 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is -** a no-op if is called with a NULL pointer. Passing a NULL pointer +** a no-op if it is called with a NULL pointer. Passing a NULL pointer ** to sqlite3_free() is harmless. After being freed, memory ** should neither be read nor written. Even reading previously freed ** memory might result in a segmentation fault or other severe error. @@ -3547,13 +3601,13 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_free(X). ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation ** of at least N bytes in size or NULL if insufficient memory is available. -** ^If M is the size of the prior allocation, then min(N,M) bytes -** of the prior allocation are copied into the beginning of buffer returned +** ^If M is the size of the prior allocation, then min(N,M) bytes of the +** prior allocation are copied into the beginning of the buffer returned ** by sqlite3_realloc(X,N) and the prior allocation is freed. ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the ** prior allocation is not freed. ** -** ^The sqlite3_realloc64(X,N) interfaces works the same as +** ^The sqlite3_realloc64(X,N) interface works the same as ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead ** of a 32-bit signed integer. ** @@ -3603,7 +3657,7 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** was last reset. ^The values returned by [sqlite3_memory_used()] and ** [sqlite3_memory_highwater()] include any overhead ** added by SQLite in its implementation of [sqlite3_malloc()], -** but not overhead added by the any underlying system library +** but not overhead added by any underlying system library ** routines that [sqlite3_malloc()] may call. ** ** ^The memory high-water mark is reset to the current value of @@ -4055,7 +4109,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** there is no harm in trying.) ** ** ^(<dt>[SQLITE_OPEN_SHAREDCACHE]</dt> -** <dd>The database is opened [shared cache] enabled, overriding +** <dd>The database is opened with [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** The [use of shared cache mode is discouraged] and hence shared cache @@ -4063,7 +4117,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** this option is a no-op. ** ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt> -** <dd>The database is opened [shared cache] disabled, overriding +** <dd>The database is opened with [shared cache] disabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** @@ -4469,6 +4523,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** <li> sqlite3_errmsg() ** <li> sqlite3_errmsg16() ** <li> sqlite3_error_offset() +** <li> sqlite3_db_handle() ** </ul> ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language @@ -4481,7 +4536,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** subsequent calls to other SQLite interface functions.)^ ** ** ^The sqlite3_errstr(E) interface returns the English-language text -** that describes the [result code] E, as UTF-8, or NULL if E is not an +** that describes the [result code] E, as UTF-8, or NULL if E is not a ** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. @@ -4489,7 +4544,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** ^If the most recent error references a specific token in the input ** SQL, the sqlite3_error_offset() interface returns the byte offset ** of the start of that token. ^The byte offset returned by -** sqlite3_error_offset() assumes that the input SQL is UTF8. +** sqlite3_error_offset() assumes that the input SQL is UTF-8. ** ^If the most recent error does not reference a specific token in the input ** SQL, then the sqlite3_error_offset() function returns -1. ** @@ -4515,6 +4570,34 @@ SQLITE_API const char *sqlite3_errstr(int); SQLITE_API int sqlite3_error_offset(sqlite3 *db); /* +** CAPI3REF: Set Error Code And Message +** METHOD: sqlite3 +** +** Set the error code of the database handle passed as the first argument +** to errcode, and the error message to a copy of nul-terminated string +** zErrMsg. If zErrMsg is passed NULL, then the error message is set to +** the default message associated with the supplied error code. Subsequent +** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will +** return the values set by this routine in place of what was previously +** set by SQLite itself. +** +** This function returns SQLITE_OK if the error code and error message are +** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if +** the database handle is NULL or invalid. +** +** The error code and message set by this routine remains in effect until +** they are changed, either by another call to this routine or until they are +** changed to by SQLite itself to reflect the result of some subsquent +** API call. +** +** This function is intended for use by SQLite extensions or wrappers. The +** idea is that an extension or wrapper can use this routine to set error +** messages and error codes and thus behave more like a core SQLite +** feature from the point of view of an application. +*/ +SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg); + +/* ** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} ** @@ -4588,8 +4671,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** These constants define various performance limits ** that can be lowered at run-time using [sqlite3_limit()]. -** The synopsis of the meanings of the various limits is shown below. -** Additional information is available at [limits | Limits in SQLite]. +** A concise description of these limits follows, and additional information +** is available at [limits | Limits in SQLite]. ** ** <dl> ** [[SQLITE_LIMIT_LENGTH]] ^(<dt>SQLITE_LIMIT_LENGTH</dt> @@ -4606,6 +4689,10 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt> ** <dd>The maximum depth of the parse tree on any expression.</dd>)^ ** +** [[SQLITE_LIMIT_PARSER_DEPTH]] ^(<dt>SQLITE_LIMIT_PARSER_DEPTH</dt> +** <dd>The maximum depth of the LALR(1) parser stack used to analyze +** input SQL statements.</dd>)^ +** ** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt> ** <dd>The maximum number of terms in a compound SELECT statement.</dd>)^ ** @@ -4650,11 +4737,12 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 +#define SQLITE_LIMIT_PARSER_DEPTH 12 /* ** CAPI3REF: Prepare Flags ** -** These constants define various flags that can be passed into +** These constants define various flags that can be passed into the ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and ** [sqlite3_prepare16_v3()] interfaces. ** @@ -4694,12 +4782,29 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** fails, the sqlite3_prepare_v3() call returns the same error indications ** with or without this flag; it just omits the call to [sqlite3_log()] that ** logs the error. +** +** [[SQLITE_PREPARE_FROM_DDL]] <dt>SQLITE_PREPARE_FROM_DDL</dt> +** <dd>The SQLITE_PREPARE_FROM_DDL flag causes the SQL compiler to enforce +** security constraints that would otherwise only be enforced when parsing +** the database schema. In other words, the SQLITE_PREPARE_FROM_DDL flag +** causes the SQL compiler to treat the SQL statement being prepared as if +** it had come from an attacker. When SQLITE_PREPARE_FROM_DDL is used and +** [SQLITE_DBCONFIG_TRUSTED_SCHEMA] is off, SQL functions may only be called +** if they are tagged with [SQLITE_INNOCUOUS] and virtual tables may only +** be used if they are tagged with [SQLITE_VTAB_INNOCUOUS]. Best practice +** is to use the SQLITE_PREPARE_FROM_DDL option when preparing any SQL that +** is derived from parts of the database schema. In particular, virtual +** table implementations that run SQL statements that are derived from +** arguments to their CREATE VIRTUAL TABLE statement should always use +** [sqlite3_prepare_v3()] and set the SQLITE_PREPARE_FROM_DDL flag to +** prevent bypass of the [SQLITE_DBCONFIG_TRUSTED_SCHEMA] security checks. ** </dl> */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 #define SQLITE_PREPARE_DONT_LOG 0x10 +#define SQLITE_PREPARE_FROM_DDL 0x20 /* ** CAPI3REF: Compiling An SQL Statement @@ -4713,8 +4818,9 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. -** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used -** for special purposes. +** [sqlite3_prepare_v3()] has an extra +** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is sometimes +** needed for special purpose or to pass along security restrictions. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently ** does all parsing using UTF-8. The UTF-16 interfaces are provided @@ -4741,7 +4847,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** there is a small performance advantage to passing an nByte parameter that ** is the number of bytes in the input string <i>including</i> ** the nul-terminator. -** Note that nByte measure the length of the input in bytes, not +** Note that nByte measures the length of the input in bytes, not ** characters, even for the UTF-16 interfaces. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte @@ -4875,7 +4981,7 @@ SQLITE_API int sqlite3_prepare16_v3( ** ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory ** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** maximum string length determined by the [SQLITE_LIMIT_LENGTH]. ** ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time @@ -5063,7 +5169,7 @@ typedef struct sqlite3_value sqlite3_value; ** ** The context in which an SQL function executes is stored in an ** sqlite3_context object. ^A pointer to an sqlite3_context object -** is always first parameter to [application-defined SQL functions]. +** is always the first parameter to [application-defined SQL functions]. ** The application-defined SQL function implementation will pass this ** pointer through into calls to [sqlite3_result_int | sqlite3_result()], ** [sqlite3_aggregate_context()], [sqlite3_user_data()], @@ -5119,8 +5225,8 @@ typedef struct sqlite3_context sqlite3_context; ** it should be a pointer to well-formed UTF16 text. ** ^If the third parameter to sqlite3_bind_text64() is not NULL, then ** it should be a pointer to a well-formed unicode string that is -** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 -** otherwise. +** either UTF8 if the sixth parameter is SQLITE_UTF8 or SQLITE_UTF8_ZT, +** or UTF16 otherwise. ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) @@ -5166,10 +5272,15 @@ typedef struct sqlite3_context sqlite3_context; ** object and pointer to it must remain valid until then. ^SQLite will then ** manage the lifetime of its private copy. ** -** ^The sixth argument to sqlite3_bind_text64() must be one of -** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] -** to specify the encoding of the text in the third parameter. If -** the sixth argument to sqlite3_bind_text64() is not one of the +** ^The sixth argument (the E argument) +** to sqlite3_bind_text64(S,K,Z,N,D,E) must be one of +** [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE] to specify the encoding of the text in the +** third parameter, Z. The special value [SQLITE_UTF8_ZT] means that the +** string argument is both UTF-8 encoded and is zero-terminated. In other +** words, SQLITE_UTF8_ZT means that the Z array is allocated to hold at +** least N+1 bytes and that the Z[N] byte is zero. If +** the E argument to sqlite3_bind_text64(S,K,Z,N,D,E) is not one of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. @@ -5187,9 +5298,11 @@ typedef struct sqlite3_context sqlite3_context; ** associated with the pointer P of type T. ^D is either a NULL pointer or ** a pointer to a destructor function for P. ^SQLite will invoke the ** destructor D with a single argument of P when it is finished using -** P. The T parameter should be a static string, preferably a string -** literal. The sqlite3_bind_pointer() routine is part of the -** [pointer passing interface] added for SQLite 3.20.0. +** P, even if the call to sqlite3_bind_pointer() fails. Due to a +** historical design quirk, results are undefined if D is +** SQLITE_TRANSIENT. The T parameter should be a static string, +** preferably a string literal. The sqlite3_bind_pointer() routine is +** part of the [pointer passing interface] added for SQLite 3.20.0. ** ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which @@ -5800,7 +5913,7 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors -** or if the statement is never been evaluated, then sqlite3_finalize() returns +** or if the statement has never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or ** [extended error code]. @@ -6032,8 +6145,54 @@ SQLITE_API int sqlite3_create_window_function( /* ** CAPI3REF: Text Encodings ** -** These constant define integer codes that represent the various +** These constants define integer codes that represent the various ** text encodings supported by SQLite. +** +** <dl> +** [[SQLITE_UTF8]] <dt>SQLITE_UTF8</dt><dd>Text is encoding as UTF-8</dd> +** +** [[SQLITE_UTF16LE]] <dt>SQLITE_UTF16LE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "little endian" - the least significant +** byte first. This is the usual encoding, for example on Windows.</dd> +** +** [[SQLITE_UTF16BE]] <dt>SQLITE_UTF16BE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "big endian" - the most significant +** byte first. This encoding is less common, but is still sometimes seen, +** specially on older systems. +** +** [[SQLITE_UTF16]] <dt>SQLITE_UTF16</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed either little endian or as big +** endian, according to the native endianness of the host computer. +** +** [[SQLITE_ANY]] <dt>SQLITE_ANY</dt><dd>This encoding value may only be used +** to declare the preferred text for [application-defined SQL functions] +** created using [sqlite3_create_function()] and similar. If the preferred +** encoding (the 4th parameter to sqlite3_create_function() - the eTextRep +** parameter) is SQLITE_ANY, that indicates that the function does not have +** a preference regarding the text encoding of its parameters and can take +** any text encoding that the SQLite core find convenient to supply. This +** option is deprecated. Please do not use it in new applications. +** +** [[SQLITE_UTF16_ALIGNED]] <dt>SQLITE_UTF16_ALIGNED</dt><dd>This encoding +** value may be used as the 3rd parameter (the eTextRep parameter) to +** [sqlite3_create_collation()] and similar. This encoding value means +** that the application-defined collating sequence created expects its +** input strings to be in UTF16 in native byte order, and that the start +** of the strings must be aligned to a 2-byte boundary. +** +** [[SQLITE_UTF8_ZT]] <dt>SQLITE_UTF8_ZT</dt><dd>This option can only be +** used to specify the text encoding to strings input to +** [sqlite3_result_text64()] and [sqlite3_bind_text64()]. +** The SQLITE_UTF8_ZT encoding means that the input string (call it "z") +** is UTF-8 encoded and that it is zero-terminated. If the length parameter +** (call it "n") is non-negative, this encoding option means that the caller +** guarantees that z array contains at least n+1 bytes and that the z[n] +** byte has a value of zero. +** This option gives the same output as SQLITE_UTF8, but can be more efficient +** by avoiding the need to make a copy of the input string, in some cases. +** However, if z is allocated to hold fewer than n+1 bytes or if the +** z[n] byte is not zero, undefined behavior may result. +** </dl> */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ #define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ @@ -6041,6 +6200,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ +#define SQLITE_UTF8_ZT 16 /* Zero-terminated UTF8 */ /* ** CAPI3REF: Function Flags @@ -6124,7 +6284,7 @@ SQLITE_API int sqlite3_create_window_function( ** result. ** Every function that invokes [sqlite3_result_subtype()] should have this ** property. If it does not, then the call to [sqlite3_result_subtype()] -** might become a no-op if the function is used as term in an +** might become a no-op if the function is used as a term in an ** [expression index]. On the other hand, SQL functions that never invoke ** [sqlite3_result_subtype()] should avoid setting this property, as the ** purpose of this property is to disable certain optimizations that are @@ -6251,7 +6411,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if -** and the prior [xColumn] method call that was invoked to extracted +** the prior [xColumn] method call that was invoked to extract ** the value for that column returned without setting a result (probably ** because it queried [sqlite3_vtab_nochange()] and found that the column ** was unchanging). ^Within an [xUpdate] method, any value for which @@ -6275,26 +6435,22 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** the SQL function that supplied the [sqlite3_value*] parameters. ** ** As long as the input parameter is correct, these routines can only -** fail if an out-of-memory error occurs during a format conversion. -** Only the following subset of interfaces are subject to out-of-memory -** errors: -** -** <ul> -** <li> sqlite3_value_blob() -** <li> sqlite3_value_text() -** <li> sqlite3_value_text16() -** <li> sqlite3_value_text16le() -** <li> sqlite3_value_text16be() -** <li> sqlite3_value_bytes() -** <li> sqlite3_value_bytes16() -** </ul> -** +** fail if an out-of-memory error occurs while trying to do a +** UTF8→UTF16 or UTF16→UTF8 conversion. ** If an out-of-memory error occurs, then the return value from these ** routines is the same as if the column had contained an SQL NULL value. -** Valid SQL NULL returns can be distinguished from out-of-memory errors -** by invoking the [sqlite3_errcode()] immediately after the suspect +** If the input sqlite3_value was not obtained from [sqlite3_value_dup()], +** then valid SQL NULL returns can also be distinguished from +** out-of-memory errors after extracting the value +** by invoking the [sqlite3_errcode()] immediately after the suspicious ** return value is obtained and before any ** other SQLite interface is called on the same [database connection]. +** If the input sqlite3_value was obtained from sqlite3_value_dup() then +** it is disconnected from the database connection and so sqlite3_errcode() +** will not work. +** In that case, the only way to distinguish an out-of-memory +** condition from a true SQL NULL is to invoke sqlite3_value_type() on the +** input to see if it is NULL prior to trying to extract the value. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -6321,7 +6477,8 @@ SQLITE_API int sqlite3_value_frombind(sqlite3_value*); ** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) ** returns something other than SQLITE_TEXT, then the return value from ** sqlite3_value_encoding(X) is meaningless. ^Calls to -** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], +** [sqlite3_value_text16be(X)], ** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or ** [sqlite3_value_bytes16(X)] might change the encoding of the value X and ** thus change the return from subsequent calls to sqlite3_value_encoding(X). @@ -6452,17 +6609,17 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** query execution, under some circumstances the associated auxiliary data ** might be preserved. An example of where this might be useful is in a ** regular-expression matching function. The compiled version of the regular -** expression can be stored as auxiliary data associated with the pattern string. -** Then as long as the pattern string remains the same, +** expression can be stored as auxiliary data associated with the pattern +** string. Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data -** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument -** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no auxiliary data -** associated with the function argument, the sqlite3_get_auxdata(C,N) interface -** returns a NULL pointer. +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary +** data associated by the sqlite3_set_auxdata(C,N,P,X) function with the +** Nth argument value to the application-defined function. ^N is zero +** for the left-most function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) +** interface returns a NULL pointer. ** ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the ** N-th argument of the application-defined function. ^Subsequent @@ -6524,6 +6681,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** or a NULL pointer if there were no prior calls to ** sqlite3_set_clientdata() with the same values of D and N. ** Names are compared using strcmp() and are thus case sensitive. +** It returns 0 on success and SQLITE_NOMEM on allocation failure. ** ** If P and X are both non-NULL, then the destructor X is invoked with ** argument P on the first of the following occurrences: @@ -6545,10 +6703,14 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** ** There is no limit (other than available memory) on the number of different ** client data pointers (with different names) that can be attached to a -** single database connection. However, the implementation is optimized -** for the case of having only one or two different client data names. -** Applications and wrapper libraries are discouraged from using more than -** one client data name each. +** single database connection. However, the current implementation stores +** the content on a linked list. Insert and retrieval performance will +** be proportional to the number of entries. The design use case, and +** the use case for which the implementation is optimized, is +** that an application will store only small number of client data names, +** typically just one or two. This interface is not intended to be a +** generalized key/value store for thousands or millions of keys. It +** will work for that, but performance might be disappointing. ** ** There is no way to enumerate the client data pointers ** associated with a database connection. The N parameter can be thought @@ -6656,10 +6818,14 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** ^The sqlite3_result_text64() interface sets the return value of an +** ^The sqlite3_result_text64(C,Z,N,D,E) interface sets the return value of an ** application-defined function to be a text string in an encoding -** specified by the fifth (and last) parameter, which must be one -** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** specified the E parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that +** the result text is both UTF-8 and zero-terminated. In other words, +** SQLITE_UTF8_ZT means that the Z array holds at least N+1 bytes and that +** the Z[N] is zero. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces @@ -6746,7 +6912,7 @@ SQLITE_API void sqlite3_result_int(sqlite3_context*, int); SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); SQLITE_API void sqlite3_result_null(sqlite3_context*); SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char *z, sqlite3_uint64 n, void(*)(void*), unsigned char encoding); SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); @@ -7685,7 +7851,7 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^The sqlite3_load_extension() interface attempts to load an ** [SQLite extension] library contained in the file zFile. If ** the file cannot be loaded directly, attempts are made to load -** with various operating-system specific extensions added. +** with various operating-system specific filename extensions added. ** So for example, if "samplelib" cannot be loaded, then names like ** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might ** be tried also. @@ -7693,10 +7859,10 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where -** X consists of the lower-case equivalent of all ASCII alphabetic -** characters in the filename from the last "/" to the first following -** "." and omitting any initial "lib".)^ +** If that does not work, it tries names of the form "sqlite3_X_init" +** where X consists of the lower-case equivalent of all ASCII alphabetic +** characters or all ASCII alphanumeric characters in the filename from +** the last "/" to the first following "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -7770,7 +7936,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** <blockquote><pre> ** int xEntryPoint( ** sqlite3 *db, -** const char **pzErrMsg, +** char **pzErrMsg, ** const struct sqlite3_api_routines *pThunk ** ); ** </pre></blockquote>)^ @@ -8520,13 +8686,6 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor -** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex -** implementation is included with the library. In this case the -** application must supply a custom mutex implementation using the -** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function -** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() @@ -8881,6 +9040,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_ATOF 34 #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* @@ -8989,17 +9149,22 @@ SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3*); ** pass the returned value to [sqlite3_free()] to avoid a memory leak. ** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any ** errors were encountered during construction of the string. ^The -** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** [sqlite3_str_finish(X)] interface might also return a NULL pointer if the ** string in [sqlite3_str] object X is zero bytes long. +** +** ^The [sqlite3_str_free(X)] interface destroys both the sqlite3_str object +** X and the string content it contains. Calling sqlite3_str_free(X) is +** the equivalent of calling [sqlite3_free](sqlite3_str_finish(X)). */ SQLITE_API char *sqlite3_str_finish(sqlite3_str*); +SQLITE_API void sqlite3_str_free(sqlite3_str*); /* ** CAPI3REF: Add Content To A Dynamic String ** METHOD: sqlite3_str ** -** These interfaces add content to an sqlite3_str object previously obtained -** from [sqlite3_str_new()]. +** These interfaces add or remove content to an sqlite3_str object +** previously obtained from [sqlite3_str_new()]. ** ** ^The [sqlite3_str_appendf(X,F,...)] and ** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] @@ -9022,6 +9187,10 @@ SQLITE_API char *sqlite3_str_finish(sqlite3_str*); ** ^The [sqlite3_str_reset(X)] method resets the string under construction ** inside [sqlite3_str] object X back to zero bytes in length. ** +** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string +** under construction to be N bytes or less. This routine is a no-op if +** N is negative or if the string is already N bytes or smaller in size. +** ** These methods do not return a result code. ^If an error occurs, that fact ** is recorded in the [sqlite3_str] object and can be recovered by a ** subsequent call to [sqlite3_str_errcode(X)]. @@ -9032,6 +9201,7 @@ SQLITE_API void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); SQLITE_API void sqlite3_str_appendall(sqlite3_str*, const char *zIn); SQLITE_API void sqlite3_str_appendchar(sqlite3_str*, int N, char C); SQLITE_API void sqlite3_str_reset(sqlite3_str*); +SQLITE_API void sqlite3_str_truncate(sqlite3_str*,int N); /* ** CAPI3REF: Status Of A Dynamic String @@ -9200,9 +9370,18 @@ SQLITE_API int sqlite3_status64( ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** +** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same +** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H +** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead +** of pointers to 32-bit integers, which allows larger status values +** to be returned. If a status value exceeds 2,147,483,647 then +** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64() +** will return the full value. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); /* ** CAPI3REF: Status Parameters for database connections @@ -9299,6 +9478,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** If an IO or other error occurs while writing a page to disk, the effect ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +** <p> +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL. +** Resetting one will reduce the other.)^ ** </dd> ** ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(<dt>SQLITE_DBSTATUS_CACHE_SPILL</dt> @@ -9314,6 +9497,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** <dd>This parameter returns zero for the current value if and only if ** all foreign key constraints (deferred or immediate) have been ** resolved.)^ ^The highwater mark is always 0. +** +** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(<dt>SQLITE_DBSTATUS_TEMPBUF_SPILL</dt> +** <dd>^(This parameter returns the number of bytes written to temporary +** files on disk that could have been kept in memory had sufficient memory +** been available. This value includes writes to intermediate tables that +** are part of complex queries, external sorts that spill to disk, and +** writes to TEMP tables.)^ +** ^The highwater mark is always 0. +** <p> +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE. +** Resetting one will reduce the other.)^ ** </dd> ** </dl> */ @@ -9330,7 +9525,8 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_DEFERRED_FKS 10 #define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 #define SQLITE_DBSTATUS_CACHE_SPILL 12 -#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_TEMPBUF_SPILL 13 +#define SQLITE_DBSTATUS_MAX 13 /* Largest defined DBSTATUS */ /* @@ -10095,7 +10291,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** is the number of pages currently in the write-ahead log file, ** including those that were just committed. ** -** The callback function should normally return [SQLITE_OK]. ^If an error +** ^The callback function should normally return [SQLITE_OK]. ^If an error ** code is returned, that error will propagate back up through the ** SQLite code base to cause the statement that provoked the callback ** to report an error, though the commit will have still occurred. If the @@ -10103,13 +10299,26 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** that does not correspond to any valid SQLite error code, the results ** are undefined. ** -** A single database handle may have at most a single write-ahead log callback -** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any -** previously registered write-ahead log callback. ^The return value is -** a copy of the third parameter from the previous call, if any, or 0. -** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** overwrite any prior [sqlite3_wal_hook()] settings. +** ^A single database handle may have at most a single write-ahead log +** callback registered at one time. ^Calling [sqlite3_wal_hook()] +** replaces the default behavior or previously registered write-ahead +** log callback. +** +** ^The return value is a copy of the third parameter from the +** previous call, if any, or 0. +** +** ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and +** will overwrite any prior [sqlite3_wal_hook()] settings. +** +** ^If a write-ahead log callback is set using this function then +** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint] +** should be invoked periodically to keep the write-ahead log file +** from growing without bound. +** +** ^Passing a NULL pointer for the callback disables automatic +** checkpointing entirely. To re-enable the default behavior, call +** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint]. */ SQLITE_API void *sqlite3_wal_hook( sqlite3*, @@ -10126,7 +10335,7 @@ SQLITE_API void *sqlite3_wal_hook( ** to automatically [checkpoint] ** after committing a transaction if there are N or ** more frames in the [write-ahead log] file. ^Passing zero or -** a negative value as the nFrame parameter disables automatic +** a negative value as the N parameter disables automatic ** checkpoints entirely. ** ** ^The callback registered by this function replaces any existing callback @@ -10142,9 +10351,10 @@ SQLITE_API void *sqlite3_wal_hook( ** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] -** pages. The use of this interface -** is only necessary if the default setting is found to be suboptimal -** for a particular application. +** pages. +** +** ^The use of this interface is only necessary if the default setting +** is found to be suboptimal for a particular application. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); @@ -10209,6 +10419,11 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the ** addition that it also truncates the log file to zero bytes just prior ** to a successful return. +** +** <dt>SQLITE_CHECKPOINT_NOOP<dd> +** ^This mode always checkpoints zero frames. The only reason to invoke +** a NOOP checkpoint is to access the values returned by +** sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt. ** </dl> ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -10279,6 +10494,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the ** meaning of each of these checkpoint modes. */ +#define SQLITE_CHECKPOINT_NOOP -1 /* Do no work at all */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ #define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ @@ -10516,7 +10732,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** <tr> ** <td valign="top">sqlite3_vtab_distinct() return value ** <td valign="top">Rows are returned in aOrderBy order -** <td valign="top">Rows with the same value in all aOrderBy columns are adjacent +** <td valign="top">Rows with the same value in all aOrderBy columns are +** adjacent ** <td valign="top">Duplicates over all colUsed columns may be omitted ** <tr><td>0<td>yes<td>yes<td>no ** <tr><td>1<td>no<td>yes<td>no @@ -10525,8 +10742,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** </table> ** ** ^For the purposes of comparing virtual table output values to see if the -** values are the same value for sorting purposes, two NULL values are considered -** to be the same. In other words, the comparison operator is "IS" +** values are the same value for sorting purposes, two NULL values are +** considered to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** ** If a virtual table implementation is unable to meet the requirements @@ -10647,7 +10864,7 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** ){ ** // do something with pVal ** } -** if( rc!=SQLITE_OK ){ +** if( rc!=SQLITE_DONE ){ ** // an error has occurred ** } ** </pre></blockquote>)^ @@ -10819,9 +11036,9 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** a variable pointed to by the "pOut" parameter. ** ** The "flags" parameter must be passed a mask of flags. At present only -** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** one flag is defined - [SQLITE_SCANSTAT_COMPLEX]. If SQLITE_SCANSTAT_COMPLEX ** is specified, then status information is available for all elements -** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** of a query plan that are reported by "[EXPLAIN QUERY PLAN]" output. If ** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements ** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of ** the EXPLAIN QUERY PLAN output) are available. Invoking API @@ -10835,7 +11052,8 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. ** -** See also: [sqlite3_stmt_scanstatus_reset()] +** See also: [sqlite3_stmt_scanstatus_reset()] and the +** [nexec and ncycle] columns of the [bytecode virtual table]. */ SQLITE_API int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -11106,7 +11324,7 @@ typedef struct sqlite3_snapshot { ** The [sqlite3_snapshot_get()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( +SQLITE_API int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -11155,7 +11373,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** The [sqlite3_snapshot_open()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( +SQLITE_API int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -11172,7 +11390,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( ** The [sqlite3_snapshot_free()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. @@ -11199,7 +11417,7 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( +SQLITE_API int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); @@ -11227,7 +11445,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Serialize a database @@ -11301,12 +11519,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the ** [database connection] D to disconnect from database S and then -** reopen S as an in-memory database based on the serialization contained -** in P. The serialized database P is N bytes in size. M is the size of -** the buffer P, which might be larger than N. If M is larger than N, and -** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is -** permitted to add content to the in-memory database as long as the total -** size does not exceed M bytes. +** reopen S as an in-memory database based on the serialization +** contained in P. If S is a NULL pointer, the main database is +** used. The serialized database P is N bytes in size. M is the size +** of the buffer P, which might be larger than N. If M is larger than +** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then +** SQLite is permitted to add content to the in-memory database as +** long as the total size does not exceed M bytes. ** ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will ** invoke sqlite3_free() on the serialization buffer when the database @@ -11374,6 +11593,77 @@ SQLITE_API int sqlite3_deserialize( #define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ /* +** CAPI3REF: Bind array values to the CARRAY table-valued function +** +** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to +** parameter that is the first argument of the [carray() table-valued function]. +** The S parameter is a pointer to the [prepared statement] that uses the +** carray() functions. I is the parameter index to be bound. I must be the +** index of the parameter that is the first argument to the carray() +** table-valued function. P is a pointer to the array to be bound, and N +** is the number of elements in the array. The F argument is one of +** constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], +** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. +** +** If the X argument is not a NULL pointer or one of the special +** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke +** the function X with argument D when it is finished using the data in P. +** The call to X(D) is a destructor for the array P. The destructor X(D) +** is invoked even if the call to sqlite3_carray_bind_v2() fails. If the X +** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes +** that the data static and the destructor is never invoked. If the X +** parameter is the special-case value [SQLITE_TRANSIENT], then +** sqlite3_carray_bind_v2() makes its own private copy of the data prior +** to returning and never invokes the destructor X. +** +** The sqlite3_carray_bind() function works the same as sqlite3_carray_bind_v2() +** with a D parameter set to P. In other words, +** sqlite3_carray_bind(S,I,P,N,F,X) is same as +** sqlite3_carray_bind_v2(S,I,P,N,F,X,P). +*/ +SQLITE_API int sqlite3_carray_bind_v2( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*), /* Destructor for aData */ + void *pDel /* Optional argument to xDel() */ +); +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructor for aData */ +); + +/* +** CAPI3REF: Datatypes for the CARRAY table-valued function +** +** The fifth argument to the [sqlite3_carray_bind()] interface musts be +** one of the following constants, to specify the datatype of the array +** that is being bound into the [carray table-valued function]. +*/ +#define SQLITE_CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define SQLITE_CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define SQLITE_CARRAY_DOUBLE 2 /* Data is doubles */ +#define SQLITE_CARRAY_TEXT 3 /* Data is char* */ +#define SQLITE_CARRAY_BLOB 4 /* Data is struct iovec */ + +/* +** Versions of the above #defines that omit the initial SQLITE_, for +** legacy compatibility. +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + +/* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. */ @@ -12632,14 +12922,32 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** update the "main" database attached to handle db with the changes found in ** the changeset passed via the second and third arguments. ** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. Additionally, starting with version 3.51.0, +** an error code and error message that may be accessed using the +** [sqlite3_errcode()] and [sqlite3_errmsg()] APIs are left in the database +** handle. +** ** The fourth argument (xFilter) passed to these functions is the "filter -** callback". If it is not NULL, then for each table affected by at least one -** change in the changeset, the filter callback is invoked with -** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument as the first. If the "filter callback" -** returns zero, then no attempt is made to apply any changes to the table. -** Otherwise, if the return value is non-zero or the xFilter argument to -** is NULL, all changes related to the table are attempted. +** callback". This may be passed NULL, in which case all changes in the +** changeset are applied to the database. For sqlite3changeset_apply() and +** sqlite3_changeset_apply_v2(), if it is not NULL, then it is invoked once +** for each table affected by at least one change in the changeset. In this +** case the table name is passed as the second argument, and a copy of +** the context pointer passed as the sixth argument to apply() or apply_v2() +** as the first. If the "filter callback" returns zero, then no attempt is +** made to apply any changes to the table. Otherwise, if the return value is +** non-zero, all changes related to the table are attempted. +** +** For sqlite3_changeset_apply_v3(), the xFilter callback is invoked once +** per change. The second argument in this case is an sqlite3_changeset_iter +** that may be queried using the usual APIs for the details of the current +** change. If the "filter callback" returns zero in this case, then no attempt +** is made to apply the current change. If it returns non-zero, the change +** is applied. ** ** For each table that is not excluded by the filter callback, this function ** tests that the target database contains a compatible table. A table is @@ -12660,11 +12968,11 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** one such warning is issued for each table in the changeset. ** ** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for -** each type of change is below. +** to modify the table contents according to each UPDATE, INSERT or DELETE +** change that is not excluded by a filter callback. If a change cannot be +** applied cleanly, the conflict handler function passed as the fifth argument +** to sqlite3changeset_apply() may be invoked. A description of exactly when +** the conflict handler is invoked for each type of change is below. ** ** Unlike the xFilter argument, xConflict may not be passed NULL. The results ** of passing anything other than a valid function pointer as the xConflict @@ -12760,12 +13068,6 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** This can be used to further customize the application's conflict ** resolution strategy. ** -** All changes made by these functions are enclosed in a savepoint transaction. -** If any other error (aside from a constraint failure when attempting to -** write to the target database) occurs, then the savepoint transaction is -** rolled back, restoring the target database to its original state, and an -** SQLite error code returned. -** ** If the output parameters (ppRebase) and (pnRebase) are non-NULL and ** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() ** may set (*ppRebase) to point to a "rebase" that may be used with the @@ -12815,6 +13117,23 @@ SQLITE_API int sqlite3changeset_apply_v2( void **ppRebase, int *pnRebase, /* OUT: Rebase data */ int flags /* SESSION_CHANGESETAPPLY_* flags */ ); +SQLITE_API int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); /* ** CAPI3REF: Flags for sqlite3changeset_apply_v2 @@ -13234,6 +13553,23 @@ SQLITE_API int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ); +SQLITE_API int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); SQLITE_API int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, @@ -13326,6 +13662,232 @@ SQLITE_API int sqlite3session_config(int op, void *pArg); #define SQLITE_SESSION_CONFIG_STRMSIZE 1 /* +** CAPI3REF: Configure a changegroup object +** +** Configure the changegroup object passed as the first argument. +** At present the only valid value for the second parameter is +** [SQLITE_CHANGEGROUP_CONFIG_PATCHSET]. +*/ +SQLITE_API int sqlite3changegroup_config(sqlite3_changegroup*, int, void *pArg); + +/* +** CAPI3REF: Options for sqlite3changegroup_config(). +** +** The following values may be passed as the 2nd parameter to +** sqlite3changegroup_config(). +** +** <dt>SQLITE_CHANGEGROUP_CONFIG_PATCHSET <dd> +** A changegroup object generates either a changeset or patchset. Usually, +** this is determined by whether the first call to sqlite3changegroup_add() +** is passed a changeset or a patchset. Or, if the first changes are added +** to the changegroup object using the sqlite3changegroup_change_xxx() +** APIs, then this option may be used to configure whether the changegroup +** object generates a changeset or patchset. +** +** When this option is invoked, parameter pArg must point to a value of +** type int. If the changegroup currently contains zero changes, and the +** value of the int variable is zero or greater than zero, then the +** changegroup is configured to generate a changeset or patchset, +** respectively. It is a no-op, not an error, if the changegroup is not +** configured because it has already started accumulating changes. +** +** Before returning, the int variable is set to 0 if the changegroup is +** configured to generate a changeset, or 1 if it is configured to generate +** a patchset. +*/ +#define SQLITE_CHANGEGROUP_CONFIG_PATCHSET 1 + + +/* +** CAPI3REF: Begin adding a change to a changegroup +** +** This API is used, in concert with other sqlite3changegroup_change_xxx() +** APIs, to add changes to a changegroup object one at a time. To add a +** single change, the caller must: +** +** 1. Invoke sqlite3changegroup_change_begin() to indicate the type of +** change (INSERT, UPDATE or DELETE), the affected table and whether +** or not the change should be marked as indirect. +** +** 2. Invoke sqlite3changegroup_change_int64() or one of the other four +** value functions - _null(), _double(), _text() or _blob() - one or +** more times to specify old.* and new.* values for the change being +** constructed. +** +** 3. Invoke sqlite3changegroup_change_finish() to either finish adding +** the change to the group, or to discard the change altogether. +** +** The first argument to this function must be a pointer to the existing +** changegroup object that the change will be added to. The second argument +** must be SQLITE_INSERT, SQLITE_UPDATE or SQLITE_DELETE. The third is the +** name of the table that the change affects, and the fourth is a boolean +** flag specifying whether the change should be marked as "indirect" (if +** bIndirect is non-zero) or not indirect (if bIndirect is zero). +** +** Following a successful call to this function, this function may not be +** called again on the same changegroup object until after +** sqlite3changegroup_change_finish() has been called. Doing so is an +** SQLITE_MISUSE error. +** +** The changegroup object passed as the first argument must be already +** configured with schema data for the specified table. It may be configured +** either by calling sqlite3changegroup_schema() with a database that contains +** the table, or sqlite3changegroup_add() with a changeset that contains the +** table. If the changegroup object has not been configured with a schema for +** the specified table when this function is called, SQLITE_ERROR is returned. +** +** If successful, SQLITE_OK is returned. Otherwise, if an error occurs, an +** SQLite error code is returned. In this case, if argument pzErr is non-NULL, +** then (*pzErr) may be set to point to a buffer containing a utf-8 formated, +** nul-terminated, English language error message. It is the responsibility +** of the caller to eventually free this buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_begin( + sqlite3_changegroup*, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +); + +/* +** CAPI3REF: Add a 64-bit integer to a changegroup +** +** This function may only be called between a successful call to +** sqlite3changegroup_change_begin() and its matching +** sqlite3changegroup_change_finish() call. If it is called at any +** other time, it is an SQLITE_MISUSE error. Calling this function +** specifies a 64-bit integer value to be used in the change currently being +** added to the changegroup object passed as the first argument. +** +** The second parameter, bNew, specifies whether the value is to be part of +** the new.* (if bNew is non-zero) or old.* (if bNew is zero) record of +** the change under construction. If this does not match the type of change +** specified by the preceding call to sqlite3changegroup_change_begin() (i.e. +** an old.* value for an SQLITE_INSERT change, or a new.* value for an +** SQLITE_DELETE), then SQLITE_ERROR is returned. +** +** The third parameter specifies the column of the old.* or new.* record that +** the value will be a part of. If the specified table has an explicit primary +** key, then this is the index of the table column, numbered from 0 in the order +** specified within the CREATE TABLE statement. Or, if the table uses an +** implicit rowid key, then the column 0 is the rowid and the explicit columns +** are numbered starting from 1. If the iCol parameter is less than 0 or greater +** than the index of the last column in the table, SQLITE_RANGE is returned. +** +** The fourth parameter is the integer value to use as part of the old.* or +** new.* record. +** +** If this call is successful, SQLITE_OK is returned. Otherwise, if an +** error occurs, an SQLite error code is returned. +*/ +SQLITE_API int sqlite3changegroup_change_int64( + sqlite3_changegroup*, + int bNew, + int iCol, + sqlite3_int64 iVal +); + +/* +** CAPI3REF: Add a NULL to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently under construction with a NULL value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_null(sqlite3_changegroup*, int, int); + +/* +** CAPI3REF: Add an double to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently being constructed with a real value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_double(sqlite3_changegroup*, int, int, double); + +/* +** CAPI3REF: Add a text value to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a text value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the text encoded using +** utf-8. Parameter nVal may either be the size of the text value in bytes, or +** else a negative value, in which case the buffer pVal points to is assumed to +** be nul-terminated. +*/ +SQLITE_API int sqlite3changegroup_change_text( + sqlite3_changegroup*, int, int, const char *pVal, int nVal +); + +/* +** CAPI3REF: Add a blob to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a blob value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the blob. Parameter +** nVal is the size of the blob in bytes. +*/ +SQLITE_API int sqlite3changegroup_change_blob( + sqlite3_changegroup*, int, int, const void *pVal, int nVal +); + +/* +** CAPI3REF: Finish adding one-at-at-time changes to a changegroup +** +** This function may only be called following a successful call to +** sqlite3changegroup_change_begin(). Otherwise, it is an SQLITE_MISUSE error. +** +** If parameter bDiscard is non-zero, then the current change is simply +** discarded. In this case this function is always successful and SQLITE_OK +** returned. +** +** If parameter bDiscard is zero, then an attempt is made to add the current +** change to the changegroup. Assuming the changegroup is configured to +** produce a changeset (not a patchset), this requires that: +** +** * If the change is an INSERT or DELETE, then a value must be specified +** for all columns of the new.* or old.* record, respectively. +** +** * If the change is an UPDATE record, then values must be provided for +** the PRIMARY KEY columns of the old.* record, but must not be provided +** for PRIMARY KEY columns of the new.* record. +** +** * If the change is an UPDATE record, then for each non-PRIMARY KEY +** column in the old.* record for which a value has been provided, a +** value must also be provided for the same column in the new.* record. +** Similarly, for each non-PK column in the old.* record for which +** a value is not provided, a value must not be provided for the same +** column in the new.* record. +** +** * All values specified for PRIMARY KEY columns must be non-NULL. +** +** Otherwise, it is an error. +** +** If the changegroup already contains a change for the same row (identified +** by PRIMARY KEY columns), then the current change is combined with the +** existing change in the same way as for sqlite3changegroup_add(). +** +** For a patchset, all of the above rules apply except that it doesn't matter +** whether or not values are provided for the non-PK old.* record columns +** for an UPDATE or DELETE change. This means that code used to produce +** a changeset using the sqlite3changegroup_change_xxx() APIs may also +** be used to produce patchsets. +** +** If the call is successful, SQLITE_OK is returned. Otherwise, if an error +** occurs, an SQLite error code is returned. If an error is returned and +** parameter pzErr is not NULL, then (*pzErr) may be set to point to a buffer +** containing a nul-terminated, utf-8 encoded, English language error message. +** It is the responsibility of the caller to eventually free any such error +** message buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_finish( + sqlite3_changegroup*, + int bDiscard, + char **pzErr +); + +/* ** Make sure we can call this stuff from C++. */ #if 0 @@ -14140,6 +14702,27 @@ struct fts5_api { #define SQLITE_MIN_LENGTH 30 /* Minimum value for the length limit */ /* +** Maximum size of any single memory allocation. +** +** This is not a limit on the total amount of memory used. This is +** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). +** +** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 +** This provides a 256-byte safety margin for defense against 32-bit +** signed integer overflow bugs when computing memory allocation sizes. +** Paranoid applications might want to reduce the maximum allocation size +** further for an even larger safety margin. 0x3fffffff or 0x0fffffff +** or even smaller would be reasonable upper bounds on the size of a memory +** allocations for most applications. +*/ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 +#endif +#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 +# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 +#endif + +/* ** This is the maximum number of ** ** * Columns in a table @@ -14174,22 +14757,43 @@ struct fts5_api { ** It used to be the case that setting this value to zero would ** turn the limit off. That is no longer true. It is not possible ** to turn this limit off. +** +** The hard limit is the largest possible 32-bit signed integer less +** 1024, or 2147482624. */ #ifndef SQLITE_MAX_SQL_LENGTH # define SQLITE_MAX_SQL_LENGTH 1000000000 #endif /* -** The maximum depth of an expression tree. This is limited to -** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might -** want to place more severe limits on the complexity of an -** expression. A value of 0 means that there is no limit. +** The maximum depth of an expression tree. The expression tree depth +** is also limited indirectly by SQLITE_MAX_SQL_LENGTH and by +** SQLITE_MAX_PARSER_DEPTH. Reducing the maximum complexity of +** expressions can help prevent excess memory usage by hostile SQL. +** +** A value of 0 for this compile-time option causes all expression +** depth limiting code to be omitted. */ #ifndef SQLITE_MAX_EXPR_DEPTH # define SQLITE_MAX_EXPR_DEPTH 1000 #endif /* +** The maximum depth of the LALR(1) stack used in the parser that +** interprets SQL inputs. The parser stack depth can also be limited +** indirectly by SQLITE_MAX_SQL_LENGTH. Limiting the parser stack +** depth can help prevent excess memory usage and excess CPU stack +** usage when processing hostile SQL. +** +** Prior to version 3.45.0 (2024-01-15), the parser stack was +** hard-coded to 100 entries, and that worked fine for almost all +** applications. So the upper bound on this limit need not be large. +*/ +#ifndef SQLITE_MAX_PARSER_DEPTH +# define SQLITE_MAX_PARSER_DEPTH 2500 +#endif + +/* ** The maximum number of terms in a compound SELECT statement. ** The code generator for compound SELECT statements does one ** level of recursion for each term. A stack overflow can result @@ -14304,13 +14908,17 @@ struct fts5_api { # undef SQLITE_MAX_DEFAULT_PAGE_SIZE # define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE #endif +#if SQLITE_MAX_DEFAULT_PAGE_SIZE<SQLITE_DEFAULT_PAGE_SIZE +# undef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_DEFAULT_PAGE_SIZE +#endif /* ** Maximum number of pages in one database file. ** ** This is really just the default value for the max_page_count pragma. -** This value can be lowered (or raised) at run-time using that the +** This value can be lowered (or raised) at run-time using the ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT @@ -14368,9 +14976,7 @@ struct fts5_api { /* ** Include standard header files as necessary */ -#ifdef HAVE_STDINT_H #include <stdint.h> -#endif #ifdef HAVE_INTTYPES_H #include <inttypes.h> #endif @@ -15074,6 +15680,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define float sqlite_int64 # define fabs(X) ((X)<0?-(X):(X)) # define sqlite3IsOverflow(X) 0 +# define INFINITY (9223372036854775807LL) # ifndef SQLITE_BIG_DBL # define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) # endif @@ -15178,7 +15785,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); ** ourselves. */ #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif /* @@ -15483,6 +16090,7 @@ typedef INT16_TYPE LogEst; #else # define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif +#define TWO_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&1)==0) /* ** Disable MMAP on platforms where it is known to not work @@ -15566,6 +16174,8 @@ SQLITE_PRIVATE u32 sqlite3TreeTrace; ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated ** 0x00080000 NOT NULL strength reduction +** 0x00100000 Pointers are all shown as zero +** 0x00200000 EXISTS-to-JOIN optimization */ /* @@ -15610,6 +16220,7 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; ** 0x00020000 Show WHERE terms returned from whereScanNext() ** 0x00040000 Solver overview messages ** 0x00080000 Star-query heuristic +** 0x00100000 Pointers are all shown as zero */ @@ -15682,7 +16293,7 @@ struct BusyHandler { ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomClear) +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3RowSetClear) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -15903,8 +16514,8 @@ typedef int VList; ** must provide its own VFS implementation together with sqlite3_os_init() ** and sqlite3_os_end() routines. */ -#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ - !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +#if SQLITE_OS_KV+1<=1 && SQLITE_OS_OTHER+1<=1 && \ + SQLITE_OS_WIN+1<=1 && SQLITE_OS_UNIX+1<=1 # if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ defined(__MINGW32__) || defined(__BORLANDC__) # define SQLITE_OS_WIN 1 @@ -16750,6 +17361,7 @@ struct BtreePayload { SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes); SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); @@ -16937,6 +17549,7 @@ struct VdbeOp { SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ Table *pTab; /* Used when p4type is P4_TABLE */ SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ + Index *pIdx; /* Used when p4type is P4_INDEX */ #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif @@ -16991,20 +17604,21 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT32 (-3) /* P4 is a 32-bit signed integer */ #define P4_SUBPROGRAM (-4) /* P4 is a pointer to a SubProgram structure */ #define P4_TABLE (-5) /* P4 is a pointer to a Table structure */ +#define P4_INDEX (-6) /* P4 is a pointer to an Index structure */ /* Above do not own any resources. Must free those below */ -#define P4_FREE_IF_LE (-6) -#define P4_DYNAMIC (-6) /* Pointer to memory from sqliteMalloc() */ -#define P4_FUNCDEF (-7) /* P4 is a pointer to a FuncDef structure */ -#define P4_KEYINFO (-8) /* P4 is a pointer to a KeyInfo structure */ -#define P4_EXPR (-9) /* P4 is a pointer to an Expr tree */ -#define P4_MEM (-10) /* P4 is a pointer to a Mem* structure */ -#define P4_VTAB (-11) /* P4 is a pointer to an sqlite3_vtab structure */ -#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ -#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ -#define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ -#define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ -#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ -#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ +#define P4_FREE_IF_LE (-7) +#define P4_DYNAMIC (-7) /* Pointer to memory from sqliteMalloc() */ +#define P4_FUNCDEF (-8) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-9) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-10) /* P4 is a pointer to an Expr tree */ +#define P4_MEM (-11) /* P4 is a pointer to a Mem* structure */ +#define P4_VTAB (-12) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_REAL (-13) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-14) /* P4 is a 64-bit signed integer */ +#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-16) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-17) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-18) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -17083,20 +17697,20 @@ typedef struct VdbeOpList VdbeOpList; #define OP_SorterSort 34 /* jump */ #define OP_Sort 35 /* jump */ #define OP_Rewind 36 /* jump0 */ -#define OP_SorterNext 37 /* jump */ -#define OP_Prev 38 /* jump */ -#define OP_Next 39 /* jump */ -#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfEmpty 37 /* jump, synopsis: if( empty(P1) ) goto P2 */ +#define OP_SorterNext 38 /* jump */ +#define OP_Prev 39 /* jump */ +#define OP_Next 40 /* jump */ +#define OP_IdxLE 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 42 /* jump, synopsis: key=r[P3@P4] */ #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ -#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 48 /* jump0 */ -#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 50 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IdxLT 45 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGE 46 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IFindKey 47 /* jump */ +#define OP_RowSetRead 48 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 49 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 50 /* jump0 */ #define OP_IsNull 51 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 52 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 53 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -17106,49 +17720,49 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 57 /* jump, same as TK_LT, synopsis: IF r[P3]<r[P1] */ #define OP_Ge 58 /* jump, same as TK_GE, synopsis: IF r[P3]>=r[P1] */ #define OP_ElseEq 59 /* jump, same as TK_ESCAPE */ -#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 62 /* jump */ -#define OP_VNext 63 /* jump */ -#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ -#define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Return 67 -#define OP_EndCoroutine 68 -#define OP_HaltIfNull 69 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 70 -#define OP_Integer 71 /* synopsis: r[P2]=P1 */ -#define OP_Int64 72 /* synopsis: r[P2]=P4 */ -#define OP_String 73 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_BeginSubrtn 74 /* synopsis: r[P2]=NULL */ -#define OP_Null 75 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 76 /* synopsis: r[P1]=NULL */ -#define OP_Blob 77 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1) */ -#define OP_Move 79 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 80 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 81 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 82 /* synopsis: r[P2]=r[P1] */ -#define OP_FkCheck 83 -#define OP_ResultRow 84 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 85 -#define OP_AddImm 86 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 87 -#define OP_Cast 88 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 89 -#define OP_Compare 90 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_IsTrue 91 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_ZeroOrNull 92 /* synopsis: r[P2] = 0 OR NULL */ -#define OP_Offset 93 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 94 /* synopsis: r[P3]=PX cursor P1 column P2 */ -#define OP_TypeCheck 95 /* synopsis: typecheck(r[P1@P2]) */ -#define OP_Affinity 96 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 97 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 98 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 99 -#define OP_SetCookie 100 -#define OP_ReopenIdx 101 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 102 /* synopsis: root=P2 iDb=P3 */ +#define OP_FkIfZero 60 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 61 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 62 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 63 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 64 /* jump */ +#define OP_VNext 65 /* jump */ +#define OP_Filter 66 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ +#define OP_PureFunc 67 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 68 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 69 +#define OP_EndCoroutine 70 +#define OP_HaltIfNull 71 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 72 +#define OP_Integer 73 /* synopsis: r[P2]=P1 */ +#define OP_Int64 74 /* synopsis: r[P2]=P4 */ +#define OP_String 75 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_BeginSubrtn 76 /* synopsis: r[P2]=NULL */ +#define OP_Null 77 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 78 /* synopsis: r[P1]=NULL */ +#define OP_Blob 79 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 80 /* synopsis: r[P2]=parameter(P1) */ +#define OP_Move 81 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 82 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 83 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 84 /* synopsis: r[P2]=r[P1] */ +#define OP_FkCheck 85 +#define OP_ResultRow 86 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 87 +#define OP_AddImm 88 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 89 +#define OP_Cast 90 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 91 +#define OP_Compare 92 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 93 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 94 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 95 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 96 /* synopsis: r[P3]=PX cursor P1 column P2 */ +#define OP_TypeCheck 97 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 98 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 99 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 100 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 101 +#define OP_SetCookie 102 #define OP_BitAnd 103 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 104 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 105 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<<r[P1] */ @@ -17159,83 +17773,85 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Divide 110 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ #define OP_Remainder 111 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ #define OP_Concat 112 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_OpenWrite 113 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenDup 114 +#define OP_ReopenIdx 113 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 114 /* synopsis: root=P2 iDb=P3 */ #define OP_BitNot 115 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ -#define OP_OpenAutoindex 116 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 117 /* synopsis: nColumn=P2 */ +#define OP_OpenWrite 116 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenDup 117 #define OP_String8 118 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_SorterOpen 119 -#define OP_SequenceTest 120 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ -#define OP_OpenPseudo 121 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 122 -#define OP_ColumnsUsed 123 -#define OP_SeekScan 124 /* synopsis: Scan-ahead up to P1 rows */ -#define OP_SeekHit 125 /* synopsis: set P2<=seekHit<=P3 */ -#define OP_Sequence 126 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 127 /* synopsis: r[P2]=rowid */ -#define OP_Insert 128 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_RowCell 129 -#define OP_Delete 130 -#define OP_ResetCount 131 -#define OP_SorterCompare 132 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 133 /* synopsis: r[P2]=data */ -#define OP_RowData 134 /* synopsis: r[P2]=data */ -#define OP_Rowid 135 /* synopsis: r[P2]=PX rowid of P1 */ -#define OP_NullRow 136 -#define OP_SeekEnd 137 -#define OP_IdxInsert 138 /* synopsis: key=r[P2] */ -#define OP_SorterInsert 139 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 140 /* synopsis: key=r[P2@P3] */ -#define OP_DeferredSeek 141 /* synopsis: Move P3 to P1.rowid if needed */ -#define OP_IdxRowid 142 /* synopsis: r[P2]=rowid */ -#define OP_FinishSeek 143 -#define OP_Destroy 144 -#define OP_Clear 145 -#define OP_ResetSorter 146 -#define OP_CreateBtree 147 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ -#define OP_SqlExec 148 -#define OP_ParseSchema 149 -#define OP_LoadAnalysis 150 -#define OP_DropTable 151 -#define OP_DropIndex 152 -#define OP_DropTrigger 153 +#define OP_OpenAutoindex 119 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 120 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 121 +#define OP_SequenceTest 122 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 123 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 124 +#define OP_ColumnsUsed 125 +#define OP_SeekScan 126 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 127 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 128 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 129 /* synopsis: r[P2]=rowid */ +#define OP_Insert 130 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 131 +#define OP_Delete 132 +#define OP_ResetCount 133 +#define OP_SorterCompare 134 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 135 /* synopsis: r[P2]=data */ +#define OP_RowData 136 /* synopsis: r[P2]=data */ +#define OP_Rowid 137 /* synopsis: r[P2]=PX rowid of P1 */ +#define OP_NullRow 138 +#define OP_SeekEnd 139 +#define OP_IdxInsert 140 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 141 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 142 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 143 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 144 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 145 +#define OP_Destroy 146 +#define OP_Clear 147 +#define OP_ResetSorter 148 +#define OP_CreateBtree 149 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 150 +#define OP_ParseSchema 151 +#define OP_LoadAnalysis 152 +#define OP_DropTable 153 #define OP_Real 154 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_IntegrityCk 155 -#define OP_RowSetAdd 156 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 157 -#define OP_FkCounter 158 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 159 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 160 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 161 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 162 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 164 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 165 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 166 -#define OP_CursorLock 167 -#define OP_CursorUnlock 168 -#define OP_TableLock 169 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 170 -#define OP_VCreate 171 -#define OP_VDestroy 172 -#define OP_VOpen 173 -#define OP_VCheck 174 -#define OP_VInitIn 175 /* synopsis: r[P2]=ValueList(P1,P3) */ -#define OP_VColumn 176 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 177 -#define OP_Pagecount 178 -#define OP_MaxPgcnt 179 -#define OP_ClrSubtype 180 /* synopsis: r[P1].subtype = 0 */ -#define OP_GetSubtype 181 /* synopsis: r[P2] = r[P1].subtype */ -#define OP_SetSubtype 182 /* synopsis: r[P2].subtype = r[P1] */ -#define OP_FilterAdd 183 /* synopsis: filter(P1) += key(P3@P4) */ -#define OP_Trace 184 -#define OP_CursorHint 185 -#define OP_ReleaseReg 186 /* synopsis: release r[P1@P2] mask P3 */ -#define OP_Noop 187 -#define OP_Explain 188 -#define OP_Abortable 189 +#define OP_DropIndex 155 +#define OP_DropTrigger 156 +#define OP_IntegrityCk 157 +#define OP_RowSetAdd 158 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 159 +#define OP_FkCounter 160 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 161 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 162 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 163 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 164 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 165 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 166 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 167 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 168 +#define OP_CursorLock 169 +#define OP_CursorUnlock 170 +#define OP_TableLock 171 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 172 +#define OP_VCreate 173 +#define OP_VDestroy 174 +#define OP_VOpen 175 +#define OP_VCheck 176 +#define OP_VInitIn 177 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 178 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 179 +#define OP_Pagecount 180 +#define OP_MaxPgcnt 181 +#define OP_ClrSubtype 182 /* synopsis: r[P1].subtype = 0 */ +#define OP_GetSubtype 183 /* synopsis: r[P2] = r[P1].subtype */ +#define OP_SetSubtype 184 /* synopsis: r[P2].subtype = r[P1] */ +#define OP_FilterAdd 185 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 186 +#define OP_CursorHint 187 +#define OP_ReleaseReg 188 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 189 +#define OP_Explain 190 +#define OP_Abortable 191 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -17254,26 +17870,27 @@ typedef struct VdbeOpList VdbeOpList; /* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ /* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ /* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ -/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x41, 0x41,\ -/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ -/* 48 */ 0x81, 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x41,\ -/* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ -/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ -/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\ -/* 88 */ 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40, 0x00,\ -/* 96 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x40, 0x26,\ +/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x01, 0x41,\ +/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x41, 0x09,\ +/* 48 */ 0x23, 0x0b, 0x81, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x01, 0x03, 0x03, 0x03,\ +/* 64 */ 0x01, 0x41, 0x01, 0x00, 0x00, 0x02, 0x02, 0x08,\ +/* 72 */ 0x00, 0x10, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10,\ +/* 80 */ 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00,\ +/* 88 */ 0x02, 0x02, 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20,\ +/* 96 */ 0x40, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x26,\ /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 112 */ 0x26, 0x00, 0x40, 0x12, 0x40, 0x40, 0x10, 0x00,\ -/* 120 */ 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10, 0x10,\ -/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x50,\ -/* 136 */ 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50, 0x40,\ -/* 144 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x06, 0x10, 0x00, 0x04,\ -/* 160 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x50,\ -/* 176 */ 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12, 0x00,\ -/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} +/* 112 */ 0x26, 0x40, 0x40, 0x12, 0x00, 0x40, 0x10, 0x40,\ +/* 120 */ 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x40, 0x40,\ +/* 128 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,\ +/* 136 */ 0x00, 0x50, 0x00, 0x40, 0x04, 0x04, 0x00, 0x40,\ +/* 144 */ 0x50, 0x40, 0x10, 0x00, 0x00, 0x10, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x06, 0x10,\ +/* 160 */ 0x00, 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,\ +/* 176 */ 0x10, 0x50, 0x40, 0x00, 0x10, 0x10, 0x02, 0x12,\ +/* 184 */ 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +} /* The resolve3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -17281,7 +17898,7 @@ typedef struct VdbeOpList VdbeOpList; ** generated this include file strives to group all JUMP opcodes ** together near the beginning of the list. */ -#define SQLITE_MX_JUMP_OPCODE 64 /* Maximum JUMP opcode */ +#define SQLITE_MX_JUMP_OPCODE 66 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -17290,7 +17907,7 @@ typedef struct VdbeOpList VdbeOpList; ** Additional non-public SQLITE_PREPARE_* flags */ #define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ -#define SQLITE_PREPARE_MASK 0x1f /* Mask of public flags */ +#define SQLITE_PREPARE_MASK 0x3f /* Mask of public flags */ /* ** Prototypes for the VDBE interface. See comments on the implementation @@ -17404,8 +18021,11 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); SQLITE_PRIVATE int sqlite3BlobCompare(const Mem*, const Mem*); +#ifdef SQLITE_ENABLE_PERCENTILE +SQLITE_PRIVATE const char *sqlite3VdbeFuncName(const sqlite3_context*); +#endif -SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); @@ -17418,7 +18038,9 @@ SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); +#ifndef SQLITE_OMIT_DATETIME_FUNCS SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); +#endif #ifdef SQLITE_ENABLE_BYTECODE_VTAB SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); #endif @@ -17567,10 +18189,10 @@ struct PgHdr { PCache *pCache; /* PRIVATE: Cache that owns this page */ PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ - Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES - u32 pageHash; /* Hash of page content */ + u64 pageHash; /* Hash of page content */ #endif + Pgno pgno; /* Page number for this page */ u16 flags; /* PGHDR flags defined below */ /********************************************************************** @@ -17910,7 +18532,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_PARSER_DEPTH+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -18064,6 +18686,7 @@ struct sqlite3 { u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ u8 eOpenState; /* Current condition of the connection */ + u8 nFpDigit; /* Significant digits to keep on double->text */ int nextPagesize; /* Pagesize after VACUUM if >0 */ i64 nChange; /* Value returned by sqlite3_changes() */ i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ @@ -18074,7 +18697,7 @@ struct sqlite3 { u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ - unsigned imposterTable : 1; /* Building an imposter table */ + unsigned imposterTable : 2; /* Building an imposter table */ unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ const char **azInit; /* "type", "name", and "tbl_name" columns */ } init; @@ -18157,6 +18780,7 @@ struct sqlite3 { i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ DbClientData *pDbData; /* sqlite3_set_clientdata() content */ + u64 nSpill; /* TEMP content spilled to disk */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MAIN ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -18300,6 +18924,7 @@ struct sqlite3 { #define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ #define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ #define SQLITE_StarQuery 0x20000000 /* Heurists for star queries */ +#define SQLITE_ExistsToJoin 0x40000000 /* The EXISTS-to-JOIN optimization */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -18538,7 +19163,7 @@ struct FuncDestructor { #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_BUILTIN|\ SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, 0, #zName, } + pArg, 0, xFunc, 0, 0, 0, #zName, {0} } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } @@ -18866,6 +19491,7 @@ struct Table { #define TF_Ephemeral 0x00004000 /* An ephemeral table */ #define TF_Eponymous 0x00008000 /* An eponymous virtual table */ #define TF_Strict 0x00010000 /* STRICT mode */ +#define TF_Imposter 0x00020000 /* An imposter table */ /* ** Allowed values for Table.eTabType @@ -19021,9 +19647,15 @@ struct FKey { ** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** -** Note that aSortOrder[] and aColl[] have nField+1 slots. There -** are nField slots for the columns of an index then one extra slot -** for the rowid at the end. +** The aSortOrder[] and aColl[] arrays have nAllField slots each. There +** are nKeyField slots for the columns of an index then extra slots +** for the rowid or key at the end. The aSortOrder array is located after +** the aColl[] array. +** +** If SQLITE_ENABLE_PREUPDATE_HOOK is defined, then aSortFlags might be NULL +** to indicate that this object is for use by a preupdate hook. When aSortFlags +** is NULL, then nAllField is uninitialized and no space is allocated for +** aColl[], so those fields may not be used. */ struct KeyInfo { u32 nRef; /* Number of references to this KeyInfo object */ @@ -19035,9 +19667,18 @@ struct KeyInfo { CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */ }; -/* The size (in bytes) of a KeyInfo object with up to N fields */ +/* The size (in bytes) of a KeyInfo object with up to N fields. This includes +** the main body of the KeyInfo object and the aColl[] array of N elements, +** but does not count the memory used to hold aSortFlags[]. */ #define SZ_KEYINFO(N) (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*)) +/* The size of a bare KeyInfo with no aColl[] entries */ +#if FLEXARRAY+1 > 1 +# define SZ_KEYINFO_0 offsetof(KeyInfo,aColl) +#else +# define SZ_KEYINFO_0 sizeof(KeyInfo) +#endif + /* ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. */ @@ -19056,9 +19697,8 @@ struct KeyInfo { ** ** An instance of this object serves as a "key" for doing a search on ** an index b+tree. The goal of the search is to find the entry that -** is closed to the key described by this object. This object might hold -** just a prefix of the key. The number of fields is given by -** pKeyInfo->nField. +** is closest to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by nField. ** ** The r1 and r2 fields are the values to return if this key is less than ** or greater than a key in the btree, respectively. These are normally @@ -19068,7 +19708,7 @@ struct KeyInfo { ** The key comparison functions actually return default_rc when they find ** an equals comparison. default_rc can be -1, 0, or +1. If there are ** multiple entries in the b-tree with the same key (when only looking -** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** at the first nField elements) then default_rc can be set to -1 to ** cause the search to find the last match, or +1 to cause the search to ** find the first match. ** @@ -19080,8 +19720,8 @@ struct KeyInfo { ** b-tree. */ struct UnpackedRecord { - KeyInfo *pKeyInfo; /* Collation and sort-order information */ - Mem *aMem; /* Values */ + KeyInfo *pKeyInfo; /* Comparison info for the index that is unpacked */ + Mem *aMem; /* Values for columns of the index */ union { char *z; /* Cache of aMem[0].z for vdbeRecordCompareString() */ i64 i; /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */ @@ -19730,6 +20370,7 @@ struct SrcItem { unsigned rowidUsed :1; /* The ROWID of this table is referenced */ unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ + unsigned fromExists :1; /* Comes from WHERE EXISTS(...) */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ @@ -19940,19 +20581,6 @@ struct Upsert { /* ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. -** -** See the header comment on the computeLimitRegisters() routine for a -** detailed description of the meaning of the iLimit and iOffset fields. -** -** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. -** These addresses must be stored so that we can go back and fill in -** the P4_KEYINFO and P2 parameters later. Neither the KeyInfo nor -** the number of columns in P2 can be computed at the same time -** as the OP_OpenEphm instruction is coded because not -** enough information about the compound query is known at that point. -** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences -** for the result set. The KeyInfo for addrOpenEphm[2] contains collating -** sequences for the ORDER BY clause. */ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ @@ -19960,7 +20588,6 @@ struct Select { u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ u32 selId; /* Unique identifier number for this SELECT */ - int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ ExprList *pEList; /* The fields of the result */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -19992,7 +20619,7 @@ struct Select { #define SF_Resolved 0x0000004 /* Identifiers have been resolved */ #define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ #define SF_HasAgg 0x0000010 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ +#define SF_ClonedRhsIn 0x0000020 /* Cloned RHS of an IN operator */ #define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ #define SF_Compound 0x0000100 /* Part of a compound query */ @@ -20002,14 +20629,14 @@ struct Select { #define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ #define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ #define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ -#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ +/* 0x0008000 // available for reuse */ #define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ #define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ #define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ #define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ #define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ #define SF_View 0x0200000 /* SELECT statement is a view */ -#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ +/* 0x0400000 // available for reuse */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ #define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ @@ -20017,6 +20644,7 @@ struct Select { #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ #define SF_Correlated 0x20000000 /* True if references the outer context */ +#define SF_OnToWhere 0x40000000 /* One or more ON clauses moved to WHERE */ /* True if SrcItem X is a subquery that has SF_NestedFrom */ #define IsNestedFrom(X) \ @@ -20028,11 +20656,6 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** -** SRT_Union Store results as a key in a temporary index -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary index pDest->iSDParm. -** ** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result ** set is not empty. ** @@ -20096,30 +20719,28 @@ struct Select { ** table. (pDest->iSDParm) is the number of key columns in ** each index record in this case. */ -#define SRT_Union 1 /* Store result as keys in an index */ -#define SRT_Except 2 /* Remove result from a UNION index */ -#define SRT_Exists 3 /* Store 1 if the result is not empty */ -#define SRT_Discard 4 /* Do not save the results anywhere */ -#define SRT_DistFifo 5 /* Like SRT_Fifo, but unique results only */ -#define SRT_DistQueue 6 /* Like SRT_Queue, but unique results only */ +#define SRT_Exists 1 /* Store 1 if the result is not empty */ +#define SRT_Discard 2 /* Do not save the results anywhere */ +#define SRT_DistFifo 3 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistQueue 4 /* Like SRT_Queue, but unique results only */ /* The DISTINCT clause is ignored for all of the above. Not that ** IgnorableDistinct() implies IgnorableOrderby() */ #define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Queue 7 /* Store result in an queue */ -#define SRT_Fifo 8 /* Store result as data with an automatic rowid */ +#define SRT_Queue 5 /* Store result in an queue */ +#define SRT_Fifo 6 /* Store result as data with an automatic rowid */ /* The ORDER BY clause is ignored for all of the above */ #define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo) -#define SRT_Output 9 /* Output each row of result */ -#define SRT_Mem 10 /* Store result in a memory cell */ -#define SRT_Set 11 /* Store results as keys in an index */ -#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 13 /* Generate a single row of result */ -#define SRT_Table 14 /* Store result as data with an automatic rowid */ -#define SRT_Upfrom 15 /* Store result as data with rowid */ +#define SRT_Output 7 /* Output each row of result */ +#define SRT_Mem 8 /* Store result in a memory cell */ +#define SRT_Set 9 /* Store results as keys in an index */ +#define SRT_EphemTab 10 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 11 /* Generate a single row of result */ +#define SRT_Table 12 /* Store result as data with an automatic rowid */ +#define SRT_Upfrom 13 /* Store result as data with rowid */ /* ** An instance of this object describes where to put of the results of @@ -20255,16 +20876,12 @@ struct Parse { u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ - u8 mayAbort; /* True if statement may throw an ABORT exception */ - u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ - u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ - u8 disableTriggers; /* True to disable triggers */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -20273,10 +20890,15 @@ struct Parse { u8 isCreate; /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER) ** and ALTER TABLE ADD COLUMN. */ #endif - bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ - bft bHasWith :1; /* True if statement contains WITH */ - bft okConstFactor :1; /* OK to factor out constants */ - bft checkSchema :1; /* Causes schema cookie check after an error */ + bft disableTriggers:1; /* True to disable triggers */ + bft mayAbort :1; /* True if statement may throw an ABORT exception */ + bft hasCompound :1; /* Need to invoke convertCompoundSelectToSubquery() */ + bft bReturning :1; /* Coding a RETURNING trigger */ + bft bHasExists :1; /* Has a correlated "EXISTS (SELECT ....)" expression */ + bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ + bft bHasWith :1; /* True if statement contains WITH */ + bft okConstFactor:1; /* OK to factor out constants */ + bft checkSchema :1; /* Causes schema cookie check after an error */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -20505,19 +21127,19 @@ struct Trigger { ** orconf -> stores the ON CONFLICT algorithm ** pSelect -> The content to be inserted - either a SELECT statement or ** a VALUES clause. -** zTarget -> Dequoted name of the table to insert into. +** pSrc -> Table to insert into. ** pIdList -> If this is an INSERT INTO ... (<column-names>) VALUES ... ** statement, then this stores the column-names to be ** inserted into. ** pUpsert -> The ON CONFLICT clauses for an Upsert ** ** (op == TK_DELETE) -** zTarget -> Dequoted name of the table to delete from. +** pSrc -> Table to delete from ** pWhere -> The WHERE clause of the DELETE statement if one is specified. ** Otherwise NULL. ** ** (op == TK_UPDATE) -** zTarget -> Dequoted name of the table to update. +** pSrc -> Table to update, followed by any FROM clause tables. ** pWhere -> The WHERE clause of the UPDATE statement if one is specified. ** Otherwise NULL. ** pExprList -> A list of the columns to update and the expressions to update @@ -20537,8 +21159,7 @@ struct TriggerStep { u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ - char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ - SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ + SrcList *pSrc; /* Table to insert/update/delete */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ IdList *pIdList; /* Column names for INSERT */ @@ -20621,10 +21242,11 @@ typedef struct { /* ** Allowed values for mInitFlags */ -#define INITFLAG_AlterMask 0x0003 /* Types of ALTER */ +#define INITFLAG_AlterMask 0x0007 /* Types of ALTER */ #define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ #define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ #define INITFLAG_AlterAdd 0x0003 /* Reparse after an ADD COLUMN */ +#define INITFLAG_AlterDropCons 0x0004 /* Reparse after an ADD COLUMN */ /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled ** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning @@ -20754,6 +21376,7 @@ struct Walker { NameContext *pNC; /* Naming context */ int n; /* A counter */ int iCur; /* A cursor number */ + int sz; /* String literal length */ SrcList *pSrcList; /* FROM clause */ struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ @@ -20769,6 +21392,7 @@ struct Walker { SrcItem *pSrcItem; /* A single FROM clause item */ DbFixer *pFix; /* See sqlite3FixSelect() */ Mem *aMem; /* See sqlite3BtreeCursorHint() */ + struct CheckOnCtx *pCheckOnCtx; /* See selectCheckOnClauses() */ } u; }; @@ -21157,7 +21781,20 @@ SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3*,int*); SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void); -#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) + +/* The SQLITE_THREAD_MISUSE_WARNINGS compile-time option used to be called +** SQLITE_ENABLE_MULTITHREADED_CHECKS. Keep that older macro for backwards +** compatibility, at least for a while... */ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +# define SQLITE_THREAD_MISUSE_WARNINGS 1 +#endif + +/* SQLITE_THREAD_MISUSE_ABORT implies SQLITE_THREAD_MISUSE_WARNINGS */ +#ifdef SQLITE_THREAD_MISUSE_ABORT +# define SQLITE_THREAD_MISUSE_WARNINGS 1 +#endif + +#if defined(SQLITE_THREAD_MISUSE_WARNINGS) && !defined(SQLITE_MUTEX_OMIT) SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); #else # define sqlite3MutexWarnOnContention(x) @@ -21187,16 +21824,21 @@ struct PrintfArguments { }; /* +** Maxium number of base-10 digits in an unsigned 64-bit integer +*/ +#define SQLITE_U64_DIGITS 20 + +/* ** An instance of this object receives the decoding of a floating point ** value into an approximate decimal representation. */ struct FpDecode { - char sign; /* '+' or '-' */ - char isSpecial; /* 1: Infinity 2: NaN */ - int n; /* Significant digits in the decode */ - int iDP; /* Location of the decimal point */ - char *z; /* Start of significant digits */ - char zBuf[24]; /* Storage for significant digits */ + int n; /* Significant digits in the decode */ + int iDP; /* Location of the decimal point */ + char *z; /* Start of significant digits */ + char zBuf[SQLITE_U64_DIGITS+1]; /* Storage for significant digits */ + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ }; SQLITE_PRIVATE void sqlite3FpDecode(FpDecode*,double,int,int); @@ -21256,6 +21898,7 @@ SQLITE_PRIVATE void sqlite3ShowTriggerList(const Trigger*); SQLITE_PRIVATE void sqlite3ShowWindow(const Window*); SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window*); #endif +SQLITE_PRIVATE void sqlite3ShowBitvec(Bitvec*); #endif SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); @@ -21284,6 +21927,7 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); #endif SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); +SQLITE_PRIVATE Expr *sqlite3ExprInt32(sqlite3*,int); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); @@ -21437,6 +22081,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList Expr*,ExprList*,u32,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3*,void*); +SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*); SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); @@ -21534,6 +22179,7 @@ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*, Parse*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); +SQLITE_PRIVATE int sqlite3ExprIsLikeOperator(const Expr*); SQLITE_PRIVATE int sqlite3IsRowid(const char*); SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab); SQLITE_PRIVATE void sqlite3GenerateRowDelete( @@ -21572,13 +22218,17 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void); SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) -SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3*); +SQLITE_PRIVATE Module *sqlite3JsonVtabRegister(sqlite3*,const char*); #endif SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) +SQLITE_PRIVATE Module *sqlite3CarrayRegister(sqlite3*); +#endif + #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif @@ -21598,17 +22248,16 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, i SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,SrcList*, IdList*, Select*,u8,Upsert*, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,SrcList*,SrcList*,ExprList*, Expr*, u8, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,SrcList*, Expr*, const char*,const char*); SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); -SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) # define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else @@ -21622,7 +22271,6 @@ SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); # define sqlite3ParseToplevel(p) p # define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 -# define sqlite3TriggerStepSrc(A,B) 0 #endif SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); @@ -21655,7 +22303,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); SQLITE_PRIVATE i64 sqlite3RealToI64(double); SQLITE_PRIVATE int sqlite3Int64ToText(i64,char*); -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3GetUInt32(const char*, u32*); SQLITE_PRIVATE int sqlite3Atoi(const char*); @@ -21799,10 +22447,13 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); SQLITE_PRIVATE void sqlite3AlterFunctions(void); SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); +SQLITE_PRIVATE void sqlite3AlterDropConstraint(Parse*,SrcList*,Token*,Token*); +SQLITE_PRIVATE void sqlite3AlterAddConstraint(Parse*,SrcList*,Token*,Token*,const char*,int); +SQLITE_PRIVATE void sqlite3AlterSetNotNull(Parse*, SrcList*, Token*, Token*); +SQLITE_PRIVATE i64 sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); -SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int, int); SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse*, SrcItem*); @@ -21875,6 +22526,7 @@ SQLITE_PRIVATE char *sqlite3RCStrResize(char*,u64); SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); +SQLITE_PRIVATE int sqlite3StrAccumEnlargeIfNeeded(StrAccum*, i64); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); @@ -22565,6 +23217,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_BYTECODE_VTAB "ENABLE_BYTECODE_VTAB", #endif +#ifdef SQLITE_ENABLE_CARRAY + "ENABLE_CARRAY", +#endif #ifdef SQLITE_ENABLE_CEROD "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), #endif @@ -22655,6 +23310,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK "ENABLE_OVERSIZE_CELL_CHECK", #endif +#ifdef SQLITE_ENABLE_PERCENTILE + "ENABLE_PERCENTILE", +#endif #ifdef SQLITE_ENABLE_PREUPDATE_HOOK "ENABLE_PREUPDATE_HOOK", #endif @@ -23096,6 +23754,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_STMTJRNL_SPILL "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), #endif +#ifdef SQLITE_STRICT_SUBTYPE + "STRICT_SUBTYPE", +#endif #ifdef SQLITE_SUBSTR_COMPATIBILITY "SUBSTR_COMPATIBILITY", #endif @@ -23869,7 +24530,7 @@ struct sqlite3_value { ** MEM_Int, MEM_Real, and MEM_IntReal. ** ** * MEM_Blob|MEM_Zero A blob in Mem.z of length Mem.n plus -** MEM.u.i extra 0x00 bytes at the end. +** Mem.u.nZero extra 0x00 bytes at the end. ** ** * MEM_Int Integer stored in Mem.u.i. ** @@ -24138,7 +24799,9 @@ struct PreUpdate { Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ sqlite3_value **apDflt; /* Array of default values, if required */ - u8 keyinfoSpace[SZ_KEYINFO(0)]; /* Space to hold pKeyinfo[0] content */ + struct { + u8 keyinfoSpace[SZ_KEYINFO_0]; /* Space to hold pKeyinfo[0] content */ + } uKey; }; /* @@ -24209,6 +24872,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, i64, u8, void(*)(void*)); +SQLITE_PRIVATE int sqlite3VdbeMemSetText(Mem*, const char*, i64, void(*)(void*)); SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE_OMIT_FLOATING_POINT # define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 @@ -24227,13 +24891,14 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetZeroBlob(Mem*,int); SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); #endif SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemZeroTerminateIfAble(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem*); SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); +SQLITE_PRIVATE int sqlite3MemRealValueRC(Mem*, double*); SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); @@ -24264,6 +24929,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int,int); #endif SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); +SQLITE_PRIVATE int sqlite3VdbeFindIndexKey(BtCursor*, Index*, UnpackedRecord*, int*, int); SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); @@ -24302,9 +24968,11 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); +SQLITE_PRIVATE int sqlite3VdbeCheckFkImmediate(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeCheckFkDeferred(Vdbe*); #else -# define sqlite3VdbeCheckFk(p,i) 0 +# define sqlite3VdbeCheckFkImmediate(p) 0 +# define sqlite3VdbeCheckFkDeferred(p) 0 #endif #ifdef SQLITE_DEBUG @@ -24513,23 +25181,25 @@ SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ /* ** Query status information for a single database connection */ -SQLITE_API int sqlite3_db_status( - sqlite3 *db, /* The database connection whose status is desired */ - int op, /* Status verb */ - int *pCurrent, /* Write current value here */ - int *pHighwater, /* Write high-water mark here */ - int resetFlag /* Reset high-water mark if true */ +SQLITE_API int sqlite3_db_status64( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + sqlite3_int64 *pCurrent, /* Write current value here */ + sqlite3_int64 *pHighwtr, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){ return SQLITE_MISUSE_BKPT; } #endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { - *pCurrent = sqlite3LookasideUsed(db, pHighwater); + int H = 0; + *pCurrent = sqlite3LookasideUsed(db, &H); + *pHighwtr = H; if( resetFlag ){ LookasideSlot *p = db->lookaside.pFree; if( p ){ @@ -24560,7 +25230,7 @@ SQLITE_API int sqlite3_db_status( assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); *pCurrent = 0; - *pHighwater = (int)db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT]; + *pHighwtr = db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT]; if( resetFlag ){ db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; } @@ -24574,7 +25244,7 @@ SQLITE_API int sqlite3_db_status( */ case SQLITE_DBSTATUS_CACHE_USED_SHARED: case SQLITE_DBSTATUS_CACHE_USED: { - int totalUsed = 0; + sqlite3_int64 totalUsed = 0; int i; sqlite3BtreeEnterAll(db); for(i=0; i<db->nDb; i++){ @@ -24590,18 +25260,18 @@ SQLITE_API int sqlite3_db_status( } sqlite3BtreeLeaveAll(db); *pCurrent = totalUsed; - *pHighwater = 0; + *pHighwtr = 0; break; } /* ** *pCurrent gets an accurate estimate of the amount of memory used ** to store the schema for all databases (main, temp, and any ATTACHed - ** databases. *pHighwater is set to zero. + ** databases. *pHighwtr is set to zero. */ case SQLITE_DBSTATUS_SCHEMA_USED: { - int i; /* Used to iterate through schemas */ - int nByte = 0; /* Used to accumulate return value */ + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; @@ -24635,7 +25305,7 @@ SQLITE_API int sqlite3_db_status( db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); - *pHighwater = 0; + *pHighwtr = 0; *pCurrent = nByte; break; } @@ -24643,7 +25313,7 @@ SQLITE_API int sqlite3_db_status( /* ** *pCurrent gets an accurate estimate of the amount of memory used ** to store all prepared statements. - ** *pHighwater is set to zero. + ** *pHighwtr is set to zero. */ case SQLITE_DBSTATUS_STMT_USED: { struct Vdbe *pVdbe; /* Used to iterate through VMs */ @@ -24658,7 +25328,7 @@ SQLITE_API int sqlite3_db_status( db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; - *pHighwater = 0; /* IMP: R-64479-57858 */ + *pHighwtr = 0; /* IMP: R-64479-57858 */ *pCurrent = nByte; break; @@ -24666,7 +25336,7 @@ SQLITE_API int sqlite3_db_status( /* ** Set *pCurrent to the total cache hits or misses encountered by all - ** pagers the database handle is connected to. *pHighwater is always set + ** pagers the database handle is connected to. *pHighwtr is always set ** to zero. */ case SQLITE_DBSTATUS_CACHE_SPILL: @@ -24686,19 +25356,39 @@ SQLITE_API int sqlite3_db_status( sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); } } - *pHighwater = 0; /* IMP: R-42420-56072 */ + *pHighwtr = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = (int)nRet & 0x7fffffff; + *pCurrent = nRet; + break; + } + + /* Set *pCurrent to the number of bytes that the db database connection + ** has spilled to the filesystem in temporary files that could have been + ** stored in memory, had sufficient memory been available. + ** The *pHighwater is always set to zero. + */ + case SQLITE_DBSTATUS_TEMPBUF_SPILL: { + u64 nRet = 0; + if( db->aDb[1].pBt ){ + Pager *pPager = sqlite3BtreePager(db->aDb[1].pBt); + sqlite3PagerCacheStat(pPager, SQLITE_DBSTATUS_CACHE_WRITE, + resetFlag, &nRet); + nRet *= sqlite3BtreeGetPageSize(db->aDb[1].pBt); + } + nRet += db->nSpill; + if( resetFlag ) db->nSpill = 0; + *pHighwtr = 0; + *pCurrent = nRet; break; } /* Set *pCurrent to non-zero if there are unresolved deferred foreign ** key constraints. Set *pCurrent to zero if all foreign key constraints - ** have been satisfied. The *pHighwater is always set to zero. + ** have been satisfied. The *pHighwtr is always set to zero. */ case SQLITE_DBSTATUS_DEFERRED_FKS: { - *pHighwater = 0; /* IMP: R-11967-56545 */ + *pHighwtr = 0; /* IMP: R-11967-56545 */ *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; break; } @@ -24711,6 +25401,31 @@ SQLITE_API int sqlite3_db_status( return rc; } +/* +** 32-bit variant of sqlite3_db_status64() +*/ +SQLITE_API int sqlite3_db_status( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + int *pCurrent, /* Write current value here */ + int *pHighwtr, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ +){ + sqlite3_int64 C = 0, H = 0; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + rc = sqlite3_db_status64(db, op, &C, &H, resetFlag); + if( rc==0 ){ + *pCurrent = C & 0x7fffffff; + *pHighwtr = H & 0x7fffffff; + } + return rc; +} + /************** End of status.c **********************************************/ /************** Begin file date.c ********************************************/ /* @@ -24903,6 +25618,10 @@ static int parseTimezone(const char *zDate, DateTime *p){ } zDate += 5; p->tz = sgn*(nMn + nHr*60); + if( p->tz==0 ){ /* Forum post 2025-09-17T10:12:14z */ + p->isLocal = 0; + p->isUtc = 1; + } zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } return *zDate!=0; @@ -25140,7 +25859,7 @@ static int parseDateOrTime( return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + }else if( sqlite3AtoF(zDate, &r)>0 ){ setRawDateNumber(p, r); return 0; }else if( (sqlite3StrICmp(zDate,"subsec")==0 @@ -25586,7 +26305,7 @@ static int parseModifier( ** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && sqlite3AtoF(&z[8], &r)>0 && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); @@ -25657,9 +26376,11 @@ static int parseModifier( case '8': case '9': { double rRounder; - int i; + int i, rx; int Y,M,D,h,m,x; const char *z2 = z; + char *zCopy; + sqlite3 *db = sqlite3_context_db_handle(pCtx); char z0 = z[0]; for(n=1; z[n]; n++){ if( z[n]==':' ) break; @@ -25669,7 +26390,11 @@ static int parseModifier( if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; } } - if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + zCopy = sqlite3DbStrNDup(db, z, n); + if( zCopy==0 ) break; + rx = sqlite3AtoF(zCopy, &r)<=0; + sqlite3DbFree(db, zCopy); + if( rx ){ assert( rc==1 ); break; } @@ -26098,8 +26823,8 @@ static int daysAfterSunday(DateTime *pDate){ ** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 -** %p "am" or "pm" -** %P "AM" or "PM" +** %p "AM" or "PM" +** %P "am" or "pm" ** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 @@ -26489,7 +27214,7 @@ static void datedebugFunc( char *zJson; zJson = sqlite3_mprintf( "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," - "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "s:%.3f,validJD:%d,validYMD:%d,validHMS:%d," "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," "isUtc:%d,isLocal:%d}", x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, @@ -29268,23 +29993,28 @@ static SQLITE_WSD int mutexIsInit = 0; #ifndef SQLITE_MUTEX_OMIT -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#ifdef SQLITE_THREAD_MISUSE_WARNINGS /* -** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** This block (enclosed by SQLITE_THREAD_MISUSE_WARNINGS) contains ** the implementation of a wrapper around the system default mutex ** implementation (sqlite3DefaultMutex()). ** ** Most calls are passed directly through to the underlying default ** mutex implementation. Except, if a mutex is configured by calling ** sqlite3MutexWarnOnContention() on it, then if contention is ever -** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** encountered within xMutexEnter() then a warning is emitted via +** sqlite3_log(). Furthermore, if SQLITE_THREAD_MISUSE_ABORT is +** defined then abort() is called after the sqlite3_log() warning. ** -** This type of mutex is used as the database handle mutex when testing -** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +** This type of mutex is used on the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. A failure +** indicates that the app ought to be using SQLITE_OPEN_FULLMUTEX or +** similar because it is trying to use the same database handle from +** two different connections at the same time. */ /* -** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** Type for all mutexes used when SQLITE_THREAD_MISUSE_WARNINGS ** is defined. Variable CheckMutex.mutex is a pointer to the real mutex ** allocated by the system mutex implementation. Variable iType is usually set ** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST @@ -29320,11 +30050,12 @@ static int checkMutexNotheld(sqlite3_mutex *p){ */ static int checkMutexInit(void){ pGlobalMutexMethods = sqlite3DefaultMutex(); - return SQLITE_OK; + return pGlobalMutexMethods->xMutexInit(); } static int checkMutexEnd(void){ + int rc = pGlobalMutexMethods->xMutexEnd(); pGlobalMutexMethods = 0; - return SQLITE_OK; + return rc; } /* @@ -29401,6 +30132,9 @@ static void checkMutexEnter(sqlite3_mutex *p){ sqlite3_log(SQLITE_MISUSE, "illegal multi-threaded access to database connection" ); +#if SQLITE_THREAD_MISUSE_ABORT + abort(); +#endif } pGlobalMutexMethods->xMutexEnter(pCheck->mutex); } @@ -29452,7 +30186,7 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; } } -#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ +#endif /* ifdef SQLITE_THREAD_MISUSE_WARNINGS */ /* ** Initialize the mutex system. @@ -29469,7 +30203,7 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; if( sqlite3GlobalConfig.bCoreMutex ){ -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#ifdef SQLITE_THREAD_MISUSE_WARNINGS pFrom = multiThreadedCheckMutex(); #else pFrom = sqlite3DefaultMutex(); @@ -30318,14 +31052,6 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ #endif /* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - -/* ** For WinCE, some API function parameters do not appear to be declared as ** volatile. */ @@ -30339,7 +31065,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() ** functions are not available (e.g. those not using MSVC, Cygwin, etc). */ -#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && \ SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) # define SQLITE_OS_WIN_THREADS 1 #else @@ -30456,11 +31182,7 @@ static int winMutexInit(void){ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; for(i=0; i<ArraySize(winMutex_staticMutexes); i++){ -#if SQLITE_OS_WINRT - InitializeCriticalSectionEx(&winMutex_staticMutexes[i].mutex, 0, 0); -#else InitializeCriticalSection(&winMutex_staticMutexes[i].mutex); -#endif } winMutex_isInit = 1; }else{ @@ -30550,11 +31272,7 @@ static sqlite3_mutex *winMutexAlloc(int iType){ p->trace = 1; #endif #endif -#if SQLITE_OS_WINRT - InitializeCriticalSectionEx(&p->mutex, 0, 0); -#else InitializeCriticalSection(&p->mutex); -#endif } break; } @@ -31019,27 +31737,6 @@ static void mallocWithAlarm(int n, void **pp){ } /* -** Maximum size of any single memory allocation. -** -** This is not a limit on the total amount of memory used. This is -** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). -** -** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 -** This provides a 256-byte safety margin for defense against 32-bit -** signed integer overflow bugs when computing memory allocation sizes. -** Paranoid applications might want to reduce the maximum allocation size -** further for an even larger safety margin. 0x3fffffff or 0x0fffffff -** or even smaller would be reasonable upper bounds on the size of a memory -** allocations for most applications. -*/ -#ifndef SQLITE_MAX_ALLOCATION_SIZE -# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 -#endif -#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 -# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 -#endif - -/* ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ @@ -31262,8 +31959,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3_free(pOld); /* IMP: R-26507-47431 */ return 0; } - if( nBytes>=0x7fffff00 ){ - /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */ + if( nBytes>SQLITE_MAX_ALLOCATION_SIZE ){ return 0; } nOld = sqlite3MallocSize(pOld); @@ -31706,6 +32402,7 @@ typedef struct et_info { /* Information about each format field */ etByte type; /* Conversion paradigm */ etByte charset; /* Offset into aDigits[] of the digits string */ etByte prefix; /* Offset into aPrefix[] of the prefix string */ + char iNxt; /* Next with same hash, or 0 for end of chain */ } et_info; /* @@ -31714,44 +32411,61 @@ typedef struct et_info { /* Information about each format field */ #define FLAG_SIGNED 1 /* True if the value to convert is signed */ #define FLAG_STRING 4 /* Allow infinite precision */ - /* -** The following table is searched linearly, so it is good to put the -** most frequently used conversion types first. +** The table is searched by hash. In the case of %C where C is the character +** and that character has ASCII value j, then the hash is j%23. +** +** The order of the entries in fmtinfo[] and the hash chain was entered +** manually, but based on the output of the following TCL script: */ +#if 0 /***** Beginning of script ******/ +foreach c {d s g z q Q w c o u x X f e E G i n % p T S r} { + scan $c %c x + set n($c) $x +} +set mx [llength [array names n]] +puts "count: $mx" + +set mx 27 +puts "*********** mx=$mx ************" +for {set r 0} {$r<$mx} {incr r} { + puts -nonewline [format %2d: $r] + foreach c [array names n] { + if {($n($c))%$mx==$r} {puts -nonewline " $c"} + } + puts "" +} +#endif /***** End of script ********/ + static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; -static const et_info fmtinfo[] = { - { 'd', 10, 1, etDECIMAL, 0, 0 }, - { 's', 0, 4, etSTRING, 0, 0 }, - { 'g', 0, 1, etGENERIC, 30, 0 }, - { 'z', 0, 4, etDYNSTRING, 0, 0 }, - { 'q', 0, 4, etESCAPE_q, 0, 0 }, - { 'Q', 0, 4, etESCAPE_Q, 0, 0 }, - { 'w', 0, 4, etESCAPE_w, 0, 0 }, - { 'c', 0, 0, etCHARX, 0, 0 }, - { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etDECIMAL, 0, 0 }, - { 'x', 16, 0, etRADIX, 16, 1 }, - { 'X', 16, 0, etRADIX, 0, 4 }, -#ifndef SQLITE_OMIT_FLOATING_POINT - { 'f', 0, 1, etFLOAT, 0, 0 }, - { 'e', 0, 1, etEXP, 30, 0 }, - { 'E', 0, 1, etEXP, 14, 0 }, - { 'G', 0, 1, etGENERIC, 14, 0 }, -#endif - { 'i', 10, 1, etDECIMAL, 0, 0 }, - { 'n', 0, 0, etSIZE, 0, 0 }, - { '%', 0, 0, etPERCENT, 0, 0 }, - { 'p', 16, 0, etPOINTER, 0, 1 }, - - /* All the rest are undocumented and are for internal use only */ - { 'T', 0, 0, etTOKEN, 0, 0 }, - { 'S', 0, 0, etSRCITEM, 0, 0 }, - { 'r', 10, 1, etORDINAL, 0, 0 }, +static const et_info fmtinfo[23] = { + /* 0 */ { 's', 0, 4, etSTRING, 0, 0, 1 }, + /* 1 */ { 'E', 0, 1, etEXP, 14, 0, 0 }, /* Hash: 0 */ + /* 2 */ { 'u', 10, 0, etDECIMAL, 0, 0, 3 }, + /* 3 */ { 'G', 0, 1, etGENERIC, 14, 0, 0 }, /* Hash: 2 */ + /* 4 */ { 'w', 0, 4, etESCAPE_w, 0, 0, 0 }, + /* 5 */ { 'x', 16, 0, etRADIX, 16, 1, 0 }, + /* 6 */ { 'c', 0, 0, etCHARX, 0, 0, 0 }, /* Hash: 7 */ + /* 7 */ { 'z', 0, 4, etDYNSTRING, 0, 0, 6 }, + /* 8 */ { 'd', 10, 1, etDECIMAL, 0, 0, 0 }, + /* 9 */ { 'e', 0, 1, etEXP, 30, 0, 0 }, + /* 10 */ { 'f', 0, 1, etFLOAT, 0, 0, 0 }, + /* 11 */ { 'g', 0, 1, etGENERIC, 30, 0, 0 }, + /* 12 */ { 'Q', 0, 4, etESCAPE_Q, 0, 0, 0 }, + /* 13 */ { 'i', 10, 1, etDECIMAL, 0, 0, 0 }, + /* 14 */ { '%', 0, 0, etPERCENT, 0, 0, 16 }, + /* 15 */ { 'T', 0, 0, etTOKEN, 0, 0, 0 }, + /* 16 */ { 'S', 0, 0, etSRCITEM, 0, 0, 0 }, /* Hash: 14 */ + /* 17 */ { 'X', 16, 0, etRADIX, 0, 4, 0 }, /* Hash: 19 */ + /* 18 */ { 'n', 0, 0, etSIZE, 0, 0, 0 }, + /* 19 */ { 'o', 8, 0, etRADIX, 0, 2, 17 }, + /* 20 */ { 'p', 16, 0, etPOINTER, 0, 1, 0 }, + /* 21 */ { 'q', 0, 4, etESCAPE_q, 0, 0, 0 }, + /* 22 */ { 'r', 10, 1, etORDINAL, 0, 0, 0 } }; -/* Notes: +/* Additional Notes: ** ** %S Takes a pointer to SrcItem. Shows name or database.name ** %!S Like %S but prefer the zName over the zAlias @@ -31799,7 +32513,7 @@ static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ sqlite3StrAccumSetError(pAccum, SQLITE_TOOBIG); return 0; } - z = sqlite3DbMallocRaw(pAccum->db, n); + z = sqlite3_malloc(n); if( z==0 ){ sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); } @@ -31878,7 +32592,10 @@ SQLITE_API void sqlite3_str_vappendf( #if HAVE_STRCHRNUL fmt = strchrnul(fmt, '%'); #else - do{ fmt++; }while( *fmt && *fmt != '%' ); + fmt = strchr(fmt, '%'); + if( fmt==0 ){ + fmt = bufpt + strlen(bufpt); + } #endif sqlite3_str_append(pAccum, bufpt, (int)(fmt - bufpt)); if( *fmt==0 ) break; @@ -31992,6 +32709,9 @@ SQLITE_API void sqlite3_str_vappendf( }while( !done && (c=(*++fmt))!=0 ); /* Fetch the info entry for the field */ +#ifdef SQLITE_EBCDIC + /* The hash table only works for ASCII. For EBCDIC, we need to do + ** a linear search of the table */ infop = &fmtinfo[0]; xtype = etINVALID; for(idx=0; idx<ArraySize(fmtinfo); idx++){ @@ -32001,6 +32721,20 @@ SQLITE_API void sqlite3_str_vappendf( break; } } +#else + /* Fast hash-table lookup */ + assert( ArraySize(fmtinfo)==23 ); + idx = ((unsigned)c) % 23; + if( fmtinfo[idx].fmttype==c + || fmtinfo[idx = fmtinfo[idx].iNxt].fmttype==c + ){ + infop = &fmtinfo[idx]; + xtype = infop->type; + }else{ + infop = &fmtinfo[0]; + xtype = etINVALID; + } +#endif /* ** At this point, variables are initialized as follows: @@ -32068,6 +32802,14 @@ SQLITE_API void sqlite3_str_vappendf( } prefix = 0; } + +#if WHERETRACE_ENABLED + if( xtype==etPOINTER && sqlite3WhereTrace & 0x100000 ) longvalue = 0; +#endif +#if TREETRACE_ENABLED + if( xtype==etPOINTER && sqlite3TreeTrace & 0x100000 ) longvalue = 0; +#endif + if( longvalue==0 ) flag_alternateform = 0; if( flag_zeropad && precision<width-(prefix!=0) ){ precision = width-(prefix!=0); @@ -32102,9 +32844,11 @@ SQLITE_API void sqlite3_str_vappendf( }while( longvalue>0 ); } length = (int)(&zOut[nOut-1]-bufpt); - while( precision>length ){ - *(--bufpt) = '0'; /* Zero pad */ - length++; + if( precision>length ){ /* zero pad */ + int nn = precision-length; + bufpt -= nn; + memset(bufpt,'0',nn); + length = precision; } if( cThousand ){ int nn = (length - 1)/3; /* Number of "," to insert */ @@ -32135,6 +32879,7 @@ SQLITE_API void sqlite3_str_vappendf( FpDecode s; int iRound; int j; + i64 szBufNeeded; /* Size needed to hold the output */ if( bArgList ){ realvalue = getDoubleArg(pArgList); @@ -32155,7 +32900,7 @@ SQLITE_API void sqlite3_str_vappendf( }else{ iRound = precision+1; } - sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 20 : 16); if( s.isSpecial ){ if( s.isSpecial==2 ){ bufpt = flag_zeropad ? "null" : "NaN"; @@ -32180,7 +32925,21 @@ SQLITE_API void sqlite3_str_vappendf( } } if( s.sign=='-' ){ - prefix = '-'; + if( flag_alternateform + && !flag_prefix + && xtype==etFLOAT + && s.iDP<=iRound + ){ + /* Suppress the minus sign if all of the following are true: + ** * The value displayed is zero + ** * The '#' flag is used + ** * The '+' flag is not used, and + ** * The format is %f + */ + prefix = 0; + }else{ + prefix = '-'; + } }else{ prefix = flag_prefix; } @@ -32209,17 +32968,31 @@ SQLITE_API void sqlite3_str_vappendf( }else{ e2 = s.iDP - 1; } - bufpt = buf; - { - i64 szBufNeeded; /* Size of a temporary buffer needed */ - szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; - if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; - if( szBufNeeded > etBUFSIZE ){ - bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); - if( bufpt==0 ) return; + + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+10; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; + if( szBufNeeded + pAccum->nChar >= pAccum->nAlloc ){ + if( pAccum->mxAlloc==0 && pAccum->accError==0 ){ + /* Unable to allocate space in pAccum, perhaps because it + ** is coming from sqlite3_snprintf() or similar. We'll have + ** to render into temporary space and the memcpy() it over. */ + bufpt = sqlite3_malloc(szBufNeeded); + if( bufpt==0 ){ + sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); + return; + } + zExtra = bufpt; + }else if( sqlite3StrAccumEnlarge(pAccum, szBufNeeded)<szBufNeeded ){ + width = length = 0; + break; + }else{ + bufpt = pAccum->zText + pAccum->nChar; } + }else{ + bufpt = pAccum->zText + pAccum->nChar; } zOut = bufpt; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ @@ -32227,12 +33000,24 @@ SQLITE_API void sqlite3_str_vappendf( } /* Digits prior to the decimal point */ j = 0; + assert( s.n>0 ); if( e2<0 ){ *(bufpt++) = '0'; - }else{ + }else if( cThousand ){ for(; e2>=0; e2--){ *(bufpt++) = j<s.n ? s.z[j++] : '0'; - if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + if( (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + } + }else{ + j = e2+1; + if( j>s.n ) j = s.n; + memcpy(bufpt, s.z, j); + bufpt += j; + e2 -= j; + if( e2>=0 ){ + memset(bufpt, '0', e2+1); + bufpt += e2+1; + e2 = -1; } } /* The decimal point */ @@ -32241,12 +33026,26 @@ SQLITE_API void sqlite3_str_vappendf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0 && precision>0; precision--, e2++){ - *(bufpt++) = '0'; + if( e2<(-1) && precision>0 ){ + int nn = -1-e2; + if( nn>precision ) nn = precision; + memset(bufpt, '0', nn); + bufpt += nn; + precision -= nn; } /* Significant digits after the decimal point */ - while( (precision--)>0 ){ - *(bufpt++) = j<s.n ? s.z[j++] : '0'; + if( precision>0 ){ + int nn = s.n - j; + if( NEVER(nn>precision) ) nn = precision; + if( nn>0 ){ + memcpy(bufpt, s.z+j, nn); + bufpt += nn; + precision -= nn; + } + if( precision>0 && !flag_rtz ){ + memset(bufpt, '0', precision); + bufpt += precision; + } } /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ @@ -32276,27 +33075,39 @@ SQLITE_API void sqlite3_str_vappendf( *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ } - *bufpt = 0; - /* The converted number is in buf[] and zero terminated. Output it. - ** Note that the number is in the usual order, not reversed as with - ** integer conversions. */ length = (int)(bufpt-zOut); - bufpt = zOut; - - /* Special case: Add leading zeros if the flag_zeropad flag is - ** set and we are not left justified */ - if( flag_zeropad && !flag_leftjustify && length < width){ - int i; - int nPad = width - length; - for(i=width; i>=nPad; i--){ - bufpt[i] = bufpt[i-nPad]; + assert( length <= szBufNeeded ); + if( length<width ){ + i64 nPad = width - length; + if( flag_leftjustify ){ + memset(bufpt, ' ', nPad); + }else if( !flag_zeropad ){ + memmove(zOut+nPad, zOut, length); + memset(zOut, ' ', nPad); + }else{ + int adj = prefix!=0; + memmove(zOut+nPad+adj, zOut+adj, length-adj); + memset(zOut+adj, '0', nPad); } - i = prefix!=0; - while( nPad-- ) bufpt[i++] = '0'; length = width; } - break; + + if( zExtra==0 ){ + /* The result is being rendered directory into pAccum. This + ** is the command and fast case */ + pAccum->nChar += length; + zOut[length] = 0; + continue; + }else{ + /* We were unable to render directly into pAccum because we + ** couldn't allocate sufficient memory. We need to memcpy() + ** the rendering (or some prefix thereof) into the output + ** buffer. */ + bufpt[0] = 0; + bufpt = zExtra; + break; + } } case etSIZE: if( !bArgList ){ @@ -32340,10 +33151,9 @@ SQLITE_API void sqlite3_str_vappendf( i64 nCopyBytes; if( nPrior > precision-1 ) nPrior = precision - 1; nCopyBytes = length*nPrior; - if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ - sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + if( sqlite3StrAccumEnlargeIfNeeded(pAccum, nCopyBytes) ){ + break; } - if( pAccum->accError ) break; sqlite3_str_append(pAccum, &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); precision -= nPrior; @@ -32690,6 +33500,13 @@ SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ return (int)N; } +SQLITE_PRIVATE int sqlite3StrAccumEnlargeIfNeeded(StrAccum *p, i64 N){ + if( N + p->nChar >= p->nAlloc ){ + sqlite3StrAccumEnlarge(p, N); + } + return p->accError; +} + /* ** Append N copies of character c to the given string buffer. */ @@ -32820,6 +33637,14 @@ SQLITE_API int sqlite3_str_length(sqlite3_str *p){ return p ? p->nChar : 0; } +/* Truncate the text of the string to be no more than N bytes. */ +SQLITE_API void sqlite3_str_truncate(sqlite3_str *p, int N){ + if( p!=0 && N>=0 && (u32)N<p->nChar ){ + p->nChar = N; + p->zText[p->nChar] = 0; + } +} + /* Return the current value for p */ SQLITE_API char *sqlite3_str_value(sqlite3_str *p){ if( p==0 || p->nChar==0 ) return 0; @@ -32841,6 +33666,17 @@ SQLITE_API void sqlite3_str_reset(StrAccum *p){ } /* +** Destroy a dynamically allocate sqlite3_str object and all +** of its content, all in one call. +*/ +SQLITE_API void sqlite3_str_free(sqlite3_str *p){ + if( p!=0 && p!=&sqlite3OomStr ){ + sqlite3_str_reset(p); + sqlite3_free(p); + } +} + +/* ** Initialize a string accumulator. ** ** p: The accumulator to be initialized. @@ -33391,9 +34227,13 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) n = 0; if( pItem->fg.isSubquery ) n++; if( pItem->fg.isTabFunc ) n++; - if( pItem->fg.isUsing ) n++; + if( pItem->fg.isUsing || pItem->u3.pOn!=0 ) n++; if( pItem->fg.isUsing ){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); + }else if( pItem->u3.pOn!=0 ){ + sqlite3TreeViewItem(pView, "ON", (--n)>0); + sqlite3TreeViewExpr(pView, pItem->u3.pOn, 0); + sqlite3TreeViewPop(&pView); } if( pItem->fg.isSubquery ){ assert( n==1 ); @@ -34449,7 +35289,13 @@ SQLITE_PRIVATE void sqlite3TreeViewTrigger( SQLITE_PRIVATE void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } SQLITE_PRIVATE void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} SQLITE_PRIVATE void sqlite3ShowIdList(const IdList *p){ sqlite3TreeViewIdList(0,p,0,0); } -SQLITE_PRIVATE void sqlite3ShowSrcList(const SrcList *p){ sqlite3TreeViewSrcList(0,p); } +SQLITE_PRIVATE void sqlite3ShowSrcList(const SrcList *p){ + TreeView *pView = 0; + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "SRCLIST"); + sqlite3TreeViewSrcList(pView,p); + sqlite3TreeViewPop(&pView); +} SQLITE_PRIVATE void sqlite3ShowSelect(const Select *p){ sqlite3TreeViewSelect(0,p,0); } SQLITE_PRIVATE void sqlite3ShowWith(const With *p){ sqlite3TreeViewWith(0,p,0); } SQLITE_PRIVATE void sqlite3ShowUpsert(const Upsert *p){ sqlite3TreeViewUpsert(0,p,0); } @@ -34830,6 +35676,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); bRc = CloseHandle((HANDLE)p->tid); + (void)bRc; /* Prevent warning when assert() is a no-op */ assert( bRc ); } if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; @@ -35969,266 +36816,618 @@ SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ return h; } -/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) +#if !defined(SQLITE_DISABLE_INTRINSIC) \ + && (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__x86_64__) || defined(__aarch64__) || \ + (defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen>32))) +#define SQLITE_USE_UINT128 +#endif + +/* +** Two inputs are multiplied to get a 128-bit result. Write the +** lower 64-bits of the result into *pLo, and return the high-order +** 64 bits. +*/ +static u64 sqlite3Multiply128(u64 a, u64 b, u64 *pLo){ +#if defined(SQLITE_USE_UINT128) + __uint128_t r = (__uint128_t)a * b; + *pLo = (u64)r; + return (u64)(r>>64); +#elif defined(_WIN64) && !defined(SQLITE_DISABLE_INTRINSIC) + *pLo = a*b; + return __umulh(a, b); +#else + u64 a0 = (u32)a; + u64 a1 = a >> 32; + u64 b0 = (u32)b; + u64 b1 = b >> 32; + u64 a0b0 = a0 * b0; + u64 a1b1 = a1 * b1; + u64 a0b1 = a0 * b1; + u64 a1b0 = a1 * b0; + u64 t = (a0b0 >> 32) + (u32)a0b1 + (u32)a1b0; + *pLo = (a0b0 & UINT64_C(0xffffffff)) | (t << 32); + return a1b1 + (a0b1>>32) + (a1b0>>32) + (t>>32); +#endif +} + +/* +** A is an unsigned 96-bit integer formed by (a<<32)+aLo. +** B is an unsigned 64-bit integer. ** -** Reference: -** T. J. Dekker, "A Floating-Point Technique for Extending the -** Available Precision". 1971-07-26. +** Compute the upper 96 bits of 160-bit result of A*B. +** +** Write ((A*B)>>64 & 0xffffffff) (the middle 32 bits of A*B) +** into *pLo. Return the upper 64 bits of A*B. +** +** The lower 64 bits of A*B are discarded. */ -static void dekkerMul2(volatile double *x, double y, double yy){ - /* - ** The "volatile" keywords on parameter x[] and on local variables - ** below are needed force intermediate results to be truncated to - ** binary64 rather than be carried around in an extended-precision - ** format. The truncation is necessary for the Dekker algorithm to - ** work. Intel x86 floating point might omit the truncation without - ** the use of volatile. - */ - volatile double tx, ty, p, q, c, cc; - double hx, hy; - u64 m; - memcpy(&m, (void*)&x[0], 8); - m &= 0xfffffffffc000000LL; - memcpy(&hx, &m, 8); - tx = x[0] - hx; - memcpy(&m, &y, 8); - m &= 0xfffffffffc000000LL; - memcpy(&hy, &m, 8); - ty = y - hy; - p = hx*hy; - q = hx*ty + tx*hy; - c = p+q; - cc = p - c + q + tx*ty; - cc = x[0]*yy + x[1]*y + cc; - x[0] = c + cc; - x[1] = c - x[0]; - x[1] += cc; +static u64 sqlite3Multiply160(u64 a, u32 aLo, u64 b, u32 *pLo){ +#if defined(SQLITE_USE_UINT128) + __uint128_t r = (__uint128_t)a * b; + r += ((__uint128_t)aLo * b) >> 32; + *pLo = (r>>32)&0xffffffff; + return r>>64; +#elif defined(_WIN64) && !defined(SQLITE_DISABLE_INTRINSIC) + u64 r1_hi = __umulh(a,b); + u64 r1_lo = a*b; + u64 r2 = (__umulh((u64)aLo,b)<<32) + ((aLo*b)>>32); + u64 t = r1_lo + r2; + if( t<r1_lo ) r1_hi++; + *pLo = t>>32; + return r1_hi; +#else + u64 x2 = a>>32; + u64 x1 = a&0xffffffff; + u64 x0 = aLo; + u64 y1 = b>>32; + u64 y0 = b&0xffffffff; + u64 x2y1 = x2*y1; + u64 r4 = x2y1>>32; + u64 x2y0 = x2*y0; + u64 x1y1 = x1*y1; + u64 r3 = (x2y1 & 0xffffffff) + (x2y0 >>32) + (x1y1 >>32); + u64 x1y0 = x1*y0; + u64 x0y1 = x0*y1; + u64 r2 = (x2y0 & 0xffffffff) + (x1y1 & 0xffffffff) + + (x1y0 >>32) + (x0y1>>32); + u64 x0y0 = x0*y0; + u64 r1 = (x1y0 & 0xffffffff) + (x0y1 & 0xffffffff) + + (x0y0 >>32); + r2 += r1>>32; + r3 += r2>>32; + *pLo = r2&0xffffffff; + return (r4<<32) + r3; +#endif } +#undef SQLITE_USE_UINT128 + /* -** The string z[] is an text representation of a real number. -** Convert this string to a double and write it into *pResult. +** Return a u64 with the N-th bit set. +*/ +#define U64_BIT(N) (((u64)1)<<(N)) + +/* +** Range of powers of 10 that we need to deal with when converting +** IEEE754 doubles to and from decimal. +*/ +#define POWERSOF10_FIRST (-348) +#define POWERSOF10_LAST (+347) + +/* +** For any p between -348 and +347, return the integer part of ** -** The string z[] is length bytes in length (bytes, not characters) and -** uses the encoding enc. The string is not necessarily zero-terminated. +** pow(10,p) * pow(2,63-pow10to2(p)) ** -** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. More specifically -** return -** 1 => The input string is a pure integer -** 2 or more => The input has a decimal point or eNNN clause -** 0 or less => The input string is not a valid number -** -1 => Not a valid number, but has a valid prefix which -** includes a decimal point and/or an eNNN clause +** Or, in other words, for any p in range, return the most significant +** 64 bits of pow(10,p). The pow(10,p) value is shifted left or right, +** as appropriate so the most significant 64 bits fit exactly into a +** 64-bit unsigned integer. ** -** Valid numbers are in one of these formats: +** Write into *pLo the next 32 significant bits of the answer after +** the first 64. ** -** [+-]digits[E[+-]digits] -** [+-]digits.[digits][E[+-]digits] -** [+-].digits[E[+-]digits] +** Algorithm: +** +** (1) For p between 0 and 26, return the value directly from the aBase[] +** lookup table. ** -** Leading and trailing whitespace is ignored for the purpose of determining -** validity. +** (2) For p outside the range 0 to 26, use aScale[] for the initial value +** then refine that result (if necessary) by a single multiplication +** against aBase[]. ** -** If some prefix of the input string is a valid number, this routine -** returns FALSE but it still converts the prefix and writes the result -** into *pResult. +** The constant tables aBase[], aScale[], and aScaleLo[] are generated +** by the C program at ../tool/mkfptab.c run with the --round option. */ -#if defined(_MSC_VER) -#pragma warning(disable : 4756) -#endif -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ -#ifndef SQLITE_OMIT_FLOATING_POINT - int incr; - const char *zEnd; - /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - u64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ - int eValid = 1; /* True exponent is either not used or is well-formed */ - int nDigit = 0; /* Number of digits processed */ - int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ - u64 s2; /* round-tripped significand */ - double rr[2]; - - assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); - *pResult = 0.0; /* Default return value, in case of an error */ - if( length==0 ) return 0; +static u64 powerOfTen(int p, u32 *pLo){ + static const u64 aBase[] = { + UINT64_C(0x8000000000000000), /* 0: 1.0e+0 << 63 */ + UINT64_C(0xa000000000000000), /* 1: 1.0e+1 << 60 */ + UINT64_C(0xc800000000000000), /* 2: 1.0e+2 << 57 */ + UINT64_C(0xfa00000000000000), /* 3: 1.0e+3 << 54 */ + UINT64_C(0x9c40000000000000), /* 4: 1.0e+4 << 50 */ + UINT64_C(0xc350000000000000), /* 5: 1.0e+5 << 47 */ + UINT64_C(0xf424000000000000), /* 6: 1.0e+6 << 44 */ + UINT64_C(0x9896800000000000), /* 7: 1.0e+7 << 40 */ + UINT64_C(0xbebc200000000000), /* 8: 1.0e+8 << 37 */ + UINT64_C(0xee6b280000000000), /* 9: 1.0e+9 << 34 */ + UINT64_C(0x9502f90000000000), /* 10: 1.0e+10 << 30 */ + UINT64_C(0xba43b74000000000), /* 11: 1.0e+11 << 27 */ + UINT64_C(0xe8d4a51000000000), /* 12: 1.0e+12 << 24 */ + UINT64_C(0x9184e72a00000000), /* 13: 1.0e+13 << 20 */ + UINT64_C(0xb5e620f480000000), /* 14: 1.0e+14 << 17 */ + UINT64_C(0xe35fa931a0000000), /* 15: 1.0e+15 << 14 */ + UINT64_C(0x8e1bc9bf04000000), /* 16: 1.0e+16 << 10 */ + UINT64_C(0xb1a2bc2ec5000000), /* 17: 1.0e+17 << 7 */ + UINT64_C(0xde0b6b3a76400000), /* 18: 1.0e+18 << 4 */ + UINT64_C(0x8ac7230489e80000), /* 19: 1.0e+19 >> 0 */ + UINT64_C(0xad78ebc5ac620000), /* 20: 1.0e+20 >> 3 */ + UINT64_C(0xd8d726b7177a8000), /* 21: 1.0e+21 >> 6 */ + UINT64_C(0x878678326eac9000), /* 22: 1.0e+22 >> 10 */ + UINT64_C(0xa968163f0a57b400), /* 23: 1.0e+23 >> 13 */ + UINT64_C(0xd3c21bcecceda100), /* 24: 1.0e+24 >> 16 */ + UINT64_C(0x84595161401484a0), /* 25: 1.0e+25 >> 20 */ + UINT64_C(0xa56fa5b99019a5c8), /* 26: 1.0e+26 >> 23 */ + }; + static const u64 aScale[] = { + UINT64_C(0x8049a4ac0c5811ae), /* 0: 1.0e-351 << 1229 */ + UINT64_C(0xcf42894a5dce35ea), /* 1: 1.0e-324 << 1140 */ + UINT64_C(0xa76c582338ed2621), /* 2: 1.0e-297 << 1050 */ + UINT64_C(0x873e4f75e2224e68), /* 3: 1.0e-270 << 960 */ + UINT64_C(0xda7f5bf590966848), /* 4: 1.0e-243 << 871 */ + UINT64_C(0xb080392cc4349dec), /* 5: 1.0e-216 << 781 */ + UINT64_C(0x8e938662882af53e), /* 6: 1.0e-189 << 691 */ + UINT64_C(0xe65829b3046b0afa), /* 7: 1.0e-162 << 602 */ + UINT64_C(0xba121a4650e4ddeb), /* 8: 1.0e-135 << 512 */ + UINT64_C(0x964e858c91ba2655), /* 9: 1.0e-108 << 422 */ + UINT64_C(0xf2d56790ab41c2a2), /* 10: 1.0e-81 << 333 */ + UINT64_C(0xc428d05aa4751e4c), /* 11: 1.0e-54 << 243 */ + UINT64_C(0x9e74d1b791e07e48), /* 12: 1.0e-27 << 153 */ + UINT64_C(0xcccccccccccccccc), /* 13: 1.0e-1 << 67 (special case) */ + UINT64_C(0xcecb8f27f4200f3a), /* 14: 1.0e+27 >> 26 */ + UINT64_C(0xa70c3c40a64e6c51), /* 15: 1.0e+54 >> 116 */ + UINT64_C(0x86f0ac99b4e8dafd), /* 16: 1.0e+81 >> 206 */ + UINT64_C(0xda01ee641a708de9), /* 17: 1.0e+108 >> 295 */ + UINT64_C(0xb01ae745b101e9e4), /* 18: 1.0e+135 >> 385 */ + UINT64_C(0x8e41ade9fbebc27d), /* 19: 1.0e+162 >> 475 */ + UINT64_C(0xe5d3ef282a242e81), /* 20: 1.0e+189 >> 564 */ + UINT64_C(0xb9a74a0637ce2ee1), /* 21: 1.0e+216 >> 654 */ + UINT64_C(0x95f83d0a1fb69cd9), /* 22: 1.0e+243 >> 744 */ + UINT64_C(0xf24a01a73cf2dccf), /* 23: 1.0e+270 >> 833 */ + UINT64_C(0xc3b8358109e84f07), /* 24: 1.0e+297 >> 923 */ + UINT64_C(0x9e19db92b4e31ba9), /* 25: 1.0e+324 >> 1013 */ + }; + static const unsigned int aScaleLo[] = { + 0x205b896d, /* 0: 1.0e-351 << 1229 */ + 0x52064cad, /* 1: 1.0e-324 << 1140 */ + 0xaf2af2b8, /* 2: 1.0e-297 << 1050 */ + 0x5a7744a7, /* 3: 1.0e-270 << 960 */ + 0xaf39a475, /* 4: 1.0e-243 << 871 */ + 0xbd8d794e, /* 5: 1.0e-216 << 781 */ + 0x547eb47b, /* 6: 1.0e-189 << 691 */ + 0x0cb4a5a3, /* 7: 1.0e-162 << 602 */ + 0x92f34d62, /* 8: 1.0e-135 << 512 */ + 0x3a6a07f9, /* 9: 1.0e-108 << 422 */ + 0xfae27299, /* 10: 1.0e-81 << 333 */ + 0xaa97e14c, /* 11: 1.0e-54 << 243 */ + 0x775ea265, /* 12: 1.0e-27 << 153 */ + 0xcccccccc, /* 13: 1.0e-1 << 67 (special case) */ + 0x00000000, /* 14: 1.0e+27 >> 26 */ + 0x999090b6, /* 15: 1.0e+54 >> 116 */ + 0x69a028bb, /* 16: 1.0e+81 >> 206 */ + 0xe80e6f48, /* 17: 1.0e+108 >> 295 */ + 0x5ec05dd0, /* 18: 1.0e+135 >> 385 */ + 0x14588f14, /* 19: 1.0e+162 >> 475 */ + 0x8f1668c9, /* 20: 1.0e+189 >> 564 */ + 0x6d953e2c, /* 21: 1.0e+216 >> 654 */ + 0x4abdaf10, /* 22: 1.0e+243 >> 744 */ + 0xbc633b39, /* 23: 1.0e+270 >> 833 */ + 0x0a862f81, /* 24: 1.0e+297 >> 923 */ + 0x6c07a2c2, /* 25: 1.0e+324 >> 1013 */ + }; + int g, n; + u64 s, x; + u32 lo; - if( enc==SQLITE_UTF8 ){ - incr = 1; - zEnd = z + length; + assert( p>=POWERSOF10_FIRST && p<=POWERSOF10_LAST ); + if( p<0 ){ + if( p==(-1) ){ + *pLo = aScaleLo[13]; + return aScale[13]; + } + g = p/27; + n = p%27; + if( n ){ + g--; + n += 27; + } + }else if( p<27 ){ + *pLo = 0; + return aBase[p]; }else{ - int i; - incr = 2; - length &= ~1; - assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); - testcase( enc==SQLITE_UTF16LE ); - testcase( enc==SQLITE_UTF16BE ); - for(i=3-enc; i<length && z[i]==0; i+=2){} - if( i<length ) eType = -100; - zEnd = &z[i^1]; - z += (enc&1); + g = p/27; + n = p%27; + } + s = aScale[g+13]; + if( n==0 ){ + *pLo = aScaleLo[g+13]; + return s; + } + x = sqlite3Multiply160(s,aScaleLo[g+13],aBase[n],&lo); + if( (U64_BIT(63) & x)==0 ){ + x = x<<1 | ((lo>>31)&1); + lo = (lo<<1) | 1; } + *pLo = lo; + return x; +} + +/* +** pow10to2(x) computes floor(log2(pow(10,x))). +** pow2to10(y) computes floor(log10(pow(2,y))). +** +** Conceptually, pow10to2(p) converts a base-10 exponent p into +** a corresponding base-2 exponent, and pow2to10(e) converts a base-2 +** exponent into a base-10 exponent. +** +** The conversions are based on the observation that: +** +** ln(10.0)/ln(2.0) == 108853/32768 (approximately) +** ln(2.0)/ln(10.0) == 78913/262144 (approximately) +** +** These ratios are approximate, but they are accurate to 5 digits, +** which is close enough for the usage here. Right-shift is used +** for division so that rounding of negative numbers happens in the +** right direction. +*/ +static int pwr10to2(int p){ return (p*108853) >> 15; } +static int pwr2to10(int p){ return (p*78913) >> 18; } + +/* +** Count leading zeros for a 64-bit unsigned integer. +*/ +static int countLeadingZeros(u64 m){ +#if (defined(__GNUC__) || defined(__clang__)) \ + && !defined(SQLITE_DISABLE_INTRINSIC) + return __builtin_clzll(m); +#else + int n = 0; + if( m <= 0x00000000ffffffffULL) { n += 32; m <<= 32; } + if( m <= 0x0000ffffffffffffULL) { n += 16; m <<= 16; } + if( m <= 0x00ffffffffffffffULL) { n += 8; m <<= 8; } + if( m <= 0x0fffffffffffffffULL) { n += 4; m <<= 4; } + if( m <= 0x3fffffffffffffffULL) { n += 2; m <<= 2; } + if( m <= 0x7fffffffffffffffULL) { n += 1; } + return n; +#endif +} - /* skip leading spaces */ - while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; - if( z>=zEnd ) return 0; +/* +** Given m and e, which represent a quantity r == m*pow(2,e), +** return values *pD and *pP such that r == (*pD)*pow(10,*pP), +** approximately. *pD should contain at least n significant digits. +** +** The input m is required to have its highest bit set. In other words, +** m should be left-shifted, and e decremented, to maximize the value of m. +*/ +static void sqlite3Fp2Convert10(u64 m, int e, int n, u64 *pD, int *pP){ + int p; + u64 h, d1; + u32 d2; + assert( n>=1 && n<=18 ); + p = n - 1 - pwr2to10(e+63); + h = sqlite3Multiply128(m, powerOfTen(p,&d2), &d1); + assert( -(e + pwr10to2(p) + 2) >= 0 ); + assert( -(e + pwr10to2(p) + 1) <= 63 ); + if( n==18 ){ + h >>= -(e + pwr10to2(p) + 2); + *pD = (h + ((h<<1)&2))>>1; + }else{ + *pD = h >> -(e + pwr10to2(p) + 1); + } + *pP = -p; +} - /* get sign of significand */ - if( *z=='-' ){ - sign = -1; - z+=incr; - }else if( *z=='+' ){ - z+=incr; +/* +** Return an IEEE754 floating point value that approximates d*pow(10,p). +** +** The (current) algorithm is adapted from the work of Ross Cox at +** https://github.com/rsc/fpfmt +*/ +static double sqlite3Fp10Convert2(u64 d, int p){ + int b, lp, e, adj, s; + u32 pwr10l, mid1; + u64 pwr10h, x, hi, lo, sticky, u, m; + double r; + if( p<POWERSOF10_FIRST ) return 0.0; + if( p>POWERSOF10_LAST ) return INFINITY; + b = 64 - countLeadingZeros(d); + lp = pwr10to2(p); + e = 53 - b - lp; + if( e > 1074 ){ + if( e>=1130 ) return 0.0; + e = 1074; + } + s = -(e-(64-b) + lp + 3); + pwr10h = powerOfTen(p, &pwr10l); + if( pwr10l!=0 ){ + pwr10h++; + pwr10l = ~pwr10l; + } + x = d<<(64-b); + hi = sqlite3Multiply128(x,pwr10h,&lo); + mid1 = lo>>32; + sticky = 1; + if( (hi & (U64_BIT(s)-1))==0 ) { + u32 mid2 = sqlite3Multiply128(x,((u64)pwr10l)<<32,&lo)>>32; + sticky = (mid1-mid2 > 1); + hi -= mid1 < mid2; + } + u = (hi>>s) | sticky; + adj = (u >= U64_BIT(55)-2); + if( adj ){ + u = (u>>adj) | (u&1); + e -= adj; } + m = (u + 1 + ((u>>2)&1)) >> 2; + if( e<=(-972) ) return INFINITY; + if((m & U64_BIT(52)) != 0){ + m = (m & ~U64_BIT(52)) | ((u64)(1075-e)<<52); + } + memcpy(&r,&m,8); + return r; +} + +/* +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. +** +** z[] must be UTF-8 and zero-terminated. +** +** Return positive if the result is a valid real number (or integer) and +** zero or negative if the string is empty or contains extraneous text. +** Lower bits of the return value contain addition information about the +** parse: +** +** bit 0 => Set if any prefix of the input is valid. Clear if +** there is no prefix of the input that can be seen as +** a valid floating point number. +** bit 1 => Set if the input contains a decimal point or eNNN +** clause. Zero if the input is an integer. +** bit 2 => The input is exactly 0.0, not an underflow from +** some value near zero. +** bit 3 => Set if there are more than about 19 significant +** digits in the input. +** +** If the input contains a syntax error but begins with text that might +** be a valid number of some kind, then the result is negative. The +** result is only zero if no prefix of the input could be interpreted as +** a number. +** +** Leading and trailing whitespace is ignored. Valid numbers are in +** one of the formats below: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Algorithm sketch: Compute an unsigned 64-bit integer s and a base-10 +** exponent d such that the value encoding by the input is s*pow(10,d). +** Then invoke sqlite3Fp10Convert2() to calculated the closest possible +** IEEE754 double. The sign is added back afterwards, if the input string +** starts with a "-". The use of an unsigned 64-bit s mantissa means that +** only about the first 19 significant digits of the input can contribute +** to the result. This can result in suboptimal rounding decisions when +** correct rounding requires more than 19 input digits. For example, +** this routine renders "3500000000000000.2500001" as +** 3500000000000000.0 instead of 3500000000000000.5 because the decision +** to round up instead of using banker's rounding to round down is determined +** by the 23rd significant digit, which this routine ignores. It is not +** possible to do better without some kind of BigNum. +*/ +SQLITE_PRIVATE int sqlite3AtoF(const char *zIn, double *pResult){ +#ifndef SQLITE_OMIT_FLOATING_POINT + const unsigned char *z = (const unsigned char*)zIn; + int neg = 0; /* True for a negative value */ + u64 s = 0; /* mantissa */ + int d = 0; /* Value is s * pow(10,d) */ + int mState = 0; /* 1: digit seen 2: fp 4: hard-zero */ + unsigned v; /* Value of a single digit */ - /* copy max significant digits to significand */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - s = s*10 + (*z - '0'); - z+=incr; nDigit++; - if( s>=((LARGEST_UINT64-9)/10) ){ - /* skip non-significant significand digits - ** (increase exponent by d to shift decimal left) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; } + start_of_text: + if( (v = (unsigned)z[0] - '0')<10 ){ + parse_integer_part: + mState = 1; + s = v; + z++; + while( (v = (unsigned)z[0] - '0')<10 ){ + s = s*10 + v; + z++; + if( s>=(LARGEST_UINT64-9)/10 ){ + mState = 9; + while( sqlite3Isdigit(z[0]) ){ z++; d++; } + break; + } } + }else if( z[0]=='-' ){ + neg = 1; + z++; + if( (v = (unsigned)z[0] - '0')<10 ) goto parse_integer_part; + }else if( z[0]=='+' ){ + z++; + if( (v = (unsigned)z[0] - '0')<10 ) goto parse_integer_part; + }else if( sqlite3Isspace(z[0]) ){ + do{ z++; }while( sqlite3Isspace(z[0]) ); + goto start_of_text; + }else{ + s = 0; } - if( z>=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z+=incr; - eType++; - /* copy digits from after decimal to significand - ** (decrease exponent by d to shift decimal right) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - if( s<((LARGEST_UINT64-9)/10) ){ - s = s*10 + (*z - '0'); - d--; - nDigit++; - } - z+=incr; + z++; + if( sqlite3Isdigit(z[0]) ){ + mState |= 1; + do{ + if( s<(LARGEST_UINT64-9)/10 ){ + s = s*10 + z[0] - '0'; + d--; + }else{ + mState = 11; + } + }while( sqlite3Isdigit(*++z) ); + }else if( mState==0 ){ + *pResult = 0.0; + return 0; } + mState |= 2; + }else if( mState==0 ){ + *pResult = 0.0; + return 0; } - if( z>=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - z+=incr; - eValid = 0; - eType++; - - /* This branch is needed to avoid a (harmless) buffer overread. The - ** special comment alerts the mutation tester that the correct answer - ** is obtained even if the branch is omitted */ - if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ + int esign; + z++; /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z+=incr; - }else if( *z=='+' ){ - z+=incr; + z++; + }else{ + esign = +1; + if( *z=='+' ){ + z++; + } } /* copy digits to exponent */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - e = e<10000 ? (e*10 + (*z - '0')) : 10000; - z+=incr; - eValid = 1; + if( (v = (unsigned)z[0] - '0')<10 ){ + int exp = v; + z++; + mState |= 2; + while( (v = (unsigned)z[0] - '0')<10 ){ + exp = exp<10000 ? (exp*10 + v) : 10000; + z++; + } + d += esign*exp; + }else{ + z--; /* Leave z[0] at 'e' or '+' or '-', + ** so that the return is 0 or -1 */ } } - /* skip trailing spaces */ - while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; - -do_atof_calc: - /* Zero is a special case */ + /* Convert s*pow(10,d) into real */ if( s==0 ){ - *pResult = sign<0 ? -0.0 : +0.0; - goto atof_return; - } - - /* adjust exponent by d, and update sign */ - e = (e*esign) + d; - - /* Try to adjust the exponent to make it smaller */ - while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){ - s *= 10; - e--; - } - while( e<0 && (s%10)==0 ){ - s /= 10; - e++; - } - - rr[0] = (double)s; - assert( sizeof(s2)==sizeof(rr[0]) ); -#ifdef SQLITE_DEBUG - rr[1] = 18446744073709549568.0; - memcpy(&s2, &rr[1], sizeof(s2)); - assert( s2==0x43efffffffffffffLL ); -#endif - /* Largest double that can be safely converted to u64 - ** vvvvvvvvvvvvvvvvvvvvvv */ - if( rr[0]<=18446744073709549568.0 ){ - s2 = (u64)rr[0]; - rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); - }else{ - rr[1] = 0.0; - } - assert( rr[1]<=1.0e-10*rr[0] ); /* Equal only when rr[0]==0.0 */ - - if( e>0 ){ - while( e>=100 ){ - e -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( e>=10 ){ - e -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( e>=1 ){ - e -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + *pResult = 0.0; + mState |= 4; }else{ - while( e<=-100 ){ - e += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( e<=-10 ){ - e += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( e<=-1 ){ - e += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + *pResult = sqlite3Fp10Convert2(s,d); } - *pResult = rr[0]+rr[1]; - if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; - if( sign<0 ) *pResult = -*pResult; + if( neg ) *pResult = -*pResult; assert( !sqlite3IsNaN(*pResult) ); -atof_return: /* return true if number and no extra non-whitespace characters after */ - if( z==zEnd && nDigit>0 && eValid && eType>0 ){ - return eType; - }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ - return -1; - }else{ - return 0; + if( z[0]==0 ){ + return mState; + } + if( sqlite3Isspace(z[0]) ){ + do{ z++; }while( sqlite3Isspace(*z) ); + if( z[0]==0 ){ + return mState; + } } + return 0xfffffff0 | mState; #else - return !sqlite3Atoi64(z, pResult, length, enc); + return sqlite3Atoi64(z, pResult, strlen(z), SQLITE_UTF8)==0; #endif /* SQLITE_OMIT_FLOATING_POINT */ } -#if defined(_MSC_VER) -#pragma warning(default : 4756) + +/* +** Digit pairs used to convert a U64 or I64 into text, two digits +** at a time. +*/ +static const union { + char a[201]; + short int forceAlignment; +} sqlite3DigitPairs = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; + +/* +** ARMv6, ARMv7, PPC32 are known to not support hardware u64 division. +*/ +#if (defined(__arm__) && !defined(__aarch64__)) || \ + (defined(__ppc__) && !defined(__ppc64__)) +# define SQLITE_AVOID_U64_DIVIDE 1 #endif +#ifdef SQLITE_AVOID_U64_DIVIDE +/* +** Render an unsigned 64-bit integer as text onto the end of a 2-byte +** aligned buffer that is SQLITE_U64_DIGIT+1 bytes long. The last byte +** of the buffer will be filled with a \000 byte. +** +** Return the index into the buffer of the first byte. +** +** This routine is used on platforms where u64-division is slow because +** it is not available in hardware and has to be emulated in software. +** It seeks to minimize the number of u64 divisions and use u32 divisions +** instead. It is slower on platforms that have hardware u64 division, +** but much faster on platforms that do not. +*/ +static int sqlite3UInt64ToText(u64 v, char *zOut){ + u32 x32, kk; + int i; + zOut[SQLITE_U64_DIGITS] = 0; + i = SQLITE_U64_DIGITS; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[0]) ); + assert( TWO_BYTE_ALIGNMENT(zOut) ); + while( (v>>32)!=0 ){ + u32 y, x0, x1, y0, y1; + x32 = v % 100000000; + v = v / 100000000; + y = x32 % 10000; + x32 /= 10000; + x1 = x32 / 100; + x0 = x32 % 100; + y1 = y / 100; + y0 = y % 100; + assert( i>=8 ); + i -= 8; + *(u16*)(&zOut[i]) = *(u16*)&sqlite3DigitPairs.a[x1*2]; + *(u16*)(&zOut[i+2]) = *(u16*)&sqlite3DigitPairs.a[x0*2]; + *(u16*)(&zOut[i+4]) = *(u16*)&sqlite3DigitPairs.a[y1*2]; + *(u16*)(&zOut[i+6]) = *(u16*)&sqlite3DigitPairs.a[y0*2]; + } + x32 = v; + while( x32>=10 ){ + kk = x32 % 100; + x32 = x32 / 100; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk*2]) ); + assert( i>=2 ); + i -= 2; + assert( TWO_BYTE_ALIGNMENT(&zOut[i]) ); + *(u16*)(&zOut[i]) = *(u16*)&sqlite3DigitPairs.a[kk*2]; + } + if( x32 ){ + assert( i>0 ); + zOut[--i] = x32 + '0'; + } + return i; +} +#endif /* defined(SQLITE_AVOID_U64_DIVIDE) */ + /* ** Render an signed 64-bit integer as text. Store the result in zOut[] and ** return the length of the string that was stored, in bytes. The value @@ -36240,23 +37439,39 @@ atof_return: SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; - char zTemp[22]; - if( v<0 ){ - x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; - }else{ + union { + char a[SQLITE_U64_DIGITS+1]; + u16 forceAlignment; + } u; + if( v>0 ){ x = v; + }else if( v==0 ){ + zOut[0] = '0'; + zOut[1] = 0; + return 1; + }else{ + x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; } - i = sizeof(zTemp)-2; - zTemp[sizeof(zTemp)-1] = 0; - while( 1 /*exit-by-break*/ ){ - zTemp[i] = (x%10) + '0'; - x = x/10; - if( x==0 ) break; - i--; - }; - if( v<0 ) zTemp[--i] = '-'; - memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); - return sizeof(zTemp)-1-i; +#ifdef SQLITE_AVOID_U64_DIVIDE + i = sqlite3UInt64ToText(x, u.a); +#else + i = sizeof(u.a)-1; + u.a[i] = 0; + while( x>=10 ){ + int kk = (x%100)*2; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); + assert( TWO_BYTE_ALIGNMENT(&u.a[i-2]) ); + *(u16*)(&u.a[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; + i -= 2; + x /= 100; + } + if( x ){ + u.a[--i] = x + '0'; + } +#endif /* SQLITE_AVOID_U64_DIVIDE */ + if( v<0 ) u.a[--i] = '-'; + memcpy(zOut, &u.a[i], sizeof(u.a)-i); + return sizeof(u.a)-1-i; } /* @@ -36310,8 +37525,8 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc int incr; u64 u = 0; int neg = 0; /* assume positive */ - int i; - int c = 0; + int i, j; + unsigned int c = 0; int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ int rc; /* Baseline return code */ const char *zStart; @@ -36339,8 +37554,8 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc } zStart = zNum; while( zNum<zEnd && zNum[0]=='0' ){ zNum+=incr; } /* Skip leading zeros. */ - for(i=0; &zNum[i]<zEnd && (c=zNum[i])>='0' && c<='9'; i+=incr){ - u = u*10 + c - '0'; + for(i=0; &zNum[i]<zEnd && (c=(unsigned)zNum[i]-'0')<=9; i+=incr){ + u = u*10 + c; } testcase( i==18*incr ); testcase( i==19*incr ); @@ -36377,14 +37592,14 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc return rc; }else{ /* zNum is a 19-digit numbers. Compare it against 9223372036854775808. */ - c = i>19*incr ? 1 : compare2pow63(zNum, incr); - if( c<0 ){ + j = i>19*incr ? 1 : compare2pow63(zNum, incr); + if( j<0 ){ /* zNum is less than 9223372036854775808 so it fits */ assert( u<=LARGEST_INT64 ); return rc; }else{ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; - if( c>0 ){ + if( j>0 ){ /* zNum is greater than 9223372036854775808 so it overflows */ return 2; }else{ @@ -36513,7 +37728,7 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ ** representation. ** ** If iRound<=0 then round to -iRound significant digits to the -** the left of the decimal point, or to a maximum of mxRound total +** the right of the decimal point, or to a maximum of mxRound total ** significant digits. ** ** If iRound>0 round to min(iRound,mxRound) significant digits total. @@ -36526,13 +37741,14 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ ** The p->z[] array is *not* zero-terminated. */ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ - int i; - u64 v; - int e, exp = 0; - double rr[2]; + int i; /* Index into zBuf[] where to put next character */ + int n; /* Number of digits */ + u64 v; /* mantissa */ + int e, exp = 0; /* Base-2 and base-10 exponent */ + char *zBuf; /* Local alias for p->zBuf */ + char *z; /* Local alias for p->z */ p->isSpecial = 0; - p->z = p->zBuf; assert( mxRound>0 ); /* Convert negative numbers to positive. Deal with Infinity, 0.0, and @@ -36550,78 +37766,100 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou p->sign = '+'; } memcpy(&v,&r,8); - e = v>>52; - if( (e&0x7ff)==0x7ff ){ + e = (v>>52)&0x7ff; + if( e==0x7ff ){ p->isSpecial = 1 + (v!=0x7ff0000000000000LL); p->n = 0; p->iDP = 0; + p->z = p->zBuf; return; } - - /* Multiply r by powers of ten until it lands somewhere in between - ** 1.0e+19 and 1.0e+17. - ** - ** Use Dekker-style double-double computation to increase the - ** precision. - ** - ** The error terms on constants like 1.0e+100 computed using the - ** decimal extension, for example as follows: - ** - ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); - */ - rr[0] = r; - rr[1] = 0.0; - if( rr[0]>9.223372036854774784e+18 ){ - while( rr[0]>9.223372036854774784e+118 ){ - exp += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( rr[0]>9.223372036854774784e+28 ){ - exp += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( rr[0]>9.223372036854774784e+18 ){ - exp += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + v &= 0x000fffffffffffffULL; + if( e==0 ){ + int nn = countLeadingZeros(v); + v <<= nn; + e = -1074 - nn; }else{ - while( rr[0]<9.223372036854774784e-83 ){ - exp -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( rr[0]<9.223372036854774784e+07 ){ - exp -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( rr[0]<9.22337203685477478e+17 ){ - exp -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + v = (v<<11) | U64_BIT(63); + e -= 1086; } - v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; + sqlite3Fp2Convert10(v, e, (iRound<=0||iRound>=18)?18:iRound+1, &v, &exp); - /* Extract significant digits. */ - i = sizeof(p->zBuf)-1; + /* Extract significant digits, start at the right-most slot in p->zBuf + ** and working back to the right. "i" keeps track of the next slot in + ** which to store a digit. */ + assert( sizeof(p->zBuf)==SQLITE_U64_DIGITS+1 ); assert( v>0 ); - while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } - assert( i>=0 && i<sizeof(p->zBuf)-1 ); - p->n = sizeof(p->zBuf) - 1 - i; - assert( p->n>0 ); - assert( p->n<sizeof(p->zBuf) ); - p->iDP = p->n + exp; + zBuf = p->zBuf; +#ifdef SQLITE_AVOID_U64_DIVIDE + i = sqlite3UInt64ToText(v, zBuf); +#else + i = SQLITE_U64_DIGITS; + while( v>=10 ){ + int kk = (v%100)*2; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); + assert( TWO_BYTE_ALIGNMENT(&zBuf[i]) ); + assert( i-2>=0 ); + *(u16*)(&zBuf[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; + i -= 2; + v /= 100; + } + if( v ){ + assert( v<10 ); + assert( i>0 ); + zBuf[--i] = v + '0'; + } +#endif /* SQLITE_AVOID_U64_DIVIDE */ + assert( i>=0 && i<SQLITE_U64_DIGITS ); + n = SQLITE_U64_DIGITS - i; /* Total number of digits extracted */ + assert( n>0 ); + assert( n<=SQLITE_U64_DIGITS ); + p->iDP = n + exp; if( iRound<=0 ){ iRound = p->iDP - iRound; - if( iRound==0 && p->zBuf[i+1]>='5' ){ + if( iRound==0 && zBuf[i]>='5' ){ iRound = 1; - p->zBuf[i--] = '0'; - p->n++; + zBuf[--i] = '0'; + n++; p->iDP++; } } - if( iRound>0 && (iRound<p->n || p->n>mxRound) ){ - char *z = &p->zBuf[i+1]; + z = &zBuf[i]; /* z points to the first digit */ + if( iRound>0 && (iRound<n || n>mxRound) ){ if( iRound>mxRound ) iRound = mxRound; - p->n = iRound; + if( iRound==17 ){ + /* If the precision is exactly 17, which only happens with the "!" + ** flag (ex: "%!.17g") then try to reduce the precision if that + ** yields text that will round-trip to the original floating-point. + ** value. Thus, for exaple, 49.47 will render as 49.47, rather than + ** as 49.469999999999999. */ + if( z[15]=='9' && z[14]=='9' ){ + int jj, kk; + u64 v2; + for(jj=14; jj>0 && z[jj-1]=='9'; jj--){} + if( jj==0 ){ + v2 = 1; + }else{ + v2 = z[0] - '0'; + for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; + v2++; + } + if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ + iRound = jj+1; + } + }else if( p->iDP>=n || (z[15]=='0' && z[14]=='0' && z[13]=='0') ){ + int jj, kk; + u64 v2; + assert( z[0]!='0' ); + for(jj=13; z[jj-1]=='0'; jj--){} + v2 = z[0] - '0'; + for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; + if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ + iRound = jj+1; + } + } + } + n = iRound; if( z[iRound]>='5' ){ int j = iRound-1; while( 1 /*exit-by-break*/ ){ @@ -36629,8 +37867,9 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou if( z[j]<='9' ) break; z[j] = '0'; if( j==0 ){ - p->z[i--] = '1'; - p->n++; + z--; + z[0] = '1'; + n++; p->iDP++; break; }else{ @@ -36639,13 +37878,13 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou } } } - p->z = &p->zBuf[i+1]; - assert( i+p->n < sizeof(p->zBuf) ); - assert( p->n>0 ); - while( p->z[p->n-1]=='0' ){ - p->n--; - assert( p->n>0 ); + assert( n>0 ); + while( z[n-1]=='0' ){ + n--; + assert( n>0 ); } + p->n = n; + p->z = z; } /* @@ -37648,6 +38887,7 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ return 0; } + /************** End of hash.c ************************************************/ /************** Begin file opcodes.c *****************************************/ /* Automatically generated. Do not edit */ @@ -37699,20 +38939,20 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 34 */ "SorterSort" OpHelp(""), /* 35 */ "Sort" OpHelp(""), /* 36 */ "Rewind" OpHelp(""), - /* 37 */ "SorterNext" OpHelp(""), - /* 38 */ "Prev" OpHelp(""), - /* 39 */ "Next" OpHelp(""), - /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 37 */ "IfEmpty" OpHelp("if( empty(P1) ) goto P2"), + /* 38 */ "SorterNext" OpHelp(""), + /* 39 */ "Prev" OpHelp(""), + /* 40 */ "Next" OpHelp(""), + /* 41 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxGT" OpHelp("key=r[P3@P4]"), /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 48 */ "Program" OpHelp(""), - /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 50 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 45 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 46 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 47 */ "IFindKey" OpHelp(""), + /* 48 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 49 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 50 */ "Program" OpHelp(""), /* 51 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 52 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 53 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), @@ -37722,49 +38962,49 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 57 */ "Lt" OpHelp("IF r[P3]<r[P1]"), /* 58 */ "Ge" OpHelp("IF r[P3]>=r[P1]"), /* 59 */ "ElseEq" OpHelp(""), - /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 62 */ "IncrVacuum" OpHelp(""), - /* 63 */ "VNext" OpHelp(""), - /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), - /* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), - /* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), - /* 67 */ "Return" OpHelp(""), - /* 68 */ "EndCoroutine" OpHelp(""), - /* 69 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 70 */ "Halt" OpHelp(""), - /* 71 */ "Integer" OpHelp("r[P2]=P1"), - /* 72 */ "Int64" OpHelp("r[P2]=P4"), - /* 73 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 74 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), - /* 75 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 76 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 77 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1)"), - /* 79 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 80 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 81 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 82 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 83 */ "FkCheck" OpHelp(""), - /* 84 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 85 */ "CollSeq" OpHelp(""), - /* 86 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 87 */ "RealAffinity" OpHelp(""), - /* 88 */ "Cast" OpHelp("affinity(r[P1])"), - /* 89 */ "Permutation" OpHelp(""), - /* 90 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 91 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 92 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), - /* 93 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 94 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), - /* 95 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), - /* 96 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 97 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 98 */ "Count" OpHelp("r[P2]=count()"), - /* 99 */ "ReadCookie" OpHelp(""), - /* 100 */ "SetCookie" OpHelp(""), - /* 101 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 102 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 60 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 61 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 62 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 63 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 64 */ "IncrVacuum" OpHelp(""), + /* 65 */ "VNext" OpHelp(""), + /* 66 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), + /* 67 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 68 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 69 */ "Return" OpHelp(""), + /* 70 */ "EndCoroutine" OpHelp(""), + /* 71 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 72 */ "Halt" OpHelp(""), + /* 73 */ "Integer" OpHelp("r[P2]=P1"), + /* 74 */ "Int64" OpHelp("r[P2]=P4"), + /* 75 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 76 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), + /* 77 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 78 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 79 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 80 */ "Variable" OpHelp("r[P2]=parameter(P1)"), + /* 81 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 82 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 83 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 84 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 85 */ "FkCheck" OpHelp(""), + /* 86 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 87 */ "CollSeq" OpHelp(""), + /* 88 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 89 */ "RealAffinity" OpHelp(""), + /* 90 */ "Cast" OpHelp("affinity(r[P1])"), + /* 91 */ "Permutation" OpHelp(""), + /* 92 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 93 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 94 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 95 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 96 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), + /* 97 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 98 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 99 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 100 */ "Count" OpHelp("r[P2]=count()"), + /* 101 */ "ReadCookie" OpHelp(""), + /* 102 */ "SetCookie" OpHelp(""), /* 103 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 104 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 105 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<<r[P1]"), @@ -37775,83 +39015,85 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 110 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), /* 111 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), /* 112 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 113 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 114 */ "OpenDup" OpHelp(""), + /* 113 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 114 */ "OpenRead" OpHelp("root=P2 iDb=P3"), /* 115 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), - /* 116 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 117 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 116 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 117 */ "OpenDup" OpHelp(""), /* 118 */ "String8" OpHelp("r[P2]='P4'"), - /* 119 */ "SorterOpen" OpHelp(""), - /* 120 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 121 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 122 */ "Close" OpHelp(""), - /* 123 */ "ColumnsUsed" OpHelp(""), - /* 124 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), - /* 125 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), - /* 126 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 127 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 128 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 129 */ "RowCell" OpHelp(""), - /* 130 */ "Delete" OpHelp(""), - /* 131 */ "ResetCount" OpHelp(""), - /* 132 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 133 */ "SorterData" OpHelp("r[P2]=data"), - /* 134 */ "RowData" OpHelp("r[P2]=data"), - /* 135 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), - /* 136 */ "NullRow" OpHelp(""), - /* 137 */ "SeekEnd" OpHelp(""), - /* 138 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 139 */ "SorterInsert" OpHelp("key=r[P2]"), - /* 140 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 141 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), - /* 142 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 143 */ "FinishSeek" OpHelp(""), - /* 144 */ "Destroy" OpHelp(""), - /* 145 */ "Clear" OpHelp(""), - /* 146 */ "ResetSorter" OpHelp(""), - /* 147 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), - /* 148 */ "SqlExec" OpHelp(""), - /* 149 */ "ParseSchema" OpHelp(""), - /* 150 */ "LoadAnalysis" OpHelp(""), - /* 151 */ "DropTable" OpHelp(""), - /* 152 */ "DropIndex" OpHelp(""), - /* 153 */ "DropTrigger" OpHelp(""), + /* 119 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 120 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 121 */ "SorterOpen" OpHelp(""), + /* 122 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 123 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 124 */ "Close" OpHelp(""), + /* 125 */ "ColumnsUsed" OpHelp(""), + /* 126 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 127 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 128 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 129 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 130 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 131 */ "RowCell" OpHelp(""), + /* 132 */ "Delete" OpHelp(""), + /* 133 */ "ResetCount" OpHelp(""), + /* 134 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 135 */ "SorterData" OpHelp("r[P2]=data"), + /* 136 */ "RowData" OpHelp("r[P2]=data"), + /* 137 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), + /* 138 */ "NullRow" OpHelp(""), + /* 139 */ "SeekEnd" OpHelp(""), + /* 140 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 141 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 142 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 143 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 144 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 145 */ "FinishSeek" OpHelp(""), + /* 146 */ "Destroy" OpHelp(""), + /* 147 */ "Clear" OpHelp(""), + /* 148 */ "ResetSorter" OpHelp(""), + /* 149 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 150 */ "SqlExec" OpHelp(""), + /* 151 */ "ParseSchema" OpHelp(""), + /* 152 */ "LoadAnalysis" OpHelp(""), + /* 153 */ "DropTable" OpHelp(""), /* 154 */ "Real" OpHelp("r[P2]=P4"), - /* 155 */ "IntegrityCk" OpHelp(""), - /* 156 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 157 */ "Param" OpHelp(""), - /* 158 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 159 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 160 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 161 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 162 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 163 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 164 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 165 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 166 */ "Expire" OpHelp(""), - /* 167 */ "CursorLock" OpHelp(""), - /* 168 */ "CursorUnlock" OpHelp(""), - /* 169 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 170 */ "VBegin" OpHelp(""), - /* 171 */ "VCreate" OpHelp(""), - /* 172 */ "VDestroy" OpHelp(""), - /* 173 */ "VOpen" OpHelp(""), - /* 174 */ "VCheck" OpHelp(""), - /* 175 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), - /* 176 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 177 */ "VRename" OpHelp(""), - /* 178 */ "Pagecount" OpHelp(""), - /* 179 */ "MaxPgcnt" OpHelp(""), - /* 180 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), - /* 181 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), - /* 182 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), - /* 183 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), - /* 184 */ "Trace" OpHelp(""), - /* 185 */ "CursorHint" OpHelp(""), - /* 186 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), - /* 187 */ "Noop" OpHelp(""), - /* 188 */ "Explain" OpHelp(""), - /* 189 */ "Abortable" OpHelp(""), + /* 155 */ "DropIndex" OpHelp(""), + /* 156 */ "DropTrigger" OpHelp(""), + /* 157 */ "IntegrityCk" OpHelp(""), + /* 158 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 159 */ "Param" OpHelp(""), + /* 160 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 161 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 162 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 163 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 164 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 165 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 166 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 167 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 168 */ "Expire" OpHelp(""), + /* 169 */ "CursorLock" OpHelp(""), + /* 170 */ "CursorUnlock" OpHelp(""), + /* 171 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 172 */ "VBegin" OpHelp(""), + /* 173 */ "VCreate" OpHelp(""), + /* 174 */ "VDestroy" OpHelp(""), + /* 175 */ "VOpen" OpHelp(""), + /* 176 */ "VCheck" OpHelp(""), + /* 177 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 178 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 179 */ "VRename" OpHelp(""), + /* 180 */ "Pagecount" OpHelp(""), + /* 181 */ "MaxPgcnt" OpHelp(""), + /* 182 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 183 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), + /* 184 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), + /* 185 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 186 */ "Trace" OpHelp(""), + /* 187 */ "CursorHint" OpHelp(""), + /* 188 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 189 */ "Noop" OpHelp(""), + /* 190 */ "Explain" OpHelp(""), + /* 191 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -37882,7 +39124,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** Debugging logic */ -/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +/* SQLITE_KV_TRACE() is used for tracing calls to kvrecord routines. */ #if 0 #define SQLITE_KV_TRACE(X) printf X #else @@ -37896,7 +39138,6 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #define SQLITE_KV_LOG(X) #endif - /* ** Forward declaration of objects used by this VFS implementation */ @@ -37904,6 +39145,11 @@ typedef struct KVVfsFile KVVfsFile; /* A single open file. There are only two files represented by this ** VFS - the database and the rollback journal. +** +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion member. */ struct KVVfsFile { sqlite3_file base; /* IO methods */ @@ -37953,7 +39199,7 @@ static int kvvfsCurrentTime(sqlite3_vfs*, double*); static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); static sqlite3_vfs sqlite3OsKvvfsObject = { - 1, /* iVersion */ + 2, /* iVersion */ sizeof(KVVfsFile), /* szOsFile */ 1024, /* mxPathname */ 0, /* pNext */ @@ -38029,23 +39275,37 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = { /* Forward declarations for the low-level storage engine */ -static int kvstorageWrite(const char*, const char *zKey, const char *zData); -static int kvstorageDelete(const char*, const char *zKey); -static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); -#define KVSTORAGE_KEY_SZ 32 +#ifndef SQLITE_WASM +/* In WASM builds these are implemented in JS. */ +static int kvrecordWrite(const char*, const char *zKey, const char *zData); +static int kvrecordDelete(const char*, const char *zKey); +static int kvrecordRead(const char*, const char *zKey, char *zBuf, int nBuf); +#endif +#ifndef KVRECORD_KEY_SZ +#define KVRECORD_KEY_SZ 32 +#endif /* Expand the key name with an appropriate prefix and put the result -** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least -** KVSTORAGE_KEY_SZ bytes. +** in zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** KVRECORD_KEY_SZ bytes. */ -static void kvstorageMakeKey( +static void kvrecordMakeKey( const char *zClass, const char *zKeyIn, char *zKeyOut ){ - sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); + assert( zKeyIn ); + assert( zKeyOut ); + assert( zClass ); + sqlite3_snprintf(KVRECORD_KEY_SZ, zKeyOut, "kvvfs-%s-%s", + zClass, zKeyIn); } +#ifndef SQLITE_WASM +/* In WASM builds do not define APIs which use fopen(), fwrite(), +** and the like because those APIs are a portability issue for +** WASM. +*/ /* Write content into a key. zClass is the particular namespace of the ** underlying key/value store to use - either "local" or "session". ** @@ -38053,14 +39313,14 @@ static void kvstorageMakeKey( ** ** Return the number of errors. */ -static int kvstorageWrite( +static int kvrecordWrite( const char *zClass, const char *zKey, const char *zData ){ FILE *fd; - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); fd = fopen(zXKey, "wb"); if( fd ){ SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, @@ -38078,9 +39338,9 @@ static int kvstorageWrite( ** namespace given by zClass. If the key does not previously exist, ** this routine is a no-op. */ -static int kvstorageDelete(const char *zClass, const char *zKey){ - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); +static int kvrecordDelete(const char *zClass, const char *zKey){ + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); unlink(zXKey); SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); return 0; @@ -38094,12 +39354,14 @@ static int kvstorageDelete(const char *zClass, const char *zKey){ ** ** Return the total number of bytes in the data, without truncation, and ** not counting the final zero terminator. Return -1 if the key does -** not exist. +** not exist or its key cannot be read. ** -** If nBuf<=0 then this routine simply returns the size of the data without -** actually reading it. +** If nBuf<=0 then this routine simply returns the size of the data +** without actually reading it. Similarly, if nBuf==1 then it +** zero-terminates zBuf at zBuf[0] and returns the size of the data +** without reading it. */ -static int kvstorageRead( +static int kvrecordRead( const char *zClass, const char *zKey, char *zBuf, @@ -38107,8 +39369,8 @@ static int kvstorageRead( ){ FILE *fd; struct stat buf; - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); if( access(zXKey, R_OK)!=0 || stat(zXKey, &buf)!=0 || !S_ISREG(buf.st_mode) @@ -38140,26 +39402,36 @@ static int kvstorageRead( return (int)n; } } +#endif /* #ifndef SQLITE_WASM */ + /* ** An internal level of indirection which enables us to replace the ** kvvfs i/o methods with JavaScript implementations in WASM builds. ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in -** sqlite3_wasm_enum_json(). There are no binary compatibility -** concerns, so it does not need an iVersion member. This file is -** necessarily always compiled together with sqlite3_wasm_enum_json(), -** and JS code dynamically creates the mapping of members based on -** that JSON description. +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { - int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); - int (*xWrite)(const char *zClass, const char *zKey, const char *zData); - int (*xDelete)(const char *zClass, const char *zKey); + int (*xRcrdRead)(const char*, const char *zKey, char *zBuf, int nBuf); + int (*xRcrdWrite)(const char*, const char *zKey, const char *zData); + int (*xRcrdDelete)(const char*, const char *zKey); const int nKeySize; + const int nBufferSize; +#ifndef SQLITE_WASM +# define MAYBE_CONST const +#else +# define MAYBE_CONST +#endif + MAYBE_CONST sqlite3_vfs * pVfs; + MAYBE_CONST sqlite3_io_methods *pIoDb; + MAYBE_CONST sqlite3_io_methods *pIoJrnl; +#undef MAYBE_CONST }; + /* ** This object holds the kvvfs I/O methods which may be swapped out ** for JavaScript-side implementations in WASM builds. In such builds @@ -38167,17 +39439,27 @@ struct sqlite3_kvvfs_methods { ** the compiler can hopefully optimize this level of indirection out. ** That said, kvvfs is intended primarily for use in WASM builds. ** -** Note that this is not explicitly flagged as static because the -** amalgamation build will tag it with SQLITE_PRIVATE. +** This is not explicitly flagged as static because the amalgamation +** build will tag it with SQLITE_PRIVATE. */ #ifndef SQLITE_WASM const #endif SQLITE_PRIVATE sqlite3_kvvfs_methods sqlite3KvvfsMethods = { -kvstorageRead, -kvstorageWrite, -kvstorageDelete, -KVSTORAGE_KEY_SZ +#ifndef SQLITE_WASM + .xRcrdRead = kvrecordRead, + .xRcrdWrite = kvrecordWrite, + .xRcrdDelete = kvrecordDelete, +#else + .xRcrdRead = 0, + .xRcrdWrite = 0, + .xRcrdDelete = 0, +#endif + .nKeySize = KVRECORD_KEY_SZ, + .nBufferSize = SQLITE_KVOS_SZ, + .pVfs = &sqlite3OsKvvfsObject, + .pIoDb = &kvvfs_db_io_methods, + .pIoJrnl = &kvvfs_jrnl_io_methods }; /****** Utility subroutines ************************************************/ @@ -38204,7 +39486,10 @@ KVSTORAGE_KEY_SZ ** of hexadecimal and base-26 numbers, it is always clear where ** one stops and the next begins. */ -static int kvvfsEncode(const char *aData, int nData, char *aOut){ +#ifndef SQLITE_WASM +static +#endif +int kvvfsEncode(const char *aData, int nData, char *aOut){ int i, j; const unsigned char *a = (const unsigned char*)aData; for(i=j=0; i<nData; i++){ @@ -38255,9 +39540,13 @@ static const signed char kvvfsHexValue[256] = { ** Decode the text encoding back to binary. The binary content is ** written into pOut, which must be at least nOut bytes in length. ** -** The return value is the number of bytes actually written into aOut[]. +** The return value is the number of bytes actually written into aOut[], or +** -1 for malformed inputs. */ -static int kvvfsDecode(const char *a, char *aOut, int nOut){ +#ifndef SQLITE_WASM +static +#endif +int kvvfsDecode(const char *a, char *aOut, int nOut){ int i, j; int c; const unsigned char *aIn = (const unsigned char*)a; @@ -38282,7 +39571,7 @@ static int kvvfsDecode(const char *a, char *aOut, int nOut){ }else{ aOut[j] = c<<4; c = kvvfsHexValue[aIn[++i]]; - if( c<0 ) break; + if( c<0 ) return -1 /* hex bytes are always in pairs */; aOut[j++] += c; i++; } @@ -38335,13 +39624,14 @@ static void kvvfsDecodeJournal( static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ char zData[50]; zData[0] = 0; - sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "sz", zData, + sizeof(zData)-1); return strtoll(zData, 0, 0); } static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ char zData[50]; sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); - return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); + return sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "sz", zData); } /****** sqlite3_io_methods methods ******************************************/ @@ -38356,6 +39646,9 @@ static int kvvfsClose(sqlite3_file *pProtoFile){ pFile->isJournal ? "journal" : "db")); sqlite3_free(pFile->aJrnl); sqlite3_free(pFile->aData); +#ifdef SQLITE_WASM + memset(pFile, 0, sizeof(*pFile)); +#endif return SQLITE_OK; } @@ -38372,16 +39665,22 @@ static int kvvfsReadJrnl( assert( pFile->isJournal ); SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); if( pFile->aJrnl==0 ){ - int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + int rc; + int szTxt = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", + 0, 0); char *aTxt; if( szTxt<=4 ){ return SQLITE_IOERR; } aTxt = sqlite3_malloc64( szTxt+1 ); if( aTxt==0 ) return SQLITE_NOMEM; - kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); - kvvfsDecodeJournal(pFile, aTxt, szTxt); + rc = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", + aTxt, szTxt+1); + if( rc>=0 ){ + kvvfsDecodeJournal(pFile, aTxt, szTxt); + } sqlite3_free(aTxt); + if( rc ) return rc; if( pFile->aJrnl==0 ) return SQLITE_IOERR; } if( iOfst+iAmt>pFile->nJrnl ){ @@ -38421,8 +39720,8 @@ static int kvvfsReadDb( pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, - aData, SQLITE_KVOS_SZ-1); + got = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ @@ -38490,6 +39789,7 @@ static int kvvfsWriteDb( unsigned int pgno; char zKey[30]; char *aData = pFile->aData; + int rc; SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); assert( iAmt>=512 && iAmt<=65536 ); assert( (iAmt & (iAmt-1))==0 ); @@ -38498,13 +39798,13 @@ static int kvvfsWriteDb( pgno = 1 + iOfst/iAmt; sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); kvvfsEncode(zBuf, iAmt, aData); - if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ - return SQLITE_IOERR; - } - if( iOfst+iAmt > pFile->szDb ){ - pFile->szDb = iOfst + iAmt; + rc = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, zKey, aData); + if( 0==rc ){ + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } } - return SQLITE_OK; + return rc; } /* @@ -38514,7 +39814,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); assert( size==0 ); - sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, "jrnl"); sqlite3_free(pFile->aJrnl); pFile->aJrnl = 0; pFile->nJrnl = 0; @@ -38533,7 +39833,7 @@ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ pgnoMax = 2 + pFile->szDb/pFile->szPage; while( pgno<=pgnoMax ){ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, zKey); pgno++; } pFile->szDb = size; @@ -38565,7 +39865,7 @@ static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ }while( n>0 ); zOut[i++] = ' '; kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); - i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + i = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "jrnl", zOut); sqlite3_free(zOut); return i ? SQLITE_IOERR : SQLITE_OK; } @@ -38679,33 +39979,32 @@ static int kvvfsOpen( KVVfsFile *pFile = (KVVfsFile*)pProtoFile; if( zName==0 ) zName = ""; SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); - if( strcmp(zName, "local")==0 - || strcmp(zName, "session")==0 - ){ - pFile->isJournal = 0; - pFile->base.pMethods = &kvvfs_db_io_methods; - }else - if( strcmp(zName, "local-journal")==0 - || strcmp(zName, "session-journal")==0 - ){ + assert(!pFile->zClass); + assert(!pFile->aData); + assert(!pFile->aJrnl); + assert(!pFile->nJrnl); + assert(!pFile->base.pMethods); + pFile->szPage = -1; + pFile->szDb = -1; + if( 0==sqlite3_strglob("*-journal", zName) ){ pFile->isJournal = 1; pFile->base.pMethods = &kvvfs_jrnl_io_methods; + if( 0==strcmp("session-journal",zName) ){ + pFile->zClass = "session"; + }else if( 0==strcmp("local-journal",zName) ){ + pFile->zClass = "local"; + } }else{ - return SQLITE_CANTOPEN; + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; } - if( zName[0]=='s' ){ - pFile->zClass = "session"; - }else{ - pFile->zClass = "local"; + if( !pFile->zClass ){ + pFile->zClass = zName; } pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); if( pFile->aData==0 ){ return SQLITE_NOMEM; } - pFile->aJrnl = 0; - pFile->nJrnl = 0; - pFile->szPage = -1; - pFile->szDb = -1; return SQLITE_OK; } @@ -38715,13 +40014,17 @@ static int kvvfsOpen( ** returning. */ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc /* The JS impl can fail with OOM in argument conversion */; if( strcmp(zPath, "local-journal")==0 ){ - sqlite3KvvfsMethods.xDelete("local", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("local", "jrnl"); }else if( strcmp(zPath, "session-journal")==0 ){ - sqlite3KvvfsMethods.xDelete("session", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("session", "jrnl"); } - return SQLITE_OK; + else{ + rc = 0; + } + return rc; } /* @@ -38735,21 +40038,42 @@ static int kvvfsAccess( int *pResOut ){ SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); +#if 0 && defined(SQLITE_WASM) + /* + ** This is not having the desired effect in the JS bindings. + ** It's ostensibly the same logic as the #else block, but + ** it's not behaving that way. + ** + ** In JS we map all zPaths to Storage objects, and -journal files + ** are mapped to the storage for the main db (which is is exactly + ** what the mapping of "local-journal" -> "local" is doing). + */ + const char *zKey = (0==sqlite3_strglob("*-journal", zPath)) + ? "jrnl" : "sz"; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead(zPath, zKey, 0, 0)>0; +#else if( strcmp(zPath, "local-journal")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("local", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "session-journal")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("session", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "local")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("local", "sz", 0, 0)>0; }else if( strcmp(zPath, "session")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("session", "sz", 0, 0)>0; }else { *pResOut = 0; } + /*all current JS tests avoid triggering: assert( *pResOut == 0 ); */ +#endif SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); return SQLITE_OK; } @@ -39341,10 +40665,11 @@ static struct unix_syscall { #if defined(HAVE_FCHMOD) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #else { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#define osFchmod(FID,MODE) 0 #endif -#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, @@ -39438,6 +40763,119 @@ static struct unix_syscall { }; /* End of the overrideable system calls */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Extract Posix Advisory Locking information about file description fd +** from the /proc/PID/fdinfo/FD pseudo-file. Fill the string buffer a[16] +** with characters to indicate which SQLite-relevant locks are held. +** a[16] will be a 15-character zero-terminated string with the following +** schema: +** +** AAA/B.DDD.DDDDD +** +** Each of character A-D will be "w" or "r" or "-" to indicate either a +** write-lock, a read-lock, or no-lock, respectively. The "." and "/" +** characters are delimiters intended to make the string more easily +** readable by humans. Here are the meaning of the specific letters: +** +** AAA -> The main database locks. PENDING_BYTE, RESERVED_BYTE, +** and SHARED_FIRST, respectively. +** +** B -> The deadman switch lock. Offset 128 of the -shm file. +** +** CCC -> WAL locks: WRITE, CKPT, RECOVER +** +** DDDDD -> WAL read-locks 0 through 5 +** +** Note that elements before the "/" apply to the main database file and +** elements after the "/" apply to the -shm file in WAL mode. +** +** Here is another way of thinking about the meaning of the result string: +** +** AAA/B.CCC.DDDDD +** ||| | ||| \___/ +** PENDING--'|| | ||| `----- READ 0-5 +** RESERVED--'| | ||`---- RECOVER +** SHARED ----' | |`----- CKPT +** DMS ------' `------ WRITE +** +** Return SQLITE_OK on success and SQLITE_ERROR_UNABLE if the /proc +** pseudo-filesystem is unavailable. +*/ +static int unixPosixAdvisoryLocks( + int fd, /* The file descriptor to analyze */ + char a[16] /* Write a text description of PALs here */ +){ + int in; + ssize_t n; + char *p, *pNext, *x; + char z[2000]; + + /* 1 */ + /* 012 4 678 01234 */ + memcpy(a, "---/-.---.-----", 16); + sqlite3_snprintf(sizeof(z), z, "/proc/%d/fdinfo/%d", getpid(), fd); + in = osOpen(z, O_RDONLY, 0); + if( in<0 ){ + return SQLITE_ERROR_UNABLE; + } + n = osRead(in, z, sizeof(z)-1); + osClose(in); + if( n<=0 ) return SQLITE_ERROR_UNABLE; + z[n] = 0; + + /* We are looking for lines that begin with "lock:\t". Examples: + ** + ** lock: 1: POSIX ADVISORY READ 494716 08:02:5277597 1073741826 1073742335 + ** lock: 1: POSIX ADVISORY WRITE 494716 08:02:5282282 120 120 + ** lock: 2: POSIX ADVISORY READ 494716 08:02:5282282 123 123 + ** lock: 3: POSIX ADVISORY READ 494716 08:02:5282282 128 128 + */ + pNext = strstr(z, "lock:\t"); + while( pNext ){ + char cType = 0; + sqlite3_int64 iFirst, iLast; + p = pNext+6; + pNext = strstr(p, "lock:\t"); + if( pNext ) pNext[-1] = 0; + if( (x = strstr(p, " READ "))!=0 ){ + cType = 'r'; + x += 6; + }else if( (x = strstr(p, " WRITE "))!=0 ){ + cType = 'w'; + x += 7; + }else{ + continue; + } + x = strrchr(x, ' '); + if( x==0 ) continue; + iLast = strtoll(x+1, 0, 10); + *x = 0; + x = strrchr(p, ' '); + if( x==0 ) continue; + iFirst = strtoll(x+1, 0, 10); + if( iLast>=PENDING_BYTE ){ + if( iFirst<=PENDING_BYTE && iLast>=PENDING_BYTE ) a[0] = cType; + if( iFirst<=PENDING_BYTE+1 && iLast>=PENDING_BYTE+1 ) a[1] = cType; + if( iFirst<=PENDING_BYTE+2 && iLast>=PENDING_BYTE+510 ) a[2] = cType; + }else if( iLast<=128 ){ + if( iFirst<=128 && iLast>=128 ) a[4] = cType; + if( iFirst<=120 && iLast>=120 ) a[6] = cType; + if( iFirst<=121 && iLast>=121 ) a[7] = cType; + if( iFirst<=122 && iLast>=122 ) a[8] = cType; + if( iFirst<=123 && iLast>=123 ) a[10] = cType; + if( iFirst<=124 && iLast>=124 ) a[11] = cType; + if( iFirst<=125 && iLast>=125 ) a[12] = cType; + if( iFirst<=126 && iLast>=126 ) a[13] = cType; + if( iFirst<=127 && iLast>=127 ) a[14] = cType; + } + } + return SQLITE_OK; +} +#else +# define unixPosixAdvisoryLocks(A,B) SQLITE_ERROR_UNABLE +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + /* ** On some systems, calls to fchown() will trigger a message in a security ** log if they come from non-root processes. So avoid calling fchown() if @@ -39602,9 +41040,8 @@ static int robust_open(const char *z, int f, mode_t m){ /* ** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the unixInodeInfo and -** vxworksFileId objects used by this file, all of which may be -** shared by multiple threads. +** global mutex is used to protect the unixInodeInfo objects used by +** this file, all of which may be shared by multiple threads. ** ** Function unixMutexHeld() is used to assert() that the global mutex ** is held when required. This function is only used as part of assert() @@ -39806,6 +41243,7 @@ struct vxworksFileId { ** variable: */ static struct vxworksFileId *vxworksFileList = 0; +static sqlite3_mutex *vxworksMutex = 0; /* ** Simplify a filename into its canonical form @@ -39871,14 +41309,14 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ ** If found, increment the reference count and return a pointer to ** the existing file ID. */ - unixEnterMutex(); + sqlite3_mutex_enter(vxworksMutex); for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ if( pCandidate->nName==n && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 ){ sqlite3_free(pNew); pCandidate->nRef++; - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); return pCandidate; } } @@ -39888,7 +41326,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ pNew->nName = n; pNew->pNext = vxworksFileList; vxworksFileList = pNew; - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); return pNew; } @@ -39897,7 +41335,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ ** the object when the reference count reaches zero. */ static void vxworksReleaseFileId(struct vxworksFileId *pId){ - unixEnterMutex(); + sqlite3_mutex_enter(vxworksMutex); assert( pId->nRef>0 ); pId->nRef--; if( pId->nRef==0 ){ @@ -39907,7 +41345,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ *pp = pId->pNext; sqlite3_free(pId); } - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); } #endif /* OS_VXWORKS */ /*************** End of Unique File ID Utility Used By VxWorks **************** @@ -40295,6 +41733,10 @@ static int findInodeInfo( storeLastErrno(pFile, errno); return SQLITE_IOERR; } + if( fsync(fd) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_FSYNC; + } rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); @@ -40464,18 +41906,42 @@ static int osSetPosixAdvisoryLock( struct flock *pLock, /* The description of the lock */ unixFile *pFile /* Structure holding timeout value */ ){ - int tm = pFile->iBusyTimeout; - int rc = osFcntl(h,F_SETLK,pLock); - while( rc<0 && tm>0 ){ - /* On systems that support some kind of blocking file lock with a timeout, - ** make appropriate changes here to invoke that blocking file lock. On - ** generic posix, however, there is no such API. So we simply try the - ** lock once every millisecond until either the timeout expires, or until - ** the lock is obtained. */ - unixSleep(0,1000); + int rc = 0; + + if( pFile->iBusyTimeout==0 ){ + /* unixFile->iBusyTimeout is set to 0. In this case, attempt a + ** non-blocking lock. */ + rc = osFcntl(h,F_SETLK,pLock); + }else{ + /* unixFile->iBusyTimeout is set to greater than zero. In this case, + ** attempt a blocking-lock with a unixFile->iBusyTimeout ms timeout. + ** + ** On systems that support some kind of blocking file lock operation, + ** this block should be replaced by code to attempt a blocking lock + ** with a timeout of unixFile->iBusyTimeout ms. The code below is + ** placeholder code. If SQLITE_TEST is defined, the placeholder code + ** retries the lock once every 1ms until it succeeds or the timeout + ** is reached. Or, if SQLITE_TEST is not defined, the placeholder + ** code attempts a non-blocking lock and sets unixFile->iBusyTimeout + ** to 0. This causes the caller to return SQLITE_BUSY, instead of + ** SQLITE_BUSY_TIMEOUT to SQLite - as required by a VFS that does not + ** support blocking locks. + */ +#ifdef SQLITE_TEST + int tm = pFile->iBusyTimeout; + while( tm>0 ){ + rc = osFcntl(h,F_SETLK,pLock); + if( rc==0 ) break; + unixSleep(0,1000); + tm--; + } +#else rc = osFcntl(h,F_SETLK,pLock); - tm--; + pFile->iBusyTimeout = 0; +#endif + /* End of code to replace with real blocking-locks code. */ } + return rc; } #endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ @@ -40533,6 +41999,13 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ return rc; } +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) +/* Forward reference */ +static int unixIsSharingShmNode(unixFile*); +#else +#define unixIsSharingShmNode(pFile) (0) +#endif + /* ** Lock the file with the lock specified by parameter eFileLock - one ** of the following: @@ -40725,6 +42198,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; + }else if( unixIsSharingShmNode(pFile) ){ + /* We are in WAL mode and attempting to delete the SHM and WAL + ** files due to closing the connection or changing out of WAL mode, + ** but another process still holds locks on the SHM file, thus + ** indicating that database locks have been broken, perhaps due + ** to a rogue close(open(dbFile)) or similar. + */ + rc = SQLITE_BUSY; }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file @@ -42816,6 +44297,10 @@ static int unixGetTempname(int nBuf, char *zBuf); #if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) static int unixFcntlExternalReader(unixFile*, int*); #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + static void unixDescribeShm(sqlite3_str*,unixShm*); +#endif + /* ** Information and control of an open file handle. @@ -42958,6 +44443,66 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; #endif } + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + case SQLITE_FCNTL_FILESTAT: { + sqlite3_str *pStr = (sqlite3_str*)pArg; + char aLck[16]; + unixInodeInfo *pInode; + static const char *azLock[] = { "SHARED", "RESERVED", + "PENDING", "EXCLUSIVE" }; + sqlite3_str_appendf(pStr, "{\"h\":%d", pFile->h); + sqlite3_str_appendf(pStr, ",\"vfs\":\"%s\"", pFile->pVfs->zName); + if( pFile->eFileLock ){ + sqlite3_str_appendf(pStr, ",\"eFileLock\":\"%s\"", + azLock[pFile->eFileLock-1]); + if( unixPosixAdvisoryLocks(pFile->h, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + } + unixEnterMutex(); + if( pFile->pShm ){ + sqlite3_str_appendall(pStr, ",\"shm\":"); + unixDescribeShm(pStr, pFile->pShm); + } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSize ){ + sqlite3_str_appendf(pStr, ",\"mmapSize\":%lld", pFile->mmapSize); + sqlite3_str_appendf(pStr, ",\"nFetchOut\":%d", pFile->nFetchOut); + } +#endif + if( (pInode = pFile->pInode)!=0 ){ + sqlite3_str_appendf(pStr, ",\"inode\":{\"nRef\":%d",pInode->nRef); + sqlite3_mutex_enter(pInode->pLockMutex); + sqlite3_str_appendf(pStr, ",\"nShared\":%d", pInode->nShared); + if( pInode->eFileLock ){ + sqlite3_str_appendf(pStr, ",\"eFileLock\":\"%s\"", + azLock[pInode->eFileLock-1]); + } + if( pInode->pUnused ){ + char cSep = '['; + UnixUnusedFd *pUFd = pFile->pInode->pUnused; + sqlite3_str_appendall(pStr, ",\"unusedFd\":"); + while( pUFd ){ + sqlite3_str_appendf(pStr, "%c{\"fd\":%d,\"flags\":%d", + cSep, pUFd->fd, pUFd->flags); + cSep = ','; + if( unixPosixAdvisoryLocks(pUFd->fd, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + sqlite3_str_append(pStr, "}", 1); + pUFd = pUFd->pNext; + } + sqlite3_str_append(pStr, "]", 1); + } + sqlite3_mutex_leave(pInode->pLockMutex); + sqlite3_str_append(pStr, "}", 1); + } + unixLeaveMutex(); + sqlite3_str_append(pStr, "}", 1); + return SQLITE_OK; + } +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ } return SQLITE_NOTFOUND; } @@ -43224,6 +44769,26 @@ struct unixShm { #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Describe the pShm object using JSON. Used for diagnostics only. +*/ +static void unixDescribeShm(sqlite3_str *pStr, unixShm *pShm){ + unixShmNode *pNode = pShm->pShmNode; + char aLck[16]; + sqlite3_str_appendf(pStr, "{\"h\":%d", pNode->hShm); + assert( unixMutexHeld() ); + sqlite3_str_appendf(pStr, ",\"nRef\":%d", pNode->nRef); + sqlite3_str_appendf(pStr, ",\"id\":%d", pShm->id); + sqlite3_str_appendf(pStr, ",\"sharedMask\":%d", pShm->sharedMask); + sqlite3_str_appendf(pStr, ",\"exclMask\":%d", pShm->exclMask); + if( unixPosixAdvisoryLocks(pNode->hShm, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + sqlite3_str_append(pStr, "}", 1); +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + /* ** Use F_GETLK to check whether or not there are any readers with open ** wal-mode transactions in other processes on database file pFile. If @@ -43257,6 +44822,44 @@ static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ return rc; } +/* +** If pFile has a -shm file open and it is sharing that file with some +** other connection, either in the same process or in a separate process, +** then return true. Return false if either pFile does not have a -shm +** file open or if it is the only connection to that -shm file across the +** entire system. +** +** This routine is not required for correct operation. It can always return +** false and SQLite will continue to operate according to spec. However, +** when this routine does its job, it adds extra robustness in cases +** where database file locks have been erroneously deleted in a WAL-mode +** database by doing close(open(DATABASE_PATHNAME)) or similar. +** +** With false negatives, SQLite still operates to spec, though with less +** robustness. With false positives, the last database connection on a +** WAL-mode database will fail to unlink the -wal and -shm files, which +** is annoying but harmless. False positives will also prevent a database +** connection from running "PRAGMA journal_mode=DELETE" in order to take +** the database out of WAL mode, which is perhaps more serious, but is +** still not a disaster. +*/ +static int unixIsSharingShmNode(unixFile *pFile){ + unixShmNode *pShmNode; + struct flock lock; + if( pFile->pShm==0 ) return 0; + if( pFile->ctrlFlags & UNIXFILE_EXCL ) return 0; + pShmNode = pFile->pShm->pShmNode; +#if SQLITE_ATOMIC_INTRINSICS + assert( AtomicLoad(&pShmNode->nRef)==1 ); +#endif + memset(&lock, 0, sizeof(lock)); + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + osFcntl(pShmNode->hShm, F_GETLK, &lock); + return (lock.l_type!=F_UNLCK); +} /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -43302,7 +44905,8 @@ static int unixShmSystemLock( /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); - assert( ofst>=UNIX_SHM_BASE && ofst<=(UNIX_SHM_DMS+SQLITE_SHM_NLOCK) ); + assert( ofst>=UNIX_SHM_BASE && ofst<=UNIX_SHM_DMS ); + assert( ofst+n-1<=UNIX_SHM_DMS ); if( pShmNode->hShm>=0 ){ int res; @@ -43753,7 +45357,7 @@ static int unixShmMap( } /* Map the requested memory region into this processes address space. */ - apNew = (char **)sqlite3_realloc( + apNew = (char **)sqlite3_realloc64( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ @@ -43834,7 +45438,7 @@ static int assertLockingArrayOk(unixShmNode *pShmNode){ return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); #endif } -#endif +#endif /* !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) */ /* ** Change the lock state for a shared-memory segment. @@ -44796,10 +46400,17 @@ static int fillInUnixFile( storeLastErrno(pNew, 0); #if OS_VXWORKS if( rc!=SQLITE_OK ){ - if( h>=0 ) robust_close(pNew, h, __LINE__); - h = -1; - osUnlink(zFilename); - pNew->ctrlFlags |= UNIXFILE_DELETE; + if( h>=0 ){ + robust_close(pNew, h, __LINE__); + h = -1; + } + if( pNew->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(zFilename); + } + if( pNew->pId ){ + vxworksReleaseFileId(pNew->pId); + pNew->pId = 0; + } } #endif if( rc!=SQLITE_OK ){ @@ -44843,6 +46454,9 @@ static const char *unixTempFileDir(void){ while(1){ if( zDir!=0 +#if OS_VXWORKS + && zDir[0]=='/' +#endif && osStat(zDir, &buf)==0 && S_ISDIR(buf.st_mode) && osAccess(zDir, 03)==0 @@ -45157,6 +46771,12 @@ static int unixOpen( || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); +#if OS_VXWORKS + /* The file-ID mechanism used in Vxworks requires that all pathnames + ** provided to unixOpen must be absolute pathnames. */ + if( zPath!=0 && zPath[0]!='/' ){ return SQLITE_CANTOPEN; } +#endif + /* Detect a pid change and reset the PRNG. There is a race condition ** here such that two or more threads all trying to open databases at ** the same instant might all reset the PRNG. But multiple resets @@ -45357,8 +46977,11 @@ static int unixOpen( } #endif - assert( zPath==0 || zPath[0]=='/' - || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL + assert( zPath==0 + || zPath[0]=='/' + || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_TEMP_JOURNAL ); rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); @@ -47087,6 +48710,9 @@ SQLITE_API int sqlite3_os_init(void){ sqlite3KvvfsInit(); #endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#if OS_VXWORKS + vxworksMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS2); +#endif #ifndef SQLITE_OMIT_WAL /* Validate lock assumptions */ @@ -47121,6 +48747,9 @@ SQLITE_API int sqlite3_os_init(void){ */ SQLITE_API int sqlite3_os_end(void){ unixBigLock = 0; +#if OS_VXWORKS + vxworksMutex = 0; +#endif return SQLITE_OK; } @@ -47173,7 +48802,7 @@ SQLITE_API int sqlite3_os_end(void){ ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_ANSI) # define SQLITE_WIN32_HAS_ANSI #endif @@ -47181,7 +48810,7 @@ SQLITE_API int sqlite3_os_end(void){ ** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT) && \ !defined(SQLITE_WIN32_NO_WIDE) # define SQLITE_WIN32_HAS_WIDE #endif @@ -47320,16 +48949,7 @@ SQLITE_API int sqlite3_os_end(void){ */ #if SQLITE_WIN32_FILEMAPPING_API && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) -/* -** Two of the file mapping APIs are different under WinRT. Figure out which -** set we need. -*/ -#if SQLITE_OS_WINRT -WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ - LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); -WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); -#else #if defined(SQLITE_WIN32_HAS_ANSI) WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ DWORD, DWORD, DWORD, LPCSTR); @@ -47341,7 +48961,6 @@ WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ #endif /* defined(SQLITE_WIN32_HAS_WIDE) */ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); -#endif /* SQLITE_OS_WINRT */ /* ** These file mapping APIs are common to both Win32 and WinRT. @@ -47632,7 +49251,7 @@ static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; ** This function is not available on Windows CE or WinRT. */ -#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +#if SQLITE_OS_WINCE # define osAreFileApisANSI() 1 #endif @@ -47647,7 +49266,7 @@ static struct win_syscall { sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, #else { "AreFileApisANSI", (SYSCALL)0, 0 }, @@ -47686,7 +49305,7 @@ static struct win_syscall { #define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "CreateFileW", (SYSCALL)CreateFileW, 0 }, #else { "CreateFileW", (SYSCALL)0, 0 }, @@ -47695,7 +49314,7 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ +#if defined(SQLITE_WIN32_HAS_ANSI) && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ SQLITE_WIN32_CREATEFILEMAPPINGA { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, @@ -47706,8 +49325,8 @@ static struct win_syscall { #define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if (SQLITE_OS_WINCE || defined(SQLITE_WIN32_HAS_WIDE)) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -47716,7 +49335,7 @@ static struct win_syscall { #define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, #else { "CreateMutexW", (SYSCALL)0, 0 }, @@ -47802,7 +49421,7 @@ static struct win_syscall { #define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ LPDWORD))aSyscall[18].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, #else { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, @@ -47819,7 +49438,7 @@ static struct win_syscall { #define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, #else { "GetFileAttributesW", (SYSCALL)0, 0 }, @@ -47836,11 +49455,7 @@ static struct win_syscall { #define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ LPVOID))aSyscall[22].pCurrent) -#if !SQLITE_OS_WINRT { "GetFileSize", (SYSCALL)GetFileSize, 0 }, -#else - { "GetFileSize", (SYSCALL)0, 0 }, -#endif #define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) @@ -47853,7 +49468,7 @@ static struct win_syscall { #define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ LPSTR*))aSyscall[24].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, #else { "GetFullPathNameW", (SYSCALL)0, 0 }, @@ -47888,16 +49503,10 @@ static struct win_syscall { #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ LPCSTR))aSyscall[27].pCurrent) -#if !SQLITE_OS_WINRT { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, -#else - { "GetSystemInfo", (SYSCALL)0, 0 }, -#endif - #define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, - #define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) #if !SQLITE_OS_WINCE @@ -47917,7 +49526,7 @@ static struct win_syscall { #define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, #else { "GetTempPathW", (SYSCALL)0, 0 }, @@ -47925,11 +49534,7 @@ static struct win_syscall { #define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) -#if !SQLITE_OS_WINRT { "GetTickCount", (SYSCALL)GetTickCount, 0 }, -#else - { "GetTickCount", (SYSCALL)0, 0 }, -#endif #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) @@ -47942,7 +49547,7 @@ static struct win_syscall { #define osGetVersionExA ((BOOL(WINAPI*)( \ LPOSVERSIONINFOA))aSyscall[34].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ +#if defined(SQLITE_WIN32_HAS_WIDE) && \ SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else @@ -47957,20 +49562,12 @@ static struct win_syscall { #define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ SIZE_T))aSyscall[36].pCurrent) -#if !SQLITE_OS_WINRT { "HeapCreate", (SYSCALL)HeapCreate, 0 }, -#else - { "HeapCreate", (SYSCALL)0, 0 }, -#endif #define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ SIZE_T))aSyscall[37].pCurrent) -#if !SQLITE_OS_WINRT { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, -#else - { "HeapDestroy", (SYSCALL)0, 0 }, -#endif #define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) @@ -47988,16 +49585,12 @@ static struct win_syscall { #define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[41].pCurrent) -#if !SQLITE_OS_WINRT { "HeapValidate", (SYSCALL)HeapValidate, 0 }, -#else - { "HeapValidate", (SYSCALL)0, 0 }, -#endif #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "HeapCompact", (SYSCALL)HeapCompact, 0 }, #else { "HeapCompact", (SYSCALL)0, 0 }, @@ -48013,7 +49606,7 @@ static struct win_syscall { #define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ +#if defined(SQLITE_WIN32_HAS_WIDE) && \ !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else @@ -48022,15 +49615,11 @@ static struct win_syscall { #define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) -#if !SQLITE_OS_WINRT { "LocalFree", (SYSCALL)LocalFree, 0 }, -#else - { "LocalFree", (SYSCALL)0, 0 }, -#endif #define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "LockFile", (SYSCALL)LockFile, 0 }, #else { "LockFile", (SYSCALL)0, 0 }, @@ -48052,8 +49641,7 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -48081,20 +49669,12 @@ static struct win_syscall { #define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) -#if !SQLITE_OS_WINRT { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, -#else - { "SetFilePointer", (SYSCALL)0, 0 }, -#endif #define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ DWORD))aSyscall[54].pCurrent) -#if !SQLITE_OS_WINRT { "Sleep", (SYSCALL)Sleep, 0 }, -#else - { "Sleep", (SYSCALL)0, 0 }, -#endif #define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) @@ -48103,7 +49683,7 @@ static struct win_syscall { #define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "UnlockFile", (SYSCALL)UnlockFile, 0 }, #else { "UnlockFile", (SYSCALL)0, 0 }, @@ -48141,15 +49721,6 @@ static struct win_syscall { #define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ LPOVERLAPPED))aSyscall[61].pCurrent) -#if SQLITE_OS_WINRT - { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, -#else - { "CreateEventExW", (SYSCALL)0, 0 }, -#endif - -#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ - DWORD,DWORD))aSyscall[62].pCurrent) - /* ** For WaitForSingleObject(), MSDN says: ** @@ -48159,7 +49730,7 @@ static struct win_syscall { { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ - DWORD))aSyscall[63].pCurrent) + DWORD))aSyscall[62].pCurrent) #if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, @@ -48168,69 +49739,12 @@ static struct win_syscall { #endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ - BOOL))aSyscall[64].pCurrent) - -#if SQLITE_OS_WINRT - { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, -#else - { "SetFilePointerEx", (SYSCALL)0, 0 }, -#endif - -#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ - PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) - -#if SQLITE_OS_WINRT - { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, -#else - { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, -#endif - -#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ - FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) - -#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) - { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, -#else - { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, -#endif - -#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ - SIZE_T))aSyscall[67].pCurrent) - -#if SQLITE_OS_WINRT - { "CreateFile2", (SYSCALL)CreateFile2, 0 }, -#else - { "CreateFile2", (SYSCALL)0, 0 }, -#endif - -#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ - LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) - -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) - { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, -#else - { "LoadPackagedLibrary", (SYSCALL)0, 0 }, -#endif + BOOL))aSyscall[63].pCurrent) -#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ - DWORD))aSyscall[69].pCurrent) - -#if SQLITE_OS_WINRT - { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, -#else - { "GetTickCount64", (SYSCALL)0, 0 }, -#endif - -#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) - -#if SQLITE_OS_WINRT { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, -#else - { "GetNativeSystemInfo", (SYSCALL)0, 0 }, -#endif #define osGetNativeSystemInfo ((VOID(WINAPI*)( \ - LPSYSTEM_INFO))aSyscall[71].pCurrent) + LPSYSTEM_INFO))aSyscall[64].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, @@ -48238,7 +49752,7 @@ static struct win_syscall { { "OutputDebugStringA", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[65].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, @@ -48246,20 +49760,11 @@ static struct win_syscall { { "OutputDebugStringW", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[66].pCurrent) { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, -#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) - -#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) - { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, -#else - { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, -#endif - -#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ - LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[67].pCurrent) /* ** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" @@ -48274,25 +49779,25 @@ static struct win_syscall { { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, #define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ - SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[68].pCurrent) #endif /* defined(InterlockedCompareExchange) */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { "UuidCreate", (SYSCALL)UuidCreate, 0 }, #else { "UuidCreate", (SYSCALL)0, 0 }, #endif -#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[69].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, #else { "UuidCreateSequential", (SYSCALL)0, 0 }, #endif #define osUuidCreateSequential \ - ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[70].pCurrent) #if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, @@ -48301,7 +49806,7 @@ static struct win_syscall { #endif #define osFlushViewOfFile \ - ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[71].pCurrent) /* ** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() @@ -48318,7 +49823,7 @@ static struct win_syscall { #define osCreateEvent ( \ (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ - aSyscall[80].pCurrent \ + aSyscall[72].pCurrent \ ) /* @@ -48335,7 +49840,7 @@ static struct win_syscall { { "CancelIo", (SYSCALL)0, 0 }, #endif -#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[73].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, @@ -48343,7 +49848,7 @@ static struct win_syscall { { "GetModuleHandleW", (SYSCALL)0, 0 }, #endif -#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[74].pCurrent) #ifndef _WIN32 { "getenv", (SYSCALL)getenv, 0 }, @@ -48351,7 +49856,7 @@ static struct win_syscall { { "getenv", (SYSCALL)0, 0 }, #endif -#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) +#define osGetenv ((const char *(*)(const char *))aSyscall[75].pCurrent) #ifndef _WIN32 { "getcwd", (SYSCALL)getcwd, 0 }, @@ -48359,7 +49864,7 @@ static struct win_syscall { { "getcwd", (SYSCALL)0, 0 }, #endif -#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[76].pCurrent) #ifndef _WIN32 { "readlink", (SYSCALL)readlink, 0 }, @@ -48367,7 +49872,7 @@ static struct win_syscall { { "readlink", (SYSCALL)0, 0 }, #endif -#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[77].pCurrent) #ifndef _WIN32 { "lstat", (SYSCALL)lstat, 0 }, @@ -48375,7 +49880,7 @@ static struct win_syscall { { "lstat", (SYSCALL)0, 0 }, #endif -#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[78].pCurrent) #ifndef _WIN32 { "__errno", (SYSCALL)__errno, 0 }, @@ -48383,7 +49888,7 @@ static struct win_syscall { { "__errno", (SYSCALL)0, 0 }, #endif -#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) +#define osErrno (*((int*(*)(void))aSyscall[79].pCurrent)()) #ifndef _WIN32 { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, @@ -48392,7 +49897,7 @@ static struct win_syscall { #endif #define osCygwin_conv_path ((size_t(*)(unsigned int, \ - const void *, void *, size_t))aSyscall[88].pCurrent) + const void *, void *, size_t))aSyscall[80].pCurrent) }; /* End of the overrideable system calls */ @@ -48496,10 +50001,10 @@ SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE if( (nLargest=osHeapCompact(hHeap, SQLITE_WIN32_HEAP_FLAGS))==0 ){ DWORD lastErrno = osGetLastError(); if( lastErrno==NO_ERROR ){ @@ -48612,28 +50117,11 @@ SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif /* _WIN32 */ -/* -** The following routine suspends the current thread for at least ms -** milliseconds. This is equivalent to the Win32 Sleep() interface. -*/ -#if SQLITE_OS_WINRT -static HANDLE sleepObj = NULL; -#endif - SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ -#if SQLITE_OS_WINRT - if ( sleepObj==NULL ){ - sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, - SYNCHRONIZE); - } - assert( sleepObj!=NULL ); - osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); -#else osSleep(milliseconds); -#endif } -#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && \ SQLITE_THREADSAFE>0 SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; @@ -48657,7 +50145,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ #if !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) -#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) +#elif SQLITE_OS_WINCE || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) @@ -48670,13 +50158,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ ** based on the NT kernel. */ SQLITE_API int sqlite3_win32_is_nt(void){ -#if SQLITE_OS_WINRT - /* - ** NOTE: The WinRT sub-platform is always assumed to be based on the NT - ** kernel. - */ - return 1; -#elif SQLITE_WIN32_GETVERSIONEX +#if SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; @@ -48718,7 +50200,7 @@ static void *winMemMalloc(int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif assert( nBytes>=0 ); @@ -48740,7 +50222,7 @@ static void winMemFree(void *pPrior){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ @@ -48761,7 +50243,7 @@ static void *winMemRealloc(void *pPrior, int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif assert( nBytes>=0 ); @@ -48789,7 +50271,7 @@ static int winMemSize(void *p){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); #endif if( !p ) return 0; @@ -48819,7 +50301,7 @@ static int winMemInit(void *pAppData){ assert( pWinMemData->magic1==WINMEM_MAGIC1 ); assert( pWinMemData->magic2==WINMEM_MAGIC2 ); -#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE +#if SQLITE_WIN32_HEAP_CREATE if( !pWinMemData->hHeap ){ DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; @@ -48852,7 +50334,7 @@ static int winMemInit(void *pAppData){ #endif assert( pWinMemData->hHeap!=0 ); assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif return SQLITE_OK; @@ -48870,7 +50352,7 @@ static void winMemShutdown(void *pAppData){ if( pWinMemData->hHeap ){ assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif if( pWinMemData->bOwned ){ @@ -49251,17 +50733,6 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ char *zOut = 0; if( osIsNT() ){ -#if SQLITE_OS_WINRT - WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; - dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - lastErrno, - 0, - zTempWide, - SQLITE_WIN32_MAX_ERRMSG_CHARS, - 0); -#else LPWSTR zTempWide = NULL; dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | @@ -49272,16 +50743,13 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ (LPWSTR) &zTempWide, 0, 0); -#endif if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); zOut = winUnicodeToUtf8(zTempWide); sqlite3EndBenignMalloc(); -#if !SQLITE_OS_WINRT /* free the system buffer allocated by FormatMessage */ osLocalFree(zTempWide); -#endif } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -49793,6 +51261,7 @@ static BOOL winLockFile( #endif } +#ifndef SQLITE_OMIT_WAL /* ** Lock a region of nByte bytes starting at offset offset of file hFile. ** Take an EXCLUSIVE lock if parameter bExclusive is true, or a SHARED lock @@ -49875,6 +51344,7 @@ static int winHandleLockTimeout( } return rc; } +#endif /* #ifndef SQLITE_OMIT_WAL */ /* ** Unlock a file region. @@ -49909,6 +51379,7 @@ static BOOL winUnlockFile( #endif } +#ifndef SQLITE_OMIT_WAL /* ** Remove an nByte lock starting at offset iOff from HANDLE h. */ @@ -49916,6 +51387,7 @@ static int winHandleUnlock(HANDLE h, int iOff, int nByte){ BOOL ret = winUnlockFile(&h, iOff, 0, nByte, 0); return (ret ? SQLITE_OK : SQLITE_IOERR_UNLOCK); } +#endif /***************************************************************************** ** The next group of routines implement the I/O methods specified @@ -49938,7 +51410,6 @@ static int winHandleUnlock(HANDLE h, int iOff, int nByte){ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ int rc = SQLITE_OK; /* Return value */ -#if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ @@ -49960,20 +51431,7 @@ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ rc = SQLITE_IOERR_SEEK; } } -#else - /* This implementation works for WinRT. */ - LARGE_INTEGER x; /* The new offset */ - BOOL bRet; /* Value returned by SetFilePointerEx() */ - - x.QuadPart = iOffset; - bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); - - if(!bRet){ - rc = SQLITE_IOERR_SEEK; - } -#endif - - OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset,sqlite3ErrName(rc))); return rc; } @@ -50253,6 +51711,7 @@ static int winWrite( return SQLITE_OK; } +#ifndef SQLITE_OMIT_WAL /* ** Truncate the file opened by handle h to nByte bytes in size. */ @@ -50273,17 +51732,6 @@ static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ */ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ int rc = SQLITE_OK; - -#if SQLITE_OS_WINRT - FILE_STANDARD_INFO info; - BOOL b; - b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); - if( b ){ - *pnByte = info.EndOfFile.QuadPart; - }else{ - rc = SQLITE_IOERR_FSTAT; - } -#else DWORD upperBits = 0; DWORD lowerBits = 0; @@ -50293,8 +51741,6 @@ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ rc = SQLITE_IOERR_FSTAT; } -#endif - return rc; } @@ -50306,6 +51752,7 @@ static void winHandleClose(HANDLE h){ osCloseHandle(h); } } +#endif /* #ifndef SQLITE_OMIT_WAL */ /* ** Truncate an open file to a specified size @@ -50492,20 +51939,6 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); - -#if SQLITE_OS_WINRT - { - FILE_STANDARD_INFO info; - if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, - &info, sizeof(info)) ){ - *pSize = info.EndOfFile.QuadPart; - }else{ - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, - "winFileSize", pFile->zPath); - } - } -#else { DWORD upperBits; DWORD lowerBits; @@ -50520,7 +51953,6 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ "winFileSize", pFile->zPath); } } -#endif OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; @@ -51083,6 +52515,28 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } #endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + case SQLITE_FCNTL_FILESTAT: { + sqlite3_str *pStr = (sqlite3_str*)pArg; + sqlite3_str_appendf(pStr, "{\"h\":%llu", (sqlite3_uint64)pFile->h); + sqlite3_str_appendf(pStr, ",\"vfs\":\"%s\"", pFile->pVfs->zName); + if( pFile->locktype ){ + static const char *azLock[] = { "SHARED", "RESERVED", + "PENDING", "EXCLUSIVE" }; + sqlite3_str_appendf(pStr, ",\"locktype\":\"%s\"", + azLock[pFile->locktype-1]); + } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSize ){ + sqlite3_str_appendf(pStr, ",\"mmapSize\":%lld", pFile->mmapSize); + sqlite3_str_appendf(pStr, ",\"nFetchOut\":%d", pFile->nFetchOut); + } +#endif + sqlite3_str_append(pStr, "}", 1); + return SQLITE_OK; + } +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + } OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; @@ -51120,6 +52574,103 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ static SYSTEM_INFO winSysInfo; +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function +** +** On Cygwin, 3 possible input forms are accepted: +** - If the filename starts with "<drive>:/" or "<drive>:\", +** it is converted to UTF-16 as-is. +** - If the filename contains '/', it is assumed to be a +** Cygwin absolute path, it is converted to a win32 +** absolute path in UTF-16. +** - Otherwise it must be a filename only, the win32 filename +** is returned in UTF-16. +** Note: If the function cygwin_conv_path() fails, only +** UTF-8 -> UTF-16 conversion will be done. This can only +** happen when the file path >32k, in which case winUtf8ToUnicode() +** will fail too. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ +#ifdef __CYGWIN__ + int nChar; + LPWSTR zWideFilename; + + if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2])) ){ + i64 nByte; + int convertflag = CCP_POSIX_TO_WIN_W; + if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; + nByte = (i64)osCygwin_conv_path(convertflag, + zFilename, 0, 0); + if( nByte>0 ){ + zConverted = sqlite3MallocZero(12+(u64)nByte); + if ( zConverted==0 ){ + return zConverted; + } + zWideFilename = zConverted; + /* Filenames should be prefixed, except when converted + * full path already starts with "\\?\". */ + if( osCygwin_conv_path(convertflag, zFilename, + zWideFilename+4, nByte)==0 ){ + if( (convertflag&CCP_RELATIVE) ){ + memmove(zWideFilename, zWideFilename+4, nByte); + }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( zWideFilename[6]!='?' ){ + memmove(zWideFilename+6, zWideFilename+4, nByte); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + }else{ + memmove(zWideFilename, zWideFilename+4, nByte); + } + return zConverted; + } + sqlite3_free(zConverted); + } + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); + if( zWideFilename==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, + zWideFilename, nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + }else if( nChar>MAX_PATH + && winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2]) ){ + memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); + zWideFilename[2] = '\\'; + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( nChar>MAX_PATH + && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) + && zFilename[2] != '?' ){ + memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + } + zConverted = zWideFilename; +#else + zConverted = winUtf8ToUnicode(zFilename); +#endif /* __CYGWIN__ */ + } +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + #ifndef SQLITE_OMIT_WAL /* @@ -51156,29 +52707,35 @@ static int winShmMutexHeld(void) { ** log-summary is opened only once per process. ** ** winShmMutexHeld() must be true when creating or destroying -** this object or while reading or writing the following fields: +** this object, or while editing the global linked list that starts +** at winShmNodeList. ** -** nRef -** pNext +** When reading or writing the linked list starting at winShmNode.pWinShmList, +** pShmNode->mutex must be held. ** -** The following fields are read-only after the object is created: +** The following fields are constant after the object is created: ** ** zFilename +** hSharedShm +** mutex +** bUseSharedLockHandle ** -** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** Either winShmNode.mutex must be held or winShmNode.pWinShmList==0 and ** winShmMutexHeld() is true when reading or writing any other field ** in this structure. ** -** File-handle hSharedShm is used to (a) take the DMS lock, (b) truncate -** the *-shm file if the DMS-locking protocol demands it, and (c) map -** regions of the *-shm file into memory using MapViewOfFile() or -** similar. Other locks are taken by individual clients using the -** winShm.hShm handles. +** File-handle hSharedShm is always used to (a) take the DMS lock, (b) +** truncate the *-shm file if the DMS-locking protocol demands it, and +** (c) map regions of the *-shm file into memory using MapViewOfFile() +** or similar. If bUseSharedLockHandle is true, then other locks are also +** taken on hSharedShm. Or, if bUseSharedLockHandle is false, then other +** locks are taken using each connection's winShm.hShm handles. */ struct winShmNode { sqlite3_mutex *mutex; /* Mutex to access this object */ char *zFilename; /* Name of the file */ HANDLE hSharedShm; /* File handle open on zFilename */ + int bUseSharedLockHandle; /* True to use hSharedShm for everything */ int isUnlocked; /* DMS lock has not yet been obtained */ int isReadonly; /* True if read-only */ @@ -51191,7 +52748,8 @@ struct winShmNode { } *aRegion; DWORD lastErrno; /* The Windows errno from the last I/O error */ - int nRef; /* Number of winShm objects pointing to this */ + winShm *pWinShmList; /* List of winShm objects with ptrs to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ @@ -51219,6 +52777,7 @@ struct winShm { #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif + winShm *pWinShmNext; /* Next winShm object on same winShmNode */ }; /* @@ -51232,7 +52791,7 @@ static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); static int winDelete(sqlite3_vfs *,const char*,int); /* -** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** Purge the winShmNodeList list of all entries with winShmNode.pWinShmList==0. ** ** This is not a VFS shared-memory method; it is a utility function called ** by VFS shared-memory methods. @@ -51245,7 +52804,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ - if( p->nRef==0 ){ + if( p->pWinShmList==0 ){ int i; if( p->mutex ){ sqlite3_mutex_free(p->mutex); } for(i=0; i<p->nRegion; i++){ @@ -51315,103 +52874,6 @@ static int winLockSharedMemory(winShmNode *pShmNode, DWORD nMs){ /* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from malloc and must be freed by the calling -** function -** -** On Cygwin, 3 possible input forms are accepted: -** - If the filename starts with "<drive>:/" or "<drive>:\", -** it is converted to UTF-16 as-is. -** - If the filename contains '/', it is assumed to be a -** Cygwin absolute path, it is converted to a win32 -** absolute path in UTF-16. -** - Otherwise it must be a filename only, the win32 filename -** is returned in UTF-16. -** Note: If the function cygwin_conv_path() fails, only -** UTF-8 -> UTF-16 conversion will be done. This can only -** happen when the file path >32k, in which case winUtf8ToUnicode() -** will fail too. -*/ -static void *winConvertFromUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( osIsNT() ){ -#ifdef __CYGWIN__ - int nChar; - LPWSTR zWideFilename; - - if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) - && winIsDirSep(zFilename[2])) ){ - i64 nByte; - int convertflag = CCP_POSIX_TO_WIN_W; - if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; - nByte = (i64)osCygwin_conv_path(convertflag, - zFilename, 0, 0); - if( nByte>0 ){ - zConverted = sqlite3MallocZero(12+(u64)nByte); - if ( zConverted==0 ){ - return zConverted; - } - zWideFilename = zConverted; - /* Filenames should be prefixed, except when converted - * full path already starts with "\\?\". */ - if( osCygwin_conv_path(convertflag, zFilename, - zWideFilename+4, nByte)==0 ){ - if( (convertflag&CCP_RELATIVE) ){ - memmove(zWideFilename, zWideFilename+4, nByte); - }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ - memcpy(zWideFilename, L"\\\\?\\", 8); - }else if( zWideFilename[6]!='?' ){ - memmove(zWideFilename+6, zWideFilename+4, nByte); - memcpy(zWideFilename, L"\\\\?\\UNC", 14); - }else{ - memmove(zWideFilename, zWideFilename+4, nByte); - } - return zConverted; - } - sqlite3_free(zConverted); - } - } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); - if( nChar==0 ){ - return 0; - } - zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); - if( zWideFilename==0 ){ - return 0; - } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, - zWideFilename, nChar); - if( nChar==0 ){ - sqlite3_free(zWideFilename); - zWideFilename = 0; - }else if( nChar>MAX_PATH - && winIsDriveLetterAndColon(zFilename) - && winIsDirSep(zFilename[2]) ){ - memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); - zWideFilename[2] = '\\'; - memcpy(zWideFilename, L"\\\\?\\", 8); - }else if( nChar>MAX_PATH - && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) - && zFilename[2] != '?' ){ - memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); - memcpy(zWideFilename, L"\\\\?\\UNC", 14); - } - zConverted = zWideFilename; -#else - zConverted = winUtf8ToUnicode(zFilename); -#endif /* __CYGWIN__ */ - } -#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) - else{ - zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} - -/* ** This function is used to open a handle on a *-shm file. ** ** If SQLITE_ENABLE_SETLK_TIMEOUT is defined at build time, then the file @@ -51452,20 +52914,6 @@ static int winHandleOpen( ** TODO: retry-on-ioerr. */ if( osIsNT() ){ -#if SQLITE_OS_WINRT - CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; - memset(&extendedParameters, 0, sizeof(extendedParameters)); - extendedParameters.dwSize = sizeof(extendedParameters); - extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; - extendedParameters.dwFileFlags = flag_overlapped; - extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; - h = osCreateFile2((LPCWSTR)zConverted, - (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ - FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ - OPEN_ALWAYS, /* dwCreationDisposition */ - &extendedParameters - ); -#else h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ @@ -51474,7 +52922,6 @@ static int winHandleOpen( FILE_ATTRIBUTE_NORMAL|flag_overlapped, NULL ); -#endif }else{ /* Due to pre-processor directives earlier in this file, ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ @@ -51506,6 +52953,60 @@ static int winHandleOpen( return rc; } +/* +** Close pDbFd's connection to shared-memory. Delete the underlying +** *-shm file if deleteFlag is true. +*/ +static int winCloseSharedMemory(winFile *pDbFd, int deleteFlag){ + winShm *p; /* The connection to be closed */ + winShm **pp; /* Iterator for pShmNode->pWinShmList */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + if( p->hShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(p->hShm); + } + + winShmEnterMutex(); + pShmNode = p->pShmNode; + + /* Remove this connection from the winShmNode.pWinShmList list */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pWinShmList; *pp!=p; pp=&(*pp)->pWinShmNext){} + *pp = p->pWinShmNext; + sqlite3_mutex_leave(pShmNode->mutex); + + winShmPurge(pDbFd->pVfs, deleteFlag); + winShmLeaveMutex(); + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + return SQLITE_OK; +} + +/* +** testfixture builds may set this global variable to true via a +** Tcl interface. This forces the VFS to use the locking normally +** only used for UNC paths for all files. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_win_test_unc_locking = 0; +#else +# define sqlite3_win_test_unc_locking 0 +#endif + +/* +** Return true if the string passed as the only argument is likely +** to be a UNC path. In other words, if it starts with "\\". +*/ +static int winIsUNCPath(const char *zFile){ + if( zFile[0]=='\\' && zFile[1]=='\\' ){ + return 1; + } + return sqlite3_win_test_unc_locking; +} /* ** Open the shared-memory area associated with database file pDbFd. @@ -51532,15 +53033,10 @@ static int winOpenSharedMemory(winFile *pDbFd){ pNew->zFilename = (char*)&pNew[1]; pNew->hSharedShm = INVALID_HANDLE_VALUE; pNew->isUnlocked = 1; + pNew->bUseSharedLockHandle = winIsUNCPath(pDbFd->zPath); sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); - /* Open a file-handle on the *-shm file for this connection. This file-handle - ** is only used for locking. The mapping of the *-shm file is created using - ** the shared file handle in winShmNode.hSharedShm. */ - p->bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0); - rc = winHandleOpen(pNew->zFilename, &p->bReadonly, &p->hShm); - /* Look to see if there is an existing winShmNode that can be used. ** If no matching winShmNode currently exists, then create a new one. */ winShmEnterMutex(); @@ -51561,7 +53057,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Open a file-handle to use for mappings, and for the DMS lock. */ if( rc==SQLITE_OK ){ HANDLE h = INVALID_HANDLE_VALUE; - pShmNode->isReadonly = p->bReadonly; + pShmNode->isReadonly = sqlite3_uri_boolean(pDbFd->zPath,"readonly_shm",0); rc = winHandleOpen(pNew->zFilename, &pShmNode->isReadonly, &h); pShmNode->hSharedShm = h; } @@ -51583,20 +53079,35 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* If no error has occurred, link the winShm object to the winShmNode and ** the winShm to pDbFd. */ if( rc==SQLITE_OK ){ + sqlite3_mutex_enter(pShmNode->mutex); p->pShmNode = pShmNode; - pShmNode->nRef++; + p->pWinShmNext = pShmNode->pWinShmList; + pShmNode->pWinShmList = p; #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) p->id = pShmNode->nextShmId++; #endif pDbFd->pShm = p; + sqlite3_mutex_leave(pShmNode->mutex); }else if( p ){ - winHandleClose(p->hShm); sqlite3_free(p); } assert( rc!=SQLITE_OK || pShmNode->isUnlocked==0 || pShmNode->nRegion==0 ); winShmLeaveMutex(); sqlite3_free(pNew); + + /* Open a file-handle on the *-shm file for this connection. This file-handle + ** is only used for locking. The mapping of the *-shm file is created using + ** the shared file handle in winShmNode.hSharedShm. */ + if( rc==SQLITE_OK && pShmNode->bUseSharedLockHandle==0 ){ + p->bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0); + rc = winHandleOpen(pShmNode->zFilename, &p->bReadonly, &p->hShm); + if( rc!=SQLITE_OK ){ + assert( p->hShm==INVALID_HANDLE_VALUE ); + winCloseSharedMemory(pDbFd, 0); + } + } + return rc; } @@ -51608,33 +53119,7 @@ static int winShmUnmap( sqlite3_file *fd, /* Database holding shared memory */ int deleteFlag /* Delete after closing if true */ ){ - winFile *pDbFd; /* Database holding shared-memory */ - winShm *p; /* The connection to be closed */ - winShmNode *pShmNode; /* The underlying shared-memory file */ - - pDbFd = (winFile*)fd; - p = pDbFd->pShm; - if( p==0 ) return SQLITE_OK; - if( p->hShm!=INVALID_HANDLE_VALUE ){ - osCloseHandle(p->hShm); - } - - pShmNode = p->pShmNode; - winShmEnterMutex(); - - /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too. */ - assert( pShmNode->nRef>0 ); - pShmNode->nRef--; - if( pShmNode->nRef==0 ){ - winShmPurge(pDbFd->pVfs, deleteFlag); - } - winShmLeaveMutex(); - - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; - return SQLITE_OK; + return winCloseSharedMemory((winFile*)fd, deleteFlag); } /* @@ -51703,6 +53188,7 @@ static int winShmLock( || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) ){ + HANDLE h = p->hShm; if( flags & SQLITE_SHM_UNLOCK ){ /* Case (a) - unlock. */ @@ -51711,7 +53197,27 @@ static int winShmLock( assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); - rc = winHandleUnlock(p->hShm, ofst+WIN_SHM_BASE, n); + assert( !(flags & SQLITE_SHM_SHARED) || n==1 ); + if( pShmNode->bUseSharedLockHandle ){ + h = pShmNode->hSharedShm; + if( flags & SQLITE_SHM_SHARED ){ + winShm *pShm; + sqlite3_mutex_enter(pShmNode->mutex); + for(pShm=pShmNode->pWinShmList; pShm; pShm=pShm->pWinShmNext){ + if( pShm!=p && (pShm->sharedMask & mask) ){ + /* Another connection within this process is also holding this + ** SHARED lock. So do not actually release the OS lock. */ + h = INVALID_HANDLE_VALUE; + break; + } + } + sqlite3_mutex_leave(pShmNode->mutex); + } + } + + if( h!=INVALID_HANDLE_VALUE ){ + rc = winHandleUnlock(h, ofst+WIN_SHM_BASE, n); + } /* If successful, also clear the bits in sharedMask/exclMask */ if( rc==SQLITE_OK ){ @@ -51721,7 +53227,32 @@ static int winShmLock( }else{ int bExcl = ((flags & SQLITE_SHM_EXCLUSIVE) ? 1 : 0); DWORD nMs = winFileBusyTimeout(pDbFd); - rc = winHandleLockTimeout(p->hShm, ofst+WIN_SHM_BASE, n, bExcl, nMs); + + if( pShmNode->bUseSharedLockHandle ){ + winShm *pShm; + h = pShmNode->hSharedShm; + sqlite3_mutex_enter(pShmNode->mutex); + for(pShm=pShmNode->pWinShmList; pShm; pShm=pShm->pWinShmNext){ + if( bExcl ){ + if( (pShm->sharedMask|pShm->exclMask) & mask ){ + rc = SQLITE_BUSY; + h = INVALID_HANDLE_VALUE; + } + }else{ + if( pShm->sharedMask & mask ){ + h = INVALID_HANDLE_VALUE; + }else if( pShm->exclMask & mask ){ + rc = SQLITE_BUSY; + h = INVALID_HANDLE_VALUE; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); + } + + if( h!=INVALID_HANDLE_VALUE ){ + rc = winHandleLockTimeout(h, ofst+WIN_SHM_BASE, n, bExcl, nMs); + } if( rc==SQLITE_OK ){ if( bExcl ){ p->exclMask = (p->exclMask | mask); @@ -51858,9 +53389,7 @@ static int winShmMap( HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ -#if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); -#elif defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); @@ -51872,15 +53401,9 @@ static int winShmMap( if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; -#if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, flags, - iOffset - iOffsetShift, szRegion + iOffsetShift - ); -#else pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); -#endif OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); @@ -52013,9 +53536,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #endif -#if SQLITE_OS_WINRT - pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); -#elif defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -52035,11 +53556,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nMap % winSysInfo.dwPageSize)==0 ); assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); -#if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); -#else pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); -#endif if( pNew==NULL ){ osCloseHandle(pFd->hMap); pFd->hMap = NULL; @@ -52374,7 +53891,6 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #endif -#if !SQLITE_OS_WINRT && defined(_WIN32) else if( osIsNT() ){ char *zMulti; LPWSTR zWidePath = sqlite3MallocZero( nMax*sizeof(WCHAR) ); @@ -52428,7 +53944,6 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } } #endif /* SQLITE_WIN32_HAS_ANSI */ -#endif /* !SQLITE_OS_WINRT */ /* ** Check to make sure the temporary directory ends with an appropriate @@ -52603,13 +54118,6 @@ static int winOpen( memset(pFile, 0, sizeof(winFile)); pFile->h = INVALID_HANDLE_VALUE; -#if SQLITE_OS_WINRT - if( !zUtf8Name && !sqlite3_temp_directory ){ - sqlite3_log(SQLITE_ERROR, - "sqlite3_temp_directory variable should be set for WinRT"); - } -#endif - /* If the second argument to this function is NULL, generate a ** temporary file name to use */ @@ -52692,31 +54200,6 @@ static int winOpen( #endif if( osIsNT() ){ -#if SQLITE_OS_WINRT - CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; - extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); - extendedParameters.dwFileAttributes = - dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; - extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; - extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; - extendedParameters.lpSecurityAttributes = NULL; - extendedParameters.hTemplateFile = NULL; - do{ - h = osCreateFile2((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, - dwCreationDisposition, - &extendedParameters); - if( h!=INVALID_HANDLE_VALUE ) break; - if( isReadWrite ){ - int rc2; - sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); - sqlite3EndBenignMalloc(); - if( rc2==SQLITE_OK && isRO ) break; - } - }while( winRetryIoerr(&cnt, &lastErrno) ); -#else do{ h = osCreateFileW((LPCWSTR)zConverted, dwDesiredAccess, @@ -52733,7 +54216,6 @@ static int winOpen( if( rc2==SQLITE_OK && isRO ) break; } }while( winRetryIoerr(&cnt, &lastErrno) ); -#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -52870,25 +54352,7 @@ static int winDelete( } if( osIsNT() ){ do { -#if SQLITE_OS_WINRT - WIN32_FILE_ATTRIBUTE_DATA sAttrData; - memset(&sAttrData, 0, sizeof(sAttrData)); - if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, - &sAttrData) ){ - attr = sAttrData.dwFileAttributes; - }else{ - lastErrno = osGetLastError(); - if( lastErrno==ERROR_FILE_NOT_FOUND - || lastErrno==ERROR_PATH_NOT_FOUND ){ - rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ - }else{ - rc = SQLITE_ERROR; - } - break; - } -#else attr = osGetFileAttributesW(zConverted); -#endif if ( attr==INVALID_FILE_ATTRIBUTES ){ lastErrno = osGetLastError(); if( lastErrno==ERROR_FILE_NOT_FOUND @@ -53011,6 +54475,7 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ + if( noRetry ) lastErrno = osGetLastError(); winLogIoerr(cnt, __LINE__); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ sqlite3_free(zConverted); @@ -53179,7 +54644,7 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE int nByte; void *zConverted; char *zOut; @@ -53268,7 +54733,7 @@ static int winFullPathnameNoMutex( } #endif /* __CYGWIN__ */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) +#if SQLITE_OS_WINCE && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -53287,7 +54752,7 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE #if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -53419,11 +54884,7 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ return 0; } if( osIsNT() ){ -#if SQLITE_OS_WINRT - h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); -#else h = osLoadLibraryW((LPCWSTR)zConverted); -#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -53505,23 +54966,16 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ DWORD pid = osGetCurrentProcessId(); xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } -#if SQLITE_OS_WINRT - { - ULONGLONG cnt = osGetTickCount64(); - xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); - } -#else { DWORD cnt = osGetTickCount(); xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } -#endif /* SQLITE_OS_WINRT */ { LARGE_INTEGER i; osQueryPerformanceCounter(&i); xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { UUID id; memset(&id, 0, sizeof(UUID)); @@ -53531,7 +54985,7 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ osUuidCreateSequential(&id); xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); } -#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ +#endif /* !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID */ return e.nXor>nBuf ? nBuf : e.nXor; #endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -53762,15 +55216,16 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==89 ); + assert( ArraySize(aSyscall)==81 ); + assert( strcmp(aSyscall[0].zName,"AreFileApisANSI")==0 ); + assert( strcmp(aSyscall[20].zName,"GetFileAttributesA")==0 ); + assert( strcmp(aSyscall[40].zName,"HeapReAlloc")==0 ); + assert( strcmp(aSyscall[60].zName,"WideCharToMultiByte")==0 ); + assert( strcmp(aSyscall[80].zName,"cygwin_conv_path")==0 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); -#if SQLITE_OS_WINRT - osGetNativeSystemInfo(&winSysInfo); -#else osGetSystemInfo(&winSysInfo); -#endif assert( winSysInfo.dwAllocationGranularity>0 ); assert( winSysInfo.dwPageSize>0 ); @@ -53794,17 +55249,9 @@ SQLITE_API int sqlite3_os_init(void){ } SQLITE_API int sqlite3_os_end(void){ -#if SQLITE_OS_WINRT - if( sleepObj!=NULL ){ - osCloseHandle(sleepObj); - sleepObj = NULL; - } -#endif - #ifndef SQLITE_OMIT_WAL winBigLock = 0; #endif - return SQLITE_OK; } @@ -54690,10 +56137,10 @@ SQLITE_API int sqlite3_deserialize( if( rc ) goto end_deserialize; db->init.iDb = (u8)iDb; db->init.reopenMemdb = 1; - rc = sqlite3_step(pStmt); + sqlite3_step(pStmt); db->init.reopenMemdb = 0; - if( rc!=SQLITE_DONE ){ - rc = SQLITE_ERROR; + rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK ){ goto end_deserialize; } p = memdbFromDbSchema(db, zSchema); @@ -54714,7 +56161,6 @@ SQLITE_API int sqlite3_deserialize( } end_deserialize: - sqlite3_finalize(pStmt); if( pData && (mFlags & SQLITE_DESERIALIZE_FREEONCLOSE)!=0 ){ sqlite3_free(pData); } @@ -54860,6 +56306,7 @@ struct Bitvec { } u; }; + /* ** Create a new bitmap object able to handle bits between 0 and iSize, ** inclusive. Return a pointer to the new object. Return NULL if @@ -55048,6 +56495,52 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ return p->iSize; } +#ifdef SQLITE_DEBUG +/* +** Show the content of a Bitvec option and its children. Indent +** everything by n spaces. Add x to each bitvec value. +** +** From a debugger such as gdb, one can type: +** +** call sqlite3ShowBitvec(p) +** +** For some Bitvec p and see a recursive view of the Bitvec's content. +*/ +static void showBitvec(Bitvec *p, int n, unsigned x){ + int i; + if( p==0 ){ + printf("NULL\n"); + return; + } + printf("Bitvec 0x%p iSize=%u", p, p->iSize); + if( p->iSize<=BITVEC_NBIT ){ + printf(" bitmap\n"); + printf("%*s bits:", n, ""); + for(i=1; i<=BITVEC_NBIT; i++){ + if( sqlite3BitvecTest(p,i) ) printf(" %u", x+(unsigned)i); + } + printf("\n"); + }else if( p->iDivisor==0 ){ + printf(" hash with %u entries\n", p->nSet); + printf("%*s bits:", n, ""); + for(i=0; i<BITVEC_NINT; i++){ + if( p->u.aHash[i] ) printf(" %u", x+(unsigned)p->u.aHash[i]); + } + printf("\n"); + }else{ + printf(" sub-bitvec with iDivisor=%u\n", p->iDivisor); + for(i=0; i<BITVEC_NPTR; i++){ + if( p->u.apSub[i]==0 ) continue; + printf("%*s apSub[%d]=", n, "", i); + showBitvec(p->u.apSub[i], n+4, i*p->iDivisor); + } + } +} +SQLITE_PRIVATE void sqlite3ShowBitvec(Bitvec *p){ + showBitvec(p, 0, 0); +} +#endif + #ifndef SQLITE_UNTESTABLE /* ** Let V[] be an array of unsigned characters sufficient to hold @@ -55059,6 +56552,7 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ #define CLEARBIT(V,I) V[I>>3] &= ~(BITVEC_TELEM)(1<<(I&7)) #define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + /* ** This routine runs an extensive test of the Bitvec code. ** @@ -55067,7 +56561,7 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ ** by 0, 1, or 3 operands, depending on the opcode. Another ** opcode follows immediately after the last operand. ** -** There are 6 opcodes numbered from 0 through 5. 0 is the +** There are opcodes numbered starting with 0. 0 is the ** "halt" opcode and causes the test to end. ** ** 0 Halt and return the number of errors @@ -55076,18 +56570,25 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ ** 3 N Set N randomly chosen bits ** 4 N Clear N randomly chosen bits ** 5 N S X Set N bits from S increment X in array only, not in bitvec +** 6 Invoice sqlite3ShowBitvec() on the Bitvec object so far +** 7 X Show compile-time parameters and the hash of X ** ** The opcodes 1 through 4 perform set and clear operations are performed ** on both a Bitvec object and on a linear array of bits obtained from malloc. ** Opcode 5 works on the linear array only, not on the Bitvec. ** Opcode 5 is used to deliberately induce a fault in order to -** confirm that error detection works. +** confirm that error detection works. Opcodes 6 and greater are +** state output opcodes. Opcodes 6 and greater are no-ops unless +** SQLite has been compiled with SQLITE_DEBUG. ** ** At the conclusion of the test the linear array is compared ** against the Bitvec object. If there are any differences, ** an error is returned. If they are the same, zero is returned. ** ** If a memory allocation error occurs, return -1. +** +** sz is the size of the Bitvec. Or if sz is negative, make the size +** 2*(unsigned)(-sz) and disabled the linear vector check. */ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ Bitvec *pBitvec = 0; @@ -55098,10 +56599,15 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Allocate the Bitvec to be tested and a linear array of ** bits to act as the reference */ - pBitvec = sqlite3BitvecCreate( sz ); - pV = sqlite3MallocZero( (7+(i64)sz)/8 + 1 ); + if( sz<=0 ){ + pBitvec = sqlite3BitvecCreate( 2*(unsigned)(-sz) ); + pV = 0; + }else{ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3MallocZero( (7+(i64)sz)/8 + 1 ); + } pTmpSpace = sqlite3_malloc64(BITVEC_SZ); - if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; + if( pBitvec==0 || pTmpSpace==0 || (pV==0 && sz>0) ) goto bitvec_end; /* NULL pBitvec tests */ sqlite3BitvecSet(0, 1); @@ -55110,6 +56616,24 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Run the program */ pc = i = 0; while( (op = aOp[pc])!=0 ){ + if( op>=6 ){ +#ifdef SQLITE_DEBUG + if( op==6 ){ + sqlite3ShowBitvec(pBitvec); + }else if( op==7 ){ + printf("BITVEC_SZ = %d (%d by sizeof)\n", + BITVEC_SZ, (int)sizeof(Bitvec)); + printf("BITVEC_USIZE = %d\n", (int)BITVEC_USIZE); + printf("BITVEC_NELEM = %d\n", (int)BITVEC_NELEM); + printf("BITVEC_NBIT = %d\n", (int)BITVEC_NBIT); + printf("BITVEC_NINT = %d\n", (int)BITVEC_NINT); + printf("BITVEC_MXHASH = %d\n", (int)BITVEC_MXHASH); + printf("BITVEC_NPTR = %d\n", (int)BITVEC_NPTR); + } +#endif + pc++; + continue; + } switch( op ){ case 1: case 2: @@ -55131,12 +56655,12 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ pc += nx; i = (i & 0x7fffffff)%sz; if( (op & 1)!=0 ){ - SETBIT(pV, (i+1)); + if( pV ) SETBIT(pV, (i+1)); if( op!=5 ){ if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; } }else{ - CLEARBIT(pV, (i+1)); + if( pV ) CLEARBIT(pV, (i+1)); sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); } } @@ -55146,14 +56670,18 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ** match (rc==0). Change rc to non-zero if a discrepancy ** is found. */ - rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) - + sqlite3BitvecTest(pBitvec, 0) - + (sqlite3BitvecSize(pBitvec) - sz); - for(i=1; i<=sz; i++){ - if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ - rc = i; - break; + if( pV ){ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } } + }else{ + rc = 0; } /* Free allocated structure */ @@ -58871,6 +60399,8 @@ SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); if( iRead ) return 0; /* Case (4) */ } +#else + UNUSED_PARAMETER(pgno); #endif assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) @@ -59291,17 +60821,17 @@ static int jrnlBufferSize(Pager *pPager){ */ #ifdef SQLITE_CHECK_PAGES /* -** Return a 32-bit hash of the page data for pPage. +** Return a 64-bit hash of the page data for pPage. */ -static u32 pager_datahash(int nByte, unsigned char *pData){ - u32 hash = 0; +static u64 pager_datahash(int nByte, unsigned char *pData){ + u64 hash = 0; int i; for(i=0; i<nByte; i++){ hash = (hash*1039) + pData[i]; } return hash; } -static u32 pager_pagehash(PgHdr *pPage){ +static u64 pager_pagehash(PgHdr *pPage){ return pager_datahash(pPage->pPager->pageSize, (unsigned char *)pPage->pData); } static void pager_set_pagehash(PgHdr *pPage){ @@ -59914,7 +61444,7 @@ static void pager_unlock(Pager *pPager){ ** have sqlite3WalEndReadTransaction() drop the write-lock, as it once ** did, because this would break "BEGIN EXCLUSIVE" handling for ** SQLITE_ENABLE_SETLK_TIMEOUT builds. */ - sqlite3WalEndWriteTransaction(pPager->pWal); + (void)sqlite3WalEndWriteTransaction(pPager->pWal); } sqlite3WalEndReadTransaction(pPager->pWal); pPager->eState = PAGER_OPEN; @@ -61670,14 +63200,27 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - if( pPager->tempFile ){ + if( pPager->tempFile || level==PAGER_SYNCHRONOUS_OFF ){ pPager->noSync = 1; pPager->fullSync = 0; pPager->extraSync = 0; }else{ - pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->noSync = 0; pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; - pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + + /* Set Pager.extraSync if "PRAGMA synchronous=EXTRA" is requested, or + ** if the file-system supports F2FS style atomic writes. If this flag + ** is set, SQLite syncs the directory to disk immediately after deleting + ** a journal file in "PRAGMA journal_mode=DELETE" mode. */ + if( level==PAGER_SYNCHRONOUS_EXTRA +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (sqlite3OsDeviceCharacteristics(pPager->fd) & SQLITE_IOCAP_BATCH_ATOMIC) +#endif + ){ + pPager->extraSync = 1; + }else{ + pPager->extraSync = 0; + } } if( pPager->noSync ){ pPager->syncFlags = 0; @@ -62237,6 +63780,8 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); pPager->pWal = 0; } +#else + UNUSED_PARAMETER(db); #endif pager_reset(pPager); if( MEMDB ){ @@ -65570,7 +67115,7 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint( } if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, - (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + (eMode<=SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt @@ -66480,7 +68025,7 @@ struct WalIterator { /* Size (in bytes) of a WalIterator object suitable for N or fewer segments */ #define SZ_WALITERATOR(N) \ - (offsetof(WalIterator,aSegment)*(N)*sizeof(struct WalSegment)) + (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment)) /* ** Define the parameters of the hash tables in the wal-index file. There @@ -67004,7 +68549,7 @@ static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ /* ** Compute a hash on a page number. The resulting hash value must land -** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances +** between 0 and (HASHTABLE_NSLOT-1). The walNextHash() function advances ** the hash to the next value in the event of a collision. */ static int walHash(u32 iPage){ @@ -67212,7 +68757,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - sLoc.aPgno[idx-1] = iPage; + sLoc.aPgno[(idx-1)&(HASHTABLE_NPAGE-1)] = iPage; AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -68132,68 +69677,82 @@ static int walCheckpoint( && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ u32 nBackfill = pInfo->nBackfill; - pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; + WalIndexHdr *pLive = (WalIndexHdr*)walIndexHdr(pWal); - /* Sync the WAL to disk */ - rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); + /* Now that read-lock slot 0 is locked, check that the wal has not been + ** wrapped since the header was read for this checkpoint. If it was, then + ** there was no work to do anyway. In this case the + ** (pInfo->nBackfill<pWal->hdr.mxFrame) test above only passed because + ** pInfo->nBackfill had already been set to 0 by the writer that wrapped + ** the wal file. It would also be dangerous to proceed, as there may be + ** fewer than pWal->hdr.mxFrame valid frames in the wal file. */ + int bChg = memcmp(pLive->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)); + if( 0==bChg ){ + pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; - /* If the database may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ - if( rc==SQLITE_OK ){ - i64 nReq = ((i64)mxPage * szPage); - i64 nSize; /* Current size of database file */ - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSize<nReq ){ - if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){ - /* If the size of the final database is larger than the current - ** database plus the amount of data in the wal file, plus the - ** maximum size of the pending-byte page (65536 bytes), then - ** must be corruption somewhere. */ - rc = SQLITE_CORRUPT_BKPT; - }else{ - sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); - } - } + /* Sync the WAL to disk */ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); - } + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSize<nReq ){ + if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){ + /* If the size of the final database is larger than the current + ** database plus the amount of data in the wal file, plus the + ** maximum size of the pending-byte page (65536 bytes), then + ** must be corruption somewhere. */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } - /* Iterate through the contents of the WAL, copying data to the db file */ - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ - i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); - SEH_INJECT_FAULT; - if( AtomicLoad(&db->u1.isInterrupted) ){ - rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; - break; - } - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ - continue; } - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - iOffset = (iDbpage-1)*(i64)szPage; - testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - } - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); - /* If work was actually accomplished... */ - if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK ){ - rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + /* Iterate through the contents of the WAL, copying data to the + ** db file */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + SEH_INJECT_FAULT; + if( AtomicLoad(&db->u1.isInterrupted) ){ + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; + break; + } + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ + continue; } + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; } + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); + + /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ - AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + } + } + if( rc==SQLITE_OK ){ + AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; + } } } @@ -69366,7 +70925,7 @@ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ assert( pWal->writeLock==0 || pWal->readLock<0 ); #endif if( pWal->readLock>=0 ){ - sqlite3WalEndWriteTransaction(pWal); + (void)sqlite3WalEndWriteTransaction(pWal); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; } @@ -69446,7 +71005,10 @@ static int walFindFrame( SEH_INJECT_FAULT; while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){ + if( iFrame<=iLast + && iFrame>=pWal->minFrame + && sLoc.aPgno[(iH-1)&(HASHTABLE_NPAGE-1)]==pgno + ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -70175,7 +71737,8 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ - assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + assert( SQLITE_CHECKPOINT_NOOP<SQLITE_CHECKPOINT_PASSIVE ); + assert( eMode>SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); @@ -70192,31 +71755,35 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, ** it will not be invoked in this case. */ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); - testcase( rc==SQLITE_BUSY ); - testcase( rc!=SQLITE_OK && xBusy2!=0 ); - if( rc==SQLITE_OK ){ - pWal->ckptLock = 1; + if( eMode!=SQLITE_CHECKPOINT_NOOP ){ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ + pWal->ckptLock = 1; - /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and - ** TRUNCATE modes also obtain the exclusive "writer" lock on the database - ** file. - ** - ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained - ** immediately, and a busy-handler is configured, it is invoked and the - ** writer lock retried until either the busy-handler returns 0 or the - ** lock is successfully obtained. - */ - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ - rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ){ - pWal->writeLock = 1; - }else if( rc==SQLITE_BUSY ){ - eMode2 = SQLITE_CHECKPOINT_PASSIVE; - xBusy2 = 0; - rc = SQLITE_OK; + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART + ** and TRUNCATE modes also obtain the exclusive "writer" lock on the + ** database file. + ** + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; + rc = SQLITE_OK; + } } } + }else{ + rc = SQLITE_OK; } @@ -70230,7 +71797,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( ** immediately and do a partial checkpoint if it cannot obtain it. */ walDisableBlocking(pWal); rc = walIndexReadHdr(pWal, &isChanged); - if( eMode2!=SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); + if( eMode2>SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ sqlite3OsUnfetch(pWal->pDbFd, 0, 0); } @@ -70238,9 +71805,10 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ + sqlite3FaultSim(660); if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; - }else{ + }else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){ rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); } @@ -70268,7 +71836,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( sqlite3WalDb(pWal, 0); /* Release the locks. */ - sqlite3WalEndWriteTransaction(pWal); + (void)sqlite3WalEndWriteTransaction(pWal); if( pWal->ckptLock ){ walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; @@ -72425,7 +73993,7 @@ static int btreeMoveto( assert( nKey==(i64)(int)nKey ); pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + sqlite3VdbeRecordUnpack((int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ rc = SQLITE_CORRUPT_BKPT; }else{ @@ -72815,7 +74383,7 @@ static void btreeParseCellPtr( CellInfo *pInfo /* Fill in this structure */ ){ u8 *pIter; /* For scanning through pCell */ - u32 nPayload; /* Number of bytes of cell payload */ + u64 nPayload; /* Number of bytes of cell payload */ u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -72837,6 +74405,7 @@ static void btreeParseCellPtr( do{ nPayload = (nPayload<<7) | (*++pIter & 0x7f); }while( (*pIter)>=0x80 && pIter<pEnd ); + nPayload &= 0xffffffff; } pIter++; @@ -72880,11 +74449,10 @@ static void btreeParseCellPtr( pIter++; pInfo->nKey = *(i64*)&iKey; - pInfo->nPayload = nPayload; + pInfo->nPayload = (u32)nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); - assert( nPayload>=0 ); assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits @@ -73482,10 +75050,10 @@ static int freeSpace(MemPage *pPage, int iStart, int iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= (int)pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=(int)pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. @@ -74409,6 +75977,7 @@ static int removeFromSharingList(BtShared *pBt){ sqlite3_mutex_leave(pMainMtx); return removed; #else + UNUSED_PARAMETER( pBt ); return 1; #endif } @@ -74626,6 +76195,10 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, sqlite3BtreeEnter(p); pBt->nReserveWanted = (u8)nReserve; x = pBt->pageSize - pBt->usableSize; + if( x==nReserve && (pageSize==0 || (u32)pageSize==pBt->pageSize) ){ + sqlite3BtreeLeave(p); + return SQLITE_OK; + } if( nReserve<x ) nReserve = x; if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); @@ -75221,6 +76794,30 @@ static SQLITE_NOINLINE int btreeBeginTrans( } #endif +#ifdef SQLITE_EXPERIMENTAL_PRAGMA_20251114 + /* If both a read and write transaction will be opened by this call, + ** then issue a file-control as if the following pragma command had + ** been evaluated: + ** + ** PRAGMA experimental_pragma_20251114 = 1|2 + ** + ** where the RHS is "1" if wrflag is 1 (RESERVED lock), or "2" if wrflag + ** is 2 (EXCLUSIVE lock). Ignore any result or error returned by the VFS. + ** + ** WARNING: This code will likely remain part of SQLite only temporarily - + ** it exists to allow users to experiment with certain types of blocking + ** locks in custom VFS implementations. It MAY BE REMOVED AT ANY TIME. */ + if( pBt->pPage1==0 && wrflag ){ + sqlite3_file *fd = sqlite3PagerFile(pPager); + char *aFcntl[3] = {0,0,0}; + aFcntl[1] = "experimental_pragma_20251114"; + assert( wrflag==1 || wrflag==2 ); + aFcntl[2] = (wrflag==1 ? "1" : "2"); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + sqlite3_free(aFcntl[0]); + } +#endif + /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after @@ -76668,7 +78265,7 @@ static int accessPayload( getCellInfo(pCur); aPayload = pCur->info.pPayload; - assert( offset+amt <= pCur->info.nPayload ); + assert( (u64)offset+(u64)amt <= (u64)pCur->info.nPayload ); assert( aPayload > pPage->aData ); if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ @@ -77215,6 +78812,30 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +/* Set *pRes to 1 (true) if the BTree pointed to by cursor pCur contains zero +** rows of content. Set *pRes to 0 (false) if the table contains content. +** Return SQLITE_OK on success or some error code (ex: SQLITE_NOMEM) if +** something goes wrong. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + if( NEVER(pCur->eState==CURSOR_VALID) ){ + *pRes = 0; + return SQLITE_OK; + } + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ + *pRes = 1; + rc = SQLITE_OK; + }else{ + *pRes = 0; + } + return rc; +} + #ifdef SQLITE_DEBUG /* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that ** this flags are true for a consistent database. @@ -77434,8 +79055,8 @@ moveto_table_finish: } /* -** Compare the "idx"-th cell on the page the cursor pCur is currently -** pointing to to pIdxKey using xRecordCompare. Return negative or +** Compare the "idx"-th cell on the page pPage against the key +** pointing to by pIdxKey using xRecordCompare. Return negative or ** zero if the cell is less than or equal pIdxKey. Return positive ** if unknown. ** @@ -77450,12 +79071,11 @@ moveto_table_finish: ** a positive value as that will cause the optimization to be skipped. */ static int indexCellCompare( - BtCursor *pCur, + MemPage *pPage, int idx, UnpackedRecord *pIdxKey, RecordCompare xRecordCompare ){ - MemPage *pPage = pCur->pPage; int c; int nCell; /* Size of the pCell cell in bytes */ u8 *pCell = findCellPastPtr(pPage, idx); @@ -77564,14 +79184,14 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( ){ int c; if( pCur->ix==pCur->pPage->nCell-1 - && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0 + && (c = indexCellCompare(pCur->pPage,pCur->ix,pIdxKey,xRecordCompare))<=0 && pIdxKey->errCode==SQLITE_OK ){ *pRes = c; return SQLITE_OK; /* Cursor already pointing at the correct spot */ } if( pCur->iPage>0 - && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 + && indexCellCompare(pCur->pPage, 0, pIdxKey, xRecordCompare)<=0 && pIdxKey->errCode==SQLITE_OK ){ pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); @@ -77788,7 +79408,7 @@ SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ n = pCur->pPage->nCell; for(i=0; i<pCur->iPage; i++){ - n *= pCur->apPage[i]->nCell; + n *= pCur->apPage[i]->nCell+1; } return n; } @@ -80245,7 +81865,12 @@ static int balance_nonroot( ** of the right-most new sibling page is set to the value that was ** originally in the same field of the right-most old sibling page. */ if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ - MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + MemPage *pOld; + if( nNew>nOld ){ + pOld = apNew[nOld-1]; + }else{ + pOld = apOld[nOld-1]; + } memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); } @@ -81278,7 +82903,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 }while( rc==SQLITE_OK && nOut>0 ); if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ - Pgno pgnoNew; + Pgno pgnoNew = 0; /* Prevent harmless static-analyzer warning */ MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); @@ -82877,6 +84502,7 @@ SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void */ SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ int rc; + UNUSED_PARAMETER(p); /* only used in DEBUG builds */ assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); @@ -83943,21 +85569,27 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ StrAccum acc; assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); assert( sz>22 ); - if( p->flags & MEM_Int ){ -#if GCC_VERSION>=7000000 - /* Work-around for GCC bug - ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 */ + if( p->flags & (MEM_Int|MEM_IntReal) ){ +#if GCC_VERSION>=7000000 && GCC_VERSION<15000000 && defined(__i386__) + /* Work-around for GCC bug or bugs: + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114659 + ** The problem appears to be fixed in GCC 15 */ i64 x; - assert( (p->flags&MEM_Int)*2==sizeof(x) ); - memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); + assert( (MEM_Str&~p->flags)*4==sizeof(x) ); + memcpy(&x, (char*)&p->u, (MEM_Str&~p->flags)*4); p->n = sqlite3Int64ToText(x, zBuf); #else p->n = sqlite3Int64ToText(p->u.i, zBuf); #endif + if( p->flags & MEM_IntReal ){ + memcpy(zBuf+p->n,".0", 3); + p->n += 2; + } }else{ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); - sqlite3_str_appendf(&acc, "%!.15g", - (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); + sqlite3_str_appendf(&acc, "%!.*g", + (p->db ? p->db->nFpDigit : 17), p->u.r); assert( acc.zText==zBuf && acc.mxAlloc<=0 ); zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ p->n = acc.nChar; @@ -84006,6 +85638,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + if( p->db==0 ){ + return 1; /* db->nFpDigit required to validate p->z[] */ + } memcpy(&tmp, p, sizeof(tmp)); vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); z = p->z; @@ -84156,13 +85791,16 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ ** ** This is an optimization. Correct operation continues even if ** this routine is a no-op. +** +** Return true if the strig is zero-terminated after this routine is +** called and false if it is not. */ -SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ +SQLITE_PRIVATE int sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ /* pMem must be a string, and it cannot be an ephemeral or static string */ - return; + return 0; } - if( pMem->enc!=SQLITE_UTF8 ) return; + if( pMem->enc!=SQLITE_UTF8 ) return 0; assert( pMem->z!=0 ); if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free @@ -84170,18 +85808,19 @@ SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return; + return 1; } if( pMem->xDel==sqlite3RCStrUnref ){ /* Blindly assume that all RCStr objects are zero-terminated */ pMem->flags |= MEM_Term; - return; + return 1; } }else if( pMem->szMalloc >= pMem->n+1 ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return; + return 1; } + return 0; } /* @@ -84480,17 +86119,116 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem *pMem){ } /* +** This routine implements the uncommon and slower path for +** sqlite3MemRealValueRC() that has to deal with input strings +** that are not UTF8 or that are not zero-terminated. It is +** broken out into a separate no-inline routine so that the +** main sqlite3MemRealValueRC() routine can avoid unnecessary +** stack pushes. +** +** A text->float translation of pMem->z is written into *pValue. +** +** Result code invariants: +** +** rc==0 => ERROR: Input string not well-formed, or OOM +** rc<0 => Some prefix of the input is well-formed +** rc>0 => All of the input is well-formed +** (rc&2)==0 => The number is expressed as an integer, with no +** decimal point or eNNN suffix. +*/ +static SQLITE_NOINLINE int sqlite3MemRealValueRCSlowPath( + Mem *pMem, + double *pValue +){ + int rc = SQLITE_OK; + *pValue = 0.0; + if( pMem->enc==SQLITE_UTF8 ){ + char *zCopy = sqlite3DbStrNDup(pMem->db, pMem->z, pMem->n); + if( zCopy ){ + rc = sqlite3AtoF(zCopy, pValue); + sqlite3DbFree(pMem->db, zCopy); + } + return rc; + }else{ + int n, i, j; + char *zCopy; + const char *z; + + n = pMem->n & ~1; + zCopy = sqlite3DbMallocRaw(pMem->db, n/2 + 2); + if( zCopy ){ + z = pMem->z; + if( pMem->enc==SQLITE_UTF16LE ){ + for(i=j=0; i<n-1; i+=2, j++){ + zCopy[j] = z[i]; + if( z[i+1]!=0 ) break; + } + }else{ + for(i=j=0; i<n-1; i+=2, j++){ + if( z[i]!=0 ) break; + zCopy[j] = z[i+1]; + } + } + assert( j<=n/2 ); + zCopy[j] = 0; + rc = sqlite3AtoF(zCopy, pValue); + if( i<n ) rc = -100; + sqlite3DbFree(pMem->db, zCopy); + } + return rc; + } +} + +/* +** Invoke sqlite3AtoF() on the text value of pMem. Write the +** translation of the text input into *pValue. +** +** The caller must ensure that pMem->db!=0 and that pMem is in +** mode MEM_Str or MEM_Blob. +** +** Result code invariants: +** +** rc==0 => ERROR: Input string not well-formed, or OOM +** rc<0 => Some prefix of the input is well-formed +** rc>0 => All of the input is well-formed +** (rc&2)==0 => The number is expressed as an integer, with no +** decimal point or eNNN suffix. +*/ +SQLITE_PRIVATE int sqlite3MemRealValueRC(Mem *pMem, double *pValue){ + testcase( pMem->db==0 ); + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + if( pMem->z==0 ){ + *pValue = 0.0; + return 0; + }else if( pMem->enc==SQLITE_UTF8 + && ((pMem->flags & MEM_Term)!=0 || sqlite3VdbeMemZeroTerminateIfAble(pMem)) + ){ + return sqlite3AtoF(pMem->z, pValue); + }else if( pMem->n==0 ){ + *pValue = 0.0; + return 0; + }else{ + return sqlite3MemRealValueRCSlowPath(pMem, pValue); + } +} + +/* +** This routine acts as a bridge from sqlite3VdbeRealValue() to +** sqlite3VdbeRealValueRC, allowing sqlite3VdbeRealValue() to avoid +** stuffing values onto the stack. +*/ +static SQLITE_NOINLINE double sqlite3MemRealValueNoRC(Mem *pMem){ + double r; + (void)sqlite3MemRealValueRC(pMem, &r); + return r; +} + +/* ** Return the best representation of pMem that we can get into a ** double. If pMem is already a double or an integer, return its ** value. If it is a string or blob, try to convert it to a double. ** If it is a NULL, return 0.0. */ -static SQLITE_NOINLINE double memRealValue(Mem *pMem){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - double val = (double)0; - sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); - return val; -} SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -84501,7 +86239,7 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - return memRealValue(pMem); + return sqlite3MemRealValueNoRC(pMem); }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; @@ -84625,8 +86363,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); - if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + rc = sqlite3MemRealValueRC(pMem, &pMem->u.r); + if( ((rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<2) || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; @@ -85062,6 +86800,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } + assert( pMem->z!=0 ); memcpy(pMem->z, z, nAlloc); }else{ sqlite3VdbeMemRelease(pMem); @@ -85089,6 +86828,84 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( return SQLITE_OK; } +/* Like sqlite3VdbeMemSetStr() except: +** +** enc is always SQLITE_UTF8 +** pMem->db is always non-NULL +*/ +SQLITE_PRIVATE int sqlite3VdbeMemSetText( + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + i64 n, /* Bytes in string, or negative */ + void (*xDel)(void*) /* Destructor function */ +){ + i64 nByte = n; /* New value for pMem->n */ + u16 flags; + + assert( pMem!=0 ); + assert( pMem->db!=0 ); + assert( sqlite3_mutex_held(pMem->db->mutex) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + + /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ + if( !z ){ + sqlite3VdbeMemSetNull(pMem); + return SQLITE_OK; + } + + if( nByte<0 ){ + nByte = strlen(z); + flags = MEM_Str|MEM_Term; + }else{ + flags = MEM_Str; + } + if( nByte>(i64)pMem->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + if( xDel && xDel!=SQLITE_TRANSIENT ){ + if( xDel==SQLITE_DYNAMIC ){ + sqlite3DbFree(pMem->db, (void*)z); + }else{ + xDel((void*)z); + } + } + sqlite3VdbeMemSetNull(pMem); + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); + } + + /* The following block sets the new values of Mem.z and Mem.xDel. It + ** also sets a flag in local variable "flags" to indicate the memory + ** management (one of MEM_Dyn or MEM_Static). + */ + if( xDel==SQLITE_TRANSIENT ){ + i64 nAlloc = nByte + 1; + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ + return SQLITE_NOMEM_BKPT; + } + assert( pMem->z!=0 ); + memcpy(pMem->z, z, nByte); + pMem->z[nByte] = 0; + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->z = (char *)z; + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + pMem->xDel = 0; + }else if( xDel==SQLITE_STATIC ){ + pMem->xDel = xDel; + flags |= MEM_Static; + }else{ + pMem->xDel = xDel; + flags |= MEM_Dyn; + } + } + pMem->flags = flags; + pMem->n = (int)(nByte & 0x7fffffff); + pMem->enc = SQLITE_UTF8; + return SQLITE_OK; +} + /* ** Move data out of a btree key or data field and into a Mem structure. ** The data is payload from the entry that pCur is currently pointing @@ -85112,7 +86929,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( ){ int rc; pMem->flags = MEM_Null; - if( sqlite3BtreeMaxRecordSize(pCur)<offset+amt ){ + testcase( amt==SQLITE_MAX_ALLOCATION_SIZE-1 ); + testcase( amt==SQLITE_MAX_ALLOCATION_SIZE ); + if( amt>=SQLITE_MAX_ALLOCATION_SIZE ){ + return SQLITE_NOMEM_BKPT; + } + if( (u64)amt + (u64)offset > (u64)sqlite3BtreeMaxRecordSize(pCur) ){ return SQLITE_CORRUPT_BKPT; } if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){ @@ -85512,7 +87334,7 @@ static int valueFromExpr( if( affinity==SQLITE_AFF_BLOB ){ if( op==TK_FLOAT ){ assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); - sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + sqlite3AtoF(pVal->z, &pVal->u.r); pVal->flags = MEM_Real; }else if( op==TK_INTEGER ){ /* This case is required by -9223372036854775808 and other strings @@ -85780,6 +87602,11 @@ SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr( ** ** If *ppVal is initially NULL then the caller is responsible for ** ensuring that the value written into *ppVal is eventually freed. +** +** If the buffer does not contain a well-formed record, this routine may +** read several bytes past the end of the buffer. Callers must therefore +** ensure that any buffer which may contain a corrupt record is padded +** with at least 8 bytes of addressable memory. */ SQLITE_PRIVATE int sqlite3Stat4Column( sqlite3 *db, /* Database handle */ @@ -87898,6 +89725,10 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = pOp->p4.pTab->zName; break; } + case P4_INDEX: { + zP4 = pOp->p4.pIdx->zName; + break; + } case P4_SUBRTNSIG: { SubrtnSig *pSig = pOp->p4.pSubrtnSig; sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff); @@ -88796,7 +90627,7 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResAlloc]); - rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); + rc = sqlite3VdbeMemSetText(pColName, zName, -1, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } @@ -88889,10 +90720,12 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) || nTrans<=1 ){ - for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + if( needXcommit ){ + for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)>=SQLITE_TXN_WRITE ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } } } @@ -88903,7 +90736,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ */ for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ){ + int txn = sqlite3BtreeTxnState(pBt); + if( txn!=SQLITE_TXN_NONE ){ + assert( needXcommit || txn==SQLITE_TXN_READ ); rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); } } @@ -89158,28 +90993,31 @@ SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* -** This function is called when a transaction opened by the database +** These functions are called when a transaction opened by the database ** handle associated with the VM passed as an argument is about to be -** committed. If there are outstanding deferred foreign key constraint -** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** committed. If there are outstanding foreign key constraint violations +** return an error code. Otherwise, SQLITE_OK. ** ** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY -** and write an error message to it. Then return SQLITE_ERROR. +** non-zero, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. */ #ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ +static SQLITE_NOINLINE int vdbeFkError(Vdbe *p){ + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; + p->errorAction = OE_Abort; + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; + return SQLITE_CONSTRAINT_FOREIGNKEY; +} +SQLITE_PRIVATE int sqlite3VdbeCheckFkImmediate(Vdbe *p){ + if( p->nFkConstraint==0 ) return SQLITE_OK; + return vdbeFkError(p); +} +SQLITE_PRIVATE int sqlite3VdbeCheckFkDeferred(Vdbe *p){ sqlite3 *db = p->db; - if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) - || (!deferred && p->nFkConstraint>0) - ){ - p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; - p->errorAction = OE_Abort; - sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; - return SQLITE_CONSTRAINT_FOREIGNKEY; - } - return SQLITE_OK; + if( (db->nDeferredCons+db->nDeferredImmCons)==0 ) return SQLITE_OK; + return vdbeFkError(p); } #endif @@ -89273,7 +91111,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - (void)sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFkImmediate(p); } /* If the auto-commit flag is set and this is the only active writer @@ -89287,7 +91125,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ && db->nVdbeWrite==(p->readOnly==0) ){ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - rc = sqlite3VdbeCheckFk(p, 1); + rc = sqlite3VdbeCheckFkDeferred(p); if( rc!=SQLITE_OK ){ if( NEVER(p->readOnly) ){ sqlite3VdbeLeave(p); @@ -90097,30 +91935,22 @@ SQLITE_PRIVATE void sqlite3VdbeSerialGet( return; } /* -** This routine is used to allocate sufficient space for an UnpackedRecord -** structure large enough to be used with sqlite3VdbeRecordUnpack() if -** the first argument is a pointer to KeyInfo structure pKeyInfo. +** Allocate sufficient space for an UnpackedRecord structure large enough +** to hold a decoded index record for pKeyInfo. ** -** The space is either allocated using sqlite3DbMallocRaw() or from within -** the unaligned buffer passed via the second and third arguments (presumably -** stack space). If the former, then *ppFree is set to a pointer that should -** be eventually freed by the caller using sqlite3DbFree(). Or, if the -** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL -** before returning. -** -** If an OOM error occurs, NULL is returned. +** The space is allocated using sqlite3DbMallocRaw(). If an OOM error +** occurs, NULL is returned. */ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( KeyInfo *pKeyInfo /* Description of the record */ ){ UnpackedRecord *p; /* Unpacked record to return */ - int nByte; /* Number of bytes required for *p */ + u64 nByte; /* Number of bytes required for *p */ assert( sizeof(UnpackedRecord) + sizeof(Mem)*65536 < 0x7fffffff ); nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; return p; @@ -90132,7 +91962,6 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( ** contents of the decoded record. */ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ int nKey, /* Size of the binary record */ const void *pKey, /* The binary record */ UnpackedRecord *p /* Populate this structure before returning. */ @@ -90143,6 +91972,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( u16 u; /* Unsigned loop counter */ u32 szHdr; Mem *pMem = p->aMem; + KeyInfo *pKeyInfo = p->pKeyInfo; p->default_rc = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -90160,16 +91990,18 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( pMem->z = 0; sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); d += sqlite3VdbeSerialTypeLen(serial_type); - pMem++; if( (++u)>=p->nField ) break; + pMem++; } if( d>(u32)nKey && u ){ assert( CORRUPT_DB ); /* In a corrupt record entry, the last pMem might have been set up using ** uninitialized memory. Overwrite its value with NULL, to prevent ** warnings from MSAN. */ - sqlite3VdbeMemSetNull(pMem-1); + sqlite3VdbeMemSetNull(pMem-(u<p->nField)); } + testcase( u == pKeyInfo->nKeyField + 1 ); + testcase( u < pKeyInfo->nKeyField + 1 ); assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } @@ -90337,6 +92169,32 @@ static void vdbeAssertFieldCountWithinLimits( ** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ +static SQLITE_NOINLINE int vdbeCompareMemStringWithEncodingChange( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ +){ + int rc; + const void *v1, *v2; + Mem c1; + Mem c2; + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + if( (v1==0 || v2==0) ){ + if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; + rc = 0; + }else{ + rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); + } + sqlite3VdbeMemReleaseMalloc(&c1); + sqlite3VdbeMemReleaseMalloc(&c2); + return rc; +} static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, @@ -90348,25 +92206,7 @@ static int vdbeCompareMemString( ** comparison function directly */ return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); }else{ - int rc; - const void *v1, *v2; - Mem c1; - Mem c2; - sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); - sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - if( (v1==0 || v2==0) ){ - if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; - rc = 0; - }else{ - rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); - } - sqlite3VdbeMemReleaseMalloc(&c1); - sqlite3VdbeMemReleaseMalloc(&c2); - return rc; + return vdbeCompareMemStringWithEncodingChange(pMem1,pMem2,pColl,prcErr); } } @@ -91029,6 +92869,7 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** The easiest way to enforce this limit is to consider only records with ** 13 fields or less. If the first field is an integer, the maximum legal ** header size is (12*5 + 1 + 1) bytes. */ + assert( p->pKeyInfo->aSortFlags!=0 ); if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortFlags[0] ){ @@ -91279,6 +93120,224 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } /* +** Helper function for vdbeIsMatchingIndexKey(). Return true if column +** iCol should be ignored when comparing a record with a record from +** an index on disk. The field should be ignored if: +** +** * the corresponding bit in mask is set, and +** * either: +** - bIntegrity is false, or +** - the two Mem values are both real values that differ by +** BTREE_ULPDISTORTION or fewer ULPs. +*/ +static int vdbeSkipField( + Bitmask mask, /* Mask of indexed expression fields */ + int iCol, /* Column of index being considered */ + Mem *pMem1, /* Expected index value */ + Mem *pMem2, /* Actual indexed value */ + int bIntegrity /* True if running PRAGMA integrity_check */ +){ +#define BTREE_ULPDISTORTION 2 + if( iCol>=BMS || (mask & MASKBIT(iCol))==0 ) return 0; + if( bIntegrity==0 ) return 1; + if( (pMem1->flags & MEM_Real) && (pMem2->flags & MEM_Real) ){ + u64 m1, m2; + memcpy(&m1,&pMem1->u.r,8); + memcpy(&m2,&pMem2->u.r,8); + if( (m1<m2 ? m2-m1 : m1-m2) <= BTREE_ULPDISTORTION ){ + return 1; + } + } + return 0; +} + +/* +** This function compares the unpacked record with the current key that +** cursor pCur points to. If bInt is false, all fields for which the +** corresponding bit in parameter "mask" is set are ignored. Or, if +** bInt is true, then a difference of BTREE_ULPDISTORTION or fewer ULPs +** in real values is overlooked for fields with the corresponding bit +** set in mask. +** +** Return the usual less than zero, zero, or greater than zero if the +** remaining fields of the cursor cursor key are less than, equal to or +** greater than those in (*p). +*/ +static int vdbeIsMatchingIndexKey( + BtCursor *pCur, /* Cursor open on index */ + int bInt, /* True for integrity_check-style search */ + Bitmask mask, /* Mask of columns to skip */ + UnpackedRecord *p, /* Index key being deleted */ + int *piRes /* 0 for a match, non-zero for not a match */ +){ + u8 *aRec = 0; + u32 nRec = 0; + Mem mem; + int rc = SQLITE_OK; + + memset(&mem, 0, sizeof(mem)); + mem.enc = p->pKeyInfo->enc; + mem.db = p->pKeyInfo->db; + nRec = sqlite3BtreePayloadSize(pCur); + if( nRec>0x7fffffff ){ + return SQLITE_CORRUPT_BKPT; + } + + /* Allocate 5 extra bytes at the end of the buffer. This allows the + ** getVarint32() call below to read slightly past the end of the buffer + ** if the record is corrupt. */ + aRec = sqlite3MallocZero(nRec+5); + if( aRec==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3BtreePayload(pCur, 0, nRec, aRec); + } + + if( rc==SQLITE_OK ){ + u32 szHdr = 0; /* Size of record header in bytes */ + u32 idxHdr = 0; /* Current index in header */ + + idxHdr = getVarint32(aRec, szHdr); + if( szHdr>98307 ){ + rc = SQLITE_CORRUPT; + }else{ + int res = 0; /* Result of this function call */ + u32 idxRec = szHdr; /* Index of next field in record body */ + int ii = 0; /* Iterator variable */ + + int nCol = p->pKeyInfo->nAllField; + for(ii=0; ii<nCol && rc==SQLITE_OK; ii++){ + u32 iSerial = 0; + int nSerial = 0; + + if( idxHdr>=szHdr ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + idxHdr += getVarint32(&aRec[idxHdr], iSerial); + nSerial = sqlite3VdbeSerialTypeLen(iSerial); + if( (idxRec+nSerial)>nRec ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3VdbeSerialGet(&aRec[idxRec], iSerial, &mem); + if( vdbeSkipField(mask, ii, &p->aMem[ii], &mem, bInt)==0 ){ + res = sqlite3MemCompare(&mem, &p->aMem[ii], p->pKeyInfo->aColl[ii]); + if( res!=0 ) break; + } + } + idxRec += sqlite3VdbeSerialTypeLen(iSerial); + } + + *piRes = res; + } + } + + sqlite3_free(aRec); + return rc; +} + +/* +** This is called when the record in (*p) should be found in the index +** opened by cursor pCur, but was not. This may happen as part of a DELETE +** operation or an integrity check. +** +** One reason that an exact match was not found may be the EIIB bug - that +** a text-to-float conversion may have caused a real value in record (*p) +** to be slightly different from its counterpart on disk. This function +** attempts to find the right index record. If it does find the right +** record, it leaves *pCur pointing to it and sets (*pRes) to 0 before +** returning. Otherwise, (*pRes) is set to non-zero and an SQLite error +** code returned. +** +** The algorithm used to find the correct record is: +** +** * Scan up to BTREE_FDK_RANGE entries either side of the current entry. +** If parameter bIntegrity is false, then all fields that are indexed +** expressions or virtual table columns are omitted from the comparison. +** If bIntegrity is true, then small differences in real values in +** such fields are overlooked, but they are not omitted from the comparison +** altogether. +** +** * If the above fails to find an entry and bIntegrity is false, search +** the entire index. +*/ +SQLITE_PRIVATE int sqlite3VdbeFindIndexKey( + BtCursor *pCur, + Index *pIdx, + UnpackedRecord *p, + int *pRes, + int bIntegrity +){ +#define BTREE_FDK_RANGE 10 + int nStep = 0; + int res = 1; + int rc = SQLITE_OK; + int ii = 0; + + /* Calculate a mask based on the first 64 columns of the index. The mask + ** bit is set if the corresponding index field is either an expression + ** or a virtual column of the table. */ + Bitmask mask = 0; + for(ii=0; ii<MIN(pIdx->nColumn, BMS); ii++){ + int iCol = pIdx->aiColumn[ii]; + if( (iCol==XN_EXPR) + || (iCol>=0 && (pIdx->pTable->aCol[iCol].colFlags & COLFLAG_VIRTUAL)) + ){ + mask |= MASKBIT(ii); + } + } + + /* If the mask is 0 at this point, then the index contains no expressions + ** or virtual columns. So do not search for a match - return so that the + ** caller may declare the db corrupt immediately. Or, if mask is non-zero, + ** proceed. */ + if( mask!=0 ){ + + /* Move the cursor back BTREE_FDK_RANGE entries. If this hits an EOF, + ** position the cursor at the first entry in the index and set nStep + ** to -1 so that the first loop below scans the entire index. Otherwise, + ** set nStep to BTREE_FDK_RANGE*2 so that the first loop below scans + ** just that many entries. */ + for(ii=0; sqlite3BtreeEof(pCur)==0 && ii<BTREE_FDK_RANGE; ii++){ + rc = sqlite3BtreePrevious(pCur, 0); + } + if( rc==SQLITE_DONE ){ + rc = sqlite3BtreeFirst(pCur, &res); + nStep = -1; + }else{ + nStep = BTREE_FDK_RANGE*2; + } + + /* This loop runs at most twice to search for a key with matching PK + ** fields in the index. The second iteration always searches the entire + ** index. The first iteration searches nStep entries starting with the + ** current cursor entry if (nStep>=0), or the entire index if (nStep<0). */ + while( sqlite3BtreeCursorIsValidNN(pCur) ){ + for(ii=0; rc==SQLITE_OK && (ii<nStep || nStep<0); ii++){ + rc = vdbeIsMatchingIndexKey(pCur, bIntegrity, mask, p, &res); + if( res==0 || rc!=SQLITE_OK ) break; + rc = sqlite3BtreeNext(pCur, 0); + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + assert( res!=0 ); + } + if( nStep<0 || rc!=SQLITE_OK || res==0 || bIntegrity ) break; + + /* The first, non-exhaustive, search failed to find an entry with + ** matching PK fields. So restart for an exhaustive search of the + ** entire index. */ + nStep = -1; + rc = sqlite3BtreeFirst(pCur, &res); + } + } + + *pRes = res; + return rc; +} + +#ifndef SQLITE_OMIT_DATETIME_FUNCS +/* ** Cause a function to throw an error if it was call from OP_PureFunc ** rather than OP_Function. ** @@ -91311,6 +93370,7 @@ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ } return 1; } +#endif /* SQLITE_OMIT_DATETIME_FUNCS */ #if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) /* @@ -91387,7 +93447,6 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( i64 iKey2; PreUpdate preupdate; const char *zTbl = pTab->zName; - static const u8 fakeSortOrder = 0; #ifdef SQLITE_DEBUG int nRealCol; if( pTab->tabFlags & TF_WithoutRowid ){ @@ -91422,11 +93481,11 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; - preupdate.pKeyinfo = (KeyInfo*)&preupdate.keyinfoSpace; + preupdate.pKeyinfo = (KeyInfo*)&preupdate.uKey; preupdate.pKeyinfo->db = db; preupdate.pKeyinfo->enc = ENC(db); preupdate.pKeyinfo->nKeyField = pTab->nCol; - preupdate.pKeyinfo->aSortFlags = (u8*)&fakeSortOrder; + preupdate.pKeyinfo->aSortFlags = 0; /* Indicate .aColl, .nAllField uninit */ preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -91456,6 +93515,17 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +#ifdef SQLITE_ENABLE_PERCENTILE +/* +** Return the name of an SQL function associated with the sqlite3_context. +*/ +SQLITE_PRIVATE const char *sqlite3VdbeFuncName(const sqlite3_context *pCtx){ + assert( pCtx!=0 ); + assert( pCtx->pFunc!=0 ); + return pCtx->pFunc->zName; +} +#endif /* SQLITE_ENABLE_PERCENTILE */ + /************** End of vdbeaux.c *********************************************/ /************** Begin file vdbeapi.c *****************************************/ /* @@ -91852,7 +93922,23 @@ static void setResultStrOrError( void (*xDel)(void*) /* Destructor function */ ){ Mem *pOut = pCtx->pOut; - int rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + int rc; + if( enc==SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); + }else if( enc==SQLITE_UTF8_ZT ){ + /* It is usually considered improper to assert() on an input. However, + ** the following assert() is checking for inputs that are documented + ** to result in undefined behavior. */ + assert( z==0 + || n<0 + || n>pOut->db->aLimit[SQLITE_LIMIT_LENGTH] + || z[n]==0 + ); + rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); + pOut->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + } if( rc ){ if( rc==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); @@ -92045,7 +94131,7 @@ SQLITE_API void sqlite3_result_text64( #endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 ){ + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; n &= ~(u64)1; } @@ -92196,6 +94282,8 @@ static int doWalCallbacks(sqlite3 *db){ } } } +#else + UNUSED_PARAMETER(db); #endif return rc; } @@ -92503,7 +94591,7 @@ static int valueFromValueList( Mem sMem; /* Raw content of current row */ memset(&sMem, 0, sizeof(sMem)); sz = sqlite3BtreePayloadSize(pRhs->pCsr); - rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,(int)sz,&sMem); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,sz,&sMem); if( rc==SQLITE_OK ){ u8 *zBuf = (u8*)sMem.z; u32 iSerial; @@ -93152,7 +95240,23 @@ static int bindText( assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ if( zData!=0 ){ pVar = &p->aVar[i-1]; - rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( encoding==SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); + }else if( encoding==SQLITE_UTF8_ZT ){ + /* It is usually consider improper to assert() on an input. + ** However, the following assert() is checking for inputs + ** that are documented to result in undefined behavior. */ + assert( zData==0 + || nData<0 + || nData>pVar->db->aLimit[SQLITE_LIMIT_LENGTH] + || ((u8*)zData)[nData]==0 + ); + rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); + pVar->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( encoding==0 ) pVar->enc = ENC(p->db); + } if( rc==SQLITE_OK && encoding!=0 ){ rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } @@ -93266,7 +95370,7 @@ SQLITE_API int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 ){ + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; nData &= ~(u64)1; } @@ -93623,7 +95727,7 @@ static UnpackedRecord *vdbeUnpackRecord( pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pRet ){ memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); + sqlite3VdbeRecordUnpack(nKey, pKey, pRet); } return pRet; } @@ -93652,6 +95756,9 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa } if( p->pPk ){ iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_old_out; }else{ iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } @@ -93807,6 +95914,8 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa } if( p->pPk && p->op!=SQLITE_UPDATE ){ iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + return SQLITE_MISUSE_BKPT; }else{ iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } @@ -94082,10 +96191,10 @@ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ ** a host parameter. If the text contains no host parameters, return ** the total number of bytes in the text. */ -static int findNextHostParameter(const char *zSql, int *pnToken){ +static i64 findNextHostParameter(const char *zSql, i64 *pnToken){ int tokenType; - int nTotal = 0; - int n; + i64 nTotal = 0; + i64 n; *pnToken = 0; while( zSql[0] ){ @@ -94132,8 +96241,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( sqlite3 *db; /* The database connection */ int idx = 0; /* Index of a host parameter */ int nextIndex = 1; /* Index of next ? host parameter */ - int n; /* Length of a token prefix */ - int nToken; /* Length of the parameter token */ + i64 n; /* Length of a token prefix */ + i64 nToken; /* Length of the parameter token */ int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ @@ -94298,17 +96407,19 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H -/* -** The following routine only works on Pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if !defined(__STRICT_ANSI__) && \ - (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if defined(_MSC_VER) && defined(_WIN32) + +/* #include "windows.h" */ + #include <profileapi.h> - #if defined(__GNUC__) + __inline sqlite3_uint64 sqlite3Hwtime(void){ + LARGE_INTEGER tm; + QueryPerformanceCounter(&tm); + return (sqlite3_uint64)tm.QuadPart; + } + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned int lo, hi; @@ -94316,17 +96427,6 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( return (sqlite_uint64)hi << 32 | lo; } - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ @@ -94335,6 +96435,14 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( return (sqlite_uint64)hi << 32 | lo; } +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && defined(__aarch64__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + sqlite3_uint64 cnt; + __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (cnt)); + return cnt; + } + #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ @@ -94693,12 +96801,11 @@ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - u8 enc = pRec->enc; int rc; assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); - rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + rc = sqlite3MemRealValueRC(pRec, &rValue); if( rc<=0 ) return; - if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ + if( (rc&2)==0 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue; @@ -94778,7 +96885,10 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; + assert( pMem->db!=0 ); + sqlite3_mutex_enter(pMem->db->mutex); applyNumericAffinity(pMem, 0); + sqlite3_mutex_leave(pMem->db->mutex); eType = sqlite3_value_type(pVal); } return eType; @@ -94811,15 +96921,15 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ pMem->u.i = 0; return MEM_Int; } - rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + rc = sqlite3MemRealValueRC(pMem, &pMem->u.r); if( rc<=0 ){ - if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + if( (rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ pMem->u.i = ix; return MEM_Int; }else{ return MEM_Real; } - }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + }else if( (rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ pMem->u.i = ix; return MEM_Int; } @@ -95057,7 +97167,7 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ static SQLITE_NOINLINE int vdbeColumnFromOverflow( VdbeCursor *pC, /* The BTree cursor from which we are reading */ int iCol, /* The column to read */ - int t, /* The serial-type code for the column value */ + u32 t, /* The serial-type code for the column value */ i64 iOffset, /* Offset to the start of the content value */ u32 cacheStatus, /* Current Vdbe.cacheCtr value */ u32 colCacheCtr, /* Current value of the column cache counter */ @@ -95132,6 +97242,36 @@ static SQLITE_NOINLINE int vdbeColumnFromOverflow( return rc; } +/* +** Send a "statement aborts" message to the error log. +*/ +static SQLITE_NOINLINE void sqlite3VdbeLogAbort( + Vdbe *p, /* The statement that is running at the time of failure */ + int rc, /* Error code */ + Op *pOp, /* Opcode that filed */ + Op *aOp /* All opcodes */ +){ + const char *zSql = p->zSql; /* Original SQL text */ + const char *zPrefix = ""; /* Prefix added to SQL text */ + int pc; /* Opcode address */ + char zXtra[100]; /* Buffer space to store zPrefix */ + + if( p->pFrame ){ + assert( aOp[0].opcode==OP_Init ); + if( aOp[0].p4.z!=0 ){ + assert( aOp[0].p4.z[0]=='-' + && aOp[0].p4.z[1]=='-' + && aOp[0].p4.z[2]==' ' ); + sqlite3_snprintf(sizeof(zXtra), zXtra,"/* %s */ ",aOp[0].p4.z+3); + zPrefix = zXtra; + }else{ + zPrefix = "/* unknown trigger */ "; + } + } + pc = (int)(pOp - aOp); + sqlite3_log(rc, "statement aborts at %d: %s; [%s%s]", + pc, p->zErrMsg, zPrefix, zSql); +} /* ** Return the symbolic name for the data type of a pMem @@ -95657,8 +97797,7 @@ case OP_Halt: { }else{ sqlite3VdbeError(p, "%s", pOp->p4.z); } - pcx = (int)(pOp - aOp); - sqlite3_log(pOp->p1, "abort at %d: %s; [%s]", pcx, p->zErrMsg, p->zSql); + sqlite3VdbeLogAbort(p, pOp->p1, pOp, aOp); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -96037,7 +98176,7 @@ case OP_IntCopy: { /* out2 */ ** RETURNING clause. */ case OP_FkCheck: { - if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkImmediate(p))!=SQLITE_OK ){ goto abort_due_to_error; } break; @@ -96129,10 +98268,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; flags2 = pIn2->flags & ~MEM_Str; } - nByte = pIn1->n + pIn2->n; + nByte = pIn1->n; + nByte += pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } +#if SQLITE_MAX_LENGTH>2147483645 + if( nByte>2147483645 ){ goto too_big; } +#endif if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ goto no_mem; } @@ -96816,6 +98959,7 @@ case OP_Compare: { pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); assert( pKeyInfo!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); p1 = pOp->p1; p2 = pOp->p2; #ifdef SQLITE_DEBUG @@ -97578,6 +99722,15 @@ op_column_corrupt: ** Take the affinities from the Table object in P4. If any value ** cannot be coerced into the correct type, then raise an error. ** +** If P3==0, then omit checking of VIRTUAL columns. +** +** If P3==1, then omit checking of all generated column, both VIRTUAL +** and STORED. +** +** If P3>=2, then only check column number P3-2 in the table (which will +** be a VIRTUAL column) against the value in reg[P1]. In this case, +** P2 will be 1. +** ** This opcode is similar to OP_Affinity except that this opcode ** forces the register type to the Table column type. This is used ** to implement "strict affinity". @@ -97591,8 +99744,8 @@ op_column_corrupt: ** ** <ul> ** <li> P2 should be the number of non-virtual columns in the -** table of P4. -** <li> Table P4 should be a STRICT table. +** table of P4 unless P3>1, in which case P2 will be 1. +** <li> Table P4 is a STRICT table. ** </ul> ** ** If any precondition is false, an assertion fault occurs. @@ -97601,16 +99754,28 @@ case OP_TypeCheck: { Table *pTab; Column *aCol; int i; + int nCol; assert( pOp->p4type==P4_TABLE ); pTab = pOp->p4.pTab; assert( pTab->tabFlags & TF_Strict ); - assert( pTab->nNVCol==pOp->p2 ); + assert( pOp->p3>=0 && pOp->p3<pTab->nCol+2 ); aCol = pTab->aCol; pIn1 = &aMem[pOp->p1]; - for(i=0; i<pTab->nCol; i++){ - if( aCol[i].colFlags & COLFLAG_GENERATED ){ - if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue; + if( pOp->p3<2 ){ + assert( pTab->nNVCol==pOp->p2 ); + i = 0; + nCol = pTab->nCol; + }else{ + i = pOp->p3-2; + nCol = i+1; + assert( i<pTab->nCol ); + assert( aCol[i].colFlags & COLFLAG_VIRTUAL ); + assert( pOp->p2==1 ); + } + for(; i<nCol; i++){ + if( (aCol[i].colFlags & COLFLAG_GENERATED)!=0 && pOp->p3<2 ){ + if( (aCol[i].colFlags & COLFLAG_VIRTUAL)!=0 ) continue; if( pOp->p3 ){ pIn1++; continue; } } assert( pIn1 < &aMem[pOp->p1+pOp->p2] ); @@ -97932,7 +100097,7 @@ case OP_MakeRecord: { len = (u32)pRec->n; serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); if( pRec->flags & MEM_Zero ){ - serial_type += pRec->u.nZero*2; + serial_type += (u32)pRec->u.nZero*2; if( nData ){ if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; len += pRec->u.nZero; @@ -98199,7 +100364,7 @@ case OP_Savepoint: { */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; if( isTransaction && p1==SAVEPOINT_RELEASE ){ - if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; } db->autoCommit = 1; @@ -98317,7 +100482,7 @@ case OP_AutoCommit: { "SQL statements in progress"); rc = SQLITE_BUSY; goto abort_due_to_error; - }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + }else if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; @@ -99689,7 +101854,7 @@ case OP_Found: { /* jump, in3, ncycle */ if( rc ) goto no_mem; pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); if( pIdxKey==0 ) goto no_mem; - sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey); + sqlite3VdbeRecordUnpack(r.aMem->n, r.aMem->z, pIdxKey); pIdxKey->default_rc = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult); sqlite3DbFreeNN(db, pIdxKey); @@ -100687,6 +102852,32 @@ case OP_Rewind: { /* jump0, ncycle */ break; } +/* Opcode: IfEmpty P1 P2 * * * +** Synopsis: if( empty(P1) ) goto P2 +** +** Check to see if the b-tree table that cursor P1 references is empty +** and jump to P2 if it is. +*/ +case OP_IfEmpty: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1<p->nCursor ); + assert( pOp->p2>=0 && pOp->p2<p->nOp ); + + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeIsEmpty(pCrsr, &res); + if( rc ) goto abort_due_to_error; + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its @@ -100879,20 +103070,17 @@ case OP_SorterInsert: { /* in2 */ break; } -/* Opcode: IdxDelete P1 P2 P3 * P5 +/* Opcode: IdxDelete P1 P2 P3 P4 * ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form ** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. ** -** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error -** if no matching index entry is found. This happens when running -** an UPDATE or DELETE statement and the index entry to be updated -** or deleted is not found. For some uses of IdxDelete -** (example: the EXCEPT operator) it does not matter that no matching -** entry is found. For those cases, P5 is zero. Also, do not raise -** this (self-correcting and non-critical) error if in writable_schema mode. +** P4 is a pointer to an Index structure. +** +** Raise an SQLITE_CORRUPT_INDEX error if no matching index entry is found +** and not in writable_schema mode. */ case OP_IdxDelete: { VdbeCursor *pC; @@ -100915,13 +103103,22 @@ case OP_IdxDelete: { r.aMem = &aMem[pOp->p2]; rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ - rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); - if( rc ) goto abort_due_to_error; - }else if( pOp->p5 && !sqlite3WritableSchema(db) ){ - rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); - goto abort_due_to_error; + if( res!=0 ){ + rc = sqlite3VdbeFindIndexKey(pCrsr, pOp->p4.pIdx, &r, &res, 0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + if( res!=0 ){ + if( !sqlite3WritableSchema(db) ){ + rc = sqlite3ReportError( + SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); + goto abort_due_to_error; + } + pC->cacheStatus = CACHE_STALE; + pC->seekResult = 0; + break; + } } + rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); + if( rc ) goto abort_due_to_error; assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; pC->seekResult = 0; @@ -101548,6 +103745,58 @@ case OP_IntegrityCk: { sqlite3VdbeChangeEncoding(pIn1, encoding); goto check_for_interrupt; } + +/* Opcode: IFindKey P1 P2 P3 P4 * +** +** This instruction always follows an OP_Found with the same P1, P2 and P3 +** values as this instruction and a non-zero P4 value. The P4 value to +** this opcode is of type P4_INDEX and contains a pointer to the Index +** object of for the index being searched. +** +** This opcode uses sqlite3VdbeFindIndexKey() to search around the current +** cursor location for an index key that exactly matches all fields that +** are not indexed expressions or references to VIRTUAL generated columns, +** and either exactly match or are real numbers that are within 2 ULPs of +** each other if the don't match. +** +** To put it another way, this opcode looks for nearby index entries that +** are very close to the search key, but which might have small differences +** in floating-point values that come via an expression. +** +** If no nearby alternative entry is found in cursor P1, then jump to P2. +** But if a close match is found, fall through. +** +** This opcode is used by PRAGMA integrity_check to help distinguish +** between truely corrupt indexes and expression indexes that are holding +** floating-point values that are off by one or two ULPs. +*/ +case OP_IFindKey: { /* jump, in3 */ + VdbeCursor *pC; + int res; + UnpackedRecord r; + + assert( pOp[-1].opcode==OP_Found ); + assert( pOp[-1].p1==pOp->p1 ); + assert( pOp[-1].p3==pOp->p3 ); + pC = p->apCsr[pOp->p1]; + assert( pOp->p4type==P4_INDEX ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + assert( pC->isTable==0 ); + + memset(&r, 0, sizeof(r)); + r.aMem = &aMem[pOp->p3]; + r.nField = pOp->p4.pIdx->nColumn; + r.pKeyInfo = pC->pKeyInfo; + + rc = sqlite3VdbeFindIndexKey(pC->uc.pCursor, pOp->p4.pIdx, &r, &res, 1); + if( rc || res!=0 ){ + rc = SQLITE_OK; + goto jump_to_p2; + } + pC->nullRow = 0; + break; +}; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* Opcode: RowSetAdd P1 P2 * * * @@ -102223,6 +104472,7 @@ case OP_Checkpoint: { || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE + || pOp->p2==SQLITE_CHECKPOINT_NOOP ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc ){ @@ -102558,7 +104808,14 @@ case OP_VOpen: { /* ncycle */ const sqlite3_module *pModule; assert( p->bIsReader ); - pCur = 0; + pCur = p->apCsr[pOp->p1]; + if( pCur!=0 + && ALWAYS( pCur->eCurType==CURTYPE_VTAB ) + && ALWAYS( pCur->uc.pVCur->pVtab==pOp->p4.pVtab->pVtab ) + ){ + /* This opcode is a no-op if the cursor is already open */ + break; + } pVCur = 0; pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ @@ -103500,8 +105757,7 @@ abort_due_to_error: p->rc = rc; sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: %s; [%s]", - (int)(pOp - aOp), p->zErrMsg, p->zSql); + sqlite3VdbeLogAbort(p, rc, pOp, aOp); if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ @@ -103962,7 +106218,7 @@ static int blobReadWrite( int iOffset, int (*xCall)(BtCursor*, u32, u32, void*) ){ - int rc; + int rc = SQLITE_OK; Incrblob *p = (Incrblob *)pBlob; Vdbe *v; sqlite3 *db; @@ -104002,17 +106258,32 @@ static int blobReadWrite( ** using the incremental-blob API, this works. For the sessions module ** anyhow. */ - sqlite3_int64 iKey; - iKey = sqlite3BtreeIntegerKey(p->pCsr); - assert( v->apCsr[0]!=0 ); - assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); - sqlite3VdbePreUpdateHook( - v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol - ); + if( sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ){ + /* If the cursor is not currently valid, try to reseek it. This + ** always either fails or finds the correct row - the cursor will + ** have been marked permanently CURSOR_INVALID if the open row has + ** been deleted. */ + int bDiff = 0; + rc = sqlite3BtreeCursorRestore(p->pCsr, &bDiff); + assert( bDiff==0 || sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ); + } + if( sqlite3BtreeCursorIsValidNN(p->pCsr) ){ + sqlite3_int64 iKey; + iKey = sqlite3BtreeIntegerKey(p->pCsr); + assert( v->apCsr[0]!=0 ); + assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); + sqlite3VdbePreUpdateHook( + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol + ); + } + } + if( rc==SQLITE_OK ){ + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); } +#else + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); #endif - rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); sqlite3BtreeLeaveCursor(p->pCsr); if( rc==SQLITE_ABORT ){ sqlite3VdbeFinalize(v); @@ -104401,6 +106672,7 @@ struct SortSubtask { SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ + u64 nSpill; /* Total bytes written by this task */ }; @@ -104521,6 +106793,7 @@ struct PmaWriter { int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ sqlite3_file *pFd; /* File handle to write to */ + u64 nPmaSpill; /* Total number of bytes written */ }; /* @@ -104865,7 +107138,7 @@ static int vdbeSorterCompareTail( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( *pbKey2Cached==0 ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); @@ -104892,7 +107165,7 @@ static int vdbeSorterCompare( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( !*pbKey2Cached ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); @@ -104932,6 +107205,7 @@ static int vdbeSorterCompareText( ); } }else{ + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; @@ -104995,6 +107269,7 @@ static int vdbeSorterCompareInt( } } + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); if( res==0 ){ if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( @@ -105068,7 +107343,8 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( assert( pCsr->eCurType==CURTYPE_SORTER ); assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*) < 0x7fffffff ); - szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nKeyField); + assert( pCsr->pKeyInfo->nKeyField<=pCsr->pKeyInfo->nAllField ); + szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nAllField); sz = SZ_VDBESORTER(nWorker+1); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); @@ -105082,7 +107358,12 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( pKeyInfo->db = 0; if( nField && nWorker==0 ){ pKeyInfo->nKeyField = nField; + assert( nField<=pCsr->pKeyInfo->nAllField ); } + /* It is OK that pKeyInfo reuses the aSortFlags field from pCsr->pKeyInfo, + ** since the pCsr->pKeyInfo->aSortFlags[] array is invariant and lives + ** longer that pSorter. */ + assert( pKeyInfo->aSortFlags==pCsr->pKeyInfo->aSortFlags ); sqlite3BtreeEnter(pBt); pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt); sqlite3BtreeLeave(pBt); @@ -105371,6 +107652,12 @@ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ assert( pCsr->eCurType==CURTYPE_SORTER ); pSorter = pCsr->uc.pSorter; if( pSorter ){ + /* Increment db->nSpill by the total number of bytes of data written + ** to temp files by this sort operation. */ + int ii; + for(ii=0; ii<pSorter->nTask; ii++){ + db->nSpill += pSorter->aTask[ii].nSpill; + } sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); @@ -105596,6 +107883,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); p->iBufStart = p->iBufEnd = 0; p->iWriteOff += p->nBuffer; } @@ -105612,17 +107900,20 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the -** last byte written to the file. +** last byte written to the file. Also, increment (*pnSpill) by the total +** number of bytes written to the file. */ -static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof, u64 *pnSpill){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); } *piEof = (p->iWriteOff + p->iBufEnd); + *pnSpill += p->nPmaSpill; sqlite3_free(p->aBuffer); rc = p->eFWErr; memset(p, 0, sizeof(PmaWriter)); @@ -105702,7 +107993,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ if( pList->aMemory==0 ) sqlite3_free(p); } pList->pList = p; - rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof, &pTask->nSpill); } vdbeSorterWorkDebug(pTask, "exit"); @@ -106016,7 +108307,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); } - rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof, &pTask->nSpill); if( rc==SQLITE_OK ) rc = rc2; vdbeSorterPopulateDebug(pTask, "exit"); return rc; @@ -106862,7 +109153,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( assert( r2->nField==nKeyCol ); pKey = vdbeSorterRowkey(pSorter, &nKey); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + sqlite3VdbeRecordUnpack(nKey, pKey, r2); for(i=0; i<nKeyCol; i++){ if( r2->aMem[i].flags & MEM_Null ){ *pRes = -1; @@ -108407,10 +110698,13 @@ static int lookupName( if( cnt>0 ){ if( pItem->fg.isUsing==0 || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + || pMatch==pItem ){ /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ + ** not joined by USING. Or, a single table has two columns + ** that match a USING term (if pMatch==pItem). These are both + ** "ambiguous column name" errors. Signal as much by clearing + ** pFJMatch and letting cnt go above 1. */ sqlite3ExprListDelete(db, pFJMatch); pFJMatch = 0; }else @@ -108960,14 +111254,14 @@ static void notValidImpl( /* ** Expression p should encode a floating point value between 1.0 and 0.0. -** Return 1024 times this value. Or return -1 if p is not a floating point -** value between 1.0 and 0.0. +** Return 134,217,728 (2^27) times this value. Or return -1 if p is not +** a floating point value between 1.0 and 0.0. */ static int exprProbability(Expr *p){ double r = -1.0; if( p->op!=TK_FLOAT ) return -1; assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + sqlite3AtoF(p->u.zToken, &r); assert( r>=0.0 ); if( r>1.0 ) return -1; return (int)(r*134217728.0); @@ -109392,11 +111686,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: case TK_SELECT: - case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); #endif case TK_IN: { testcase( pExpr->op==TK_IN ); + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); if( ExprUseXSelect(pExpr) ){ int nRef = pNC->nRef; testcase( pNC->ncFlags & NC_IsCheck ); @@ -109404,6 +111700,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); assert( pExpr->x.pSelect ); + if( pExpr->op==TK_EXISTS ) pParse->bHasExists = 1; if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -109684,10 +111981,8 @@ static int resolveCompoundOrderBy( /* Convert the ORDER BY term into an integer column number iCol, ** taking care to preserve the COLLATE clause if it exists. */ if( !IN_RENAME_OBJECT ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pNew = sqlite3ExprInt32(db, iCol); if( pNew==0 ) return 1; - pNew->flags |= EP_IntValue; - pNew->u.iValue = iCol; if( pItem->pExpr==pE ){ pItem->pExpr = pNew; }else{ @@ -110041,10 +112336,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } #endif - /* The ORDER BY and GROUP BY clauses may not refer to terms in - ** outer queries - */ - sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; /* If this is a converted compound query, move the ORDER BY clause from @@ -110107,6 +112398,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ return WRC_Abort; } + /* If the SELECT statement contains ON clauses that were moved into + ** the WHERE clause, go through and verify that none of the terms + ** in the ON clauses reference tables to the right of the ON clause. */ + if( (p->selFlags & SF_OnToWhere) ){ + sqlite3SelectCheckOnClauses(pParse, p); + if( pParse->nErr ) return WRC_Abort; + } + /* Advance to the next term of the compound */ p = p->pPrior; @@ -110314,14 +112613,17 @@ SQLITE_PRIVATE int sqlite3ResolveSelfReference( SrcList *pSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; - u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ + union { + SrcList sSrc; + u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ + } uSrc; assert( type==0 || pTab!=0 ); assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); - pSrc = (SrcList*)srcSpace; - memset(pSrc, 0, SZ_SRCLIST_1); + memset(&uSrc, 0, sizeof(uSrc)); + pSrc = &uSrc.sSrc; if( pTab ){ pSrc->nSrc = 1; pSrc->a[0].zName = pTab->zName; @@ -111280,34 +113582,22 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( int dequote /* True to dequote */ ){ Expr *pNew; - int nExtra = 0; - int iValue = 0; + int nExtra = pToken ? pToken->n+1 : 0; assert( db!=0 ); - if( pToken ){ - if( op!=TK_INTEGER || pToken->z==0 - || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; /* tag-20240227-a */ - assert( iValue>=0 ); - } - } pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; - if( pToken ){ - if( nExtra==0 ){ - pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); - pNew->u.iValue = iValue; - }else{ - pNew->u.zToken = (char*)&pNew[1]; - assert( pToken->z!=0 || pToken->n==0 ); - if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); - pNew->u.zToken[pToken->n] = 0; - if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - sqlite3DequoteExpr(pNew); - } + if( nExtra ){ + assert( pToken!=0 ); + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + sqlite3DequoteExpr(pNew); } } #if SQLITE_MAX_EXPR_DEPTH>0 @@ -111333,6 +113623,24 @@ SQLITE_PRIVATE Expr *sqlite3Expr( } /* +** Allocate an expression for a 32-bit signed integer literal. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprInt32(sqlite3 *db, int iVal){ + Expr *pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)); + if( pNew ){ + memset(pNew, 0, sizeof(Expr)); + pNew->op = TK_INTEGER; + pNew->iAgg = -1; + pNew->flags = EP_IntValue|EP_Leaf|(iVal?EP_IsTrue:EP_IsFalse); + pNew->u.iValue = iVal; +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif + } + return pNew; +} + +/* ** Attach subtrees pLeft and pRight to the Expr node pRoot. ** ** If pRoot==NULL that means that a memory allocation error has occurred. @@ -111494,7 +113802,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ ){ sqlite3ExprDeferredDelete(pParse, pLeft); sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); + return sqlite3ExprInt32(db, 0); }else{ return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } @@ -111584,6 +113892,11 @@ SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy( sqlite3ExprListDelete(db, pOrderBy); return; } + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + sqlite3ExprListDelete(db, pOrderBy); + return; + } pOB = sqlite3ExprAlloc(db, TK_ORDER, 0, 0); if( pOB==0 ){ @@ -111614,7 +113927,9 @@ SQLITE_PRIVATE void sqlite3ExprFunctionUsable( ){ assert( !IN_RENAME_OBJECT ); assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); - if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( ExprHasProperty(pExpr, EP_FromDDL) + || pParse->prepFlags & SQLITE_PREPARE_FROM_DDL + ){ if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 || (pParse->db->flags & SQLITE_TrustedSchema)==0 ){ @@ -112310,9 +114625,7 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int fla pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; + pNew->selFlags = p->selFlags; pNew->nSelectRow = p->nSelectRow; pNew->pWith = sqlite3WithDup(db, p->pWith); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -112719,6 +115032,85 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ } /* +** Return true if it might be advantageous to compute the right operand +** of expression pExpr first, before the left operand. +** +** Normally the left operand is computed before the right operand. But if +** the left operand contains a subquery and the right does not, then it +** might be more efficient to compute the right operand first. +*/ +static int exprEvalRhsFirst(Expr *pExpr){ + if( ExprHasProperty(pExpr->pLeft, EP_Subquery) + && !ExprHasProperty(pExpr->pRight, EP_Subquery) + ){ + return 1; + }else{ + return 0; + } +} + +/* +** Compute the two operands of a binary operator. +** +** If either operand contains a subquery, then the code strives to +** compute the operand containing the subquery second. If the other +** operand evalutes to NULL, then a jump is made. The address of the +** IsNull operand that does this jump is returned. The caller can use +** this to optimize the computation so as to avoid doing the potentially +** expensive subquery. +** +** If no optimization opportunities exist, return 0. +*/ +static int exprComputeOperands( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The comparison expression */ + int *pR1, /* OUT: Register holding the left operand */ + int *pR2, /* OUT: Register holding the right operand */ + int *pFree1, /* OUT: Temp register to free if not zero */ + int *pFree2 /* OUT: Another temp register to free if not zero */ +){ + int addrIsNull; + int r1, r2; + Vdbe *v = pParse->pVdbe; + + assert( v!=0 ); + /* + ** If the left operand contains a (possibly expensive) subquery and the + ** right operand does not and the right operation might be NULL, + ** then compute the right operand first and do an IsNull jump if the + ** right operand evalutes to NULL. + */ + if( exprEvalRhsFirst(pExpr) && sqlite3ExprCanBeNull(pExpr->pRight) ){ + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + }else{ + r2 = 0; /* Silence a false-positive uninit-var warning in MSVC */ + addrIsNull = 0; + } + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pFree1); + if( addrIsNull==0 ){ + /* + ** If the right operand contains a subquery and the left operand does not + ** and the left operand might be NULL, then do an IsNull check + ** check on the left operand before computing the right operand. + */ + if( ExprHasProperty(pExpr->pRight, EP_Subquery) + && sqlite3ExprCanBeNull(pExpr->pLeft) + ){ + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + } + *pR1 = r1; + *pR2 = r2; + return addrIsNull; +} + +/* ** pExpr is a TK_FUNCTION node. Try to determine whether or not the ** function is a constant function. A function is constant if all of ** the following are true: @@ -112885,7 +115277,7 @@ static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ /* ** Walk an expression tree. Return non-zero if the expression is constant -** and 0 if it involves variables or function calls. +** or return zero if the expression involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is @@ -113675,6 +116067,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( */ u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; + int bloomOk = (inFlags & IN_INDEX_MEMBERSHIP)!=0; eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; @@ -113682,7 +116075,13 @@ SQLITE_PRIVATE int sqlite3FindInIndex( *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } assert( pX->op==TK_IN ); - sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( !bloomOk + && ExprUseXSelect(pX) + && (pX->x.pSelect->selFlags & SF_ClonedRhsIn)!=0 + ){ + bloomOk = 1; + } + sqlite3CodeRhsOfIN(pParse, pX, iTab, bloomOk); if( rMayHaveNull ){ sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); } @@ -113840,7 +116239,8 @@ static int findCompatibleInRhsSubrtn( SQLITE_PRIVATE void sqlite3CodeRhsOfIN( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN operator */ - int iTab /* Use this cursor number */ + int iTab, /* Use this cursor number */ + int allowBloom /* True to allow the use of a Bloom filter */ ){ int addrOnce = 0; /* Address of the OP_Once instruction at top */ int addr; /* Address of OP_OpenEphemeral instruction */ @@ -113962,7 +116362,10 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; - if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( addrOnce + && allowBloom + && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) + ){ int regBloom = ++pParse->nMem; addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); VdbeComment((v, "Bloom filter")); @@ -114152,9 +116555,22 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ pParse->nMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; - dest.iSdst = dest.iSDParm; + if( (pSel->selFlags&SF_Distinct) && pSel->pLimit && pSel->pLimit->pRight ){ + /* If there is both a DISTINCT and an OFFSET clause, then allocate + ** a separate dest.iSdst array for sqlite3Select() and other + ** routines to populate. In this case results will be copied over + ** into the dest.iSDParm array only after OFFSET processing. This + ** ensures that in the case where OFFSET excludes all rows, the + ** dest.iSDParm array is not left populated with the contents of the + ** last row visited - it should be all NULLs if all rows were + ** excluded by OFFSET. */ + dest.iSdst = pParse->nMem+1; + pParse->nMem += nReg; + }else{ + dest.iSdst = dest.iSDParm; + } dest.nSdst = nReg; - sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, pParse->nMem); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; @@ -114162,20 +116578,26 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ VdbeComment((v, "Init EXISTS result")); } if( pSel->pLimit ){ - /* The subquery already has a limit. If the pre-existing limit is X - ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ - sqlite3 *db = pParse->db; - pLimit = sqlite3Expr(db, TK_INTEGER, "0"); - if( pLimit ){ - pLimit->affExpr = SQLITE_AFF_NUMERIC; - pLimit = sqlite3PExpr(pParse, TK_NE, - sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); + /* The subquery already has a limit. If the pre-existing limit X is + ** not already integer value 1 or 0, then make the new limit X<>0 so that + ** the new limit is either 1 or 0 */ + Expr *pLeft = pSel->pLimit->pLeft; + if( ExprHasProperty(pLeft, EP_IntValue)==0 + || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) + ){ + sqlite3 *db = pParse->db; + pLimit = sqlite3ExprInt32(db, 0); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pLeft, 0), pLimit); + } + sqlite3ExprDeferredDelete(pParse, pLeft); + pSel->pLimit->pLeft = pLimit; } - sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); - pSel->pLimit->pLeft = pLimit; }else{ /* If there is no pre-existing limit add a limit of 1 */ - pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pLimit = sqlite3ExprInt32(pParse->db, 1); pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } pSel->iLimit = 0; @@ -114260,7 +116682,6 @@ static void sqlite3ExprCodeIN( int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ int eType; /* Type of the RHS */ int rLhs; /* Register(s) holding the LHS values */ - int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ int *aiMap = 0; /* Map from vector field to index column */ char *zAff = 0; /* Affinity string for comparisons */ @@ -114323,19 +116744,8 @@ static void sqlite3ExprCodeIN( ** by code generated below. */ assert( pParse->okConstFactor==okConstFactor ); pParse->okConstFactor = 0; - rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + rLhs = exprCodeVector(pParse, pLeft, &iDummy); pParse->okConstFactor = okConstFactor; - for(i=0; i<nVector && aiMap[i]==i; i++){} /* Are LHS fields reordered? */ - if( i==nVector ){ - /* LHS fields are not reordered */ - rLhs = rLhsOrig; - }else{ - /* Need to reorder the LHS fields according to aiMap */ - rLhs = sqlite3GetTempRange(pParse, nVector); - for(i=0; i<nVector; i++){ - sqlite3VdbeAddOp3(v, OP_Copy, rLhsOrig+i, rLhs+aiMap[i], 0); - } - } /* If sqlite3FindInIndex() did not find or create an index that is ** suitable for evaluating the IN operator, then evaluate using a @@ -114350,6 +116760,7 @@ static void sqlite3ExprCodeIN( int r2, regToFree; int regCkNull = 0; int ii; + assert( nVector==1 ); assert( ExprUseXList(pExpr) ); pList = pExpr->x.pList; pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); @@ -114391,6 +116802,26 @@ static void sqlite3ExprCodeIN( goto sqlite3ExprCodeIN_finished; } + if( eType!=IN_INDEX_ROWID ){ + /* If this IN operator will use an index, then the order of columns in the + ** vector might be different from the order in the index. In that case, + ** we need to reorder the LHS values to be in index order. Run Affinity + ** before reordering the columns, so that the affinity is correct. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + for(i=0; i<nVector && aiMap[i]==i; i++){} /* Are LHS fields reordered? */ + if( i!=nVector ){ + /* Need to reorder the LHS fields according to aiMap */ + int rLhsOrig = rLhs; + rLhs = sqlite3GetTempRange(pParse, nVector); + for(i=0; i<nVector; i++){ + sqlite3VdbeAddOp3(v, OP_Copy, rLhsOrig+i, rLhs+aiMap[i], 0); + } + sqlite3ReleaseTempReg(pParse, rLhsOrig); + } + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the ** LHS does contain NULLs then the result must be either FALSE or NULL. ** We will then skip the binary search of the RHS. @@ -114417,18 +116848,19 @@ static void sqlite3ExprCodeIN( /* In this case, the RHS is the ROWID of table b-tree and so we also ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 ** into a single opcode. */ + assert( nVector==1 ); sqlite3VdbeAddOp3(v, OP_SeekRowid, iTab, destIfFalse, rLhs); VdbeCoverage(v); addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ }else{ - sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); if( destIfFalse==destIfNull ){ /* Combine Step 3 and Step 5 into a single opcode */ if( ExprHasProperty(pExpr, EP_Subrtn) ){ const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); assert( pOp->opcode==OP_Once || pParse->nErr ); - if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ - assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + if( pOp->p3>0 ){ /* tag-202407032019 */ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) + || pParse->nErr ); sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, rLhs, nVector); VdbeCoverage(v); } @@ -114499,7 +116931,6 @@ static void sqlite3ExprCodeIN( sqlite3VdbeJumpHere(v, addrTruthOp); sqlite3ExprCodeIN_finished: - if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); VdbeComment((v, "end IN expr")); sqlite3ExprCodeIN_oom_error: sqlite3DbFree(pParse->db, aiMap); @@ -114519,7 +116950,7 @@ sqlite3ExprCodeIN_oom_error: static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3AtoF(z, &value); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); @@ -114614,7 +117045,12 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( iAddr = 0; } sqlite3ExprCodeCopy(pParse, sqlite3ColumnExpr(pTab,pCol), regOut); - if( pCol->affinity>=SQLITE_AFF_TEXT ){ + if( (pCol->colFlags & COLFLAG_VIRTUAL)!=0 + && (pTab->tabFlags & TF_Strict)!=0 + ){ + int p3 = 2+(int)(pCol - pTab->aCol); + sqlite3VdbeAddOp4(v, OP_TypeCheck, regOut, 1, p3, (char*)pTab, P4_TABLE); + }else if( pCol->affinity>=SQLITE_AFF_TEXT ){ sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); @@ -115052,6 +117488,80 @@ static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ return 0; } +/* +** Generate code that evaluates an AND or OR operator leaving a +** boolean result in a register. pExpr is the AND/OR expression. +** Store the result in the "target" register. Use short-circuit +** evaluation to avoid computing both operands, if possible. +** +** The code generated might require the use of a temporary register. +** If it does, then write the number of that temporary register +** into *pTmpReg. If not, leave *pTmpReg unchanged. +*/ +static SQLITE_NOINLINE int exprCodeTargetAndOr( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* AND or OR expression to be coded */ + int target, /* Put result in this register, guaranteed */ + int *pTmpReg /* Write a temporary register here */ +){ + int op; /* The opcode. TK_AND or TK_OR */ + int skipOp; /* Opcode for the branch that skips one operand */ + int addrSkip; /* Branch instruction that skips one of the operands */ + int regSS = 0; /* Register holding computed operand when other omitted */ + int r1, r2; /* Registers for left and right operands, respectively */ + Expr *pAlt; /* Alternative, simplified expression */ + Vdbe *v; /* statement being coded */ + + assert( pExpr!=0 ); + op = pExpr->op; + assert( op==TK_AND || op==TK_OR ); + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( pParse->pVdbe!=0 ); + v = pParse->pVdbe; + pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + r1 = sqlite3ExprCodeTarget(pParse, pAlt, target); + sqlite3VdbeAddOp3(v, OP_And, r1, r1, target); + return target; + } + skipOp = op==TK_AND ? OP_IfNot : OP_If; + if( exprEvalRhsFirst(pExpr) ){ + /* Compute the right operand first. Skip the computation of the left + ** operand if the right operand fully determines the result */ + r2 = regSS = sqlite3ExprCodeTarget(pParse, pExpr->pRight, target); + addrSkip = sqlite3VdbeAddOp1(v, skipOp, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pTmpReg); + }else{ + /* Compute the left operand first */ + r1 = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + if( ExprHasProperty(pExpr->pRight, EP_Subquery) ){ + /* Skip over the computation of the right operand if the right + ** operand is a subquery and the left operand completely determines + ** the result */ + regSS = r1; + addrSkip = sqlite3VdbeAddOp1(v, skipOp, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + }else{ + addrSkip = regSS = 0; + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pTmpReg); + } + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( (*pTmpReg)==0 ); + if( addrSkip ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrSkip); + sqlite3VdbeAddOp3(v, OP_Or, regSS, regSS, target); + VdbeComment((v, "short-circut value")); + } + return target; +} + + /* ** Generate code into the current Vdbe to evaluate the given @@ -115299,7 +117809,7 @@ expr_code_doover: case TK_ISNOT: op = (op==TK_IS) ? TK_EQ : TK_NE; p5 = SQLITE_NULLEQ; - /* fall-through */ + /* no break */ deliberate_fall_through case TK_LT: case TK_LE: case TK_GT: @@ -115307,11 +117817,17 @@ expr_code_doover: case TK_NE: case TK_EQ: { Expr *pLeft = pExpr->pLeft; + int addrIsNull = 0; if( sqlite3ExprIsVector(pLeft) ){ codeVectorCompare(pParse, pExpr, target, op, p5); }else{ - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && p5!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + } sqlite3VdbeAddOp2(v, OP_Integer, 1, inReg); codeCompare(pParse, pLeft, pExpr->pRight, op, r1, r2, sqlite3VdbeCurrentAddr(v)+2, p5, @@ -115326,6 +117842,11 @@ expr_code_doover: sqlite3VdbeAddOp2(v, OP_Integer, 0, inReg); }else{ sqlite3VdbeAddOp3(v, OP_ZeroOrNull, r1, inReg, r2); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, inReg); + } } testcase( regFree1==0 ); testcase( regFree2==0 ); @@ -115333,7 +117854,10 @@ expr_code_doover: break; } case TK_AND: - case TK_OR: + case TK_OR: { + inReg = exprCodeTargetAndOr(pParse, pExpr, target, ®Free1); + break; + } case TK_PLUS: case TK_STAR: case TK_MINUS: @@ -115344,8 +117868,7 @@ expr_code_doover: case TK_LSHIFT: case TK_RSHIFT: case TK_CONCAT: { - assert( TK_AND==OP_And ); testcase( op==TK_AND ); - assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + int addrIsNull; assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); @@ -115355,11 +117878,23 @@ expr_code_doover: assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } sqlite3VdbeAddOp3(v, op, r2, r1, target); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + VdbeComment((v, "short-circut value")); + } break; } case TK_UMINUS: { @@ -116227,17 +118762,27 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); - }else if( op==TK_AND ){ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); }else{ - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + } } break; } @@ -116276,10 +118821,16 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -116294,6 +118845,13 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: @@ -116401,17 +118959,27 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); - }else if( pExpr->op==TK_AND ){ - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); }else{ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } } break; } @@ -116453,10 +119021,16 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -116471,6 +119045,13 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: @@ -117470,7 +120051,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; for(i=0; i<pSrcList->nSrc; i++){ - if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + if( pSrcList->a[i].iCursor==pIEpr->iDataCur ){ + testcase( i>0 ); + break; + } } if( i>=pSrcList->nSrc ) break; if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ @@ -118259,7 +120843,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ /* Look up the table being altered. */ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); - if( db->mallocFailed ) goto exit_begin_add_column; + if( NEVER(db->mallocFailed) ) goto exit_begin_add_column; pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; @@ -118331,7 +120915,7 @@ exit_begin_add_column: ** Or, if pTab is not a view or virtual table, zero is returned. */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) -static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ +static int isRealTable(Parse *pParse, Table *pTab, int iOp){ const char *zType = 0; #ifndef SQLITE_OMIT_VIEW if( IsView(pTab) ){ @@ -118344,9 +120928,12 @@ static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ } #endif if( zType ){ + const char *azMsg[] = { + "rename columns of", "drop column from", "edit constraints of" + }; + assert( iOp>=0 && iOp<ArraySize(azMsg) ); sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", - (bDrop ? "drop column from" : "rename columns of"), - zType, pTab->zName + azMsg[iOp], zType, pTab->zName ); return 1; } @@ -118818,6 +121405,25 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ } /* +** Set the error message of the context passed as the first argument to +** the result of formatting zFmt using printf() style formatting. +*/ +static void errorMPrintf(sqlite3_context *pCtx, const char *zFmt, ...){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3VMPrintf(db, zFmt, ap); + va_end(ap); + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3DbFree(db, zErr); + }else{ + sqlite3_result_error_nomem(pCtx); + } +} + +/* ** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an ** ALTER TABLE RENAME COLUMN program. The error message emitted by the @@ -119114,8 +121720,8 @@ static int renameResolveTrigger(Parse *pParse){ sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); if( pParse->nErr ) rc = pParse->rc; } - if( rc==SQLITE_OK && pStep->zTarget ){ - SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); + if( rc==SQLITE_OK && pStep->pSrc ){ + SrcList *pSrc = sqlite3SrcListDup(db, pStep->pSrc, 0); if( pSrc ){ Select *pSel = sqlite3SelectNew( pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0 @@ -119143,10 +121749,10 @@ static int renameResolveTrigger(Parse *pParse){ pSel->pSrc = 0; sqlite3SelectDelete(db, pSel); } - if( pStep->pFrom ){ + if( ALWAYS(pStep->pSrc) ){ int i; - for(i=0; i<pStep->pFrom->nSrc && rc==SQLITE_OK; i++){ - SrcItem *p = &pStep->pFrom->a[i]; + for(i=0; i<pStep->pSrc->nSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pStep->pSrc->a[i]; if( p->fg.isSubquery ){ assert( p->u4.pSubq!=0 ); sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); @@ -119215,13 +121821,13 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); } - if( pStep->pFrom ){ + if( pStep->pSrc ){ int i; - SrcList *pFrom = pStep->pFrom; - for(i=0; i<pFrom->nSrc; i++){ - if( pFrom->a[i].fg.isSubquery ){ - assert( pFrom->a[i].u4.pSubq!=0 ); - sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + SrcList *pSrc = pStep->pSrc; + for(i=0; i<pSrc->nSrc; i++){ + if( pSrc->a[i].fg.isSubquery ){ + assert( pSrc->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pSrc->a[i].u4.pSubq->pSelect); } } } @@ -119392,8 +121998,8 @@ static void renameColumnFunc( if( rc!=SQLITE_OK ) goto renameColumnFunc_done; for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget ){ - Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pStep->pSrc ){ + Table *pTarget = sqlite3LocateTableItem(&sParse, 0, &pStep->pSrc->a[0]); if( pTarget==pTab ){ if( pStep->pUpsert ){ ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; @@ -119405,7 +122011,6 @@ static void renameColumnFunc( } } - /* Find tokens to edit in UPDATE OF clause */ if( sParse.pTriggerTab==pTab ){ renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); @@ -119607,13 +122212,10 @@ static void renameTableFunc( if( rc==SQLITE_OK ){ renameWalkTrigger(&sWalker, pTrigger); for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ - renameTokenFind(&sParse, &sCtx, pStep->zTarget); - } - if( pStep->pFrom ){ + if( pStep->pSrc ){ int i; - for(i=0; i<pStep->pFrom->nSrc; i++){ - SrcItem *pItem = &pStep->pFrom->a[i]; + for(i=0; i<pStep->pSrc->nSrc; i++){ + SrcItem *pItem = &pStep->pSrc->a[i]; if( 0==sqlite3_stricmp(pItem->zName, zOld) ){ renameTokenFind(&sParse, &sCtx, pItem->zName); } @@ -119860,6 +122462,57 @@ static void renameTableTest( #endif } + +/* +** Return the number of bytes until the end of the next non-whitespace and +** non-comment token. For the purpose of this function, a "(" token includes +** all of the bytes through and including the matching ")", or until the +** first illegal token, whichever comes first. +** +** Write the token type into *piToken. +** +** The value returned is the number of bytes in the token itself plus +** the number of bytes of leading whitespace and comments skipped plus +** all bytes through the next matching ")" if the token is TK_LP. +** +** Example: (Note: '.' used in place of '*' in the example z[] text) +** +** ,--------- *piToken := TK_RP +** v +** z[] = " /.comment./ --comment\n (two three four) five" +** | | +** |<-------------------------------------->| +** | +** `--- return value +*/ +static int getConstraintToken(const u8 *z, int *piToken){ + int iOff = 0; + int t = 0; + do { + iOff += sqlite3GetToken(&z[iOff], &t); + }while( t==TK_SPACE || t==TK_COMMENT ); + + *piToken = t; + + if( t==TK_LP ){ + int nNest = 1; + while( nNest>0 ){ + iOff += sqlite3GetToken(&z[iOff], &t); + if( t==TK_LP ){ + nNest++; + }else if( t==TK_RP ){ + t = TK_LP; + nNest--; + }else if( t==TK_ILLEGAL ){ + break; + } + } + } + + *piToken = t; + return iOff; +} + /* ** The implementation of internal UDF sqlite_drop_column(). ** @@ -119904,15 +122557,24 @@ static void dropColumnFunc( goto drop_column_done; } - pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); if( iCol<pTab->nCol-1 ){ RenameToken *pEnd; + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zCnName); zEnd = (const char*)pEnd->t.z; }else{ + int eTok; assert( IsOrdinaryTable(pTab) ); + assert( iCol!=0 ); + /* Point pCol->t.z at the "," immediately preceding the definition of + ** the column being dropped. To do this, start at the name of the + ** previous column, and tokenize until the next ",". */ + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol-1].zCnName); + do { + pCol->t.z += getConstraintToken((const u8*)pCol->t.z, &eTok); + }while( eTok!=TK_COMMA ); + pCol->t.z--; zEnd = (const char*)&zSql[pTab->u.tab.addColOffset]; - while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; } zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); @@ -120082,6 +122744,653 @@ exit_drop_column: } /* +** Return the number of bytes of leading whitespace/comments in string z[]. +*/ +static int getWhitespace(const u8 *z){ + int nRet = 0; + while( 1 ){ + int t = 0; + int n = sqlite3GetToken(&z[nRet], &t); + if( t!=TK_SPACE && t!=TK_COMMENT ) break; + nRet += n; + } + return nRet; +} + + +/* +** Argument z points into the body of a constraint - specifically the +** second token of the constraint definition. For a named constraint, +** z points to the first token past the CONSTRAINT keyword. For an +** unnamed NOT NULL constraint, z points to the first byte past the NOT +** keyword. +** +** Return the number of bytes until the end of the constraint. +*/ +static int getConstraint(const u8 *z){ + int iOff = 0; + int t = 0; + + /* Now, the current constraint proceeds until the next occurence of one + ** of the following tokens: + ** + ** CONSTRAINT, PRIMARY, NOT, UNIQUE, CHECK, DEFAULT, + ** COLLATE, REFERENCES, FOREIGN, GENERATED, AS, RP, or COMMA + ** + ** Also exit the loop if ILLEGAL turns up. + */ + while( 1 ){ + int n = getConstraintToken(&z[iOff], &t); + if( t==TK_CONSTRAINT || t==TK_PRIMARY || t==TK_NOT || t==TK_UNIQUE + || t==TK_CHECK || t==TK_DEFAULT || t==TK_COLLATE || t==TK_REFERENCES + || t==TK_FOREIGN || t==TK_RP || t==TK_COMMA || t==TK_ILLEGAL + || t==TK_AS || t==TK_GENERATED + ){ + break; + } + iOff += n; + } + + return iOff; +} + +/* +** Compare two constraint names. +** +** Summary: *pRes := zQuote != zCmp +** +** Details: +** Compare the (possibly quoted) constraint name zQuote[0..nQuote-1] +** against zCmp[]. Write zero into *pRes if they are the same and +** non-zero if they differ. Normally return SQLITE_OK, except if there +** is an OOM, set the OOM error condition on ctx and return SQLITE_NOMEM. +*/ +static int quotedCompare( + sqlite3_context *ctx, /* Function context on which to report errors */ + int t, /* Token type */ + const u8 *zQuote, /* Possibly quoted text. Not zero-terminated. */ + int nQuote, /* Length of zQuote in bytes */ + const u8 *zCmp, /* Zero-terminated, unquoted name to compare against */ + int *pRes /* OUT: Set to 0 if equal, non-zero if unequal */ +){ + char *zCopy = 0; /* De-quoted, zero-terminated copy of zQuote[] */ + + if( t==TK_ILLEGAL ){ + *pRes = 1; + return SQLITE_OK; + } + zCopy = sqlite3MallocZero(nQuote+1); + if( zCopy==0 ){ + sqlite3_result_error_nomem(ctx); + return SQLITE_NOMEM_BKPT; + } + memcpy(zCopy, zQuote, nQuote); + sqlite3Dequote(zCopy); + *pRes = sqlite3_stricmp((const char*)zCopy, (const char*)zCmp); + sqlite3_free(zCopy); + return SQLITE_OK; +} + +/* +** zSql[] is a CREATE TABLE statement, supposedly. Find the offset +** into zSql[] of the first character past the first "(" and write +** that offset into *piOff and return SQLITE_OK. Or, if not found, +** set the SQLITE_CORRUPT error code and return SQLITE_ERROR. +*/ +static int skipCreateTable(sqlite3_context *ctx, const u8 *zSql, int *piOff){ + int iOff = 0; + + if( zSql==0 ) return SQLITE_ERROR; + + /* Jump past the "CREATE TABLE" bit. */ + while( 1 ){ + int t = 0; + iOff += sqlite3GetToken(&zSql[iOff], &t); + if( t==TK_LP ) break; + if( t==TK_ILLEGAL ){ + sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); + return SQLITE_ERROR; + } + } + + *piOff = iOff; + return SQLITE_OK; +} + +/* +** Internal SQL function sqlite3_drop_constraint(): Given an input +** CREATE TABLE statement, return a revised CREATE TABLE statement +** with a constraint removed. Two forms, depending on the datatype +** of argv[2]: +** +** sqlite_drop_constraint(SQL, INT) -- Omit NOT NULL from the INT-th column +** sqlite_drop_constraint(SQL, TEXT) -- OMIT constraint with name TEXT +** +** In the first case, the left-most column is 0. +*/ +static void dropConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = sqlite3_value_text(argv[0]); + const u8 *zCons = 0; + int iNotNull = -1; + int ii; + int iOff = 0; + int iStart = 0; + int iEnd = 0; + char *zNew = 0; + int t = 0; + sqlite3 *db; + UNUSED_PARAMETER(NotUsed); + + if( zSql==0 ) return; + + /* Jump past the "CREATE TABLE" bit. */ + if( skipCreateTable(ctx, zSql, &iOff) ) return; + + if( sqlite3_value_type(argv[1])==SQLITE_INTEGER ){ + iNotNull = sqlite3_value_int(argv[1]); + }else{ + zCons = sqlite3_value_text(argv[1]); + } + + /* Search for the named constraint within column definitions. */ + for(ii=0; iEnd==0; ii++){ + + /* Now parse the column or table constraint definition. Search + ** for the token CONSTRAINT if this is a DROP CONSTRAINT command, or + ** NOT in the right column if this is a DROP NOT NULL. */ + while( 1 ){ + iStart = iOff; + iOff += getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT && (zCons || iNotNull==ii) ){ + /* Check if this is the constraint we are searching for. */ + int nTok = 0; + int cmp = 1; + + /* Skip past any whitespace. */ + iOff += getWhitespace(&zSql[iOff]); + + /* Compare the next token - which may be quoted - with the name of + ** the constraint being dropped. */ + nTok = getConstraintToken(&zSql[iOff], &t); + if( zCons ){ + if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; + } + iOff += nTok; + + /* The next token is usually the first token of the constraint + ** definition. This is enough to tell the type of the constraint - + ** TK_NOT means it is a NOT NULL, TK_CHECK a CHECK constraint etc. + ** + ** There is also the chance that the next token is TK_CONSTRAINT + ** (or TK_DEFAULT or TK_COLLATE), for example if a table has been + ** created as follows: + ** + ** CREATE TABLE t1(cols, CONSTRAINT one CONSTRAINT two NOT NULL); + ** + ** In this case, allow the "CONSTRAINT one" bit to be dropped by + ** this command if that is what is requested, or to advance to + ** the next iteration of the loop with &zSql[iOff] still pointing + ** to the CONSTRAINT keyword. */ + nTok = getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT || t==TK_DEFAULT || t==TK_COLLATE + || t==TK_COMMA || t==TK_RP || t==TK_GENERATED || t==TK_AS + ){ + t = TK_CHECK; + }else{ + iOff += nTok; + iOff += getConstraint(&zSql[iOff]); + } + + if( cmp==0 || (iNotNull>=0 && t==TK_NOT) ){ + if( t!=TK_NOT && t!=TK_CHECK ){ + errorMPrintf(ctx, "constraint may not be dropped: %s", zCons); + return; + } + iEnd = iOff; + break; + } + + }else if( t==TK_NOT && iNotNull==ii ){ + iEnd = iOff + getConstraint(&zSql[iOff]); + break; + }else if( t==TK_RP || t==TK_ILLEGAL ){ + iEnd = -1; + break; + }else if( t==TK_COMMA ){ + break; + } + } + } + + /* If the constraint has not been found it is an error. */ + if( iEnd<=0 ){ + if( zCons ){ + errorMPrintf(ctx, "no such constraint: %s", zCons); + }else{ + /* SQLite follows postgres in that a DROP NOT NULL on a column that is + ** not NOT NULL is not an error. So just return the original SQL here. */ + sqlite3_result_text(ctx, (const char*)zSql, -1, SQLITE_TRANSIENT); + } + }else{ + + /* Figure out if an extra space should be inserted after the constraint + ** is removed. And if an additional comma preceding the constraint + ** should be removed. */ + const char *zSpace = " "; + iEnd += getWhitespace(&zSql[iEnd]); + sqlite3GetToken(&zSql[iEnd], &t); + if( t==TK_RP || t==TK_COMMA ){ + zSpace = ""; + if( zSql[iStart-1]==',' ) iStart--; + } + + db = sqlite3_context_db_handle(ctx); + zNew = sqlite3MPrintf(db, "%.*s%s%s", iStart, zSql, zSpace, &zSql[iEnd]); + sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); + } +} + +/* +** Internal SQL function: +** +** sqlite_add_constraint(SQL, CONSTRAINT-TEXT, ICOL) +** +** SQL is a CREATE TABLE statement. Return a modified version of +** SQL that adds CONSTRAINT-TEXT at the end of the ICOL-th column +** definition. (The left-most column defintion is 0.) +*/ +static void addConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = sqlite3_value_text(argv[0]); + const char *zCons = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + int iOff = 0; + int ii; + char *zNew = 0; + int t = 0; + sqlite3 *db; + UNUSED_PARAMETER(NotUsed); + + if( skipCreateTable(ctx, zSql, &iOff) ) return; + + for(ii=0; ii<=iCol || (iCol<0 && t!=TK_RP); ii++){ + iOff += getConstraintToken(&zSql[iOff], &t); + while( 1 ){ + int nTok = getConstraintToken(&zSql[iOff], &t); + if( t==TK_COMMA || t==TK_RP ) break; + if( t==TK_ILLEGAL ){ + sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); + return; + } + iOff += nTok; + } + } + + iOff += getWhitespace(&zSql[iOff]); + + db = sqlite3_context_db_handle(ctx); + if( iCol<0 ){ + zNew = sqlite3MPrintf(db, "%.*s, %s%s", iOff, zSql, zCons, &zSql[iOff]); + }else{ + zNew = sqlite3MPrintf(db, "%.*s %s%s", iOff, zSql, zCons, &zSql[iOff]); + } + sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); +} + +/* +** Find a column named pCol in table pTab. If successful, set output +** parameter *piCol to the index of the column in the table and return +** SQLITE_OK. Otherwise, set *piCol to -1 and return an SQLite error +** code. +*/ +static int alterFindCol(Parse *pParse, Table *pTab, Token *pCol, int *piCol){ + sqlite3 *db = pParse->db; + char *zName = sqlite3NameFromToken(db, pCol); + int rc = SQLITE_NOMEM; + int iCol = -1; + + if( zName ){ + iCol = sqlite3ColumnIndex(pTab, zName); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: %s", zName); + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + } + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( rc==SQLITE_OK ){ + const char *zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + const char *zCol = pTab->aCol[iCol].zCnName; + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, zCol) ){ + pTab = 0; + } + } +#endif + + sqlite3DbFree(db, zName); + *piCol = iCol; + return rc; +} + + +/* +** Find the table named by the first entry in source list pSrc. If successful, +** return a pointer to the Table structure and set output variable (*pzDb) +** to point to the name of the database containin the table (i.e. "main", +** "temp" or the name of an attached database). +** +** If the table cannot be located, return NULL. The value of the two output +** parameters is undefined in this case. +*/ +static Table *alterFindTable( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Name of the table to look for */ + int *piDb, /* OUT: write the iDb here */ + const char **pzDb, /* OUT: write name of schema here */ + int bAuth /* Do ALTER TABLE authorization checks if true */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( pTab ){ + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + *pzDb = db->aDb[iDb].zDbSName; + *piDb = iDb; + + if( SQLITE_OK!=isRealTable(pParse, pTab, 2) + || SQLITE_OK!=isAlterableTable(pParse, pTab) + ){ + pTab = 0; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pTab && bAuth ){ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, *pzDb, pTab->zName, 0) ){ + pTab = 0; + } + } +#endif + sqlite3SrcListDelete(db, pSrc); + return pTab; +} + +/* +** Generate bytecode for one of: +** +** (1) ALTER TABLE pSrc DROP CONSTRAINT pCons +** (2) ALTER TABLE pSrc ALTER pCol DROP NOT NULL +** +** One of pCons and pCol must be NULL and the other non-null. +*/ +SQLITE_PRIVATE void sqlite3AlterDropConstraint( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* The table being altered */ + Token *pCons, /* Name of the constraint to drop */ + Token *pCol /* Name of the column from which to remove the NOT NULL */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; + int iDb = 0; + const char *zDb = 0; + char *zArg = 0; + + assert( (pCol==0)!=(pCons==0) ); + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, pCons!=0); + if( !pTab ) return; + + if( pCons ){ + char *z = sqlite3NameFromToken(db, pCons); + zArg = sqlite3MPrintf(db, "%Q", z); + sqlite3DbFree(db, z); + }else{ + int iCol; + if( alterFindCol(pParse, pTab, pCol, &iCol) ) return; + zArg = sqlite3MPrintf(db, "%d", iCol); + } + + /* Edit the SQL for the named table. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_drop_constraint(sql, %s) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, zArg, pTab->zName + ); + sqlite3DbFree(db, zArg); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* +** The implementation of SQL function sqlite_fail(MSG). This takes a single +** argument, and returns it as an error message with the error code set to +** SQLITE_CONSTRAINT. +*/ +static void failConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + int err = sqlite3_value_int(argv[1]); + (void)NotUsed; + sqlite3_result_error(ctx, zText, -1); + sqlite3_result_error_code(ctx, err); +} + +/* +** Buffer pCons, which is nCons bytes in size, contains the text of a +** NOT NULL or CHECK constraint that will be inserted into a CREATE TABLE +** statement. If successful, this function returns the size of the buffer in +** bytes not including any trailing whitespace or "--" style comments. Or, +** if an OOM occurs, it returns 0 and sets db->mallocFailed to true. +** +** C-style comments at the end are preserved. "--" style comments are +** removed because the comment terminator might be \000, and we are about +** to insert the pCons[] text into the middle of a larger string, and that +** will have the effect of removing the comment terminator and messing up +** the syntax. +*/ +static int alterRtrimConstraint( + sqlite3 *db, /* used to record OOM error */ + const char *pCons, /* Buffer containing constraint */ + int nCons /* Size of pCons in bytes */ +){ + u8 *zTmp = (u8*)sqlite3MPrintf(db, "%.*s", nCons, pCons); + int iOff = 0; + int iEnd = 0; + + if( zTmp==0 ) return 0; + + while( 1 ){ + int t = 0; + int nToken = sqlite3GetToken(&zTmp[iOff], &t); + if( t==TK_ILLEGAL ) break; + if( t!=TK_SPACE && (t!=TK_COMMENT || zTmp[iOff]!='-') ){ + iEnd = iOff+nToken; + } + iOff += nToken; + } + + sqlite3DbFree(db, zTmp); + return iEnd; +} + +/* +** Prepare a statement of the form: +** +** ALTER TABLE pSrc ALTER pCol SET NOT NULL +*/ +SQLITE_PRIVATE void sqlite3AlterSetNotNull( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Name of the table being altered */ + Token *pCol, /* Name of the column to add a NOT NULL constraint to */ + Token *pFirst /* The NOT token of the NOT NULL constraint text */ +){ + Table *pTab = 0; + int iCol = 0; + int iDb = 0; + const char *zDb = 0; + const char *pCons = 0; + int nCons = 0; + + /* Look up the table being altered. */ + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 0); + if( !pTab ) return; + + /* Find the column being altered. */ + if( alterFindCol(pParse, pTab, pCol, &iCol) ){ + return; + } + + /* Find the length in bytes of the constraint definition */ + pCons = pFirst->z; + nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); + + /* Search for a constraint violation. Throw an exception if one is found. */ + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint failed', %d) " + "FROM %Q.%Q AS x WHERE x.%.*s IS NULL", + SQLITE_CONSTRAINT, zDb, pTab->zName, (int)pCol->n, pCol->z + ); + + /* Edit the SQL for the named table. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_add_constraint(sqlite_drop_constraint(sql, %d), %.*Q, %d) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, iCol, nCons, pCons, iCol, pTab->zName + ); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* +** Implementation of internal SQL function: +** +** sqlite_find_constraint(SQL, CONSTRAINT-NAME) +** +** This function returns true if the SQL passed as the first argument is a +** CREATE TABLE that contains a constraint with the name CONSTRAINT-NAME, +** or false otherwise. +*/ +static void findConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = 0; + const u8 *zCons = 0; + int iOff = 0; + int t = 0; + + (void)NotUsed; + zSql = sqlite3_value_text(argv[0]); + zCons = sqlite3_value_text(argv[1]); + + if( zSql==0 || zCons==0 ) return; + while( t!=TK_LP && t!=TK_ILLEGAL ){ + iOff += sqlite3GetToken(&zSql[iOff], &t); + } + + while( 1 ){ + iOff += getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT ){ + int nTok = 0; + int cmp = 0; + iOff += getWhitespace(&zSql[iOff]); + nTok = getConstraintToken(&zSql[iOff], &t); + if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; + if( cmp==0 ){ + sqlite3_result_int(ctx, 1); + return; + } + }else if( t==TK_ILLEGAL ){ + break; + } + } + + sqlite3_result_int(ctx, 0); +} + +/* +** Generate bytecode to implement: +** +** ALTER TABLE pSrc ADD [CONSTRAINT pName] CHECK(pExpr) +** +** Any "ON CONFLICT" text that occurs after the "CHECK(...)", up +** until pParse->sLastToken, is included as part of the new constraint. +*/ +SQLITE_PRIVATE void sqlite3AlterAddConstraint( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* Table to add constraint to */ + Token *pFirst, /* First token of new constraint */ + Token *pName, /* Name of new constraint. NULL if name omitted. */ + const char *pExpr, /* Text of CHECK expression */ + int nExpr /* Size of pExpr in bytes */ +){ + Table *pTab = 0; /* Table identified by pSrc */ + int iDb = 0; /* Which schema does pTab live in */ + const char *zDb = 0; /* Name of the schema in which pTab lives */ + const char *pCons = 0; /* Text of the constraint */ + int nCons; /* Bytes of text to use from pCons[] */ + + /* Look up the table being altered. */ + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 1); + if( !pTab ) return; + + /* If this new constraint has a name, check that it is not a duplicate of + ** an existing constraint. It is an error if it is. */ + if( pName ){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint %q already exists', %d) " + "FROM \"%w\"." LEGACY_SCHEMA_TABLE " " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase " + "AND sqlite_find_constraint(sql, %Q)", + zName, SQLITE_ERROR, zDb, pTab->zName, zName + ); + sqlite3DbFree(pParse->db, zName); + } + + /* Search for a constraint violation. Throw an exception if one is found. */ + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint failed', %d) " + "FROM %Q.%Q WHERE (%.*s) IS NOT TRUE", + SQLITE_CONSTRAINT, zDb, pTab->zName, nExpr, pExpr + ); + + /* Edit the SQL for the named table. */ + pCons = pFirst->z; + nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); + + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_add_constraint(sql, %.*Q, -1) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, nCons, pCons, pTab->zName + ); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* ** Register built-in functions used to help implement ALTER TABLE */ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ @@ -120091,6 +123400,10 @@ SQLITE_PRIVATE void sqlite3AlterFunctions(void){ INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), INTERNAL_FUNCTION(sqlite_rename_quotefix,2, renameQuotefixFunc), + INTERNAL_FUNCTION(sqlite_drop_constraint,2, dropConstraintFunc), + INTERNAL_FUNCTION(sqlite_fail, 2, failConstraintFunc), + INTERNAL_FUNCTION(sqlite_add_constraint, 3, addConstraintFunc), + INTERNAL_FUNCTION(sqlite_find_constraint,2, findConstraintFunc), }; sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); } @@ -122221,6 +125534,16 @@ static void attachFunc( ** from sqlite3_deserialize() to close database db->init.iDb and ** reopen it as a MemDB */ Btree *pNewBt = 0; + + pNew = &db->aDb[db->init.iDb]; + assert( pNew->pBt!=0 ); + if( sqlite3BtreeTxnState(pNew->pBt)!=SQLITE_TXN_NONE + || sqlite3BtreeIsInBackup(pNew->pBt) + ){ + rc = SQLITE_BUSY; + goto attach_error; + } + pVfs = sqlite3_vfs_find("memdb"); if( pVfs==0 ) return; rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNewBt, 0, SQLITE_OPEN_MAIN_DB); @@ -122230,8 +125553,7 @@ static void attachFunc( /* Both the Btree and the new Schema were allocated successfully. ** Close the old db and update the aDb[] slot with the new memdb ** values. */ - pNew = &db->aDb[db->init.iDb]; - if( ALWAYS(pNew->pBt) ) sqlite3BtreeClose(pNew->pBt); + sqlite3BtreeClose(pNew->pBt); pNew->pBt = pNewBt; pNew->pSchema = pNewSchema; }else{ @@ -122711,7 +126033,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) || sqlite3WalkExpr(&pFix->w, pStep->pWhere) || sqlite3WalkExprList(&pFix->w, pStep->pExprList) - || sqlite3FixSrcList(pFix, pStep->pFrom) + || sqlite3FixSrcList(pFix, pStep->pSrc) ){ return 1; } @@ -122818,7 +126140,7 @@ SQLITE_API int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); + sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -123436,6 +126758,16 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); } +#ifndef SQLITE_OMIT_JSON + if( pMod==0 && sqlite3_strnicmp(zName, "json", 4)==0 ){ + pMod = sqlite3JsonVtabRegister(db, zName); + } +#endif +#ifdef SQLITE_ENABLE_CARRAY + if( pMod==0 && sqlite3_stricmp(zName, "carray")==0 ){ + pMod = sqlite3CarrayRegister(db); + } +#endif if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ testcase( pMod->pEpoTab==0 ); return pMod->pEpoTab; @@ -123479,6 +126811,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTableItem( const char *zDb; if( p->fg.fixedSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); + assert( iDb>=0 && iDb<pParse->db->nDb ); zDb = pParse->db->aDb[iDb].zDbSName; }else{ assert( !p->fg.isSubquery ); @@ -124074,7 +127407,7 @@ SQLITE_PRIVATE int sqlite3TableColumnToIndex(Index *pIdx, int iCol){ int i; i16 iCol16; assert( iCol>=(-1) && iCol<=SQLITE_MAX_COLUMN ); - assert( pIdx->nColumn<=SQLITE_MAX_COLUMN+1 ); + assert( pIdx->nColumn<=SQLITE_MAX_COLUMN*2 ); iCol16 = iCol; for(i=0; i<pIdx->nColumn; i++){ if( iCol16==pIdx->aiColumn[i] ){ @@ -124371,6 +127704,9 @@ SQLITE_PRIVATE void sqlite3StartTable( sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); + }else if( db->init.imposterTable ){ + pTable->tabFlags |= TF_Imposter; + if( db->init.imposterTable>=2 ) pTable->tabFlags |= TF_Readonly; } /* Normal (non-error) return. */ @@ -125049,8 +128385,8 @@ SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ ** The estimate is conservative. It might be larger that what is ** really needed. */ -static int identLength(const char *z){ - int n; +static i64 identLength(const char *z){ + i64 n; for(n=0; *z; n++, z++){ if( *z=='"' ){ n++; } } @@ -125483,9 +128819,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, j, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + const char *zColl = sqlite3ColumnColl(&pTab->aCol[i]); assert( j<pPk->nColumn ); pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; + pPk->azColl[j] = zColl ? zColl : sqlite3StrBINARY; j++; } } @@ -125560,13 +128897,14 @@ SQLITE_PRIVATE void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){ ** restored to its original value prior to this routine returning. */ SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ - char *zTail; /* Pointer to the last "_" in zName */ + const char *zTail; /* Pointer to the last "_" in zName */ Table *pTab; /* Table that zName is a shadow of */ + char *zCopy; zTail = strrchr(zName, '_'); if( zTail==0 ) return 0; - *zTail = 0; - pTab = sqlite3FindTable(db, zName, 0); - *zTail = '_'; + zCopy = sqlite3DbStrNDup(db, zName, (int)(zTail-zName)); + pTab = zCopy ? sqlite3FindTable(db, zCopy, 0) : 0; + sqlite3DbFree(db, zCopy); if( pTab==0 ) return 0; if( !IsVirtual(pTab) ) return 0; return sqlite3IsShadowTableOf(db, pTab, zName); @@ -125719,6 +129057,7 @@ SQLITE_PRIVATE void sqlite3EndTable( convertToWithoutRowidTable(pParse, p); } iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( iDb>=0 && iDb<=db->nDb ); #ifndef SQLITE_OMIT_CHECK /* Resolve names in all CHECK constraint expressions. @@ -126014,6 +129353,7 @@ SQLITE_PRIVATE void sqlite3CreateView( sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( iDb>=0 && iDb<db->nDb ); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; @@ -127610,6 +130950,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists goto exit_drop_index; } iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); + assert( iDb>=0 && iDb<db->nDb ); #ifndef SQLITE_OMIT_AUTHORIZATION { int code = SQLITE_DROP_INDEX; @@ -128140,16 +131481,22 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI ** are deleted by this function. */ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ - assert( p1 && p1->nSrc==1 ); + assert( p1 ); + assert( p2 || pParse->nErr ); + assert( p2==0 || p2->nSrc>=1 ); + testcase( p1->nSrc==0 ); if( p2 ){ - SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + int nOld = p1->nSrc; + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, nOld); if( pNew==0 ){ sqlite3SrcListDelete(pParse->db, p2); }else{ p1 = pNew; - memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + memcpy(&p1->a[nOld], p2->a, p2->nSrc*sizeof(SrcItem)); + assert( nOld==1 || (p2->a[0].fg.jointype & JT_LTORJ)==0 ); + assert( p1->nSrc>=1 ); + p1->a[0].fg.jointype |= (JT_LTORJ & p2->a[0].fg.jointype); sqlite3DbFree(pParse->db, p2); - p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype); } } return p1; @@ -128501,8 +131848,7 @@ SQLITE_PRIVATE void sqlite3RowidConstraint( } /* -** Check to see if pIndex uses the collating sequence pColl. Return -** true if it does and false if it does not. +** Return true if any column of pIndex uses the zColl collation */ #ifndef SQLITE_OMIT_REINDEX static int collationMatch(const char *zColl, Index *pIndex){ @@ -128510,8 +131856,8 @@ static int collationMatch(const char *zColl, Index *pIndex){ assert( zColl!=0 ); for(i=0; i<pIndex->nColumn; i++){ const char *z = pIndex->azColl[i]; - assert( z!=0 || pIndex->aiColumn[i]<0 ); - if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ + assert( z!=0 ); + if( 0==sqlite3StrICmp(z, zColl) ){ return 1; } } @@ -128520,72 +131866,38 @@ static int collationMatch(const char *zColl, Index *pIndex){ #endif /* -** Recompute all indices of pTab that use the collating sequence pColl. -** If pColl==0 then recompute all indices of pTab. -*/ -#ifndef SQLITE_OMIT_REINDEX -static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ - if( !IsVirtual(pTab) ){ - Index *pIndex; /* An index associated with pTab */ - - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( zColl==0 || collationMatch(zColl, pIndex) ){ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - } - } - } -} -#endif - -/* -** Recompute all indices of all tables in all databases where the -** indices use the collating sequence pColl. If pColl==0 then recompute -** all indices everywhere. -*/ -#ifndef SQLITE_OMIT_REINDEX -static void reindexDatabases(Parse *pParse, char const *zColl){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - sqlite3 *db = pParse->db; /* The database connection */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - - assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ - for(iDb=0, pDb=db->aDb; iDb<db->nDb; iDb++, pDb++){ - assert( pDb!=0 ); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - reindexTable(pParse, pTab, zColl); - } - } -} -#endif - -/* ** Generate code for the REINDEX command. ** ** REINDEX -- 1 ** REINDEX <collation> -- 2 -** REINDEX ?<database>.?<tablename> -- 3 -** REINDEX ?<database>.?<indexname> -- 4 +** REINDEX ?<database>.?<indexname> -- 3 +** REINDEX ?<database>.?<tablename> -- 4 +** REINDEX EXPRESSIONS -- 5 ** -** Form 1 causes all indices in all attached databases to be rebuilt. -** Form 2 rebuilds all indices in all databases that use the named +** Form 1 causes all indexes in all attached databases to be rebuilt. +** Form 2 rebuilds all indexes in all databases that use the named ** collating function. Forms 3 and 4 rebuild the named index or all -** indices associated with the named table. +** indexes associated with the named table, respectively. Form 5 +** rebuilds all expression indexes in addition to all collations, +** indexes, or tables named "EXPRESSIONS". +** +** If the name is ambiguous such that it matches two or more of +** forms 2 through 5, then rebuild the union of all matching indexes, +** taken care to avoid rebuilding the same index more than once. */ #ifndef SQLITE_OMIT_REINDEX SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ - CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ - char *z; /* Name of a table or index */ - const char *zDb; /* Name of the database */ - Table *pTab; /* A table in the database */ - Index *pIndex; /* An index associated with pTab */ - int iDb; /* The database index number */ + char *z = 0; /* Name of a table or index or collation */ + const char *zDb = 0; /* Name of the database */ + int iReDb = -1; /* The database index number */ sqlite3 *db = pParse->db; /* The database connection */ Token *pObjName; /* Name of the table or index to be reindexed */ + int bMatch = 0; /* At least one name match */ + const char *zColl = 0; /* Rebuild indexes using this collation */ + Table *pReTab = 0; /* Rebuild all indexes of this table */ + Index *pReIndex = 0; /* Rebuild this index */ + int isExprIdx = 0; /* Rebuild all expression indexes */ + int bAll = 0; /* Rebuild all indexes */ /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -128594,41 +131906,66 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ } if( pName1==0 ){ - reindexDatabases(pParse, 0); - return; + /* rebuild all indexes */ + bMatch = 1; + bAll = 1; }else if( NEVER(pName2==0) || pName2->z==0 ){ - char *zColl; assert( pName1->z ); - zColl = sqlite3NameFromToken(pParse->db, pName1); - if( !zColl ) return; - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - if( pColl ){ - reindexDatabases(pParse, zColl); - sqlite3DbFree(db, zColl); - return; + z = sqlite3NameFromToken(pParse->db, pName1); + if( z==0 ) return; + }else{ + iReDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iReDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iReDb].zDbSName; + } + if( !bAll ){ + if( zDb==0 && sqlite3StrICmp(z, "expressions")==0 ){ + isExprIdx = 1; + bMatch = 1; + } + if( zDb==0 && sqlite3FindCollSeq(db, ENC(db), z, 0)!=0 ){ + zColl = z; + bMatch = 1; + } + if( zColl==0 && (pReTab = sqlite3FindTable(db, z, zDb))!=0 ){ + bMatch = 1; + } + if( zColl==0 && (pReIndex = sqlite3FindIndex(db, z, zDb))!=0 ){ + bMatch = 1; } - sqlite3DbFree(db, zColl); } - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); - if( iDb<0 ) return; - z = sqlite3NameFromToken(db, pObjName); - if( z==0 ) return; - zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; - pTab = sqlite3FindTable(db, z, zDb); - if( pTab ){ - reindexTable(pParse, pTab, 0); - sqlite3DbFree(db, z); - return; + if( bMatch ){ + int iDb; + HashElem *k; + Table *pTab; + Index *pIdx; + Db *pDb; + for(iDb=0, pDb=db->aDb; iDb<db->nDb; iDb++, pDb++){ + assert( pDb!=0 ); + if( iReDb>=0 && iReDb!=iDb ) continue; + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + if( IsVirtual(pTab) ) continue; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( bAll + || pTab==pReTab + || pIdx==pReIndex + || (isExprIdx && pIdx->bHasExpr) + || (zColl!=0 && collationMatch(zColl,pIdx)) + ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIdx, -1); + } + } /* End loop over indexes of pTab */ + } /* End loop over tables of iDb */ + } /* End loop over databases */ + }else{ + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); } - pIndex = sqlite3FindIndex(db, z, zDb); sqlite3DbFree(db, z); - if( pIndex ){ - iDb = sqlite3SchemaToIndex(db, pIndex->pTable->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - return; - } - sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); + return; } #endif @@ -128660,14 +131997,19 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ } if( pParse->nErr ){ assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); - if( pIdx->bNoQuery==0 ){ + if( pIdx->bNoQuery==0 + && sqlite3HashFind(&pIdx->pSchema->idxHash, pIdx->zName) + ){ /* Deactivate the index because it contains an unknown collating ** sequence. The only way to reactive the index is to reload the ** schema. Adding the missing collating sequence later does not ** reactive the index. The application had the chance to register ** the missing index using the collation-needed callback. For ** simplicity, SQLite will not give the application a second chance. - */ + ** + ** Except, do not do this if the index is not in the schema hash + ** table. In this case the index is currently being constructed + ** by a CREATE INDEX statement, and retrying will not help. */ pIdx->bNoQuery = 1; pParse->rc = SQLITE_ERROR_RETRY; } @@ -129304,6 +132646,7 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } + sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ @@ -129432,7 +132775,7 @@ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS ** virtual tables if PRAGMA trusted_schema=ON. */ - if( pParse->pToplevel!=0 + if( (pParse->pToplevel!=0 || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) && pTab->u.vtab.p->eVtabRisk > ((pParse->db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -130269,8 +133612,9 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); - sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn + ); + sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -130845,7 +134189,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); + sqlite3AtoF(zBuf, &r); sqlite3_free(zBuf); } sqlite3_result_double(context, r); @@ -130864,7 +134208,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3 *db = sqlite3_context_db_handle(context); assert( nByte>0 ); testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + testcase( nByte==(i64)db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); z = 0; @@ -131477,18 +134821,11 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int switch( sqlite3_value_type(pValue) ){ case SQLITE_FLOAT: { - double r1, r2; - const char *zVal; - r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!0.15g", r1); - zVal = sqlite3_str_value(pStr); - if( zVal ){ - sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); - if( r1!=r2 ){ - sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!0.20e", r1); - } - } + /* ,--- Show infinity as 9.0e+999 + ** | + ** | ,--- 17 precision guarantees round-trip + ** v v */ + sqlite3_str_appendf(pStr, "%!0.17g", sqlite3_value_double(pValue)); break; } case SQLITE_INTEGER: { @@ -131535,7 +134872,7 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int */ static int isNHex(const char *z, int N, u32 *pVal){ int i; - int v = 0; + u32 v = 0; for(i=0; i<N; i++){ if( !sqlite3Isxdigit(z[i]) ) return 0; v = (v<<4) + sqlite3HexToInt(z[i]); @@ -131580,7 +134917,7 @@ static void unistrFunc( } i = j = 0; while( i<nIn ){ - char *z = strchr(&zIn[i],'\\'); + const char *z = strchr(&zIn[i],'\\'); if( z==0 ){ n = nIn - i; memmove(&zOut[j], &zIn[i], n); @@ -131617,7 +134954,7 @@ static void unistrFunc( } } zOut[j] = 0; - sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8_ZT); return; unistr_error: @@ -131710,7 +135047,7 @@ static void charFunc( } \ } *zOut = 0; - sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(context, (char*)z, zOut-z,sqlite3_free,SQLITE_UTF8_ZT); } /* @@ -131739,7 +135076,7 @@ static void hexFunc( } *z = 0; sqlite3_result_text64(context, zHex, (u64)(z-zHex), - sqlite3_free, SQLITE_UTF8); + sqlite3_free, SQLITE_UTF8_ZT); } } @@ -132048,6 +135385,7 @@ static void concatFuncCore( ){ i64 j, n = 0; int i; + int bNotNull = 0; /* True after at least NOT NULL argument seen */ char *z; for(i=0; i<argc; i++){ n += sqlite3_value_bytes(argv[i]); @@ -132064,18 +135402,19 @@ static void concatFuncCore( int k = sqlite3_value_bytes(argv[i]); const char *v = (const char*)sqlite3_value_text(argv[i]); if( v!=0 ){ - if( j>0 && nSep>0 ){ + if( bNotNull && nSep>0 ){ memcpy(&z[j], zSep, nSep); j += nSep; } memcpy(&z[j], v, k); j += k; + bNotNull = 1; } } } z[j] = 0; assert( j<=n ); - sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8_ZT); } /* @@ -132741,6 +136080,8 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); + assert( pDef!=0 ); /* The sqlite3CreateFunc() call above cannot fail + ** because the "like" SQL-function already exists */ pDef->funcFlags |= flags; pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; } @@ -133015,6 +136356,502 @@ static void signFunc( sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); } +#if defined(SQLITE_ENABLE_PERCENTILE) +/*********************************************************************** +** This section implements the percentile(Y,P) SQL function and similar. +** Requirements: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cont(Y,P) function is equivalent to +** percentile(Y,P/100.0). In other words, the fraction value in +** the second argument is in the range of 0 to 1 instead of 0 to 100. +** +** (15) A separate percentile_disc(Y,P) function is like +** percentile_cont(Y,P) except that instead of returning the weighted +** average of the nearest two input values, it returns the next lower +** value. So the percentile_disc(Y,P) will always return a value +** that was one of the inputs. +** +** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and +** percentile_disc(Y,P) can be used as window functions. +** +** Differences from standard SQL: +** +** * The percentile_cont(X,P) function is equivalent to the following in +** standard SQL: +** +** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) +** +** The SQLite syntax is much more compact. The standard SQL syntax +** is also supported if SQLite is compiled with the +** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. +** +** * No median(X) function exists in the SQL standard. App developers +** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". +** +** * No percentile(Y,P) function exists in the SQL standard. Instead of +** percential(Y,P), developers must write this: +** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that +** the fraction parameter to percentile() goes from 0 to 100 whereas +** the fraction parameter in SQL standard percentile_cont() goes from +** 0 to 1. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of these routines work +** by accumulating all values in an array of doubles, then sorting +** that array using quicksort before computing the answer. Thus +** the runtime is O(NlogN) where N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can imagine alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. +*/ + +/* The following object is the group context for a single percentile() +** aggregate. Remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + u64 nAlloc; /* Number of slots allocated for a[] */ + u64 nUsed; /* Number of slots actually used in a[] */ + char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ + char bPctValid; /* True if rPct is valid */ + double rPct; /* Fraction. 0.0 to 1.0 */ + double *a; /* Array of Y values */ +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int percentIsInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less. +*/ +static int percentSameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static i64 percentBinarySearch(Percentile *p, double y, int bExact){ + i64 iFirst = 0; /* First element of search range */ + i64 iLast = (i64)p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + i64 iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( x<y ){ + iFirst = iMid + 1; + }else if( x>y ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + +/* +** Generate an error for a percentile function. +** +** The error format string must have exactly one occurrence of "%%s()" +** (with two '%' characters). That substring will be replaced by the name +** of the function. +*/ +static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ + char *zMsg1; + char *zMsg2; + va_list ap; + + va_start(ap, zFormat); + zMsg1 = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, sqlite3VdbeFuncName(pCtx)) : 0; + sqlite3_result_error(pCtx, zMsg2, -1); + sqlite3_free(zMsg1); + sqlite3_free(zMsg2); +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 0.5; + }else{ + /* P must be a number between 0 and 100 for percentile() or between + ** 0.0 and 1.0 for percentile_cont() and percentile_disc(). + ** + ** The user-data is an integer which is 10 times the upper bound. + */ + double mxFrac = (SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx))&2)? 100.0 : 1.0; + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1])/mxFrac; + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 + ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not between 0.0 and %.1f", + (double)mxFrac); + return; + } + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( !p->bPctValid ){ + p->rPct = rPct; + p->bPctValid = 1; + }else if( !percentSameValue(p->rPct,rPct) ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not the same for all input rows"); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + percentError(pCtx, "input to %%s() is not numeric"); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + percentError(pCtx, "Inf input to %%s()"); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + u64 n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc64(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; + p->bSorted = 1; + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + i64 i; + i = percentBinarySearch(p, y, 0); + if( i<(int)p->nUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; + p->bSorted = 0; + } +} + +/* +** Interchange two doubles. +*/ +#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} + +/* +** Sort an array of doubles. +** +** Algorithm: quicksort +** +** This is implemented separately rather than using the qsort() routine +** from the standard library because: +** +** (1) To avoid a dependency on qsort() +** (2) To avoid the function call to the comparison routine for each +** comparison. +*/ +static void percentSort(double *a, unsigned int n){ + int iLt; /* Entries before a[iLt] are less than rPivot */ + int iGt; /* Entries at or after a[iGt] are greater than rPivot */ + int i; /* Loop counter */ + double rPivot; /* The pivot value */ + + assert( n>=2 ); + if( a[0]>a[n-1] ){ + SWAP_DOUBLE(a[0],a[n-1]) + } + if( n==2 ) return; + iGt = n-1; + i = n/2; + if( a[0]>a[i] ){ + SWAP_DOUBLE(a[0],a[i]) + }else if( a[i]>a[iGt] ){ + SWAP_DOUBLE(a[i],a[iGt]) + } + if( n==3 ) return; + rPivot = a[i]; + iLt = i = 1; + do{ + if( a[i]<rPivot ){ + if( i>iLt ) SWAP_DOUBLE(a[i],a[iLt]) + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + SWAP_DOUBLE(a[i],a[iGt]) + }else{ + i++; + } + }while( i<iGt ); + if( iLt>=2 ) percentSort(a, iLt); + if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; i<n-1; i++){ + assert( a[i]<=a[i+1] ); + } +#endif +} + + +/* +** The "inverse" function for percentile(Y,P) is called to remove a +** row that was previously inserted by "step". +*/ +static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){ + Percentile *p; + int eType; + double y; + i64 i; + assert( argc==2 || argc==1 ); + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + assert( p!=0 ); + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + return; + } + + /* Ignore the Y value if it is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + return; + } + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( i<(int)p->nUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. +*/ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ + Percentile *p; + int settings = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx))&1; /* Discrete? */ + unsigned i1, i2; + double v1, v2; + double ix, vx; + p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); + if( p==0 ) return; + if( p->a==0 ) return; + if( p->nUsed ){ + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + ix = p->rPct*(p->nUsed-1); + i1 = (unsigned)ix; + if( settings & 1 ){ + vx = p->a[i1]; + }else{ + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + } + sqlite3_result_double(pCtx, vx); + } + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; + } +} +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); +} +/****** End of percentile family of functions ******/ +#endif /* SQLITE_ENABLE_PERCENTILE */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Implementation of sqlite_filestat(SCHEMA). +** +** Return JSON text that describes low-level debug/diagnostic information +** about the sqlite3_file object associated with SCHEMA. +*/ +static void filestatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDbName; + sqlite3_str *pStr; + Btree *pBtree; + + zDbName = (const char*)sqlite3_value_text(argv[0]); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + int rc; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + pStr = sqlite3_str_new(db); + if( pStr==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_str_append(pStr, "{\"db\":", 6); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_FILESTAT, pStr); + if( rc ) sqlite3_str_append(pStr, "null", 4); + fd = sqlite3PagerJrnlFile(pPager); + if( fd && fd->pMethods!=0 ){ + sqlite3_str_appendall(pStr, ",\"journal\":"); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_FILESTAT, pStr); + if( rc ) sqlite3_str_append(pStr, "null", 4); + } + sqlite3_str_append(pStr, "}", 1); + sqlite3_result_text(context, sqlite3_str_finish(pStr), -1, + sqlite3_free); + } + sqlite3BtreeLeave(pBtree); + }else{ + sqlite3_result_text(context, "{}", 2, SQLITE_STATIC); + } +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + #ifdef SQLITE_DEBUG /* ** Implementation of fpdecode(x,y,z) function. @@ -133173,6 +137010,9 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC INLINE_FUNC(sqlite_offset, 1, INLINEFUNC_sqlite_offset, 0 ), #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + FUNCTION(sqlite_filestat, 1, 0, 0, filestatFunc ), +#endif FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), FUNCTION(rtrim, 1, 2, 0, trimFunc ), @@ -133245,6 +137085,21 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ WAGGREGATE(string_agg, 2, 0, 0, groupConcatStep, groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), +#ifdef SQLITE_ENABLE_PERCENTILE + WAGGREGATE(median, 1, 0,0, percentStep, + percentFinal, percentValue, percentInverse, + SQLITE_INNOCUOUS|SQLITE_SELFORDER1), + WAGGREGATE(percentile, 2, 0x2,0, percentStep, + percentFinal, percentValue, percentInverse, + SQLITE_INNOCUOUS|SQLITE_SELFORDER1), + WAGGREGATE(percentile_cont, 2, 0,0, percentStep, + percentFinal, percentValue, percentInverse, + SQLITE_INNOCUOUS|SQLITE_SELFORDER1), + WAGGREGATE(percentile_disc, 2, 0x1,0, percentStep, + percentFinal, percentValue, percentInverse, + SQLITE_INNOCUOUS|SQLITE_SELFORDER1), +#endif /* SQLITE_ENABLE_PERCENTILE */ + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), #ifdef SQLITE_CASE_SENSITIVE_LIKE LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), @@ -134015,6 +137870,7 @@ SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ if( p ){ TriggerStep *pStep = p->step_list; + sqlite3SrcListDelete(dbMem, pStep->pSrc); sqlite3ExprDelete(dbMem, pStep->pWhere); sqlite3ExprListDelete(dbMem, pStep->pExprList); sqlite3SelectDelete(dbMem, pStep->pSelect); @@ -134234,6 +138090,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( !IsOrdinaryTable(pTab) ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=00 && iDb<db->nDb ); zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the @@ -134651,7 +138508,6 @@ static Trigger *fkActionTrigger( nFrom = sqlite3Strlen30(zFrom); if( action==OE_Restrict ){ - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); SrcList *pSrc; Expr *pRaise; @@ -134662,10 +138518,10 @@ static Trigger *fkActionTrigger( } pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ - assert( pSrc->nSrc==1 ); - pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); - pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + SrcItem *pItem = &pSrc->a[0]; + pItem->zName = sqlite3DbStrDup(db, zFrom); + pItem->fg.fixedSchema = 1; + pItem->u4.pSchema = pTab->pSchema; } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), @@ -134681,14 +138537,17 @@ static Trigger *fkActionTrigger( pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ - sizeof(TriggerStep) + /* Single step in trigger program */ - nFrom + 1 /* Space for pStep->zTarget */ + sizeof(TriggerStep) /* Single step in trigger program */ ); if( pTrigger ){ pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->zTarget = (char *)&pStep[1]; - memcpy((char *)pStep->zTarget, zFrom, nFrom); - + pStep->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pStep->pSrc ){ + SrcItem *pItem = &pStep->pSrc->a[0]; + pItem->zName = sqlite3DbStrNDup(db, zFrom, nFrom); + pItem->u4.pSchema = pTab->pSchema; + pItem->fg.fixedSchema = 1; + } pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); @@ -134999,12 +138858,15 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** by one slot and insert a new OP_TypeCheck where the current ** OP_MakeRecord is found */ VdbeOp *pPrev; + int p3; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; - sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, pPrev->p3); + p3 = pPrev->p3; + pPrev->p3 = 0; + sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, p3); }else{ /* Insert an isolated OP_Typecheck */ sqlite3VdbeAddOp2(v, OP_TypeCheck, iReg, pTab->nNVCol); @@ -138739,6 +142601,14 @@ struct sqlite3_api_routines { int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); /* Version 3.50.0 and later */ int (*setlk_timeout)(sqlite3*,int,int); + /* Version 3.51.0 and later */ + int (*set_errmsg)(sqlite3*,int,const char*); + int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); + /* Version 3.52.0 and later */ + void (*str_truncate)(sqlite3_str*,int); + void (*str_free)(sqlite3_str*); + int (*carray_bind)(sqlite3_stmt*,int,void*,int,int,void(*)(void*)); + int (*carray_bind_v2)(sqlite3_stmt*,int,void*,int,int,void(*)(void*),void*); }; /* @@ -139074,6 +142944,14 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_set_clientdata sqlite3_api->set_clientdata /* Version 3.50.0 and later */ #define sqlite3_setlk_timeout sqlite3_api->setlk_timeout +/* Version 3.51.0 and later */ +#define sqlite3_set_errmsg sqlite3_api->set_errmsg +#define sqlite3_db_status64 sqlite3_api->db_status64 +/* Version 3.52.0 and later */ +#define sqlite3_str_truncate sqlite3_api->str_truncate +#define sqlite3_str_free sqlite3_api->str_free +#define sqlite3_carray_bind sqlite3_api->carray_bind +#define sqlite3_carray_bind_v2 sqlite3_api->carray_bind_v2 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -139597,7 +143475,20 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_get_clientdata, sqlite3_set_clientdata, /* Version 3.50.0 and later */ - sqlite3_setlk_timeout + sqlite3_setlk_timeout, + /* Version 3.51.0 and later */ + sqlite3_set_errmsg, + sqlite3_db_status64, + /* Version 3.52.0 and later */ + sqlite3_str_truncate, + sqlite3_str_free, +#ifdef SQLITE_ENABLE_CARRAY + sqlite3_carray_bind, + sqlite3_carray_bind_v2 +#else + 0, + 0 +#endif }; /* True if x is the directory separator character @@ -139699,33 +143590,42 @@ static int sqlite3LoadExtension( ** entry point name "sqlite3_extension_init" was not found, then ** construct an entry point name "sqlite3_X_init" where the X is ** replaced by the lowercase value of every ASCII alphabetic - ** character in the filename after the last "/" upto the first ".", - ** and eliding the first three characters if they are "lib". + ** character in the filename after the last "/" up to the first ".", + ** and skipping the first three characters if they are "lib". ** Examples: ** ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example_init ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + ** + ** If that still finds no entry point, repeat a second time but this + ** time include both alphabetic and numeric characters up to the first + ** ".". Example: + ** + ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example5_init */ if( xInit==0 && zProc==0 ){ int iFile, iEntry, c; int ncFile = sqlite3Strlen30(zFile); + int cnt = 0; zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); return SQLITE_NOMEM_BKPT; } - memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} - iFile++; - if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; - for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ - if( sqlite3Isalpha(c) ){ - zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + do{ + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) || (cnt && sqlite3Isdigit(c)) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } } - } - memcpy(zAltEntry+iEntry, "_init", 6); - zEntry = zAltEntry; - xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + }while( xInit==0 && (++cnt)<2 ); } if( xInit==0 ){ if( pzErrMsg ){ @@ -141060,6 +144960,22 @@ static int integrityCheckResultRow(Vdbe *v){ } /* +** Should table pTab be skipped when doing an integrity_check? +** Return true or false. +** +** If pObjTab is not null, the return true if pTab matches pObjTab. +** +** If pObjTab is null, then return true only if pTab is an imposter table. +*/ +static int tableSkipIntegrityCheck(const Table *pTab, const Table *pObjTab){ + if( pObjTab ){ + return pTab!=pObjTab; + }else{ + return (pTab->tabFlags & TF_Imposter)!=0; + } +} + +/* ** Process a pragma statement. ** ** Pragmas are of this form: @@ -142404,7 +146320,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab = sqliteHashData(x); /* Current table */ Index *pIdx; /* An index on pTab */ int nIdx; /* Number of indexes on pTab */ - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } } @@ -142417,7 +146333,7 @@ SQLITE_PRIVATE void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ aRoot[++cnt] = pIdx->tnum; @@ -142448,7 +146364,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int iTab = 0; Table *pTab = sqliteHashData(x); Index *pIdx; - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ){ iTab = cnt++; }else{ @@ -142484,7 +146400,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int r2; /* Previous key for WITHOUT ROWID tables */ int mxCol; /* Maximum non-virtual column number */ - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( !IsOrdinaryTable(pTab) ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; @@ -142719,8 +146635,20 @@ SQLITE_PRIVATE void sqlite3Pragma( pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ /* Verify that an index entry exists for the current table row */ - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); + jmp2 = sqlite3VdbeAddOp3(v, OP_IFindKey, iIdxCur+j, ckUniq, r1); + VdbeCoverage(v); + sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "index %s stores an imprecise floating-point " + "value for row ", pIdx->zName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + integrityCheckResultRow(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ckUniq); + + sqlite3VdbeJumpHere(v, jmp2); sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); @@ -142728,7 +146656,7 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp4 = integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeResolveLabel(v, ckUniq); /* The OP_IdxRowid opcode is an optimized version of OP_Column ** that extracts the rowid off the end of the index record. @@ -142808,7 +146736,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab = sqliteHashData(x); sqlite3_vtab *pVTab; int a1; - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( IsOrdinaryTable(pTab) ) continue; if( !IsVirtual(pTab) ) continue; if( pTab->nCol<=0 ){ @@ -143040,6 +146968,8 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_RESTART; }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ eMode = SQLITE_CHECKPOINT_TRUNCATE; + }else if( sqlite3StrICmp(zRight, "noop")==0 ){ + eMode = SQLITE_CHECKPOINT_NOOP; } } pParse->nMem = 3; @@ -143779,7 +147709,8 @@ static void corruptSchema( static const char *azAlterType[] = { "rename", "drop column", - "add column" + "add column", + "drop constraint" }; *pData->pzErrMsg = sqlite3MPrintf(db, "error in %s %s after %s: %s", azObj[0], azObj[1], @@ -144606,9 +148537,11 @@ static int sqlite3LockAndPrepare( rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); assert( rc==SQLITE_OK || *ppStmt==0 ); if( rc==SQLITE_OK || db->mallocFailed ) break; - }while( (rc==SQLITE_ERROR_RETRY && (cnt++)<SQLITE_MAX_PREPARE_RETRY) - || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt++)==0) ); + cnt++; + }while( (rc==SQLITE_ERROR_RETRY && ALWAYS(cnt<=SQLITE_MAX_PREPARE_RETRY)) + || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt)==1) ); sqlite3BtreeLeaveAll(db); + assert( rc!=SQLITE_ERROR_RETRY ); rc = sqlite3ApiExit(db, rc); assert( (rc&db->errMask)==rc ); db->busyHandler.nBusy = 0; @@ -144860,7 +148793,7 @@ SQLITE_API int sqlite3_prepare16_v3( */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ + u8 isTnct; /* 0: Not distinct. 1: DISTINCT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -144990,8 +148923,6 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->iLimit = 0; pNew->iOffset = 0; pNew->selId = ++pParse->nSelect; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; @@ -145223,7 +149154,7 @@ static int tableAndColumnIndex( int iEnd, /* Last member of pSrc->a[] to check */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ - int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pSTab->aCol[] here */ int bIgnoreHidden /* Ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ @@ -145282,8 +149213,7 @@ SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->w.iJoin = iTable; - if( p->op==TK_FUNCTION ){ - assert( ExprUseXList(p) ); + if( ExprUseXList(p) ){ if( p->x.pList ){ int i; for(i=0; i<p->x.pList->nExpr; i++){ @@ -145499,6 +149429,11 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn); pRight->u3.pOn = 0; pRight->fg.isOn = 1; + p->selFlags |= SF_OnToWhere; + } + + if( IsVirtual(pRightTab) && joinType==EP_OuterON && pRight->u1.pFuncArg ){ + p->selFlags |= SF_OnToWhere; } } return 0; @@ -146139,29 +150074,6 @@ static void selectInnerLoop( } switch( eDest ){ - /* In this mode, write each query result to the key of the temporary - ** table iParm. - */ -#ifndef SQLITE_OMIT_COMPOUND_SELECT - case SRT_Union: { - int r1; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); - sqlite3ReleaseTempReg(pParse, r1); - break; - } - - /* Construct a record from the query result, but instead of - ** saving that record, use it as a key to delete elements from - ** the temporary table iParm. - */ - case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); - break; - } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - /* Store the result as data using a unique key. */ case SRT_Fifo: @@ -146284,9 +150196,14 @@ static void selectInnerLoop( assert( nResultCol<=pDest->nSdst ); pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm = regResult; }else{ assert( nResultCol==pDest->nSdst ); - assert( regResult==iParm ); + if( regResult!=iParm ){ + /* This occurs in cases where the SELECT had both a DISTINCT and + ** an OFFSET clause. */ + sqlite3VdbeAddOp3(v, OP_Copy, regResult, iParm, nResultCol-1); + } /* The LIMIT clause will jump out of the loop for us */ } break; @@ -146385,7 +150302,10 @@ static void selectInnerLoop( */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ int nExtra = (N+X)*(sizeof(CollSeq*)+1); - KeyInfo *p = sqlite3DbMallocRawNN(db, SZ_KEYINFO(0) + nExtra); + KeyInfo *p; + assert( X>=0 ); + if( NEVER(N+X>0xffff) ) return (KeyInfo*)sqlite3OomFault(db); + p = sqlite3DbMallocRawNN(db, SZ_KEYINFO(0) + nExtra); if( p ){ p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; @@ -146952,6 +150872,10 @@ static void generateColumnTypes( #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(pTabList); + UNUSED_PARAMETER(pEList); #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } @@ -147262,8 +151186,8 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( } } if( zType ){ - const i64 k = sqlite3Strlen30(zType); - n = sqlite3Strlen30(pCol->zCnName); + const i64 k = strlen(zType); + n = strlen(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ @@ -147436,9 +151360,9 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ ** function is responsible for ensuring that this structure is eventually ** freed. */ -static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ +static KeyInfo *multiSelectByMergeKeyInfo(Parse *pParse, Select *p, int nExtra){ ExprList *pOrderBy = p->pOrderBy; - int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0; + int nOrderBy = (pOrderBy!=0) ? pOrderBy->nExpr : 0; sqlite3 *db = pParse->db; KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); if( pRet ){ @@ -147571,7 +151495,7 @@ static void generateWithRecursiveQuery( regCurrent = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); if( pOrderBy ){ - KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + KeyInfo *pKeyInfo = multiSelectByMergeKeyInfo(pParse, p, 1); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); destQueue.pOrderBy = pOrderBy; @@ -147580,8 +151504,28 @@ static void generateWithRecursiveQuery( } VdbeComment((v, "Queue table")); if( iDistinct ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); - p->selFlags |= SF_UsesEphemeral; + /* Generate an ephemeral table used to enforce distinctness on the + ** output of the recursive part of the CTE. + */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + + assert( p->pNext==0 ); + assert( p->pEList!=0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nCol, 1); + if( pKeyInfo ){ + for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ + *apColl = multiSelectCollSeq(pParse, p, i); + if( 0==*apColl ){ + *apColl = pParse->db->pDfltColl; + } + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iDistinct, nCol, 0, + (void*)pKeyInfo, P4_KEYINFO); + }else{ + assert( pParse->nErr>0 ); + } } /* Detach the ORDER BY clause from the compound SELECT */ @@ -147656,7 +151600,7 @@ end_of_recursive_query: #endif /* SQLITE_OMIT_CTE */ /* Forward references */ -static int multiSelectOrderBy( +static int multiSelectByMerge( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -147805,12 +151749,26 @@ static int multiSelect( generateWithRecursiveQuery(pParse, p, &dest); }else #endif - - /* Compound SELECTs that have an ORDER BY clause are handled separately. - */ if( p->pOrderBy ){ - return multiSelectOrderBy(pParse, p, pDest); + /* If the compound has an ORDER BY clause, then always use the merge + ** algorithm. */ + return multiSelectByMerge(pParse, p, pDest); + }else if( p->op!=TK_ALL ){ + /* If the compound is EXCEPT, INTERSECT, or UNION (anything other than + ** UNION ALL) then also always use the merge algorithm. However, the + ** multiSelectByMerge() routine requires that the compound have an + ** ORDER BY clause, and it doesn't right now. So invent one first. */ + Expr *pOne = sqlite3ExprInt32(db, 1); + p->pOrderBy = sqlite3ExprListAppend(pParse, 0, pOne); + if( pParse->nErr ) goto multi_select_end; + assert( p->pOrderBy!=0 ); + p->pOrderBy->a[0].u.x.iOrderByCol = 1; + return multiSelectByMerge(pParse, p, pDest); }else{ + /* For a UNION ALL compound without ORDER BY, simply run the left + ** query, then run the right query */ + int addr = 0; + int nLimit = 0; /* Initialize to suppress harmless compiler warning */ #ifndef SQLITE_OMIT_EXPLAIN if( pPrior->pPrior==0 ){ @@ -147818,277 +151776,49 @@ static int multiSelect( ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); } #endif - - /* Generate code for the left and right SELECT statements. - */ - switch( p->op ){ - case TK_ALL: { - int addr = 0; - int nLimit = 0; /* Initialize to suppress harmless compiler warning */ - assert( !pPrior->pLimit ); - pPrior->iLimit = p->iLimit; - pPrior->iOffset = p->iOffset; - pPrior->pLimit = p->pLimit; - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); - rc = sqlite3Select(pParse, pPrior, &dest); - pPrior->pLimit = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); - VdbeComment((v, "Jump ahead if LIMIT reached")); - if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_OffsetLimit, - p->iLimit, p->iOffset+1, p->iOffset); - } - } - ExplainQueryPlan((pParse, 1, "UNION ALL")); - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); - rc = sqlite3Select(pParse, p, &dest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) - ){ - p->nSelectRow = sqlite3LogEst((u64)nLimit); - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); - } - break; - } - case TK_EXCEPT: - case TK_UNION: { - int unionTab; /* Cursor number of the temp table holding result */ - u8 op = 0; /* One of the SRT_ operations to apply to self */ - int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit; /* Saved values of p->nLimit */ - int addr; - SelectDest uniondest; - - testcase( p->op==TK_EXCEPT ); - testcase( p->op==TK_UNION ); - priorOp = SRT_Union; - if( dest.eDest==priorOp ){ - /* We can reuse a temporary table generated by a SELECT to our - ** right. - */ - assert( p->pLimit==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iSDParm; - }else{ - /* We will need to create our own temporary table to hold the - ** intermediate results. - */ - unionTab = pParse->nTab++; - assert( p->pOrderBy==0 ); - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - } - - - /* Code the SELECT statements to our left - */ - assert( !pPrior->pOrderBy ); - sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); - rc = sqlite3Select(pParse, pPrior, &uniondest); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT statement - */ - if( p->op==TK_EXCEPT ){ - op = SRT_Except; - }else{ - assert( p->op==TK_UNION ); - op = SRT_Union; - } - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - uniondest.eDest = op; - ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", - sqlite3SelectOpName(p->op))); - TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); - rc = sqlite3Select(pParse, p, &uniondest); - testcase( rc!=SQLITE_OK ); - assert( p->pOrderBy==0 ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->pOrderBy = 0; - if( p->op==TK_UNION ){ - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - } - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->iLimit = 0; - p->iOffset = 0; - - /* Convert the data in the temporary table into whatever form - ** it is that we currently need. - */ - assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); - assert( p->pEList || db->mallocFailed ); - if( dest.eDest!=priorOp && db->mallocFailed==0 ){ - int iCont, iBreak, iStart; - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); - iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, unionTab, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); - } - break; - } - default: assert( p->op==TK_INTERSECT ); { - int tab1, tab2; - int iCont, iBreak, iStart; - Expr *pLimit; - int addr; - SelectDest intersectdest; - int r1; - - /* INTERSECT is different from the others since it requires - ** two temporary tables. Hence it has its own case. Begin - ** by allocating the tables we will need. - */ - tab1 = pParse->nTab++; - tab2 = pParse->nTab++; - assert( p->pOrderBy==0 ); - - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - - /* Code the SELECTs to our left into temporary table "tab1". - */ - sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); - rc = sqlite3Select(pParse, pPrior, &intersectdest); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT into temporary table "tab2" - */ - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); - assert( p->addrOpenEphm[1] == -1 ); - p->addrOpenEphm[1] = addr; - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - intersectdest.iSDParm = tab2; - ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", - sqlite3SelectOpName(p->op))); - TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); - rc = sqlite3Select(pParse, p, &intersectdest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - if( p->nSelectRow>pPrior->nSelectRow ){ - p->nSelectRow = pPrior->nSelectRow; - } - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - - /* Generate code to take the intersection of the two temporary - ** tables. - */ - if( rc ) break; - assert( p->pEList ); - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); - r1 = sqlite3GetTempReg(pParse); - iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); - VdbeCoverage(v); - sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, tab1, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); - sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); - break; - } - } - - #ifndef SQLITE_OMIT_EXPLAIN - if( p->pNext==0 ){ - ExplainQueryPlanPop(pParse); - } - #endif - } - if( pParse->nErr ) goto multi_select_end; - - /* Compute collating sequences used by - ** temporary tables needed to implement the compound select. - ** Attach the KeyInfo structure to all temporary tables. - ** - ** This section is run by the right-most SELECT statement only. - ** SELECT statements to the left always skip this part. The right-most - ** SELECT might also skip this part if it has no ORDER BY clause and - ** no temp tables are required. - */ - if( p->selFlags & SF_UsesEphemeral ){ - int i; /* Loop counter */ - KeyInfo *pKeyInfo; /* Collating sequence for the result set */ - Select *pLoop; /* For looping through SELECT statements */ - CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ - int nCol; /* Number of columns in result set */ - - assert( p->pNext==0 ); - assert( p->pEList!=0 ); - nCol = p->pEList->nExpr; - pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); - if( !pKeyInfo ){ - rc = SQLITE_NOMEM_BKPT; + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = sqlite3ExprDup(db, p->pLimit, 0); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); + rc = sqlite3Select(pParse, pPrior, &dest); + sqlite3ExprDelete(db, pPrior->pLimit); + pPrior->pLimit = 0; + if( rc ){ goto multi_select_end; } - for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ - *apColl = multiSelectCollSeq(pParse, p, i); - if( 0==*apColl ){ - *apColl = db->pDfltColl; + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); } } - - for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ - for(i=0; i<2; i++){ - int addr = pLoop->addrOpenEphm[i]; - if( addr<0 ){ - /* If [0] is unused then [1] is also unused. So we can - ** always safely abort as soon as the first unused slot is found */ - assert( pLoop->addrOpenEphm[1]<0 ); - break; - } - sqlite3VdbeChangeP2(v, addr, nCol); - sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), - P4_KEYINFO); - pLoop->addrOpenEphm[i] = -1; - } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } +#ifndef SQLITE_OMIT_EXPLAIN + if( p->pNext==0 ){ + ExplainQueryPlanPop(pParse); } - sqlite3KeyInfoUnref(pKeyInfo); +#endif } multi_select_end: @@ -148120,8 +151850,8 @@ SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ ** Code an output subroutine for a coroutine implementation of a ** SELECT statement. ** -** The data to be output is contained in pIn->iSdst. There are -** pIn->nSdst columns to be output. pDest is where the output should +** The data to be output is contained in an array of pIn->nSdst registers +** starting at register pIn->iSdst. pDest is where the output should ** be sent. ** ** regReturn is the number of the register holding the subroutine @@ -148150,6 +151880,8 @@ static int generateOutputSubroutine( int iContinue; int addr; + assert( pIn->eDest==SRT_Coroutine ); + addr = sqlite3VdbeCurrentAddr(v); iContinue = sqlite3VdbeMakeLabel(pParse); @@ -148171,23 +151903,60 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); - assert( pDest->eDest!=SRT_Exists ); - assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ + case SRT_Fifo: + case SRT_DistFifo: + case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); + int iParm = pDest->iSDParm; + testcase( pDest->eDest==SRT_Table ); + testcase( pDest->eDest==SRT_EphemTab ); + testcase( pDest->eDest==SRT_Fifo ); + testcase( pDest->eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); - sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( pDest->eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif +#ifndef SQLITE_OMIT_CTE + if( pDest->eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index that is used to enforce uniqueness on the + ** total result. At this point, we are processing the setup portion + ** of the recursive CTE using the merge algorithm, so the results are + ** guaranteed to be unique anyhow. But we still need to populate the + ** (iParm+1) cursor for use by the subsequent recursive phase. + */ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1, + pIn->iSdst, pIn->nSdst); + } +#endif + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); sqlite3ReleaseTempReg(pParse, r1); break; } + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + #ifndef SQLITE_OMIT_SUBQUERY /* If we are creating a set for an "expr IN (SELECT ...)". */ @@ -148234,9 +152003,51 @@ static int generateOutputSubroutine( break; } +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3, ii; + ExprList *pSO; + int iParm = pDest->iSDParm; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r3); + if( pDest->eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + } + for(ii=0; ii<nKey; ii++){ + sqlite3VdbeAddOp2(v, OP_SCopy, + pIn->iSdst + pSO->a[ii].u.x.iOrderByCol - 1, + r2+ii); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + /* Ignore the output */ + case SRT_Discard: { + break; + } + /* If none of the above, then the result destination must be - ** SRT_Output. This routine is never called with any other - ** destination other than the ones handled above or SRT_Output. + ** SRT_Output. ** ** For SRT_Output, results are stored in a sequence of registers. ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to @@ -148264,8 +152075,9 @@ static int generateOutputSubroutine( } /* -** Alternative compound select code generator for cases when there -** is an ORDER BY clause. +** Generate code for a compound SELECT statement using a merge +** algorithm. The compound must have an ORDER BY clause for this +** to work. ** ** We assume a query of the following form: ** @@ -148282,7 +152094,7 @@ static int generateOutputSubroutine( ** ** outB: Move the output of the selectB coroutine into the output ** of the compound query. (Only generated for UNION and -** UNION ALL. EXCEPT and INSERTSECT never output a row that +** UNION ALL. EXCEPT and INTERSECT never output a row that ** appears only in B.) ** ** AltB: Called when there is data from both coroutines and A<B. @@ -148335,10 +152147,8 @@ static int generateOutputSubroutine( ** AeqB: ... ** AgtB: ... ** Init: initialize coroutine registers -** yield coA -** if eof(A) goto EofA -** yield coB -** if eof(B) goto EofB +** yield coA, on eof goto EofA +** yield coB, on eof goto EofB ** Cmpr: Compare A, B ** Jump AltB, AeqB, AgtB ** End: ... @@ -148346,10 +152156,10 @@ static int generateOutputSubroutine( ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop ** until all data is exhausted then jump to the "end" label. AltB, AeqB, -** and AgtB jump to either L2 or to one of EofA or EofB. +** and AgtB jump to either Cmpr or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT -static int multiSelectOrderBy( +static int multiSelectByMerge( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -148421,10 +152231,8 @@ static int multiSelectOrderBy( if( pItem->u.x.iOrderByCol==i ) break; } if( j==nOrderBy ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pNew = sqlite3ExprInt32(db, i); if( pNew==0 ) return SQLITE_NOMEM_BKPT; - pNew->flags |= EP_IntValue; - pNew->u.iValue = i; p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; } @@ -148432,26 +152240,29 @@ static int multiSelectOrderBy( } /* Compute the comparison permutation and keyinfo that is used with - ** the permutation used to determine if the next - ** row of results comes from selectA or selectB. Also add explicit - ** collations to the ORDER BY clause terms so that when the subqueries - ** to the right and the left are evaluated, they use the correct - ** collation. + ** the permutation to determine if the next row of results comes + ** from selectA or selectB. Also add literal collations to the + ** ORDER BY clause terms so that when selectA and selectB are + ** evaluated, they use the correct collation. */ aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; + int bKeep = 0; aPermute[0] = nOrderBy; for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ assert( pItem!=0 ); assert( pItem->u.x.iOrderByCol>0 ); assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; + if( aPermute[i]!=(u32)i-1 ) bKeep = 1; + } + if( bKeep==0 ){ + sqlite3DbFreeNN(db, aPermute); + aPermute = 0; } - pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); - }else{ - pKeyMerge = 0; } + pKeyMerge = multiSelectByMergeKeyInfo(pParse, p, 1); /* Allocate a range of temporary registers and the KeyInfo needed ** for the logic that removes duplicate result rows when the @@ -148530,7 +152341,7 @@ static int multiSelectOrderBy( */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); - VdbeComment((v, "left SELECT")); + VdbeComment((v, "SUBR: next-A")); pPrior->iLimit = regLimitA; ExplainQueryPlan((pParse, 1, "LEFT")); sqlite3Select(pParse, pPrior, &destA); @@ -148542,7 +152353,7 @@ static int multiSelectOrderBy( */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); - VdbeComment((v, "right SELECT")); + VdbeComment((v, "SUBR: next-B")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; @@ -148556,7 +152367,7 @@ static int multiSelectOrderBy( /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. */ - VdbeNoopComment((v, "Output routine for A")); + VdbeNoopComment((v, "SUBR: out-A")); addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, regPrev, pKeyDup, labelEnd); @@ -148565,7 +152376,7 @@ static int multiSelectOrderBy( ** select as the next output row of the compound select. */ if( op==TK_ALL || op==TK_UNION ){ - VdbeNoopComment((v, "Output routine for B")); + VdbeNoopComment((v, "SUBR: out-B")); addrOutB = generateOutputSubroutine(pParse, p, &destB, pDest, regOutB, regPrev, pKeyDup, labelEnd); @@ -148578,10 +152389,12 @@ static int multiSelectOrderBy( if( op==TK_EXCEPT || op==TK_INTERSECT ){ addrEofA_noB = addrEofA = labelEnd; }else{ - VdbeNoopComment((v, "eof-A subroutine")); + VdbeNoopComment((v, "SUBR: eof-A")); addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + VdbeComment((v, "out-B")); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); + VdbeComment((v, "next-B")); sqlite3VdbeGoto(v, addrEofA); p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } @@ -148593,17 +152406,20 @@ static int multiSelectOrderBy( addrEofB = addrEofA; if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ - VdbeNoopComment((v, "eof-B subroutine")); + VdbeNoopComment((v, "SUBR: eof-B")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); + VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of A<B */ - VdbeNoopComment((v, "A-lt-B subroutine")); addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); + VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, labelCmpr); /* Generate code to handle the case of A==B @@ -148614,36 +152430,48 @@ static int multiSelectOrderBy( addrAeqB = addrAltB; addrAltB++; }else{ - VdbeNoopComment((v, "A-eq-B subroutine")); - addrAeqB = - sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); - sqlite3VdbeGoto(v, labelCmpr); + addrAeqB = addrAltB + 1; } /* Generate code to handle the case of A>B */ - VdbeNoopComment((v, "A-gt-B subroutine")); addrAgtB = sqlite3VdbeCurrentAddr(v); if( op==TK_ALL || op==TK_UNION ){ sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + VdbeComment((v, "out-B")); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + VdbeComment((v, "next-B")); + sqlite3VdbeGoto(v, labelCmpr); + }else{ + addrAgtB++; /* Just do next-B. Might as well use the next-B call + ** in the next code block */ } - sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + VdbeComment((v, "next-A")); + /* v--- Also the A>B case for EXCEPT and INTERSECT */ sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + VdbeComment((v, "next-B")); /* Implement the main merge loop */ + if( aPermute!=0 ){ + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + } sqlite3VdbeResolveLabel(v, labelCmpr); - sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO); - sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); + if( aPermute!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + } + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); + VdbeCoverageIf(v, op==TK_ALL); + VdbeCoverageIf(v, op==TK_UNION); + VdbeCoverageIf(v, op==TK_EXCEPT); + VdbeCoverageIf(v, op==TK_INTERSECT); /* Jump to the this point in order to terminate the query. */ @@ -148677,7 +152505,7 @@ static int multiSelectOrderBy( ** ## About "isOuterJoin": ** ** The isOuterJoin column indicates that the replacement will occur into a -** position in the parent that NULL-able due to an OUTER JOIN. Either the +** position in the parent that is NULL-able due to an OUTER JOIN. Either the ** target slot in the parent is the right operand of a LEFT JOIN, or one of ** the left operands of a RIGHT JOIN. In either case, we need to potentially ** bypass the substituted expression with OP_IfNullRow. @@ -148707,6 +152535,7 @@ typedef struct SubstContext { int iTable; /* Replace references to this table */ int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + int nSelDepth; /* Depth of sub-query recursion. Top==1 */ ExprList *pEList; /* Replacement expressions */ ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; @@ -148814,6 +152643,9 @@ static Expr *substExpr( if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ pExpr->iTable = pSubst->iNewTable; } + if( pExpr->op==TK_AGG_FUNCTION && pExpr->op2>=pSubst->nSelDepth ){ + pExpr->op2--; + } pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); pExpr->pRight = substExpr(pSubst, pExpr->pRight); if( ExprUseXSelect(pExpr) ){ @@ -148851,6 +152683,7 @@ static void substSelect( SrcItem *pItem; int i; if( !p ) return; + pSubst->nSelDepth++; do{ substExprList(pSubst, p->pEList); substExprList(pSubst, p->pGroupBy); @@ -148868,6 +152701,7 @@ static void substSelect( } } }while( doPrior && (p = p->pPrior)!=0 ); + pSubst->nSelDepth--; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -149479,7 +153313,7 @@ static int flattenSubquery( ** complete, since there may still exist Expr.pTab entries that ** refer to the subquery even after flattening. Ticket #3346. ** - ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + ** pSubitem->pSTab is always non-NULL by test restrictions and tests above. */ if( ALWAYS(pSubitem->pSTab!=0) ){ Table *pTabToDel = pSubitem->pSTab; @@ -149509,17 +153343,12 @@ static int flattenSubquery( pSub = pSub1; for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; - u8 jointype = 0; - u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ; + u8 jointype = pSubitem->fg.jointype; assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ pSrc = pParent->pSrc; /* FROM clause of the outer query */ - if( pParent==p ){ - jointype = pSubitem->fg.jointype; /* First time through the loop */ - } - /* The subquery uses a single slot of the FROM clause of the outer ** query. If the subquery has more than one element in its FROM clause, ** then expand the outer query to make space for it to hold all elements @@ -149539,6 +153368,7 @@ static int flattenSubquery( pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); if( pSrc==0 ) break; pParent->pSrc = pSrc; + pSubitem = &pSrc->a[iFrom]; } /* Transfer the FROM clause terms from the subquery into the @@ -149553,13 +153383,12 @@ static int flattenSubquery( || pItem->u4.zDatabase==0 ); if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); *pItem = pSubSrc->a[i]; - pItem->fg.jointype |= ltorj; + pItem->fg.jointype |= (jointype & JT_LTORJ); memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].fg.jointype &= JT_LTORJ; - pSrc->a[iFrom].fg.jointype |= jointype | ltorj; + pSubitem->fg.jointype |= jointype; - /* Now begin substituting subquery result set expressions for + /* Begin substituting subquery result set expressions for ** references to the iParent in the outer query. ** ** Example: @@ -149571,7 +153400,7 @@ static int flattenSubquery( ** We look at every expression in the outer query and every place we see ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". */ - if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ + if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values @@ -149579,9 +153408,9 @@ static int flattenSubquery( ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this - ** function attempts to flatten a compound sub-query into pParent - ** (the only way this can happen is if the compound sub-query is - ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ** function attempts to flatten a compound sub-query into pParent. + ** See ticket [d11a6e908f]. + */ ExprList *pOrderBy = pSub->pOrderBy; for(i=0; i<pOrderBy->nExpr; i++){ pOrderBy->a[i].u.x.iOrderByCol = 0; @@ -149609,6 +153438,7 @@ static int flattenSubquery( x.iTable = iParent; x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; + x.nSelDepth = 0; x.pEList = pSub->pEList; x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); @@ -150194,9 +154024,20 @@ static int pushDownWhereTerms( x.iTable = pSrc->iCursor; x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; + x.nSelDepth = 0; x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); + assert( pNew!=0 || pParse->nErr!=0 ); + if( pParse->nErr==0 && pNew->op==TK_IN && ExprUseXSelect(pNew) ){ + assert( pNew->x.pSelect!=0 ); + pNew->x.pSelect->selFlags |= SF_ClonedRhsIn; + assert( pWhere!=0 ); + assert( pWhere->op==TK_IN ); + assert( ExprUseXSelect(pWhere) ); + assert( pWhere->x.pSelect!=0 ); + pWhere->x.pSelect->selFlags |= SF_ClonedRhsIn; + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ /* Restriction 6c has prevented push-down in this case */ @@ -150431,14 +154272,14 @@ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ ** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) ** ORDER BY ... COLLATE ... ** -** This transformation is necessary because the multiSelectOrderBy() routine +** This transformation is necessary because the multiSelectByMerge() routine ** above that generates the code for a compound SELECT with an ORDER BY clause ** uses a merge algorithm that requires the same collating sequence on the ** result columns as on the ORDER BY clause. See ticket ** http://sqlite.org/src/info/6709574d2a ** ** This transformation is only needed for EXCEPT, INTERSECT, and UNION. -** The UNION ALL operator works fine with multiSelectOrderBy() even when +** The UNION ALL operator works fine with multiSelectByMerge() even when ** there are COLLATE terms in the ORDER BY. */ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ @@ -150591,7 +154432,7 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ ** CTE expression, through routine checks to see if the reference is ** a recursive reference to the CTE. ** -** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** If pFrom matches a CTE according to either of these two above, pFrom->pSTab ** and other fields are populated accordingly. ** ** Return 0 if no match is found. @@ -150984,7 +154825,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) - && pFrom->fg.fromDDL + && (pFrom->fg.fromDDL || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) && ALWAYS(pTab->u.vtab.p!=0) && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -151629,6 +155470,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pFunc->bOBPayload ){ /* extra columns for the function arguments */ assert( ExprUseXList(pFunc->pFExpr) ); + assert( pFunc->pFExpr->x.pList!=0 ); nExtra += pFunc->pFExpr->x.pList->nExpr; } if( pFunc->bUseSubtype ){ @@ -151936,7 +155778,7 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ && pExpr->pAggInfo==0 ){ sqlite3 *db = pWalker->pParse->db; - Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); + Expr *pNew = sqlite3ExprInt32(db, 1); if( pNew ){ Expr *pWhere = pS->pWhere; SWAP(Expr, *pNew, *pExpr); @@ -152219,6 +156061,246 @@ static int fromClauseTermCanBeCoroutine( } /* +** Argument pWhere is the WHERE clause belonging to SELECT statement p. This +** function attempts to transform expressions of the form: +** +** EXISTS (SELECT ...) +** +** into joins. For example, given +** +** CREATE TABLE sailors(sid INTEGER PRIMARY KEY, name TEXT); +** CREATE TABLE reserves(sid INT, day DATE, PRIMARY KEY(sid, day)); +** +** SELECT name FROM sailors AS S WHERE EXISTS ( +** SELECT * FROM reserves AS R WHERE S.sid = R.sid AND R.day = '2022-10-25' +** ); +** +** the SELECT statement may be transformed as follows: +** +** SELECT name FROM sailors AS S, reserves AS R +** WHERE S.sid = R.sid AND R.day = '2022-10-25'; +** +** **Approximately**. Really, we have to ensure that the FROM-clause term +** that was formerly inside the EXISTS is only executed once. This is handled +** by setting the SrcItem.fg.fromExists flag, which then causes code in +** the where.c file to exit the corresponding loop after the first successful +** match (if any). +*/ +static SQLITE_NOINLINE void existsToJoin( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement being optimized */ + Expr *pWhere /* part of the WHERE clause currently being examined */ +){ + if( pParse->nErr==0 + && pWhere!=0 + && !ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) + && ALWAYS(p->pSrc!=0) + && p->pSrc->nSrc<BMS + && (p->pLimit==0 || p->pLimit->pRight==0) + ){ + if( pWhere->op==TK_AND ){ + Expr *pRight = pWhere->pRight; + existsToJoin(pParse, p, pWhere->pLeft); + existsToJoin(pParse, p, pRight); + } + else if( pWhere->op==TK_EXISTS ){ + Select *pSub = pWhere->x.pSelect; + Expr *pSubWhere = pSub->pWhere; + if( pSub->pSrc->nSrc==1 + && (pSub->selFlags & SF_Aggregate)==0 + && !pSub->pSrc->a[0].fg.isSubquery + && pSub->pLimit==0 + && pSub->pPrior==0 + ){ + /* Before combining the sub-select with the parent, renumber the + ** cursor used by the subselect. This is because the EXISTS expression + ** might be a copy of another EXISTS expression from somewhere + ** else in the tree, and in this case it is important that it use + ** a unique cursor number. */ + sqlite3 *db = pParse->db; + int *aCsrMap = sqlite3DbMallocZero(db, (pParse->nTab+2)*sizeof(int)); + if( aCsrMap==0 ) return; + aCsrMap[0] = (pParse->nTab+1); + renumberCursors(pParse, pSub, -1, aCsrMap); + sqlite3DbFree(db, aCsrMap); + + memset(pWhere, 0, sizeof(*pWhere)); + pWhere->op = TK_INTEGER; + pWhere->u.iValue = 1; + ExprSetProperty(pWhere, EP_IntValue); + assert( p->pWhere!=0 ); + pSub->pSrc->a[0].fg.fromExists = 1; + p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); + if( pSubWhere ){ + p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); + pSub->pWhere = 0; + } + pSub->pSrc = 0; + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSub); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + TREETRACE(0x100000,pParse,p, + ("After EXISTS-to-JOIN optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + } + } +} + +/* +** Type used for Walker callbacks by selectCheckOnClauses(). +*/ +typedef struct CheckOnCtx CheckOnCtx; +struct CheckOnCtx { + SrcList *pSrc; /* SrcList for this context */ + int iJoin; /* Cursor numbers must be =< than this */ + int bFuncArg; /* True for table-function arg */ + CheckOnCtx *pParent; /* Parent context */ +}; + +/* +** True if the SrcList passed as the only argument contains at least +** one RIGHT or FULL JOIN. False otherwise. +*/ +#define hasRightJoin(pSrc) (((pSrc)->a[0].fg.jointype & JT_LTORJ)!=0) + +/* +** The xExpr callback for the search of invalid ON clause terms. +*/ +static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ + CheckOnCtx *pCtx = pWalker->u.pCheckOnCtx; + + /* Check if pExpr is root or near-root of an ON clause constraint that needs + ** to be checked to ensure that it does not refer to tables in its FROM + ** clause to the right of itself. i.e. it is either: + ** + ** + an ON clause on an OUTER join, or + ** + an ON clause on an INNER join within a FROM that features at + ** least one RIGHT or FULL join. + */ + if( (ExprHasProperty(pExpr, EP_OuterON)) + || (ExprHasProperty(pExpr, EP_InnerON) && hasRightJoin(pCtx->pSrc)) + ){ + /* If CheckOnCtx.iJoin is already set, then fall through and process + ** this expression node as normal. Or, if CheckOnCtx.iJoin is still 0, + ** set it to the cursor number of the RHS of the join to which this + ** ON expression was attached and then iterate through the entire + ** expression. */ + assert( pCtx->iJoin==0 || pCtx->iJoin==pExpr->w.iJoin ); + if( pCtx->iJoin==0 ){ + pCtx->iJoin = pExpr->w.iJoin; + sqlite3WalkExprNN(pWalker, pExpr); + pCtx->iJoin = 0; + return WRC_Prune; + } + } + + if( pExpr->op==TK_COLUMN ){ + /* A column expression. Find the SrcList (if any) to which it refers. + ** Then, if CheckOnCtx.iJoin indicates that this expression is part of an + ** ON clause from that SrcList (i.e. if iJoin is non-zero), check that it + ** does not refer to a table to the right of CheckOnCtx.iJoin. */ + do { + SrcList *pSrc = pCtx->pSrc; + int iTab = pExpr->iTable; + if( iTab>=pSrc->a[0].iCursor && iTab<=pSrc->a[pSrc->nSrc-1].iCursor ){ + if( pCtx->iJoin && iTab>pCtx->iJoin ){ + sqlite3ErrorMsg(pWalker->pParse, + "%s references tables to its right", + (pCtx->bFuncArg ? "table-function argument" : "ON clause") + ); + return WRC_Abort; + } + break; + } + pCtx = pCtx->pParent; + }while( pCtx ); + } + return WRC_Continue; +} + +/* +** The xSelect callback for the search of invalid ON clause terms. +*/ +static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ + CheckOnCtx *pCtx = pWalker->u.pCheckOnCtx; + if( pSelect->pSrc==pCtx->pSrc || pSelect->pSrc->nSrc==0 ){ + return WRC_Continue; + }else{ + CheckOnCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pSrc = pSelect->pSrc; + sCtx.pParent = pCtx; + pWalker->u.pCheckOnCtx = &sCtx; + sqlite3WalkSelect(pWalker, pSelect); + pWalker->u.pCheckOnCtx = pCtx; + pSelect->selFlags &= ~SF_OnToWhere; + return WRC_Prune; + } +} + +/* +** Check all ON clauses in pSelect to verify that they do not reference +** columns to the right. +*/ +SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ + Walker w; + CheckOnCtx sCtx; + int ii; + assert( pSelect->selFlags & SF_OnToWhere ); + assert( pSelect->pSrc!=0 && pSelect->pSrc->nSrc>=2 ); + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = selectCheckOnClausesExpr; + w.xSelectCallback = selectCheckOnClausesSelect; + w.u.pCheckOnCtx = &sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pSrc = pSelect->pSrc; + sqlite3WalkExpr(&w, pSelect->pWhere); + pSelect->selFlags &= ~SF_OnToWhere; + + /* Check for any table-function args that are attached to virtual tables + ** on the RHS of an outer join. They are subject to the same constraints + ** as ON clauses. */ + sCtx.bFuncArg = 1; + for(ii=0; ii<pSelect->pSrc->nSrc; ii++){ + SrcItem *pItem = &pSelect->pSrc->a[ii]; + if( pItem->fg.isTabFunc + && (pItem->fg.jointype & JT_OUTER) + ){ + sCtx.iJoin = pItem->iCursor; + sqlite3WalkExprList(&w, pItem->u1.pFuncArg); + } + } +} + +/* +** If p2 exists and p1 and p2 have the same number of terms, then change +** every term of p1 to have the same sort order as p2 and return true. +** +** If p2 is NULL or p1 and p2 are different lengths, then make no changes +** and return false. +** +** p1 must be non-NULL. +*/ +static int sqlite3CopySortOrder(ExprList *p1, ExprList *p2){ + assert( p1 ); + if( p2 && p1->nExpr==p2->nExpr ){ + int ii; + for(ii=0; ii<p1->nExpr; ii++){ + u8 sortFlags; + sortFlags = p2->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; + p1->a[ii].fg.sortFlags = sortFlags; + } + return 1; + }else{ + return 0; + } +} + +/* ** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. @@ -152313,8 +156395,7 @@ SQLITE_PRIVATE int sqlite3Select( assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableDistinct(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Discard || pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ @@ -152330,7 +156411,6 @@ SQLITE_PRIVATE int sqlite3Select( p->pOrderBy = 0; } p->selFlags &= ~(u32)SF_Distinct; - p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); if( pParse->nErr ){ @@ -152586,6 +156666,13 @@ SQLITE_PRIVATE int sqlite3Select( } #endif + /* If there may be an "EXISTS (SELECT ...)" in the WHERE clause, attempt + ** to change it into a join. */ + if( pParse->bHasExists && OptimizationEnabled(db,SQLITE_ExistsToJoin) ){ + existsToJoin(pParse, p, p->pWhere); + pTabList = p->pSrc; + } + /* Do the WHERE-clause constant propagation optimization if this is ** a join. No need to spend time on this operation for non-join queries ** as the equivalent optimization will be handled by query planner in @@ -152851,7 +156938,8 @@ SQLITE_PRIVATE int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && sqlite3CopySortOrder(pEList, sSort.pOrderBy) + && sqlite3ExprListCompare(pEList, sSort.pOrderBy, -1)==0 && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 @@ -153065,21 +157153,10 @@ SQLITE_PRIVATE int sqlite3Select( ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp ** variable. */ - if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ - int ii; - /* The GROUP BY processing doesn't care whether rows are delivered in - ** ASC or DESC order - only that each group is returned contiguously. - ** So set the ASC/DESC flags in the GROUP BY to match those in the - ** ORDER BY to maximize the chances of rows being delivered in an - ** order that makes the ORDER BY redundant. */ - for(ii=0; ii<pGroupBy->nExpr; ii++){ - u8 sortFlags; - sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; - pGroupBy->a[ii].fg.sortFlags = sortFlags; - } - if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ - orderByGrp = 1; - } + if( sqlite3CopySortOrder(pGroupBy, sSort.pOrderBy) + && sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 + ){ + orderByGrp = 1; } }else{ assert( 0==sqlite3LogEst(1) ); @@ -153373,12 +157450,12 @@ SQLITE_PRIVATE int sqlite3Select( ** for the next GROUP BY batch. */ sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output one row")); + VdbeComment((v, "output one row of %d", p->selId)); sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - VdbeComment((v, "reset accumulator")); + VdbeComment((v, "reset accumulator %d", p->selId)); /* Update the aggregate accumulators based on the content of ** the current row @@ -153386,7 +157463,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); - VdbeComment((v, "indicate data in accumulator")); + VdbeComment((v, "indicate data in accumulator %d", p->selId)); /* End of the loop */ @@ -153403,7 +157480,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Output the final row of result */ sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output final row")); + VdbeComment((v, "output final row of %d", p->selId)); /* Jump over the subroutines */ @@ -153424,7 +157501,7 @@ SQLITE_PRIVATE int sqlite3Select( addrOutputRow = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); - VdbeComment((v, "Groupby result generator entry point")); + VdbeComment((v, "Groupby result generator entry point %d", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, pAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); @@ -153432,14 +157509,14 @@ SQLITE_PRIVATE int sqlite3Select( &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - VdbeComment((v, "end groupby result generator")); + VdbeComment((v, "end groupby result generator %d", p->selId)); /* Generate a subroutine that will reset the group-by accumulator */ sqlite3VdbeResolveLabel(v, addrReset); resetAccumulator(pParse, pAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); + VdbeComment((v, "indicate accumulator %d empty", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regReset); if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ @@ -153889,7 +157966,7 @@ SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerS sqlite3SelectDelete(db, pTmp->pSelect); sqlite3IdListDelete(db, pTmp->pIdList); sqlite3UpsertDelete(db, pTmp->pUpsert); - sqlite3SrcListDelete(db, pTmp->pFrom); + sqlite3SrcListDelete(db, pTmp->pSrc); sqlite3DbFree(db, pTmp->zSpan); sqlite3DbFree(db, pTmp); @@ -154078,11 +158155,16 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( } } + /* NB: The SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES compile-time option is + ** experimental and unsupported. Do not use it unless understand the + ** implications and you cannot get by without this capability. */ +#if !defined(SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES) /* Experimental */ /* Do not create a trigger on a system table */ if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); goto trigger_cleanup; } +#endif /* INSTEAD of triggers are only for views and views only support INSTEAD ** of triggers. @@ -154194,6 +158276,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; zName = pTrig->zName; iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + assert( iDb>=00 && iDb<db->nDb ); pTrig->step_list = pStepList; while( pStepList ){ pStepList->pTrig = pTrig; @@ -154228,12 +158311,12 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( if( sqlite3ReadOnlyShadowTables(db) ){ TriggerStep *pStep; for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget!=0 - && sqlite3ShadowTableName(db, pStep->zTarget) + if( pStep->pSrc!=0 + && sqlite3ShadowTableName(db, pStep->pSrc->a[0].zName) ){ sqlite3ErrorMsg(pParse, "trigger \"%s\" may not write to shadow table \"%s\"", - pTrig->zName, pStep->zTarget); + pTrig->zName, pStep->pSrc->a[0].zName); goto triggerfinish_cleanup; } } @@ -154324,26 +158407,39 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep( static TriggerStep *triggerStepAllocate( Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ - Token *pName, /* The target name */ + SrcList *pTabList, /* Target table */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + Trigger *pNew = pParse->pNewTrigger; sqlite3 *db = pParse->db; - TriggerStep *pTriggerStep; + TriggerStep *pTriggerStep = 0; - if( pParse->nErr ) return 0; - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); - if( pTriggerStep ){ - char *z = (char*)&pTriggerStep[1]; - memcpy(z, pName->z, pName->n); - sqlite3Dequote(z); - pTriggerStep->zTarget = z; - pTriggerStep->op = op; - pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); - if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + if( pParse->nErr==0 ){ + if( pNew + && pNew->pSchema!=db->aDb[1].pSchema + && pTabList->a[0].u4.zDatabase + ){ + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); + }else{ + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); + if( pTriggerStep ){ + pTriggerStep->pSrc = sqlite3SrcListDup(db, pTabList, EXPRDUP_REDUCE); + pTriggerStep->op = op; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( pTriggerStep->pSrc && IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, + pTriggerStep->pSrc->a[0].zName, + pTabList->a[0].zName + ); + } + } } } + + sqlite3SrcListDelete(db, pTabList); return pTriggerStep; } @@ -154356,7 +158452,7 @@ static TriggerStep *triggerStepAllocate( */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( Parse *pParse, /* Parser */ - Token *pTableName, /* Name of the table into which we insert */ + SrcList *pTabList, /* Table to INSERT into */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf, /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ @@ -154369,7 +158465,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTabList, zStart, zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pSelect = pSelect; @@ -154401,7 +158497,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( Parse *pParse, /* Parser */ - Token *pTableName, /* Name of the table to be updated */ + SrcList *pTabList, /* Name of the table to be updated */ SrcList *pFrom, /* FROM clause for an UPDATE-FROM, or NULL */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ @@ -154412,21 +158508,36 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTabList, zStart, zEnd); if( pTriggerStep ){ + SrcList *pFromDup = 0; if( IN_RENAME_OBJECT ){ pTriggerStep->pExprList = pEList; pTriggerStep->pWhere = pWhere; - pTriggerStep->pFrom = pFrom; + pFromDup = pFrom; pEList = 0; pWhere = 0; pFrom = 0; }else{ pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pTriggerStep->pFrom = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); + pFromDup = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); } pTriggerStep->orconf = orconf; + + if( pFromDup && !IN_RENAME_OBJECT){ + Select *pSub; + Token as = {0, 0}; + pSub = sqlite3SelectNew(pParse, 0, pFromDup, 0,0,0,0, SF_NestedFrom, 0); + pFromDup = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &as, pSub ,0); + } + if( pFromDup && pTriggerStep->pSrc ){ + pTriggerStep->pSrc = sqlite3SrcListAppendList( + pParse, pTriggerStep->pSrc, pFromDup + ); + }else{ + sqlite3SrcListDelete(db, pFromDup); + } } sqlite3ExprListDelete(db, pEList); sqlite3ExprDelete(db, pWhere); @@ -154441,7 +158552,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( Parse *pParse, /* Parser */ - Token *pTableName, /* The table from which rows are deleted */ + SrcList *pTabList, /* The table from which rows are deleted */ Expr *pWhere, /* The WHERE clause */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ @@ -154449,7 +158560,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTabList, zStart, zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pWhere = pWhere; @@ -154649,6 +158760,7 @@ static SQLITE_NOINLINE Trigger *triggersReallyExist( p = pList; if( (pParse->db->flags & SQLITE_EnableTrigger)==0 && pTab->pTrigger!=0 + && sqlite3SchemaToIndex(pParse->db, pTab->pTrigger->pSchema)!=1 ){ /* The SQLITE_DBCONFIG_ENABLE_TRIGGER setting is off. That means that ** only TEMP triggers are allowed. Truncate the pList so that it @@ -154712,52 +158824,6 @@ SQLITE_PRIVATE Trigger *sqlite3TriggersExist( } /* -** Convert the pStep->zTarget string into a SrcList and return a pointer -** to that SrcList. -** -** This routine adds a specific database name, if needed, to the target when -** forming the SrcList. This prevents a trigger in one database from -** referring to a target in another database. An exception is when the -** trigger is in TEMP in which case it can refer to any other database it -** wants. -*/ -SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc( - Parse *pParse, /* The parsing context */ - TriggerStep *pStep /* The trigger containing the target token */ -){ - sqlite3 *db = pParse->db; - SrcList *pSrc; /* SrcList to be returned */ - char *zName = sqlite3DbStrDup(db, pStep->zTarget); - pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); - assert( pSrc==0 || pSrc->nSrc==1 ); - assert( zName || pSrc==0 ); - if( pSrc ){ - Schema *pSchema = pStep->pTrig->pSchema; - pSrc->a[0].zName = zName; - if( pSchema!=db->aDb[1].pSchema ){ - assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); - pSrc->a[0].u4.pSchema = pSchema; - pSrc->a[0].fg.fixedSchema = 1; - } - if( pStep->pFrom ){ - SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); - if( pDup && pDup->nSrc>1 && !IN_RENAME_OBJECT ){ - Select *pSubquery; - Token as; - pSubquery = sqlite3SelectNew(pParse,0,pDup,0,0,0,0,SF_NestedFrom,0); - as.n = 0; - as.z = 0; - pDup = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); - } - pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); - } - }else{ - sqlite3DbFree(db, zName); - } - return pSrc; -} - -/* ** Return true if the pExpr term from the RETURNING clause argument ** list is of the form "*". Raise an error if the terms if of the ** form "table.*". @@ -154903,7 +158969,10 @@ static void codeReturningTrigger( Returning *pReturning; Select sSelect; SrcList *pFrom; - u8 fromSpace[SZ_SRCLIST_1]; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; assert( v!=0 ); if( !pParse->bReturning ){ @@ -154919,8 +158988,8 @@ static void codeReturningTrigger( return; } memset(&sSelect, 0, sizeof(sSelect)); - pFrom = (SrcList*)fromSpace; - memset(pFrom, 0, SZ_SRCLIST_1); + memset(&uSrc, 0, sizeof(uSrc)); + pFrom = &uSrc.sSrc; sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); sSelect.pSrc = pFrom; pFrom->nSrc = 1; @@ -155019,7 +159088,7 @@ static int codeTriggerProgram( switch( pStep->op ){ case TK_UPDATE: { sqlite3Update(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3ExprDup(db, pStep->pWhere, 0), pParse->eOrconf, 0, 0, 0 @@ -155029,7 +159098,7 @@ static int codeTriggerProgram( } case TK_INSERT: { sqlite3Insert(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf, @@ -155040,7 +159109,7 @@ static int codeTriggerProgram( } case TK_DELETE: { sqlite3DeleteFrom(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 ); sqlite3VdbeAddOp0(v, OP_ResetCount); @@ -157327,7 +161396,8 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments + | SQLITE_AttachCreate | SQLITE_AttachWrite; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); @@ -157361,9 +161431,11 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; + nRes = sqlite3BtreeGetRequestedReserve(pMain); if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); i64 sz = 0; + const char *zFilename; if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ rc = SQLITE_ERROR; sqlite3SetString(pzErrMsg, db, "output file already exists"); @@ -157375,8 +161447,16 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( ** they are for the database being vacuumed, except that PAGER_CACHESPILL ** is always set. */ pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); + + /* If the VACUUM INTO target file is a URI filename and if the + ** "reserve=N" query parameter is present, reset the reserve to the + ** amount specified, if the amount is within range */ + zFilename = sqlite3BtreeGetFilename(pTemp); + if( ALWAYS(zFilename) ){ + int nNew = (int)sqlite3_uri_int64(zFilename, "reserve", nRes); + if( nNew>=0 && nNew<=255 ) nRes = nNew; + } } - nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); @@ -158832,9 +162912,12 @@ SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); addModuleArgument(pParse, pTab, 0); addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + db->nSchemaLock++; rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + db->nSchemaLock--; if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; sqlite3DbFree(db, zErr); sqlite3VtabEponymousTableClear(db, pMod); } @@ -159030,6 +163113,7 @@ struct WhereLevel { int iTabCur; /* The VDBE cursor used to access the table */ int iIdxCur; /* The VDBE cursor used to access pIdx */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* Abort the query due to empty table or similar */ int addrNxt; /* Jump here to start the next IN combination */ int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ @@ -159235,6 +163319,9 @@ struct WhereTerm { u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X <op> <expr>" */ +#ifdef SQLITE_DEBUG + int iTerm; /* Which WhereTerm is this, for debug purposes */ +#endif union { struct { int leftColumn; /* Column number of X in "X <op> <expr>" */ @@ -159727,7 +163814,6 @@ SQLITE_PRIVATE void sqlite3WhereAddExplainText( #endif { VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe, addr); - SrcItem *pItem = &pTabList->a[pLevel->iFrom]; sqlite3 *db = pParse->db; /* Database handle */ int isSearch; /* True for a SEARCH. False for SCAN. */ @@ -159750,7 +163836,10 @@ SQLITE_PRIVATE void sqlite3WhereAddExplainText( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); str.printfFlags = SQLITE_PRINTF_INTERNAL; - sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem); + sqlite3_str_appendf(&str, "%s %S%s", + isSearch ? "SEARCH" : "SCAN", + pItem, + pItem->fg.fromExists ? " EXISTS" : ""); if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ const char *zFmt = 0; Index *pIdx; @@ -160994,6 +165083,7 @@ static SQLITE_NOINLINE void filterPullDown( int addrNxt, /* Jump here to bypass inner loops */ Bitmask notReady /* Loops that are not ready */ ){ + int saved_addrBrk; while( ++iLevel < pWInfo->nLevel ){ WhereLevel *pLevel = &pWInfo->a[iLevel]; WhereLoop *pLoop = pLevel->pWLoop; @@ -161002,7 +165092,7 @@ static SQLITE_NOINLINE void filterPullDown( /* ,--- Because sqlite3ConstructBloomFilter() has will not have set ** vvvvv--' pLevel->regFilter if this were true. */ if( NEVER(pLoop->prereq & notReady) ) continue; - assert( pLevel->addrBrk==0 ); + saved_addrBrk = pLevel->addrBrk; pLevel->addrBrk = addrNxt; if( pLoop->wsFlags & WHERE_IPK ){ WhereTerm *pTerm = pLoop->aLTerm[0]; @@ -161032,7 +165122,7 @@ static SQLITE_NOINLINE void filterPullDown( VdbeCoverage(pParse->pVdbe); } pLevel->regFilter = 0; - pLevel->addrBrk = 0; + pLevel->addrBrk = saved_addrBrk; } } @@ -161079,7 +165169,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3 *db; /* Database connection */ SrcItem *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ - int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ @@ -161123,7 +165212,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** there are no IN operators in the constraints, the "addrNxt" label ** is the same as "addrBrk". */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrBrk = pLevel->addrNxt = pLevel->addrBrk; addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); /* If this is the right table of a LEFT OUTER JOIN, allocate and @@ -161139,14 +165228,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeComment((v, "init LEFT JOIN match flag")); } - /* Compute a safe address to jump to if we discover that the table for - ** this loop is empty and can never contribute content. */ - for(j=iLevel; j>0; j--){ - if( pWInfo->a[j].iLeftJoin ) break; - if( pWInfo->a[j].pRJ ) break; - } - addrHalt = pWInfo->a[j].addrBrk; - /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ int regYield; @@ -161180,7 +165261,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( SMASKBIT32(j) & pLoop->u.vtab.mHandleIn ){ int iTab = pParse->nTab++; int iCache = ++pParse->nMem; - sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab); + sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab, 0); sqlite3VdbeAddOp3(v, OP_VInitIn, iTab, iTarget, iCache); }else{ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); @@ -161385,7 +165466,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, pX->op==TK_GE); sqlite3ReleaseTempReg(pParse, rTemp); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -161425,36 +165506,36 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. + /* Case 4: Search using an index. ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" or "IS" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==", "IN", or "IS" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: + ** The z<10 term of the following cannot be used, only + ** the x=5 term: ** - ** x=5 AND z<10 + ** x=5 AND z<10 ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. */ static const u8 aStartOp[] = { 0, @@ -161904,7 +165985,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). */ - if( pWInfo->nLevel>1 ){ + if( pWInfo->nLevel>1 || pTabItem->fg.fromExists ){ int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; @@ -161917,6 +165998,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( for(k=1; k<=nNotReady; k++){ memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); } + + /* Clear the fromExists flag on the OR-optimized table entry so that + ** the calls to sqlite3WhereEnd() do not code early-exits after the + ** first row is visited. The early exit applies to this table's + ** overall loop - including the multiple OR branches and any WHERE + ** conditions not passed to the sub-loops - not to the sub-loops. */ + pOrTab->a[0].fg.fromExists = 0; }else{ pOrTab = pWInfo->pTabList; } @@ -162160,7 +166248,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->op==OP_Return ); pLevel->p2 = sqlite3VdbeCurrentAddr(v); - if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } + if( pWInfo->pTabList!=pOrTab ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -162180,7 +166268,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev],iCur,pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; @@ -162452,7 +166540,10 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( WhereLoop *pLoop = pLevel->pWLoop; SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; SrcList *pFrom; - u8 fromSpace[SZ_SRCLIST_1]; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; Bitmask mAll = 0; int k; @@ -162496,7 +166587,16 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } - pFrom = (SrcList*)fromSpace; + if( pLevel->iIdxCur ){ + /* pSubWhere may contain expressions that read from an index on the + ** table on the RHS of the right join. All such expressions first test + ** if the index is pointing at a NULL row, and if so, read from the + ** table cursor instead. So ensure that the index cursor really is + ** pointing at a NULL row here, so that no values are read from it during + ** the scan of the RHS of the RIGHT join below. */ + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } + pFrom = &uSrc.sSrc; pFrom->nSrc = 1; pFrom->nAlloc = 1; memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); @@ -162836,13 +166936,14 @@ static int isLikeOrGlob( ){ int isNum; double rDummy; - isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + assert( zNew[iTo]==0 ); + isNum = sqlite3AtoF(zNew, &rDummy); if( isNum<=0 ){ if( iTo==1 && zNew[0]=='-' ){ isNum = +1; }else{ zNew[iTo-1]++; - isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + isNum = sqlite3AtoF(zNew, &rDummy); zNew[iTo-1]--; } } @@ -162885,6 +166986,34 @@ static int isLikeOrGlob( } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ +/* +** If pExpr is one of "like", "glob", "match", or "regexp", then +** return the corresponding SQLITE_INDEX_CONSTRAINT_xxxx value. +** If not, return 0. +** +** pExpr is guaranteed to be a TK_FUNCTION. +*/ +SQLITE_PRIVATE int sqlite3ExprIsLikeOperator(const Expr *pExpr){ + static const struct { + const char *zOp; + unsigned char eOp; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; + int i; + assert( pExpr->op==TK_FUNCTION ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + for(i=0; i<ArraySize(aOp); i++){ + if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ + return aOp[i].eOp; + } + } + return 0; +} + #ifndef SQLITE_OMIT_VIRTUALTABLE /* @@ -162921,15 +167050,6 @@ static int isAuxiliaryVtabOperator( Expr **ppRight /* Expression to left of MATCH/op2 */ ){ if( pExpr->op==TK_FUNCTION ){ - static const struct Op2 { - const char *zOp; - unsigned char eOp2; - } aOp[] = { - { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, - { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, - { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, - { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } - }; ExprList *pList; Expr *pCol; /* Column reference */ int i; @@ -162949,16 +167069,11 @@ static int isAuxiliaryVtabOperator( */ pCol = pList->a[1].pExpr; assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); - if( ExprIsVtab(pCol) ){ - for(i=0; i<ArraySize(aOp); i++){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - *ppRight = pList->a[0].pExpr; - *ppLeft = pCol; - return 1; - } - } + if( ExprIsVtab(pCol) && (i = sqlite3ExprIsLikeOperator(pExpr))!=0 ){ + *peOp2 = i; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; } /* We can also match against the first column of overloaded @@ -163092,16 +167207,22 @@ static void whereCombineDisjuncts( Expr *pNew; /* New virtual expression */ int op; /* Operator for the combined expression */ int idxNew; /* Index in pWC of the next virtual term */ + Expr *pA, *pB; /* Expressions associated with pOne and pTwo */ if( (pOne->wtFlags | pTwo->wtFlags) & TERM_VNULL ) return; if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; - assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); - assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); - if( sqlite3ExprCompare(0,pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; - if( sqlite3ExprCompare(0,pOne->pExpr->pRight, pTwo->pExpr->pRight,-1) )return; + pA = pOne->pExpr; + pB = pTwo->pExpr; + assert( pA->pLeft!=0 && pA->pRight!=0 ); + assert( pB->pLeft!=0 && pB->pRight!=0 ); + if( sqlite3ExprCompare(0,pA->pLeft, pB->pLeft, -1) ) return; + if( sqlite3ExprCompare(0,pA->pRight, pB->pRight,-1) ) return; + if( ExprHasProperty(pA,EP_Commuted)!=ExprHasProperty(pB,EP_Commuted) ){ + return; + } /* If we reach this point, it means the two subterms can be combined */ if( (eOp & (eOp-1))!=0 ){ if( eOp & (WO_LT|WO_LE) ){ @@ -163112,7 +167233,7 @@ static void whereCombineDisjuncts( } } db = pWC->pWInfo->pParse->db; - pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + pNew = sqlite3ExprDup(db, pA, 0); if( pNew==0 ) return; for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( op<TK_GE ); } pNew->op = op; @@ -163491,7 +167612,7 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr, SrcList *pSrc){ if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* (3) */ assert( pSrc!=0 ); if( pExpr->op==TK_IS - && pSrc->nSrc + && pSrc->nSrc>=2 && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ return 0; /* (4) */ @@ -163667,6 +167788,9 @@ static void exprAnalyze( } assert( pWC->nTerm > idxTerm ); pTerm = &pWC->a[idxTerm]; +#ifdef SQLITE_DEBUG + pTerm->iTerm = idxTerm; +#endif pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; assert( pExpr!=0 ); /* Because malloc() has not failed */ @@ -163710,21 +167834,7 @@ static void exprAnalyze( prereqAll |= x; extraRight = x-1; /* ON clause terms may not be used with an index ** on left table of a LEFT JOIN. Ticket #3015 */ - if( (prereqAll>>1)>=x ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; - } }else if( (prereqAll>>1)>=x ){ - /* The ON clause of an INNER JOIN references a table to its right. - ** Most other SQL database engines raise an error. But SQLite versions - ** 3.0 through 3.38 just put the ON clause constraint into the WHERE - ** clause and carried on. Beginning with 3.39, raise an error only - ** if there is a RIGHT or FULL JOIN in the query. This makes SQLite - ** more like other systems, and also preserves legacy. */ - if( ALWAYS(pSrc->nSrc>0) && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; - } ExprClearProperty(pExpr, EP_InnerON); } } @@ -164081,7 +168191,7 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; + pNewTerm->prereqRight = prereqExpr | extraRight; pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.x.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_AUX; @@ -164163,13 +168273,11 @@ static void whereAddLimitExpr( int iVal = 0; if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ - Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pVal = sqlite3ExprInt32(db, iVal); if( pVal==0 ) return; - ExprSetProperty(pVal, EP_IntValue); - pVal->u.iValue = iVal; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); }else{ - Expr *pVal = sqlite3Expr(db, TK_REGISTER, 0); + Expr *pVal = sqlite3ExprAlloc(db, TK_REGISTER, 0, 0); if( pVal==0 ) return; pVal->iTable = iReg; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); @@ -164192,7 +168300,7 @@ static void whereAddLimitExpr( ** ** 1. The SELECT statement has a LIMIT clause, and ** 2. The SELECT statement is not an aggregate or DISTINCT query, and -** 3. The SELECT statement has exactly one object in its from clause, and +** 3. The SELECT statement has exactly one object in its FROM clause, and ** that object is a virtual table, and ** 4. There are no terms in the WHERE clause that will not be passed ** to the virtual table xBestIndex method. @@ -164229,8 +168337,22 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec ** (leftCursor==iCsr) test below. */ continue; } - if( pWC->a[ii].leftCursor!=iCsr ) return; - if( pWC->a[ii].prereqRight!=0 ) return; + if( pWC->a[ii].leftCursor==iCsr && pWC->a[ii].prereqRight==0 ) continue; + + /* If this term has a parent with exactly one child, and the parent will + ** be passed through to xBestIndex, then this term can be ignored. */ + if( pWC->a[ii].iParent>=0 ){ + WhereTerm *pParent = &pWC->a[ pWC->a[ii].iParent ]; + if( pParent->leftCursor==iCsr + && pParent->prereqRight==0 + && pParent->nChild==1 + ){ + continue; + } + } + + /* This term will not be passed through. Do not add a LIMIT clause. */ + return; } /* Check condition (5). Return early if it is not met. */ @@ -164894,11 +169016,11 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ pScan->pWC = pWC; pScan->k = k+1; #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x20000)!=0 && pScan->nEquiv>1 ){ int ii; - sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d", - pTerm, pScan->nEquiv); - for(ii=0; ii<pScan->nEquiv; ii++){ + sqlite3DebugPrintf("EQUIVALENT TO {%d:%d} (due to TERM-%d):", + pScan->aiCur[0], pScan->aiColumn[0], pTerm->iTerm); + for(ii=1; ii<pScan->nEquiv; ii++){ sqlite3DebugPrintf(" {%d:%d}", pScan->aiCur[ii], pScan->aiColumn[ii]); } @@ -165669,7 +169791,9 @@ static SQLITE_NOINLINE void constructAutomaticIndex( VdbeCoverage(v); VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + assert( pLevel->addrHalt ); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind,pLevel->iTabCur,pLevel->addrHalt); + VdbeCoverage(v); } if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(pParse); @@ -165697,11 +169821,14 @@ static SQLITE_NOINLINE void constructAutomaticIndex( pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; + sqlite3VdbeJumpHere(v, addrTop); }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + if( (pSrc->fg.jointype & JT_LEFT)!=0 ){ + sqlite3VdbeJumpHere(v, addrTop); + } } - sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); /* Jump here when skipping the initialization */ @@ -165977,11 +170104,14 @@ static sqlite3_index_info *allocateIndexInfo( break; } if( i==n ){ + int bSortByGroup = (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0; nOrderBy = n; if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ - eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); + eDistinct = 2 + bSortByGroup; }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ - eDistinct = 1; + eDistinct = 1 - bSortByGroup; + }else if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + eDistinct = 3; } } } @@ -166853,6 +170983,7 @@ SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ }else{ sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); } + iTerm = pTerm->iTerm = MAX(iTerm,pTerm->iTerm); sqlite3DebugPrintf( "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); @@ -166908,11 +171039,16 @@ SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ WhereInfo *pWInfo; if( pWC ){ + int nb; + SrcItem *pItem; + Table *pTab; + Bitmask mAll; + pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pSTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + nb = 1+(pWInfo->pTabList->nSrc+3)/4; + pItem = pWInfo->pTabList->a + p->iTab; + pTab = pItem->pSTab; + mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); sqlite3DebugPrintf(" %12s", @@ -167392,6 +171528,67 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } /* +** Callback for estLikePatternLength(). +** +** If this node is a string literal that is longer pWalker->sz, then set +** pWalker->sz to the byte length of that string literal. +** +** pWalker->eCode indicates how to count characters: +** +** eCode==0 Count as a GLOB pattern +** eCode==1 Count as a LIKE pattern +*/ +static int exprNodePatternLengthEst(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_STRING ){ + int sz = 0; /* Pattern size in bytes */ + u8 *z = (u8*)pExpr->u.zToken; /* The pattern */ + u8 c; /* Next character of the pattern */ + u8 c1, c2, c3; /* Wildcards */ + if( pWalker->eCode ){ + c1 = '%'; + c2 = '_'; + c3 = 0; + }else{ + c1 = '*'; + c2 = '?'; + c3 = '['; + } + while( (c = *(z++))!=0 ){ + if( c==c3 ){ + if( *z ) z++; + while( *z && *z!=']' ) z++; + }else if( c!=c1 && c!=c2 ){ + sz++; + } + } + if( sz>pWalker->u.sz ) pWalker->u.sz = sz; + } + return WRC_Continue; +} + +/* +** Return the length of the longest string literal in the given +** expression. +** +** eCode indicates how to count characters: +** +** eCode==0 Count as a GLOB pattern +** eCode==1 Count as a LIKE pattern +*/ +static int estLikePatternLength(Expr *p, u16 eCode){ + Walker w; + w.u.sz = 0; + w.eCode = eCode; + w.xExprCallback = exprNodePatternLengthEst; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + sqlite3WalkExpr(&w, p); + return w.u.sz; +} + +/* ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. @@ -167419,6 +171616,13 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** "x" column is boolean or else -1 or 0 or 1 is a common default value ** on the "x" column and so in that case only cap the output row estimate ** at 1/2 instead of 1/4. +** +** Heuristic 3: If there is a LIKE or GLOB (or REGEXP or MATCH) operator +** with a large constant pattern, then reduce the size of the search +** space according to the length of the pattern, under the theory that +** longer patterns are less likely to match. This heuristic was added +** to give better output-row count estimates when preparing queries for +** the Join-Order Benchmarks. See forum thread 2026-01-30T09:57:54z */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -167468,13 +171672,14 @@ static void whereLoopOutputAdjust( }else{ /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ + Expr *pOpExpr = pTerm->pExpr; pLoop->nOut--; if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ ){ - Expr *pRight = pTerm->pExpr->pRight; + Expr *pRight = pOpExpr->pRight; int k = 0; - testcase( pTerm->pExpr->op==TK_IS ); + testcase( pOpExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ @@ -167484,6 +171689,23 @@ static void whereLoopOutputAdjust( pTerm->wtFlags |= TERM_HEURTRUTH; iReduce = k; } + }else + if( ExprHasProperty(pOpExpr, EP_InfixFunc) + && pOpExpr->op==TK_FUNCTION + ){ + int eOp; + assert( ExprUseXList(pOpExpr) ); + assert( pOpExpr->x.pList->nExpr>=2 ); + eOp = sqlite3ExprIsLikeOperator(pOpExpr); + if( ALWAYS(eOp>0) ){ + int szPattern; + Expr *pRHS = pOpExpr->x.pList->a[0].pExpr; + eOp = eOp==SQLITE_INDEX_CONSTRAINT_LIKE; + szPattern = estLikePatternLength(pRHS, eOp); + if( szPattern>0 ){ + pLoop->nOut -= szPattern*2; + } + } } } } @@ -167555,7 +171777,7 @@ static int whereRangeVectorLen( idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); if( aff!=idxaff ) break; - pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + pColl = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); if( pColl==0 ) break; if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; } @@ -167944,6 +172166,7 @@ static int whereLoopAddBtreeIndex( pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); + if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -167994,6 +172217,7 @@ static int whereLoopAddBtreeIndex( && pProbe->hasStat1!=0 && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && pSrc->fg.fromExists==0 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -168538,7 +172762,14 @@ static int whereLoopAddBtree( whereLoopOutputAdjust(pWC, pNew, rSize); if( pSrc->fg.isSubquery ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; - pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + /* Do not set btree.pOrderBy for a recursive CTE. In this case + ** the ORDER BY clause does not determine the overall order that + ** rows are emitted from the CTE in. */ + if( (pSrc->u4.pSubq->pSelect->selFlags & SF_Recursive)==0 ){ + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + } + }else if( pSrc->fg.fromExists ){ + pNew->nOut = 0; } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -168641,6 +172872,7 @@ static int whereLoopAddBtree( ** positioned to the correct row during the right-join no-match ** loop. */ }else{ + if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); } pNew->nOut = rSize; @@ -169303,7 +173535,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; int bFirstPastRJ = 0; - int hasRightJoin = 0; + int hasRightCrossJoin = 0; WhereLoop *pNew; @@ -169330,15 +173562,34 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** prevents the right operand of a RIGHT JOIN from being swapped with ** other elements even further to the right. ** - ** The JT_LTORJ case and the hasRightJoin flag work together to - ** prevent FROM-clause terms from moving from the right side of - ** a LEFT JOIN over to the left side of that join if the LEFT JOIN - ** is itself on the left side of a RIGHT JOIN. + ** The hasRightCrossJoin flag prevent FROM-clause terms from moving + ** from the right side of a LEFT JOIN or CROSS JOIN over to the + ** left side of that same join. This is a required restriction in + ** the case of LEFT JOIN - an incorrect answer may results if it is + ** not enforced. This restriction is not required for CROSS JOIN. + ** It is provided merely as a means of controlling join order, under + ** the theory that no real-world queries that care about performance + ** actually use the CROSS JOIN syntax. */ - if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; + if( pItem->fg.jointype & (JT_LTORJ|JT_CROSS) ){ + testcase( pItem->fg.jointype & JT_LTORJ ); + testcase( pItem->fg.jointype & JT_CROSS ); + hasRightCrossJoin = 1; + } mPrereq |= mPrior; bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; - }else if( !hasRightJoin ){ + }else if( pItem->fg.fromExists ){ + /* joins that result from the EXISTS-to-JOIN optimization should not + ** be moved to the left of any of their dependencies */ + WhereClause *pWC = &pWInfo->sWC; + WhereTerm *pTerm; + int i; + for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pNew->maskSelf & pTerm->prereqAll)!=0 ){ + mPrereq |= (pTerm->prereqAll & (pNew->maskSelf-1)); + } + } + }else if( !hasRightCrossJoin ){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -169561,10 +173812,12 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered - && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) - ){ + if( pLoop->u.vtab.isOrdered && pWInfo->pOrderBy==pOrderBy ){ obSat = obDone; + }else{ + /* No further ORDER BY terms may be matched. So this call should + ** return >=0, not -1. Clear isOrderDistinct to ensure it does so. */ + isOrderDistinct = 0; } break; } @@ -169935,12 +174188,21 @@ static LogEst whereSortingCost( ** 12 otherwise ** ** For the purposes of this heuristic, a star-query is defined as a query -** with a large central table that is joined using an INNER JOIN, -** not CROSS or OUTER JOINs, against four or more smaller tables. -** The central table is called the "fact" table. The smaller tables -** that get joined are "dimension tables". Also, any table that is -** self-joined cannot be a dimension table; we assume that dimension -** tables may only be joined against fact tables. +** with a central "fact" table that is joined against multiple +** "dimension" tables, subject to the following constraints: +** +** (aa) Only a five-way or larger join is considered for this +** optimization. If there are fewer than four terms in the FROM +** clause, this heuristic does not apply. +** +** (bb) The join between the fact table and the dimension tables must +** be an INNER join. CROSS and OUTER JOINs do not qualify. +** +** (cc) A table must have 3 or more dimension tables in order to be +** considered a fact table. (Was 4 prior to 2026-02-10.) +** +** (dd) A table that is a self-join cannot be a dimension table. +** Dimension tables are joined against fact tables. ** ** SIDE EFFECT: (and really the whole point of this subroutine) ** @@ -169993,7 +174255,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ } #endif /* SQLITE_DEBUG */ - if( nLoop>=5 + if( nLoop>=4 /* Constraint (aa) */ && !pWInfo->bStarDone && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery) ){ @@ -170005,7 +174267,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWInfo->bStarDone = 1; /* Only do this computation once */ - /* Look for fact tables with four or more dimensions where the + /* Look for fact tables with three or more dimensions where the ** dimension tables are not separately from the fact tables by an outer ** or cross join. Adjust cost weights if found. */ @@ -170022,18 +174284,17 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( (pFactTab->fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ /* If the candidate fact-table is the right table of an outer join ** restrict the search for dimension-tables to be tables to the right - ** of the fact-table. */ - if( iFromIdx+4 > nLoop ) break; /* Impossible to reach nDep>=4 */ + ** of the fact-table. Constraint (bb) */ + if( iFromIdx+3 > nLoop ){ + break; /* ^-- Impossible to reach nDep>=2 - Constraint (cc) */ + } while( pStart && pStart->iTab<=iFromIdx ){ pStart = pStart->pNextLoop; } } for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ - /* Fact-tables and dimension-tables cannot be separated by an - ** outer join (at least for the definition of fact- and dimension- - ** used by this heuristic). */ - break; + break; /* Constraint (bb) */ } if( (pWLoop->prereq & m)!=0 /* pWInfo depends on iFromIdx */ && (pWLoop->maskSelf & mSeen)==0 /* pWInfo not already a dependency */ @@ -170047,7 +174308,9 @@ static int computeMxChoice(WhereInfo *pWInfo){ } } } - if( nDep<=3 ) continue; + if( nDep<=2 ){ + continue; /* Constraint (cc) */ + } /* If we reach this point, it means that pFactTab is a fact table ** with four or more dimensions connected by inner joins. Proceed @@ -170061,6 +174324,23 @@ static int computeMxChoice(WhereInfo *pWInfo){ } } #endif +#ifdef WHERETRACE_ENABLED /* 0x80000 */ + if( sqlite3WhereTrace & 0x80000 ){ + Bitmask mShow = mSeen; + sqlite3DebugPrintf("Fact table %s(%d), dimensions:", + pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, + iFromIdx); + for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( mShow & pWLoop->maskSelf ){ + SrcItem *pDim = aFromTabs + pWLoop->iTab; + mShow &= ~pWLoop->maskSelf; + sqlite3DebugPrintf(" %s(%d)", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab); + } + } + sqlite3DebugPrintf("\n"); + } +#endif pWInfo->bStarUsed = 1; /* Compute the maximum cost of any WhereLoop for the @@ -170083,10 +174363,8 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( sqlite3WhereTrace & 0x80000 ){ SrcItem *pDim = aFromTabs + pWLoop->iTab; sqlite3DebugPrintf( - "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n", - pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab, - pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, - iFromIdx, mxRun + "Increase SCAN cost of %s to %d\n", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, mxRun ); } pWLoop->rStarDelta = mxRun - pWLoop->rRun; @@ -170310,8 +174588,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** mxChoice best-so-far paths. ** ** First look for an existing path among best-so-far paths - ** that covers the same set of loops and has the same isOrdered - ** setting as the current path candidate. + ** that: + ** (1) covers the same set of loops, and + ** (2) has a compatible isOrdered value. + ** + ** "Compatible isOrdered value" means either + ** (A) both have isOrdered==-1, or + ** (B) both have isOrder>=0, or + ** (C) ordering does not matter because this is the last round + ** of the solver. ** ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range @@ -170320,7 +174605,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ testcase( nTo==0 ); for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){ if( pTo->maskLoop==maskNew - && ((pTo->isOrdered^isOrdered)&0x80)==0 + && ( ((pTo->isOrdered^isOrdered)&0x80)==0 || iLoop==nLoop-1 ) ){ testcase( jj==nTo-1 ); break; @@ -170475,11 +174760,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_ERROR; } - /* Find the lowest cost path. pFrom will be left pointing to that path */ + /* Only one path is available, which is the best path */ + assert( nFrom==1 ); pFrom = aFrom; - for(ii=1; ii<nFrom; ii++){ - if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; - } + assert( pWInfo->nLevel==nLoop ); /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLoop<nLoop; iLoop++){ @@ -170612,7 +174896,10 @@ static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ for(i=0; i<pWInfo->nLevel; i++){ WhereLoop *p = pWInfo->a[i].pWLoop; if( p==0 ) break; - if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Treat a vtab scan as similar to a full-table scan */ + break; + } if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ u8 iTab = p->iTab; WhereLoop *pLoop; @@ -170891,6 +175178,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ pTerm->wtFlags |= TERM_CODED; + pTerm->prereqAll = 0; } } if( i!=pWInfo->nLevel-1 ){ @@ -171550,6 +175838,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; + pLevel->addrBrk = sqlite3VdbeMakeLabel(pParse); + if( ii==0 || (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->addrHalt = pLevel->addrBrk; + }else if( pWInfo->a[ii-1].pRJ ){ + pLevel->addrHalt = pWInfo->a[ii-1].addrBrk; + }else{ + pLevel->addrHalt = pWInfo->a[ii-1].addrHalt; + } if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ /* Do nothing */ }else @@ -171601,6 +175897,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, (const u8*)&pTabItem->colUsed, P4_INT64); #endif + if( ii>=2 + && (pTabItem[0].fg.jointype & (JT_LTORJ|JT_LEFT))==0 + && pLevel->addrHalt==pWInfo->a[0].addrHalt + ){ + sqlite3VdbeAddOp2(v, OP_IfEmpty, pTabItem->iCursor, pWInfo->iBreak); + VdbeCoverage(v); + } }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -171813,6 +176116,9 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3 *db = pParse->db; int iEnd = sqlite3VdbeCurrentAddr(v); int nRJ = 0; +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + int addrSeek = 0; +#endif /* Generate loop termination code. */ @@ -171825,7 +176131,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the RIGHT JOIN table */ WhereRightJoin *pRJ = pLevel->pRJ; sqlite3VdbeResolveLabel(v, pLevel->addrCont); - pLevel->addrCont = 0; + /* Replace addrCont with a new label that will never be used, just so + ** the subsequent call to resolve pLevel->addrCont will have something + ** to resolve. */ + pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1); VdbeCoverage(v); @@ -171834,7 +176143,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; if( pLevel->op!=OP_Noop ){ #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT - int addrSeek = 0; Index *pIdx; int n; if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED @@ -171846,6 +176154,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ int r1 = pParse->nMem+1; int j, op; + int addrIfNull = 0; /* Init to avoid false-positive compiler warning */ + if( pLevel->iLeftJoin ){ + addrIfNull = sqlite3VdbeAddOp2(v, OP_IfNullRow, pLevel->iIdxCur, r1); + } for(j=0; j<n; j++){ sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j); } @@ -171855,10 +176167,20 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, op==OP_SeekLT); VdbeCoverageIf(v, op==OP_SeekGT); sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + if( pLevel->iLeftJoin ){ + sqlite3VdbeJumpHere(v, addrIfNull); + } } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ - /* The common case: Advance to the next row */ - if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); + } + if( pTabList->a[pLevel->iFrom].fg.fromExists ){ + /* This is an EXISTS-to-JOIN optimization loop. If this loop sees a + ** successful row, it should break out of itself. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); + VdbeComment((v, "EXISTS break %d", i)); + } + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + if( pLevel->op!=OP_Noop ){ sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); VdbeCoverage(v); @@ -171871,10 +176193,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverage(v); } #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT - if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); + if( addrSeek ){ + sqlite3VdbeJumpHere(v, addrSeek); + addrSeek = 0; + } #endif - }else if( pLevel->addrCont ){ - sqlite3VdbeResolveLabel(v, pLevel->addrCont); } if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; @@ -172871,7 +177194,7 @@ SQLITE_PRIVATE void sqlite3WindowUpdate( pWin->eEnd = aUp[i].eEnd; pWin->eExclude = 0; if( pWin->eStart==TK_FOLLOWING ){ - pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + pWin->pStart = sqlite3ExprInt32(db, 1); } break; } @@ -173216,9 +177539,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ ** keep everything legal in this case. */ if( pSublist==0 ){ - pSublist = sqlite3ExprListAppend(pParse, 0, - sqlite3Expr(db, TK_INTEGER, "0") - ); + pSublist = sqlite3ExprListAppend(pParse, 0, sqlite3ExprInt32(db, 0)); } pSub = sqlite3SelectNew( @@ -174707,7 +179028,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){ ** ** ROWS BETWEEN <expr1> FOLLOWING AND <expr2> FOLLOWING ** -** ... loop started by sqlite3WhereBegin() ... +** ... loop started by sqlite3WhereBegin() ... ** if( new partition ){ ** Gosub flush ** } @@ -175225,6 +179546,12 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); }else{ assert( pMWin->eEnd==TK_FOLLOWING ); + /* assert( regStart>=0 ); + ** regEnd = regEnd - regStart; + ** regStart = 0; */ + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regEnd); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regStart); + addrStart = sqlite3VdbeCurrentAddr(v); addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); @@ -175436,8 +179763,23 @@ static void updateDeleteLimitError( ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate ** testing. */ - static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ - return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + static void *parserStackRealloc( + void *pOld, /* Prior allocation */ + sqlite3_uint64 newSize, /* Requested new alloation size */ + Parse *pParse /* Parsing context */ + ){ + void *p = sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + if( p==0 ) sqlite3OomFault(pParse->db); + return p; + } + static void parserStackFree(void *pOld, Parse *pParse){ + (void)pParse; + sqlite3_free(pOld); + } + + /* Return an integer that is the maximum allowed stack size */ + static int parserStackSizeLimit(Parse *pParse){ + return pParse->db->aLimit[SQLITE_LIMIT_PARSER_DEPTH]; } @@ -175476,15 +179818,46 @@ static void updateDeleteLimitError( } - /* A routine to convert a binary TK_IS or TK_ISNOT expression into a - ** unary TK_ISNULL or TK_NOTNULL expression. */ - static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ - sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ - pA->op = (u8)op; - sqlite3ExprDelete(db, pA->pRight); - pA->pRight = 0; + /* Create a TK_ISNULL or TK_NOTNULL expression, perhaps optimized to + ** to TK_TRUEFALSE, if possible */ + static Expr *sqlite3PExprIsNull( + Parse *pParse, /* Parsing context */ + int op, /* TK_ISNULL or TK_NOTNULL */ + Expr *pLeft /* Operand */ + ){ + Expr *p = pLeft; + assert( op==TK_ISNULL || op==TK_NOTNULL ); + assert( pLeft!=0 ); + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + assert( p!=0 ); + } + switch( p->op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + sqlite3ExprDeferredDelete(pParse, pLeft); + return sqlite3ExprInt32(pParse->db, op==TK_NOTNULL); + default: + break; } + return sqlite3PExpr(pParse, op, pLeft, 0); + } + + /* Create a TK_IS or TK_ISNOT operator, perhaps optimized to + ** TK_ISNULL or TK_NOTNULL or TK_TRUEFALSE. */ + static Expr *sqlite3PExprIs( + Parse *pParse, /* Parsing context */ + int op, /* TK_IS or TK_ISNOT */ + Expr *pLeft, /* Left operand */ + Expr *pRight /* Right operand */ + ){ + if( pRight && pRight->op==TK_NULL ){ + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3PExprIsNull(pParse, op==TK_IS ? TK_ISNULL : TK_NOTNULL, pLeft); + } + return sqlite3PExpr(pParse, op, pLeft, pRight); } /* Add a single new term to an ExprList that is used to store a @@ -175767,63 +180140,72 @@ static void updateDeleteLimitError( #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 323 +#define YYNOCODE 322 #define YYACTIONTYPE unsigned short int #define YYWILDCARD 102 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - u32 yy9; - struct TrigEvent yy28; - With* yy125; - IdList* yy204; - struct FrameBound yy205; - TriggerStep* yy319; - const char* yy342; - Cte* yy361; - ExprList* yy402; - Upsert* yy403; - OnOrUsing yy421; - u8 yy444; - struct {int value; int mask;} yy481; - Window* yy483; - int yy502; - SrcList* yy563; - Expr* yy590; - Select* yy637; + ExprList* yy14; + With* yy59; + Cte* yy67; + Upsert* yy122; + IdList* yy132; + int yy144; + const char* yy168; + SrcList* yy203; + Window* yy211; + OnOrUsing yy269; + struct TrigEvent yy286; + struct {int value; int mask;} yy383; + u32 yy391; + TriggerStep* yy427; + Expr* yy454; + u8 yy462; + struct FrameBound yy509; + Select* yy555; } YYMINORTYPE; #ifndef YYSTACKDEPTH -#define YYSTACKDEPTH 100 +#define YYSTACKDEPTH 50 #endif #define sqlite3ParserARG_SDECL #define sqlite3ParserARG_PDECL #define sqlite3ParserARG_PARAM #define sqlite3ParserARG_FETCH #define sqlite3ParserARG_STORE +#undef YYREALLOC #define YYREALLOC parserStackRealloc -#define YYFREE sqlite3_free +#undef YYFREE +#define YYFREE parserStackFree +#undef YYDYNSTACK #define YYDYNSTACK 1 +#undef YYSIZELIMIT +#define YYSIZELIMIT parserStackSizeLimit +#define sqlite3ParserCTX(P) ((P)->pParse) #define sqlite3ParserCTX_SDECL Parse *pParse; #define sqlite3ParserCTX_PDECL ,Parse *pParse #define sqlite3ParserCTX_PARAM ,pParse #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; +#undef YYERRORSYMBOL +#undef YYERRSYMDT +#undef YYFALLBACK #define YYFALLBACK 1 -#define YYNSTATE 583 -#define YYNRULE 409 -#define YYNRULE_WITH_ACTION 344 +#define YYNSTATE 600 +#define YYNRULE 412 +#define YYNRULE_WITH_ACTION 348 #define YYNTOKEN 187 -#define YY_MAX_SHIFT 582 -#define YY_MIN_SHIFTREDUCE 845 -#define YY_MAX_SHIFTREDUCE 1253 -#define YY_ERROR_ACTION 1254 -#define YY_ACCEPT_ACTION 1255 -#define YY_NO_ACTION 1256 -#define YY_MIN_REDUCE 1257 -#define YY_MAX_REDUCE 1665 +#define YY_MAX_SHIFT 599 +#define YY_MIN_SHIFTREDUCE 867 +#define YY_MAX_SHIFTREDUCE 1278 +#define YY_ERROR_ACTION 1279 +#define YY_ACCEPT_ACTION 1280 +#define YY_NO_ACTION 1281 +#define YY_MIN_REDUCE 1282 +#define YY_MAX_REDUCE 1693 #define YY_MIN_DSTRCTR 206 -#define YY_MAX_DSTRCTR 320 +#define YY_MAX_DSTRCTR 319 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -175906,643 +180288,680 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2207) +#define YY_ACTTAB_COUNT (2379) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 130, 127, 234, 282, 282, 1328, 576, 1307, 460, 289, - /* 10 */ 289, 576, 1622, 381, 576, 1328, 573, 576, 562, 413, - /* 20 */ 1300, 1542, 573, 481, 562, 524, 460, 459, 558, 82, - /* 30 */ 82, 983, 294, 375, 51, 51, 498, 61, 61, 984, - /* 40 */ 82, 82, 1577, 137, 138, 91, 7, 1228, 1228, 1063, - /* 50 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 413, - /* 60 */ 288, 288, 182, 288, 288, 481, 536, 288, 288, 130, - /* 70 */ 127, 234, 432, 573, 525, 562, 573, 557, 562, 1290, - /* 80 */ 573, 421, 562, 137, 138, 91, 559, 1228, 1228, 1063, - /* 90 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 296, - /* 100 */ 460, 398, 1249, 134, 134, 134, 134, 133, 133, 132, - /* 110 */ 132, 132, 131, 128, 451, 451, 1050, 1050, 1064, 1067, - /* 120 */ 1255, 1, 1, 582, 2, 1259, 581, 1174, 1259, 1174, - /* 130 */ 321, 413, 155, 321, 1584, 155, 379, 112, 481, 1341, - /* 140 */ 456, 299, 1341, 134, 134, 134, 134, 133, 133, 132, - /* 150 */ 132, 132, 131, 128, 451, 137, 138, 91, 498, 1228, - /* 160 */ 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, - /* 170 */ 136, 1204, 862, 1281, 288, 288, 283, 288, 288, 523, - /* 180 */ 523, 1250, 139, 578, 7, 578, 1345, 573, 1169, 562, - /* 190 */ 573, 1054, 562, 136, 136, 136, 136, 129, 573, 547, - /* 200 */ 562, 1169, 245, 1541, 1169, 245, 133, 133, 132, 132, - /* 210 */ 132, 131, 128, 451, 302, 134, 134, 134, 134, 133, - /* 220 */ 133, 132, 132, 132, 131, 128, 451, 1575, 1204, 1205, - /* 230 */ 1204, 7, 470, 550, 455, 413, 550, 455, 130, 127, - /* 240 */ 234, 134, 134, 134, 134, 133, 133, 132, 132, 132, - /* 250 */ 131, 128, 451, 136, 136, 136, 136, 538, 483, 137, - /* 260 */ 138, 91, 1019, 1228, 1228, 1063, 1066, 1053, 1053, 135, - /* 270 */ 135, 136, 136, 136, 136, 1085, 576, 1204, 132, 132, - /* 280 */ 132, 131, 128, 451, 93, 214, 134, 134, 134, 134, - /* 290 */ 133, 133, 132, 132, 132, 131, 128, 451, 401, 19, - /* 300 */ 19, 134, 134, 134, 134, 133, 133, 132, 132, 132, - /* 310 */ 131, 128, 451, 1498, 426, 267, 344, 467, 332, 134, - /* 320 */ 134, 134, 134, 133, 133, 132, 132, 132, 131, 128, - /* 330 */ 451, 1281, 576, 6, 1204, 1205, 1204, 257, 576, 413, - /* 340 */ 511, 508, 507, 1279, 94, 1019, 464, 1204, 551, 551, - /* 350 */ 506, 1224, 1571, 44, 38, 51, 51, 411, 576, 413, - /* 360 */ 45, 51, 51, 137, 138, 91, 530, 1228, 1228, 1063, - /* 370 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 398, - /* 380 */ 1148, 82, 82, 137, 138, 91, 39, 1228, 1228, 1063, - /* 390 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 344, - /* 400 */ 44, 288, 288, 375, 1204, 1205, 1204, 209, 1204, 1224, - /* 410 */ 320, 567, 471, 576, 573, 576, 562, 576, 316, 264, - /* 420 */ 231, 46, 160, 134, 134, 134, 134, 133, 133, 132, - /* 430 */ 132, 132, 131, 128, 451, 303, 82, 82, 82, 82, - /* 440 */ 82, 82, 442, 134, 134, 134, 134, 133, 133, 132, - /* 450 */ 132, 132, 131, 128, 451, 1582, 544, 320, 567, 1250, - /* 460 */ 874, 1582, 380, 382, 413, 1204, 1205, 1204, 360, 182, - /* 470 */ 288, 288, 1576, 557, 1339, 557, 7, 557, 1277, 472, - /* 480 */ 346, 526, 531, 573, 556, 562, 439, 1511, 137, 138, - /* 490 */ 91, 219, 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, - /* 500 */ 136, 136, 136, 136, 465, 1511, 1513, 532, 413, 288, - /* 510 */ 288, 423, 512, 288, 288, 411, 288, 288, 874, 130, - /* 520 */ 127, 234, 573, 1107, 562, 1204, 573, 1107, 562, 573, - /* 530 */ 560, 562, 137, 138, 91, 1293, 1228, 1228, 1063, 1066, - /* 540 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 134, 134, - /* 550 */ 134, 134, 133, 133, 132, 132, 132, 131, 128, 451, - /* 560 */ 493, 503, 1292, 1204, 257, 288, 288, 511, 508, 507, - /* 570 */ 1204, 1628, 1169, 123, 568, 275, 4, 506, 573, 1511, - /* 580 */ 562, 331, 1204, 1205, 1204, 1169, 548, 548, 1169, 261, - /* 590 */ 571, 7, 134, 134, 134, 134, 133, 133, 132, 132, - /* 600 */ 132, 131, 128, 451, 108, 533, 130, 127, 234, 1204, - /* 610 */ 448, 447, 413, 1451, 452, 983, 886, 96, 1598, 1233, - /* 620 */ 1204, 1205, 1204, 984, 1235, 1450, 565, 1204, 1205, 1204, - /* 630 */ 229, 522, 1234, 534, 1333, 1333, 137, 138, 91, 1449, - /* 640 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, - /* 650 */ 136, 136, 373, 1595, 971, 1040, 413, 1236, 418, 1236, - /* 660 */ 879, 121, 121, 948, 373, 1595, 1204, 1205, 1204, 122, - /* 670 */ 1204, 452, 577, 452, 363, 417, 1028, 882, 373, 1595, - /* 680 */ 137, 138, 91, 462, 1228, 1228, 1063, 1066, 1053, 1053, - /* 690 */ 135, 135, 136, 136, 136, 136, 134, 134, 134, 134, - /* 700 */ 133, 133, 132, 132, 132, 131, 128, 451, 1028, 1028, - /* 710 */ 1030, 1031, 35, 570, 570, 570, 197, 423, 1040, 198, - /* 720 */ 1204, 123, 568, 1204, 4, 320, 567, 1204, 1205, 1204, - /* 730 */ 40, 388, 576, 384, 882, 1029, 423, 1188, 571, 1028, - /* 740 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, - /* 750 */ 128, 451, 529, 1568, 1204, 19, 19, 1204, 575, 492, - /* 760 */ 413, 157, 452, 489, 1187, 1331, 1331, 5, 1204, 949, - /* 770 */ 431, 1028, 1028, 1030, 565, 22, 22, 1204, 1205, 1204, - /* 780 */ 1204, 1205, 1204, 477, 137, 138, 91, 212, 1228, 1228, - /* 790 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, - /* 800 */ 1188, 48, 111, 1040, 413, 1204, 213, 970, 1041, 121, - /* 810 */ 121, 1204, 1205, 1204, 1204, 1205, 1204, 122, 221, 452, - /* 820 */ 577, 452, 44, 487, 1028, 1204, 1205, 1204, 137, 138, - /* 830 */ 91, 378, 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, - /* 840 */ 136, 136, 136, 136, 134, 134, 134, 134, 133, 133, - /* 850 */ 132, 132, 132, 131, 128, 451, 1028, 1028, 1030, 1031, - /* 860 */ 35, 461, 1204, 1205, 1204, 1569, 1040, 377, 214, 1149, - /* 870 */ 1657, 535, 1657, 437, 902, 320, 567, 1568, 364, 320, - /* 880 */ 567, 412, 329, 1029, 519, 1188, 3, 1028, 134, 134, - /* 890 */ 134, 134, 133, 133, 132, 132, 132, 131, 128, 451, - /* 900 */ 1659, 399, 1169, 307, 893, 307, 515, 576, 413, 214, - /* 910 */ 498, 944, 1024, 540, 903, 1169, 943, 392, 1169, 1028, - /* 920 */ 1028, 1030, 406, 298, 1204, 50, 1149, 1658, 413, 1658, - /* 930 */ 145, 145, 137, 138, 91, 293, 1228, 1228, 1063, 1066, - /* 940 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 1188, 1147, - /* 950 */ 514, 1568, 137, 138, 91, 1505, 1228, 1228, 1063, 1066, - /* 960 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 434, 323, - /* 970 */ 435, 539, 111, 1506, 274, 291, 372, 517, 367, 516, - /* 980 */ 262, 1204, 1205, 1204, 1574, 481, 363, 576, 7, 1569, - /* 990 */ 1568, 377, 134, 134, 134, 134, 133, 133, 132, 132, - /* 1000 */ 132, 131, 128, 451, 1568, 576, 1147, 576, 232, 576, - /* 1010 */ 19, 19, 134, 134, 134, 134, 133, 133, 132, 132, - /* 1020 */ 132, 131, 128, 451, 1169, 433, 576, 1207, 19, 19, - /* 1030 */ 19, 19, 19, 19, 1627, 576, 911, 1169, 47, 120, - /* 1040 */ 1169, 117, 413, 306, 498, 438, 1125, 206, 336, 19, - /* 1050 */ 19, 1435, 49, 449, 449, 449, 1368, 315, 81, 81, - /* 1060 */ 576, 304, 413, 1570, 207, 377, 137, 138, 91, 115, - /* 1070 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, - /* 1080 */ 136, 136, 576, 82, 82, 1207, 137, 138, 91, 1340, - /* 1090 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, - /* 1100 */ 136, 136, 1569, 386, 377, 82, 82, 463, 1126, 1552, - /* 1110 */ 333, 463, 335, 131, 128, 451, 1569, 161, 377, 16, - /* 1120 */ 317, 387, 428, 1127, 448, 447, 134, 134, 134, 134, - /* 1130 */ 133, 133, 132, 132, 132, 131, 128, 451, 1128, 576, - /* 1140 */ 1105, 10, 445, 267, 576, 1554, 134, 134, 134, 134, - /* 1150 */ 133, 133, 132, 132, 132, 131, 128, 451, 532, 576, - /* 1160 */ 922, 576, 19, 19, 576, 1573, 576, 147, 147, 7, - /* 1170 */ 923, 1236, 498, 1236, 576, 487, 413, 552, 285, 1224, - /* 1180 */ 969, 215, 82, 82, 66, 66, 1435, 67, 67, 21, - /* 1190 */ 21, 1110, 1110, 495, 334, 297, 413, 53, 53, 297, - /* 1200 */ 137, 138, 91, 119, 1228, 1228, 1063, 1066, 1053, 1053, - /* 1210 */ 135, 135, 136, 136, 136, 136, 413, 1336, 1311, 446, - /* 1220 */ 137, 138, 91, 227, 1228, 1228, 1063, 1066, 1053, 1053, - /* 1230 */ 135, 135, 136, 136, 136, 136, 574, 1224, 936, 936, - /* 1240 */ 137, 126, 91, 141, 1228, 1228, 1063, 1066, 1053, 1053, - /* 1250 */ 135, 135, 136, 136, 136, 136, 533, 429, 472, 346, - /* 1260 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, - /* 1270 */ 128, 451, 576, 457, 233, 343, 1435, 403, 498, 1550, - /* 1280 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, - /* 1290 */ 128, 451, 576, 324, 576, 82, 82, 487, 576, 969, - /* 1300 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, - /* 1310 */ 128, 451, 288, 288, 546, 68, 68, 54, 54, 553, - /* 1320 */ 413, 69, 69, 351, 6, 573, 944, 562, 410, 409, - /* 1330 */ 1435, 943, 450, 545, 260, 259, 258, 576, 158, 576, - /* 1340 */ 413, 222, 1180, 479, 969, 138, 91, 430, 1228, 1228, - /* 1350 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, - /* 1360 */ 70, 70, 71, 71, 576, 1126, 91, 576, 1228, 1228, - /* 1370 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, - /* 1380 */ 1127, 166, 850, 851, 852, 1282, 419, 72, 72, 108, - /* 1390 */ 73, 73, 1310, 358, 1180, 1128, 576, 305, 576, 123, - /* 1400 */ 568, 494, 4, 488, 134, 134, 134, 134, 133, 133, - /* 1410 */ 132, 132, 132, 131, 128, 451, 571, 564, 534, 55, - /* 1420 */ 55, 56, 56, 576, 134, 134, 134, 134, 133, 133, - /* 1430 */ 132, 132, 132, 131, 128, 451, 576, 1104, 233, 1104, - /* 1440 */ 452, 1602, 582, 2, 1259, 576, 57, 57, 576, 321, - /* 1450 */ 576, 155, 565, 1435, 485, 353, 576, 356, 1341, 59, - /* 1460 */ 59, 576, 44, 969, 569, 419, 576, 238, 60, 60, - /* 1470 */ 261, 74, 74, 75, 75, 287, 231, 576, 1366, 76, - /* 1480 */ 76, 1040, 420, 184, 20, 20, 576, 121, 121, 77, - /* 1490 */ 77, 97, 218, 288, 288, 122, 125, 452, 577, 452, - /* 1500 */ 143, 143, 1028, 576, 520, 576, 573, 576, 562, 144, - /* 1510 */ 144, 474, 227, 1244, 478, 123, 568, 576, 4, 320, - /* 1520 */ 567, 245, 411, 576, 443, 411, 78, 78, 62, 62, - /* 1530 */ 79, 79, 571, 319, 1028, 1028, 1030, 1031, 35, 418, - /* 1540 */ 63, 63, 576, 290, 411, 9, 80, 80, 1144, 576, - /* 1550 */ 400, 576, 486, 455, 576, 1223, 452, 576, 325, 342, - /* 1560 */ 576, 111, 576, 1188, 242, 64, 64, 473, 565, 576, - /* 1570 */ 23, 576, 170, 170, 171, 171, 576, 87, 87, 328, - /* 1580 */ 65, 65, 542, 83, 83, 146, 146, 541, 123, 568, - /* 1590 */ 341, 4, 84, 84, 168, 168, 576, 1040, 576, 148, - /* 1600 */ 148, 576, 1380, 121, 121, 571, 1021, 576, 266, 576, - /* 1610 */ 424, 122, 576, 452, 577, 452, 576, 553, 1028, 142, - /* 1620 */ 142, 169, 169, 576, 162, 162, 528, 889, 371, 452, - /* 1630 */ 152, 152, 151, 151, 1379, 149, 149, 109, 370, 150, - /* 1640 */ 150, 565, 576, 480, 576, 266, 86, 86, 576, 1092, - /* 1650 */ 1028, 1028, 1030, 1031, 35, 542, 482, 576, 266, 466, - /* 1660 */ 543, 123, 568, 1616, 4, 88, 88, 85, 85, 475, - /* 1670 */ 1040, 52, 52, 222, 901, 900, 121, 121, 571, 1188, - /* 1680 */ 58, 58, 244, 1032, 122, 889, 452, 577, 452, 908, - /* 1690 */ 909, 1028, 300, 347, 504, 111, 263, 361, 165, 111, - /* 1700 */ 111, 1088, 452, 263, 974, 1153, 266, 1092, 986, 987, - /* 1710 */ 942, 939, 125, 125, 565, 1103, 872, 1103, 159, 941, - /* 1720 */ 1309, 125, 1557, 1028, 1028, 1030, 1031, 35, 542, 337, - /* 1730 */ 1530, 205, 1529, 541, 499, 1589, 490, 348, 1376, 352, - /* 1740 */ 355, 1032, 357, 1040, 359, 1324, 1308, 366, 563, 121, - /* 1750 */ 121, 376, 1188, 1389, 1434, 1362, 280, 122, 1374, 452, - /* 1760 */ 577, 452, 167, 1439, 1028, 1289, 1280, 1268, 1267, 1269, - /* 1770 */ 1609, 1359, 312, 313, 314, 397, 12, 237, 224, 1421, - /* 1780 */ 295, 1416, 1409, 1426, 339, 484, 340, 509, 1371, 1612, - /* 1790 */ 1372, 1425, 1244, 404, 301, 228, 1028, 1028, 1030, 1031, - /* 1800 */ 35, 1601, 1192, 454, 345, 1307, 292, 369, 1502, 1501, - /* 1810 */ 270, 396, 396, 395, 277, 393, 1370, 1369, 859, 1549, - /* 1820 */ 186, 123, 568, 235, 4, 1188, 391, 210, 211, 223, - /* 1830 */ 1547, 239, 1241, 327, 422, 96, 220, 195, 571, 180, - /* 1840 */ 188, 326, 468, 469, 190, 191, 502, 192, 193, 566, - /* 1850 */ 247, 109, 1430, 491, 199, 251, 102, 281, 402, 476, - /* 1860 */ 405, 1496, 452, 497, 253, 1422, 13, 1428, 14, 1427, - /* 1870 */ 203, 1507, 241, 500, 565, 354, 407, 92, 95, 1270, - /* 1880 */ 175, 254, 518, 43, 1327, 255, 1326, 1325, 436, 1518, - /* 1890 */ 350, 1318, 104, 229, 893, 1626, 440, 441, 1625, 408, - /* 1900 */ 240, 1296, 268, 1040, 310, 269, 1297, 527, 444, 121, - /* 1910 */ 121, 368, 1295, 1594, 1624, 311, 1394, 122, 1317, 452, - /* 1920 */ 577, 452, 374, 1580, 1028, 1393, 140, 553, 11, 90, - /* 1930 */ 568, 385, 4, 116, 318, 414, 1579, 110, 1483, 537, - /* 1940 */ 320, 567, 1350, 555, 42, 579, 571, 1349, 1198, 383, - /* 1950 */ 276, 390, 216, 389, 278, 279, 1028, 1028, 1030, 1031, - /* 1960 */ 35, 172, 580, 1265, 458, 1260, 415, 416, 185, 156, - /* 1970 */ 452, 1534, 1535, 173, 1533, 1532, 89, 308, 225, 226, - /* 1980 */ 846, 174, 565, 453, 217, 1188, 322, 236, 1102, 154, - /* 1990 */ 1100, 330, 187, 176, 1223, 243, 189, 925, 338, 246, - /* 2000 */ 1116, 194, 177, 425, 178, 427, 98, 196, 99, 100, - /* 2010 */ 101, 1040, 179, 1119, 1115, 248, 249, 121, 121, 163, - /* 2020 */ 24, 250, 349, 1238, 496, 122, 1108, 452, 577, 452, - /* 2030 */ 1192, 454, 1028, 266, 292, 200, 252, 201, 861, 396, - /* 2040 */ 396, 395, 277, 393, 15, 501, 859, 370, 292, 256, - /* 2050 */ 202, 554, 505, 396, 396, 395, 277, 393, 103, 239, - /* 2060 */ 859, 327, 25, 26, 1028, 1028, 1030, 1031, 35, 326, - /* 2070 */ 362, 510, 891, 239, 365, 327, 513, 904, 105, 309, - /* 2080 */ 164, 181, 27, 326, 106, 521, 107, 1185, 1069, 1155, - /* 2090 */ 17, 1154, 230, 1188, 284, 286, 265, 204, 125, 1171, - /* 2100 */ 241, 28, 978, 972, 29, 41, 1175, 1179, 175, 1173, - /* 2110 */ 30, 43, 31, 8, 241, 1178, 32, 1160, 208, 549, - /* 2120 */ 33, 111, 175, 1083, 1070, 43, 1068, 1072, 240, 113, - /* 2130 */ 114, 34, 561, 118, 1124, 271, 1073, 36, 18, 572, - /* 2140 */ 1033, 873, 240, 124, 37, 935, 272, 273, 1617, 183, - /* 2150 */ 153, 394, 1194, 1193, 1256, 1256, 1256, 1256, 1256, 1256, - /* 2160 */ 1256, 1256, 1256, 414, 1256, 1256, 1256, 1256, 320, 567, - /* 2170 */ 1256, 1256, 1256, 1256, 1256, 1256, 1256, 414, 1256, 1256, - /* 2180 */ 1256, 1256, 320, 567, 1256, 1256, 1256, 1256, 1256, 1256, - /* 2190 */ 1256, 1256, 458, 1256, 1256, 1256, 1256, 1256, 1256, 1256, - /* 2200 */ 1256, 1256, 1256, 1256, 1256, 1256, 458, + /* 0 */ 134, 131, 238, 290, 290, 1353, 593, 1332, 478, 1606, + /* 10 */ 593, 1315, 593, 7, 593, 1353, 590, 593, 579, 424, + /* 20 */ 1566, 134, 131, 238, 1318, 541, 478, 477, 575, 84, + /* 30 */ 84, 1005, 303, 84, 84, 51, 51, 63, 63, 1006, + /* 40 */ 84, 84, 498, 141, 142, 93, 442, 1254, 1254, 1085, + /* 50 */ 1088, 1075, 1075, 139, 139, 140, 140, 140, 140, 424, + /* 60 */ 296, 296, 498, 296, 296, 567, 553, 296, 296, 1306, + /* 70 */ 574, 1358, 1358, 590, 542, 579, 590, 574, 579, 548, + /* 80 */ 590, 1304, 579, 141, 142, 93, 576, 1254, 1254, 1085, + /* 90 */ 1088, 1075, 1075, 139, 139, 140, 140, 140, 140, 399, + /* 100 */ 478, 395, 6, 138, 138, 138, 138, 137, 137, 136, + /* 110 */ 136, 136, 135, 132, 463, 44, 342, 593, 305, 1127, + /* 120 */ 1280, 1, 1, 599, 2, 1284, 598, 1200, 1284, 1200, + /* 130 */ 330, 424, 158, 330, 1613, 158, 390, 116, 308, 1366, + /* 140 */ 51, 51, 1366, 138, 138, 138, 138, 137, 137, 136, + /* 150 */ 136, 136, 135, 132, 463, 141, 142, 93, 515, 1254, + /* 160 */ 1254, 1085, 1088, 1075, 1075, 139, 139, 140, 140, 140, + /* 170 */ 140, 1230, 329, 584, 296, 296, 212, 296, 296, 568, + /* 180 */ 568, 488, 143, 1072, 1072, 1086, 1089, 590, 1195, 579, + /* 190 */ 590, 340, 579, 140, 140, 140, 140, 133, 392, 564, + /* 200 */ 536, 1195, 250, 425, 1195, 250, 137, 137, 136, 136, + /* 210 */ 136, 135, 132, 463, 291, 138, 138, 138, 138, 137, + /* 220 */ 137, 136, 136, 136, 135, 132, 463, 966, 1230, 1231, + /* 230 */ 1230, 412, 965, 467, 412, 424, 467, 489, 357, 1611, + /* 240 */ 391, 138, 138, 138, 138, 137, 137, 136, 136, 136, + /* 250 */ 135, 132, 463, 463, 134, 131, 238, 555, 1076, 141, + /* 260 */ 142, 93, 593, 1254, 1254, 1085, 1088, 1075, 1075, 139, + /* 270 */ 139, 140, 140, 140, 140, 1317, 134, 131, 238, 424, + /* 280 */ 549, 1597, 1531, 333, 97, 83, 83, 140, 140, 140, + /* 290 */ 140, 138, 138, 138, 138, 137, 137, 136, 136, 136, + /* 300 */ 135, 132, 463, 141, 142, 93, 1657, 1254, 1254, 1085, + /* 310 */ 1088, 1075, 1075, 139, 139, 140, 140, 140, 140, 138, + /* 320 */ 138, 138, 138, 137, 137, 136, 136, 136, 135, 132, + /* 330 */ 463, 591, 1230, 958, 958, 138, 138, 138, 138, 137, + /* 340 */ 137, 136, 136, 136, 135, 132, 463, 44, 398, 547, + /* 350 */ 1306, 136, 136, 136, 135, 132, 463, 386, 593, 442, + /* 360 */ 595, 145, 595, 138, 138, 138, 138, 137, 137, 136, + /* 370 */ 136, 136, 135, 132, 463, 500, 1230, 112, 550, 460, + /* 380 */ 459, 51, 51, 424, 296, 296, 479, 334, 1259, 1230, + /* 390 */ 1231, 1230, 1599, 1261, 388, 312, 444, 590, 246, 579, + /* 400 */ 546, 1260, 271, 235, 329, 584, 551, 141, 142, 93, + /* 410 */ 429, 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, 140, + /* 420 */ 140, 140, 140, 22, 22, 1230, 1262, 424, 1262, 216, + /* 430 */ 296, 296, 98, 1230, 1231, 1230, 264, 884, 45, 528, + /* 440 */ 525, 524, 1041, 590, 1269, 579, 421, 420, 393, 523, + /* 450 */ 44, 141, 142, 93, 498, 1254, 1254, 1085, 1088, 1075, + /* 460 */ 1075, 139, 139, 140, 140, 140, 140, 138, 138, 138, + /* 470 */ 138, 137, 137, 136, 136, 136, 135, 132, 463, 593, + /* 480 */ 1611, 561, 1230, 1231, 1230, 23, 264, 515, 200, 528, + /* 490 */ 525, 524, 127, 585, 509, 4, 355, 487, 506, 523, + /* 500 */ 593, 498, 84, 84, 134, 131, 238, 329, 584, 588, + /* 510 */ 1627, 138, 138, 138, 138, 137, 137, 136, 136, 136, + /* 520 */ 135, 132, 463, 19, 19, 435, 1230, 1460, 297, 297, + /* 530 */ 311, 424, 1565, 464, 1631, 599, 2, 1284, 437, 574, + /* 540 */ 1107, 590, 330, 579, 158, 582, 489, 357, 573, 593, + /* 550 */ 592, 1366, 409, 1274, 1230, 141, 142, 93, 1364, 1254, + /* 560 */ 1254, 1085, 1088, 1075, 1075, 139, 139, 140, 140, 140, + /* 570 */ 140, 389, 84, 84, 1062, 567, 1230, 313, 1523, 593, + /* 580 */ 125, 125, 970, 1230, 1231, 1230, 296, 296, 126, 46, + /* 590 */ 464, 594, 464, 296, 296, 1050, 1230, 218, 439, 590, + /* 600 */ 1604, 579, 84, 84, 7, 403, 590, 515, 579, 325, + /* 610 */ 417, 1230, 1231, 1230, 250, 138, 138, 138, 138, 137, + /* 620 */ 137, 136, 136, 136, 135, 132, 463, 1050, 1050, 1052, + /* 630 */ 1053, 35, 1275, 1230, 1231, 1230, 424, 1370, 993, 574, + /* 640 */ 371, 414, 274, 412, 1597, 467, 1302, 552, 451, 590, + /* 650 */ 543, 579, 1530, 1230, 1231, 1230, 1214, 201, 409, 1174, + /* 660 */ 141, 142, 93, 223, 1254, 1254, 1085, 1088, 1075, 1075, + /* 670 */ 139, 139, 140, 140, 140, 140, 296, 296, 1250, 593, + /* 680 */ 424, 296, 296, 236, 529, 296, 296, 515, 100, 590, + /* 690 */ 1600, 579, 48, 1605, 590, 1230, 579, 7, 590, 577, + /* 700 */ 579, 904, 84, 84, 141, 142, 93, 496, 1254, 1254, + /* 710 */ 1085, 1088, 1075, 1075, 139, 139, 140, 140, 140, 140, + /* 720 */ 138, 138, 138, 138, 137, 137, 136, 136, 136, 135, + /* 730 */ 132, 463, 1365, 1230, 296, 296, 1250, 115, 1275, 326, + /* 740 */ 233, 539, 1062, 40, 282, 127, 585, 590, 4, 579, + /* 750 */ 329, 584, 1230, 1231, 1230, 1598, 593, 388, 904, 1051, + /* 760 */ 1356, 1356, 588, 1050, 138, 138, 138, 138, 137, 137, + /* 770 */ 136, 136, 136, 135, 132, 463, 185, 593, 1230, 19, + /* 780 */ 19, 1230, 971, 1597, 424, 1651, 464, 129, 908, 1195, + /* 790 */ 1230, 1231, 1230, 1325, 443, 1050, 1050, 1052, 582, 1603, + /* 800 */ 149, 149, 1195, 7, 5, 1195, 1687, 410, 141, 142, + /* 810 */ 93, 1536, 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, + /* 820 */ 140, 140, 140, 140, 1214, 397, 593, 1062, 424, 1536, + /* 830 */ 1538, 50, 901, 125, 125, 1230, 1231, 1230, 1230, 1231, + /* 840 */ 1230, 126, 1230, 464, 594, 464, 515, 1230, 1050, 84, + /* 850 */ 84, 3, 141, 142, 93, 924, 1254, 1254, 1085, 1088, + /* 860 */ 1075, 1075, 139, 139, 140, 140, 140, 140, 138, 138, + /* 870 */ 138, 138, 137, 137, 136, 136, 136, 135, 132, 463, + /* 880 */ 1050, 1050, 1052, 1053, 35, 442, 457, 532, 433, 1230, + /* 890 */ 1062, 1361, 540, 540, 1598, 925, 388, 7, 1129, 1230, + /* 900 */ 1231, 1230, 1129, 1536, 1230, 1231, 1230, 1051, 570, 1214, + /* 910 */ 593, 1050, 138, 138, 138, 138, 137, 137, 136, 136, + /* 920 */ 136, 135, 132, 463, 6, 185, 1195, 1230, 231, 593, + /* 930 */ 382, 992, 424, 151, 151, 510, 1213, 557, 482, 1195, + /* 940 */ 381, 160, 1195, 1050, 1050, 1052, 1230, 1231, 1230, 422, + /* 950 */ 593, 447, 84, 84, 593, 217, 141, 142, 93, 593, + /* 960 */ 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, 140, 140, + /* 970 */ 140, 140, 1214, 19, 19, 593, 424, 19, 19, 442, + /* 980 */ 1063, 442, 19, 19, 1230, 1231, 1230, 515, 445, 458, + /* 990 */ 1597, 386, 315, 1175, 1685, 556, 1685, 450, 84, 84, + /* 1000 */ 141, 142, 93, 505, 1254, 1254, 1085, 1088, 1075, 1075, + /* 1010 */ 139, 139, 140, 140, 140, 140, 138, 138, 138, 138, + /* 1020 */ 137, 137, 136, 136, 136, 135, 132, 463, 442, 1147, + /* 1030 */ 454, 1597, 362, 1041, 593, 462, 1460, 1233, 47, 1393, + /* 1040 */ 324, 565, 565, 115, 1148, 449, 7, 460, 459, 307, + /* 1050 */ 375, 354, 593, 113, 593, 329, 584, 19, 19, 1149, + /* 1060 */ 138, 138, 138, 138, 137, 137, 136, 136, 136, 135, + /* 1070 */ 132, 463, 209, 1173, 563, 19, 19, 19, 19, 49, + /* 1080 */ 424, 944, 1175, 1686, 1046, 1686, 218, 355, 484, 343, + /* 1090 */ 210, 945, 569, 562, 1262, 1233, 1262, 490, 314, 423, + /* 1100 */ 424, 1598, 1206, 388, 141, 142, 93, 440, 1254, 1254, + /* 1110 */ 1085, 1088, 1075, 1075, 139, 139, 140, 140, 140, 140, + /* 1120 */ 352, 316, 531, 316, 141, 142, 93, 549, 1254, 1254, + /* 1130 */ 1085, 1088, 1075, 1075, 139, 139, 140, 140, 140, 140, + /* 1140 */ 446, 10, 1598, 274, 388, 915, 281, 299, 383, 534, + /* 1150 */ 378, 533, 269, 593, 1206, 587, 587, 587, 374, 293, + /* 1160 */ 1579, 991, 1173, 302, 138, 138, 138, 138, 137, 137, + /* 1170 */ 136, 136, 136, 135, 132, 463, 53, 53, 520, 1250, + /* 1180 */ 593, 1147, 1576, 431, 138, 138, 138, 138, 137, 137, + /* 1190 */ 136, 136, 136, 135, 132, 463, 1148, 301, 593, 1577, + /* 1200 */ 593, 1307, 431, 54, 54, 593, 268, 593, 461, 461, + /* 1210 */ 461, 1149, 347, 492, 424, 135, 132, 463, 1146, 1195, + /* 1220 */ 474, 68, 68, 69, 69, 550, 332, 287, 21, 21, + /* 1230 */ 55, 55, 1195, 581, 424, 1195, 309, 1250, 141, 142, + /* 1240 */ 93, 119, 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, + /* 1250 */ 140, 140, 140, 140, 593, 237, 480, 1476, 141, 142, + /* 1260 */ 93, 593, 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, + /* 1270 */ 140, 140, 140, 140, 344, 430, 346, 70, 70, 494, + /* 1280 */ 991, 1132, 1132, 512, 56, 56, 1269, 593, 268, 593, + /* 1290 */ 369, 374, 593, 481, 215, 384, 1624, 481, 138, 138, + /* 1300 */ 138, 138, 137, 137, 136, 136, 136, 135, 132, 463, + /* 1310 */ 71, 71, 72, 72, 225, 73, 73, 593, 138, 138, + /* 1320 */ 138, 138, 137, 137, 136, 136, 136, 135, 132, 463, + /* 1330 */ 586, 431, 593, 872, 873, 874, 593, 911, 593, 1602, + /* 1340 */ 74, 74, 593, 7, 1460, 242, 593, 306, 424, 1578, + /* 1350 */ 472, 306, 364, 219, 367, 75, 75, 430, 345, 57, + /* 1360 */ 57, 58, 58, 432, 187, 59, 59, 593, 424, 61, + /* 1370 */ 61, 1475, 141, 142, 93, 123, 1254, 1254, 1085, 1088, + /* 1380 */ 1075, 1075, 139, 139, 140, 140, 140, 140, 424, 570, + /* 1390 */ 62, 62, 141, 142, 93, 911, 1254, 1254, 1085, 1088, + /* 1400 */ 1075, 1075, 139, 139, 140, 140, 140, 140, 161, 384, + /* 1410 */ 1624, 1474, 141, 130, 93, 441, 1254, 1254, 1085, 1088, + /* 1420 */ 1075, 1075, 139, 139, 140, 140, 140, 140, 267, 266, + /* 1430 */ 265, 1460, 138, 138, 138, 138, 137, 137, 136, 136, + /* 1440 */ 136, 135, 132, 463, 593, 1336, 593, 1269, 1460, 384, + /* 1450 */ 1624, 231, 138, 138, 138, 138, 137, 137, 136, 136, + /* 1460 */ 136, 135, 132, 463, 593, 163, 593, 76, 76, 77, + /* 1470 */ 77, 593, 138, 138, 138, 138, 137, 137, 136, 136, + /* 1480 */ 136, 135, 132, 463, 475, 593, 483, 78, 78, 20, + /* 1490 */ 20, 1249, 424, 491, 79, 79, 495, 422, 295, 235, + /* 1500 */ 1574, 38, 511, 896, 422, 335, 240, 422, 147, 147, + /* 1510 */ 112, 593, 424, 593, 101, 222, 991, 142, 93, 455, + /* 1520 */ 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, 140, 140, + /* 1530 */ 140, 140, 593, 39, 148, 148, 80, 80, 93, 551, + /* 1540 */ 1254, 1254, 1085, 1088, 1075, 1075, 139, 139, 140, 140, + /* 1550 */ 140, 140, 328, 923, 922, 64, 64, 502, 1656, 1005, + /* 1560 */ 933, 896, 124, 422, 121, 254, 593, 1006, 593, 226, + /* 1570 */ 593, 127, 585, 164, 4, 16, 138, 138, 138, 138, + /* 1580 */ 137, 137, 136, 136, 136, 135, 132, 463, 588, 81, + /* 1590 */ 81, 65, 65, 82, 82, 593, 138, 138, 138, 138, + /* 1600 */ 137, 137, 136, 136, 136, 135, 132, 463, 593, 226, + /* 1610 */ 237, 966, 464, 593, 298, 593, 965, 593, 66, 66, + /* 1620 */ 593, 1170, 593, 411, 582, 353, 469, 115, 593, 471, + /* 1630 */ 169, 173, 173, 593, 44, 991, 174, 174, 89, 89, + /* 1640 */ 67, 67, 593, 85, 85, 150, 150, 1114, 1043, 593, + /* 1650 */ 273, 86, 86, 1062, 593, 503, 171, 171, 593, 125, + /* 1660 */ 125, 497, 593, 273, 336, 152, 152, 126, 1335, 464, + /* 1670 */ 594, 464, 146, 146, 1050, 593, 545, 172, 172, 593, + /* 1680 */ 1054, 165, 165, 256, 339, 156, 156, 127, 585, 1586, + /* 1690 */ 4, 329, 584, 499, 358, 273, 115, 348, 155, 155, + /* 1700 */ 930, 931, 153, 153, 588, 1114, 1050, 1050, 1052, 1053, + /* 1710 */ 35, 1554, 521, 593, 270, 1008, 1009, 9, 593, 372, + /* 1720 */ 593, 115, 593, 168, 593, 115, 593, 1110, 464, 270, + /* 1730 */ 996, 964, 273, 129, 1645, 1214, 154, 154, 1054, 1404, + /* 1740 */ 582, 88, 88, 90, 90, 87, 87, 52, 52, 60, + /* 1750 */ 60, 1405, 504, 537, 559, 1179, 961, 507, 129, 558, + /* 1760 */ 127, 585, 1126, 4, 1126, 1125, 894, 1125, 162, 1062, + /* 1770 */ 963, 359, 129, 1401, 363, 125, 125, 588, 366, 368, + /* 1780 */ 370, 1349, 1334, 126, 1333, 464, 594, 464, 377, 387, + /* 1790 */ 1050, 1391, 1414, 1618, 1459, 1387, 1399, 208, 580, 1464, + /* 1800 */ 1314, 464, 243, 516, 1305, 1293, 1384, 1292, 1294, 1638, + /* 1810 */ 288, 170, 228, 582, 12, 408, 321, 322, 241, 323, + /* 1820 */ 245, 1446, 1050, 1050, 1052, 1053, 35, 559, 304, 350, + /* 1830 */ 351, 501, 560, 127, 585, 1441, 4, 1451, 1434, 310, + /* 1840 */ 1450, 526, 1062, 1332, 415, 380, 232, 1527, 125, 125, + /* 1850 */ 588, 1214, 1396, 356, 1526, 583, 126, 1397, 464, 594, + /* 1860 */ 464, 1641, 535, 1050, 1581, 1395, 1269, 1583, 1582, 213, + /* 1870 */ 402, 277, 214, 227, 464, 1573, 239, 1571, 1266, 1394, + /* 1880 */ 434, 198, 100, 224, 96, 183, 582, 191, 485, 193, + /* 1890 */ 486, 194, 195, 196, 519, 1050, 1050, 1052, 1053, 35, + /* 1900 */ 559, 113, 252, 413, 1447, 558, 493, 13, 1455, 416, + /* 1910 */ 1453, 1452, 14, 202, 1521, 1062, 1532, 508, 258, 106, + /* 1920 */ 514, 125, 125, 99, 1214, 1543, 289, 260, 206, 126, + /* 1930 */ 365, 464, 594, 464, 361, 517, 1050, 261, 448, 1295, + /* 1940 */ 262, 418, 1352, 1351, 108, 1350, 1655, 1654, 1343, 915, + /* 1950 */ 419, 1322, 233, 452, 319, 379, 1321, 453, 1623, 320, + /* 1960 */ 1320, 275, 1653, 544, 276, 1609, 1608, 1342, 1050, 1050, + /* 1970 */ 1052, 1053, 35, 1630, 1218, 466, 385, 456, 300, 1419, + /* 1980 */ 144, 1418, 570, 407, 407, 406, 284, 404, 11, 1508, + /* 1990 */ 881, 396, 120, 127, 585, 394, 4, 1214, 327, 114, + /* 2000 */ 1375, 1374, 220, 247, 400, 338, 401, 554, 42, 1224, + /* 2010 */ 588, 596, 283, 337, 285, 286, 188, 597, 1290, 1285, + /* 2020 */ 175, 1558, 176, 1559, 1557, 1556, 159, 317, 229, 177, + /* 2030 */ 868, 230, 91, 465, 464, 221, 331, 468, 1165, 470, + /* 2040 */ 473, 94, 244, 95, 249, 189, 582, 1124, 1122, 341, + /* 2050 */ 427, 190, 178, 1249, 179, 43, 192, 947, 349, 428, + /* 2060 */ 1138, 197, 251, 180, 181, 436, 102, 182, 438, 103, + /* 2070 */ 104, 199, 248, 1140, 253, 1062, 105, 255, 1137, 166, + /* 2080 */ 24, 125, 125, 257, 1264, 273, 360, 513, 259, 126, + /* 2090 */ 15, 464, 594, 464, 204, 883, 1050, 518, 263, 373, + /* 2100 */ 381, 92, 585, 1130, 4, 203, 205, 426, 107, 522, + /* 2110 */ 25, 26, 329, 584, 913, 572, 527, 376, 588, 926, + /* 2120 */ 530, 109, 184, 318, 167, 110, 27, 538, 1050, 1050, + /* 2130 */ 1052, 1053, 35, 1211, 1091, 17, 476, 111, 1181, 234, + /* 2140 */ 292, 1180, 464, 294, 207, 994, 129, 1201, 272, 1000, + /* 2150 */ 28, 1197, 29, 30, 582, 1199, 1205, 1214, 31, 1204, + /* 2160 */ 32, 1186, 41, 566, 33, 1105, 211, 8, 115, 1092, + /* 2170 */ 1090, 1094, 34, 278, 578, 1095, 117, 122, 118, 1145, + /* 2180 */ 36, 18, 128, 1062, 1055, 895, 957, 37, 589, 125, + /* 2190 */ 125, 279, 186, 280, 1646, 157, 405, 126, 1220, 464, + /* 2200 */ 594, 464, 1218, 466, 1050, 1219, 300, 1281, 1281, 1281, + /* 2210 */ 1281, 407, 407, 406, 284, 404, 1281, 1281, 881, 1281, + /* 2220 */ 300, 1281, 1281, 571, 1281, 407, 407, 406, 284, 404, + /* 2230 */ 1281, 247, 881, 338, 1281, 1281, 1050, 1050, 1052, 1053, + /* 2240 */ 35, 337, 1281, 1281, 1281, 247, 1281, 338, 1281, 1281, + /* 2250 */ 1281, 1281, 1281, 1281, 1281, 337, 1281, 1281, 1281, 1281, + /* 2260 */ 1281, 1281, 1281, 1281, 1281, 1214, 1281, 1281, 1281, 1281, + /* 2270 */ 1281, 1281, 249, 1281, 1281, 1281, 1281, 1281, 1281, 1281, + /* 2280 */ 178, 1281, 1281, 43, 1281, 1281, 249, 1281, 1281, 1281, + /* 2290 */ 1281, 1281, 1281, 1281, 178, 1281, 1281, 43, 1281, 1281, + /* 2300 */ 248, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, + /* 2310 */ 1281, 1281, 1281, 1281, 248, 1281, 1281, 1281, 1281, 1281, + /* 2320 */ 1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, + /* 2330 */ 1281, 1281, 1281, 1281, 1281, 426, 1281, 1281, 1281, 1281, + /* 2340 */ 329, 584, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 426, + /* 2350 */ 1281, 1281, 1281, 1281, 329, 584, 1281, 1281, 1281, 1281, + /* 2360 */ 1281, 1281, 1281, 1281, 476, 1281, 1281, 1281, 1281, 1281, + /* 2370 */ 1281, 1281, 1281, 1281, 1281, 1281, 1281, 1281, 476, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 277, 278, 279, 241, 242, 225, 195, 227, 195, 241, - /* 10 */ 242, 195, 217, 221, 195, 235, 254, 195, 256, 19, - /* 20 */ 225, 298, 254, 195, 256, 206, 213, 214, 206, 218, - /* 30 */ 219, 31, 206, 195, 218, 219, 195, 218, 219, 39, - /* 40 */ 218, 219, 313, 43, 44, 45, 317, 47, 48, 49, + /* 0 */ 277, 278, 279, 241, 242, 225, 195, 227, 195, 312, + /* 10 */ 195, 218, 195, 316, 195, 235, 254, 195, 256, 19, + /* 20 */ 297, 277, 278, 279, 218, 206, 213, 214, 206, 218, + /* 30 */ 219, 31, 206, 218, 219, 218, 219, 218, 219, 39, + /* 40 */ 218, 219, 195, 43, 44, 45, 195, 47, 48, 49, /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 19, - /* 60 */ 241, 242, 195, 241, 242, 195, 255, 241, 242, 277, - /* 70 */ 278, 279, 234, 254, 255, 256, 254, 255, 256, 218, - /* 80 */ 254, 240, 256, 43, 44, 45, 264, 47, 48, 49, - /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 271, - /* 100 */ 287, 22, 23, 103, 104, 105, 106, 107, 108, 109, - /* 110 */ 110, 111, 112, 113, 114, 114, 47, 48, 49, 50, + /* 60 */ 241, 242, 195, 241, 242, 195, 255, 241, 242, 195, + /* 70 */ 255, 237, 238, 254, 255, 256, 254, 255, 256, 264, + /* 80 */ 254, 207, 256, 43, 44, 45, 264, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 251, + /* 100 */ 287, 253, 215, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 113, 114, 82, 265, 195, 271, 11, /* 120 */ 187, 188, 189, 190, 191, 192, 190, 87, 192, 89, - /* 130 */ 197, 19, 199, 197, 318, 199, 320, 25, 195, 206, - /* 140 */ 299, 271, 206, 103, 104, 105, 106, 107, 108, 109, + /* 130 */ 197, 19, 199, 197, 317, 199, 319, 25, 271, 206, + /* 140 */ 218, 219, 206, 103, 104, 105, 106, 107, 108, 109, /* 150 */ 110, 111, 112, 113, 114, 43, 44, 45, 195, 47, /* 160 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 170 */ 58, 60, 21, 195, 241, 242, 215, 241, 242, 312, - /* 180 */ 313, 102, 70, 205, 317, 207, 242, 254, 77, 256, - /* 190 */ 254, 122, 256, 55, 56, 57, 58, 59, 254, 88, - /* 200 */ 256, 90, 269, 240, 93, 269, 107, 108, 109, 110, - /* 210 */ 111, 112, 113, 114, 271, 103, 104, 105, 106, 107, - /* 220 */ 108, 109, 110, 111, 112, 113, 114, 313, 117, 118, - /* 230 */ 119, 317, 81, 195, 301, 19, 195, 301, 277, 278, - /* 240 */ 279, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 250 */ 112, 113, 114, 55, 56, 57, 58, 146, 195, 43, - /* 260 */ 44, 45, 74, 47, 48, 49, 50, 51, 52, 53, - /* 270 */ 54, 55, 56, 57, 58, 124, 195, 60, 109, 110, - /* 280 */ 111, 112, 113, 114, 68, 195, 103, 104, 105, 106, - /* 290 */ 107, 108, 109, 110, 111, 112, 113, 114, 208, 218, - /* 300 */ 219, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 310 */ 112, 113, 114, 162, 233, 24, 128, 129, 130, 103, + /* 170 */ 58, 60, 139, 140, 241, 242, 289, 241, 242, 309, + /* 180 */ 310, 294, 70, 47, 48, 49, 50, 254, 77, 256, + /* 190 */ 254, 195, 256, 55, 56, 57, 58, 59, 221, 88, + /* 200 */ 109, 90, 269, 240, 93, 269, 107, 108, 109, 110, + /* 210 */ 111, 112, 113, 114, 215, 103, 104, 105, 106, 107, + /* 220 */ 108, 109, 110, 111, 112, 113, 114, 136, 117, 118, + /* 230 */ 119, 298, 141, 300, 298, 19, 300, 129, 130, 317, + /* 240 */ 318, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 250 */ 112, 113, 114, 114, 277, 278, 279, 146, 122, 43, + /* 260 */ 44, 45, 195, 47, 48, 49, 50, 51, 52, 53, + /* 270 */ 54, 55, 56, 57, 58, 218, 277, 278, 279, 19, + /* 280 */ 19, 195, 286, 23, 68, 218, 219, 55, 56, 57, + /* 290 */ 58, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 300 */ 112, 113, 114, 43, 44, 45, 232, 47, 48, 49, + /* 310 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 103, /* 320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - /* 330 */ 114, 195, 195, 215, 117, 118, 119, 120, 195, 19, - /* 340 */ 123, 124, 125, 207, 24, 74, 246, 60, 310, 311, - /* 350 */ 133, 60, 311, 82, 22, 218, 219, 257, 195, 19, - /* 360 */ 73, 218, 219, 43, 44, 45, 206, 47, 48, 49, - /* 370 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 22, - /* 380 */ 23, 218, 219, 43, 44, 45, 54, 47, 48, 49, - /* 390 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 128, - /* 400 */ 82, 241, 242, 195, 117, 118, 119, 289, 60, 118, - /* 410 */ 139, 140, 294, 195, 254, 195, 256, 195, 255, 259, - /* 420 */ 260, 73, 22, 103, 104, 105, 106, 107, 108, 109, - /* 430 */ 110, 111, 112, 113, 114, 206, 218, 219, 218, 219, - /* 440 */ 218, 219, 234, 103, 104, 105, 106, 107, 108, 109, - /* 450 */ 110, 111, 112, 113, 114, 318, 319, 139, 140, 102, - /* 460 */ 60, 318, 319, 221, 19, 117, 118, 119, 23, 195, - /* 470 */ 241, 242, 313, 255, 206, 255, 317, 255, 206, 129, - /* 480 */ 130, 206, 264, 254, 264, 256, 264, 195, 43, 44, - /* 490 */ 45, 151, 47, 48, 49, 50, 51, 52, 53, 54, - /* 500 */ 55, 56, 57, 58, 246, 213, 214, 19, 19, 241, - /* 510 */ 242, 195, 23, 241, 242, 257, 241, 242, 118, 277, - /* 520 */ 278, 279, 254, 29, 256, 60, 254, 33, 256, 254, - /* 530 */ 206, 256, 43, 44, 45, 218, 47, 48, 49, 50, - /* 540 */ 51, 52, 53, 54, 55, 56, 57, 58, 103, 104, - /* 550 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - /* 560 */ 66, 19, 218, 60, 120, 241, 242, 123, 124, 125, - /* 570 */ 60, 232, 77, 19, 20, 26, 22, 133, 254, 287, - /* 580 */ 256, 265, 117, 118, 119, 90, 312, 313, 93, 47, - /* 590 */ 36, 317, 103, 104, 105, 106, 107, 108, 109, 110, - /* 600 */ 111, 112, 113, 114, 116, 117, 277, 278, 279, 60, - /* 610 */ 107, 108, 19, 276, 60, 31, 23, 152, 195, 116, - /* 620 */ 117, 118, 119, 39, 121, 276, 72, 117, 118, 119, - /* 630 */ 166, 167, 129, 145, 237, 238, 43, 44, 45, 276, - /* 640 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 650 */ 57, 58, 315, 316, 144, 101, 19, 154, 116, 156, - /* 660 */ 23, 107, 108, 109, 315, 316, 117, 118, 119, 115, - /* 670 */ 60, 117, 118, 119, 132, 200, 122, 60, 315, 316, - /* 680 */ 43, 44, 45, 272, 47, 48, 49, 50, 51, 52, - /* 690 */ 53, 54, 55, 56, 57, 58, 103, 104, 105, 106, - /* 700 */ 107, 108, 109, 110, 111, 112, 113, 114, 154, 155, - /* 710 */ 156, 157, 158, 212, 213, 214, 22, 195, 101, 22, - /* 720 */ 60, 19, 20, 60, 22, 139, 140, 117, 118, 119, - /* 730 */ 22, 251, 195, 253, 117, 118, 195, 183, 36, 122, - /* 740 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 750 */ 113, 114, 195, 195, 60, 218, 219, 60, 195, 284, - /* 760 */ 19, 25, 60, 288, 23, 237, 238, 22, 60, 109, - /* 770 */ 233, 154, 155, 156, 72, 218, 219, 117, 118, 119, - /* 780 */ 117, 118, 119, 116, 43, 44, 45, 265, 47, 48, - /* 790 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - /* 800 */ 183, 243, 25, 101, 19, 60, 265, 144, 23, 107, - /* 810 */ 108, 117, 118, 119, 117, 118, 119, 115, 151, 117, - /* 820 */ 118, 119, 82, 195, 122, 117, 118, 119, 43, 44, - /* 830 */ 45, 195, 47, 48, 49, 50, 51, 52, 53, 54, - /* 840 */ 55, 56, 57, 58, 103, 104, 105, 106, 107, 108, - /* 850 */ 109, 110, 111, 112, 113, 114, 154, 155, 156, 157, - /* 860 */ 158, 121, 117, 118, 119, 307, 101, 309, 195, 22, - /* 870 */ 23, 195, 25, 19, 35, 139, 140, 195, 24, 139, - /* 880 */ 140, 208, 195, 118, 109, 183, 22, 122, 103, 104, - /* 890 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, - /* 900 */ 304, 305, 77, 230, 127, 232, 67, 195, 19, 195, - /* 910 */ 195, 136, 23, 88, 75, 90, 141, 203, 93, 154, - /* 920 */ 155, 156, 208, 295, 60, 243, 22, 23, 19, 25, - /* 930 */ 218, 219, 43, 44, 45, 100, 47, 48, 49, 50, - /* 940 */ 51, 52, 53, 54, 55, 56, 57, 58, 183, 102, - /* 950 */ 96, 195, 43, 44, 45, 240, 47, 48, 49, 50, - /* 960 */ 51, 52, 53, 54, 55, 56, 57, 58, 114, 134, - /* 970 */ 131, 146, 25, 286, 120, 121, 122, 123, 124, 125, - /* 980 */ 126, 117, 118, 119, 313, 195, 132, 195, 317, 307, - /* 990 */ 195, 309, 103, 104, 105, 106, 107, 108, 109, 110, - /* 1000 */ 111, 112, 113, 114, 195, 195, 102, 195, 195, 195, - /* 1010 */ 218, 219, 103, 104, 105, 106, 107, 108, 109, 110, - /* 1020 */ 111, 112, 113, 114, 77, 233, 195, 60, 218, 219, - /* 1030 */ 218, 219, 218, 219, 23, 195, 25, 90, 243, 159, - /* 1040 */ 93, 161, 19, 233, 195, 233, 23, 233, 16, 218, - /* 1050 */ 219, 195, 243, 212, 213, 214, 262, 263, 218, 219, - /* 1060 */ 195, 271, 19, 307, 233, 309, 43, 44, 45, 160, - /* 1070 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1080 */ 57, 58, 195, 218, 219, 118, 43, 44, 45, 240, - /* 1090 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1100 */ 57, 58, 307, 195, 309, 218, 219, 263, 12, 195, - /* 1110 */ 78, 267, 80, 112, 113, 114, 307, 22, 309, 24, - /* 1120 */ 255, 281, 266, 27, 107, 108, 103, 104, 105, 106, - /* 1130 */ 107, 108, 109, 110, 111, 112, 113, 114, 42, 195, - /* 1140 */ 11, 22, 255, 24, 195, 195, 103, 104, 105, 106, - /* 1150 */ 107, 108, 109, 110, 111, 112, 113, 114, 19, 195, - /* 1160 */ 64, 195, 218, 219, 195, 313, 195, 218, 219, 317, - /* 1170 */ 74, 154, 195, 156, 195, 195, 19, 233, 23, 60, - /* 1180 */ 25, 24, 218, 219, 218, 219, 195, 218, 219, 218, - /* 1190 */ 219, 128, 129, 130, 162, 263, 19, 218, 219, 267, - /* 1200 */ 43, 44, 45, 160, 47, 48, 49, 50, 51, 52, - /* 1210 */ 53, 54, 55, 56, 57, 58, 19, 240, 228, 255, - /* 1220 */ 43, 44, 45, 25, 47, 48, 49, 50, 51, 52, - /* 1230 */ 53, 54, 55, 56, 57, 58, 135, 118, 137, 138, - /* 1240 */ 43, 44, 45, 22, 47, 48, 49, 50, 51, 52, - /* 1250 */ 53, 54, 55, 56, 57, 58, 117, 266, 129, 130, - /* 1260 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 1270 */ 113, 114, 195, 195, 119, 295, 195, 206, 195, 195, - /* 1280 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 1290 */ 113, 114, 195, 195, 195, 218, 219, 195, 195, 144, - /* 1300 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 1310 */ 113, 114, 241, 242, 67, 218, 219, 218, 219, 146, - /* 1320 */ 19, 218, 219, 240, 215, 254, 136, 256, 107, 108, - /* 1330 */ 195, 141, 255, 86, 128, 129, 130, 195, 165, 195, - /* 1340 */ 19, 143, 95, 272, 25, 44, 45, 266, 47, 48, - /* 1350 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - /* 1360 */ 218, 219, 218, 219, 195, 12, 45, 195, 47, 48, - /* 1370 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - /* 1380 */ 27, 23, 7, 8, 9, 210, 211, 218, 219, 116, - /* 1390 */ 218, 219, 228, 16, 147, 42, 195, 295, 195, 19, - /* 1400 */ 20, 266, 22, 294, 103, 104, 105, 106, 107, 108, - /* 1410 */ 109, 110, 111, 112, 113, 114, 36, 64, 145, 218, - /* 1420 */ 219, 218, 219, 195, 103, 104, 105, 106, 107, 108, - /* 1430 */ 109, 110, 111, 112, 113, 114, 195, 154, 119, 156, - /* 1440 */ 60, 189, 190, 191, 192, 195, 218, 219, 195, 197, - /* 1450 */ 195, 199, 72, 195, 19, 78, 195, 80, 206, 218, - /* 1460 */ 219, 195, 82, 144, 210, 211, 195, 15, 218, 219, - /* 1470 */ 47, 218, 219, 218, 219, 259, 260, 195, 261, 218, - /* 1480 */ 219, 101, 302, 303, 218, 219, 195, 107, 108, 218, - /* 1490 */ 219, 150, 151, 241, 242, 115, 25, 117, 118, 119, - /* 1500 */ 218, 219, 122, 195, 146, 195, 254, 195, 256, 218, - /* 1510 */ 219, 246, 25, 61, 246, 19, 20, 195, 22, 139, - /* 1520 */ 140, 269, 257, 195, 266, 257, 218, 219, 218, 219, - /* 1530 */ 218, 219, 36, 246, 154, 155, 156, 157, 158, 116, - /* 1540 */ 218, 219, 195, 22, 257, 49, 218, 219, 23, 195, - /* 1550 */ 25, 195, 117, 301, 195, 25, 60, 195, 195, 23, - /* 1560 */ 195, 25, 195, 183, 24, 218, 219, 130, 72, 195, - /* 1570 */ 22, 195, 218, 219, 218, 219, 195, 218, 219, 195, - /* 1580 */ 218, 219, 86, 218, 219, 218, 219, 91, 19, 20, - /* 1590 */ 153, 22, 218, 219, 218, 219, 195, 101, 195, 218, - /* 1600 */ 219, 195, 195, 107, 108, 36, 23, 195, 25, 195, - /* 1610 */ 62, 115, 195, 117, 118, 119, 195, 146, 122, 218, - /* 1620 */ 219, 218, 219, 195, 218, 219, 19, 60, 122, 60, - /* 1630 */ 218, 219, 218, 219, 195, 218, 219, 150, 132, 218, - /* 1640 */ 219, 72, 195, 23, 195, 25, 218, 219, 195, 60, - /* 1650 */ 154, 155, 156, 157, 158, 86, 23, 195, 25, 195, - /* 1660 */ 91, 19, 20, 142, 22, 218, 219, 218, 219, 130, - /* 1670 */ 101, 218, 219, 143, 121, 122, 107, 108, 36, 183, - /* 1680 */ 218, 219, 142, 60, 115, 118, 117, 118, 119, 7, - /* 1690 */ 8, 122, 153, 23, 23, 25, 25, 23, 23, 25, - /* 1700 */ 25, 23, 60, 25, 23, 98, 25, 118, 84, 85, - /* 1710 */ 23, 23, 25, 25, 72, 154, 23, 156, 25, 23, - /* 1720 */ 228, 25, 195, 154, 155, 156, 157, 158, 86, 195, - /* 1730 */ 195, 258, 195, 91, 291, 322, 195, 195, 195, 195, - /* 1740 */ 195, 118, 195, 101, 195, 195, 195, 195, 238, 107, - /* 1750 */ 108, 195, 183, 195, 195, 195, 290, 115, 195, 117, - /* 1760 */ 118, 119, 244, 195, 122, 195, 195, 195, 195, 195, - /* 1770 */ 195, 258, 258, 258, 258, 193, 245, 300, 216, 274, - /* 1780 */ 247, 270, 270, 274, 296, 296, 248, 222, 262, 198, - /* 1790 */ 262, 274, 61, 274, 248, 231, 154, 155, 156, 157, - /* 1800 */ 158, 0, 1, 2, 247, 227, 5, 221, 221, 221, - /* 1810 */ 142, 10, 11, 12, 13, 14, 262, 262, 17, 202, - /* 1820 */ 300, 19, 20, 300, 22, 183, 247, 251, 251, 245, - /* 1830 */ 202, 30, 38, 32, 202, 152, 151, 22, 36, 43, - /* 1840 */ 236, 40, 18, 202, 239, 239, 18, 239, 239, 283, - /* 1850 */ 201, 150, 236, 202, 236, 201, 159, 202, 248, 248, - /* 1860 */ 248, 248, 60, 63, 201, 275, 273, 275, 273, 275, - /* 1870 */ 22, 286, 71, 223, 72, 202, 223, 297, 297, 202, - /* 1880 */ 79, 201, 116, 82, 220, 201, 220, 220, 65, 293, - /* 1890 */ 292, 229, 22, 166, 127, 226, 24, 114, 226, 223, - /* 1900 */ 99, 222, 202, 101, 285, 92, 220, 308, 83, 107, - /* 1910 */ 108, 220, 220, 316, 220, 285, 268, 115, 229, 117, - /* 1920 */ 118, 119, 223, 321, 122, 268, 149, 146, 22, 19, - /* 1930 */ 20, 202, 22, 159, 282, 134, 321, 148, 280, 147, - /* 1940 */ 139, 140, 252, 141, 25, 204, 36, 252, 13, 251, - /* 1950 */ 196, 248, 250, 249, 196, 6, 154, 155, 156, 157, - /* 1960 */ 158, 209, 194, 194, 163, 194, 306, 306, 303, 224, - /* 1970 */ 60, 215, 215, 209, 215, 215, 215, 224, 216, 216, - /* 1980 */ 4, 209, 72, 3, 22, 183, 164, 15, 23, 16, - /* 1990 */ 23, 140, 152, 131, 25, 24, 143, 20, 16, 145, - /* 2000 */ 1, 143, 131, 62, 131, 37, 54, 152, 54, 54, - /* 2010 */ 54, 101, 131, 117, 1, 34, 142, 107, 108, 5, - /* 2020 */ 22, 116, 162, 76, 41, 115, 69, 117, 118, 119, - /* 2030 */ 1, 2, 122, 25, 5, 69, 142, 116, 20, 10, - /* 2040 */ 11, 12, 13, 14, 24, 19, 17, 132, 5, 126, - /* 2050 */ 22, 141, 68, 10, 11, 12, 13, 14, 22, 30, - /* 2060 */ 17, 32, 22, 22, 154, 155, 156, 157, 158, 40, - /* 2070 */ 23, 68, 60, 30, 24, 32, 97, 28, 22, 68, - /* 2080 */ 23, 37, 34, 40, 150, 22, 25, 23, 23, 23, - /* 2090 */ 22, 98, 142, 183, 23, 23, 34, 22, 25, 89, - /* 2100 */ 71, 34, 117, 144, 34, 22, 76, 76, 79, 87, - /* 2110 */ 34, 82, 34, 44, 71, 94, 34, 23, 25, 24, - /* 2120 */ 34, 25, 79, 23, 23, 82, 23, 23, 99, 143, - /* 2130 */ 143, 22, 25, 25, 23, 22, 11, 22, 22, 25, - /* 2140 */ 23, 23, 99, 22, 22, 136, 142, 142, 142, 25, - /* 2150 */ 23, 15, 1, 1, 323, 323, 323, 323, 323, 323, - /* 2160 */ 323, 323, 323, 134, 323, 323, 323, 323, 139, 140, - /* 2170 */ 323, 323, 323, 323, 323, 323, 323, 134, 323, 323, - /* 2180 */ 323, 323, 139, 140, 323, 323, 323, 323, 323, 323, - /* 2190 */ 323, 323, 163, 323, 323, 323, 323, 323, 323, 323, - /* 2200 */ 323, 323, 323, 323, 323, 323, 163, 323, 323, 323, - /* 2210 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2220 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2230 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2240 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2250 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2260 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2270 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2280 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2290 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2300 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2310 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2320 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2330 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, - /* 2340 */ 323, 187, 187, 187, 187, 187, 187, 187, 187, 187, - /* 2350 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, - /* 2360 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, - /* 2370 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, - /* 2380 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, - /* 2390 */ 187, 187, 187, 187, + /* 330 */ 114, 135, 60, 137, 138, 103, 104, 105, 106, 107, + /* 340 */ 108, 109, 110, 111, 112, 113, 114, 82, 281, 206, + /* 350 */ 195, 109, 110, 111, 112, 113, 114, 195, 195, 195, + /* 360 */ 205, 22, 207, 103, 104, 105, 106, 107, 108, 109, + /* 370 */ 110, 111, 112, 113, 114, 195, 60, 116, 117, 107, + /* 380 */ 108, 218, 219, 19, 241, 242, 121, 23, 116, 117, + /* 390 */ 118, 119, 306, 121, 308, 206, 234, 254, 15, 256, + /* 400 */ 195, 129, 259, 260, 139, 140, 145, 43, 44, 45, + /* 410 */ 200, 47, 48, 49, 50, 51, 52, 53, 54, 55, + /* 420 */ 56, 57, 58, 218, 219, 60, 154, 19, 156, 265, + /* 430 */ 241, 242, 24, 117, 118, 119, 120, 21, 73, 123, + /* 440 */ 124, 125, 74, 254, 61, 256, 107, 108, 221, 133, + /* 450 */ 82, 43, 44, 45, 195, 47, 48, 49, 50, 51, + /* 460 */ 52, 53, 54, 55, 56, 57, 58, 103, 104, 105, + /* 470 */ 106, 107, 108, 109, 110, 111, 112, 113, 114, 195, + /* 480 */ 317, 318, 117, 118, 119, 22, 120, 195, 22, 123, + /* 490 */ 124, 125, 19, 20, 284, 22, 128, 81, 288, 133, + /* 500 */ 195, 195, 218, 219, 277, 278, 279, 139, 140, 36, + /* 510 */ 195, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 520 */ 112, 113, 114, 218, 219, 62, 60, 195, 241, 242, + /* 530 */ 271, 19, 240, 60, 189, 190, 191, 192, 233, 255, + /* 540 */ 124, 254, 197, 256, 199, 72, 129, 130, 264, 195, + /* 550 */ 195, 206, 22, 23, 60, 43, 44, 45, 206, 47, + /* 560 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 570 */ 58, 195, 218, 219, 101, 195, 60, 271, 162, 195, + /* 580 */ 107, 108, 109, 117, 118, 119, 241, 242, 115, 73, + /* 590 */ 117, 118, 119, 241, 242, 122, 60, 195, 266, 254, + /* 600 */ 312, 256, 218, 219, 316, 203, 254, 195, 256, 255, + /* 610 */ 208, 117, 118, 119, 269, 103, 104, 105, 106, 107, + /* 620 */ 108, 109, 110, 111, 112, 113, 114, 154, 155, 156, + /* 630 */ 157, 158, 102, 117, 118, 119, 19, 242, 144, 255, + /* 640 */ 23, 206, 24, 298, 195, 300, 206, 195, 264, 254, + /* 650 */ 206, 256, 240, 117, 118, 119, 183, 22, 22, 23, + /* 660 */ 43, 44, 45, 151, 47, 48, 49, 50, 51, 52, + /* 670 */ 53, 54, 55, 56, 57, 58, 241, 242, 60, 195, + /* 680 */ 19, 241, 242, 195, 23, 241, 242, 195, 152, 254, + /* 690 */ 310, 256, 243, 312, 254, 60, 256, 316, 254, 206, + /* 700 */ 256, 60, 218, 219, 43, 44, 45, 272, 47, 48, + /* 710 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 720 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 730 */ 113, 114, 240, 60, 241, 242, 118, 25, 102, 255, + /* 740 */ 166, 167, 101, 22, 26, 19, 20, 254, 22, 256, + /* 750 */ 139, 140, 117, 118, 119, 306, 195, 308, 117, 118, + /* 760 */ 237, 238, 36, 122, 103, 104, 105, 106, 107, 108, + /* 770 */ 109, 110, 111, 112, 113, 114, 195, 195, 60, 218, + /* 780 */ 219, 60, 109, 195, 19, 217, 60, 25, 23, 77, + /* 790 */ 117, 118, 119, 225, 233, 154, 155, 156, 72, 312, + /* 800 */ 218, 219, 90, 316, 22, 93, 303, 304, 43, 44, + /* 810 */ 45, 195, 47, 48, 49, 50, 51, 52, 53, 54, + /* 820 */ 55, 56, 57, 58, 183, 195, 195, 101, 19, 213, + /* 830 */ 214, 243, 23, 107, 108, 117, 118, 119, 117, 118, + /* 840 */ 119, 115, 60, 117, 118, 119, 195, 60, 122, 218, + /* 850 */ 219, 22, 43, 44, 45, 35, 47, 48, 49, 50, + /* 860 */ 51, 52, 53, 54, 55, 56, 57, 58, 103, 104, + /* 870 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 880 */ 154, 155, 156, 157, 158, 195, 255, 67, 195, 60, + /* 890 */ 101, 240, 311, 312, 306, 75, 308, 316, 29, 117, + /* 900 */ 118, 119, 33, 287, 117, 118, 119, 118, 146, 183, + /* 910 */ 195, 122, 103, 104, 105, 106, 107, 108, 109, 110, + /* 920 */ 111, 112, 113, 114, 215, 195, 77, 60, 25, 195, + /* 930 */ 122, 144, 19, 218, 219, 66, 23, 88, 246, 90, + /* 940 */ 132, 25, 93, 154, 155, 156, 117, 118, 119, 257, + /* 950 */ 195, 131, 218, 219, 195, 265, 43, 44, 45, 195, + /* 960 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 970 */ 57, 58, 183, 218, 219, 195, 19, 218, 219, 195, + /* 980 */ 23, 195, 218, 219, 117, 118, 119, 195, 233, 255, + /* 990 */ 195, 195, 233, 22, 23, 146, 25, 233, 218, 219, + /* 1000 */ 43, 44, 45, 294, 47, 48, 49, 50, 51, 52, + /* 1010 */ 53, 54, 55, 56, 57, 58, 103, 104, 105, 106, + /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 114, 195, 12, + /* 1030 */ 234, 195, 240, 74, 195, 255, 195, 60, 243, 262, + /* 1040 */ 263, 311, 312, 25, 27, 19, 316, 107, 108, 265, + /* 1050 */ 24, 265, 195, 150, 195, 139, 140, 218, 219, 42, + /* 1060 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1070 */ 113, 114, 233, 102, 67, 218, 219, 218, 219, 243, + /* 1080 */ 19, 64, 22, 23, 23, 25, 195, 128, 129, 130, + /* 1090 */ 233, 74, 233, 86, 154, 118, 156, 130, 265, 208, + /* 1100 */ 19, 306, 95, 308, 43, 44, 45, 266, 47, 48, + /* 1110 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 1120 */ 153, 230, 96, 232, 43, 44, 45, 19, 47, 48, + /* 1130 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 1140 */ 114, 22, 306, 24, 308, 127, 120, 121, 122, 123, + /* 1150 */ 124, 125, 126, 195, 147, 212, 213, 214, 132, 23, + /* 1160 */ 195, 25, 102, 100, 103, 104, 105, 106, 107, 108, + /* 1170 */ 109, 110, 111, 112, 113, 114, 218, 219, 19, 60, + /* 1180 */ 195, 12, 210, 211, 103, 104, 105, 106, 107, 108, + /* 1190 */ 109, 110, 111, 112, 113, 114, 27, 134, 195, 195, + /* 1200 */ 195, 210, 211, 218, 219, 195, 47, 195, 212, 213, + /* 1210 */ 214, 42, 16, 130, 19, 112, 113, 114, 23, 77, + /* 1220 */ 195, 218, 219, 218, 219, 117, 163, 164, 218, 219, + /* 1230 */ 218, 219, 90, 64, 19, 93, 153, 118, 43, 44, + /* 1240 */ 45, 160, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1250 */ 55, 56, 57, 58, 195, 119, 272, 276, 43, 44, + /* 1260 */ 45, 195, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1270 */ 55, 56, 57, 58, 78, 116, 80, 218, 219, 116, + /* 1280 */ 144, 128, 129, 130, 218, 219, 61, 195, 47, 195, + /* 1290 */ 16, 132, 195, 263, 195, 314, 315, 267, 103, 104, + /* 1300 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 1310 */ 218, 219, 218, 219, 151, 218, 219, 195, 103, 104, + /* 1320 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 1330 */ 210, 211, 195, 7, 8, 9, 195, 60, 195, 312, + /* 1340 */ 218, 219, 195, 316, 195, 120, 195, 263, 19, 195, + /* 1350 */ 125, 267, 78, 24, 80, 218, 219, 116, 162, 218, + /* 1360 */ 219, 218, 219, 301, 302, 218, 219, 195, 19, 218, + /* 1370 */ 219, 276, 43, 44, 45, 160, 47, 48, 49, 50, + /* 1380 */ 51, 52, 53, 54, 55, 56, 57, 58, 19, 146, + /* 1390 */ 218, 219, 43, 44, 45, 118, 47, 48, 49, 50, + /* 1400 */ 51, 52, 53, 54, 55, 56, 57, 58, 165, 314, + /* 1410 */ 315, 276, 43, 44, 45, 266, 47, 48, 49, 50, + /* 1420 */ 51, 52, 53, 54, 55, 56, 57, 58, 128, 129, + /* 1430 */ 130, 195, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1440 */ 111, 112, 113, 114, 195, 228, 195, 61, 195, 314, + /* 1450 */ 315, 25, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1460 */ 111, 112, 113, 114, 195, 22, 195, 218, 219, 218, + /* 1470 */ 219, 195, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1480 */ 111, 112, 113, 114, 195, 195, 246, 218, 219, 218, + /* 1490 */ 219, 25, 19, 246, 218, 219, 246, 257, 259, 260, + /* 1500 */ 195, 22, 266, 60, 257, 195, 120, 257, 218, 219, + /* 1510 */ 116, 195, 19, 195, 150, 151, 25, 44, 45, 266, + /* 1520 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1530 */ 57, 58, 195, 54, 218, 219, 218, 219, 45, 145, + /* 1540 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1550 */ 57, 58, 246, 121, 122, 218, 219, 19, 23, 31, + /* 1560 */ 25, 118, 159, 257, 161, 24, 195, 39, 195, 143, + /* 1570 */ 195, 19, 20, 22, 22, 24, 103, 104, 105, 106, + /* 1580 */ 107, 108, 109, 110, 111, 112, 113, 114, 36, 218, + /* 1590 */ 219, 218, 219, 218, 219, 195, 103, 104, 105, 106, + /* 1600 */ 107, 108, 109, 110, 111, 112, 113, 114, 195, 143, + /* 1610 */ 119, 136, 60, 195, 22, 195, 141, 195, 218, 219, + /* 1620 */ 195, 23, 195, 25, 72, 23, 131, 25, 195, 134, + /* 1630 */ 23, 218, 219, 195, 82, 144, 218, 219, 218, 219, + /* 1640 */ 218, 219, 195, 218, 219, 218, 219, 60, 23, 195, + /* 1650 */ 25, 218, 219, 101, 195, 117, 218, 219, 195, 107, + /* 1660 */ 108, 23, 195, 25, 195, 218, 219, 115, 228, 117, + /* 1670 */ 118, 119, 218, 219, 122, 195, 19, 218, 219, 195, + /* 1680 */ 60, 218, 219, 142, 195, 218, 219, 19, 20, 195, + /* 1690 */ 22, 139, 140, 23, 23, 25, 25, 195, 218, 219, + /* 1700 */ 7, 8, 218, 219, 36, 118, 154, 155, 156, 157, + /* 1710 */ 158, 195, 23, 195, 25, 84, 85, 49, 195, 23, + /* 1720 */ 195, 25, 195, 23, 195, 25, 195, 23, 60, 25, + /* 1730 */ 23, 23, 25, 25, 142, 183, 218, 219, 118, 195, + /* 1740 */ 72, 218, 219, 218, 219, 218, 219, 218, 219, 218, + /* 1750 */ 219, 195, 195, 146, 86, 98, 23, 195, 25, 91, + /* 1760 */ 19, 20, 154, 22, 156, 154, 23, 156, 25, 101, + /* 1770 */ 23, 195, 25, 195, 195, 107, 108, 36, 195, 195, + /* 1780 */ 195, 195, 228, 115, 195, 117, 118, 119, 195, 195, + /* 1790 */ 122, 261, 195, 321, 195, 195, 195, 258, 238, 195, + /* 1800 */ 195, 60, 299, 291, 195, 195, 258, 195, 195, 195, + /* 1810 */ 290, 244, 216, 72, 245, 193, 258, 258, 299, 258, + /* 1820 */ 299, 274, 154, 155, 156, 157, 158, 86, 247, 295, + /* 1830 */ 248, 295, 91, 19, 20, 270, 22, 274, 270, 248, + /* 1840 */ 274, 222, 101, 227, 274, 221, 231, 221, 107, 108, + /* 1850 */ 36, 183, 262, 247, 221, 283, 115, 262, 117, 118, + /* 1860 */ 119, 198, 116, 122, 220, 262, 61, 220, 220, 251, + /* 1870 */ 247, 142, 251, 245, 60, 202, 299, 202, 38, 262, + /* 1880 */ 202, 22, 152, 151, 296, 43, 72, 236, 18, 239, + /* 1890 */ 202, 239, 239, 239, 18, 154, 155, 156, 157, 158, + /* 1900 */ 86, 150, 201, 248, 275, 91, 248, 273, 236, 248, + /* 1910 */ 275, 275, 273, 236, 248, 101, 286, 202, 201, 159, + /* 1920 */ 63, 107, 108, 296, 183, 293, 202, 201, 22, 115, + /* 1930 */ 202, 117, 118, 119, 292, 223, 122, 201, 65, 202, + /* 1940 */ 201, 223, 220, 220, 22, 220, 226, 226, 229, 127, + /* 1950 */ 223, 220, 166, 24, 285, 220, 222, 114, 315, 285, + /* 1960 */ 220, 202, 220, 307, 92, 320, 320, 229, 154, 155, + /* 1970 */ 156, 157, 158, 0, 1, 2, 223, 83, 5, 268, + /* 1980 */ 149, 268, 146, 10, 11, 12, 13, 14, 22, 280, + /* 1990 */ 17, 202, 159, 19, 20, 251, 22, 183, 282, 148, + /* 2000 */ 252, 252, 250, 30, 249, 32, 248, 147, 25, 13, + /* 2010 */ 36, 204, 196, 40, 196, 6, 302, 194, 194, 194, + /* 2020 */ 209, 215, 209, 215, 215, 215, 224, 224, 216, 209, + /* 2030 */ 4, 216, 215, 3, 60, 22, 122, 19, 122, 19, + /* 2040 */ 125, 22, 15, 22, 71, 16, 72, 23, 23, 140, + /* 2050 */ 305, 152, 79, 25, 131, 82, 143, 20, 16, 305, + /* 2060 */ 1, 143, 145, 131, 131, 62, 54, 131, 37, 54, + /* 2070 */ 54, 152, 99, 117, 34, 101, 54, 24, 1, 5, + /* 2080 */ 22, 107, 108, 116, 76, 25, 162, 41, 142, 115, + /* 2090 */ 24, 117, 118, 119, 116, 20, 122, 19, 126, 23, + /* 2100 */ 132, 19, 20, 69, 22, 69, 22, 134, 22, 68, + /* 2110 */ 22, 22, 139, 140, 60, 141, 68, 24, 36, 28, + /* 2120 */ 97, 22, 37, 68, 23, 150, 34, 22, 154, 155, + /* 2130 */ 156, 157, 158, 23, 23, 22, 163, 25, 23, 142, + /* 2140 */ 23, 98, 60, 23, 22, 144, 25, 76, 34, 117, + /* 2150 */ 34, 89, 34, 34, 72, 87, 76, 183, 34, 94, + /* 2160 */ 34, 23, 22, 24, 34, 23, 25, 44, 25, 23, + /* 2170 */ 23, 23, 22, 22, 25, 11, 143, 25, 143, 23, + /* 2180 */ 22, 22, 22, 101, 23, 23, 136, 22, 25, 107, + /* 2190 */ 108, 142, 25, 142, 142, 23, 15, 115, 1, 117, + /* 2200 */ 118, 119, 1, 2, 122, 1, 5, 322, 322, 322, + /* 2210 */ 322, 10, 11, 12, 13, 14, 322, 322, 17, 322, + /* 2220 */ 5, 322, 322, 141, 322, 10, 11, 12, 13, 14, + /* 2230 */ 322, 30, 17, 32, 322, 322, 154, 155, 156, 157, + /* 2240 */ 158, 40, 322, 322, 322, 30, 322, 32, 322, 322, + /* 2250 */ 322, 322, 322, 322, 322, 40, 322, 322, 322, 322, + /* 2260 */ 322, 322, 322, 322, 322, 183, 322, 322, 322, 322, + /* 2270 */ 322, 322, 71, 322, 322, 322, 322, 322, 322, 322, + /* 2280 */ 79, 322, 322, 82, 322, 322, 71, 322, 322, 322, + /* 2290 */ 322, 322, 322, 322, 79, 322, 322, 82, 322, 322, + /* 2300 */ 99, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2310 */ 322, 322, 322, 322, 99, 322, 322, 322, 322, 322, + /* 2320 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2330 */ 322, 322, 322, 322, 322, 134, 322, 322, 322, 322, + /* 2340 */ 139, 140, 322, 322, 322, 322, 322, 322, 322, 134, + /* 2350 */ 322, 322, 322, 322, 139, 140, 322, 322, 322, 322, + /* 2360 */ 322, 322, 322, 322, 163, 322, 322, 322, 322, 322, + /* 2370 */ 322, 322, 322, 322, 322, 322, 322, 322, 163, 322, + /* 2380 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2390 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2400 */ 322, 322, 322, 322, 322, 322, 322, 322, 187, 187, + /* 2410 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2420 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2430 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2440 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2450 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2460 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2470 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2480 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2490 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2500 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2510 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2520 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2530 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2540 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2550 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2560 */ 187, 187, 187, 187, 187, 187, }; -#define YY_SHIFT_COUNT (582) +#define YY_SHIFT_COUNT (599) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2152) +#define YY_SHIFT_MAX (2215) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 2029, 1801, 2043, 1380, 1380, 318, 271, 1496, 1569, 1642, - /* 10 */ 702, 702, 702, 740, 318, 318, 318, 318, 318, 0, - /* 20 */ 0, 216, 1177, 702, 702, 702, 702, 702, 702, 702, - /* 30 */ 702, 702, 702, 702, 702, 702, 702, 702, 503, 503, - /* 40 */ 111, 111, 217, 287, 348, 610, 610, 736, 736, 736, - /* 50 */ 736, 40, 112, 320, 340, 445, 489, 593, 637, 741, - /* 60 */ 785, 889, 909, 1023, 1043, 1157, 1177, 1177, 1177, 1177, - /* 70 */ 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, - /* 80 */ 1177, 1177, 1177, 1177, 1197, 1177, 1301, 1321, 1321, 554, - /* 90 */ 1802, 1910, 702, 702, 702, 702, 702, 702, 702, 702, - /* 100 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 110 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 120 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 130 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, - /* 140 */ 702, 702, 138, 198, 198, 198, 198, 198, 198, 198, - /* 150 */ 183, 99, 169, 549, 610, 151, 542, 610, 610, 1017, - /* 160 */ 1017, 610, 1001, 350, 464, 464, 464, 586, 1, 1, - /* 170 */ 2207, 2207, 854, 854, 854, 465, 694, 694, 694, 694, - /* 180 */ 1096, 1096, 825, 549, 847, 904, 610, 610, 610, 610, - /* 190 */ 610, 610, 610, 610, 610, 610, 610, 610, 610, 610, - /* 200 */ 610, 610, 610, 610, 610, 488, 947, 947, 610, 1129, - /* 210 */ 495, 495, 1139, 1139, 967, 967, 1173, 2207, 2207, 2207, - /* 220 */ 2207, 2207, 2207, 2207, 617, 765, 765, 697, 444, 708, - /* 230 */ 660, 745, 510, 663, 864, 610, 610, 610, 610, 610, - /* 240 */ 610, 610, 610, 610, 610, 188, 610, 610, 610, 610, - /* 250 */ 610, 610, 610, 610, 610, 610, 610, 610, 839, 839, - /* 260 */ 839, 610, 610, 610, 1155, 610, 610, 610, 1119, 1247, - /* 270 */ 610, 1353, 610, 610, 610, 610, 610, 610, 610, 610, - /* 280 */ 1063, 494, 1101, 291, 291, 291, 291, 1319, 1101, 1101, - /* 290 */ 775, 1221, 1375, 1452, 667, 1341, 1198, 1341, 1435, 1487, - /* 300 */ 667, 667, 1487, 667, 1198, 1435, 777, 1011, 1423, 584, - /* 310 */ 584, 584, 1273, 1273, 1273, 1273, 1471, 1471, 880, 1530, - /* 320 */ 1190, 1095, 1731, 1731, 1668, 1668, 1794, 1794, 1668, 1683, - /* 330 */ 1685, 1815, 1796, 1824, 1824, 1824, 1824, 1668, 1828, 1701, - /* 340 */ 1685, 1685, 1701, 1815, 1796, 1701, 1796, 1701, 1668, 1828, - /* 350 */ 1697, 1800, 1668, 1828, 1848, 1668, 1828, 1668, 1828, 1848, - /* 360 */ 1766, 1766, 1766, 1823, 1870, 1870, 1848, 1766, 1767, 1766, - /* 370 */ 1823, 1766, 1766, 1727, 1872, 1783, 1783, 1848, 1668, 1813, - /* 380 */ 1813, 1825, 1825, 1777, 1781, 1906, 1668, 1774, 1777, 1789, - /* 390 */ 1792, 1701, 1919, 1935, 1935, 1949, 1949, 1949, 2207, 2207, - /* 400 */ 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, - /* 410 */ 2207, 2207, 2207, 69, 1032, 79, 357, 1377, 1206, 400, - /* 420 */ 1525, 835, 332, 1540, 1437, 1539, 1536, 1548, 1583, 1620, - /* 430 */ 1633, 1670, 1671, 1674, 1567, 1553, 1682, 1506, 1675, 1358, - /* 440 */ 1607, 1589, 1678, 1681, 1624, 1687, 1688, 1283, 1561, 1693, - /* 450 */ 1696, 1623, 1521, 1976, 1980, 1962, 1822, 1972, 1973, 1965, - /* 460 */ 1967, 1851, 1840, 1862, 1969, 1969, 1971, 1853, 1977, 1854, - /* 470 */ 1982, 1999, 1858, 1871, 1969, 1873, 1941, 1968, 1969, 1855, - /* 480 */ 1952, 1954, 1955, 1956, 1881, 1896, 1981, 1874, 2013, 2014, - /* 490 */ 1998, 1905, 1860, 1957, 2008, 1966, 1947, 1983, 1894, 1921, - /* 500 */ 2020, 2018, 2026, 1915, 1923, 2028, 1984, 2036, 2040, 2047, - /* 510 */ 2041, 2003, 2012, 2050, 1979, 2049, 2056, 2011, 2044, 2057, - /* 520 */ 2048, 1934, 2063, 2064, 2065, 2061, 2066, 2068, 1993, 1950, - /* 530 */ 2071, 2072, 1985, 2062, 2075, 1959, 2073, 2067, 2070, 2076, - /* 540 */ 2078, 2010, 2030, 2022, 2069, 2031, 2021, 2082, 2094, 2083, - /* 550 */ 2095, 2093, 2096, 2086, 1986, 1987, 2100, 2073, 2101, 2103, - /* 560 */ 2104, 2109, 2107, 2108, 2111, 2113, 2125, 2115, 2116, 2117, - /* 570 */ 2118, 2121, 2122, 2114, 2009, 2004, 2005, 2006, 2124, 2127, - /* 580 */ 2136, 2151, 2152, + /* 0 */ 2201, 1973, 2215, 1552, 1552, 33, 368, 1668, 1741, 1814, + /* 10 */ 726, 726, 726, 265, 33, 33, 33, 33, 33, 0, + /* 20 */ 0, 216, 1349, 726, 726, 726, 726, 726, 726, 726, + /* 30 */ 726, 726, 726, 726, 726, 726, 726, 726, 272, 272, + /* 40 */ 111, 111, 316, 365, 516, 867, 867, 916, 916, 916, + /* 50 */ 916, 40, 112, 260, 364, 408, 512, 617, 661, 765, + /* 60 */ 809, 913, 957, 1061, 1081, 1195, 1215, 1329, 1349, 1349, + /* 70 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, + /* 80 */ 1349, 1349, 1349, 1349, 1349, 1349, 1369, 1349, 1473, 1493, + /* 90 */ 1493, 473, 1974, 2082, 726, 726, 726, 726, 726, 726, + /* 100 */ 726, 726, 726, 726, 726, 726, 726, 726, 726, 726, + /* 110 */ 726, 726, 726, 726, 726, 726, 726, 726, 726, 726, + /* 120 */ 726, 726, 726, 726, 726, 726, 726, 726, 726, 726, + /* 130 */ 726, 726, 726, 726, 726, 726, 726, 726, 726, 726, + /* 140 */ 726, 726, 726, 726, 726, 726, 138, 232, 232, 232, + /* 150 */ 232, 232, 232, 232, 188, 99, 242, 718, 416, 1159, + /* 160 */ 867, 867, 940, 940, 867, 1103, 417, 574, 574, 574, + /* 170 */ 611, 139, 139, 2379, 2379, 1026, 1026, 1026, 536, 466, + /* 180 */ 466, 466, 466, 1017, 1017, 849, 718, 971, 1060, 867, + /* 190 */ 867, 867, 867, 867, 867, 867, 867, 867, 867, 867, + /* 200 */ 867, 867, 867, 867, 867, 867, 867, 867, 261, 712, + /* 210 */ 712, 867, 108, 1142, 1142, 977, 1108, 1108, 977, 977, + /* 220 */ 1243, 2379, 2379, 2379, 2379, 2379, 2379, 2379, 641, 789, + /* 230 */ 789, 635, 366, 721, 673, 782, 494, 787, 829, 867, + /* 240 */ 867, 867, 867, 867, 867, 867, 867, 867, 867, 867, + /* 250 */ 959, 867, 867, 867, 867, 867, 867, 867, 867, 867, + /* 260 */ 867, 867, 867, 867, 867, 820, 820, 820, 867, 867, + /* 270 */ 867, 1136, 867, 867, 867, 1119, 1007, 867, 1169, 867, + /* 280 */ 867, 867, 867, 867, 867, 867, 867, 1225, 1153, 869, + /* 290 */ 196, 618, 618, 618, 618, 1491, 196, 196, 91, 339, + /* 300 */ 1326, 1386, 383, 1163, 1364, 1426, 1364, 1538, 903, 1163, + /* 310 */ 1163, 903, 1163, 1426, 1538, 1018, 1535, 1241, 1528, 1528, + /* 320 */ 1528, 1394, 1394, 1394, 1394, 762, 762, 1403, 1466, 1475, + /* 330 */ 1551, 1746, 1805, 1746, 1746, 1729, 1729, 1840, 1840, 1729, + /* 340 */ 1730, 1732, 1859, 1842, 1870, 1870, 1870, 1870, 1729, 1876, + /* 350 */ 1751, 1732, 1732, 1751, 1859, 1842, 1751, 1842, 1751, 1729, + /* 360 */ 1876, 1760, 1857, 1729, 1876, 1906, 1729, 1876, 1729, 1876, + /* 370 */ 1906, 1746, 1746, 1746, 1873, 1922, 1922, 1906, 1746, 1822, + /* 380 */ 1746, 1873, 1746, 1746, 1786, 1929, 1843, 1843, 1906, 1729, + /* 390 */ 1872, 1872, 1894, 1894, 1831, 1836, 1966, 1729, 1833, 1831, + /* 400 */ 1851, 1860, 1751, 1983, 1996, 1996, 2009, 2009, 2009, 2379, + /* 410 */ 2379, 2379, 2379, 2379, 2379, 2379, 2379, 2379, 2379, 2379, + /* 420 */ 2379, 2379, 2379, 2379, 136, 1063, 1196, 530, 636, 1274, + /* 430 */ 1300, 1443, 1598, 1495, 1479, 967, 1083, 1602, 463, 1625, + /* 440 */ 1638, 1670, 1541, 1671, 1689, 1696, 1277, 1432, 1693, 808, + /* 450 */ 1700, 1607, 1657, 1587, 1704, 1707, 1631, 1708, 1733, 1608, + /* 460 */ 1611, 1743, 1747, 1620, 1592, 2026, 2030, 2013, 1914, 2018, + /* 470 */ 1916, 2020, 2019, 2021, 1915, 2027, 2029, 2024, 2025, 1909, + /* 480 */ 1899, 1923, 2028, 2028, 1913, 2037, 1917, 2042, 2059, 1918, + /* 490 */ 1932, 2028, 1933, 2003, 2031, 2028, 1919, 2012, 2015, 2016, + /* 500 */ 2022, 1936, 1956, 2040, 2053, 2077, 2074, 2058, 1967, 1924, + /* 510 */ 2034, 2060, 2036, 2008, 2046, 1946, 1978, 2066, 2075, 2078, + /* 520 */ 1968, 1972, 2084, 2041, 2086, 2088, 2076, 2089, 2048, 2054, + /* 530 */ 2093, 2023, 2091, 2099, 2055, 2085, 2101, 2092, 1975, 2105, + /* 540 */ 2110, 2111, 2112, 2115, 2113, 2043, 1997, 2117, 2120, 2032, + /* 550 */ 2114, 2122, 2001, 2121, 2116, 2118, 2119, 2124, 2062, 2071, + /* 560 */ 2068, 2123, 2080, 2065, 2126, 2138, 2140, 2139, 2141, 2143, + /* 570 */ 2130, 2033, 2035, 2142, 2121, 2146, 2147, 2148, 2150, 2149, + /* 580 */ 2152, 2156, 2151, 2164, 2158, 2159, 2161, 2162, 2160, 2165, + /* 590 */ 2163, 2050, 2049, 2051, 2052, 2167, 2172, 2181, 2197, 2204, }; -#define YY_REDUCE_COUNT (412) -#define YY_REDUCE_MIN (-277) -#define YY_REDUCE_MAX (1772) +#define YY_REDUCE_COUNT (423) +#define YY_REDUCE_MIN (-303) +#define YY_REDUCE_MAX (1825) static const short yy_reduce_ofst[] = { - /* 0 */ -67, 1252, -64, -178, -181, 160, 1071, 143, -184, 137, - /* 10 */ 218, 220, 222, -174, 229, 268, 272, 275, 324, -208, - /* 20 */ 242, -277, -39, 81, 537, 792, 810, 812, -189, 814, - /* 30 */ 831, 163, 865, 944, 887, 840, 964, 1077, -187, 292, - /* 40 */ -133, 274, 673, 558, 682, 795, 809, -238, -232, -238, - /* 50 */ -232, 329, 329, 329, 329, 329, 329, 329, 329, 329, - /* 60 */ 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, - /* 70 */ 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, - /* 80 */ 329, 329, 329, 329, 329, 329, 329, 329, 329, 557, - /* 90 */ 712, 949, 966, 969, 971, 979, 1097, 1099, 1103, 1142, - /* 100 */ 1144, 1169, 1172, 1201, 1203, 1228, 1241, 1250, 1253, 1255, - /* 110 */ 1261, 1266, 1271, 1282, 1291, 1308, 1310, 1312, 1322, 1328, - /* 120 */ 1347, 1354, 1356, 1359, 1362, 1365, 1367, 1374, 1376, 1381, - /* 130 */ 1401, 1403, 1406, 1412, 1414, 1417, 1421, 1428, 1447, 1449, - /* 140 */ 1453, 1462, 329, 329, 329, 329, 329, 329, 329, 329, - /* 150 */ 329, 329, 329, -22, -159, 475, -220, 756, 38, 501, - /* 160 */ 841, 714, 329, 118, 337, 349, 363, -56, 329, 329, - /* 170 */ 329, 329, -205, -205, -205, 687, -172, -130, -57, 790, - /* 180 */ 397, 528, -271, 136, 596, 596, 90, 316, 522, 541, - /* 190 */ -37, 715, 849, 977, 628, 856, 980, 991, 1081, 1102, - /* 200 */ 1135, 1083, -162, 208, 1258, 794, -86, 159, 41, 1109, - /* 210 */ 671, 852, 844, 932, 1175, 1254, 480, 1180, 100, 258, - /* 220 */ 1265, 1268, 1216, 1287, -139, 317, 344, 63, 339, 423, - /* 230 */ 563, 636, 676, 813, 908, 914, 950, 1078, 1084, 1098, - /* 240 */ 1363, 1384, 1407, 1439, 1464, 411, 1527, 1534, 1535, 1537, - /* 250 */ 1541, 1542, 1543, 1544, 1545, 1547, 1549, 1550, 990, 1164, - /* 260 */ 1492, 1551, 1552, 1556, 1217, 1558, 1559, 1560, 1473, 1413, - /* 270 */ 1563, 1510, 1568, 563, 1570, 1571, 1572, 1573, 1574, 1575, - /* 280 */ 1443, 1466, 1518, 1513, 1514, 1515, 1516, 1217, 1518, 1518, - /* 290 */ 1531, 1562, 1582, 1477, 1505, 1511, 1533, 1512, 1488, 1538, - /* 300 */ 1509, 1517, 1546, 1519, 1557, 1489, 1565, 1564, 1578, 1586, - /* 310 */ 1587, 1588, 1526, 1528, 1554, 1555, 1576, 1577, 1566, 1579, - /* 320 */ 1584, 1591, 1520, 1523, 1617, 1628, 1580, 1581, 1632, 1585, - /* 330 */ 1590, 1593, 1604, 1605, 1606, 1608, 1609, 1641, 1649, 1610, - /* 340 */ 1592, 1594, 1611, 1595, 1616, 1612, 1618, 1613, 1651, 1654, - /* 350 */ 1596, 1598, 1655, 1663, 1650, 1673, 1680, 1677, 1684, 1653, - /* 360 */ 1664, 1666, 1667, 1662, 1669, 1672, 1676, 1686, 1679, 1691, - /* 370 */ 1689, 1692, 1694, 1597, 1599, 1619, 1630, 1699, 1700, 1602, - /* 380 */ 1615, 1648, 1657, 1690, 1698, 1658, 1729, 1652, 1695, 1702, - /* 390 */ 1704, 1703, 1741, 1754, 1758, 1768, 1769, 1771, 1660, 1661, - /* 400 */ 1665, 1752, 1756, 1757, 1759, 1760, 1764, 1745, 1753, 1762, - /* 410 */ 1763, 1761, 1772, + /* 0 */ -67, 345, -64, -178, -181, 143, 435, -78, -183, 163, + /* 10 */ -185, 284, 384, -174, 189, 352, 440, 444, 493, -23, + /* 20 */ 227, -277, -1, 305, 561, 755, 759, 764, -189, 839, + /* 30 */ 857, 354, 484, 859, 631, 67, 734, 780, -187, 616, + /* 40 */ 581, 730, 891, 449, 588, 795, 836, -238, 287, -238, + /* 50 */ 287, -256, -256, -256, -256, -256, -256, -256, -256, -256, + /* 60 */ -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, + /* 70 */ -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, + /* 80 */ -256, -256, -256, -256, -256, -256, -256, -256, -256, -256, + /* 90 */ -256, 205, 582, 715, 958, 985, 1003, 1005, 1010, 1012, + /* 100 */ 1059, 1066, 1092, 1094, 1097, 1122, 1137, 1141, 1143, 1147, + /* 110 */ 1151, 1172, 1249, 1251, 1269, 1271, 1276, 1290, 1316, 1318, + /* 120 */ 1337, 1371, 1373, 1375, 1400, 1413, 1418, 1420, 1422, 1425, + /* 130 */ 1427, 1433, 1438, 1447, 1454, 1459, 1463, 1467, 1480, 1484, + /* 140 */ 1518, 1523, 1525, 1527, 1529, 1531, -256, -256, -256, -256, + /* 150 */ -256, -256, -256, -256, -256, -256, -256, 155, 210, -220, + /* 160 */ 86, -130, 943, 996, 402, -256, -113, 981, 1095, 1135, + /* 170 */ 395, -256, -256, -256, -256, 568, 568, 568, -4, -153, + /* 180 */ -133, 259, 306, -166, 523, -303, -126, 503, 503, -37, + /* 190 */ -149, 164, 690, 292, 412, 492, 651, 784, 332, 786, + /* 200 */ 841, 1149, 833, 1236, 792, 162, 796, 1253, 777, 288, + /* 210 */ 381, 380, 709, 487, 1027, 972, 1030, 1084, 991, 1120, + /* 220 */ -152, 1062, 692, 1240, 1247, 1250, 1239, 1306, -207, -194, + /* 230 */ 57, 180, 74, 315, 355, 376, 452, 488, 630, 693, + /* 240 */ 965, 1004, 1025, 1099, 1154, 1289, 1305, 1310, 1469, 1489, + /* 250 */ 984, 1494, 1502, 1516, 1544, 1556, 1557, 1562, 1576, 1578, + /* 260 */ 1579, 1583, 1584, 1585, 1586, 1217, 1440, 1554, 1589, 1593, + /* 270 */ 1594, 1530, 1597, 1599, 1600, 1539, 1472, 1601, 1560, 1604, + /* 280 */ 355, 1605, 1609, 1610, 1612, 1613, 1614, 1503, 1512, 1520, + /* 290 */ 1567, 1548, 1558, 1559, 1561, 1530, 1567, 1567, 1569, 1596, + /* 300 */ 1622, 1519, 1521, 1547, 1565, 1581, 1568, 1534, 1582, 1563, + /* 310 */ 1566, 1591, 1570, 1606, 1536, 1619, 1615, 1616, 1624, 1626, + /* 320 */ 1633, 1590, 1595, 1603, 1617, 1618, 1621, 1572, 1623, 1628, + /* 330 */ 1663, 1644, 1577, 1647, 1648, 1673, 1675, 1588, 1627, 1678, + /* 340 */ 1630, 1629, 1634, 1651, 1650, 1652, 1653, 1654, 1688, 1701, + /* 350 */ 1655, 1635, 1636, 1658, 1639, 1672, 1661, 1677, 1666, 1715, + /* 360 */ 1717, 1632, 1642, 1724, 1726, 1712, 1728, 1736, 1737, 1739, + /* 370 */ 1718, 1722, 1723, 1725, 1719, 1720, 1721, 1727, 1731, 1734, + /* 380 */ 1735, 1738, 1740, 1742, 1643, 1656, 1669, 1674, 1753, 1759, + /* 390 */ 1645, 1646, 1711, 1713, 1748, 1744, 1709, 1789, 1716, 1749, + /* 400 */ 1752, 1755, 1758, 1807, 1816, 1818, 1823, 1824, 1825, 1745, + /* 410 */ 1754, 1714, 1811, 1806, 1808, 1809, 1810, 1813, 1802, 1803, + /* 420 */ 1812, 1815, 1817, 1820, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1663, 1663, 1663, 1491, 1254, 1367, 1254, 1254, 1254, 1254, - /* 10 */ 1491, 1491, 1491, 1254, 1254, 1254, 1254, 1254, 1254, 1397, - /* 20 */ 1397, 1544, 1287, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 30 */ 1254, 1254, 1254, 1254, 1254, 1490, 1254, 1254, 1254, 1254, - /* 40 */ 1578, 1578, 1254, 1254, 1254, 1254, 1254, 1563, 1562, 1254, - /* 50 */ 1254, 1254, 1406, 1254, 1413, 1254, 1254, 1254, 1254, 1254, - /* 60 */ 1492, 1493, 1254, 1254, 1254, 1254, 1543, 1545, 1508, 1420, - /* 70 */ 1419, 1418, 1417, 1526, 1385, 1411, 1404, 1408, 1487, 1488, - /* 80 */ 1486, 1641, 1493, 1492, 1254, 1407, 1455, 1471, 1454, 1254, - /* 90 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 100 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 110 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 120 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 130 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 140 */ 1254, 1254, 1463, 1470, 1469, 1468, 1477, 1467, 1464, 1457, - /* 150 */ 1456, 1458, 1459, 1278, 1254, 1275, 1329, 1254, 1254, 1254, - /* 160 */ 1254, 1254, 1460, 1287, 1448, 1447, 1446, 1254, 1474, 1461, - /* 170 */ 1473, 1472, 1551, 1615, 1614, 1509, 1254, 1254, 1254, 1254, - /* 180 */ 1254, 1254, 1578, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 190 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 200 */ 1254, 1254, 1254, 1254, 1254, 1387, 1578, 1578, 1254, 1287, - /* 210 */ 1578, 1578, 1388, 1388, 1283, 1283, 1391, 1558, 1358, 1358, - /* 220 */ 1358, 1358, 1367, 1358, 1254, 1254, 1254, 1254, 1254, 1254, - /* 230 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1548, - /* 240 */ 1546, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 250 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 260 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1363, 1254, - /* 270 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1608, - /* 280 */ 1254, 1521, 1343, 1363, 1363, 1363, 1363, 1365, 1344, 1342, - /* 290 */ 1357, 1288, 1261, 1655, 1423, 1412, 1364, 1412, 1652, 1410, - /* 300 */ 1423, 1423, 1410, 1423, 1364, 1652, 1304, 1630, 1299, 1397, - /* 310 */ 1397, 1397, 1387, 1387, 1387, 1387, 1391, 1391, 1489, 1364, - /* 320 */ 1357, 1254, 1655, 1655, 1373, 1373, 1654, 1654, 1373, 1509, - /* 330 */ 1638, 1432, 1332, 1338, 1338, 1338, 1338, 1373, 1272, 1410, - /* 340 */ 1638, 1638, 1410, 1432, 1332, 1410, 1332, 1410, 1373, 1272, - /* 350 */ 1525, 1649, 1373, 1272, 1499, 1373, 1272, 1373, 1272, 1499, - /* 360 */ 1330, 1330, 1330, 1319, 1254, 1254, 1499, 1330, 1304, 1330, - /* 370 */ 1319, 1330, 1330, 1596, 1254, 1503, 1503, 1499, 1373, 1588, - /* 380 */ 1588, 1400, 1400, 1405, 1391, 1494, 1373, 1254, 1405, 1403, - /* 390 */ 1401, 1410, 1322, 1611, 1611, 1607, 1607, 1607, 1660, 1660, - /* 400 */ 1558, 1623, 1287, 1287, 1287, 1287, 1623, 1306, 1306, 1288, - /* 410 */ 1288, 1287, 1623, 1254, 1254, 1254, 1254, 1254, 1254, 1618, - /* 420 */ 1254, 1553, 1510, 1377, 1254, 1254, 1254, 1254, 1254, 1254, - /* 430 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 440 */ 1564, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 450 */ 1254, 1254, 1437, 1254, 1257, 1555, 1254, 1254, 1254, 1254, - /* 460 */ 1254, 1254, 1254, 1254, 1414, 1415, 1378, 1254, 1254, 1254, - /* 470 */ 1254, 1254, 1254, 1254, 1429, 1254, 1254, 1254, 1424, 1254, - /* 480 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1651, 1254, 1254, - /* 490 */ 1254, 1254, 1254, 1254, 1524, 1523, 1254, 1254, 1375, 1254, - /* 500 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 510 */ 1254, 1254, 1302, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 520 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 530 */ 1254, 1254, 1254, 1254, 1254, 1254, 1402, 1254, 1254, 1254, - /* 540 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 550 */ 1254, 1593, 1392, 1254, 1254, 1254, 1254, 1642, 1254, 1254, - /* 560 */ 1254, 1254, 1352, 1254, 1254, 1254, 1254, 1254, 1254, 1254, - /* 570 */ 1254, 1254, 1254, 1634, 1346, 1438, 1254, 1441, 1276, 1254, - /* 580 */ 1266, 1254, 1254, + /* 0 */ 1691, 1691, 1691, 1516, 1279, 1392, 1279, 1279, 1279, 1279, + /* 10 */ 1516, 1516, 1516, 1279, 1279, 1279, 1279, 1279, 1279, 1422, + /* 20 */ 1422, 1568, 1312, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 30 */ 1279, 1279, 1279, 1279, 1279, 1515, 1279, 1279, 1279, 1279, + /* 40 */ 1607, 1607, 1279, 1279, 1279, 1279, 1279, 1592, 1591, 1279, + /* 50 */ 1279, 1279, 1431, 1279, 1279, 1279, 1438, 1279, 1279, 1279, + /* 60 */ 1279, 1279, 1517, 1518, 1279, 1279, 1279, 1279, 1567, 1569, + /* 70 */ 1533, 1445, 1444, 1443, 1442, 1551, 1410, 1436, 1429, 1433, + /* 80 */ 1512, 1513, 1511, 1670, 1518, 1517, 1279, 1432, 1480, 1496, + /* 90 */ 1479, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 100 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 110 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 120 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 130 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 140 */ 1279, 1279, 1279, 1279, 1279, 1279, 1488, 1495, 1494, 1493, + /* 150 */ 1502, 1492, 1489, 1482, 1481, 1483, 1484, 1303, 1300, 1354, + /* 160 */ 1279, 1279, 1279, 1279, 1279, 1485, 1312, 1473, 1472, 1471, + /* 170 */ 1279, 1499, 1486, 1498, 1497, 1575, 1644, 1643, 1534, 1279, + /* 180 */ 1279, 1279, 1279, 1279, 1279, 1607, 1279, 1279, 1279, 1279, + /* 190 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 200 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1412, 1607, + /* 210 */ 1607, 1279, 1312, 1607, 1607, 1308, 1413, 1413, 1308, 1308, + /* 220 */ 1416, 1587, 1383, 1383, 1383, 1383, 1392, 1383, 1279, 1279, + /* 230 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 240 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1572, 1570, 1279, + /* 250 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 260 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 270 */ 1279, 1279, 1279, 1279, 1279, 1388, 1279, 1279, 1279, 1279, + /* 280 */ 1279, 1279, 1279, 1279, 1279, 1279, 1637, 1683, 1279, 1546, + /* 290 */ 1368, 1388, 1388, 1388, 1388, 1390, 1369, 1367, 1382, 1313, + /* 300 */ 1286, 1683, 1683, 1448, 1437, 1389, 1437, 1680, 1435, 1448, + /* 310 */ 1448, 1435, 1448, 1389, 1680, 1329, 1659, 1324, 1422, 1422, + /* 320 */ 1422, 1412, 1412, 1412, 1412, 1416, 1416, 1514, 1389, 1382, + /* 330 */ 1279, 1355, 1683, 1355, 1355, 1398, 1398, 1682, 1682, 1398, + /* 340 */ 1534, 1667, 1457, 1357, 1363, 1363, 1363, 1363, 1398, 1297, + /* 350 */ 1435, 1667, 1667, 1435, 1457, 1357, 1435, 1357, 1435, 1398, + /* 360 */ 1297, 1550, 1678, 1398, 1297, 1524, 1398, 1297, 1398, 1297, + /* 370 */ 1524, 1355, 1355, 1355, 1344, 1279, 1279, 1524, 1355, 1329, + /* 380 */ 1355, 1344, 1355, 1355, 1625, 1279, 1528, 1528, 1524, 1398, + /* 390 */ 1617, 1617, 1425, 1425, 1430, 1416, 1519, 1398, 1279, 1430, + /* 400 */ 1428, 1426, 1435, 1347, 1640, 1640, 1636, 1636, 1636, 1688, + /* 410 */ 1688, 1587, 1652, 1312, 1312, 1312, 1312, 1652, 1331, 1331, + /* 420 */ 1313, 1313, 1312, 1652, 1279, 1279, 1279, 1279, 1279, 1279, + /* 430 */ 1279, 1647, 1279, 1279, 1535, 1279, 1279, 1279, 1279, 1279, + /* 440 */ 1279, 1279, 1402, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 450 */ 1279, 1279, 1593, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 460 */ 1279, 1279, 1279, 1279, 1462, 1279, 1282, 1584, 1279, 1279, + /* 470 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 480 */ 1279, 1279, 1439, 1440, 1279, 1279, 1279, 1279, 1279, 1279, + /* 490 */ 1279, 1454, 1279, 1279, 1279, 1449, 1279, 1279, 1279, 1279, + /* 500 */ 1279, 1279, 1279, 1279, 1403, 1279, 1279, 1279, 1279, 1279, + /* 510 */ 1279, 1549, 1548, 1279, 1279, 1400, 1279, 1279, 1279, 1279, + /* 520 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1327, + /* 530 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 540 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 550 */ 1279, 1279, 1279, 1427, 1279, 1279, 1279, 1279, 1279, 1279, + /* 560 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1622, 1417, + /* 570 */ 1279, 1279, 1279, 1279, 1671, 1279, 1279, 1279, 1279, 1377, + /* 580 */ 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, 1279, + /* 590 */ 1663, 1371, 1463, 1279, 1466, 1301, 1279, 1291, 1279, 1279, }; /********** End of lemon-generated parsing tables *****************************/ @@ -177127,34 +181546,33 @@ static const char *const yyTokenName[] = { /* 292 */ "foreach_clause", /* 293 */ "when_clause", /* 294 */ "trigger_cmd", - /* 295 */ "trnm", - /* 296 */ "tridxby", - /* 297 */ "database_kw_opt", - /* 298 */ "key_opt", - /* 299 */ "add_column_fullname", - /* 300 */ "kwcolumn_opt", - /* 301 */ "create_vtab", - /* 302 */ "vtabarglist", - /* 303 */ "vtabarg", - /* 304 */ "vtabargtoken", - /* 305 */ "lp", - /* 306 */ "anylist", - /* 307 */ "wqitem", - /* 308 */ "wqas", - /* 309 */ "withnm", - /* 310 */ "windowdefn_list", - /* 311 */ "windowdefn", - /* 312 */ "window", - /* 313 */ "frame_opt", - /* 314 */ "part_opt", - /* 315 */ "filter_clause", - /* 316 */ "over_clause", - /* 317 */ "range_or_rows", - /* 318 */ "frame_bound", - /* 319 */ "frame_bound_s", - /* 320 */ "frame_bound_e", - /* 321 */ "frame_exclude_opt", - /* 322 */ "frame_exclude", + /* 295 */ "tridxby", + /* 296 */ "database_kw_opt", + /* 297 */ "key_opt", + /* 298 */ "alter_add", + /* 299 */ "kwcolumn_opt", + /* 300 */ "create_vtab", + /* 301 */ "vtabarglist", + /* 302 */ "vtabarg", + /* 303 */ "vtabargtoken", + /* 304 */ "lp", + /* 305 */ "anylist", + /* 306 */ "wqitem", + /* 307 */ "wqas", + /* 308 */ "withnm", + /* 309 */ "windowdefn_list", + /* 310 */ "windowdefn", + /* 311 */ "window", + /* 312 */ "frame_opt", + /* 313 */ "part_opt", + /* 314 */ "filter_clause", + /* 315 */ "over_clause", + /* 316 */ "range_or_rows", + /* 317 */ "frame_bound", + /* 318 */ "frame_bound_s", + /* 319 */ "frame_bound_e", + /* 320 */ "frame_exclude_opt", + /* 321 */ "frame_exclude", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -177284,8 +181702,8 @@ static const char *const yyRuleName[] = { /* 119 */ "fullname ::= nm DOT nm", /* 120 */ "xfullname ::= nm", /* 121 */ "xfullname ::= nm DOT nm", - /* 122 */ "xfullname ::= nm DOT nm AS nm", - /* 123 */ "xfullname ::= nm AS nm", + /* 122 */ "xfullname ::= nm AS nm", + /* 123 */ "xfullname ::= nm DOT nm AS nm", /* 124 */ "joinop ::= COMMA|JOIN", /* 125 */ "joinop ::= JOIN_KW JOIN", /* 126 */ "joinop ::= JOIN_KW nm JOIN", @@ -177434,143 +181852,146 @@ static const char *const yyRuleName[] = { /* 269 */ "when_clause ::= WHEN expr", /* 270 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", /* 271 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 272 */ "trnm ::= nm DOT nm", - /* 273 */ "tridxby ::= INDEXED BY nm", - /* 274 */ "tridxby ::= NOT INDEXED", - /* 275 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 276 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 277 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 278 */ "trigger_cmd ::= scanpt select scanpt", - /* 279 */ "expr ::= RAISE LP IGNORE RP", - /* 280 */ "expr ::= RAISE LP raisetype COMMA expr RP", - /* 281 */ "raisetype ::= ROLLBACK", - /* 282 */ "raisetype ::= ABORT", - /* 283 */ "raisetype ::= FAIL", - /* 284 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 285 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 286 */ "cmd ::= DETACH database_kw_opt expr", - /* 287 */ "key_opt ::=", - /* 288 */ "key_opt ::= KEY expr", - /* 289 */ "cmd ::= REINDEX", - /* 290 */ "cmd ::= REINDEX nm dbnm", - /* 291 */ "cmd ::= ANALYZE", - /* 292 */ "cmd ::= ANALYZE nm dbnm", - /* 293 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 294 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 272 */ "tridxby ::= INDEXED BY nm", + /* 273 */ "tridxby ::= NOT INDEXED", + /* 274 */ "trigger_cmd ::= UPDATE orconf xfullname tridxby SET setlist from where_opt scanpt", + /* 275 */ "trigger_cmd ::= scanpt insert_cmd INTO xfullname idlist_opt select upsert scanpt", + /* 276 */ "trigger_cmd ::= DELETE FROM xfullname tridxby where_opt scanpt", + /* 277 */ "trigger_cmd ::= scanpt select scanpt", + /* 278 */ "expr ::= RAISE LP IGNORE RP", + /* 279 */ "expr ::= RAISE LP raisetype COMMA expr RP", + /* 280 */ "raisetype ::= ROLLBACK", + /* 281 */ "raisetype ::= ABORT", + /* 282 */ "raisetype ::= FAIL", + /* 283 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 284 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 285 */ "cmd ::= DETACH database_kw_opt expr", + /* 286 */ "key_opt ::=", + /* 287 */ "key_opt ::= KEY expr", + /* 288 */ "cmd ::= REINDEX", + /* 289 */ "cmd ::= REINDEX nm dbnm", + /* 290 */ "cmd ::= ANALYZE", + /* 291 */ "cmd ::= ANALYZE nm dbnm", + /* 292 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 293 */ "cmd ::= alter_add carglist", + /* 294 */ "alter_add ::= ALTER TABLE fullname ADD kwcolumn_opt nm typetoken", /* 295 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 296 */ "add_column_fullname ::= fullname", - /* 297 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 298 */ "cmd ::= create_vtab", - /* 299 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 300 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 301 */ "vtabarg ::=", - /* 302 */ "vtabargtoken ::= ANY", - /* 303 */ "vtabargtoken ::= lp anylist RP", - /* 304 */ "lp ::= LP", - /* 305 */ "with ::= WITH wqlist", - /* 306 */ "with ::= WITH RECURSIVE wqlist", - /* 307 */ "wqas ::= AS", - /* 308 */ "wqas ::= AS MATERIALIZED", - /* 309 */ "wqas ::= AS NOT MATERIALIZED", - /* 310 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", - /* 311 */ "withnm ::= nm", - /* 312 */ "wqlist ::= wqitem", - /* 313 */ "wqlist ::= wqlist COMMA wqitem", - /* 314 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 315 */ "windowdefn ::= nm AS LP window RP", - /* 316 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 317 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 318 */ "window ::= ORDER BY sortlist frame_opt", - /* 319 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 320 */ "window ::= nm frame_opt", - /* 321 */ "frame_opt ::=", - /* 322 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 323 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 324 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 325 */ "frame_bound_s ::= frame_bound", - /* 326 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 327 */ "frame_bound_e ::= frame_bound", - /* 328 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 329 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 330 */ "frame_bound ::= CURRENT ROW", - /* 331 */ "frame_exclude_opt ::=", - /* 332 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 333 */ "frame_exclude ::= NO OTHERS", - /* 334 */ "frame_exclude ::= CURRENT ROW", - /* 335 */ "frame_exclude ::= GROUP|TIES", - /* 336 */ "window_clause ::= WINDOW windowdefn_list", - /* 337 */ "filter_over ::= filter_clause over_clause", - /* 338 */ "filter_over ::= over_clause", - /* 339 */ "filter_over ::= filter_clause", - /* 340 */ "over_clause ::= OVER LP window RP", - /* 341 */ "over_clause ::= OVER nm", - /* 342 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 343 */ "term ::= QNUMBER", - /* 344 */ "input ::= cmdlist", - /* 345 */ "cmdlist ::= cmdlist ecmd", - /* 346 */ "cmdlist ::= ecmd", - /* 347 */ "ecmd ::= SEMI", - /* 348 */ "ecmd ::= cmdx SEMI", - /* 349 */ "ecmd ::= explain cmdx SEMI", - /* 350 */ "trans_opt ::=", - /* 351 */ "trans_opt ::= TRANSACTION", - /* 352 */ "trans_opt ::= TRANSACTION nm", - /* 353 */ "savepoint_opt ::= SAVEPOINT", - /* 354 */ "savepoint_opt ::=", - /* 355 */ "cmd ::= create_table create_table_args", - /* 356 */ "table_option_set ::= table_option", - /* 357 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 358 */ "columnlist ::= columnname carglist", - /* 359 */ "nm ::= ID|INDEXED|JOIN_KW", - /* 360 */ "nm ::= STRING", - /* 361 */ "typetoken ::= typename", - /* 362 */ "typename ::= ID|STRING", - /* 363 */ "signed ::= plus_num", - /* 364 */ "signed ::= minus_num", - /* 365 */ "carglist ::= carglist ccons", - /* 366 */ "carglist ::=", - /* 367 */ "ccons ::= NULL onconf", - /* 368 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 369 */ "ccons ::= AS generated", - /* 370 */ "conslist_opt ::= COMMA conslist", - /* 371 */ "conslist ::= conslist tconscomma tcons", - /* 372 */ "conslist ::= tcons", - /* 373 */ "tconscomma ::=", - /* 374 */ "defer_subclause_opt ::= defer_subclause", - /* 375 */ "resolvetype ::= raisetype", - /* 376 */ "selectnowith ::= oneselect", - /* 377 */ "oneselect ::= values", - /* 378 */ "sclp ::= selcollist COMMA", - /* 379 */ "as ::= ID|STRING", - /* 380 */ "indexed_opt ::= indexed_by", - /* 381 */ "returning ::=", - /* 382 */ "expr ::= term", - /* 383 */ "likeop ::= LIKE_KW|MATCH", - /* 384 */ "case_operand ::= expr", - /* 385 */ "exprlist ::= nexprlist", - /* 386 */ "nmnum ::= plus_num", - /* 387 */ "nmnum ::= nm", - /* 388 */ "nmnum ::= ON", - /* 389 */ "nmnum ::= DELETE", - /* 390 */ "nmnum ::= DEFAULT", - /* 391 */ "plus_num ::= INTEGER|FLOAT", - /* 392 */ "foreach_clause ::=", - /* 393 */ "foreach_clause ::= FOR EACH ROW", - /* 394 */ "trnm ::= nm", - /* 395 */ "tridxby ::=", - /* 396 */ "database_kw_opt ::= DATABASE", - /* 397 */ "database_kw_opt ::=", - /* 398 */ "kwcolumn_opt ::=", - /* 399 */ "kwcolumn_opt ::= COLUMNKW", - /* 400 */ "vtabarglist ::= vtabarg", - /* 401 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 402 */ "vtabarg ::= vtabarg vtabargtoken", - /* 403 */ "anylist ::=", - /* 404 */ "anylist ::= anylist LP anylist RP", - /* 405 */ "anylist ::= anylist ANY", - /* 406 */ "with ::=", - /* 407 */ "windowdefn_list ::= windowdefn", - /* 408 */ "window ::= frame_opt", + /* 296 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 297 */ "cmd ::= ALTER TABLE fullname DROP CONSTRAINT nm", + /* 298 */ "cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm DROP NOT NULL", + /* 299 */ "cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm SET NOT NULL onconf", + /* 300 */ "cmd ::= ALTER TABLE fullname ADD CONSTRAINT nm CHECK LP expr RP onconf", + /* 301 */ "cmd ::= ALTER TABLE fullname ADD CHECK LP expr RP onconf", + /* 302 */ "cmd ::= create_vtab", + /* 303 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 304 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 305 */ "vtabarg ::=", + /* 306 */ "vtabargtoken ::= ANY", + /* 307 */ "vtabargtoken ::= lp anylist RP", + /* 308 */ "lp ::= LP", + /* 309 */ "with ::= WITH wqlist", + /* 310 */ "with ::= WITH RECURSIVE wqlist", + /* 311 */ "wqas ::= AS", + /* 312 */ "wqas ::= AS MATERIALIZED", + /* 313 */ "wqas ::= AS NOT MATERIALIZED", + /* 314 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", + /* 315 */ "withnm ::= nm", + /* 316 */ "wqlist ::= wqitem", + /* 317 */ "wqlist ::= wqlist COMMA wqitem", + /* 318 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 319 */ "windowdefn ::= nm AS LP window RP", + /* 320 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 321 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 322 */ "window ::= ORDER BY sortlist frame_opt", + /* 323 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 324 */ "window ::= nm frame_opt", + /* 325 */ "frame_opt ::=", + /* 326 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 327 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 328 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 329 */ "frame_bound_s ::= frame_bound", + /* 330 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 331 */ "frame_bound_e ::= frame_bound", + /* 332 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 333 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 334 */ "frame_bound ::= CURRENT ROW", + /* 335 */ "frame_exclude_opt ::=", + /* 336 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 337 */ "frame_exclude ::= NO OTHERS", + /* 338 */ "frame_exclude ::= CURRENT ROW", + /* 339 */ "frame_exclude ::= GROUP|TIES", + /* 340 */ "window_clause ::= WINDOW windowdefn_list", + /* 341 */ "filter_over ::= filter_clause over_clause", + /* 342 */ "filter_over ::= over_clause", + /* 343 */ "filter_over ::= filter_clause", + /* 344 */ "over_clause ::= OVER LP window RP", + /* 345 */ "over_clause ::= OVER nm", + /* 346 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 347 */ "term ::= QNUMBER", + /* 348 */ "input ::= cmdlist", + /* 349 */ "cmdlist ::= cmdlist ecmd", + /* 350 */ "cmdlist ::= ecmd", + /* 351 */ "ecmd ::= SEMI", + /* 352 */ "ecmd ::= cmdx SEMI", + /* 353 */ "ecmd ::= explain cmdx SEMI", + /* 354 */ "trans_opt ::=", + /* 355 */ "trans_opt ::= TRANSACTION", + /* 356 */ "trans_opt ::= TRANSACTION nm", + /* 357 */ "savepoint_opt ::= SAVEPOINT", + /* 358 */ "savepoint_opt ::=", + /* 359 */ "cmd ::= create_table create_table_args", + /* 360 */ "table_option_set ::= table_option", + /* 361 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 362 */ "columnlist ::= columnname carglist", + /* 363 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 364 */ "nm ::= STRING", + /* 365 */ "typetoken ::= typename", + /* 366 */ "typename ::= ID|STRING", + /* 367 */ "signed ::= plus_num", + /* 368 */ "signed ::= minus_num", + /* 369 */ "carglist ::= carglist ccons", + /* 370 */ "carglist ::=", + /* 371 */ "ccons ::= NULL onconf", + /* 372 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 373 */ "ccons ::= AS generated", + /* 374 */ "conslist_opt ::= COMMA conslist", + /* 375 */ "conslist ::= conslist tconscomma tcons", + /* 376 */ "conslist ::= tcons", + /* 377 */ "tconscomma ::=", + /* 378 */ "defer_subclause_opt ::= defer_subclause", + /* 379 */ "resolvetype ::= raisetype", + /* 380 */ "selectnowith ::= oneselect", + /* 381 */ "oneselect ::= values", + /* 382 */ "sclp ::= selcollist COMMA", + /* 383 */ "as ::= ID|STRING", + /* 384 */ "indexed_opt ::= indexed_by", + /* 385 */ "returning ::=", + /* 386 */ "expr ::= term", + /* 387 */ "likeop ::= LIKE_KW|MATCH", + /* 388 */ "case_operand ::= expr", + /* 389 */ "exprlist ::= nexprlist", + /* 390 */ "nmnum ::= plus_num", + /* 391 */ "nmnum ::= nm", + /* 392 */ "nmnum ::= ON", + /* 393 */ "nmnum ::= DELETE", + /* 394 */ "nmnum ::= DEFAULT", + /* 395 */ "plus_num ::= INTEGER|FLOAT", + /* 396 */ "foreach_clause ::=", + /* 397 */ "foreach_clause ::= FOR EACH ROW", + /* 398 */ "tridxby ::=", + /* 399 */ "database_kw_opt ::= DATABASE", + /* 400 */ "database_kw_opt ::=", + /* 401 */ "kwcolumn_opt ::=", + /* 402 */ "kwcolumn_opt ::= COLUMNKW", + /* 403 */ "vtabarglist ::= vtabarg", + /* 404 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 405 */ "vtabarg ::= vtabarg vtabargtoken", + /* 406 */ "anylist ::=", + /* 407 */ "anylist ::= anylist LP anylist RP", + /* 408 */ "anylist ::= anylist ANY", + /* 409 */ "with ::=", + /* 410 */ "windowdefn_list ::= windowdefn", + /* 411 */ "window ::= frame_opt", }; #endif /* NDEBUG */ @@ -177585,15 +182006,24 @@ static int yyGrowStack(yyParser *p){ int newSize; int idx; yyStackEntry *pNew; +#ifdef YYSIZELIMIT + int nLimit = YYSIZELIMIT(sqlite3ParserCTX(p)); +#endif newSize = oldSize*2 + 100; +#ifdef YYSIZELIMIT + if( newSize>nLimit ){ + newSize = nLimit; + if( newSize<=oldSize ) return 1; + } +#endif idx = (int)(p->yytos - p->yystack); if( p->yystack==p->yystk0 ){ - pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(0, newSize*sizeof(pNew[0]), sqlite3ParserCTX(p)); if( pNew==0 ) return 1; memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]), sqlite3ParserCTX(p)); if( pNew==0 ) return 1; } p->yystack = pNew; @@ -177700,7 +182130,7 @@ static void yy_destructor( case 254: /* values */ case 256: /* mvalues */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy637)); +sqlite3SelectDelete(pParse->db, (yypminor->yy555)); } break; case 218: /* term */ @@ -177712,10 +182142,10 @@ sqlite3SelectDelete(pParse->db, (yypminor->yy637)); case 283: /* case_else */ case 286: /* vinto */ case 293: /* when_clause */ - case 298: /* key_opt */ - case 315: /* filter_clause */ + case 297: /* key_opt */ + case 314: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy590)); +sqlite3ExprDelete(pParse->db, (yypminor->yy454)); } break; case 223: /* eidlist_opt */ @@ -177730,9 +182160,9 @@ sqlite3ExprDelete(pParse->db, (yypminor->yy590)); case 271: /* setlist */ case 280: /* paren_exprlist */ case 282: /* case_exprlist */ - case 314: /* part_opt */ + case 313: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy402)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; case 240: /* fullname */ @@ -177741,51 +182171,51 @@ sqlite3ExprListDelete(pParse->db, (yypminor->yy402)); case 260: /* stl_prefix */ case 265: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy563)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy203)); } break; case 243: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy125)); +sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; case 253: /* window_clause */ - case 310: /* windowdefn_list */ + case 309: /* windowdefn_list */ { -sqlite3WindowListDelete(pParse->db, (yypminor->yy483)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy211)); } break; case 266: /* idlist */ case 273: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy204)); +sqlite3IdListDelete(pParse->db, (yypminor->yy132)); } break; case 276: /* filter_over */ - case 311: /* windowdefn */ - case 312: /* window */ - case 313: /* frame_opt */ - case 316: /* over_clause */ + case 310: /* windowdefn */ + case 311: /* window */ + case 312: /* frame_opt */ + case 315: /* over_clause */ { -sqlite3WindowDelete(pParse->db, (yypminor->yy483)); +sqlite3WindowDelete(pParse->db, (yypminor->yy211)); } break; case 289: /* trigger_cmd_list */ case 294: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy319)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy427)); } break; case 291: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy28).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy286).b); } break; - case 318: /* frame_bound */ - case 319: /* frame_bound_s */ - case 320: /* frame_bound_e */ + case 317: /* frame_bound */ + case 318: /* frame_bound_s */ + case 319: /* frame_bound_e */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy205).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy509).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -177838,7 +182268,9 @@ SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ } #if YYGROWABLESTACK - if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); + if( pParser->yystack!=pParser->yystk0 ){ + YYFREE(pParser->yystack, sqlite3ParserCTX(pParser)); + } #endif } @@ -178021,7 +182453,7 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ - sqlite3OomFault(pParse->db); + if( pParse->nErr==0 ) sqlite3ErrorMsg(pParse, "Recursion limit"); /******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ sqlite3ParserCTX_STORE @@ -178209,8 +182641,8 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 240, /* (119) fullname ::= nm DOT nm */ 265, /* (120) xfullname ::= nm */ 265, /* (121) xfullname ::= nm DOT nm */ - 265, /* (122) xfullname ::= nm DOT nm AS nm */ - 265, /* (123) xfullname ::= nm AS nm */ + 265, /* (122) xfullname ::= nm AS nm */ + 265, /* (123) xfullname ::= nm DOT nm AS nm */ 261, /* (124) joinop ::= COMMA|JOIN */ 261, /* (125) joinop ::= JOIN_KW JOIN */ 261, /* (126) joinop ::= JOIN_KW nm JOIN */ @@ -178359,143 +182791,146 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 293, /* (269) when_clause ::= WHEN expr */ 289, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ 289, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ - 295, /* (272) trnm ::= nm DOT nm */ - 296, /* (273) tridxby ::= INDEXED BY nm */ - 296, /* (274) tridxby ::= NOT INDEXED */ - 294, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 294, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 294, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 294, /* (278) trigger_cmd ::= scanpt select scanpt */ - 219, /* (279) expr ::= RAISE LP IGNORE RP */ - 219, /* (280) expr ::= RAISE LP raisetype COMMA expr RP */ - 238, /* (281) raisetype ::= ROLLBACK */ - 238, /* (282) raisetype ::= ABORT */ - 238, /* (283) raisetype ::= FAIL */ - 192, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ - 192, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 192, /* (286) cmd ::= DETACH database_kw_opt expr */ - 298, /* (287) key_opt ::= */ - 298, /* (288) key_opt ::= KEY expr */ - 192, /* (289) cmd ::= REINDEX */ - 192, /* (290) cmd ::= REINDEX nm dbnm */ - 192, /* (291) cmd ::= ANALYZE */ - 192, /* (292) cmd ::= ANALYZE nm dbnm */ - 192, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 192, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 295, /* (272) tridxby ::= INDEXED BY nm */ + 295, /* (273) tridxby ::= NOT INDEXED */ + 294, /* (274) trigger_cmd ::= UPDATE orconf xfullname tridxby SET setlist from where_opt scanpt */ + 294, /* (275) trigger_cmd ::= scanpt insert_cmd INTO xfullname idlist_opt select upsert scanpt */ + 294, /* (276) trigger_cmd ::= DELETE FROM xfullname tridxby where_opt scanpt */ + 294, /* (277) trigger_cmd ::= scanpt select scanpt */ + 219, /* (278) expr ::= RAISE LP IGNORE RP */ + 219, /* (279) expr ::= RAISE LP raisetype COMMA expr RP */ + 238, /* (280) raisetype ::= ROLLBACK */ + 238, /* (281) raisetype ::= ABORT */ + 238, /* (282) raisetype ::= FAIL */ + 192, /* (283) cmd ::= DROP TRIGGER ifexists fullname */ + 192, /* (284) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 192, /* (285) cmd ::= DETACH database_kw_opt expr */ + 297, /* (286) key_opt ::= */ + 297, /* (287) key_opt ::= KEY expr */ + 192, /* (288) cmd ::= REINDEX */ + 192, /* (289) cmd ::= REINDEX nm dbnm */ + 192, /* (290) cmd ::= ANALYZE */ + 192, /* (291) cmd ::= ANALYZE nm dbnm */ + 192, /* (292) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 192, /* (293) cmd ::= alter_add carglist */ + 298, /* (294) alter_add ::= ALTER TABLE fullname ADD kwcolumn_opt nm typetoken */ 192, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 299, /* (296) add_column_fullname ::= fullname */ - 192, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 192, /* (298) cmd ::= create_vtab */ - 192, /* (299) cmd ::= create_vtab LP vtabarglist RP */ - 301, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 303, /* (301) vtabarg ::= */ - 304, /* (302) vtabargtoken ::= ANY */ - 304, /* (303) vtabargtoken ::= lp anylist RP */ - 305, /* (304) lp ::= LP */ - 269, /* (305) with ::= WITH wqlist */ - 269, /* (306) with ::= WITH RECURSIVE wqlist */ - 308, /* (307) wqas ::= AS */ - 308, /* (308) wqas ::= AS MATERIALIZED */ - 308, /* (309) wqas ::= AS NOT MATERIALIZED */ - 307, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ - 309, /* (311) withnm ::= nm */ - 243, /* (312) wqlist ::= wqitem */ - 243, /* (313) wqlist ::= wqlist COMMA wqitem */ - 310, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 311, /* (315) windowdefn ::= nm AS LP window RP */ - 312, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 312, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 312, /* (318) window ::= ORDER BY sortlist frame_opt */ - 312, /* (319) window ::= nm ORDER BY sortlist frame_opt */ - 312, /* (320) window ::= nm frame_opt */ - 313, /* (321) frame_opt ::= */ - 313, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 313, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 317, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ - 319, /* (325) frame_bound_s ::= frame_bound */ - 319, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ - 320, /* (327) frame_bound_e ::= frame_bound */ - 320, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 318, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ - 318, /* (330) frame_bound ::= CURRENT ROW */ - 321, /* (331) frame_exclude_opt ::= */ - 321, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 322, /* (333) frame_exclude ::= NO OTHERS */ - 322, /* (334) frame_exclude ::= CURRENT ROW */ - 322, /* (335) frame_exclude ::= GROUP|TIES */ - 253, /* (336) window_clause ::= WINDOW windowdefn_list */ - 276, /* (337) filter_over ::= filter_clause over_clause */ - 276, /* (338) filter_over ::= over_clause */ - 276, /* (339) filter_over ::= filter_clause */ - 316, /* (340) over_clause ::= OVER LP window RP */ - 316, /* (341) over_clause ::= OVER nm */ - 315, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ - 218, /* (343) term ::= QNUMBER */ - 187, /* (344) input ::= cmdlist */ - 188, /* (345) cmdlist ::= cmdlist ecmd */ - 188, /* (346) cmdlist ::= ecmd */ - 189, /* (347) ecmd ::= SEMI */ - 189, /* (348) ecmd ::= cmdx SEMI */ - 189, /* (349) ecmd ::= explain cmdx SEMI */ - 194, /* (350) trans_opt ::= */ - 194, /* (351) trans_opt ::= TRANSACTION */ - 194, /* (352) trans_opt ::= TRANSACTION nm */ - 196, /* (353) savepoint_opt ::= SAVEPOINT */ - 196, /* (354) savepoint_opt ::= */ - 192, /* (355) cmd ::= create_table create_table_args */ - 205, /* (356) table_option_set ::= table_option */ - 203, /* (357) columnlist ::= columnlist COMMA columnname carglist */ - 203, /* (358) columnlist ::= columnname carglist */ - 195, /* (359) nm ::= ID|INDEXED|JOIN_KW */ - 195, /* (360) nm ::= STRING */ - 210, /* (361) typetoken ::= typename */ - 211, /* (362) typename ::= ID|STRING */ - 212, /* (363) signed ::= plus_num */ - 212, /* (364) signed ::= minus_num */ - 209, /* (365) carglist ::= carglist ccons */ - 209, /* (366) carglist ::= */ - 217, /* (367) ccons ::= NULL onconf */ - 217, /* (368) ccons ::= GENERATED ALWAYS AS generated */ - 217, /* (369) ccons ::= AS generated */ - 204, /* (370) conslist_opt ::= COMMA conslist */ - 230, /* (371) conslist ::= conslist tconscomma tcons */ - 230, /* (372) conslist ::= tcons */ - 231, /* (373) tconscomma ::= */ - 235, /* (374) defer_subclause_opt ::= defer_subclause */ - 237, /* (375) resolvetype ::= raisetype */ - 241, /* (376) selectnowith ::= oneselect */ - 242, /* (377) oneselect ::= values */ - 257, /* (378) sclp ::= selcollist COMMA */ - 258, /* (379) as ::= ID|STRING */ - 267, /* (380) indexed_opt ::= indexed_by */ - 275, /* (381) returning ::= */ - 219, /* (382) expr ::= term */ - 277, /* (383) likeop ::= LIKE_KW|MATCH */ - 281, /* (384) case_operand ::= expr */ - 264, /* (385) exprlist ::= nexprlist */ - 287, /* (386) nmnum ::= plus_num */ - 287, /* (387) nmnum ::= nm */ - 287, /* (388) nmnum ::= ON */ - 287, /* (389) nmnum ::= DELETE */ - 287, /* (390) nmnum ::= DEFAULT */ - 213, /* (391) plus_num ::= INTEGER|FLOAT */ - 292, /* (392) foreach_clause ::= */ - 292, /* (393) foreach_clause ::= FOR EACH ROW */ - 295, /* (394) trnm ::= nm */ - 296, /* (395) tridxby ::= */ - 297, /* (396) database_kw_opt ::= DATABASE */ - 297, /* (397) database_kw_opt ::= */ - 300, /* (398) kwcolumn_opt ::= */ - 300, /* (399) kwcolumn_opt ::= COLUMNKW */ - 302, /* (400) vtabarglist ::= vtabarg */ - 302, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ - 303, /* (402) vtabarg ::= vtabarg vtabargtoken */ - 306, /* (403) anylist ::= */ - 306, /* (404) anylist ::= anylist LP anylist RP */ - 306, /* (405) anylist ::= anylist ANY */ - 269, /* (406) with ::= */ - 310, /* (407) windowdefn_list ::= windowdefn */ - 312, /* (408) window ::= frame_opt */ + 192, /* (296) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 192, /* (297) cmd ::= ALTER TABLE fullname DROP CONSTRAINT nm */ + 192, /* (298) cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm DROP NOT NULL */ + 192, /* (299) cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm SET NOT NULL onconf */ + 192, /* (300) cmd ::= ALTER TABLE fullname ADD CONSTRAINT nm CHECK LP expr RP onconf */ + 192, /* (301) cmd ::= ALTER TABLE fullname ADD CHECK LP expr RP onconf */ + 192, /* (302) cmd ::= create_vtab */ + 192, /* (303) cmd ::= create_vtab LP vtabarglist RP */ + 300, /* (304) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 302, /* (305) vtabarg ::= */ + 303, /* (306) vtabargtoken ::= ANY */ + 303, /* (307) vtabargtoken ::= lp anylist RP */ + 304, /* (308) lp ::= LP */ + 269, /* (309) with ::= WITH wqlist */ + 269, /* (310) with ::= WITH RECURSIVE wqlist */ + 307, /* (311) wqas ::= AS */ + 307, /* (312) wqas ::= AS MATERIALIZED */ + 307, /* (313) wqas ::= AS NOT MATERIALIZED */ + 306, /* (314) wqitem ::= withnm eidlist_opt wqas LP select RP */ + 308, /* (315) withnm ::= nm */ + 243, /* (316) wqlist ::= wqitem */ + 243, /* (317) wqlist ::= wqlist COMMA wqitem */ + 309, /* (318) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 310, /* (319) windowdefn ::= nm AS LP window RP */ + 311, /* (320) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (321) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (322) window ::= ORDER BY sortlist frame_opt */ + 311, /* (323) window ::= nm ORDER BY sortlist frame_opt */ + 311, /* (324) window ::= nm frame_opt */ + 312, /* (325) frame_opt ::= */ + 312, /* (326) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 312, /* (327) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 316, /* (328) range_or_rows ::= RANGE|ROWS|GROUPS */ + 318, /* (329) frame_bound_s ::= frame_bound */ + 318, /* (330) frame_bound_s ::= UNBOUNDED PRECEDING */ + 319, /* (331) frame_bound_e ::= frame_bound */ + 319, /* (332) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 317, /* (333) frame_bound ::= expr PRECEDING|FOLLOWING */ + 317, /* (334) frame_bound ::= CURRENT ROW */ + 320, /* (335) frame_exclude_opt ::= */ + 320, /* (336) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 321, /* (337) frame_exclude ::= NO OTHERS */ + 321, /* (338) frame_exclude ::= CURRENT ROW */ + 321, /* (339) frame_exclude ::= GROUP|TIES */ + 253, /* (340) window_clause ::= WINDOW windowdefn_list */ + 276, /* (341) filter_over ::= filter_clause over_clause */ + 276, /* (342) filter_over ::= over_clause */ + 276, /* (343) filter_over ::= filter_clause */ + 315, /* (344) over_clause ::= OVER LP window RP */ + 315, /* (345) over_clause ::= OVER nm */ + 314, /* (346) filter_clause ::= FILTER LP WHERE expr RP */ + 218, /* (347) term ::= QNUMBER */ + 187, /* (348) input ::= cmdlist */ + 188, /* (349) cmdlist ::= cmdlist ecmd */ + 188, /* (350) cmdlist ::= ecmd */ + 189, /* (351) ecmd ::= SEMI */ + 189, /* (352) ecmd ::= cmdx SEMI */ + 189, /* (353) ecmd ::= explain cmdx SEMI */ + 194, /* (354) trans_opt ::= */ + 194, /* (355) trans_opt ::= TRANSACTION */ + 194, /* (356) trans_opt ::= TRANSACTION nm */ + 196, /* (357) savepoint_opt ::= SAVEPOINT */ + 196, /* (358) savepoint_opt ::= */ + 192, /* (359) cmd ::= create_table create_table_args */ + 205, /* (360) table_option_set ::= table_option */ + 203, /* (361) columnlist ::= columnlist COMMA columnname carglist */ + 203, /* (362) columnlist ::= columnname carglist */ + 195, /* (363) nm ::= ID|INDEXED|JOIN_KW */ + 195, /* (364) nm ::= STRING */ + 210, /* (365) typetoken ::= typename */ + 211, /* (366) typename ::= ID|STRING */ + 212, /* (367) signed ::= plus_num */ + 212, /* (368) signed ::= minus_num */ + 209, /* (369) carglist ::= carglist ccons */ + 209, /* (370) carglist ::= */ + 217, /* (371) ccons ::= NULL onconf */ + 217, /* (372) ccons ::= GENERATED ALWAYS AS generated */ + 217, /* (373) ccons ::= AS generated */ + 204, /* (374) conslist_opt ::= COMMA conslist */ + 230, /* (375) conslist ::= conslist tconscomma tcons */ + 230, /* (376) conslist ::= tcons */ + 231, /* (377) tconscomma ::= */ + 235, /* (378) defer_subclause_opt ::= defer_subclause */ + 237, /* (379) resolvetype ::= raisetype */ + 241, /* (380) selectnowith ::= oneselect */ + 242, /* (381) oneselect ::= values */ + 257, /* (382) sclp ::= selcollist COMMA */ + 258, /* (383) as ::= ID|STRING */ + 267, /* (384) indexed_opt ::= indexed_by */ + 275, /* (385) returning ::= */ + 219, /* (386) expr ::= term */ + 277, /* (387) likeop ::= LIKE_KW|MATCH */ + 281, /* (388) case_operand ::= expr */ + 264, /* (389) exprlist ::= nexprlist */ + 287, /* (390) nmnum ::= plus_num */ + 287, /* (391) nmnum ::= nm */ + 287, /* (392) nmnum ::= ON */ + 287, /* (393) nmnum ::= DELETE */ + 287, /* (394) nmnum ::= DEFAULT */ + 213, /* (395) plus_num ::= INTEGER|FLOAT */ + 292, /* (396) foreach_clause ::= */ + 292, /* (397) foreach_clause ::= FOR EACH ROW */ + 295, /* (398) tridxby ::= */ + 296, /* (399) database_kw_opt ::= DATABASE */ + 296, /* (400) database_kw_opt ::= */ + 299, /* (401) kwcolumn_opt ::= */ + 299, /* (402) kwcolumn_opt ::= COLUMNKW */ + 301, /* (403) vtabarglist ::= vtabarg */ + 301, /* (404) vtabarglist ::= vtabarglist COMMA vtabarg */ + 302, /* (405) vtabarg ::= vtabarg vtabargtoken */ + 305, /* (406) anylist ::= */ + 305, /* (407) anylist ::= anylist LP anylist RP */ + 305, /* (408) anylist ::= anylist ANY */ + 269, /* (409) with ::= */ + 309, /* (410) windowdefn_list ::= windowdefn */ + 311, /* (411) window ::= frame_opt */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -178623,8 +183058,8 @@ static const signed char yyRuleInfoNRhs[] = { -3, /* (119) fullname ::= nm DOT nm */ -1, /* (120) xfullname ::= nm */ -3, /* (121) xfullname ::= nm DOT nm */ - -5, /* (122) xfullname ::= nm DOT nm AS nm */ - -3, /* (123) xfullname ::= nm AS nm */ + -3, /* (122) xfullname ::= nm AS nm */ + -5, /* (123) xfullname ::= nm DOT nm AS nm */ -1, /* (124) joinop ::= COMMA|JOIN */ -2, /* (125) joinop ::= JOIN_KW JOIN */ -3, /* (126) joinop ::= JOIN_KW nm JOIN */ @@ -178773,143 +183208,146 @@ static const signed char yyRuleInfoNRhs[] = { -2, /* (269) when_clause ::= WHEN expr */ -3, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ -2, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (272) trnm ::= nm DOT nm */ - -3, /* (273) tridxby ::= INDEXED BY nm */ - -2, /* (274) tridxby ::= NOT INDEXED */ - -9, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (278) trigger_cmd ::= scanpt select scanpt */ - -4, /* (279) expr ::= RAISE LP IGNORE RP */ - -6, /* (280) expr ::= RAISE LP raisetype COMMA expr RP */ - -1, /* (281) raisetype ::= ROLLBACK */ - -1, /* (282) raisetype ::= ABORT */ - -1, /* (283) raisetype ::= FAIL */ - -4, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (286) cmd ::= DETACH database_kw_opt expr */ - 0, /* (287) key_opt ::= */ - -2, /* (288) key_opt ::= KEY expr */ - -1, /* (289) cmd ::= REINDEX */ - -3, /* (290) cmd ::= REINDEX nm dbnm */ - -1, /* (291) cmd ::= ANALYZE */ - -3, /* (292) cmd ::= ANALYZE nm dbnm */ - -6, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -3, /* (272) tridxby ::= INDEXED BY nm */ + -2, /* (273) tridxby ::= NOT INDEXED */ + -9, /* (274) trigger_cmd ::= UPDATE orconf xfullname tridxby SET setlist from where_opt scanpt */ + -8, /* (275) trigger_cmd ::= scanpt insert_cmd INTO xfullname idlist_opt select upsert scanpt */ + -6, /* (276) trigger_cmd ::= DELETE FROM xfullname tridxby where_opt scanpt */ + -3, /* (277) trigger_cmd ::= scanpt select scanpt */ + -4, /* (278) expr ::= RAISE LP IGNORE RP */ + -6, /* (279) expr ::= RAISE LP raisetype COMMA expr RP */ + -1, /* (280) raisetype ::= ROLLBACK */ + -1, /* (281) raisetype ::= ABORT */ + -1, /* (282) raisetype ::= FAIL */ + -4, /* (283) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (284) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (285) cmd ::= DETACH database_kw_opt expr */ + 0, /* (286) key_opt ::= */ + -2, /* (287) key_opt ::= KEY expr */ + -1, /* (288) cmd ::= REINDEX */ + -3, /* (289) cmd ::= REINDEX nm dbnm */ + -1, /* (290) cmd ::= ANALYZE */ + -3, /* (291) cmd ::= ANALYZE nm dbnm */ + -6, /* (292) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -2, /* (293) cmd ::= alter_add carglist */ + -7, /* (294) alter_add ::= ALTER TABLE fullname ADD kwcolumn_opt nm typetoken */ -6, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (296) add_column_fullname ::= fullname */ - -8, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (298) cmd ::= create_vtab */ - -4, /* (299) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (301) vtabarg ::= */ - -1, /* (302) vtabargtoken ::= ANY */ - -3, /* (303) vtabargtoken ::= lp anylist RP */ - -1, /* (304) lp ::= LP */ - -2, /* (305) with ::= WITH wqlist */ - -3, /* (306) with ::= WITH RECURSIVE wqlist */ - -1, /* (307) wqas ::= AS */ - -2, /* (308) wqas ::= AS MATERIALIZED */ - -3, /* (309) wqas ::= AS NOT MATERIALIZED */ - -6, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ - -1, /* (311) withnm ::= nm */ - -1, /* (312) wqlist ::= wqitem */ - -3, /* (313) wqlist ::= wqlist COMMA wqitem */ - -3, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (315) windowdefn ::= nm AS LP window RP */ - -5, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (318) window ::= ORDER BY sortlist frame_opt */ - -5, /* (319) window ::= nm ORDER BY sortlist frame_opt */ - -2, /* (320) window ::= nm frame_opt */ - 0, /* (321) frame_opt ::= */ - -3, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (325) frame_bound_s ::= frame_bound */ - -2, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (327) frame_bound_e ::= frame_bound */ - -2, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (330) frame_bound ::= CURRENT ROW */ - 0, /* (331) frame_exclude_opt ::= */ - -2, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (333) frame_exclude ::= NO OTHERS */ - -2, /* (334) frame_exclude ::= CURRENT ROW */ - -1, /* (335) frame_exclude ::= GROUP|TIES */ - -2, /* (336) window_clause ::= WINDOW windowdefn_list */ - -2, /* (337) filter_over ::= filter_clause over_clause */ - -1, /* (338) filter_over ::= over_clause */ - -1, /* (339) filter_over ::= filter_clause */ - -4, /* (340) over_clause ::= OVER LP window RP */ - -2, /* (341) over_clause ::= OVER nm */ - -5, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (343) term ::= QNUMBER */ - -1, /* (344) input ::= cmdlist */ - -2, /* (345) cmdlist ::= cmdlist ecmd */ - -1, /* (346) cmdlist ::= ecmd */ - -1, /* (347) ecmd ::= SEMI */ - -2, /* (348) ecmd ::= cmdx SEMI */ - -3, /* (349) ecmd ::= explain cmdx SEMI */ - 0, /* (350) trans_opt ::= */ - -1, /* (351) trans_opt ::= TRANSACTION */ - -2, /* (352) trans_opt ::= TRANSACTION nm */ - -1, /* (353) savepoint_opt ::= SAVEPOINT */ - 0, /* (354) savepoint_opt ::= */ - -2, /* (355) cmd ::= create_table create_table_args */ - -1, /* (356) table_option_set ::= table_option */ - -4, /* (357) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (358) columnlist ::= columnname carglist */ - -1, /* (359) nm ::= ID|INDEXED|JOIN_KW */ - -1, /* (360) nm ::= STRING */ - -1, /* (361) typetoken ::= typename */ - -1, /* (362) typename ::= ID|STRING */ - -1, /* (363) signed ::= plus_num */ - -1, /* (364) signed ::= minus_num */ - -2, /* (365) carglist ::= carglist ccons */ - 0, /* (366) carglist ::= */ - -2, /* (367) ccons ::= NULL onconf */ - -4, /* (368) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (369) ccons ::= AS generated */ - -2, /* (370) conslist_opt ::= COMMA conslist */ - -3, /* (371) conslist ::= conslist tconscomma tcons */ - -1, /* (372) conslist ::= tcons */ - 0, /* (373) tconscomma ::= */ - -1, /* (374) defer_subclause_opt ::= defer_subclause */ - -1, /* (375) resolvetype ::= raisetype */ - -1, /* (376) selectnowith ::= oneselect */ - -1, /* (377) oneselect ::= values */ - -2, /* (378) sclp ::= selcollist COMMA */ - -1, /* (379) as ::= ID|STRING */ - -1, /* (380) indexed_opt ::= indexed_by */ - 0, /* (381) returning ::= */ - -1, /* (382) expr ::= term */ - -1, /* (383) likeop ::= LIKE_KW|MATCH */ - -1, /* (384) case_operand ::= expr */ - -1, /* (385) exprlist ::= nexprlist */ - -1, /* (386) nmnum ::= plus_num */ - -1, /* (387) nmnum ::= nm */ - -1, /* (388) nmnum ::= ON */ - -1, /* (389) nmnum ::= DELETE */ - -1, /* (390) nmnum ::= DEFAULT */ - -1, /* (391) plus_num ::= INTEGER|FLOAT */ - 0, /* (392) foreach_clause ::= */ - -3, /* (393) foreach_clause ::= FOR EACH ROW */ - -1, /* (394) trnm ::= nm */ - 0, /* (395) tridxby ::= */ - -1, /* (396) database_kw_opt ::= DATABASE */ - 0, /* (397) database_kw_opt ::= */ - 0, /* (398) kwcolumn_opt ::= */ - -1, /* (399) kwcolumn_opt ::= COLUMNKW */ - -1, /* (400) vtabarglist ::= vtabarg */ - -3, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (402) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (403) anylist ::= */ - -4, /* (404) anylist ::= anylist LP anylist RP */ - -2, /* (405) anylist ::= anylist ANY */ - 0, /* (406) with ::= */ - -1, /* (407) windowdefn_list ::= windowdefn */ - -1, /* (408) window ::= frame_opt */ + -8, /* (296) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -6, /* (297) cmd ::= ALTER TABLE fullname DROP CONSTRAINT nm */ + -9, /* (298) cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm DROP NOT NULL */ + -10, /* (299) cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm SET NOT NULL onconf */ + -11, /* (300) cmd ::= ALTER TABLE fullname ADD CONSTRAINT nm CHECK LP expr RP onconf */ + -9, /* (301) cmd ::= ALTER TABLE fullname ADD CHECK LP expr RP onconf */ + -1, /* (302) cmd ::= create_vtab */ + -4, /* (303) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (304) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (305) vtabarg ::= */ + -1, /* (306) vtabargtoken ::= ANY */ + -3, /* (307) vtabargtoken ::= lp anylist RP */ + -1, /* (308) lp ::= LP */ + -2, /* (309) with ::= WITH wqlist */ + -3, /* (310) with ::= WITH RECURSIVE wqlist */ + -1, /* (311) wqas ::= AS */ + -2, /* (312) wqas ::= AS MATERIALIZED */ + -3, /* (313) wqas ::= AS NOT MATERIALIZED */ + -6, /* (314) wqitem ::= withnm eidlist_opt wqas LP select RP */ + -1, /* (315) withnm ::= nm */ + -1, /* (316) wqlist ::= wqitem */ + -3, /* (317) wqlist ::= wqlist COMMA wqitem */ + -3, /* (318) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (319) windowdefn ::= nm AS LP window RP */ + -5, /* (320) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (321) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (322) window ::= ORDER BY sortlist frame_opt */ + -5, /* (323) window ::= nm ORDER BY sortlist frame_opt */ + -2, /* (324) window ::= nm frame_opt */ + 0, /* (325) frame_opt ::= */ + -3, /* (326) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (327) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (328) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (329) frame_bound_s ::= frame_bound */ + -2, /* (330) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (331) frame_bound_e ::= frame_bound */ + -2, /* (332) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (333) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (334) frame_bound ::= CURRENT ROW */ + 0, /* (335) frame_exclude_opt ::= */ + -2, /* (336) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (337) frame_exclude ::= NO OTHERS */ + -2, /* (338) frame_exclude ::= CURRENT ROW */ + -1, /* (339) frame_exclude ::= GROUP|TIES */ + -2, /* (340) window_clause ::= WINDOW windowdefn_list */ + -2, /* (341) filter_over ::= filter_clause over_clause */ + -1, /* (342) filter_over ::= over_clause */ + -1, /* (343) filter_over ::= filter_clause */ + -4, /* (344) over_clause ::= OVER LP window RP */ + -2, /* (345) over_clause ::= OVER nm */ + -5, /* (346) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (347) term ::= QNUMBER */ + -1, /* (348) input ::= cmdlist */ + -2, /* (349) cmdlist ::= cmdlist ecmd */ + -1, /* (350) cmdlist ::= ecmd */ + -1, /* (351) ecmd ::= SEMI */ + -2, /* (352) ecmd ::= cmdx SEMI */ + -3, /* (353) ecmd ::= explain cmdx SEMI */ + 0, /* (354) trans_opt ::= */ + -1, /* (355) trans_opt ::= TRANSACTION */ + -2, /* (356) trans_opt ::= TRANSACTION nm */ + -1, /* (357) savepoint_opt ::= SAVEPOINT */ + 0, /* (358) savepoint_opt ::= */ + -2, /* (359) cmd ::= create_table create_table_args */ + -1, /* (360) table_option_set ::= table_option */ + -4, /* (361) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (362) columnlist ::= columnname carglist */ + -1, /* (363) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (364) nm ::= STRING */ + -1, /* (365) typetoken ::= typename */ + -1, /* (366) typename ::= ID|STRING */ + -1, /* (367) signed ::= plus_num */ + -1, /* (368) signed ::= minus_num */ + -2, /* (369) carglist ::= carglist ccons */ + 0, /* (370) carglist ::= */ + -2, /* (371) ccons ::= NULL onconf */ + -4, /* (372) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (373) ccons ::= AS generated */ + -2, /* (374) conslist_opt ::= COMMA conslist */ + -3, /* (375) conslist ::= conslist tconscomma tcons */ + -1, /* (376) conslist ::= tcons */ + 0, /* (377) tconscomma ::= */ + -1, /* (378) defer_subclause_opt ::= defer_subclause */ + -1, /* (379) resolvetype ::= raisetype */ + -1, /* (380) selectnowith ::= oneselect */ + -1, /* (381) oneselect ::= values */ + -2, /* (382) sclp ::= selcollist COMMA */ + -1, /* (383) as ::= ID|STRING */ + -1, /* (384) indexed_opt ::= indexed_by */ + 0, /* (385) returning ::= */ + -1, /* (386) expr ::= term */ + -1, /* (387) likeop ::= LIKE_KW|MATCH */ + -1, /* (388) case_operand ::= expr */ + -1, /* (389) exprlist ::= nexprlist */ + -1, /* (390) nmnum ::= plus_num */ + -1, /* (391) nmnum ::= nm */ + -1, /* (392) nmnum ::= ON */ + -1, /* (393) nmnum ::= DELETE */ + -1, /* (394) nmnum ::= DEFAULT */ + -1, /* (395) plus_num ::= INTEGER|FLOAT */ + 0, /* (396) foreach_clause ::= */ + -3, /* (397) foreach_clause ::= FOR EACH ROW */ + 0, /* (398) tridxby ::= */ + -1, /* (399) database_kw_opt ::= DATABASE */ + 0, /* (400) database_kw_opt ::= */ + 0, /* (401) kwcolumn_opt ::= */ + -1, /* (402) kwcolumn_opt ::= COLUMNKW */ + -1, /* (403) vtabarglist ::= vtabarg */ + -3, /* (404) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (405) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (406) anylist ::= */ + -4, /* (407) anylist ::= anylist LP anylist RP */ + -2, /* (408) anylist ::= anylist ANY */ + 0, /* (409) with ::= */ + -1, /* (410) windowdefn_list ::= windowdefn */ + -1, /* (411) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -178961,16 +183399,16 @@ static YYACTIONTYPE yy_reduce( { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy502);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy144);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy502 = TK_DEFERRED;} +{yymsp[1].minor.yy144 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324); -{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/} + case 328: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==328); +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); @@ -178993,7 +183431,7 @@ static YYACTIONTYPE yy_reduce( break; case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy502,0,0,yymsp[-2].minor.yy502); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy144,0,0,yymsp[-2].minor.yy144); } break; case 14: /* createkw ::= CREATE */ @@ -179009,38 +183447,38 @@ static YYACTIONTYPE yy_reduce( case 81: /* ifexists ::= */ yytestcase(yyruleno==81); case 100: /* distinct ::= */ yytestcase(yyruleno==100); case 246: /* collate ::= */ yytestcase(yyruleno==246); -{yymsp[1].minor.yy502 = 0;} +{yymsp[1].minor.yy144 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy502 = 1;} +{yymsp[-2].minor.yy144 = 1;} break; case 17: /* temp ::= TEMP */ -{yymsp[0].minor.yy502 = pParse->db->init.busy==0;} +{yymsp[0].minor.yy144 = pParse->db->init.busy==0;} break; case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy9,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy391,0); } break; case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy637); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy637); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy555); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 21: /* table_option_set ::= */ -{yymsp[1].minor.yy9 = 0;} +{yymsp[1].minor.yy391 = 0;} break; case 22: /* table_option_set ::= table_option_set COMMA table_option */ -{yylhsminor.yy9 = yymsp[-2].minor.yy9|yymsp[0].minor.yy9;} - yymsp[-2].minor.yy9 = yylhsminor.yy9; +{yylhsminor.yy391 = yymsp[-2].minor.yy391|yymsp[0].minor.yy391;} + yymsp[-2].minor.yy391 = yylhsminor.yy391; break; case 23: /* table_option ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy9 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy391 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy9 = 0; + yymsp[-1].minor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -179048,13 +183486,13 @@ static YYACTIONTYPE yy_reduce( case 24: /* table_option ::= nm */ { if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ - yylhsminor.yy9 = TF_Strict; + yylhsminor.yy391 = TF_Strict; }else{ - yylhsminor.yy9 = 0; + yylhsminor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } - yymsp[0].minor.yy9 = yylhsminor.yy9; + yymsp[0].minor.yy391 = yylhsminor.yy391; break; case 25: /* columnname ::= nm typetoken */ {sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} @@ -179080,7 +183518,7 @@ static YYACTIONTYPE yy_reduce( case 30: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy342 = yyLookaheadToken.z; + yymsp[1].minor.yy168 = yyLookaheadToken.z; } break; case 31: /* scantok ::= */ @@ -179094,17 +183532,17 @@ static YYACTIONTYPE yy_reduce( {ASSERT_IS_CREATE; pParse->u1.cr.constraintName = yymsp[0].minor.yy0;} break; case 33: /* ccons ::= DEFAULT scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy590,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 34: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy590,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; case 35: /* ccons ::= DEFAULT PLUS scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy590,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 36: /* ccons ::= DEFAULT MINUS scantok term */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy590, 0); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy454, 0); sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; @@ -179119,133 +183557,133 @@ static YYACTIONTYPE yy_reduce( } break; case 38: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy502);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy144);} break; case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy502,yymsp[0].minor.yy502,yymsp[-2].minor.yy502);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy144,yymsp[0].minor.yy144,yymsp[-2].minor.yy144);} break; case 40: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy502,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 41: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy590,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} break; case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy402,yymsp[0].minor.yy502);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy144);} break; case 43: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy502);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy144);} break; case 44: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 45: /* generated ::= LP expr RP */ -{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy590,0);} +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy454,0);} break; case 46: /* generated ::= LP expr RP ID */ -{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy590,&yymsp[0].minor.yy0);} +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy454,&yymsp[0].minor.yy0);} break; case 48: /* autoinc ::= AUTOINCR */ -{yymsp[0].minor.yy502 = 1;} +{yymsp[0].minor.yy144 = 1;} break; case 49: /* refargs ::= */ -{ yymsp[1].minor.yy502 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yymsp[1].minor.yy144 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 50: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy502 = (yymsp[-1].minor.yy502 & ~yymsp[0].minor.yy481.mask) | yymsp[0].minor.yy481.value; } +{ yymsp[-1].minor.yy144 = (yymsp[-1].minor.yy144 & ~yymsp[0].minor.yy383.mask) | yymsp[0].minor.yy383.value; } break; case 51: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy481.value = 0; yymsp[-1].minor.yy481.mask = 0x000000; } +{ yymsp[-1].minor.yy383.value = 0; yymsp[-1].minor.yy383.mask = 0x000000; } break; case 52: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy481.value = 0; yymsp[-2].minor.yy481.mask = 0x000000; } +{ yymsp[-2].minor.yy383.value = 0; yymsp[-2].minor.yy383.mask = 0x000000; } break; case 53: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy481.value = yymsp[0].minor.yy502; yymsp[-2].minor.yy481.mask = 0x0000ff; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144; yymsp[-2].minor.yy383.mask = 0x0000ff; } break; case 54: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy481.value = yymsp[0].minor.yy502<<8; yymsp[-2].minor.yy481.mask = 0x00ff00; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144<<8; yymsp[-2].minor.yy383.mask = 0x00ff00; } break; case 55: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy502 = OE_SetNull; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetNull; /* EV: R-33326-45252 */} break; case 56: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy502 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 57: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy502 = OE_Cascade; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Cascade; /* EV: R-33326-45252 */} break; case 58: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy502 = OE_Restrict; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Restrict; /* EV: R-33326-45252 */} break; case 59: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy502 = OE_None; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_None; /* EV: R-33326-45252 */} break; case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy502 = 0;} +{yymsp[-2].minor.yy144 = 0;} break; case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173); -{yymsp[-1].minor.yy502 = yymsp[0].minor.yy502;} +{yymsp[-1].minor.yy144 = yymsp[0].minor.yy144;} break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); -{yymsp[-1].minor.yy502 = 1;} +{yymsp[-1].minor.yy144 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy502 = 0;} +{yymsp[-1].minor.yy144 = 0;} break; case 66: /* tconscomma ::= COMMA */ {ASSERT_IS_CREATE; pParse->u1.cr.constraintName.n = 0;} break; case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy402,yymsp[0].minor.yy502,yymsp[-2].minor.yy502,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy144,yymsp[-2].minor.yy144,0);} break; case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy402,yymsp[0].minor.yy502,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 70: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy590,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy454,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} break; case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy402, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy402, yymsp[-1].minor.yy502); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy502); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy144); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy144); } break; case 73: /* onconf ::= */ case 75: /* orconf ::= */ yytestcase(yyruleno==75); -{yymsp[1].minor.yy502 = OE_Default;} +{yymsp[1].minor.yy144 = OE_Default;} break; case 74: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy502 = yymsp[0].minor.yy502;} +{yymsp[-2].minor.yy144 = yymsp[0].minor.yy144;} break; case 77: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy502 = OE_Ignore;} +{yymsp[0].minor.yy144 = OE_Ignore;} break; case 78: /* resolvetype ::= REPLACE */ case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174); -{yymsp[0].minor.yy502 = OE_Replace;} +{yymsp[0].minor.yy144 = OE_Replace;} break; case 79: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy563, 0, yymsp[-1].minor.yy502); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 0, yymsp[-1].minor.yy144); } break; case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy402, yymsp[0].minor.yy637, yymsp[-7].minor.yy502, yymsp[-5].minor.yy502); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy555, yymsp[-7].minor.yy144, yymsp[-5].minor.yy144); } break; case 83: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy563, 1, yymsp[-1].minor.yy502); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 1, yymsp[-1].minor.yy144); } break; case 84: /* cmd ::= select */ @@ -179254,20 +183692,20 @@ static YYACTIONTYPE yy_reduce( if( (pParse->db->mDbFlags & DBFLAG_EncodingFixed)!=0 || sqlite3ReadSchema(pParse)==SQLITE_OK ){ - sqlite3Select(pParse, yymsp[0].minor.yy637, &dest); + sqlite3Select(pParse, yymsp[0].minor.yy555, &dest); } - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy637); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 85: /* select ::= WITH wqlist selectnowith */ -{yymsp[-2].minor.yy637 = attachWithToSelect(pParse,yymsp[0].minor.yy637,yymsp[-1].minor.yy125);} +{yymsp[-2].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ -{yymsp[-3].minor.yy637 = attachWithToSelect(pParse,yymsp[0].minor.yy637,yymsp[-1].minor.yy125);} +{yymsp[-3].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 87: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy637; + Select *p = yymsp[0].minor.yy555; if( p ){ parserDoubleLinkSelect(pParse, p); } @@ -179275,8 +183713,8 @@ static YYACTIONTYPE yy_reduce( break; case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy637; - Select *pLhs = yymsp[-2].minor.yy637; + Select *pRhs = yymsp[0].minor.yy555; + Select *pLhs = yymsp[-2].minor.yy555; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -179286,60 +183724,60 @@ static YYACTIONTYPE yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy502; + pRhs->op = (u8)yymsp[-1].minor.yy144; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~(u32)SF_MultiValue; pRhs->selFlags &= ~(u32)SF_MultiValue; - if( yymsp[-1].minor.yy502!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy144!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy637 = pRhs; + yymsp[-2].minor.yy555 = pRhs; } break; case 89: /* multiselect_op ::= UNION */ case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); -{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-OP*/} +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-OP*/} break; case 90: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy502 = TK_ALL;} +{yymsp[-1].minor.yy144 = TK_ALL;} break; case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yymsp[-8].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy402,yymsp[-5].minor.yy563,yymsp[-4].minor.yy590,yymsp[-3].minor.yy402,yymsp[-2].minor.yy590,yymsp[-1].minor.yy402,yymsp[-7].minor.yy502,yymsp[0].minor.yy590); + yymsp[-8].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy203,yymsp[-4].minor.yy454,yymsp[-3].minor.yy14,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[-7].minor.yy144,yymsp[0].minor.yy454); } break; case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-9].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy402,yymsp[-6].minor.yy563,yymsp[-5].minor.yy590,yymsp[-4].minor.yy402,yymsp[-3].minor.yy590,yymsp[-1].minor.yy402,yymsp[-8].minor.yy502,yymsp[0].minor.yy590); - if( yymsp[-9].minor.yy637 ){ - yymsp[-9].minor.yy637->pWinDefn = yymsp[-2].minor.yy483; + yymsp[-9].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy14,yymsp[-6].minor.yy203,yymsp[-5].minor.yy454,yymsp[-4].minor.yy14,yymsp[-3].minor.yy454,yymsp[-1].minor.yy14,yymsp[-8].minor.yy144,yymsp[0].minor.yy454); + if( yymsp[-9].minor.yy555 ){ + yymsp[-9].minor.yy555->pWinDefn = yymsp[-2].minor.yy211; }else{ - sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy483); + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy211); } } break; case 94: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy402,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0); } break; case 95: /* oneselect ::= mvalues */ { - sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy637); + sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy555); } break; case 96: /* mvalues ::= values COMMA LP nexprlist RP */ case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97); { - yymsp[-4].minor.yy637 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy637, yymsp[-1].minor.yy402); + yymsp[-4].minor.yy555 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy555, yymsp[-1].minor.yy14); } break; case 98: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy502 = SF_Distinct;} +{yymsp[0].minor.yy144 = SF_Distinct;} break; case 99: /* distinct ::= ALL */ -{yymsp[0].minor.yy502 = SF_All;} +{yymsp[0].minor.yy144 = SF_All;} break; case 101: /* sclp ::= */ case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134); @@ -179347,20 +183785,20 @@ static YYACTIONTYPE yy_reduce( case 234: /* exprlist ::= */ yytestcase(yyruleno==234); case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237); case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242); -{yymsp[1].minor.yy402 = 0;} +{yymsp[1].minor.yy14 = 0;} break; case 102: /* selcollist ::= sclp scanpt expr scanpt as */ { - yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy402, yymsp[-2].minor.yy590); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy402, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy402,yymsp[-3].minor.yy342,yymsp[-1].minor.yy342); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy14,yymsp[-3].minor.yy168,yymsp[-1].minor.yy168); } break; case 103: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); - yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy402, p); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, p); } break; case 104: /* selcollist ::= sclp scanpt nm DOT STAR */ @@ -179370,7 +183808,7 @@ static YYACTIONTYPE yy_reduce( sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, pDot); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, pDot); } break; case 105: /* as ::= AS nm */ @@ -179381,50 +183819,50 @@ static YYACTIONTYPE yy_reduce( break; case 107: /* from ::= */ case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110); -{yymsp[1].minor.yy563 = 0;} +{yymsp[1].minor.yy203 = 0;} break; case 108: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy563 = yymsp[0].minor.yy563; - sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy563); + yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; + sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy203); } break; case 109: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy563 && yymsp[-1].minor.yy563->nSrc>0) ) yymsp[-1].minor.yy563->a[yymsp[-1].minor.yy563->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy502; + if( ALWAYS(yymsp[-1].minor.yy203 && yymsp[-1].minor.yy203->nSrc>0) ) yymsp[-1].minor.yy203->a[yymsp[-1].minor.yy203->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy144; } break; case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */ { - yymsp[-4].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy563,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421); + yymsp[-4].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy203,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); } break; case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ { - yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy421); - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy563, &yymsp[-1].minor.yy0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-1].minor.yy0); } break; case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ { - yymsp[-7].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy563,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421); - sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy563, yymsp[-3].minor.yy402); + yymsp[-7].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy203,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy203, yymsp[-3].minor.yy14); } break; case 114: /* seltablist ::= stl_prefix LP select RP as on_using */ { - yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy637,&yymsp[0].minor.yy421); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy555,&yymsp[0].minor.yy269); } break; case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ { - if( yymsp[-5].minor.yy563==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy421.pOn==0 && yymsp[0].minor.yy421.pUsing==0 ){ - yymsp[-5].minor.yy563 = yymsp[-3].minor.yy563; - }else if( ALWAYS(yymsp[-3].minor.yy563!=0) && yymsp[-3].minor.yy563->nSrc==1 ){ - yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421); - if( yymsp[-5].minor.yy563 ){ - SrcItem *pNew = &yymsp[-5].minor.yy563->a[yymsp[-5].minor.yy563->nSrc-1]; - SrcItem *pOld = yymsp[-3].minor.yy563->a; + if( yymsp[-5].minor.yy203==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy269.pOn==0 && yymsp[0].minor.yy269.pUsing==0 ){ + yymsp[-5].minor.yy203 = yymsp[-3].minor.yy203; + }else if( ALWAYS(yymsp[-3].minor.yy203!=0) && yymsp[-3].minor.yy203->nSrc==1 ){ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + if( yymsp[-5].minor.yy203 ){ + SrcItem *pNew = &yymsp[-5].minor.yy203->a[yymsp[-5].minor.yy203->nSrc-1]; + SrcItem *pOld = yymsp[-3].minor.yy203->a; assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; assert( pOld->fg.fixedSchema==0 ); @@ -179449,12 +183887,12 @@ static YYACTIONTYPE yy_reduce( } pOld->zName = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy563); + sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy203); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy563); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy563,0,0,0,0,SF_NestedFrom,0); - yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy421); + sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy203); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy203,0,0,0,0,SF_NestedFrom,0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy269); } } break; @@ -179463,57 +183901,67 @@ static YYACTIONTYPE yy_reduce( {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; case 118: /* fullname ::= nm */ + case 120: /* xfullname ::= nm */ yytestcase(yyruleno==120); { - yylhsminor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); - if( IN_RENAME_OBJECT && yylhsminor.yy563 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy563->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy563 = yylhsminor.yy563; + yymsp[0].minor.yy203 = yylhsminor.yy203; break; case 119: /* fullname ::= nm DOT nm */ + case 121: /* xfullname ::= nm DOT nm */ yytestcase(yyruleno==121); { - yylhsminor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - if( IN_RENAME_OBJECT && yylhsminor.yy563 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy563->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy563 = yylhsminor.yy563; - break; - case 120: /* xfullname ::= nm */ -{yymsp[0].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + yymsp[-2].minor.yy203 = yylhsminor.yy203; break; - case 121: /* xfullname ::= nm DOT nm */ -{yymsp[-2].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 122: /* xfullname ::= nm DOT nm AS nm */ + case 122: /* xfullname ::= nm AS nm */ { - yymsp[-4].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy563 ) yymsp[-4].minor.yy563->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); + if( yylhsminor.yy203 ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[-2].minor.yy0); + }else{ + yylhsminor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + } + } } + yymsp[-2].minor.yy203 = yylhsminor.yy203; break; - case 123: /* xfullname ::= nm AS nm */ + case 123: /* xfullname ::= nm DOT nm AS nm */ { - yymsp[-2].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy563 ) yymsp[-2].minor.yy563->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); + if( yylhsminor.yy203 ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[-2].minor.yy0); + }else{ + yylhsminor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + } + } } + yymsp[-4].minor.yy203 = yylhsminor.yy203; break; case 124: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy502 = JT_INNER; } +{ yymsp[0].minor.yy144 = JT_INNER; } break; case 125: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} +{yymsp[-1].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; case 126: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} +{yymsp[-2].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; case 127: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} +{yymsp[-3].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; case 128: /* on_using ::= ON expr */ -{yymsp[-1].minor.yy421.pOn = yymsp[0].minor.yy590; yymsp[-1].minor.yy421.pUsing = 0;} +{yymsp[-1].minor.yy269.pOn = yymsp[0].minor.yy454; yymsp[-1].minor.yy269.pUsing = 0;} break; case 129: /* on_using ::= USING LP idlist RP */ -{yymsp[-3].minor.yy421.pOn = 0; yymsp[-3].minor.yy421.pUsing = yymsp[-1].minor.yy204;} +{yymsp[-3].minor.yy269.pOn = 0; yymsp[-3].minor.yy269.pUsing = yymsp[-1].minor.yy132;} break; case 130: /* on_using ::= */ -{yymsp[1].minor.yy421.pOn = 0; yymsp[1].minor.yy421.pUsing = 0;} +{yymsp[1].minor.yy269.pOn = 0; yymsp[1].minor.yy269.pUsing = 0;} break; case 132: /* indexed_by ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} @@ -179523,35 +183971,35 @@ static YYACTIONTYPE yy_reduce( break; case 135: /* orderby_opt ::= ORDER BY sortlist */ case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145); -{yymsp[-2].minor.yy402 = yymsp[0].minor.yy402;} +{yymsp[-2].minor.yy14 = yymsp[0].minor.yy14;} break; case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402,yymsp[-2].minor.yy590); - sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy402,yymsp[-1].minor.yy502,yymsp[0].minor.yy502); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; case 137: /* sortlist ::= expr sortorder nulls */ { - yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy590); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy402,yymsp[-1].minor.yy502,yymsp[0].minor.yy502); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy454); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; case 138: /* sortorder ::= ASC */ -{yymsp[0].minor.yy502 = SQLITE_SO_ASC;} +{yymsp[0].minor.yy144 = SQLITE_SO_ASC;} break; case 139: /* sortorder ::= DESC */ -{yymsp[0].minor.yy502 = SQLITE_SO_DESC;} +{yymsp[0].minor.yy144 = SQLITE_SO_DESC;} break; case 140: /* sortorder ::= */ case 143: /* nulls ::= */ yytestcase(yyruleno==143); -{yymsp[1].minor.yy502 = SQLITE_SO_UNDEFINED;} +{yymsp[1].minor.yy144 = SQLITE_SO_UNDEFINED;} break; case 141: /* nulls ::= NULLS FIRST */ -{yymsp[-1].minor.yy502 = SQLITE_SO_ASC;} +{yymsp[-1].minor.yy144 = SQLITE_SO_ASC;} break; case 142: /* nulls ::= NULLS LAST */ -{yymsp[-1].minor.yy502 = SQLITE_SO_DESC;} +{yymsp[-1].minor.yy144 = SQLITE_SO_DESC;} break; case 146: /* having_opt ::= */ case 148: /* limit_opt ::= */ yytestcase(yyruleno==148); @@ -179560,42 +184008,42 @@ static YYACTIONTYPE yy_reduce( case 232: /* case_else ::= */ yytestcase(yyruleno==232); case 233: /* case_operand ::= */ yytestcase(yyruleno==233); case 252: /* vinto ::= */ yytestcase(yyruleno==252); -{yymsp[1].minor.yy590 = 0;} +{yymsp[1].minor.yy454 = 0;} break; case 147: /* having_opt ::= HAVING expr */ case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154); case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156); case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251); -{yymsp[-1].minor.yy590 = yymsp[0].minor.yy590;} +{yymsp[-1].minor.yy454 = yymsp[0].minor.yy454;} break; case 149: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy590,0);} +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,0);} break; case 150: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);} +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; case 151: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy590,yymsp[-2].minor.yy590);} +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,yymsp[-2].minor.yy454);} break; case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy563, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy563,yymsp[0].minor.yy590,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy203, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy203,yymsp[0].minor.yy454,0,0); } break; case 157: /* where_opt_ret ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy402); yymsp[-1].minor.yy590 = 0;} +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-1].minor.yy454 = 0;} break; case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy402); yymsp[-3].minor.yy590 = yymsp[-2].minor.yy590;} +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-3].minor.yy454 = yymsp[-2].minor.yy454;} break; case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy563, &yymsp[-4].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy402,"set list"); - if( yymsp[-1].minor.yy563 ){ - SrcList *pFromClause = yymsp[-1].minor.yy563; + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy14,"set list"); + if( yymsp[-1].minor.yy203 ){ + SrcList *pFromClause = yymsp[-1].minor.yy203; if( pFromClause->nSrc>1 ){ Select *pSubquery; Token as; @@ -179604,90 +184052,90 @@ static YYACTIONTYPE yy_reduce( as.z = 0; pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); } - yymsp[-5].minor.yy563 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy563, pFromClause); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy203, pFromClause); } - sqlite3Update(pParse,yymsp[-5].minor.yy563,yymsp[-2].minor.yy402,yymsp[0].minor.yy590,yymsp[-6].minor.yy502,0,0,0); + sqlite3Update(pParse,yymsp[-5].minor.yy203,yymsp[-2].minor.yy14,yymsp[0].minor.yy454,yymsp[-6].minor.yy144,0,0,0); } break; case 160: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy402, yymsp[0].minor.yy590); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy402, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, 1); } break; case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy402 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy402, yymsp[-3].minor.yy204, yymsp[0].minor.yy590); + yymsp[-6].minor.yy14 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy14, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; case 162: /* setlist ::= nm EQ expr */ { - yylhsminor.yy402 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy590); - sqlite3ExprListSetName(pParse, yylhsminor.yy402, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yylhsminor.yy14, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy402 = yylhsminor.yy402; + yymsp[-2].minor.yy14 = yylhsminor.yy14; break; case 163: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy402 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy204, yymsp[0].minor.yy590); + yymsp[-4].minor.yy14 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy563, yymsp[-1].minor.yy637, yymsp[-2].minor.yy204, yymsp[-5].minor.yy502, yymsp[0].minor.yy403); + sqlite3Insert(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy555, yymsp[-2].minor.yy132, yymsp[-5].minor.yy144, yymsp[0].minor.yy122); } break; case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ { - sqlite3Insert(pParse, yymsp[-4].minor.yy563, 0, yymsp[-3].minor.yy204, yymsp[-6].minor.yy502, 0); + sqlite3Insert(pParse, yymsp[-4].minor.yy203, 0, yymsp[-3].minor.yy132, yymsp[-6].minor.yy144, 0); } break; case 166: /* upsert ::= */ -{ yymsp[1].minor.yy403 = 0; } +{ yymsp[1].minor.yy122 = 0; } break; case 167: /* upsert ::= RETURNING selcollist */ -{ yymsp[-1].minor.yy403 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy402); } +{ yymsp[-1].minor.yy122 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy14); } break; case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ -{ yymsp[-11].minor.yy403 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy402,yymsp[-6].minor.yy590,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590,yymsp[0].minor.yy403);} +{ yymsp[-11].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy14,yymsp[-6].minor.yy454,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,yymsp[0].minor.yy122);} break; case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ -{ yymsp[-8].minor.yy403 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy402,yymsp[-3].minor.yy590,0,0,yymsp[0].minor.yy403); } +{ yymsp[-8].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy14,yymsp[-3].minor.yy454,0,0,yymsp[0].minor.yy122); } break; case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */ -{ yymsp[-4].minor.yy403 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } +{ yymsp[-4].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } break; case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ -{ yymsp[-7].minor.yy403 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590,0);} +{ yymsp[-7].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,0);} break; case 172: /* returning ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy402);} +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14);} break; case 175: /* idlist_opt ::= */ -{yymsp[1].minor.yy204 = 0;} +{yymsp[1].minor.yy132 = 0;} break; case 176: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy204 = yymsp[-1].minor.yy204;} +{yymsp[-2].minor.yy132 = yymsp[-1].minor.yy132;} break; case 177: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy204 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy204,&yymsp[0].minor.yy0);} +{yymsp[-2].minor.yy132 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy132,&yymsp[0].minor.yy0);} break; case 178: /* idlist ::= nm */ -{yymsp[0].minor.yy204 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} +{yymsp[0].minor.yy132 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; case 179: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy590 = yymsp[-1].minor.yy590;} +{yymsp[-2].minor.yy454 = yymsp[-1].minor.yy454;} break; case 180: /* expr ::= ID|INDEXED|JOIN_KW */ -{yymsp[0].minor.yy590=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} +{yymsp[0].minor.yy454=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; case 181: /* expr ::= nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); - yylhsminor.yy590 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy590 = yylhsminor.yy590; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; case 182: /* expr ::= nm DOT nm DOT nm */ { @@ -179698,27 +184146,32 @@ static YYACTIONTYPE yy_reduce( if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, 0, temp1); } - yylhsminor.yy590 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy590 = yylhsminor.yy590; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; case 183: /* term ::= NULL|FLOAT|BLOB */ case 184: /* term ::= STRING */ yytestcase(yyruleno==184); -{yymsp[0].minor.yy590=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} +{yymsp[0].minor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; case 185: /* term ::= INTEGER */ { - yylhsminor.yy590 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); - if( yylhsminor.yy590 ) yylhsminor.yy590->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); + int iValue; + if( sqlite3GetInt32(yymsp[0].minor.yy0.z, &iValue)==0 ){ + yylhsminor.yy454 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 0); + }else{ + yylhsminor.yy454 = sqlite3ExprInt32(pParse->db, iValue); + } + if( yylhsminor.yy454 ) yylhsminor.yy454->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } - yymsp[0].minor.yy590 = yylhsminor.yy590; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; case 186: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy590 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy590, n); + yymsp[0].minor.yy454 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy454, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -179727,80 +184180,80 @@ static YYACTIONTYPE yy_reduce( assert( t.n>=2 ); if( pParse->nested==0 ){ parserSyntaxError(pParse, &t); - yymsp[0].minor.yy590 = 0; + yymsp[0].minor.yy454 = 0; }else{ - yymsp[0].minor.yy590 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy590 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy590->iTable); + yymsp[0].minor.yy454 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy454 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy454->iTable); } } } break; case 187: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy590 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy590, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy454 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy454, &yymsp[0].minor.yy0, 1); } break; case 188: /* expr ::= CAST LP expr AS typetoken RP */ { - yymsp[-5].minor.yy590 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy590, yymsp[-3].minor.yy590, 0); + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy454, yymsp[-3].minor.yy454, 0); } break; case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy402, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy502); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy144); } - yymsp[-4].minor.yy590 = yylhsminor.yy590; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy402, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy502); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy590, yymsp[-1].minor.yy402); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy14, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy144); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-1].minor.yy14); } - yymsp[-7].minor.yy590 = yylhsminor.yy590; + yymsp[-7].minor.yy454 = yylhsminor.yy454; break; case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-3].minor.yy590 = yylhsminor.yy590; + yymsp[-3].minor.yy454 = yylhsminor.yy454; break; case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy402, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy502); - sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy14, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-5].minor.yy590 = yylhsminor.yy590; + yymsp[-5].minor.yy454 = yylhsminor.yy454; break; case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy402, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy502); - sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy590, yymsp[-2].minor.yy402); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy14, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-2].minor.yy14); } - yymsp[-8].minor.yy590 = yylhsminor.yy590; + yymsp[-8].minor.yy454 = yylhsminor.yy454; break; case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); - sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-4].minor.yy590 = yylhsminor.yy590; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; case 195: /* term ::= CTIME_KW */ { - yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } - yymsp[0].minor.yy590 = yylhsminor.yy590; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; case 196: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy402, yymsp[-1].minor.yy590); - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy590 ){ - yymsp[-4].minor.yy590->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; if( ALWAYS(pList->nExpr) ){ - yymsp[-4].minor.yy590->flags |= pList->a[0].pExpr->flags & EP_Propagate; + yymsp[-4].minor.yy454->flags |= pList->a[0].pExpr->flags & EP_Propagate; } }else{ sqlite3ExprListDelete(pParse->db, pList); @@ -179808,7 +184261,7 @@ static YYACTIONTYPE yy_reduce( } break; case 197: /* expr ::= expr AND expr */ -{yymsp[-2].minor.yy590=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);} +{yymsp[-2].minor.yy454=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; case 198: /* expr ::= expr OR expr */ case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199); @@ -179817,7 +184270,7 @@ static YYACTIONTYPE yy_reduce( case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202); case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203); case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204); -{yymsp[-2].minor.yy590=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);} +{yymsp[-2].minor.yy454=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; case 205: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} @@ -179827,11 +184280,11 @@ static YYACTIONTYPE yy_reduce( ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy590); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy590); - yymsp[-2].minor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); - if( bNot ) yymsp[-2].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy590, 0); - if( yymsp[-2].minor.yy590 ) yymsp[-2].minor.yy590->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy454); + yymsp[-2].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy454, 0); + if( yymsp[-2].minor.yy454 ) yymsp[-2].minor.yy454->flags |= EP_InfixFunc; } break; case 207: /* expr ::= expr likeop expr ESCAPE expr */ @@ -179839,91 +184292,88 @@ static YYACTIONTYPE yy_reduce( ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy590); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy590); - yymsp[-4].minor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); - if( bNot ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); - if( yymsp[-4].minor.yy590 ) yymsp[-4].minor.yy590->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ) yymsp[-4].minor.yy454->flags |= EP_InfixFunc; } break; case 208: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy590,0);} +{yymsp[-1].minor.yy454 = sqlite3PExprIsNull(pParse,yymsp[0].major,yymsp[-1].minor.yy454);} break; case 209: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy590 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy590,0);} +{yymsp[-2].minor.yy454 = sqlite3PExprIsNull(pParse,TK_NOTNULL,yymsp[-2].minor.yy454);} break; case 210: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy590 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy590,yymsp[0].minor.yy590); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-2].minor.yy590, TK_ISNULL); + yymsp[-2].minor.yy454 = sqlite3PExprIs(pParse, TK_IS, yymsp[-2].minor.yy454, yymsp[0].minor.yy454); } break; case 211: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy590,yymsp[0].minor.yy590); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-3].minor.yy590, TK_NOTNULL); + yymsp[-3].minor.yy454 = sqlite3PExprIs(pParse, TK_ISNOT, yymsp[-3].minor.yy454, yymsp[0].minor.yy454); } break; case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */ { - yymsp[-5].minor.yy590 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy590,yymsp[0].minor.yy590); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-5].minor.yy590, TK_ISNULL); + yymsp[-5].minor.yy454 = sqlite3PExprIs(pParse, TK_IS, yymsp[-5].minor.yy454, yymsp[0].minor.yy454); } break; case 213: /* expr ::= expr IS DISTINCT FROM expr */ { - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy590,yymsp[0].minor.yy590); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-4].minor.yy590, TK_NOTNULL); + yymsp[-4].minor.yy454 = sqlite3PExprIs(pParse, TK_ISNOT, yymsp[-4].minor.yy454, yymsp[0].minor.yy454); } break; case 214: /* expr ::= NOT expr */ case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); -{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy590, 0);/*A-overwrites-B*/} +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy454, 0);/*A-overwrites-B*/} break; case 216: /* expr ::= PLUS|MINUS expr */ { - Expr *p = yymsp[0].minor.yy590; + Expr *p = yymsp[0].minor.yy454; u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS); assert( TK_UPLUS>TK_PLUS ); assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); if( p && p->op==TK_UPLUS ){ p->op = op; - yymsp[-1].minor.yy590 = p; + yymsp[-1].minor.yy454 = p; }else{ - yymsp[-1].minor.yy590 = sqlite3PExpr(pParse, op, p, 0); + yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, op, p, 0); /*A-overwrites-B*/ } } break; case 217: /* expr ::= expr PTR expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy590); - pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy590); - yylhsminor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy454); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); } - yymsp[-2].minor.yy590 = yylhsminor.yy590; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; case 218: /* between_op ::= BETWEEN */ case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); -{yymsp[0].minor.yy502 = 0;} +{yymsp[0].minor.yy144 = 0;} break; case 220: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy590); - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy590, 0); - if( yymsp[-4].minor.yy590 ){ - yymsp[-4].minor.yy590->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; case 223: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy402==0 ){ + if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -179936,145 +184386,145 @@ static YYACTIONTYPE yy_reduce( ** it is or not) and if it is an aggregate, that could change the meaning ** of the whole query. */ - Expr *pB = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy502 ? "true" : "false"); + Expr *pB = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy144 ? "true" : "false"); if( pB ) sqlite3ExprIdToTrueFalse(pB); - if( !ExprHasProperty(yymsp[-4].minor.yy590, EP_HasFunc) ){ - sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy590); - yymsp[-4].minor.yy590 = pB; + if( !ExprHasProperty(yymsp[-4].minor.yy454, EP_HasFunc) ){ + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy454); + yymsp[-4].minor.yy454 = pB; }else{ - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, yymsp[-3].minor.yy502 ? TK_OR : TK_AND, pB, yymsp[-4].minor.yy590); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, yymsp[-3].minor.yy144 ? TK_OR : TK_AND, pB, yymsp[-4].minor.yy454); } }else{ - Expr *pRHS = yymsp[-1].minor.yy402->a[0].pExpr; - if( yymsp[-1].minor.yy402->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy590->op!=TK_VECTOR ){ - yymsp[-1].minor.yy402->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + if( yymsp[-1].minor.yy14->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy454->op!=TK_VECTOR ){ + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy590, pRHS); - }else if( yymsp[-1].minor.yy402->nExpr==1 && pRHS->op==TK_SELECT ){ - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pRHS->x.pSelect); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy454, pRHS); + }else if( yymsp[-1].minor.yy14->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pRHS->x.pSelect); pRHS->x.pSelect = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); }else{ - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); - if( yymsp[-4].minor.yy590==0 ){ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); - }else if( yymsp[-4].minor.yy590->pLeft->op==TK_VECTOR ){ - int nExpr = yymsp[-4].minor.yy590->pLeft->x.pList->nExpr; - Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy402); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + }else if( yymsp[-4].minor.yy454->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy454->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy14); if( pSelectRHS ){ parserDoubleLinkSelect(pParse, pSelectRHS); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelectRHS); } }else{ - yymsp[-4].minor.yy590->x.pList = yymsp[-1].minor.yy402; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy590); + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); } } - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } } break; case 224: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy590 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy590, yymsp[-1].minor.yy637); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy454, yymsp[-1].minor.yy555); } break; case 225: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, yymsp[-1].minor.yy637); - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, yymsp[-1].minor.yy555); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy402 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy402); - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pSelect); - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); + if( yymsp[0].minor.yy14 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy14); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelect); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; case 227: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy590 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy637); + p = yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy555); } break; case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy590, 0); - if( yymsp[-4].minor.yy590 ){ - yymsp[-4].minor.yy590->x.pList = yymsp[-1].minor.yy590 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590) : yymsp[-2].minor.yy402; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy590); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy454 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy402); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy590); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } } break; case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, yymsp[-2].minor.yy590); - yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, yymsp[0].minor.yy590); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[0].minor.yy454); } break; case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy402 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590); - yymsp[-3].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy402, yymsp[0].minor.yy590); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, yymsp[0].minor.yy454); } break; case 235: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy402,yymsp[0].minor.yy590);} +{yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy454);} break; case 236: /* nexprlist ::= expr */ -{yymsp[0].minor.yy402 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy590); /*A-overwrites-Y*/} +{yymsp[0].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy454); /*A-overwrites-Y*/} break; case 238: /* paren_exprlist ::= LP exprlist RP */ case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243); -{yymsp[-2].minor.yy402 = yymsp[-1].minor.yy402;} +{yymsp[-2].minor.yy14 = yymsp[-1].minor.yy14;} break; case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy402, yymsp[-10].minor.yy502, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy590, SQLITE_SO_ASC, yymsp[-8].minor.yy502, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy144, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy454, SQLITE_SO_ASC, yymsp[-8].minor.yy144, SQLITE_IDXTYPE_APPDEF); if( IN_RENAME_OBJECT && pParse->pNewIndex ){ sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); } } break; case 240: /* uniqueflag ::= UNIQUE */ - case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282); -{yymsp[0].minor.yy502 = OE_Abort;} + case 281: /* raisetype ::= ABORT */ yytestcase(yyruleno==281); +{yymsp[0].minor.yy144 = OE_Abort;} break; case 241: /* uniqueflag ::= */ -{yymsp[1].minor.yy502 = OE_None;} +{yymsp[1].minor.yy144 = OE_None;} break; case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy402 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy402, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); + yymsp[-4].minor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); } break; case 245: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy402 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); /*A-overwrites-Y*/ + yymsp[-2].minor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); /*A-overwrites-Y*/ } break; case 248: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy563, yymsp[-1].minor.yy502);} +{sqlite3DropIndex(pParse, yymsp[0].minor.yy203, yymsp[-1].minor.yy144);} break; case 249: /* cmd ::= VACUUM vinto */ -{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy590);} +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy454);} break; case 250: /* cmd ::= VACUUM nm vinto */ -{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy590);} +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy454);} break; case 253: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} @@ -180096,12 +184546,12 @@ static YYACTIONTYPE yy_reduce( Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy319, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy427, &all); } break; case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy502, yymsp[-4].minor.yy28.a, yymsp[-4].minor.yy28.b, yymsp[-2].minor.yy563, yymsp[0].minor.yy590, yymsp[-10].minor.yy502, yymsp[-8].minor.yy502); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy144, yymsp[-4].minor.yy286.a, yymsp[-4].minor.yy286.b, yymsp[-2].minor.yy203, yymsp[0].minor.yy454, yymsp[-10].minor.yy144, yymsp[-8].minor.yy144); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ #ifdef SQLITE_DEBUG assert( pParse->isCreate ); /* Set by createkw reduce action */ @@ -180110,421 +184560,439 @@ static YYACTIONTYPE yy_reduce( } break; case 262: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/ } +{ yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/ } break; case 263: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy502 = TK_INSTEAD;} +{ yymsp[-1].minor.yy144 = TK_INSTEAD;} break; case 264: /* trigger_time ::= */ -{ yymsp[1].minor.yy502 = TK_BEFORE; } +{ yymsp[1].minor.yy144 = TK_BEFORE; } break; case 265: /* trigger_event ::= DELETE|INSERT */ case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); -{yymsp[0].minor.yy28.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy28.b = 0;} +{yymsp[0].minor.yy286.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy286.b = 0;} break; case 267: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy28.a = TK_UPDATE; yymsp[-2].minor.yy28.b = yymsp[0].minor.yy204;} +{yymsp[-2].minor.yy286.a = TK_UPDATE; yymsp[-2].minor.yy286.b = yymsp[0].minor.yy132;} break; case 268: /* when_clause ::= */ - case 287: /* key_opt ::= */ yytestcase(yyruleno==287); -{ yymsp[1].minor.yy590 = 0; } + case 286: /* key_opt ::= */ yytestcase(yyruleno==286); +{ yymsp[1].minor.yy454 = 0; } break; case 269: /* when_clause ::= WHEN expr */ - case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288); -{ yymsp[-1].minor.yy590 = yymsp[0].minor.yy590; } + case 287: /* key_opt ::= KEY expr */ yytestcase(yyruleno==287); +{ yymsp[-1].minor.yy454 = yymsp[0].minor.yy454; } break; case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy319!=0 ); - yymsp[-2].minor.yy319->pLast->pNext = yymsp[-1].minor.yy319; - yymsp[-2].minor.yy319->pLast = yymsp[-1].minor.yy319; + yymsp[-2].minor.yy427->pLast->pNext = yymsp[-1].minor.yy427; + yymsp[-2].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy319!=0 ); - yymsp[-1].minor.yy319->pLast = yymsp[-1].minor.yy319; -} - break; - case 272: /* trnm ::= nm DOT nm */ -{ - yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); + yymsp[-1].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 273: /* tridxby ::= INDEXED BY nm */ + case 272: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 274: /* tridxby ::= NOT INDEXED */ + case 273: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ -{yylhsminor.yy319 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy563, yymsp[-3].minor.yy402, yymsp[-1].minor.yy590, yymsp[-7].minor.yy502, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy342);} - yymsp[-8].minor.yy319 = yylhsminor.yy319; + case 274: /* trigger_cmd ::= UPDATE orconf xfullname tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerUpdateStep(pParse, yymsp[-6].minor.yy203, yymsp[-2].minor.yy203, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454, yymsp[-7].minor.yy144, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-8].minor.yy427 = yylhsminor.yy427; break; - case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 275: /* trigger_cmd ::= scanpt insert_cmd INTO xfullname idlist_opt select upsert scanpt */ { - yylhsminor.yy319 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy204,yymsp[-2].minor.yy637,yymsp[-6].minor.yy502,yymsp[-1].minor.yy403,yymsp[-7].minor.yy342,yymsp[0].minor.yy342);/*yylhsminor.yy319-overwrites-yymsp[-6].minor.yy502*/ + yylhsminor.yy427 = sqlite3TriggerInsertStep(pParse,yymsp[-4].minor.yy203,yymsp[-3].minor.yy132,yymsp[-2].minor.yy555,yymsp[-6].minor.yy144,yymsp[-1].minor.yy122,yymsp[-7].minor.yy168,yymsp[0].minor.yy168);/*yylhsminor.yy427-overwrites-yymsp[-6].minor.yy144*/ } - yymsp[-7].minor.yy319 = yylhsminor.yy319; + yymsp[-7].minor.yy427 = yylhsminor.yy427; break; - case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy319 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy590, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy342);} - yymsp[-5].minor.yy319 = yylhsminor.yy319; + case 276: /* trigger_cmd ::= DELETE FROM xfullname tridxby where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerDeleteStep(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy454, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-5].minor.yy427 = yylhsminor.yy427; break; - case 278: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy319 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy637, yymsp[-2].minor.yy342, yymsp[0].minor.yy342); /*yylhsminor.yy319-overwrites-yymsp[-1].minor.yy637*/} - yymsp[-2].minor.yy319 = yylhsminor.yy319; + case 277: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy427 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy555, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); /*yylhsminor.yy427-overwrites-yymsp[-1].minor.yy555*/} + yymsp[-2].minor.yy427 = yylhsminor.yy427; break; - case 279: /* expr ::= RAISE LP IGNORE RP */ + case 278: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy590 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy590 ){ - yymsp[-3].minor.yy590->affExpr = OE_Ignore; + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy454 ){ + yymsp[-3].minor.yy454->affExpr = OE_Ignore; } } break; - case 280: /* expr ::= RAISE LP raisetype COMMA expr RP */ + case 279: /* expr ::= RAISE LP raisetype COMMA expr RP */ { - yymsp[-5].minor.yy590 = sqlite3PExpr(pParse, TK_RAISE, yymsp[-1].minor.yy590, 0); - if( yymsp[-5].minor.yy590 ) { - yymsp[-5].minor.yy590->affExpr = (char)yymsp[-3].minor.yy502; + yymsp[-5].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, yymsp[-1].minor.yy454, 0); + if( yymsp[-5].minor.yy454 ) { + yymsp[-5].minor.yy454->affExpr = (char)yymsp[-3].minor.yy144; } } break; - case 281: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy502 = OE_Rollback;} + case 280: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy144 = OE_Rollback;} break; - case 283: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy502 = OE_Fail;} + case 282: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy144 = OE_Fail;} break; - case 284: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 283: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy563,yymsp[-1].minor.yy502); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy203,yymsp[-1].minor.yy144); } break; - case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 284: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy590, yymsp[-1].minor.yy590, yymsp[0].minor.yy590); + sqlite3Attach(pParse, yymsp[-3].minor.yy454, yymsp[-1].minor.yy454, yymsp[0].minor.yy454); } break; - case 286: /* cmd ::= DETACH database_kw_opt expr */ + case 285: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy590); + sqlite3Detach(pParse, yymsp[0].minor.yy454); } break; - case 289: /* cmd ::= REINDEX */ + case 288: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 290: /* cmd ::= REINDEX nm dbnm */ + case 289: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 291: /* cmd ::= ANALYZE */ + case 290: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 292: /* cmd ::= ANALYZE nm dbnm */ + case 291: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 292: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy563,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy203,&yymsp[0].minor.yy0); } break; - case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 293: /* cmd ::= alter_add carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; + case 294: /* alter_add ::= ALTER TABLE fullname ADD kwcolumn_opt nm typetoken */ +{ + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, yymsp[-4].minor.yy203); + sqlite3AddColumn(pParse, yymsp[-1].minor.yy0, yymsp[0].minor.yy0); + yymsp[-6].minor.yy0 = yymsp[-1].minor.yy0; +} + break; case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { - sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy563, &yymsp[0].minor.yy0); + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0); } break; - case 296: /* add_column_fullname ::= fullname */ + case 296: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { - disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy563); + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy203, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 297: /* cmd ::= ALTER TABLE fullname DROP CONSTRAINT nm */ +{ + sqlite3AlterDropConstraint(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0, 0); } break; - case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 298: /* cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm DROP NOT NULL */ { - sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy563, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3AlterDropConstraint(pParse, yymsp[-6].minor.yy203, 0, &yymsp[-3].minor.yy0); } break; - case 298: /* cmd ::= create_vtab */ + case 299: /* cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm SET NOT NULL onconf */ +{ + sqlite3AlterSetNotNull(pParse, yymsp[-7].minor.yy203, &yymsp[-4].minor.yy0, &yymsp[-2].minor.yy0); +} + break; + case 300: /* cmd ::= ALTER TABLE fullname ADD CONSTRAINT nm CHECK LP expr RP onconf */ +{ + sqlite3AlterAddConstraint(pParse, yymsp[-8].minor.yy203, &yymsp[-6].minor.yy0, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy0.z+1, (yymsp[-1].minor.yy0.z-yymsp[-3].minor.yy0.z-1)); +} + yy_destructor(yypParser,219,&yymsp[-2].minor); + break; + case 301: /* cmd ::= ALTER TABLE fullname ADD CHECK LP expr RP onconf */ +{ + sqlite3AlterAddConstraint(pParse, yymsp[-6].minor.yy203, &yymsp[-4].minor.yy0, 0, yymsp[-3].minor.yy0.z+1, (yymsp[-1].minor.yy0.z-yymsp[-3].minor.yy0.z-1)); +} + yy_destructor(yypParser,219,&yymsp[-2].minor); + break; + case 302: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 299: /* cmd ::= create_vtab LP vtabarglist RP */ + case 303: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 304: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy502); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy144); } break; - case 301: /* vtabarg ::= */ + case 305: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 302: /* vtabargtoken ::= ANY */ - case 303: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==303); - case 304: /* lp ::= LP */ yytestcase(yyruleno==304); + case 306: /* vtabargtoken ::= ANY */ + case 307: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==307); + case 308: /* lp ::= LP */ yytestcase(yyruleno==308); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 305: /* with ::= WITH wqlist */ - case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy125, 1); } + case 309: /* with ::= WITH wqlist */ + case 310: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==310); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy59, 1); } break; - case 307: /* wqas ::= AS */ -{yymsp[0].minor.yy444 = M10d_Any;} + case 311: /* wqas ::= AS */ +{yymsp[0].minor.yy462 = M10d_Any;} break; - case 308: /* wqas ::= AS MATERIALIZED */ -{yymsp[-1].minor.yy444 = M10d_Yes;} + case 312: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy462 = M10d_Yes;} break; - case 309: /* wqas ::= AS NOT MATERIALIZED */ -{yymsp[-2].minor.yy444 = M10d_No;} + case 313: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy462 = M10d_No;} break; - case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ + case 314: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ { - yymsp[-5].minor.yy361 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy402, yymsp[-1].minor.yy637, yymsp[-3].minor.yy444); /*A-overwrites-X*/ + yymsp[-5].minor.yy67 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy555, yymsp[-3].minor.yy462); /*A-overwrites-X*/ } break; - case 311: /* withnm ::= nm */ + case 315: /* withnm ::= nm */ {pParse->bHasWith = 1;} break; - case 312: /* wqlist ::= wqitem */ + case 316: /* wqlist ::= wqitem */ { - yymsp[0].minor.yy125 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy361); /*A-overwrites-X*/ + yymsp[0].minor.yy59 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy67); /*A-overwrites-X*/ } break; - case 313: /* wqlist ::= wqlist COMMA wqitem */ + case 317: /* wqlist ::= wqlist COMMA wqitem */ { - yymsp[-2].minor.yy125 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy125, yymsp[0].minor.yy361); + yymsp[-2].minor.yy59 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy59, yymsp[0].minor.yy67); } break; - case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 318: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { - assert( yymsp[0].minor.yy483!=0 ); - sqlite3WindowChain(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy483); - yymsp[0].minor.yy483->pNextWin = yymsp[-2].minor.yy483; - yylhsminor.yy483 = yymsp[0].minor.yy483; + assert( yymsp[0].minor.yy211!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy211); + yymsp[0].minor.yy211->pNextWin = yymsp[-2].minor.yy211; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-2].minor.yy483 = yylhsminor.yy483; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 315: /* windowdefn ::= nm AS LP window RP */ + case 319: /* windowdefn ::= nm AS LP window RP */ { - if( ALWAYS(yymsp[-1].minor.yy483) ){ - yymsp[-1].minor.yy483->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + if( ALWAYS(yymsp[-1].minor.yy211) ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); } - yylhsminor.yy483 = yymsp[-1].minor.yy483; + yylhsminor.yy211 = yymsp[-1].minor.yy211; } - yymsp[-4].minor.yy483 = yylhsminor.yy483; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 320: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { - yymsp[-4].minor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy402, yymsp[-1].minor.yy402, 0); + yymsp[-4].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, 0); } break; - case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 321: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { - yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy402, yymsp[-1].minor.yy402, &yymsp[-5].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy483 = yylhsminor.yy483; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 318: /* window ::= ORDER BY sortlist frame_opt */ + case 322: /* window ::= ORDER BY sortlist frame_opt */ { - yymsp[-3].minor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, yymsp[-1].minor.yy402, 0); + yymsp[-3].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, 0); } break; - case 319: /* window ::= nm ORDER BY sortlist frame_opt */ + case 323: /* window ::= nm ORDER BY sortlist frame_opt */ { - yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, yymsp[-1].minor.yy402, &yymsp[-4].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy483 = yylhsminor.yy483; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 320: /* window ::= nm frame_opt */ + case 324: /* window ::= nm frame_opt */ { - yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, 0, &yymsp[-1].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, 0, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy483 = yylhsminor.yy483; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 321: /* frame_opt ::= */ + case 325: /* frame_opt ::= */ { - yymsp[1].minor.yy483 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); + yymsp[1].minor.yy211 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 326: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { - yylhsminor.yy483 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy502, yymsp[-1].minor.yy205.eType, yymsp[-1].minor.yy205.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy444); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy144, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy462); } - yymsp[-2].minor.yy483 = yylhsminor.yy483; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 327: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { - yylhsminor.yy483 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy502, yymsp[-3].minor.yy205.eType, yymsp[-3].minor.yy205.pExpr, yymsp[-1].minor.yy205.eType, yymsp[-1].minor.yy205.pExpr, yymsp[0].minor.yy444); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy144, yymsp[-3].minor.yy509.eType, yymsp[-3].minor.yy509.pExpr, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, yymsp[0].minor.yy462); } - yymsp[-5].minor.yy483 = yylhsminor.yy483; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 325: /* frame_bound_s ::= frame_bound */ - case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327); -{yylhsminor.yy205 = yymsp[0].minor.yy205;} - yymsp[0].minor.yy205 = yylhsminor.yy205; + case 329: /* frame_bound_s ::= frame_bound */ + case 331: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==331); +{yylhsminor.yy509 = yymsp[0].minor.yy509;} + yymsp[0].minor.yy509 = yylhsminor.yy509; break; - case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328); - case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330); -{yylhsminor.yy205.eType = yymsp[-1].major; yylhsminor.yy205.pExpr = 0;} - yymsp[-1].minor.yy205 = yylhsminor.yy205; + case 330: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 332: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==332); + case 334: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==334); +{yylhsminor.yy509.eType = yymsp[-1].major; yylhsminor.yy509.pExpr = 0;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */ -{yylhsminor.yy205.eType = yymsp[0].major; yylhsminor.yy205.pExpr = yymsp[-1].minor.yy590;} - yymsp[-1].minor.yy205 = yylhsminor.yy205; + case 333: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy509.eType = yymsp[0].major; yylhsminor.yy509.pExpr = yymsp[-1].minor.yy454;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 331: /* frame_exclude_opt ::= */ -{yymsp[1].minor.yy444 = 0;} + case 335: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy462 = 0;} break; - case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ -{yymsp[-1].minor.yy444 = yymsp[0].minor.yy444;} + case 336: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy462 = yymsp[0].minor.yy462;} break; - case 333: /* frame_exclude ::= NO OTHERS */ - case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334); -{yymsp[-1].minor.yy444 = yymsp[-1].major; /*A-overwrites-X*/} + case 337: /* frame_exclude ::= NO OTHERS */ + case 338: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==338); +{yymsp[-1].minor.yy462 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 335: /* frame_exclude ::= GROUP|TIES */ -{yymsp[0].minor.yy444 = yymsp[0].major; /*A-overwrites-X*/} + case 339: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy462 = yymsp[0].major; /*A-overwrites-X*/} break; - case 336: /* window_clause ::= WINDOW windowdefn_list */ -{ yymsp[-1].minor.yy483 = yymsp[0].minor.yy483; } + case 340: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy211 = yymsp[0].minor.yy211; } break; - case 337: /* filter_over ::= filter_clause over_clause */ + case 341: /* filter_over ::= filter_clause over_clause */ { - if( yymsp[0].minor.yy483 ){ - yymsp[0].minor.yy483->pFilter = yymsp[-1].minor.yy590; + if( yymsp[0].minor.yy211 ){ + yymsp[0].minor.yy211->pFilter = yymsp[-1].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy590); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } - yylhsminor.yy483 = yymsp[0].minor.yy483; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-1].minor.yy483 = yylhsminor.yy483; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 338: /* filter_over ::= over_clause */ + case 342: /* filter_over ::= over_clause */ { - yylhsminor.yy483 = yymsp[0].minor.yy483; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[0].minor.yy483 = yylhsminor.yy483; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 339: /* filter_over ::= filter_clause */ + case 343: /* filter_over ::= filter_clause */ { - yylhsminor.yy483 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yylhsminor.yy483 ){ - yylhsminor.yy483->eFrmType = TK_FILTER; - yylhsminor.yy483->pFilter = yymsp[0].minor.yy590; + yylhsminor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy211 ){ + yylhsminor.yy211->eFrmType = TK_FILTER; + yylhsminor.yy211->pFilter = yymsp[0].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy590); + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy454); } } - yymsp[0].minor.yy483 = yylhsminor.yy483; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 340: /* over_clause ::= OVER LP window RP */ + case 344: /* over_clause ::= OVER LP window RP */ { - yymsp[-3].minor.yy483 = yymsp[-1].minor.yy483; - assert( yymsp[-3].minor.yy483!=0 ); + yymsp[-3].minor.yy211 = yymsp[-1].minor.yy211; + assert( yymsp[-3].minor.yy211!=0 ); } break; - case 341: /* over_clause ::= OVER nm */ + case 345: /* over_clause ::= OVER nm */ { - yymsp[-1].minor.yy483 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yymsp[-1].minor.yy483 ){ - yymsp[-1].minor.yy483->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + yymsp[-1].minor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy211 ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); } } break; - case 342: /* filter_clause ::= FILTER LP WHERE expr RP */ -{ yymsp[-4].minor.yy590 = yymsp[-1].minor.yy590; } + case 346: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy454 = yymsp[-1].minor.yy454; } break; - case 343: /* term ::= QNUMBER */ + case 347: /* term ::= QNUMBER */ { - yylhsminor.yy590=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); - sqlite3DequoteNumber(pParse, yylhsminor.yy590); + yylhsminor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); + sqlite3DequoteNumber(pParse, yylhsminor.yy454); } - yymsp[0].minor.yy590 = yylhsminor.yy590; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; default: - /* (344) input ::= cmdlist */ yytestcase(yyruleno==344); - /* (345) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==345); - /* (346) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=346); - /* (347) ecmd ::= SEMI */ yytestcase(yyruleno==347); - /* (348) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==348); - /* (349) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=349); - /* (350) trans_opt ::= */ yytestcase(yyruleno==350); - /* (351) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==351); - /* (352) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==352); - /* (353) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==353); - /* (354) savepoint_opt ::= */ yytestcase(yyruleno==354); - /* (355) cmd ::= create_table create_table_args */ yytestcase(yyruleno==355); - /* (356) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=356); - /* (357) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==357); - /* (358) columnlist ::= columnname carglist */ yytestcase(yyruleno==358); - /* (359) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==359); - /* (360) nm ::= STRING */ yytestcase(yyruleno==360); - /* (361) typetoken ::= typename */ yytestcase(yyruleno==361); - /* (362) typename ::= ID|STRING */ yytestcase(yyruleno==362); - /* (363) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); - /* (364) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); - /* (365) carglist ::= carglist ccons */ yytestcase(yyruleno==365); - /* (366) carglist ::= */ yytestcase(yyruleno==366); - /* (367) ccons ::= NULL onconf */ yytestcase(yyruleno==367); - /* (368) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==368); - /* (369) ccons ::= AS generated */ yytestcase(yyruleno==369); - /* (370) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==370); - /* (371) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==371); - /* (372) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=372); - /* (373) tconscomma ::= */ yytestcase(yyruleno==373); - /* (374) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=374); - /* (375) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=375); - /* (376) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=376); - /* (377) oneselect ::= values */ yytestcase(yyruleno==377); - /* (378) sclp ::= selcollist COMMA */ yytestcase(yyruleno==378); - /* (379) as ::= ID|STRING */ yytestcase(yyruleno==379); - /* (380) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=380); - /* (381) returning ::= */ yytestcase(yyruleno==381); - /* (382) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=382); - /* (383) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==383); - /* (384) case_operand ::= expr */ yytestcase(yyruleno==384); - /* (385) exprlist ::= nexprlist */ yytestcase(yyruleno==385); - /* (386) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=386); - /* (387) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=387); - /* (388) nmnum ::= ON */ yytestcase(yyruleno==388); - /* (389) nmnum ::= DELETE */ yytestcase(yyruleno==389); - /* (390) nmnum ::= DEFAULT */ yytestcase(yyruleno==390); - /* (391) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==391); - /* (392) foreach_clause ::= */ yytestcase(yyruleno==392); - /* (393) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==393); - /* (394) trnm ::= nm */ yytestcase(yyruleno==394); - /* (395) tridxby ::= */ yytestcase(yyruleno==395); - /* (396) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==396); - /* (397) database_kw_opt ::= */ yytestcase(yyruleno==397); - /* (398) kwcolumn_opt ::= */ yytestcase(yyruleno==398); - /* (399) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==399); - /* (400) vtabarglist ::= vtabarg */ yytestcase(yyruleno==400); - /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==401); - /* (402) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==402); - /* (403) anylist ::= */ yytestcase(yyruleno==403); - /* (404) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==404); - /* (405) anylist ::= anylist ANY */ yytestcase(yyruleno==405); - /* (406) with ::= */ yytestcase(yyruleno==406); - /* (407) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=407); - /* (408) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=408); + /* (348) input ::= cmdlist */ yytestcase(yyruleno==348); + /* (349) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==349); + /* (350) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=350); + /* (351) ecmd ::= SEMI */ yytestcase(yyruleno==351); + /* (352) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==352); + /* (353) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=353); + /* (354) trans_opt ::= */ yytestcase(yyruleno==354); + /* (355) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==355); + /* (356) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==356); + /* (357) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==357); + /* (358) savepoint_opt ::= */ yytestcase(yyruleno==358); + /* (359) cmd ::= create_table create_table_args */ yytestcase(yyruleno==359); + /* (360) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=360); + /* (361) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==361); + /* (362) columnlist ::= columnname carglist */ yytestcase(yyruleno==362); + /* (363) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==363); + /* (364) nm ::= STRING */ yytestcase(yyruleno==364); + /* (365) typetoken ::= typename */ yytestcase(yyruleno==365); + /* (366) typename ::= ID|STRING */ yytestcase(yyruleno==366); + /* (367) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=367); + /* (368) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=368); + /* (369) carglist ::= carglist ccons */ yytestcase(yyruleno==369); + /* (370) carglist ::= */ yytestcase(yyruleno==370); + /* (371) ccons ::= NULL onconf */ yytestcase(yyruleno==371); + /* (372) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==372); + /* (373) ccons ::= AS generated */ yytestcase(yyruleno==373); + /* (374) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==374); + /* (375) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==375); + /* (376) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) tconscomma ::= */ yytestcase(yyruleno==377); + /* (378) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=378); + /* (379) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=379); + /* (380) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) oneselect ::= values */ yytestcase(yyruleno==381); + /* (382) sclp ::= selcollist COMMA */ yytestcase(yyruleno==382); + /* (383) as ::= ID|STRING */ yytestcase(yyruleno==383); + /* (384) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=384); + /* (385) returning ::= */ yytestcase(yyruleno==385); + /* (386) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=386); + /* (387) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==387); + /* (388) case_operand ::= expr */ yytestcase(yyruleno==388); + /* (389) exprlist ::= nexprlist */ yytestcase(yyruleno==389); + /* (390) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=390); + /* (391) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=391); + /* (392) nmnum ::= ON */ yytestcase(yyruleno==392); + /* (393) nmnum ::= DELETE */ yytestcase(yyruleno==393); + /* (394) nmnum ::= DEFAULT */ yytestcase(yyruleno==394); + /* (395) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==395); + /* (396) foreach_clause ::= */ yytestcase(yyruleno==396); + /* (397) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==397); + /* (398) tridxby ::= */ yytestcase(yyruleno==398); + /* (399) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==399); + /* (400) database_kw_opt ::= */ yytestcase(yyruleno==400); + /* (401) kwcolumn_opt ::= */ yytestcase(yyruleno==401); + /* (402) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==402); + /* (403) vtabarglist ::= vtabarg */ yytestcase(yyruleno==403); + /* (404) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==404); + /* (405) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==405); + /* (406) anylist ::= */ yytestcase(yyruleno==406); + /* (407) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==407); + /* (408) anylist ::= anylist ANY */ yytestcase(yyruleno==408); + /* (409) with ::= */ yytestcase(yyruleno==409); + /* (410) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=410); + /* (411) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=411); break; /********** End reduce actions ************************************************/ }; @@ -181303,8 +185771,8 @@ static const unsigned char aKWCode[148] = {0, /* Check to see if z[0..n-1] is a keyword. If it is, write the ** parser symbol code for that keyword into *pType. Always ** return the integer n (the length of the token). */ -static int keywordCode(const char *z, int n, int *pType){ - int i, j; +static i64 keywordCode(const char *z, i64 n, int *pType){ + i64 i, j; const char *zKW; assert( n>=2 ); i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; @@ -181620,8 +186088,9 @@ static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; +SQLITE_PRIVATE i64 sqlite3GetToken(const unsigned char *z, int *tokenType){ + i64 i; + int c; switch( aiClass[*z] ){ /* Switch on the character-class of the first byte ** of the token. See the comment on the CC_ defines ** above. */ @@ -181857,7 +186326,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } case CC_DOLLAR: case CC_VARALPHA: { - int n = 0; + i64 n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; @@ -181949,11 +186418,11 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ int nErr = 0; /* Number of errors encountered */ void *pEngine; /* The LEMON-generated LALR(1) parser */ - int n = 0; /* Length of the next token token */ + i64 n = 0; /* Length of the next token token */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ + i64 mxSqlLen; /* Max length of an SQL string */ Parse *pParentParse = 0; /* Outer parse context, if any */ #ifdef sqlite3Parser_ENGINEALWAYSONSTACK yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ @@ -182052,13 +186521,13 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; - x.n = n; + x.n = (u32)n; sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x); break; } } pParse->sLastToken.z = zSql; - pParse->sLastToken.n = n; + pParse->sLastToken.n = (u32)n; sqlite3Parser(pEngine, tokenType, pParse->sLastToken); lastTokenParsed = tokenType; zSql += n; @@ -182083,7 +186552,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ } if( pParse->zErrMsg || (pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE) ){ if( pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + pParse->zErrMsg = sqlite3DbStrDup(db, sqlite3ErrStr(pParse->rc)); } if( (pParse->prepFlags & SQLITE_PREPARE_DONT_LOG)==0 ){ sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); @@ -182134,7 +186603,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( ){ sqlite3 *db; /* The database connection */ int i; /* Next unread byte of zSql[] */ - int n; /* length of current token */ + i64 n; /* length of current token */ int tokenType; /* type of current token */ int prevType = 0; /* Previous non-whitespace token */ int nParen; /* Number of nested levels of parentheses */ @@ -182164,7 +186633,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( sqlite3_str_append(pStr, " NULL", 5); break; } - /* Fall through */ + /* no break */ deliberate_fall_through } case TK_STRING: case TK_INTEGER: @@ -182228,7 +186697,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( } case TK_SELECT: { iStartIN = 0; - /* fall through */ + /* no break */ deliberate_fall_through } default: { if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); @@ -182712,9 +187181,6 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { sqlite3DbstatRegister, #endif sqlite3TestExtInit, -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) - sqlite3JsonTableFunctions, -#endif #ifdef SQLITE_ENABLE_STMTVTAB sqlite3StmtVtabInit, #endif @@ -183612,6 +188078,14 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ rc = setupLookaside(db, pBuf, sz, cnt); break; } + case SQLITE_DBCONFIG_FP_DIGITS: { + int nIn = va_arg(ap, int); + int *pOut = va_arg(ap, int*); + if( nIn>3 && nIn<24 ) db->nFpDigit = (u8)nIn; + if( pOut ) *pOut = db->nFpDigit; + rc = SQLITE_OK; + break; + } default: { static const struct { int op; /* The opcode */ @@ -184035,6 +188509,7 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* Clear the TEMP schema separately and last */ if( db->aDb[1].pSchema ){ sqlite3SchemaClear(db->aDb[1].pSchema); + assert( db->aDb[1].pSchema->trigHash.count==0 ); } sqlite3VtabUnlockList(db); @@ -184170,6 +188645,9 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_OK: zName = "SQLITE_OK"; break; case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_ERROR_RETRY: zName = "SQLITE_ERROR_RETRY"; break; + case SQLITE_ERROR_MISSING_COLLSEQ: + zName = "SQLITE_ERROR_MISSING_COLLSEQ"; break; case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; case SQLITE_PERM: zName = "SQLITE_PERM"; break; case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; @@ -185163,6 +189641,9 @@ SQLITE_API void *sqlite3_wal_hook( sqlite3_mutex_leave(db->mutex); return pRet; #else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(xCallback); + UNUSED_PARAMETER(pArg); return 0; #endif } @@ -185178,6 +189659,11 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( int *pnCkpt /* OUT: Total number of frames checkpointed */ ){ #ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(zDb); + UNUSED_PARAMETER(eMode); + UNUSED_PARAMETER(pnLog); + UNUSED_PARAMETER(pnCkpt); return SQLITE_OK; #else int rc; /* Return code */ @@ -185191,11 +189677,12 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; + assert( SQLITE_CHECKPOINT_NOOP==-1 ); assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eMode<SQLITE_CHECKPOINT_PASSIVE || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ + if( eMode<SQLITE_CHECKPOINT_NOOP || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ return SQLITE_MISUSE_BKPT; @@ -185352,6 +189839,29 @@ SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ } /* +** Set the error code and error message associated with the database handle. +** +** This routine is intended to be called by outside extensions (ex: the +** Session extension). Internal logic should invoke sqlite3Error() or +** sqlite3ErrorWithMsg() directly. +*/ +SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zMsg){ + int rc = SQLITE_OK; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + if( zMsg ){ + sqlite3ErrorWithMsg(db, errcode, "%s", zMsg); + }else{ + sqlite3Error(db, errcode); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* ** Return the byte offset of the most recent error */ SQLITE_API int sqlite3_error_offset(sqlite3 *db){ @@ -185536,6 +190046,7 @@ static const int aHardLimit[] = { SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, SQLITE_MAX_WORKER_THREADS, + SQLITE_MAX_PARSER_DEPTH, }; /* @@ -185550,6 +190061,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH # error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH #endif +#if SQLITE_MAX_SQL_LENGTH>2147482624 /* 1024 less than 2^31 */ +# error SQLITE_MAX_SQL_LENGTH must not be greater than 2147482624 +#endif #if SQLITE_MAX_COMPOUND_SELECT<2 # error SQLITE_MAX_COMPOUND_SELECT must be at least 2 #endif @@ -185605,6 +190119,7 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_PARSER_DEPTH]==SQLITE_MAX_PARSER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); @@ -185614,7 +190129,7 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + assert( SQLITE_LIMIT_PARSER_DEPTH==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -185978,7 +190493,7 @@ static int openDatabase( db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; if( isThreadsafe -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#if defined(SQLITE_THREAD_MISUSE_WARNINGS) || sqlite3GlobalConfig.bCoreMutex #endif ){ @@ -185999,6 +190514,7 @@ static int openDatabase( db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; db->lookaside.sz = 0; + db->nFpDigit = 17; assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); @@ -186444,6 +190960,12 @@ SQLITE_API int sqlite3_collation_needed16( */ SQLITE_API void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ DbClientData *p; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zName || !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); for(p=db->pDbData; p; p=p->pNext){ if( strcmp(p->zName, zName)==0 ){ @@ -187175,13 +191697,15 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, mode, tnum); ** ** This test control is used to create imposter tables. "db" is a pointer ** to the database connection. dbName is the database name (ex: "main" or - ** "temp") which will receive the imposter. "onOff" turns imposter mode on - ** or off. "tnum" is the root page of the b-tree to which the imposter - ** table should connect. + ** "temp") which will receive the imposter. "mode" turns imposter mode on + ** or off. mode==0 means imposter mode is off. mode==1 means imposter mode + ** is on. mode==2 means imposter mode is on but results in an imposter + ** table that is read-only unless writable_schema is on. "tnum" is the + ** root page of the b-tree to which the imposter table should connect. ** ** Enable imposter mode only when the schema has already been parsed. Then ** run a single CREATE TABLE statement to construct the imposter table in @@ -187299,6 +191823,17 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_ATOF, const char *z, double *p); + ** + ** Test access to the sqlite3AtoF() routine. + */ + case SQLITE_TESTCTRL_ATOF: { + const char *z = va_arg(ap,const char*); + double *pR = va_arg(ap,double*); + rc = sqlite3AtoF(z,pR); + break; + } + #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) ** @@ -187515,6 +192050,7 @@ SQLITE_API const char *sqlite3_filename_journal(const char *zFilename){ } SQLITE_API const char *sqlite3_filename_wal(const char *zFilename){ #ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(zFilename); return 0; #else zFilename = sqlite3_filename_journal(zFilename); @@ -188418,6 +192954,13 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ #ifndef _FTSINT_H #define _FTSINT_H +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + /* #include <assert.h> */ /* #include <stdlib.h> */ /* #include <stddef.h> */ @@ -188425,10 +192968,6 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ /* #include <string.h> */ /* #include <stdarg.h> */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - /* FTS3/FTS4 require virtual tables */ #ifdef SQLITE_OMIT_VIRTUALTABLE # undef SQLITE_ENABLE_FTS3 @@ -188872,13 +193411,6 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define UNUSED_PARAMETER(x) (void)(x) /* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - -/* ** The TESTONLY macro is used to enclose variable declarations or ** other bits of code that are needed to support the arguments ** within testcase() and assert() macros. @@ -188892,13 +193424,22 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define deliberate_fall_through +#if !defined(deliberate_fall_through) +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through +#endif /* ** Macros needed to provide flexible arrays in a portable way */ #ifndef offsetof -# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY @@ -189290,6 +193831,15 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ ) +SQLITE_PRIVATE int sqlite3Fts3PrepareStmt( + Fts3Table *p, /* Prepare for this connection */ + const char *zSql, /* SQL to prepare */ + int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ + int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ + sqlite3_stmt **pp /* OUT: Prepared statement */ +); + + /* fts3.c */ SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); @@ -190897,9 +195447,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); if( !zSql ) return SQLITE_NOMEM; p->bLock++; - rc = sqlite3_prepare_v3( - p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 - ); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); p->bLock--; sqlite3_free(zSql); } @@ -192474,9 +197022,7 @@ static int fts3FilterMethod( } if( zSql ){ p->bLock++; - rc = sqlite3_prepare_v3( - p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 - ); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); p->bLock--; sqlite3_free(zSql); }else{ @@ -193099,6 +197645,7 @@ static int fts3IntegrityMethod( UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( pVtab->zErrMsg==0 || rc!=SQLITE_OK ); assert( rc!=SQLITE_CORRUPT_VTAB ); if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" @@ -195615,7 +200162,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->aStat[1].nDoc++; } eState = 2; - /* fall through */ + /* no break */ deliberate_fall_through case 2: if( v==0 ){ /* 0x00. Next integer will be a docid. */ @@ -199536,9 +204083,9 @@ typedef struct SegmentWriter SegmentWriter; ** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { - int nData; + sqlite3_int64 nData; char *aData; - int nSpace; + sqlite3_int64 nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; @@ -199712,6 +204259,24 @@ struct SegmentNode { #define SQL_UPDATE_LEVEL 39 /* +** Wrapper around sqlite3_prepare_v3() to ensure that SQLITE_PREPARE_FROM_DDL +** is always set. +*/ +SQLITE_PRIVATE int sqlite3Fts3PrepareStmt( + Fts3Table *p, /* Prepare for this connection */ + const char *zSql, /* SQL to prepare */ + int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ + int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ + sqlite3_stmt **pp /* OUT: Prepared statement */ +){ + int f = SQLITE_PREPARE_FROM_DDL + |((bAllowVtab==0) ? SQLITE_PREPARE_NO_VTAB : 0) + |(bPersist ? SQLITE_PREPARE_PERSISTENT : 0); + + return sqlite3_prepare_v3(p->db, zSql, -1, f, pp, NULL); +} + +/* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. @@ -199836,12 +204401,12 @@ static int fts3SqlStmt( pStmt = p->aStmt[eStmt]; if( !pStmt ){ - int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + int bAllowVtab = 0; char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - f &= ~SQLITE_PREPARE_NO_VTAB; + bAllowVtab = 1; zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); @@ -199849,7 +204414,7 @@ static int fts3SqlStmt( if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, bAllowVtab, &pStmt); sqlite3_free(zSql); assert( rc==SQLITE_OK || pStmt==0 ); p->aStmt[eStmt] = pStmt; @@ -200198,7 +204763,9 @@ static int fts3PendingTermsAddOne( pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + assert( (i64)pList->nData+(i64)nToken+(i64)sizeof(Fts3HashElem) + <= (i64)p->nPendingData ); + p->nPendingData -= (int)(pList->nData + nToken + sizeof(Fts3HashElem)); } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ @@ -200211,7 +204778,9 @@ static int fts3PendingTermsAddOne( } } if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + assert( (i64)p->nPendingData + pList->nData + nToken + + sizeof(Fts3HashElem) <= 0x3fffffff ); + p->nPendingData += (int)(pList->nData + nToken + sizeof(Fts3HashElem)); } return rc; } @@ -203012,7 +207581,7 @@ static int fts3DoRebuild(Fts3Table *p){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); sqlite3_free(zSql); } @@ -203152,8 +207721,8 @@ struct NodeWriter { ** to an appendable b-tree segment. */ struct IncrmergeWriter { - int nLeafEst; /* Space allocated for leaf blocks */ - int nWork; /* Number of leaf pages flushed */ + i64 nLeafEst; /* Space allocated for leaf blocks */ + i64 nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ @@ -203899,7 +208468,7 @@ static int fts3IncrmergeWriter( ){ int rc; /* Return Code */ int i; /* Iterator variable */ - int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + i64 nLeafEst = 0; /* Blocks allocated for leaf nodes */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ @@ -203909,7 +208478,7 @@ static int fts3IncrmergeWriter( sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ - nLeafEst = sqlite3_column_int(pLeafEst, 0); + nLeafEst = sqlite3_column_int64(pLeafEst, 0); } rc = sqlite3_reset(pLeafEst); } @@ -204765,7 +209334,7 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); sqlite3_free(zSql); } @@ -204895,7 +209464,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ v = atoi(&zVal[9]); if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 11) ){ v = atoi(&zVal[11]); if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; rc = SQLITE_OK; @@ -205292,10 +209861,6 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ /* #include <string.h> */ /* #include <assert.h> */ -#ifndef SQLITE_AMALGAMATION -typedef sqlite3_int64 i64; -#endif - /* ** Characters that may appear in the second argument to matchinfo(). */ @@ -208159,7 +212724,8 @@ struct JsonString { /* Allowed values for JsonString.eErr */ #define JSTRING_OOM 0x01 /* Out of memory */ #define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ -#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ +#define JSTRING_TOODEEP 0x04 /* JSON nested too deep */ +#define JSTRING_ERR 0x08 /* Error already sent to sqlite3_result */ /* The "subtype" set for text JSON values passed through using ** sqlite3_result_subtype() and sqlite3_value_subtype(). @@ -208174,7 +212740,10 @@ struct JsonString { #define JSON_SQL 0x02 /* Result is always SQL */ #define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ #define JSON_ISSET 0x04 /* json_set(), not json_insert() */ -#define JSON_BLOB 0x08 /* Use the BLOB output format */ +#define JSON_AINS 0x08 /* json_array_insert(), not json_insert() */ +#define JSON_BLOB 0x10 /* Use the BLOB output format */ + +#define JSON_INSERT_TYPE(X) (((X)&0xC)>>2) /* A parsed JSON value. Lifecycle: @@ -208220,6 +212789,7 @@ struct JsonParse { #define JEDIT_REPL 2 /* Overwrite if exists */ #define JEDIT_INS 3 /* Insert if not exists */ #define JEDIT_SET 4 /* Insert or overwrite */ +#define JEDIT_AINS 5 /* array_insert() */ /* ** Maximum nesting depth of JSON for this implementation. @@ -208245,7 +212815,7 @@ struct JsonParse { **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); -static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); +static u32 jsonTranslateBlobToText(JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); @@ -208403,6 +212973,15 @@ static void jsonStringOom(JsonString *p){ jsonStringReset(p); } +/* Report JSON nested too deep +*/ +static void jsonStringTooDeep(JsonString *p){ + p->eErr |= JSTRING_TOODEEP; + assert( p->pCtx!=0 ); + sqlite3_result_error(p->pCtx, "JSON nested too deep", -1); + jsonStringReset(p); +} + /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ @@ -208692,6 +213271,7 @@ static void jsonReturnString( ){ assert( (pParse!=0)==(ctx!=0) ); assert( ctx==0 || ctx==p->pCtx ); + jsonStringTerminate(p); if( p->eErr==0 ){ int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); if( flags & JSON_BLOB ){ @@ -208699,7 +213279,7 @@ static void jsonReturnString( }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); - }else if( jsonStringTerminate(p) ){ + }else{ if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ int rc; pParse->zJson = sqlite3RCStrRef(p->zBuf); @@ -208715,11 +213295,11 @@ static void jsonReturnString( sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); - }else{ - sqlite3_result_error_nomem(p->pCtx); } }else if( p->eErr & JSTRING_OOM ){ sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_TOODEEP ){ + /* error already in p->pCtx */ }else if( p->eErr & JSTRING_MALFORMED ){ sqlite3_result_error(p->pCtx, "malformed JSON", -1); } @@ -209050,11 +213630,11 @@ static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ /* Slow version of jsonBlobAppendNode() that first resizes the ** pParse->aBlob structure. */ -static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static void jsonBlobAppendNode(JsonParse*,u8,u64,const void*); static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( JsonParse *pParse, u8 eType, - u32 szPayload, + u64 szPayload, const void *aPayload ){ if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; @@ -209074,7 +213654,7 @@ static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( static void jsonBlobAppendNode( JsonParse *pParse, /* The JsonParse object under construction */ u8 eType, /* Node type. One of JSONB_* */ - u32 szPayload, /* Number of bytes of payload */ + u64 szPayload, /* Number of bytes of payload */ const void *aPayload /* The payload. Might be NULL */ ){ u8 *a; @@ -209930,12 +214510,8 @@ static int jsonConvertTextToBlob( */ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; + assert( pStr->eErr==0 ); memset(&px, 0, sizeof(px)); - jsonStringTerminate(pStr); - if( pStr->eErr ){ - sqlite3_result_error_nomem(pStr->pCtx); - return; - } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); @@ -210025,7 +214601,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ static u32 jsonTranslateBlobToText( - const JsonParse *pParse, /* the complete parse of the JSON */ + JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ ){ @@ -210149,7 +214725,7 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '\''); break; case 'v': - jsonAppendRawNZ(pOut, "\\u0009", 6); + jsonAppendRawNZ(pOut, "\\u000b", 6); break; case 'x': if( sz2<4 ){ @@ -210207,10 +214783,14 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '['); j = i+n; iEnd = j+sz; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( j<iEnd && pOut->eErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, ','); } + pParse->iDepth--; if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, ']'); @@ -210221,10 +214801,14 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '{'); j = i+n; iEnd = j+sz; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( j<iEnd && pOut->eErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } + pParse->iDepth--; if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, '}'); @@ -210281,7 +214865,7 @@ static u32 jsonTranslateBlobToPrettyText( u32 i /* Start rendering at this index */ ){ u32 sz, n, j, iEnd; - const JsonParse *pParse = pPretty->pParse; + JsonParse *pParse = pPretty->pParse; JsonString *pOut = pPretty->pOut; n = jsonbPayloadSize(pParse, i, &sz); if( n==0 ){ @@ -210296,6 +214880,9 @@ static u32 jsonTranslateBlobToPrettyText( if( j<iEnd ){ jsonAppendChar(pOut, '\n'); pPretty->nIndent++; + if( pPretty->nIndent >= JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToPrettyText(pPretty, j); @@ -210317,6 +214904,10 @@ static u32 jsonTranslateBlobToPrettyText( if( j<iEnd ){ jsonAppendChar(pOut, '\n'); pPretty->nIndent++; + if( pPretty->nIndent >= JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } + pParse->iDepth = pPretty->nIndent; while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToText(pParse, j, pOut); @@ -210474,6 +215065,7 @@ static void jsonBlobEdit( u32 nIns /* Bytes of content to insert */ ){ i64 d = (i64)nIns - (i64)nDel; + assert( pParse->nBlob >= (u64)iDel + (u64)nDel ); if( d<0 && d>=(-8) && aIns!=0 && jsonBlobOverwrite(&pParse->aBlob[iDel], aIns, nIns, (int)-d) ){ @@ -210716,7 +215308,9 @@ static int jsonLabelCompare( */ #define JSON_LOOKUP_ERROR 0xffffffff #define JSON_LOOKUP_NOTFOUND 0xfffffffe -#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_NOTARRAY 0xfffffffd +#define JSON_LOOKUP_TOODEEP 0xfffffffc +#define JSON_LOOKUP_PATHERROR 0xfffffffb #define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) /* Forward declaration */ @@ -210745,7 +215339,7 @@ static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); static u32 jsonCreateEditSubstructure( JsonParse *pParse, /* The original JSONB that is being edited */ JsonParse *pIns, /* Populate this with the blob data to insert */ - const char *zTail /* Tail of the path that determins substructure */ + const char *zTail /* Tail of the path that determines substructure */ ){ static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; int rc; @@ -210763,7 +215357,12 @@ static u32 jsonCreateEditSubstructure( pIns->eEdit = pParse->eEdit; pIns->nIns = pParse->nIns; pIns->aIns = pParse->aIns; + pIns->iDepth = pParse->iDepth+1; + if( pIns->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->iDepth--; pParse->oom |= pIns->oom; } return rc; /* Error code only */ @@ -210780,9 +215379,9 @@ static u32 jsonCreateEditSubstructure( ** Return one of the JSON_LOOKUP error codes if problems are seen. ** ** This routine will also modify the blob. If pParse->eEdit is one of -** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be -** made to the selected value. If an edit is performed, then the return -** value does not necessarily point to the select element. If an edit +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, JEDIT_SET, or JEDIT_AINS, then changes +** might be made to the selected value. If an edit is performed, then the +** return value does not necessarily point to the select element. If an edit ** is performed, the return value is only useful for detecting error ** conditions. */ @@ -210808,6 +215407,13 @@ static u32 jsonLookupStep( jsonBlobEdit(pParse, iRoot, sz, 0, 0); }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ + }else if( pParse->eEdit==JEDIT_AINS ){ + /* json_array_insert() */ + if( zPath[-1]!=']' ){ + return JSON_LOOKUP_NOTARRAY; + }else{ + jsonBlobEdit(pParse, iRoot, 0, pParse->aIns, pParse->nIns); + } }else{ /* json_set() or json_replace() */ jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); @@ -210862,7 +215468,11 @@ static u32 jsonLookupStep( n = jsonbPayloadSize(pParse, v, &sz); if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; assert( j>0 ); + if( ++pParse->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pParse, v, &zPath[i], j); + pParse->iDepth--; if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } @@ -210879,6 +215489,10 @@ static u32 jsonLookupStep( JsonParse ix; /* Header of the label to be inserted */ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); + testcase( pParse->eEdit==JEDIT_AINS ); + if( pParse->eEdit==JEDIT_AINS && sqlite3_strglob("*]",&zPath[i])!=0 ){ + return JSON_LOOKUP_NOTARRAY; + } memset(&ix, 0, sizeof(ix)); ix.db = pParse->db; jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); @@ -210906,28 +215520,32 @@ static u32 jsonLookupStep( return rc; } }else if( zPath[0]=='[' ){ + u64 kk = 0; x = pParse->aBlob[iRoot] & 0x0f; if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); - k = 0; i = 1; while( sqlite3Isdigit(zPath[i]) ){ - k = k*10 + zPath[i] - '0'; + if( kk<0xffffffff ) kk = kk*10 + zPath[i] - '0'; + /* ^^^^^^^^^^--- Allow kk to be bigger than any JSON array so that + ** we get NOTFOUND instead of PATHERROR, without overflowing kk. */ i++; } if( i<2 || zPath[i]!=']' ){ if( zPath[1]=='#' ){ - k = jsonbArrayCount(pParse, iRoot); + kk = jsonbArrayCount(pParse, iRoot); i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int nn = 0; + u64 nn = 0; i = 3; do{ - nn = nn*10 + zPath[i] - '0'; + if( nn<0xffffffff ) nn = nn*10 + zPath[i] - '0'; + /* ^^^^^^^^^^--- Allow nn to be bigger than any JSON array to + ** get NOTFOUND instead of PATHERROR, without overflowing nn. */ i++; }while( sqlite3Isdigit(zPath[i]) ); - if( nn>k ) return JSON_LOOKUP_NOTFOUND; - k -= nn; + if( nn>kk ) return JSON_LOOKUP_NOTFOUND; + kk -= nn; } if( zPath[i]!=']' ){ return JSON_LOOKUP_PATHERROR; @@ -210939,21 +215557,26 @@ static u32 jsonLookupStep( j = iRoot+n; iEnd = j+sz; while( j<iEnd ){ - if( k==0 ){ + if( kk==0 ){ + if( ++pParse->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pParse, j, &zPath[i+1], 0); + pParse->iDepth--; if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } - k--; + kk--; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } if( j>iEnd ) return JSON_LOOKUP_ERROR; - if( k>0 ) return JSON_LOOKUP_NOTFOUND; + if( kk>0 ) return JSON_LOOKUP_NOTFOUND; if( pParse->eEdit>=JEDIT_INS ){ JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_AINS ); testcase( pParse->eEdit==JEDIT_SET ); rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); if( !JSON_LOOKUP_ISERROR(rc) @@ -210999,19 +215622,27 @@ static void jsonReturnTextJsonFromBlob( ** ** If the value is a primitive, return it as an SQL value. ** If the value is an array or object, return it as either -** JSON text or the BLOB encoding, depending on the JSON_B flag -** on the userdata. +** JSON text or the BLOB encoding, depending on the eMode flag +** as follows: +** +** eMode==0 JSONB if the JSON_B flag is set in userdata or +** text if the JSON_B flag is omitted from userdata. +** +** eMode==1 Text +** +** eMode==2 JSONB */ static void jsonReturnFromBlob( JsonParse *pParse, /* Complete JSON parse tree */ u32 i, /* Index of the node */ sqlite3_context *pCtx, /* Return value for this function */ - int textOnly /* return text JSON. Disregard user-data */ + int eMode /* Format of return: text of JSONB */ ){ u32 n, sz; int rc; sqlite3 *db = sqlite3_context_db_handle(pCtx); + assert( eMode>=0 && eMode<=2 ); n = jsonbPayloadSize(pParse, i, &sz); if( n==0 ){ sqlite3_result_error(pCtx, "malformed JSON", -1); @@ -211052,7 +215683,19 @@ static void jsonReturnFromBlob( rc = sqlite3DecOrHexToI64(z, &iRes); sqlite3DbFree(db, z); if( rc==0 ){ - sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + if( iRes<0 ){ + /* A hexadecimal literal with 16 significant digits and with the + ** high-order bit set is a negative integer in SQLite (and hence + ** iRes comes back as negative) but should be interpreted as a + ** positive value if it occurs within JSON. The value is too + ** large to appear as an SQLite integer so it must be converted + ** into floating point. */ + double r; + r = (double)*(sqlite3_uint64*)&iRes; + sqlite3_result_double(pCtx, bNeg ? -r : r); + }else{ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + } }else if( rc==3 && bNeg ){ sqlite3_result_int64(pCtx, SMALLEST_INT64); }else if( rc==1 ){ @@ -211071,7 +215714,7 @@ static void jsonReturnFromBlob( to_double: z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); if( z==0 ) goto returnfromblob_oom; - rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + rc = sqlite3AtoF(z, &r); sqlite3DbFree(db, z); if( rc<=0 ) goto returnfromblob_malformed; sqlite3_result_double(pCtx, r); @@ -211130,8 +215773,14 @@ static void jsonReturnFromBlob( } case JSONB_ARRAY: case JSONB_OBJECT: { - int flags = textOnly ? 0 : SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); - if( flags & JSON_BLOB ){ + if( eMode==0 ){ + if( (SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)) & JSON_BLOB)!=0 ){ + eMode = 2; + }else{ + eMode = 1; + } + } + if( eMode==2 ){ sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); }else{ jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); @@ -211245,16 +215894,35 @@ static int jsonFunctionArgToBlob( } /* -** Generate a bad path error. +** Generate a path error. +** +** The specifics of the error are determined by the rc argument. +** +** rc error +** ----------------- ---------------------- +** JSON_LOOKUP_ARRAY "not an array" +** JSON_LOOKUP_TOODEEP "JSON nested too deep" +** JSON_LOOKUP_ERROR "malformed JSON" +** otherwise... "bad JSON path" ** ** If ctx is not NULL then push the error message into ctx and return NULL. ** If ctx is NULL, then return the text of the error message. */ static char *jsonBadPathError( sqlite3_context *ctx, /* The function call containing the error */ - const char *zPath /* The path with the problem */ + const char *zPath, /* The path with the problem */ + int rc /* Maybe JSON_LOOKUP_NOTARRAY */ ){ - char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + char *zMsg; + if( rc==(int)JSON_LOOKUP_NOTARRAY ){ + zMsg = sqlite3_mprintf("not an array element: %Q", zPath); + }else if( rc==(int)JSON_LOOKUP_ERROR ){ + zMsg = sqlite3_mprintf("malformed JSON"); + }else if( rc==(int)JSON_LOOKUP_TOODEEP ){ + zMsg = sqlite3_mprintf("JSON path too deep"); + }else{ + zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + } if( ctx==0 ) return zMsg; if( zMsg ){ sqlite3_result_error(ctx, zMsg, -1); @@ -211271,13 +215939,13 @@ static char *jsonBadPathError( ** and return the result. ** ** The specific operation is determined by eEdit, which can be one -** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +** of JEDIT_INS, JEDIT_REPL, JEDIT_SET, or JEDIT_AINS. */ static void jsonInsertIntoBlob( sqlite3_context *ctx, int argc, sqlite3_value **argv, - int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ + int eEdit /* JEDIT_INS, JEDIT_REPL, JEDIT_SET, JEDIT_AINS */ ){ int i; u32 rc = 0; @@ -211314,6 +215982,7 @@ static void jsonInsertIntoBlob( p->nIns = ax.nBlob; p->aIns = ax.aBlob; p->delta = 0; + p->iDepth = 0; rc = jsonLookupStep(p, 0, zPath+1, 0); } jsonParseReset(&ax); @@ -211326,11 +215995,7 @@ static void jsonInsertIntoBlob( jsonInsertIntoBlob_patherror: jsonParseFree(p); - if( rc==JSON_LOOKUP_ERROR ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - }else{ - jsonBadPathError(ctx, zPath); - } + jsonBadPathError(ctx, zPath, rc); return; } @@ -211770,10 +216435,8 @@ static void jsonArrayLengthFunc( if( JSON_LOOKUP_ISERROR(i) ){ if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, i); } eErr = 1; i = 0; @@ -211876,7 +216539,7 @@ static void jsonExtractFunc( j = jsonLookupStep(p, 0, jx.zBuf, 0); jsonStringReset(&jx); }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_extract_error; } if( j<p->nBlob ){ @@ -211907,11 +216570,8 @@ static void jsonExtractFunc( jsonAppendSeparator(&jx); jsonAppendRawNZ(&jx, "null", 4); } - }else if( j==JSON_LOOKUP_ERROR ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - goto json_extract_error; }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, j); goto json_extract_error; } } @@ -211935,6 +216595,7 @@ json_extract_error: #define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ #define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ #define JSON_MERGE_OOM 3 /* Out-of-memory condition */ +#define JSON_MERGE_TOODEEP 4 /* Nested too deep */ /* ** RFC-7396 MergePatch for two JSONB blobs. @@ -211986,7 +216647,8 @@ static int jsonMergePatch( JsonParse *pTarget, /* The JSON parser that contains the TARGET */ u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ const JsonParse *pPatch, /* The PATCH */ - u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ + u32 iPatch, /* Index of PATCH in pPatch->aBlob[] */ + u32 iDepth /* Nesting depth */ ){ u8 x; /* Type of a single node */ u32 n, sz=0; /* Return values from jsonbPayloadSize() */ @@ -212095,7 +216757,8 @@ static int jsonMergePatch( /* Algorithm line 12 */ int rc, savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); + if( iDepth>=JSON_MAX_DEPTH ) return JSON_MERGE_TOODEEP; + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue, iDepth+1); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -212116,7 +216779,8 @@ static int jsonMergePatch( pTarget->aBlob[iTEnd+szNew] = 0x00; savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); + if( iDepth>=JSON_MAX_DEPTH ) return JSON_MERGE_TOODEEP; + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue,iDepth+1); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -212147,11 +216811,13 @@ static void jsonPatchFunc( if( pTarget==0 ) return; pPatch = jsonParseFuncArg(ctx, argv[1], 0); if( pPatch ){ - rc = jsonMergePatch(pTarget, 0, pPatch, 0); + rc = jsonMergePatch(pTarget, 0, pPatch, 0, 0); if( rc==JSON_MERGE_OK ){ jsonReturnParse(ctx, pTarget); }else if( rc==JSON_MERGE_OOM ){ sqlite3_result_error_nomem(ctx); + }else if( rc==JSON_MERGE_TOODEEP ){ + sqlite3_result_error(ctx, "JSON nested too deep", -1); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -212239,10 +216905,8 @@ static void jsonRemoveFunc( if( JSON_LOOKUP_ISERROR(rc) ){ if( rc==JSON_LOOKUP_NOTFOUND ){ continue; /* No-op */ - }else if( rc==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, rc); } goto json_remove_done; } @@ -212252,7 +216916,7 @@ static void jsonRemoveFunc( return; json_remove_patherror: - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); json_remove_done: jsonParseFree(p); @@ -212296,16 +216960,18 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - int bIsSet = (flags&JSON_ISSET)!=0; + int eInsType = JSON_INSERT_TYPE(flags); + static const char *azInsType[] = { "insert", "set", "array_insert" }; + static const u8 aEditType[] = { JEDIT_INS, JEDIT_SET, JEDIT_AINS }; if( argc<1 ) return; + assert( eInsType>=0 && eInsType<=2 ); if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + jsonWrongNumArgs(ctx, azInsType[eInsType]); return; } - jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); + jsonInsertIntoBlob(ctx, argc, argv, aEditType[eInsType]); } /* @@ -212330,17 +216996,15 @@ static void jsonTypeFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) goto json_type_done; if( zPath[0]!='$' ){ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_type_done; } i = jsonLookupStep(p, 0, zPath+1, 0); if( JSON_LOOKUP_ISERROR(i) ){ if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, i); } goto json_type_done; } @@ -212594,12 +217258,12 @@ static void jsonArrayStep( } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - int flags; pStr->pCtx = ctx; - jsonAppendChar(pStr, ']'); - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + jsonAppendRawNZ(pStr, "]", 2); + jsonStringTrimOneChar(pStr); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -212620,6 +217284,9 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } + }else if( flags & JSON_BLOB ){ + static const u8 emptyArray = 0x0b; + sqlite3_result_blob(ctx, &emptyArray, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); } @@ -212716,12 +217383,12 @@ static void jsonObjectStep( } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - int flags; - jsonAppendChar(pStr, '}'); + jsonAppendRawNZ(pStr, "}", 2); + jsonStringTrimOneChar(pStr); pStr->pCtx = ctx; - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -212742,6 +217409,9 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } + }else if( flags & JSON_BLOB ){ + static const unsigned char emptyObject = 0x0c; + sqlite3_result_blob(ctx, &emptyObject, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); } @@ -212778,6 +217448,7 @@ struct JsonEachCursor { u32 nRoot; /* Size of the root path in bytes */ u8 eType; /* Type of the container for element i */ u8 bRecursive; /* True for json_tree(). False for json_each() */ + u8 eMode; /* 1 for json_each(). 2 for jsonb_each() */ u32 nParent; /* Current nesting depth */ u32 nParentAlloc; /* Space allocated for aParent[] */ JsonParent *aParent; /* Parent elements of i */ @@ -212789,6 +217460,8 @@ typedef struct JsonEachConnection JsonEachConnection; struct JsonEachConnection { sqlite3_vtab base; /* Base class - must be first */ sqlite3 *db; /* Database connection */ + u8 eMode; /* 1 for json_each(). 2 for jsonb_each() */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ }; @@ -212831,6 +217504,8 @@ static int jsonEachConnect( if( pNew==0 ) return SQLITE_NOMEM; sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); pNew->db = db; + pNew->eMode = argv[0][4]=='b' ? 2 : 1; + pNew->bRecursive = argv[0][4+pNew->eMode]=='t'; } return rc; } @@ -212842,8 +217517,8 @@ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* constructor for a JsonEachCursor object for json_each(). */ -static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ +/* constructor for a JsonEachCursor object for json_each()/json_tree(). */ +static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ JsonEachConnection *pVtab = (JsonEachConnection*)p; JsonEachCursor *pCur; @@ -212851,21 +217526,13 @@ static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ pCur = sqlite3DbMallocZero(pVtab->db, sizeof(*pCur)); if( pCur==0 ) return SQLITE_NOMEM; pCur->db = pVtab->db; + pCur->eMode = pVtab->eMode; + pCur->bRecursive = pVtab->bRecursive; jsonStringZero(&pCur->path); *ppCursor = &pCur->base; return SQLITE_OK; } -/* constructor for a JsonEachCursor object for json_tree(). */ -static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - int rc = jsonEachOpenEach(p, ppCursor); - if( rc==SQLITE_OK ){ - JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; - pCur->bRecursive = 1; - } - return rc; -} - /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ @@ -213070,7 +217737,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { u32 i = jsonSkipLabel(p); - jsonReturnFromBlob(&p->sParse, i, ctx, 1); + jsonReturnFromBlob(&p->sParse, i, ctx, p->eMode); if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -213245,7 +217912,7 @@ static int jsonEachFilter( if( zRoot==0 ) return SQLITE_OK; if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -213263,7 +217930,7 @@ static int jsonEachFilter( return SQLITE_OK; } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -213314,36 +217981,7 @@ static sqlite3_module jsonEachModule = { jsonEachBestIndex, /* xBestIndex */ jsonEachDisconnect, /* xDisconnect */ 0, /* xDestroy */ - jsonEachOpenEach, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ -}; - -/* The methods of the json_tree virtual table. */ -static sqlite3_module jsonTreeModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachOpen, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ jsonEachNext, /* xNext - advance a cursor */ @@ -213382,6 +218020,8 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_insert, -1,1,1, 1,0,JSON_AINS, jsonSetFunc), + JFUNCTION(jsonb_array_insert,-1,1,0, 1,1,JSON_AINS, jsonSetFunc), JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), @@ -213432,22 +218072,21 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) /* -** Register the JSON table-valued functions +** Register the JSON table-valued function named zName and return a +** pointer to its Module object. Return NULL if something goes wrong. */ -SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3 *db){ - int rc = SQLITE_OK; - static const struct { - const char *zName; - sqlite3_module *pModule; - } aMod[] = { - { "json_each", &jsonEachModule }, - { "json_tree", &jsonTreeModule }, - }; +SQLITE_PRIVATE Module *sqlite3JsonVtabRegister(sqlite3 *db, const char *zName){ unsigned int i; - for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){ - rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0); + static const char *azModule[] = { + "json_each", "json_tree", "jsonb_each", "jsonb_tree" + }; + assert( sqlite3HashFind(&db->aModule, zName)==0 ); + for(i=0; i<sizeof(azModule)/sizeof(azModule[0]); i++){ + if( sqlite3StrICmp(azModule[i],zName)==0 ){ + return sqlite3VtabCreateModule(db, azModule[i], &jsonEachModule, 0, 0); + } } - return rc; + return 0; } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) */ @@ -213517,7 +218156,7 @@ SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3 *db){ #else /* #include "sqlite3.h" */ #endif -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char*,int*); /* In the SQLite core */ +SQLITE_PRIVATE sqlite3_int64 sqlite3GetToken(const unsigned char*,int*); /* In SQLite core */ /* #include <stddef.h> */ @@ -213552,7 +218191,7 @@ typedef unsigned int u32; # define NEVER(X) (X) #endif #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY @@ -214492,7 +219131,17 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; assert( pRtree->nCursor==0 ); nodeBlobReset(pRtree); - assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); + if( pRtree->nNodeRef ){ + int i; + assert( pRtree->bCorrupt ); + for(i=0; i<HASHSIZE; i++){ + while( pRtree->aHash[i] ){ + RtreeNode *pNext = pRtree->aHash[i]->pNext; + sqlite3_free(pRtree->aHash[i]); + pRtree->aHash[i] = pNext; + } + } + } sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -214590,6 +219239,12 @@ static void resetCursor(RtreeCursor *pCsr){ pCsr->base.pVtab = (sqlite3_vtab*)pRtree; pCsr->pReadAux = pStmt; + /* The following will only fail if the previous sqlite3_step() call failed, + ** in which case the error has already been caught. This statement never + ** encounters an error within an sqlite3_column_xxx() function, as it + ** calls sqlite3_column_value(), which does not use malloc(). So it is safe + ** to ignore the error code here. */ + sqlite3_reset(pStmt); } /* @@ -215778,7 +220433,7 @@ static int AdjustTree( int iCell; cnt++; - if( NEVER(cnt>100) ){ + if( cnt>100 ){ RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -216136,15 +220791,6 @@ static int SplitNode( rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRight); - pRight = 0; - } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pLeft); - pLeft = 0; - } - splitnode_out: nodeRelease(pRtree, pRight); nodeRelease(pRtree, pLeft); @@ -216329,7 +220975,7 @@ static int rtreeInsertCell( rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); - if( ALWAYS(rc==SQLITE_OK) ){ + if( rc==SQLITE_OK ){ if( iHeight==0 ){ rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); }else{ @@ -217224,7 +221870,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ if( node.zData==0 ) return; nData = sqlite3_value_bytes(apArg[1]); if( nData<4 ) return; - if( nData<NCELL(&node)*tree.nBytesPerCell ) return; + if( nData<4+NCELL(&node)*tree.nBytesPerCell ) return; pOut = sqlite3_str_new(0); for(ii=0; ii<NCELL(&node); ii++){ @@ -217957,7 +222603,7 @@ static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ /* The sqlite3AtoF() routine is much much faster than atof(), if it ** is available */ double r; - (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + (void)sqlite3AtoF((const char*)p->z, &r); *pVal = r; #else *pVal = (GeoCoord)atof((const char*)p->z); @@ -221896,7 +226542,7 @@ static int rbuDeltaApply( /* ERROR: copy exceeds output file size */ return -1; } - if( (int)(ofst+cnt) > lenSrc ){ + if( (u64)ofst+(u64)cnt > (u64)lenSrc ){ /* ERROR: copy extends past end of input */ return -1; } @@ -223542,8 +228188,8 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ /* If necessary, grow the pIter->aIdxCol[] array */ if( iIdxCol==nIdxAlloc ){ - RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( - pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64( + pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan) ); if( aIdxCol==0 ){ rc = SQLITE_NOMEM; @@ -227678,8 +232324,8 @@ typedef struct DbpageCursor DbpageCursor; struct DbpageCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ - int pgno; /* Current page number */ - int mxPgno; /* Last page to visit on this scan */ + Pgno pgno; /* Current page number */ + Pgno mxPgno; /* Last page to visit on this scan */ Pager *pPager; /* Pager being read/written */ DbPage *pPage1; /* Page 1 of the database */ int iDb; /* Index of database to analyze */ @@ -227816,7 +232462,7 @@ static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ }else{ memset(pCsr, 0, sizeof(DbpageCursor)); pCsr->base.pVtab = pVTab; - pCsr->pgno = -1; + pCsr->pgno = 0; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -227869,7 +232515,8 @@ static int dbpageFilter( sqlite3 *db = pTab->db; Btree *pBt; - (void)idxStr; + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); /* Default setting is no rows of result */ pCsr->pgno = 1; @@ -227915,12 +232562,12 @@ static int dbpageColumn( int rc = SQLITE_OK; switch( i ){ case 0: { /* pgno */ - sqlite3_result_int(ctx, pCsr->pgno); + sqlite3_result_int64(ctx, (sqlite3_int64)pCsr->pgno); break; } case 1: { /* data */ DbPage *pDbPage = 0; - if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + if( pCsr->pgno==(Pgno)((PENDING_BYTE/pCsr->szPage)+1) ){ /* The pending byte page. Assume it is zeroed out. Attempting to ** request this page from the page is an SQLITE_CORRUPT error. */ sqlite3_result_zeroblob(ctx, pCsr->szPage); @@ -227994,10 +232641,10 @@ static int dbpageUpdate( goto update_fail; } if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ - pgno = (Pgno)sqlite3_value_int(argv[2]); + pgno = (Pgno)sqlite3_value_int64(argv[2]); isInsert = 1; }else{ - pgno = sqlite3_value_int(argv[0]); + pgno = (Pgno)sqlite3_value_int64(argv[0]); if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ zErr = "cannot insert"; goto update_fail; @@ -228049,7 +232696,8 @@ static int dbpageUpdate( memcpy(aPage, pData, szPage); pTab->pgnoTrunc = 0; } - }else{ + } + if( rc!=SQLITE_OK ){ pTab->pgnoTrunc = 0; } sqlite3PagerUnref(pDbPage); @@ -228132,6 +232780,567 @@ SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; } #endif /* SQLITE_ENABLE_DBSTAT_VTAB */ /************** End of dbpage.c **********************************************/ +/************** Begin file carray.c ******************************************/ +/* +** 2016-06-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a table-valued-function that +** returns the values in a C-language array. +** Examples: +** +** SELECT * FROM carray($ptr,5) +** +** The query above returns 5 integers contained in a C-language array +** at the address $ptr. $ptr is a pointer to the array of integers. +** The pointer value must be assigned to $ptr using the +** sqlite3_bind_pointer() interface with a pointer type of "carray". +** For example: +** +** static int aX[] = { 53, 9, 17, 2231, 4, 99 }; +** int i = sqlite3_bind_parameter_index(pStmt, "$ptr"); +** sqlite3_bind_pointer(pStmt, i, aX, "carray", 0); +** +** There is an optional third parameter to determine the datatype of +** the C-language array. Allowed values of the third parameter are +** 'int32', 'int64', 'double', 'char*', 'struct iovec'. Example: +** +** SELECT * FROM carray($ptr,10,'char*'); +** +** The default value of the third parameter is 'int32'. +** +** HOW IT WORKS +** +** The carray "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE carray( +** value, +** pointer HIDDEN, +** count HIDDEN, +** ctype TEXT HIDDEN +** ); +** +** If the hidden columns "pointer" and "count" are unconstrained, then +** the virtual table has no rows. Otherwise, the virtual table interprets +** the integer value of "pointer" as a pointer to the array and "count" +** as the number of elements in the array. The virtual table steps through +** the array, element by element. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) +/* #include "sqliteInt.h" */ +#if defined(_WIN32) || defined(__RTP__) || defined(_WRS_KERNEL) + struct iovec { + void *iov_base; + size_t iov_len; + }; +#else +# include <sys/uio.h> +#endif + +/* +** Names of allowed datatypes +*/ +static const char *azCarrayType[] = { + "int32", "int64", "double", "char*", "struct iovec" +}; + +/* +** Structure used to hold the sqlite3_carray_bind() information +*/ +typedef struct carray_bind carray_bind; +struct carray_bind { + void *aData; /* The data */ + int nData; /* Number of elements */ + int mFlags; /* Control flags */ + void (*xDel)(void*); /* Destructor for aData */ + void *pDel; /* Alternative argument to xDel() */ +}; + + +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct carray_cursor carray_cursor; +struct carray_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + void *pPtr; /* Pointer to the array of values */ + sqlite3_int64 iCnt; /* Number of integers in the array */ + unsigned char eType; /* One of the CARRAY_type values */ +}; + +/* +** The carrayConnect() method is invoked to create a new +** carray_vtab that describes the carray virtual table. +** +** Think of this routine as the constructor for carray_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the carray_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against carray will look like. +*/ +static int carrayConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define CARRAY_COLUMN_VALUE 0 +#define CARRAY_COLUMN_POINTER 1 +#define CARRAY_COLUMN_COUNT 2 +#define CARRAY_COLUMN_CTYPE 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for carray_cursor objects. +*/ +static int carrayDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new carray_cursor object. +*/ +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + carray_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a carray_cursor. +*/ +static int carrayClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a carray_cursor to its next row of output. +*/ +static int carrayNext(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the carray_cursor +** is currently pointing. +*/ +static int carrayColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + carray_cursor *pCur = (carray_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case CARRAY_COLUMN_POINTER: return SQLITE_OK; + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; + case CARRAY_COLUMN_CTYPE: { + sqlite3_result_text(ctx, azCarrayType[pCur->eType], -1, SQLITE_STATIC); + return SQLITE_OK; + } + default: { + switch( pCur->eType ){ + case CARRAY_INT32: { + int *p = (int*)pCur->pPtr; + sqlite3_result_int(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_INT64: { + sqlite3_int64 *p = (sqlite3_int64*)pCur->pPtr; + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_DOUBLE: { + double *p = (double*)pCur->pPtr; + sqlite3_result_double(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_TEXT: { + const char **p = (const char**)pCur->pPtr; + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); + return SQLITE_OK; + } + default: { + const struct iovec *p = (struct iovec*)pCur->pPtr; + assert( pCur->eType==CARRAY_BLOB ); + sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base, + (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); + return SQLITE_OK; + } + } + } + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + carray_cursor *pCur = (carray_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int carrayEof(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + return pCur->iRowid>pCur->iCnt; +} + +/* +** This method is called to "rewind" the carray_cursor object back +** to the first row of output. +*/ +static int carrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + carray_cursor *pCur = (carray_cursor *)pVtabCursor; + pCur->pPtr = 0; + pCur->iCnt = 0; + switch( idxNum ){ + case 1: { + carray_bind *pBind = sqlite3_value_pointer(argv[0], "carray-bind"); + if( pBind==0 ) break; + pCur->pPtr = pBind->aData; + pCur->iCnt = pBind->nData; + pCur->eType = pBind->mFlags & 0x07; + break; + } + case 2: + case 3: { + pCur->pPtr = sqlite3_value_pointer(argv[0], "carray"); + pCur->iCnt = pCur->pPtr ? sqlite3_value_int64(argv[1]) : 0; + if( idxNum<3 ){ + pCur->eType = CARRAY_INT32; + }else{ + unsigned char i; + const char *zType = (const char*)sqlite3_value_text(argv[2]); + for(i=0; i<sizeof(azCarrayType)/sizeof(azCarrayType[0]); i++){ + if( sqlite3_stricmp(zType, azCarrayType[i])==0 ) break; + } + if( i>=sizeof(azCarrayType)/sizeof(azCarrayType[0]) ){ + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "unknown datatype: %Q", zType); + return SQLITE_ERROR; + }else{ + pCur->eType = i; + } + } + break; + } + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the carray virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** idxNum is: +** +** 1 If only the pointer= constraint exists. In this case, the +** parameter must be bound using sqlite3_carray_bind(). +** +** 2 if the pointer= and count= constraints exist. +** +** 3 if the ctype= constraint also exists. +** +** idxNum is 0 otherwise and carray becomes an empty table. +*/ +static int carrayBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + unsigned seen = 0; /* Bitmask of == constrainted columns */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn>=0 ) seen |= 1 << pConstraint->iColumn; + if( pConstraint->usable==0 ) continue; + switch( pConstraint->iColumn ){ + case CARRAY_COLUMN_POINTER: + ptrIdx = i; + break; + case CARRAY_COLUMN_COUNT: + cntIdx = i; + break; + case CARRAY_COLUMN_CTYPE: + ctypeIdx = i; + break; + } + } + if( ptrIdx>=0 ){ + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 1; + if( cntIdx>=0 ){ + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; + pIdxInfo->idxNum = 2; + if( ctypeIdx>=0 ){ + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; + pIdxInfo->idxNum = 3; + }else if( seen & (1<<CARRAY_COLUMN_CTYPE) ){ + /* In a three-argument carray(), we need to know the value of all + ** three arguments */ + return SQLITE_CONSTRAINT; + } + }else if( seen & (1<<CARRAY_COLUMN_COUNT) ){ + /* In a two-argument carray(), we need to know the value of both + ** arguments */ + return SQLITE_CONSTRAINT; + } + }else{ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + pIdxInfo->idxNum = 0; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** carray virtual table. +*/ +static sqlite3_module carrayModule = { + 0, /* iVersion */ + 0, /* xCreate */ + carrayConnect, /* xConnect */ + carrayBestIndex, /* xBestIndex */ + carrayDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + carrayOpen, /* xOpen - open a cursor */ + carrayClose, /* xClose - close a cursor */ + carrayFilter, /* xFilter - configure scan constraints */ + carrayNext, /* xNext - advance a cursor */ + carrayEof, /* xEof - check for end of scan */ + carrayColumn, /* xColumn - read data */ + carrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadow */ + 0 /* xIntegrity */ +}; + +/* +** Destructor for the carray_bind object +*/ +static void carrayBindDel(void *pPtr){ + carray_bind *p = (carray_bind*)pPtr; + if( p->xDel!=SQLITE_STATIC ){ + p->xDel(p->pDel); + } + sqlite3_free(p); +} + +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). +** +** pStmt The prepared statement to which to bind +** idx The index of the parameter of pStmt to which to bind +** aData The data to be bound +** nData The number of elements in aData +** mFlags One of SQLITE_CARRAY_xxxx indicating datatype of aData +** xDestroy Destructor for pDestroy or aData if pDestroy==NULL. +** pDestroy Invoke xDestroy on this pointer if not NULL +** +** The destructor is called pDestroy if pDestroy!=NULL, or against +** aData if pDestroy==NULL. +*/ +SQLITE_API int sqlite3_carray_bind_v2( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*), + void *pDestroy +){ + carray_bind *pNew = 0; + int i; + int rc = SQLITE_OK; + + /* Ensure that the mFlags value is acceptable. */ + assert( CARRAY_INT32==0 && CARRAY_INT64==1 && CARRAY_DOUBLE==2 ); + assert( CARRAY_TEXT==3 && CARRAY_BLOB==4 ); + if( mFlags<CARRAY_INT32 || mFlags>CARRAY_BLOB ){ + rc = SQLITE_ERROR; + goto carray_bind_error; + } + + pNew = sqlite3_malloc64(sizeof(*pNew)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto carray_bind_error; + } + + pNew->nData = nData; + pNew->mFlags = mFlags; + if( xDestroy==SQLITE_TRANSIENT ){ + sqlite3_int64 sz = nData; + switch( mFlags ){ + case CARRAY_INT32: sz *= 4; break; + case CARRAY_INT64: sz *= 8; break; + case CARRAY_DOUBLE: sz *= 8; break; + case CARRAY_TEXT: sz *= sizeof(char*); break; + default: sz *= sizeof(struct iovec); break; + } + if( mFlags==CARRAY_TEXT ){ + for(i=0; i<nData; i++){ + const char *z = ((char**)aData)[i]; + if( z ) sz += strlen(z) + 1; + } + }else if( mFlags==CARRAY_BLOB ){ + for(i=0; i<nData; i++){ + sz += ((struct iovec*)aData)[i].iov_len; + } + } + + pNew->aData = sqlite3_malloc64( sz ); + if( pNew->aData==0 ){ + rc = SQLITE_NOMEM; + goto carray_bind_error; + } + + if( mFlags==CARRAY_TEXT ){ + char **az = (char**)pNew->aData; + char *z = (char*)&az[nData]; + for(i=0; i<nData; i++){ + const char *zData = ((char**)aData)[i]; + sqlite3_int64 n; + if( zData==0 ){ + az[i] = 0; + continue; + } + az[i] = z; + n = strlen(zData); + memcpy(z, zData, n+1); + z += n+1; + } + }else if( mFlags==CARRAY_BLOB ){ + struct iovec *p = (struct iovec*)pNew->aData; + unsigned char *z = (unsigned char*)&p[nData]; + for(i=0; i<nData; i++){ + size_t n = ((struct iovec*)aData)[i].iov_len; + p[i].iov_len = n; + p[i].iov_base = z; + z += n; + memcpy(p[i].iov_base, ((struct iovec*)aData)[i].iov_base, n); + } + }else{ + memcpy(pNew->aData, aData, sz); + } + pNew->xDel = sqlite3_free; + pNew->pDel = pNew->aData; + }else{ + pNew->aData = aData; + pNew->xDel = xDestroy; + pNew->pDel = pDestroy; + } + return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); + + carray_bind_error: + if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ + xDestroy(pDestroy); + } + sqlite3_free(pNew); + return rc; +} + +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). Same as sqlite3_carray_bind_v2() with the +** pDestroy parameter set to NULL. +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*) +){ + return sqlite3_carray_bind_v2(pStmt,idx,aData,nData,mFlags,xDestroy,aData); +} + +/* +** Invoke this routine to register the carray() function. +*/ +SQLITE_PRIVATE Module *sqlite3CarrayRegister(sqlite3 *db){ + return sqlite3VtabCreateModule(db, "carray", &carrayModule, 0, 0); +} + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) */ + +/************** End of carray.c **********************************************/ /************** Begin file sqlite3session.c **********************************/ #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) @@ -228484,6 +233693,21 @@ static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } +/* +** Read a varint value from buffer aBuf[], size nBuf bytes, into *piVal. +** Return the number of bytes read. +*/ +static int sessionVarintGetSafe(const u8 *aBuf, int nBuf, int *piVal){ + u8 aCopy[9]; + const u8 *aRead = aBuf; + memset(aCopy, 0, sizeof(aCopy)); + if( nBuf<sizeof(aCopy) ){ + memcpy(aCopy, aBuf, nBuf); + aRead = aCopy; + } + return getVarint32(aRead, *piVal); +} + /* Load an unaligned and unsigned 32-bit integer */ #define SESSION_UINT32(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) @@ -228513,6 +233737,19 @@ static void sessionPutI64(u8 *aBuf, sqlite3_int64 i){ } /* +** Write a double value to the buffer aBuf[]. +*/ +static void sessionPutDouble(u8 *aBuf, double r){ + /* TODO: SQLite does something special to deal with mixed-endian + ** floating point values (e.g. ARM7). This code probably should + ** too. */ + u64 i; + assert( sizeof(double)==8 && sizeof(u64)==8 ); + memcpy(&i, &r, 8); + sessionPutI64(aBuf, i); +} + +/* ** This function is used to serialize the contents of value pValue (see ** comment titled "RECORD FORMAT" above). ** @@ -228549,16 +233786,13 @@ static int sessionSerializeValue( /* TODO: SQLite does something special to deal with mixed-endian ** floating point values (e.g. ARM7). This code probably should ** too. */ - u64 i; if( eType==SQLITE_INTEGER ){ - i = (u64)sqlite3_value_int64(pValue); + u64 i = (u64)sqlite3_value_int64(pValue); + sessionPutI64(&aBuf[1], i); }else{ - double r; - assert( sizeof(double)==8 && sizeof(u64)==8 ); - r = sqlite3_value_double(pValue); - memcpy(&i, &r, 8); + double r = sqlite3_value_double(pValue); + sessionPutDouble(&aBuf[1], r); } - sessionPutI64(&aBuf[1], i); } nByte = 9; break; @@ -228748,10 +233982,11 @@ static int sessionSerialLen(const u8 *a){ int n; assert( a!=0 ); e = *a; - if( e==0 || e==0xFF ) return 1; - if( e==SQLITE_NULL ) return 1; if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; - return sessionVarintGet(&a[1], &n) + 1 + n; + if( e==SQLITE_TEXT || e==SQLITE_BLOB ){ + return sessionVarintGet(&a[1], &n) + 1 + n; + } + return 1; } /* @@ -228774,31 +234009,31 @@ static unsigned int sessionChangeHash( u8 *a = aRecord; /* Used to iterate through change record */ for(i=0; i<pTab->nCol; i++){ - int eType = *a; int isPK = pTab->abPK[i]; if( bPkOnly && isPK==0 ) continue; - /* It is not possible for eType to be SQLITE_NULL here. The session - ** module does not record changes for rows with NULL values stored in - ** primary key columns. */ - assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT - || eType==SQLITE_TEXT || eType==SQLITE_BLOB - || eType==SQLITE_NULL || eType==0 - ); - assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); - if( isPK ){ - a++; + int eType = *a++; + + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_TEXT || eType==SQLITE_BLOB + || eType==SQLITE_NULL || eType==0 + ); + h = sessionHashAppendType(h, eType); if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ h = sessionHashAppendI64(h, sessionGetI64(a)); a += 8; - }else{ + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int n; a += sessionVarintGet(a, &n); h = sessionHashAppendBlob(h, n, a); a += n; } + /* It should not be possible for eType to be SQLITE_NULL or 0x00 here, + ** as the session module does not record changes for rows with NULL + ** values stored in primary key columns. But a corrupt changesets + ** may contain such a value. */ }else{ a += sessionSerialLen(a); } @@ -229490,9 +234725,7 @@ static void sessionUpdateOneChange( case SQLITE_FLOAT: { double rVal = sqlite3_column_double(pDflt, iField); - i64 iVal = 0; - memcpy(&iVal, &rVal, sizeof(rVal)); - sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + sessionPutDouble(&pNew->aRecord[pNew->nRecord], rVal); pNew->nRecord += 8; break; } @@ -230749,15 +235982,14 @@ static void sessionAppendCol( int eType = sqlite3_column_type(pStmt, iCol); sessionAppendByte(p, (u8)eType, pRc); if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - sqlite3_int64 i; u8 aBuf[8]; if( eType==SQLITE_INTEGER ){ - i = sqlite3_column_int64(pStmt, iCol); + sqlite3_int64 i = sqlite3_column_int64(pStmt, iCol); + sessionPutI64(aBuf, i); }else{ double r = sqlite3_column_double(pStmt, iCol); - memcpy(&i, &r, 8); + sessionPutDouble(aBuf, r); } - sessionPutI64(aBuf, i); sessionAppendBlob(p, aBuf, 8, pRc); } if( eType==SQLITE_BLOB || eType==SQLITE_TEXT ){ @@ -230950,6 +236182,19 @@ static int sessionAppendDelete( return rc; } +static int sessionPrepare( + sqlite3 *db, + sqlite3_stmt **pp, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, pp, 0); + if( pzErrmsg && rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + return rc; +} + /* ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. @@ -230971,12 +236216,12 @@ static int sessionSelectStmt( int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ - sqlite3_stmt **ppStmt /* OUT: Prepared SELECT statement */ + sqlite3_stmt **ppStmt, /* OUT: Prepared SELECT statement */ + char **pzErrmsg /* OUT: Error message */ ){ int rc = SQLITE_OK; char *zSql = 0; const char *zSep = ""; - int nSql = -1; int i; SessionBuffer cols = {0, 0, 0}; @@ -231056,7 +236301,7 @@ static int sessionSelectStmt( #endif if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, zSql, nSql, ppStmt, 0); + rc = sessionPrepare(db, ppStmt, pzErrmsg, zSql); } sqlite3_free(zSql); sqlite3_free(nooptest.aBuf); @@ -231194,10 +236439,13 @@ static int sessionGenerateChangeset( } if( pSession->rc ) return pSession->rc; - rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); - if( rc!=SQLITE_OK ) return rc; sqlite3_mutex_enter(sqlite3_db_mutex(db)); + rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); + if( rc!=SQLITE_OK ){ + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; + } for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ @@ -231220,7 +236468,7 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt(db, 0, pSession->zDb, - zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel + zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel, 0 ); } @@ -231680,7 +236928,8 @@ static int sessionReadRecord( u8 *aVal = &pIn->aData[pIn->iNext]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int nByte; - pIn->iNext += sessionVarintGet(aVal, &nByte); + int nRem = pIn->nData - pIn->iNext; + pIn->iNext += sessionVarintGetSafe(aVal, nRem, &nByte); rc = sessionInputBuffer(pIn, nByte); if( rc==SQLITE_OK ){ if( nByte<0 || nByte>pIn->nData-pIn->iNext ){ @@ -231733,7 +236982,8 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ rc = sessionInputBuffer(pIn, 9); if( rc==SQLITE_OK ){ - nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); + int nBuf = pIn->nData - pIn->iNext; + nRead += sessionVarintGetSafe(&pIn->aData[pIn->iNext], nBuf, &nCol); /* The hard upper limit for the number of columns in an SQLite ** database table is, according to sqliteLimit.h, 32676. So ** consider any table-header that purports to have more than 65536 @@ -231753,8 +237003,15 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ while( (pIn->iNext + nRead)<pIn->nData && pIn->aData[pIn->iNext + nRead] ){ nRead++; } + + /* Break out of the loop if if the nul-terminator byte has been found. + ** Otherwise, read some more input data and keep seeking. If there is + ** no more input data, consider the changeset corrupt. */ if( (pIn->iNext + nRead)<pIn->nData ) break; rc = sessionInputBuffer(pIn, nRead + 100); + if( rc==SQLITE_OK && (pIn->iNext + nRead)>=pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + } } *pnByte = nRead+1; return rc; @@ -231775,7 +237032,7 @@ static int sessionChangesetBufferRecord( int *pnByte /* OUT: Size of record in bytes */ ){ int rc = SQLITE_OK; - int nByte = 0; + i64 nByte = 0; int i; for(i=0; rc==SQLITE_OK && i<nCol; i++){ int eType; @@ -231784,13 +237041,19 @@ static int sessionChangesetBufferRecord( eType = pIn->aData[pIn->iNext + nByte++]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int n; - nByte += sessionVarintGet(&pIn->aData[pIn->iNext+nByte], &n); + int nRem = pIn->nData - (pIn->iNext + nByte); + nByte += sessionVarintGetSafe(&pIn->aData[pIn->iNext+nByte], nRem, &n); nByte += n; rc = sessionInputBuffer(pIn, nByte); }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ nByte += 8; + }else if( eType!=0 && eType!=SQLITE_NULL ){ + rc = SQLITE_CORRUPT_BKPT; } } + if( rc==SQLITE_OK && (pIn->iNext+nByte)>pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + } } *pnByte = nByte; return rc; @@ -231886,10 +237149,10 @@ static int sessionChangesetNextOne( memset(p->apValue, 0, sizeof(sqlite3_value*)*p->nCol*2); } - /* Make sure the buffer contains at least 10 bytes of input data, or all - ** remaining data if there are less than 10 bytes available. This is - ** sufficient either for the 'T' or 'P' byte and the varint that follows - ** it, or for the two single byte values otherwise. */ + /* Make sure the buffer contains at least 2 bytes of input data, or all + ** remaining data if there are less than 2 bytes available. This is + ** sufficient either for the 'T' or 'P' byte that begins a new table, + ** or for the "op" and "bIndirect" single bytes otherwise. */ p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; @@ -231919,11 +237182,13 @@ static int sessionChangesetNextOne( return (p->rc = SQLITE_CORRUPT_BKPT); } - p->op = op; - p->bIndirect = p->in.aData[p->in.iNext++]; - if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ + if( (op!=SQLITE_UPDATE && op!=SQLITE_DELETE && op!=SQLITE_INSERT) + || (p->in.iNext>=p->in.nData) + ){ return (p->rc = SQLITE_CORRUPT_BKPT); } + p->op = op; + p->bIndirect = p->in.aData[p->in.iNext++]; if( paRec ){ int nVal; /* Number of values to buffer */ @@ -232429,6 +237694,7 @@ struct SessionApplyCtx { u8 bRebase; /* True to collect rebase information */ u8 bIgnoreNoop; /* True to ignore no-op conflicts */ int bRowid; + char *zErr; /* Error message, if any */ }; /* Number of prepared UPDATE statements to cache. */ @@ -232654,7 +237920,7 @@ static int sessionDeleteRow( } if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pDelete, 0); + rc = sessionPrepare(db, &p->pDelete, &p->zErr, (char*)buf.aBuf); } sqlite3_free(buf.aBuf); @@ -232681,7 +237947,7 @@ static int sessionSelectRow( ){ /* TODO */ return sessionSelectStmt(db, p->bIgnoreNoop, - "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect, &p->zErr ); } @@ -232718,16 +237984,12 @@ static int sessionInsertRow( sessionAppendStr(&buf, ")", &rc); if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pInsert, 0); + rc = sessionPrepare(db, &p->pInsert, &p->zErr, (char*)buf.aBuf); } sqlite3_free(buf.aBuf); return rc; } -static int sessionPrepare(sqlite3 *db, sqlite3_stmt **pp, const char *zSql){ - return sqlite3_prepare_v2(db, zSql, -1, pp, 0); -} - /* ** Prepare statements for applying changes to the sqlite_stat1 table. ** These are similar to those created by sessionSelectRow(), @@ -232737,14 +237999,14 @@ static int sessionPrepare(sqlite3 *db, sqlite3_stmt **pp, const char *zSql){ static int sessionStat1Sql(sqlite3 *db, SessionApplyCtx *p){ int rc = sessionSelectRow(db, "sqlite_stat1", p); if( rc==SQLITE_OK ){ - rc = sessionPrepare(db, &p->pInsert, + rc = sessionPrepare(db, &p->pInsert, 0, "INSERT INTO main.sqlite_stat1 VALUES(?1, " "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END, " "?3)" ); } if( rc==SQLITE_OK ){ - rc = sessionPrepare(db, &p->pDelete, + rc = sessionPrepare(db, &p->pDelete, 0, "DELETE FROM main.sqlite_stat1 WHERE tbl=?1 AND idx IS " "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END " "AND (?4 OR stat IS ?3)" @@ -232968,7 +238230,7 @@ static int sessionConflictHandler( void *pCtx, /* First argument for conflict handler */ int *pbReplace /* OUT: Set to true if PK row is found */ ){ - int res = 0; /* Value returned by conflict handler */ + int res = SQLITE_CHANGESET_OMIT;/* Value returned by conflict handler */ int rc; int nCol; int op; @@ -232989,11 +238251,9 @@ static int sessionConflictHandler( if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - if( p->bIgnoreNoop - && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + if( 0==p->bIgnoreNoop + || 0==sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) ){ - res = SQLITE_CHANGESET_OMIT; - }else{ pIter->pConflict = p->pSelect; res = xConflict(pCtx, eType, pIter); pIter->pConflict = 0; @@ -233007,7 +238267,9 @@ static int sessionConflictHandler( int nBlob = pIter->in.iNext - pIter->in.iCurrent; sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc); return SQLITE_OK; - }else{ + }else if( p->bIgnoreNoop==0 || op!=SQLITE_DELETE + || eType==SQLITE_CHANGESET_CONFLICT + ){ /* No other row with the new.* primary key. */ res = xConflict(pCtx, eType+1, pIter); if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE; @@ -233105,7 +238367,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -233317,6 +238579,10 @@ static int sessionChangesetApply( void *pCtx, /* Copy of sixth arg to _apply() */ const char *zTab /* Table name */ ), + int(*xFilterIter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), int(*xConflict)( void *pCtx, /* Copy of fifth arg to _apply() */ int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ @@ -233457,6 +238723,9 @@ static int sessionChangesetApply( ** next change. A log message has already been issued. */ if( schemaMismatch ) continue; + /* If this is a call to apply_v3(), invoke xFilterIter here. */ + if( xFilterIter && 0==xFilterIter(pCtx, pIter) ) continue; + rc = sessionApplyOneWithRetry(db, pIter, &sApply, xConflict, pCtx); } @@ -233503,6 +238772,7 @@ static int sessionChangesetApply( assert( sApply.bRebase || sApply.rebase.nBuf==0 ); if( rc==SQLITE_OK && bPatchset==0 && sApply.bRebase ){ + assert( ppRebase!=0 && pnRebase!=0 ); *ppRebase = (void*)sApply.rebase.aBuf; *pnRebase = sApply.rebase.nBuf; sApply.rebase.aBuf = 0; @@ -233520,22 +238790,74 @@ static int sessionChangesetApply( db->flags &= ~((u64)SQLITE_FkNoAction); db->aDb[0].pSchema->schema_cookie -= 32; } + + assert( rc!=SQLITE_OK || sApply.zErr==0 ); + sqlite3_set_errmsg(db, rc, sApply.zErr); + sqlite3_free(sApply.zErr); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); return rc; } /* -** Apply the changeset passed via pChangeset/nChangeset to the main -** database attached to handle "db". +** This function is called by all six sqlite3changeset_apply() variants: +** +** + sqlite3changeset_apply() +** + sqlite3changeset_apply_v2() +** + sqlite3changeset_apply_v3() +** + sqlite3changeset_apply_strm() +** + sqlite3changeset_apply_strm_v2() +** + sqlite3changeset_apply_strm_v3() +** +** Arguments passed to this function are as follows: +** +** db: +** Database handle to apply changeset to main database of. +** +** nChangeset/pChangeset: +** These are both passed zero for the streaming variants. For the normal +** apply() functions, these are passed the size of and the buffer containing +** the changeset, respectively. +** +** xInput/pIn: +** These are both passed zero for the normal variants. For the streaming +** apply() functions, these are passed the input callback and context +** pointer, respectively. +** +** xFilter: +** The filter function as passed to apply() or apply_v2() (to filter by +** table name), if any. This is always NULL for apply_v3() calls. +** +** xFilterIter: +** The filter function as passed to apply_v3(), if any. +** +** xConflict: +** The conflict handler callback (must not be NULL). +** +** pCtx: +** The context pointer passed to the xFilter and xConflict handler callbacks. +** +** ppRebase, pnRebase: +** Zero for apply(). The rebase changeset output pointers, if any, for +** apply_v2() and apply_v3(). +** +** flags: +** Zero for apply(). The flags parameter for apply_v2() and apply_v3(). */ -SQLITE_API int sqlite3changeset_apply_v2( +static int sessionChangesetApplyV23( sqlite3 *db, /* Apply change to "main" db of this handle */ int nChangeset, /* Size of changeset in bytes */ void *pChangeset, /* Changeset blob */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ int(*xFilter)( void *pCtx, /* Copy of sixth arg to _apply() */ const char *zTab /* Table name */ ), + int(*xFilterIter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing current change */ + ), int(*xConflict)( void *pCtx, /* Copy of sixth arg to _apply() */ int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ @@ -233546,19 +238868,75 @@ SQLITE_API int sqlite3changeset_apply_v2( int flags ){ sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int bInv = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); - int rc = sessionChangesetStart(&pIter, 0, 0, nChangeset, pChangeset, bInv, 1); - + int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + int rc = sessionChangesetStart( + &pIter, xInput, pIn, nChangeset, pChangeset, bInverse, 1 + ); if( rc==SQLITE_OK ){ - rc = sessionChangesetApply( - db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags + rc = sessionChangesetApply(db, pIter, + xFilter, xFilterIter, xConflict, pCtx, ppRebase, pnRebase, flags ); } - return rc; } /* +** Apply the changeset passed via pChangeset/nChangeset to the main +** database attached to handle "db". +*/ +SQLITE_API int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + xFilter, 0, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} + +/* +** Apply the changeset passed via pChangeset/nChangeset to the main +** database attached to handle "db". +*/ +SQLITE_API int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing current change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + 0, xFilter, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} + +/* ** Apply the changeset passed via pChangeset/nChangeset to the main database ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. @@ -233578,8 +238956,10 @@ SQLITE_API int sqlite3changeset_apply( ), void *pCtx /* First argument passed to xConflict */ ){ - return sqlite3changeset_apply_v2( - db, nChangeset, pChangeset, xFilter, xConflict, pCtx, 0, 0, 0 + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + xFilter, 0, xConflict, pCtx, + 0, 0, 0 ); } @@ -233588,6 +238968,29 @@ SQLITE_API int sqlite3changeset_apply( ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. */ +SQLITE_API int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + 0, xFilter, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} SQLITE_API int sqlite3changeset_apply_v2_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ @@ -233605,15 +239008,11 @@ SQLITE_API int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ){ - sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); - int rc = sessionChangesetStart(&pIter, xInput, pIn, 0, 0, bInverse, 1); - if( rc==SQLITE_OK ){ - rc = sessionChangesetApply( - db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags - ); - } - return rc; + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + xFilter, 0, xConflict, pCtx, + ppRebase, pnRebase, flags + ); } SQLITE_API int sqlite3changeset_apply_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ @@ -233630,12 +239029,29 @@ SQLITE_API int sqlite3changeset_apply_strm( ), void *pCtx /* First argument passed to xConflict */ ){ - return sqlite3changeset_apply_v2_strm( - db, xInput, pIn, xFilter, xConflict, pCtx, 0, 0, 0 + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + xFilter, 0, xConflict, pCtx, + 0, 0, 0 ); } /* +** The parts of the sqlite3_changegroup structure used by the +** sqlite3changegroup_change_xxx() APIs. +*/ +typedef struct ChangeData ChangeData; +struct ChangeData { + SessionTable *pTab; + int bIndirect; + int eOp; + + int nBufAlloc; + SessionBuffer *aBuf; + SessionBuffer record; +}; + +/* ** sqlite3_changegroup handle. */ struct sqlite3_changegroup { @@ -233646,12 +239062,17 @@ struct sqlite3_changegroup { sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ + ChangeData cd; /* Used by changegroup_change_xxx() APIs. */ }; /* ** This function is called to merge two changes to the same row together as ** part of an sqlite3changeset_concat() operation. A new change object is ** allocated and a pointer to it stored in *ppNew. +** +** Because they have been vetted by sqlite3changegroup_add() or similar, +** both the aRec[] change and the pExist change are safe to use without +** checking for buffer overflows. */ static int sessionChangeMerge( SessionTable *pTab, /* Table structure */ @@ -233792,7 +239213,7 @@ static int sessionChangeMerge( memcpy(aCsr, aRec, nRec); aCsr += nRec; }else{ - if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist, 0,aRec,0) ){ + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist,0,aRec,0) ){ sqlite3_free(pNew); pNew = 0; } @@ -233885,15 +239306,14 @@ static int sessionChangesetExtendRecord( switch( eType ){ case SQLITE_FLOAT: case SQLITE_INTEGER: { - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); - }else{ - double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); - memcpy(&iVal, &rVal, sizeof(i64)); - } if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ - sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + if( eType==SQLITE_INTEGER ){ + sqlite3_int64 iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + }else{ + double rVal = sqlite3_column_double(pTab->pDfltStmt, ii); + sessionPutDouble(&pOut->aBuf[pOut->nBuf], rVal); + } pOut->nBuf += 8; } break; @@ -233964,13 +239384,19 @@ static int sessionChangesetFindTable( int nCol = 0; *ppTab = 0; - sqlite3changeset_pk(pIter, &abPK, &nCol); /* Search the list for an existing table */ for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; } + + if( pIter ){ + sqlite3changeset_pk(pIter, &abPK, &nCol); + }else if( !pTab && !pGrp->db ){ + return SQLITE_OK; + } + /* If one was not found above, create a new table now */ if( !pTab ){ SessionTable **ppNew; @@ -233982,15 +239408,17 @@ static int sessionChangesetFindTable( memset(pTab, 0, sizeof(SessionTable)); pTab->nCol = nCol; pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); + if( nCol>0 ){ + memcpy(pTab->abPK, abPK, nCol); + } pTab->zName = (char*)&pTab->abPK[nCol]; memcpy(pTab->zName, zTab, nTab+1); if( pGrp->db ){ pTab->nCol = 0; rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); + if( rc || pTab->nCol==0 ){ + sqlite3_free(pTab->azCol); sqlite3_free(pTab); return rc; } @@ -234005,7 +239433,7 @@ static int sessionChangesetFindTable( } /* Check that the table is compatible. */ - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + if( pIter && !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ rc = SQLITE_SCHEMA; } @@ -234014,44 +239442,27 @@ static int sessionChangesetFindTable( } /* -** Add the change currently indicated by iterator pIter to the hash table -** belonging to changegroup pGrp. +** Add a single change to the changegroup pGrp. */ static int sessionOneChangeToHash( - sqlite3_changegroup *pGrp, - sqlite3_changeset_iter *pIter, - int bRebase + sqlite3_changegroup *pGrp, /* Changegroup to update */ + SessionTable *pTab, /* Table change pertains to */ + int op, /* One of SQLITE_INSERT, UPDATE, DELETE */ + int bIndirect, /* True to flag change as "indirect" */ + int nCol, /* Number of columns in record(s) */ + u8 *aRec, /* Serialized change record(s) */ + int nRec, /* Size of aRec[] in bytes */ + int bRebase /* True if this is a rebase blob */ ){ int rc = SQLITE_OK; - int nCol = 0; - int op = 0; int iHash = 0; - int bIndirect = 0; SessionChange *pChange = 0; SessionChange *pExist = 0; SessionChange **pp = 0; - SessionTable *pTab = 0; - u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; - int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; assert( nRec>0 ); - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_OK ){ - const char *zTab = 0; - sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); - rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); - } - - if( rc==SQLITE_OK && nCol<pTab->nCol ){ + if( nCol<pTab->nCol ){ SessionBuffer *pBuf = &pGrp->rec; rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); aRec = pBuf->aBuf; @@ -234059,7 +239470,7 @@ static int sessionOneChangeToHash( assert( pGrp->db ); } - if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + if( rc==SQLITE_OK && sessionGrowHash(0, pGrp->bPatch, pTab) ){ rc = SQLITE_NOMEM; } @@ -234067,12 +239478,12 @@ static int sessionOneChangeToHash( /* Search for existing entry. If found, remove it from the hash table. ** Code below may link it back in. */ iHash = sessionChangeHash( - pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange + pTab, (pGrp->bPatch && op==SQLITE_DELETE), aRec, pTab->nChange ); for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; - if( pIter->bPatchset ){ + if( pGrp->bPatch ){ bPkOnly1 = (*pp)->op==SQLITE_DELETE; bPkOnly2 = op==SQLITE_DELETE; } @@ -234087,7 +239498,7 @@ static int sessionOneChangeToHash( if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, - pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange + pGrp->bPatch, pExist, op, bIndirect, aRec, nRec, &pChange ); } if( rc==SQLITE_OK && pChange ){ @@ -234096,6 +239507,47 @@ static int sessionOneChangeToHash( pTab->nEntry++; } + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeIterToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + const char *zTab = 0; + int nCol = 0; + int op = 0; + int bIndirect = 0; + int rc = SQLITE_OK; + SessionTable *pTab = 0; + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK ){ + rc = sessionOneChangeToHash( + pGrp, pTab, op, bIndirect, nCol, aRec, nRec, bRebase + ); + } + if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -234115,7 +239567,7 @@ static int sessionChangesetToHash( pIter->in.bNoDiscard = 1; while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ - rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + rc = sessionOneChangeIterToHash(pGrp, pIter, bRebase); if( rc!=SQLITE_OK ) break; } @@ -234206,6 +239658,33 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp){ } /* +** Configure a changegroup object. +*/ +SQLITE_API int sqlite3changegroup_config( + sqlite3_changegroup *pGrp, + int op, + void *pArg +){ + int rc = SQLITE_OK; + + switch( op ){ + case SQLITE_CHANGEGROUP_CONFIG_PATCHSET: { + int arg = *(int*)pArg; + if( pGrp->pList==0 && arg>=0 ){ + pGrp->bPatch = (arg>0); + } + *(int*)pArg = pGrp->bPatch; + break; + } + default: + rc = SQLITE_MISUSE; + break; + } + + return rc; +} + +/* ** Provide a database schema to the changegroup object. */ SQLITE_API int sqlite3changegroup_schema( @@ -234263,7 +239742,7 @@ SQLITE_API int sqlite3changegroup_add_change( rc = SQLITE_ERROR; }else{ pIter->in.bNoDiscard = 1; - rc = sessionOneChangeToHash(pGrp, pIter, 0); + rc = sessionOneChangeIterToHash(pGrp, pIter, 0); } return rc; } @@ -234315,6 +239794,12 @@ SQLITE_API int sqlite3changegroup_output_strm( */ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ + int ii; + for(ii=0; ii<pGrp->cd.nBufAlloc; ii++){ + sqlite3_free(pGrp->cd.aBuf[ii].aBuf); + } + sqlite3_free(pGrp->cd.record.aBuf); + sqlite3_free(pGrp->cd.aBuf); sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); sqlite3_free(pGrp->rec.aBuf); @@ -234745,6 +240230,328 @@ SQLITE_API int sqlite3session_config(int op, void *pArg){ return rc; } +/* +** Begin adding a change to a changegroup object. +*/ +SQLITE_API int sqlite3changegroup_change_begin( + sqlite3_changegroup *pGrp, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +){ + SessionTable *pTab = 0; + int rc = SQLITE_OK; + + if( pGrp->cd.pTab ){ + rc = SQLITE_MISUSE; + }else if( eOp!=SQLITE_INSERT && eOp!=SQLITE_UPDATE && eOp!=SQLITE_DELETE ){ + rc = SQLITE_ERROR; + }else{ + rc = sessionChangesetFindTable(pGrp, zTab, 0, &pTab); + } + if( rc==SQLITE_OK ){ + if( pTab==0 ){ + if( pzErr ){ + *pzErr = sqlite3_mprintf("no such table: %s", zTab); + } + rc = SQLITE_ERROR; + }else{ + int nReq = pTab->nCol * (eOp==SQLITE_UPDATE ? 2 : 1); + pGrp->cd.pTab = pTab; + pGrp->cd.eOp = eOp; + pGrp->cd.bIndirect = bIndirect; + + if( pGrp->cd.nBufAlloc<nReq ){ + SessionBuffer *aBuf = (SessionBuffer*)sqlite3_realloc( + pGrp->cd.aBuf, nReq * sizeof(SessionBuffer) + ); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&aBuf[pGrp->cd.nBufAlloc], 0, + sizeof(SessionBuffer) * (nReq - pGrp->cd.nBufAlloc) + ); + pGrp->cd.aBuf = aBuf; + pGrp->cd.nBufAlloc = nReq; + } + } + +#ifdef SQLITE_DEBUG + { + /* Assert that all column values are currently undefined */ + int ii; + for(ii=0; ii<pGrp->cd.nBufAlloc; ii++){ + assert( pGrp->cd.aBuf[ii].nBuf==0 ); + } + } +#endif + } + } + + return rc; +} + +/* +** This function does processing common to the _change_int64(), _change_text() +** and other similar APIs. +*/ +static int checkChangeParams( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + sqlite3_int64 nReq, + SessionBuffer **ppBuf +){ + int rc = SQLITE_OK; + if( pGrp->cd.pTab==0 ){ + rc = SQLITE_MISUSE; + }else if( iCol<0 || iCol>=pGrp->cd.pTab->nCol ){ + rc = SQLITE_RANGE; + }else if( + (bNew && pGrp->cd.eOp==SQLITE_DELETE) + || (!bNew && pGrp->cd.eOp==SQLITE_INSERT) + ){ + rc = SQLITE_ERROR; + }else{ + SessionBuffer *pBuf = &pGrp->cd.aBuf[iCol]; + if( pGrp->cd.eOp==SQLITE_UPDATE && bNew ){ + pBuf += pGrp->cd.pTab->nCol; + } + pBuf->nBuf = 0; + sessionBufferGrow(pBuf, nReq, &rc); + pBuf->nBuf = nReq; + *ppBuf = pBuf; + } + return rc; +} + +/* +** Configure the change currently under construction with an integer value. +*/ +SQLITE_API int sqlite3changegroup_change_int64( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + sqlite3_int64 iVal +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 9, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_INTEGER; + sessionPutI64(&pBuf->aBuf[1], iVal); + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a null value. +*/ +SQLITE_API int sqlite3changegroup_change_null( + sqlite3_changegroup *pGrp, + int bNew, + int iCol +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 1, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_NULL; + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a real value. +*/ +SQLITE_API int sqlite3changegroup_change_double( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + double fVal +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 9, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_FLOAT; + sessionPutDouble(&pBuf->aBuf[1], fVal); + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a text value. +*/ +SQLITE_API int sqlite3changegroup_change_text( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + const char *pVal, + int nVal +){ + int nText = nVal>=0 ? nVal : strlen(pVal); + sqlite3_int64 nByte = 1 + sessionVarintLen(nText) + nText; + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, nByte, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_TEXT; + pBuf->nBuf = (1 + sessionVarintPut(&pBuf->aBuf[1], nText)); + memcpy(&pBuf->aBuf[pBuf->nBuf], pVal, nText); + pBuf->nBuf += nText; + + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a blob value. +*/ +SQLITE_API int sqlite3changegroup_change_blob( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + const void *pVal, + int nVal +){ + sqlite3_int64 nByte = 1 + sessionVarintLen(nVal) + nVal; + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, nByte, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_BLOB; + pBuf->nBuf = (1 + sessionVarintPut(&pBuf->aBuf[1], nVal)); + memcpy(&pBuf->aBuf[pBuf->nBuf], pVal, nVal); + pBuf->nBuf += nVal; + + return SQLITE_OK; +} + +/* +** Finish any change currently being constructed by the changegroup object. +*/ +SQLITE_API int sqlite3changegroup_change_finish( + sqlite3_changegroup *pGrp, + int bDiscard, + char **pzErr +){ + int rc = SQLITE_OK; + if( pGrp->cd.pTab ){ + SessionBuffer *aBuf = pGrp->cd.aBuf; + int ii; + + if( bDiscard==0 ){ + int nBuf = pGrp->cd.pTab->nCol; + u8 eUndef = SQLITE_NULL; + if( pGrp->cd.eOp==SQLITE_UPDATE ){ + for(ii=0; ii<nBuf; ii++){ + if( pGrp->cd.pTab->abPK[ii] ){ + if( aBuf[ii].nBuf<=1 ){ + *pzErr = sqlite3_mprintf( + "invalid change: %s value in PK of old.* record", + aBuf[ii].nBuf==1 ? "null" : "undefined" + ); + rc = SQLITE_ERROR; + break; + }else if( aBuf[ii + nBuf].nBuf>0 ){ + *pzErr = sqlite3_mprintf( + "invalid change: defined value in PK of new.* record" + ); + rc = SQLITE_ERROR; + break; + } + }else + if( pGrp->bPatch==0 && (aBuf[ii].nBuf>0)!=(aBuf[ii+nBuf].nBuf>0) ){ + *pzErr = sqlite3_mprintf( + "invalid change: column %d " + "- old.* value is %sdefined but new.* is %sdefined", + ii, aBuf[ii].nBuf ? "" : "un", aBuf[ii+nBuf].nBuf ? "" : "un" + ); + rc = SQLITE_ERROR; + break; + } + } + eUndef = 0x00; + if( pGrp->bPatch==0 ) nBuf = nBuf * 2; + }else{ + for(ii=0; ii<nBuf; ii++){ + int isPK = pGrp->cd.pTab->abPK[ii]; + if( (pGrp->cd.eOp==SQLITE_INSERT || pGrp->bPatch==0 || isPK) + && aBuf[ii].nBuf==0 + ){ + *pzErr = sqlite3_mprintf( + "invalid change: column %d is undefined", ii + ); + rc = SQLITE_ERROR; + break; + } + if( aBuf[ii].nBuf==1 && isPK ){ + *pzErr = sqlite3_mprintf( + "invalid change: null value in PK" + ); + rc = SQLITE_ERROR; + break; + } + } + } + + pGrp->cd.record.nBuf = 0; + for(ii=0; ii<nBuf; ii++){ + SessionBuffer *p = &pGrp->cd.aBuf[ii]; + if( pGrp->bPatch ){ + if( pGrp->cd.pTab->abPK[ii]==0 ){ + if( pGrp->cd.eOp==SQLITE_UPDATE ){ + p += pGrp->cd.pTab->nCol; + }else if( pGrp->cd.eOp==SQLITE_DELETE ){ + continue; + } + } + } + if( 0==sessionBufferGrow(&pGrp->cd.record, p->nBuf?p->nBuf:1, &rc) ){ + if( p->nBuf ){ + memcpy(&pGrp->cd.record.aBuf[pGrp->cd.record.nBuf],p->aBuf,p->nBuf); + pGrp->cd.record.nBuf += p->nBuf; + }else{ + pGrp->cd.record.aBuf[pGrp->cd.record.nBuf++] = eUndef; + } + } + } + if( rc==SQLITE_OK ){ + rc = sessionOneChangeToHash( + pGrp, pGrp->cd.pTab, + pGrp->cd.eOp, pGrp->cd.bIndirect, pGrp->cd.pTab->nCol, + pGrp->cd.record.aBuf, pGrp->cd.record.nBuf, 0 + ); + } + } + + /* Reset all aBuf[] entries to "undefined". */ + { + int nZero = pGrp->cd.pTab->nCol; + if( pGrp->cd.eOp==SQLITE_UPDATE ) nZero += nZero; + for(ii=0; ii<nZero; ii++){ + pGrp->cd.aBuf[ii].nBuf = 0; + } + } + pGrp->cd.pTab = 0; + } + + return rc; +} + #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ /************** End of sqlite3session.c **************************************/ @@ -235603,27 +241410,20 @@ typedef sqlite3_uint64 u64; # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -/* The uptr type is an unsigned integer large enough to hold a pointer +/* +** This macro is used in a single assert() within fts5 to check that an +** allocation is aligned to an 8-byte boundary. But it is a complicated +** macro to get right for multiple platforms without generating warnings. +** So instead of reproducing the entire definition from sqliteInt.h, we +** just do without this assert() for the rare non-amalgamation builds. */ -#if defined(HAVE_STDINT_H) - typedef uintptr_t uptr; -#elif SQLITE_PTRSIZE==4 - typedef u32 uptr; -#else - typedef u64 uptr; -#endif - -#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC -# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) -#else -# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) -#endif +#define EIGHT_BYTE_ALIGNMENT(x) 1 /* ** Macros needed to provide flexible arrays in a portable way */ #ifndef offsetof -# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY @@ -235631,7 +241431,13 @@ typedef sqlite3_uint64 u64; # define FLEXARRAY 1 #endif -#endif +#endif /* SQLITE_AMALGAMATION */ + +/* +** Constants for the largest and smallest possible 32-bit signed integers. +*/ +# define LARGEST_INT32 ((int)(0x7fffffff)) +# define SMALLEST_INT32 ((int)((-1) - LARGEST_INT32)) /* Truncate very long tokens to this many bytes. Hard limit is ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset @@ -236365,7 +242171,7 @@ static int sqlite3Fts5ExprPattern( ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); ** } */ -static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc); static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); static int sqlite3Fts5ExprEof(Fts5Expr*); static i64 sqlite3Fts5ExprRowid(Fts5Expr*); @@ -236676,14 +242482,22 @@ typedef union { #define sqlite3Fts5ParserARG_PARAM ,pParse #define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; #define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#undef fts5YYREALLOC #define fts5YYREALLOC realloc +#undef fts5YYFREE #define fts5YYFREE free +#undef fts5YYDYNSTACK #define fts5YYDYNSTACK 0 +#undef fts5YYSIZELIMIT +#define sqlite3Fts5ParserCTX(P) 0 #define sqlite3Fts5ParserCTX_SDECL #define sqlite3Fts5ParserCTX_PDECL #define sqlite3Fts5ParserCTX_PARAM #define sqlite3Fts5ParserCTX_FETCH #define sqlite3Fts5ParserCTX_STORE +#undef fts5YYERRORSYMBOL +#undef fts5YYERRSYMDT +#undef fts5YYFALLBACK #define fts5YYNSTATE 35 #define fts5YYNRULE 28 #define fts5YYNRULE_WITH_ACTION 28 @@ -237008,15 +242822,24 @@ static int fts5yyGrowStack(fts5yyParser *p){ int newSize; int idx; fts5yyStackEntry *pNew; +#ifdef fts5YYSIZELIMIT + int nLimit = fts5YYSIZELIMIT(sqlite3Fts5ParserCTX(p)); +#endif newSize = oldSize*2 + 100; +#ifdef fts5YYSIZELIMIT + if( newSize>nLimit ){ + newSize = nLimit; + if( newSize<=oldSize ) return 1; + } +#endif idx = (int)(p->fts5yytos - p->fts5yystack); if( p->fts5yystack==p->fts5yystk0 ){ - pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0])); + pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0]), sqlite3Fts5ParserCTX(p)); if( pNew==0 ) return 1; memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0])); + pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0]), sqlite3Fts5ParserCTX(p)); if( pNew==0 ) return 1; } p->fts5yystack = pNew; @@ -237196,7 +243019,9 @@ static void sqlite3Fts5ParserFinalize(void *p){ } #if fts5YYGROWABLESTACK - if( pParser->fts5yystack!=pParser->fts5yystk0 ) fts5YYFREE(pParser->fts5yystack); + if( pParser->fts5yystack!=pParser->fts5yystk0 ){ + fts5YYFREE(pParser->fts5yystack, sqlite3Fts5ParserCTX(pParser)); + } #endif } @@ -238478,7 +244303,7 @@ static void fts5SnippetFunction( iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); - aSeen = sqlite3_malloc(nPhrase); + aSeen = sqlite3_malloc64(nPhrase); if( aSeen==0 ){ rc = SQLITE_NOMEM; } @@ -239133,7 +244958,7 @@ static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ if( nIn<0 ){ nIn = (int)strlen(pIn); } - zRet = (char*)sqlite3_malloc(nIn+1); + zRet = (char*)sqlite3_malloc64((i64)nIn+1); if( zRet ){ memcpy(zRet, pIn, nIn); zRet[nIn] = '\0'; @@ -239833,7 +245658,7 @@ static int sqlite3Fts5ConfigParse( sqlite3_int64 nByte; int bUnindexed = 0; /* True if there are one or more UNINDEXED */ - *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + *ppOut = pRet = (Fts5Config*)sqlite3_malloc64(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->pGlobal = pGlobal; @@ -240381,8 +246206,6 @@ static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_end(ap); } - - /* ** 2014 May 31 ** @@ -240699,7 +246522,7 @@ static int sqlite3Fts5ExprNew( assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ - *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + *ppNew = pNew = sqlite3_malloc64(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); @@ -240851,7 +246674,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ p2->pRoot = 0; if( sParse.rc==SQLITE_OK ){ - Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64( p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) ); if( ap==0 ){ @@ -241934,7 +247757,13 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ -static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ +static int sqlite3Fts5ExprFirst( + Fts5Expr *p, + Fts5Index *pIdx, + i64 iFirst, + i64 iLast, + int bDesc +){ Fts5ExprNode *pRoot = p->pRoot; int rc; /* Return code */ @@ -241956,6 +247785,9 @@ static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bD assert( pRoot->bEof==0 ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } return rc; } @@ -243754,7 +249586,7 @@ static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte int rc = SQLITE_OK; Fts5Hash *pNew; - *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -244808,6 +250640,36 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption found reading blob %lld from table \"%s\"", + iRowid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ROWID(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid) + +static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption on page %d, segment %d, table \"%s\"", + pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ITER(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter) + +static int fts5IndexCorruptIdx(Fts5Index *pIdx){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption in table \"%s\"", pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_IDX(pIdx) fts5IndexCorruptIdx(pIdx) + + /* ** Array of tombstone pages. Reference counted. */ @@ -245097,13 +250959,13 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ ** All the reasons those functions might return SQLITE_ERROR - missing ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ - if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT_ROWID(p, iRowid); if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ - int nByte = sqlite3_blob_bytes(p->pReader); - int szData = (sizeof(Fts5Data) + 7) & ~7; - sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING; + i64 nByte = sqlite3_blob_bytes(p->pReader); + i64 szData = (sizeof(Fts5Data) + 7) & ~7; + i64 nAlloc = szData + nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); if( pRet ){ pRet->nn = nByte; @@ -245147,7 +251009,7 @@ static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = fts5DataRead(p, iRowid); if( pRet ){ if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); fts5DataRelease(pRet); pRet = 0; } @@ -245506,8 +251368,14 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->rc==SQLITE_OK ){ + if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + }else if( p->rc==SQLITE_CORRUPT_VTAB ){ + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName + ); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ @@ -246130,7 +251998,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_ITER(p, pIter); return; } iOff = 4; @@ -246162,7 +252030,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ iOff += fts5GetVarint32(&a[iOff], nNew); if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->term.n = nKeep; @@ -246292,6 +252160,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ while( 1 ){ u64 iDelta = 0; + if( i>=n ) break; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ if( i<n && a[i]==0 ){ @@ -246310,7 +252179,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ - int nNew = pIter->nRowidOffset + 8; + i64 nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; @@ -246357,7 +252226,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ if( iRowidOff>=pNew->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); }else{ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -246591,7 +252460,7 @@ static void fts5SegIterNext( } assert_nc( iOff<pLeaf->szLeaf ); if( iOff>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } } @@ -246699,18 +252568,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - iOff = fts5LeafFirstRowidOff(pLast); - if( iOff>pLast->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; + if( p->rc==SQLITE_OK ){ + iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + FTS5_CORRUPT_ITER(p, pIter); + return; + } + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; - if( fts5LeafIsTermless(pLast) ){ - pIter->iEndofDoclist = pLast->nn+1; - }else{ - pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } } } @@ -246780,7 +252651,7 @@ static void fts5LeafSeek( iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -246823,7 +252694,7 @@ static void fts5LeafSeek( iOff = iTermOff; if( iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -246845,7 +252716,7 @@ static void fts5LeafSeek( iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; }else{ nKeep = 0; @@ -246860,7 +252731,7 @@ static void fts5LeafSeek( search_success: if( (i64)iOff+nNew>n || nNew<1 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->iLeafOffset = iOff + nNew; @@ -247325,7 +253196,7 @@ static void fts5SegIterGotoPage( assert( iLeafPgno>pIter->iLeafPgno ); if( iLeafPgno>pIter->pSeg->pgnoLast ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; @@ -247340,7 +253211,7 @@ static void fts5SegIterGotoPage( u8 *a = pIter->pLeaf->p; int n = pIter->pLeaf->szLeaf; if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; @@ -247819,7 +253690,7 @@ static void fts5ChunkIterate( if( nRem<=0 ){ break; }else if( pSeg->pSeg==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); return; }else{ pgno++; @@ -248922,7 +254793,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ ** a single page has been assigned to more than one segment. In ** this case a prior iteration of this loop may have corrupted the ** segment currently being trimmed. */ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iLeafRowid); }else{ fts5BufferZero(&buf); fts5BufferGrow(&p->rc, &buf, pData->nn); @@ -249389,7 +255260,7 @@ static void fts5SecureDeleteOverflow( }else if( bDetailNone ){ break; }else if( iNext>=pLeaf->szLeaf || pLeaf->nn<pLeaf->szLeaf || iNext<4 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; }else{ int nShift = iNext - 4; @@ -249409,7 +255280,7 @@ static void fts5SecureDeleteOverflow( i1 += fts5GetVarint32(&aPg[i1], iFirst); if( iFirst<iNext ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; } aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); @@ -249455,7 +255326,7 @@ static void fts5DoSecureDelete( int iSegid = pSeg->pSeg->iSegid; u8 *aPg = pSeg->pLeaf->p; int nPg = pSeg->pLeaf->nn; - int iPgIdx = pSeg->pLeaf->szLeaf; + int iPgIdx = pSeg->pLeaf->szLeaf; /* Offset of page footer */ u64 iDelta = 0; int iNextOff = 0; @@ -249534,7 +255405,7 @@ static void fts5DoSecureDelete( iSOP += fts5GetVarint32(&aPg[iSOP], nPos); } assert_nc( iSOP==pSeg->iLeafOffset ); - iNextOff = pSeg->iLeafOffset + pSeg->nPos; + iNextOff = iSOP + pSeg->nPos; } } @@ -249614,32 +255485,32 @@ static void fts5DoSecureDelete( ** is another term following it on this page. So the subsequent term ** needs to be moved to replace the term associated with the entry ** being removed. */ - int nPrefix = 0; - int nSuffix = 0; - int nPrefix2 = 0; - int nSuffix2 = 0; + u64 nPrefix = 0; + u64 nSuffix = 0; + u64 nPrefix2 = 0; + u64 nSuffix2 = 0; iDelKeyOff = iNextOff; - iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); - iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + iNextOff += fts5GetVarint(&aPg[iNextOff], &nPrefix2); + iNextOff += fts5GetVarint(&aPg[iNextOff], &nSuffix2); if( iKey!=1 ){ - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nPrefix); } - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nSuffix); nPrefix = MIN(nPrefix, nPrefix2); nSuffix = (nPrefix2 + nSuffix2) - nPrefix; - if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ - p->rc = FTS5_CORRUPT; + if( (iKeyOff+nSuffix)>(u64)iPgIdx || (iNextOff+nSuffix2)>(u64)iPgIdx ){ + FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>pSeg->term.n ){ - p->rc = FTS5_CORRUPT; + if( nPrefix2>(u64)pSeg->term.n ){ + FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); @@ -249669,7 +255540,7 @@ static void fts5DoSecureDelete( u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; int nTermIdx = pTerm->nn - pTerm->szLeaf; int iTermIdx = 0; - int iTermOff = 0; + i64 iTermOff = 0; while( 1 ){ u32 iVal = 0; @@ -249680,12 +255551,15 @@ static void fts5DoSecureDelete( } nTermIdx = iTermIdx; - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); - - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + if( iTermOff>pTerm->szLeaf ){ + FTS5_CORRUPT_IDX(p); + }else{ + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } } } fts5DataRelease(pTerm); @@ -249708,7 +255582,9 @@ static void fts5DoSecureDelete( int iPrevKeyOut = 0; int iKeyIn = 0; - memmove(&aPg[iOff], &aPg[iNextOff], nMove); + if( nMove>0 ){ + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + } iPgIdx -= nShift; nPg = iPgIdx; fts5PutU16(&aPg[2], iPgIdx); @@ -250063,7 +255939,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( } nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); - assert( nByte==SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); + assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -250146,7 +256022,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ fts5StructureRelease(pStruct); pStruct = pNew; nMin = 1; - nMerge = nMerge*-1; + nMerge = (nMerge==SMALLEST_INT32 ? LARGEST_INT32 : (nMerge*-1)); } if( pStruct && pStruct->nLevel ){ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ @@ -250432,7 +256308,7 @@ static void fts5MergePrefixLists( } if( pHead==0 || pHead->pNext==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); break; } @@ -250469,7 +256345,7 @@ static void fts5MergePrefixLists( assert_nc( tmp.n+nTail<=nTmp ); assert( tmp.n+nTail<=nTmp+nMerge*10 ); if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_IDX(p); break; } fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); @@ -250628,16 +256504,16 @@ struct Fts5TokenDataMap { ** aMap[] variables. */ struct Fts5TokenDataIter { - int nMapAlloc; /* Allocated size of aMap[] in entries */ - int nMap; /* Number of valid entries in aMap[] */ + i64 nMapAlloc; /* Allocated size of aMap[] in entries */ + i64 nMap; /* Number of valid entries in aMap[] */ Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ /* The following are used for prefix-queries only. */ Fts5Buffer terms; /* The following are used for other full-token tokendata queries only. */ - int nIter; - int nIterAlloc; + i64 nIter; + i64 nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; Fts5Iter *apIter[FLEXARRAY]; @@ -250693,11 +256569,11 @@ static void fts5TokendataIterAppendMap( ){ if( p->rc==SQLITE_OK ){ if( pT->nMap==pT->nMapAlloc ){ - int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nAlloc = nNew * sizeof(Fts5TokenDataMap); + i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + i64 nAlloc = nNew * sizeof(Fts5TokenDataMap); Fts5TokenDataMap *aNew; - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + aNew = (Fts5TokenDataMap*)sqlite3_realloc64(pT->aMap, nAlloc); if( aNew==0 ){ p->rc = SQLITE_NOMEM; return; @@ -250723,7 +256599,7 @@ static void fts5TokendataIterAppendMap( */ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ Fts5TokenDataMap *aTmp = 0; - int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap); aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); if( aTmp ){ @@ -251038,11 +256914,14 @@ static int sqlite3Fts5IndexRollback(Fts5Index *p){ */ static int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure *pTmp; - u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + union { + Fts5Structure sFts; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + } uFts; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - pTmp = (Fts5Structure*)tmpSpace; - memset(pTmp, 0, SZ_FTS5STRUCTURE(1)); + pTmp = &uFts.sFts; + memset(uFts.tmpSpace, 0, sizeof(uFts.tmpSpace)); if( p->pConfig->bContentlessDelete ){ pTmp->nOriginCntr = 1; } @@ -251254,9 +257133,10 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ - int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); - Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); + Fts5TokenDataIter *pNew; + pNew = (Fts5TokenDataIter*)sqlite3_realloc64(pIn, nByte); if( pNew==0 ){ p->rc = SQLITE_NOMEM; @@ -251353,8 +257233,8 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Ensure the token-mapping is large enough */ if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ - int nNew = (pT->nMapAlloc + nByte) * 2; - Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + i64 nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64( pT->aMap, nNew*sizeof(Fts5TokenDataMap) ); if( aNew==0 ){ @@ -252502,19 +258382,27 @@ static int fts5TestUtf8(const char *z, int n){ /* ** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. +** +** This function sets output variable (*pbFail) to true if the test fails. Or +** leaves it unchanged if the test succeeds. */ static void fts5TestTerm( Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, - u64 *pCksum + u64 *pCksum, + int *pbFail ){ int rc = p->rc; if( pPrev->n==0 ){ fts5BufferSet(&rc, pPrev, n, (const u8*)z); }else - if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + if( *pbFail==0 + && rc==SQLITE_OK + && (pPrev->n!=n || memcmp(pPrev->p, z, n)) + && (p->pHash==0 || p->pHash->nEntry==0) + ){ u64 cksum3 = *pCksum; const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ @@ -252564,7 +258452,7 @@ static void fts5TestTerm( fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ - rc = FTS5_CORRUPT; + *pbFail = 1; } *pCksum = cksum3; } @@ -252573,7 +258461,7 @@ static void fts5TestTerm( #else # define fts5TestDlidxReverse(x,y,z) -# define fts5TestTerm(u,v,w,x,y,z) +# define fts5TestTerm(t,u,v,w,x,y,z) #endif /* @@ -252598,14 +258486,17 @@ static void fts5IndexIntegrityCheckEmpty( for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ - if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) + || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)) + ){ + FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + } } fts5DataRelease(pLeaf); } } -static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ +static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){ i64 iTermOff = 0; int ii; @@ -252623,12 +258514,12 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff = iTermOff; if( iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else if( iTermOff==nIncr ){ int nByte; iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); } @@ -252637,7 +258528,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ buf1.n = nKeep; fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); @@ -252645,7 +258536,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ if( p->rc==SQLITE_OK ){ res = fts5BufferCompare(&buf1, &buf2); - if( res<=0 ) p->rc = FTS5_CORRUPT; + if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid); } } fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); @@ -252706,7 +258597,7 @@ static void fts5IndexIntegrityCheckSegment( ** entry even if all the terms are removed from it by secure-delete ** operations. */ }else{ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); } }else{ @@ -252718,15 +258609,15 @@ static void fts5IndexIntegrityCheckSegment( iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; - if( res<0 ) p->rc = FTS5_CORRUPT; + if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow); } - fts5IntegrityCheckPgidx(p, pLeaf); + fts5IntegrityCheckPgidx(p, iRow, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -252756,7 +258647,7 @@ static void fts5IndexIntegrityCheckSegment( iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ - if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) FTS5_CORRUPT_ROWID(p, iKey); fts5DataRelease(pLeaf); } } @@ -252771,12 +258662,12 @@ static void fts5IndexIntegrityCheckSegment( int iRowidOff = fts5LeafFirstRowidOff(pLeaf); ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); }else if( bSecureDelete==0 || iRowidOff>0 ){ i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); } } fts5DataRelease(pLeaf); @@ -252828,6 +258719,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ + int bTestFail = 0; #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; @@ -252870,7 +258762,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ - fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail); if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ @@ -252888,15 +258780,26 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum } } } - fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail); fts5MultiIterFree(pIter); - if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; - - fts5StructureRelease(pStruct); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ){ + p->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName + ); + } #ifdef SQLITE_DEBUG + /* In SQLITE_DEBUG builds, expensive extra checks were run as part of + ** the integrity-check above. If no other errors were detected, but one + ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */ + if( p->rc==SQLITE_OK && bTestFail ){ + p->rc = FTS5_CORRUPT; + } fts5BufferFree(&term); #endif + + fts5StructureRelease(pStruct); fts5BufferFree(&poslist); return fts5IndexReturn(p); } @@ -254240,6 +260143,17 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ #endif } +static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008002 ) +#endif + { + pIdxInfo->estimatedRows = MAX(1, nRow); + } +#endif +} + static int fts5UsePatternMatch( Fts5Config *pConfig, struct sqlite3_index_constraint *p @@ -254304,19 +260218,30 @@ static int fts5UsePatternMatch( ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: ** -** * No other constraints: cost=1000.0 -** * One rowid range constraint: cost=750.0 -** * Both rowid range constraints: cost=500.0 -** * An == rowid constraint: cost=100.0 +** * No other constraints: cost=50000.0 +** * One rowid range constraint: cost=37500.0 +** * Both rowid range constraints: cost=30000.0 +** * An == rowid constraint: cost=25000.0 ** ** b) Otherwise, if there is no MATCH: ** -** * No other constraints: cost=1000000.0 -** * One rowid range constraint: cost=750000.0 -** * Both rowid range constraints: cost=250000.0 -** * An == rowid constraint: cost=10.0 +** * No other constraints: cost=3000000.0 +** * One rowid range constraints: cost=2250000.0 +** * Both rowid range constraint: cost=750000.0 +** * An == rowid constraint: cost=25.0 ** ** Costs are not modified by the ORDER BY clause. +** +** The ratios used in case (a) are based on informal results obtained from +** the tool/fts5cost.tcl script. The "MATCH and ==" combination has the +** cost set quite high because the query may be a prefix query. Unless +** there is a prefix index, prefix queries with rowid constraints are much +** more expensive than non-prefix queries with rowid constraints. +** +** The estimated rows returned is set to the cost/40. For simple queries, +** experimental results show that cost/4 might be about right. But for +** more complex queries that use multiple terms the number of rows might +** be far fewer than this. So we compromise and use cost/40. */ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Table *pTab = (Fts5Table*)pVTab; @@ -254349,7 +260274,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_ERROR; } - idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + idxStr = (char*)sqlite3_malloc64((i64)pInfo->nConstraint * 8 + 1); if( idxStr==0 ) return SQLITE_NOMEM; pInfo->idxStr = idxStr; pInfo->needToFreeIdxStr = 1; @@ -254375,7 +260300,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ nSeenMatch++; idxStr[iIdxStr++] = 'M'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); - idxStr += strlen(&idxStr[iIdxStr]); + iIdxStr += (int)strlen(&idxStr[iIdxStr]); assert( idxStr[iIdxStr]=='\0' ); } pInfo->aConstraintUsage[i].argvIndex = ++iCons; @@ -254394,6 +260319,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ idxStr[iIdxStr++] = '='; bSeenEq = 1; pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; } } } @@ -254441,17 +260367,35 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0; - if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo); - }else if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0; - }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0; + pInfo->estimatedCost = nSeenMatch ? 25000.0 : 25.0; + fts5SetEstimatedRows(pInfo, 1); + fts5SetUniqueFlag(pInfo); }else{ - pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0; - } - for(i=1; i<nSeenMatch; i++){ - pInfo->estimatedCost *= 0.4; + i64 nEstRows; + if( nSeenMatch ){ + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = 50000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = 37500.0; + }else{ + pInfo->estimatedCost = 50000.0; + } + nEstRows = (i64)(pInfo->estimatedCost / 40.0); + for(i=1; i<nSeenMatch; i++){ + pInfo->estimatedCost *= 2.5; + nEstRows = nEstRows / 2; + } + }else{ + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = 750000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = 2250000.0; + }else{ + pInfo->estimatedCost = 3000000.0; + } + nEstRows = (i64)(pInfo->estimatedCost / 4.0); + } + fts5SetEstimatedRows(pInfo, nEstRows); } pInfo->idxNum = idxFlags; @@ -254650,7 +260594,9 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ int bDesc = pCsr->bDesc; i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc + ); if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ *pbSkip = 1; } @@ -254822,7 +260768,9 @@ static int fts5CursorFirstSorted( static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ int rc; Fts5Expr *pExpr = pCsr->pExpr; - rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc + ); if( sqlite3Fts5ExprEof(pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } @@ -255790,6 +261738,7 @@ static int fts5UpdateMethod( } update_out: + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -257307,7 +263256,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2025-07-30 19:33:53 4d8adfb30e03f9cf27f800a2c1ba3c48fb4ca1b08b0f5ed59a4d5ecbf45e20a3", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2026-05-05 10:34:17 c88b22011a54b4f6fbd149e9f8e4de77658ce58143a1af0e3785e4e6475127e9", -1, SQLITE_TRANSIENT); } /* @@ -257330,9 +263279,9 @@ static void fts5LocaleFunc( sqlite3_value **apArg /* Function arguments */ ){ const char *zLocale = 0; - int nLocale = 0; + i64 nLocale = 0; const char *zText = 0; - int nText = 0; + i64 nText = 0; assert( nArg==2 ); UNUSED_PARAM(nArg); @@ -257349,10 +263298,10 @@ static void fts5LocaleFunc( Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); u8 *pBlob = 0; u8 *pCsr = 0; - int nBlob = 0; + i64 nBlob = 0; nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; - pBlob = (u8*)sqlite3_malloc(nBlob); + pBlob = (u8*)sqlite3_malloc64(nBlob); if( pBlob==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -257430,8 +263379,9 @@ static int fts5IntegrityMethod( " FTS5 table %s.%s: %s", zSchema, zTabname, sqlite3_errstr(rc)); } + }else if( (rc&0xff)==SQLITE_CORRUPT ){ + rc = SQLITE_OK; } - sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; @@ -257470,7 +263420,7 @@ static int fts5Init(sqlite3 *db){ int rc; Fts5Global *pGlobal = 0; - pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); + pGlobal = (Fts5Global*)sqlite3_malloc64(sizeof(Fts5Global)); if( pGlobal==0 ){ rc = SQLITE_NOMEM; }else{ @@ -259186,7 +265136,7 @@ static int fts5AsciiCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = sqlite3_malloc(sizeof(AsciiTokenizer)); + p = sqlite3_malloc64(sizeof(AsciiTokenizer)); if( p==0 ){ rc = SQLITE_NOMEM; }else{ @@ -259481,7 +265431,7 @@ static int fts5UnicodeCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + p = (Unicode61Tokenizer*)sqlite3_malloc64(sizeof(Unicode61Tokenizer)); if( p ){ const char *zCat = "L* N* Co"; int i; @@ -259704,7 +265654,7 @@ static int fts5PorterCreate( zBase = azArg[0]; } - pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + pRet = (PorterTokenizer*)sqlite3_malloc64(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); @@ -260411,7 +266361,7 @@ static int fts5TriCreate( rc = SQLITE_ERROR; }else{ int i; - pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + pNew = (TrigramTokenizer*)sqlite3_malloc64(sizeof(*pNew)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -262127,7 +268077,12 @@ static int fts5VocabOpenMethod( return rc; } +/* +** Restore cursor pCsr to the state it was in immediately after being +** created by the xOpen() method. +*/ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + int nCol = pCsr->pFts5->pConfig->nCol; pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); sqlite3Fts5StructureRelease(pCsr->pStruct); @@ -262137,6 +268092,12 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->nLeTerm = -1; pCsr->zLeTerm = 0; pCsr->bEof = 0; + pCsr->iCol = 0; + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + pCsr->colUsed = 0; + memset(pCsr->aCnt, 0, sizeof(i64)*nCol); + memset(pCsr->aDoc, 0, sizeof(i64)*nCol); } /* @@ -262386,7 +268347,7 @@ static int fts5VocabFilterMethod( const char *zCopy = (const char *)sqlite3_value_text(pLe); if( zCopy==0 ) zCopy = ""; pCsr->nLeTerm = sqlite3_value_bytes(pLe); - pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + pCsr->zLeTerm = sqlite3_malloc64((i64)pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/contrib/sqlite3/sqlite3.h b/contrib/sqlite3/sqlite3.h index c2ed750305b2..8ee26c99d86e 100644 --- a/contrib/sqlite3/sqlite3.h +++ b/contrib/sqlite3/sqlite3.h @@ -146,9 +146,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.50.4" -#define SQLITE_VERSION_NUMBER 3050004 -#define SQLITE_SOURCE_ID "2025-07-30 19:33:53 4d8adfb30e03f9cf27f800a2c1ba3c48fb4ca1b08b0f5ed59a4d5ecbf45e20a3" +#define SQLITE_VERSION "3.53.1" +#define SQLITE_VERSION_NUMBER 3053001 +#define SQLITE_SOURCE_ID "2026-05-05 10:34:17 c88b22011a54b4f6fbd149e9f8e4de77658ce58143a1af0e3785e4e6475127e9" +#define SQLITE_SCM_BRANCH "branch-3.53" +#define SQLITE_SCM_TAGS "release version-3.53.1" +#define SQLITE_SCM_DATETIME "2026-05-05T10:34:17.344Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -168,9 +171,9 @@ extern "C" { ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 ); ** </pre></blockquote>)^ ** -** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] -** macro. ^The sqlite3_libversion() function returns a pointer to the -** to the sqlite3_version[] string constant. The sqlite3_libversion() +** ^The sqlite3_version[] string constant contains the text of the +** [SQLITE_VERSION] macro. ^The sqlite3_libversion() function returns a +** pointer to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to @@ -370,7 +373,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, -** semicolon-separate SQL statements passed into its 2nd argument, +** semicolon-separated SQL statements passed into its 2nd argument, ** in the context of the [database connection] passed in as its 1st ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row @@ -403,7 +406,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** result row is NULL then the corresponding string pointer for the ** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the ** sqlite3_exec() callback is an array of pointers to strings where each -** entry represents the name of corresponding result column as obtained +** entry represents the name of a corresponding result column as obtained ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer @@ -497,6 +500,9 @@ SQLITE_API int sqlite3_exec( #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_ERROR_RESERVESIZE (SQLITE_ERROR | (4<<8)) +#define SQLITE_ERROR_KEY (SQLITE_ERROR | (5<<8)) +#define SQLITE_ERROR_UNABLE (SQLITE_ERROR | (6<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -531,6 +537,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) #define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_IOERR_BADKEY (SQLITE_IOERR | (35<<8)) +#define SQLITE_IOERR_CODEC (SQLITE_IOERR | (36<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -570,7 +578,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal only */ /* ** CAPI3REF: Flags For File Open Operations @@ -589,7 +597,7 @@ SQLITE_API int sqlite3_exec( ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into ** [sqlite3_open_v2()] does *not* cause the underlying database file ** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into -** [sqlite3_open_v2()] has historically be a no-op and might become an +** [sqlite3_open_v2()] has historically been a no-op and might become an ** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ @@ -683,7 +691,7 @@ SQLITE_API int sqlite3_exec( ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods ** of an [sqlite3_io_methods] object. These values are ordered from -** lest restrictive to most restrictive. +** least restrictive to most restrictive. ** ** The argument to xLock() is always SHARED or higher. The argument to ** xUnlock is either SHARED or NONE. @@ -924,7 +932,7 @@ struct sqlite3_io_methods { ** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] -** No longer in use. +** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used. ** ** <li>[[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and @@ -999,7 +1007,7 @@ struct sqlite3_io_methods { ** ** <li>[[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of -** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** all [VFSes] in the VFS stack. The names of all VFS shims and the ** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. @@ -1013,7 +1021,7 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level ** [VFSes] currently in use. ^(The argument X in ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X +** of type "[sqlite3_vfs] **". This opcode will set *X ** to a pointer to the top-level VFS.)^ ** ^When there are multiple VFS shims in the stack, this opcode finds the ** upper-most shim only. @@ -1203,7 +1211,7 @@ struct sqlite3_io_methods { ** <li>[[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect ** whether or not there is a database client in another process with a wal-mode -** transaction open on the database or not. It is only available on unix.The +** transaction open on the database or not. It is only available on unix. The ** (void*) argument passed with this file-control should be a pointer to a ** value of type (int). The integer value is set to 1 if the database is a wal ** mode database and there exists at least one client in another process that @@ -1221,6 +1229,15 @@ struct sqlite3_io_methods { ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control ** purges the contents of the in-memory page cache. If there is an open ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. +** +** <li>[[SQLITE_FCNTL_FILESTAT]] +** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information +** about the [sqlite3_file] objects used access the database and journal files +** for the given schema. The fourth parameter to [sqlite3_file_control()] +** should be an initialized [sqlite3_str] pointer. JSON text describing +** various aspects of the sqlite3_file object is appended to the sqlite3_str. +** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time +** options are used to enable it. ** </ul> */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1266,12 +1283,19 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_RESET_CACHE 42 #define SQLITE_FCNTL_NULL_IO 43 #define SQLITE_FCNTL_BLOCK_ON_CONNECT 44 +#define SQLITE_FCNTL_FILESTAT 45 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO +/* reserved file-control numbers: +** 101 +** 102 +** 103 +*/ + /* ** CAPI3REF: Mutex Handle @@ -1472,7 +1496,7 @@ typedef const char *sqlite3_filename; ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. ** -** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** ^The xSetSystemCall(), xGetSystemCall(), and xNextSystemCall() interfaces ** are not used by the SQLite core. These optional interfaces are provided ** by some VFSes to facilitate testing of the VFS code. By overriding ** system calls with functions under its control, a test program can @@ -1628,7 +1652,7 @@ struct sqlite3_vfs { ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically -** initialized when [sqlite3_open()] is called if it has not be initialized +** initialized when [sqlite3_open()] is called if it has not been initialized ** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly @@ -1693,7 +1717,8 @@ SQLITE_API int sqlite3_os_end(void); ** are called "anytime configuration options". ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before ** [sqlite3_shutdown()] with a first argument that is not an anytime -** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** configuration option, then the sqlite3_config() call will +** return SQLITE_MISUSE. ** Note, however, that ^sqlite3_config() can be called as part of the ** implementation of an application-defined [sqlite3_os_init()]. ** @@ -1885,21 +1910,21 @@ struct sqlite3_mem_methods { ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation -** routines with a wrapper that simulations memory allocation failure or +** routines with a wrapper that simulates memory allocation failure or ** tracks memory usage, for example. </dd> ** ** [[SQLITE_CONFIG_SMALL_MALLOC]] <dt>SQLITE_CONFIG_SMALL_MALLOC</dt> -** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of ** type int, interpreted as a boolean, which if true provides a hint to ** SQLite that it should avoid large memory allocations if possible. ** SQLite will run faster if it is free to make large memory allocations, -** but some application might prefer to run slower in exchange for +** but some applications might prefer to run slower in exchange for ** guarantees about memory fragmentation that are possible if large ** allocations are avoided. This hint is normally off. ** </dd> ** ** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt> -** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: @@ -1944,7 +1969,7 @@ struct sqlite3_mem_methods { ** ^If pMem is NULL and N is non-zero, then each database connection ** does an initial bulk allocation for page cache memory ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional +** of -1024*N bytes if N is negative. ^If additional ** page cache memory is needed beyond what is provided by the initial ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each ** additional cache line. </dd> @@ -1973,7 +1998,7 @@ struct sqlite3_mem_methods { ** <dd> ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. ** The argument specifies alternative low-level mutex routines to be used -** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** in place of the mutex routines built into SQLite.)^ ^SQLite makes a copy of ** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then @@ -2015,7 +2040,7 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]] <dt>SQLITE_CONFIG_GETPCACHE2</dt> ** <dd> ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies off ** the current page cache implementation into that object.)^ </dd> ** ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt> @@ -2032,7 +2057,7 @@ struct sqlite3_mem_methods { ** the logger function is a copy of the first parameter to the corresponding ** [sqlite3_log()] call and is intended to be a [result code] or an ** [extended result code]. ^The third parameter passed to the logger is -** log message after formatting via [sqlite3_snprintf()]. +** a log message after formatting via [sqlite3_snprintf()]. ** The SQLite logging interface is not reentrant; the logger function ** supplied by the application must not invoke any SQLite interface. ** In a multi-threaded application, the application-defined logger @@ -2223,7 +2248,7 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the second parameter to the [sqlite3_db_config()] interface. ** -** The [sqlite3_db_config()] interface is a var-args functions. It takes a +** The [sqlite3_db_config()] interface is a var-args function. It takes a ** variable number of parameters, though always at least two. The number of ** parameters passed into sqlite3_db_config() depends on which of these ** constants is given as the second parameter. This documentation page @@ -2259,9 +2284,10 @@ struct sqlite3_mem_methods { ** is less than 8. The "sz" argument should be a multiple of 8 less than ** 65536. If "sz" does not meet this constraint, it is reduced in size until ** it does. -** <li><p>The third argument ("cnt") is the number of slots. Lookaside is disabled -** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so -** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +** <li><p>The third argument ("cnt") is the number of slots. +** Lookaside is disabled if "cnt"is less than 1. +* The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" ** parameter is usually chosen so that the product of "sz" and "cnt" is less ** than 1,000,000. ** </ol> @@ -2335,17 +2361,20 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt> -** <dd> ^This option is used to enable or disable the -** [fts3_tokenizer()] function which is part of the -** [FTS3] full-text search engine extension. -** There must be two additional arguments. -** The first argument is an integer which is 0 to disable fts3_tokenizer() or -** positive to enable fts3_tokenizer() or negative to leave the setting -** unchanged. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled -** following this call. The second parameter may be a NULL pointer, in -** which case the new setting is not reported back. </dd> +** <dd> ^This option is used to enable or disable using the +** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine +** extension - without using bound parameters as the parameters. Doing so +** is disabled by default. There must be two additional arguments. The first +** argument is an integer. If it is passed 0, then using fts3_tokenizer() +** without bound parameters is disabled. If it is passed a positive value, +** then calling fts3_tokenizer without bound parameters is enabled. If it +** is passed a negative value, this setting is not modified - this can be +** used to query for the current setting. The second parameter is a pointer +** to an integer into which is written 0 or 1 to indicate the current value +** of this setting (after it is modified, if applicable). The second +** parameter may be a NULL pointer, in which case the value of the setting +** is not reported back. Refer to [FTS3] documentation for further details. +** </dd> ** ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] ** <dt>SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION</dt> @@ -2357,8 +2386,8 @@ struct sqlite3_mem_methods { ** When the first argument to this interface is 1, then only the C-API is ** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. +** If the first argument is -1, then no changes are made to the state of either +** the C-API or the SQL function. ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface ** is disabled or enabled following this call. The second parameter may @@ -2476,7 +2505,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] ** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates -** the legacy behavior of the [ALTER TABLE RENAME] command such it +** the legacy behavior of the [ALTER TABLE RENAME] command such that it ** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ** additional information. This feature can also be turned on and off @@ -2525,7 +2554,7 @@ struct sqlite3_mem_methods { ** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt> ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly -** created database file to have a schema format version number (the 4-byte +** created database files to have a schema format version number (the 4-byte ** integer found at offset 44 into the database header) of 1. This in turn ** means that the resulting database file will be readable and writable by ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, @@ -2546,13 +2575,16 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] ** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt> ** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in -** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears -** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() -** statistics. For statistics to be collected, the flag must be set on -** the database handle both when the SQL statement is prepared and when it -** is stepped. The flag is set (collection of statistics is enabled) -** by default. <p>This option takes two arguments: an integer and a pointer to -** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** [SQLITE_ENABLE_STMT_SCANSTATUS] builds. In this case, it sets or clears +** a flag that enables collection of run-time performance statistics +** used by [sqlite3_stmt_scanstatus_v2()] and the [nexec and ncycle] +** columns of the [bytecode virtual table]. +** For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is +** [sqlite3_prepare|prepared] and when it is [sqlite3_step|stepped]. +** The flag is set (collection of statistics is enabled) by default. +** <p>This option takes two arguments: an integer and a pointer to +** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after ** processing the first argument is written into the integer that the second @@ -2595,8 +2627,8 @@ struct sqlite3_mem_methods { ** <dd>The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the ** ability of the [ATTACH DATABASE] SQL command to open a database for writing. ** This capability is enabled by default. Applications can disable or -** reenable this capability using the current DBCONFIG option. If the -** the this capability is disabled, the [ATTACH] command will still work, +** reenable this capability using the current DBCONFIG option. If +** this capability is disabled, the [ATTACH] command will still work, ** but the database will be opened read-only. If this option is disabled, ** then the ability to create a new database using [ATTACH] is also disabled, ** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE] @@ -2624,16 +2656,34 @@ struct sqlite3_mem_methods { ** comments are allowed in SQL text after processing the first argument. ** </dd> ** +** [[SQLITE_DBCONFIG_FP_DIGITS]] +** <dt>SQLITE_DBCONFIG_FP_DIGITS</dt> +** <dd>The SQLITE_DBCONFIG_FP_DIGITS setting is a small integer that determines +** the number of significant digits that SQLite will attempt to preserve when +** converting floating point numbers (IEEE 754 "doubles") into text. The +** default value 17, as of SQLite version 3.52.0. The value was 15 in all +** prior versions.<p> +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is a small integer, between 3 and 23, or +** zero. The FP_DIGITS setting is changed to that small integer, or left +** unaltered if the first argument is zero or out of range. The second argument +** is a pointer to an integer. If the pointer is not NULL, then the value of +** the FP_DIGITS setting, after possibly being modified by the first +** arguments, is written into the integer to which the second argument points. +** </dd> +** ** </dl> ** ** [[DBCONFIG arguments]] <h3>Arguments To SQLITE_DBCONFIG Options</h3> ** ** <p>Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is a integer. -** The second argument is a pointer to an integer. If the first argument is 1, -** then the option becomes enabled. If the first integer argument is 0, then the -** option is disabled. If the first argument is -1, then the option setting +** The first argument (the third parameter to sqlite3_db_config()) is +** an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, +** then the option is disabled. +** If the first argument is -1, then the option setting ** is unchanged. The second argument, the pointer to an integer, may be NULL. ** If the second argument is not NULL, then a value of 0 or 1 is written into ** the integer to which the second argument points, depending on whether the @@ -2641,9 +2691,10 @@ struct sqlite3_mem_methods { ** the first argument. ** ** <p>While most SQLITE_DBCONFIG options use the argument format -** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] -** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the -** documentation of those exceptional options for details. +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME], +** [SQLITE_DBCONFIG_LOOKASIDE], and [SQLITE_DBCONFIG_FP_DIGITS] options +** are different. See the documentation of those exceptional options for +** details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2668,7 +2719,8 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_FP_DIGITS 1023 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1023 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2920,7 +2972,7 @@ SQLITE_API int sqlite3_is_interrupted(sqlite3*); ** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** ^These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements and thus ** will not detect syntactically incorrect SQL. ** ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior @@ -3037,7 +3089,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** indefinitely if possible. The results of passing any other negative value ** are undefined. ** -** Internally, each SQLite database handle store two timeout values - the +** Internally, each SQLite database handle stores two timeout values - the ** busy-timeout (used for rollback mode databases, or if the VFS does not ** support blocking locks) and the setlk-timeout (used for blocking locks ** on wal-mode databases). The sqlite3_busy_timeout() method sets both @@ -3067,7 +3119,7 @@ SQLITE_API int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. ** -** Definition: A <b>result table</b> is memory data structure created by the +** Definition: A <b>result table</b> is a memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -3210,7 +3262,7 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is -** a no-op if is called with a NULL pointer. Passing a NULL pointer +** a no-op if it is called with a NULL pointer. Passing a NULL pointer ** to sqlite3_free() is harmless. After being freed, memory ** should neither be read nor written. Even reading previously freed ** memory might result in a segmentation fault or other severe error. @@ -3228,13 +3280,13 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_free(X). ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation ** of at least N bytes in size or NULL if insufficient memory is available. -** ^If M is the size of the prior allocation, then min(N,M) bytes -** of the prior allocation are copied into the beginning of buffer returned +** ^If M is the size of the prior allocation, then min(N,M) bytes of the +** prior allocation are copied into the beginning of the buffer returned ** by sqlite3_realloc(X,N) and the prior allocation is freed. ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the ** prior allocation is not freed. ** -** ^The sqlite3_realloc64(X,N) interfaces works the same as +** ^The sqlite3_realloc64(X,N) interface works the same as ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead ** of a 32-bit signed integer. ** @@ -3284,7 +3336,7 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** was last reset. ^The values returned by [sqlite3_memory_used()] and ** [sqlite3_memory_highwater()] include any overhead ** added by SQLite in its implementation of [sqlite3_malloc()], -** but not overhead added by the any underlying system library +** but not overhead added by any underlying system library ** routines that [sqlite3_malloc()] may call. ** ** ^The memory high-water mark is reset to the current value of @@ -3736,7 +3788,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** there is no harm in trying.) ** ** ^(<dt>[SQLITE_OPEN_SHAREDCACHE]</dt> -** <dd>The database is opened [shared cache] enabled, overriding +** <dd>The database is opened with [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** The [use of shared cache mode is discouraged] and hence shared cache @@ -3744,7 +3796,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** this option is a no-op. ** ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt> -** <dd>The database is opened [shared cache] disabled, overriding +** <dd>The database is opened with [shared cache] disabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** @@ -4150,6 +4202,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** <li> sqlite3_errmsg() ** <li> sqlite3_errmsg16() ** <li> sqlite3_error_offset() +** <li> sqlite3_db_handle() ** </ul> ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language @@ -4162,7 +4215,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** subsequent calls to other SQLite interface functions.)^ ** ** ^The sqlite3_errstr(E) interface returns the English-language text -** that describes the [result code] E, as UTF-8, or NULL if E is not an +** that describes the [result code] E, as UTF-8, or NULL if E is not a ** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. @@ -4170,7 +4223,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** ^If the most recent error references a specific token in the input ** SQL, the sqlite3_error_offset() interface returns the byte offset ** of the start of that token. ^The byte offset returned by -** sqlite3_error_offset() assumes that the input SQL is UTF8. +** sqlite3_error_offset() assumes that the input SQL is UTF-8. ** ^If the most recent error does not reference a specific token in the input ** SQL, then the sqlite3_error_offset() function returns -1. ** @@ -4196,6 +4249,34 @@ SQLITE_API const char *sqlite3_errstr(int); SQLITE_API int sqlite3_error_offset(sqlite3 *db); /* +** CAPI3REF: Set Error Code And Message +** METHOD: sqlite3 +** +** Set the error code of the database handle passed as the first argument +** to errcode, and the error message to a copy of nul-terminated string +** zErrMsg. If zErrMsg is passed NULL, then the error message is set to +** the default message associated with the supplied error code. Subsequent +** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will +** return the values set by this routine in place of what was previously +** set by SQLite itself. +** +** This function returns SQLITE_OK if the error code and error message are +** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if +** the database handle is NULL or invalid. +** +** The error code and message set by this routine remains in effect until +** they are changed, either by another call to this routine or until they are +** changed to by SQLite itself to reflect the result of some subsquent +** API call. +** +** This function is intended for use by SQLite extensions or wrappers. The +** idea is that an extension or wrapper can use this routine to set error +** messages and error codes and thus behave more like a core SQLite +** feature from the point of view of an application. +*/ +SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg); + +/* ** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} ** @@ -4269,8 +4350,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** These constants define various performance limits ** that can be lowered at run-time using [sqlite3_limit()]. -** The synopsis of the meanings of the various limits is shown below. -** Additional information is available at [limits | Limits in SQLite]. +** A concise description of these limits follows, and additional information +** is available at [limits | Limits in SQLite]. ** ** <dl> ** [[SQLITE_LIMIT_LENGTH]] ^(<dt>SQLITE_LIMIT_LENGTH</dt> @@ -4287,6 +4368,10 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt> ** <dd>The maximum depth of the parse tree on any expression.</dd>)^ ** +** [[SQLITE_LIMIT_PARSER_DEPTH]] ^(<dt>SQLITE_LIMIT_PARSER_DEPTH</dt> +** <dd>The maximum depth of the LALR(1) parser stack used to analyze +** input SQL statements.</dd>)^ +** ** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt> ** <dd>The maximum number of terms in a compound SELECT statement.</dd>)^ ** @@ -4331,11 +4416,12 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 +#define SQLITE_LIMIT_PARSER_DEPTH 12 /* ** CAPI3REF: Prepare Flags ** -** These constants define various flags that can be passed into +** These constants define various flags that can be passed into the ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and ** [sqlite3_prepare16_v3()] interfaces. ** @@ -4375,12 +4461,29 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** fails, the sqlite3_prepare_v3() call returns the same error indications ** with or without this flag; it just omits the call to [sqlite3_log()] that ** logs the error. +** +** [[SQLITE_PREPARE_FROM_DDL]] <dt>SQLITE_PREPARE_FROM_DDL</dt> +** <dd>The SQLITE_PREPARE_FROM_DDL flag causes the SQL compiler to enforce +** security constraints that would otherwise only be enforced when parsing +** the database schema. In other words, the SQLITE_PREPARE_FROM_DDL flag +** causes the SQL compiler to treat the SQL statement being prepared as if +** it had come from an attacker. When SQLITE_PREPARE_FROM_DDL is used and +** [SQLITE_DBCONFIG_TRUSTED_SCHEMA] is off, SQL functions may only be called +** if they are tagged with [SQLITE_INNOCUOUS] and virtual tables may only +** be used if they are tagged with [SQLITE_VTAB_INNOCUOUS]. Best practice +** is to use the SQLITE_PREPARE_FROM_DDL option when preparing any SQL that +** is derived from parts of the database schema. In particular, virtual +** table implementations that run SQL statements that are derived from +** arguments to their CREATE VIRTUAL TABLE statement should always use +** [sqlite3_prepare_v3()] and set the SQLITE_PREPARE_FROM_DDL flag to +** prevent bypass of the [SQLITE_DBCONFIG_TRUSTED_SCHEMA] security checks. ** </dl> */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 #define SQLITE_PREPARE_DONT_LOG 0x10 +#define SQLITE_PREPARE_FROM_DDL 0x20 /* ** CAPI3REF: Compiling An SQL Statement @@ -4394,8 +4497,9 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. -** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used -** for special purposes. +** [sqlite3_prepare_v3()] has an extra +** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is sometimes +** needed for special purpose or to pass along security restrictions. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently ** does all parsing using UTF-8. The UTF-16 interfaces are provided @@ -4422,7 +4526,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** there is a small performance advantage to passing an nByte parameter that ** is the number of bytes in the input string <i>including</i> ** the nul-terminator. -** Note that nByte measure the length of the input in bytes, not +** Note that nByte measures the length of the input in bytes, not ** characters, even for the UTF-16 interfaces. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte @@ -4556,7 +4660,7 @@ SQLITE_API int sqlite3_prepare16_v3( ** ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory ** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** maximum string length determined by the [SQLITE_LIMIT_LENGTH]. ** ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time @@ -4744,7 +4848,7 @@ typedef struct sqlite3_value sqlite3_value; ** ** The context in which an SQL function executes is stored in an ** sqlite3_context object. ^A pointer to an sqlite3_context object -** is always first parameter to [application-defined SQL functions]. +** is always the first parameter to [application-defined SQL functions]. ** The application-defined SQL function implementation will pass this ** pointer through into calls to [sqlite3_result_int | sqlite3_result()], ** [sqlite3_aggregate_context()], [sqlite3_user_data()], @@ -4800,8 +4904,8 @@ typedef struct sqlite3_context sqlite3_context; ** it should be a pointer to well-formed UTF16 text. ** ^If the third parameter to sqlite3_bind_text64() is not NULL, then ** it should be a pointer to a well-formed unicode string that is -** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 -** otherwise. +** either UTF8 if the sixth parameter is SQLITE_UTF8 or SQLITE_UTF8_ZT, +** or UTF16 otherwise. ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) @@ -4847,10 +4951,15 @@ typedef struct sqlite3_context sqlite3_context; ** object and pointer to it must remain valid until then. ^SQLite will then ** manage the lifetime of its private copy. ** -** ^The sixth argument to sqlite3_bind_text64() must be one of -** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] -** to specify the encoding of the text in the third parameter. If -** the sixth argument to sqlite3_bind_text64() is not one of the +** ^The sixth argument (the E argument) +** to sqlite3_bind_text64(S,K,Z,N,D,E) must be one of +** [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE] to specify the encoding of the text in the +** third parameter, Z. The special value [SQLITE_UTF8_ZT] means that the +** string argument is both UTF-8 encoded and is zero-terminated. In other +** words, SQLITE_UTF8_ZT means that the Z array is allocated to hold at +** least N+1 bytes and that the Z[N] byte is zero. If +** the E argument to sqlite3_bind_text64(S,K,Z,N,D,E) is not one of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. @@ -4868,9 +4977,11 @@ typedef struct sqlite3_context sqlite3_context; ** associated with the pointer P of type T. ^D is either a NULL pointer or ** a pointer to a destructor function for P. ^SQLite will invoke the ** destructor D with a single argument of P when it is finished using -** P. The T parameter should be a static string, preferably a string -** literal. The sqlite3_bind_pointer() routine is part of the -** [pointer passing interface] added for SQLite 3.20.0. +** P, even if the call to sqlite3_bind_pointer() fails. Due to a +** historical design quirk, results are undefined if D is +** SQLITE_TRANSIENT. The T parameter should be a static string, +** preferably a string literal. The sqlite3_bind_pointer() routine is +** part of the [pointer passing interface] added for SQLite 3.20.0. ** ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which @@ -5481,7 +5592,7 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors -** or if the statement is never been evaluated, then sqlite3_finalize() returns +** or if the statement has never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or ** [extended error code]. @@ -5713,8 +5824,54 @@ SQLITE_API int sqlite3_create_window_function( /* ** CAPI3REF: Text Encodings ** -** These constant define integer codes that represent the various +** These constants define integer codes that represent the various ** text encodings supported by SQLite. +** +** <dl> +** [[SQLITE_UTF8]] <dt>SQLITE_UTF8</dt><dd>Text is encoding as UTF-8</dd> +** +** [[SQLITE_UTF16LE]] <dt>SQLITE_UTF16LE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "little endian" - the least significant +** byte first. This is the usual encoding, for example on Windows.</dd> +** +** [[SQLITE_UTF16BE]] <dt>SQLITE_UTF16BE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "big endian" - the most significant +** byte first. This encoding is less common, but is still sometimes seen, +** specially on older systems. +** +** [[SQLITE_UTF16]] <dt>SQLITE_UTF16</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed either little endian or as big +** endian, according to the native endianness of the host computer. +** +** [[SQLITE_ANY]] <dt>SQLITE_ANY</dt><dd>This encoding value may only be used +** to declare the preferred text for [application-defined SQL functions] +** created using [sqlite3_create_function()] and similar. If the preferred +** encoding (the 4th parameter to sqlite3_create_function() - the eTextRep +** parameter) is SQLITE_ANY, that indicates that the function does not have +** a preference regarding the text encoding of its parameters and can take +** any text encoding that the SQLite core find convenient to supply. This +** option is deprecated. Please do not use it in new applications. +** +** [[SQLITE_UTF16_ALIGNED]] <dt>SQLITE_UTF16_ALIGNED</dt><dd>This encoding +** value may be used as the 3rd parameter (the eTextRep parameter) to +** [sqlite3_create_collation()] and similar. This encoding value means +** that the application-defined collating sequence created expects its +** input strings to be in UTF16 in native byte order, and that the start +** of the strings must be aligned to a 2-byte boundary. +** +** [[SQLITE_UTF8_ZT]] <dt>SQLITE_UTF8_ZT</dt><dd>This option can only be +** used to specify the text encoding to strings input to +** [sqlite3_result_text64()] and [sqlite3_bind_text64()]. +** The SQLITE_UTF8_ZT encoding means that the input string (call it "z") +** is UTF-8 encoded and that it is zero-terminated. If the length parameter +** (call it "n") is non-negative, this encoding option means that the caller +** guarantees that z array contains at least n+1 bytes and that the z[n] +** byte has a value of zero. +** This option gives the same output as SQLITE_UTF8, but can be more efficient +** by avoiding the need to make a copy of the input string, in some cases. +** However, if z is allocated to hold fewer than n+1 bytes or if the +** z[n] byte is not zero, undefined behavior may result. +** </dl> */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ #define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ @@ -5722,6 +5879,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ +#define SQLITE_UTF8_ZT 16 /* Zero-terminated UTF8 */ /* ** CAPI3REF: Function Flags @@ -5805,7 +5963,7 @@ SQLITE_API int sqlite3_create_window_function( ** result. ** Every function that invokes [sqlite3_result_subtype()] should have this ** property. If it does not, then the call to [sqlite3_result_subtype()] -** might become a no-op if the function is used as term in an +** might become a no-op if the function is used as a term in an ** [expression index]. On the other hand, SQL functions that never invoke ** [sqlite3_result_subtype()] should avoid setting this property, as the ** purpose of this property is to disable certain optimizations that are @@ -5932,7 +6090,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if -** and the prior [xColumn] method call that was invoked to extracted +** the prior [xColumn] method call that was invoked to extract ** the value for that column returned without setting a result (probably ** because it queried [sqlite3_vtab_nochange()] and found that the column ** was unchanging). ^Within an [xUpdate] method, any value for which @@ -5956,26 +6114,22 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** the SQL function that supplied the [sqlite3_value*] parameters. ** ** As long as the input parameter is correct, these routines can only -** fail if an out-of-memory error occurs during a format conversion. -** Only the following subset of interfaces are subject to out-of-memory -** errors: -** -** <ul> -** <li> sqlite3_value_blob() -** <li> sqlite3_value_text() -** <li> sqlite3_value_text16() -** <li> sqlite3_value_text16le() -** <li> sqlite3_value_text16be() -** <li> sqlite3_value_bytes() -** <li> sqlite3_value_bytes16() -** </ul> -** +** fail if an out-of-memory error occurs while trying to do a +** UTF8→UTF16 or UTF16→UTF8 conversion. ** If an out-of-memory error occurs, then the return value from these ** routines is the same as if the column had contained an SQL NULL value. -** Valid SQL NULL returns can be distinguished from out-of-memory errors -** by invoking the [sqlite3_errcode()] immediately after the suspect +** If the input sqlite3_value was not obtained from [sqlite3_value_dup()], +** then valid SQL NULL returns can also be distinguished from +** out-of-memory errors after extracting the value +** by invoking the [sqlite3_errcode()] immediately after the suspicious ** return value is obtained and before any ** other SQLite interface is called on the same [database connection]. +** If the input sqlite3_value was obtained from sqlite3_value_dup() then +** it is disconnected from the database connection and so sqlite3_errcode() +** will not work. +** In that case, the only way to distinguish an out-of-memory +** condition from a true SQL NULL is to invoke sqlite3_value_type() on the +** input to see if it is NULL prior to trying to extract the value. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -6002,7 +6156,8 @@ SQLITE_API int sqlite3_value_frombind(sqlite3_value*); ** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) ** returns something other than SQLITE_TEXT, then the return value from ** sqlite3_value_encoding(X) is meaningless. ^Calls to -** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], +** [sqlite3_value_text16be(X)], ** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or ** [sqlite3_value_bytes16(X)] might change the encoding of the value X and ** thus change the return from subsequent calls to sqlite3_value_encoding(X). @@ -6133,17 +6288,17 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** query execution, under some circumstances the associated auxiliary data ** might be preserved. An example of where this might be useful is in a ** regular-expression matching function. The compiled version of the regular -** expression can be stored as auxiliary data associated with the pattern string. -** Then as long as the pattern string remains the same, +** expression can be stored as auxiliary data associated with the pattern +** string. Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data -** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument -** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no auxiliary data -** associated with the function argument, the sqlite3_get_auxdata(C,N) interface -** returns a NULL pointer. +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary +** data associated by the sqlite3_set_auxdata(C,N,P,X) function with the +** Nth argument value to the application-defined function. ^N is zero +** for the left-most function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) +** interface returns a NULL pointer. ** ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the ** N-th argument of the application-defined function. ^Subsequent @@ -6205,6 +6360,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** or a NULL pointer if there were no prior calls to ** sqlite3_set_clientdata() with the same values of D and N. ** Names are compared using strcmp() and are thus case sensitive. +** It returns 0 on success and SQLITE_NOMEM on allocation failure. ** ** If P and X are both non-NULL, then the destructor X is invoked with ** argument P on the first of the following occurrences: @@ -6226,10 +6382,14 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** ** There is no limit (other than available memory) on the number of different ** client data pointers (with different names) that can be attached to a -** single database connection. However, the implementation is optimized -** for the case of having only one or two different client data names. -** Applications and wrapper libraries are discouraged from using more than -** one client data name each. +** single database connection. However, the current implementation stores +** the content on a linked list. Insert and retrieval performance will +** be proportional to the number of entries. The design use case, and +** the use case for which the implementation is optimized, is +** that an application will store only small number of client data names, +** typically just one or two. This interface is not intended to be a +** generalized key/value store for thousands or millions of keys. It +** will work for that, but performance might be disappointing. ** ** There is no way to enumerate the client data pointers ** associated with a database connection. The N parameter can be thought @@ -6337,10 +6497,14 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** ^The sqlite3_result_text64() interface sets the return value of an +** ^The sqlite3_result_text64(C,Z,N,D,E) interface sets the return value of an ** application-defined function to be a text string in an encoding -** specified by the fifth (and last) parameter, which must be one -** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** specified the E parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that +** the result text is both UTF-8 and zero-terminated. In other words, +** SQLITE_UTF8_ZT means that the Z array holds at least N+1 bytes and that +** the Z[N] is zero. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces @@ -6427,7 +6591,7 @@ SQLITE_API void sqlite3_result_int(sqlite3_context*, int); SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); SQLITE_API void sqlite3_result_null(sqlite3_context*); SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char *z, sqlite3_uint64 n, void(*)(void*), unsigned char encoding); SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); @@ -7366,7 +7530,7 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^The sqlite3_load_extension() interface attempts to load an ** [SQLite extension] library contained in the file zFile. If ** the file cannot be loaded directly, attempts are made to load -** with various operating-system specific extensions added. +** with various operating-system specific filename extensions added. ** So for example, if "samplelib" cannot be loaded, then names like ** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might ** be tried also. @@ -7374,10 +7538,10 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where -** X consists of the lower-case equivalent of all ASCII alphabetic -** characters in the filename from the last "/" to the first following -** "." and omitting any initial "lib".)^ +** If that does not work, it tries names of the form "sqlite3_X_init" +** where X consists of the lower-case equivalent of all ASCII alphabetic +** characters or all ASCII alphanumeric characters in the filename from +** the last "/" to the first following "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -7451,7 +7615,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** <blockquote><pre> ** int xEntryPoint( ** sqlite3 *db, -** const char **pzErrMsg, +** char **pzErrMsg, ** const struct sqlite3_api_routines *pThunk ** ); ** </pre></blockquote>)^ @@ -8201,13 +8365,6 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor -** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex -** implementation is included with the library. In this case the -** application must supply a custom mutex implementation using the -** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function -** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() @@ -8562,6 +8719,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_ATOF 34 #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* @@ -8670,17 +8828,22 @@ SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3*); ** pass the returned value to [sqlite3_free()] to avoid a memory leak. ** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any ** errors were encountered during construction of the string. ^The -** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** [sqlite3_str_finish(X)] interface might also return a NULL pointer if the ** string in [sqlite3_str] object X is zero bytes long. +** +** ^The [sqlite3_str_free(X)] interface destroys both the sqlite3_str object +** X and the string content it contains. Calling sqlite3_str_free(X) is +** the equivalent of calling [sqlite3_free](sqlite3_str_finish(X)). */ SQLITE_API char *sqlite3_str_finish(sqlite3_str*); +SQLITE_API void sqlite3_str_free(sqlite3_str*); /* ** CAPI3REF: Add Content To A Dynamic String ** METHOD: sqlite3_str ** -** These interfaces add content to an sqlite3_str object previously obtained -** from [sqlite3_str_new()]. +** These interfaces add or remove content to an sqlite3_str object +** previously obtained from [sqlite3_str_new()]. ** ** ^The [sqlite3_str_appendf(X,F,...)] and ** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] @@ -8703,6 +8866,10 @@ SQLITE_API char *sqlite3_str_finish(sqlite3_str*); ** ^The [sqlite3_str_reset(X)] method resets the string under construction ** inside [sqlite3_str] object X back to zero bytes in length. ** +** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string +** under construction to be N bytes or less. This routine is a no-op if +** N is negative or if the string is already N bytes or smaller in size. +** ** These methods do not return a result code. ^If an error occurs, that fact ** is recorded in the [sqlite3_str] object and can be recovered by a ** subsequent call to [sqlite3_str_errcode(X)]. @@ -8713,6 +8880,7 @@ SQLITE_API void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); SQLITE_API void sqlite3_str_appendall(sqlite3_str*, const char *zIn); SQLITE_API void sqlite3_str_appendchar(sqlite3_str*, int N, char C); SQLITE_API void sqlite3_str_reset(sqlite3_str*); +SQLITE_API void sqlite3_str_truncate(sqlite3_str*,int N); /* ** CAPI3REF: Status Of A Dynamic String @@ -8881,9 +9049,18 @@ SQLITE_API int sqlite3_status64( ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** +** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same +** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H +** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead +** of pointers to 32-bit integers, which allows larger status values +** to be returned. If a status value exceeds 2,147,483,647 then +** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64() +** will return the full value. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); /* ** CAPI3REF: Status Parameters for database connections @@ -8980,6 +9157,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** If an IO or other error occurs while writing a page to disk, the effect ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +** <p> +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL. +** Resetting one will reduce the other.)^ ** </dd> ** ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(<dt>SQLITE_DBSTATUS_CACHE_SPILL</dt> @@ -8995,6 +9176,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** <dd>This parameter returns zero for the current value if and only if ** all foreign key constraints (deferred or immediate) have been ** resolved.)^ ^The highwater mark is always 0. +** +** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(<dt>SQLITE_DBSTATUS_TEMPBUF_SPILL</dt> +** <dd>^(This parameter returns the number of bytes written to temporary +** files on disk that could have been kept in memory had sufficient memory +** been available. This value includes writes to intermediate tables that +** are part of complex queries, external sorts that spill to disk, and +** writes to TEMP tables.)^ +** ^The highwater mark is always 0. +** <p> +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE. +** Resetting one will reduce the other.)^ ** </dd> ** </dl> */ @@ -9011,7 +9204,8 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_DEFERRED_FKS 10 #define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 #define SQLITE_DBSTATUS_CACHE_SPILL 12 -#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_TEMPBUF_SPILL 13 +#define SQLITE_DBSTATUS_MAX 13 /* Largest defined DBSTATUS */ /* @@ -9776,7 +9970,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** is the number of pages currently in the write-ahead log file, ** including those that were just committed. ** -** The callback function should normally return [SQLITE_OK]. ^If an error +** ^The callback function should normally return [SQLITE_OK]. ^If an error ** code is returned, that error will propagate back up through the ** SQLite code base to cause the statement that provoked the callback ** to report an error, though the commit will have still occurred. If the @@ -9784,13 +9978,26 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** that does not correspond to any valid SQLite error code, the results ** are undefined. ** -** A single database handle may have at most a single write-ahead log callback -** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any -** previously registered write-ahead log callback. ^The return value is -** a copy of the third parameter from the previous call, if any, or 0. -** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** overwrite any prior [sqlite3_wal_hook()] settings. +** ^A single database handle may have at most a single write-ahead log +** callback registered at one time. ^Calling [sqlite3_wal_hook()] +** replaces the default behavior or previously registered write-ahead +** log callback. +** +** ^The return value is a copy of the third parameter from the +** previous call, if any, or 0. +** +** ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and +** will overwrite any prior [sqlite3_wal_hook()] settings. +** +** ^If a write-ahead log callback is set using this function then +** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint] +** should be invoked periodically to keep the write-ahead log file +** from growing without bound. +** +** ^Passing a NULL pointer for the callback disables automatic +** checkpointing entirely. To re-enable the default behavior, call +** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint]. */ SQLITE_API void *sqlite3_wal_hook( sqlite3*, @@ -9807,7 +10014,7 @@ SQLITE_API void *sqlite3_wal_hook( ** to automatically [checkpoint] ** after committing a transaction if there are N or ** more frames in the [write-ahead log] file. ^Passing zero or -** a negative value as the nFrame parameter disables automatic +** a negative value as the N parameter disables automatic ** checkpoints entirely. ** ** ^The callback registered by this function replaces any existing callback @@ -9823,9 +10030,10 @@ SQLITE_API void *sqlite3_wal_hook( ** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] -** pages. The use of this interface -** is only necessary if the default setting is found to be suboptimal -** for a particular application. +** pages. +** +** ^The use of this interface is only necessary if the default setting +** is found to be suboptimal for a particular application. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); @@ -9890,6 +10098,11 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the ** addition that it also truncates the log file to zero bytes just prior ** to a successful return. +** +** <dt>SQLITE_CHECKPOINT_NOOP<dd> +** ^This mode always checkpoints zero frames. The only reason to invoke +** a NOOP checkpoint is to access the values returned by +** sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt. ** </dl> ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -9960,6 +10173,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the ** meaning of each of these checkpoint modes. */ +#define SQLITE_CHECKPOINT_NOOP -1 /* Do no work at all */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ #define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ @@ -10197,7 +10411,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** <tr> ** <td valign="top">sqlite3_vtab_distinct() return value ** <td valign="top">Rows are returned in aOrderBy order -** <td valign="top">Rows with the same value in all aOrderBy columns are adjacent +** <td valign="top">Rows with the same value in all aOrderBy columns are +** adjacent ** <td valign="top">Duplicates over all colUsed columns may be omitted ** <tr><td>0<td>yes<td>yes<td>no ** <tr><td>1<td>no<td>yes<td>no @@ -10206,8 +10421,8 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** </table> ** ** ^For the purposes of comparing virtual table output values to see if the -** values are the same value for sorting purposes, two NULL values are considered -** to be the same. In other words, the comparison operator is "IS" +** values are the same value for sorting purposes, two NULL values are +** considered to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** ** If a virtual table implementation is unable to meet the requirements @@ -10328,7 +10543,7 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** ){ ** // do something with pVal ** } -** if( rc!=SQLITE_OK ){ +** if( rc!=SQLITE_DONE ){ ** // an error has occurred ** } ** </pre></blockquote>)^ @@ -10500,9 +10715,9 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** a variable pointed to by the "pOut" parameter. ** ** The "flags" parameter must be passed a mask of flags. At present only -** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** one flag is defined - [SQLITE_SCANSTAT_COMPLEX]. If SQLITE_SCANSTAT_COMPLEX ** is specified, then status information is available for all elements -** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** of a query plan that are reported by "[EXPLAIN QUERY PLAN]" output. If ** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements ** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of ** the EXPLAIN QUERY PLAN output) are available. Invoking API @@ -10516,7 +10731,8 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. ** -** See also: [sqlite3_stmt_scanstatus_reset()] +** See also: [sqlite3_stmt_scanstatus_reset()] and the +** [nexec and ncycle] columns of the [bytecode virtual table]. */ SQLITE_API int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -10787,7 +11003,7 @@ typedef struct sqlite3_snapshot { ** The [sqlite3_snapshot_get()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( +SQLITE_API int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -10836,7 +11052,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** The [sqlite3_snapshot_open()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( +SQLITE_API int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -10853,7 +11069,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( ** The [sqlite3_snapshot_free()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. @@ -10880,7 +11096,7 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( +SQLITE_API int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); @@ -10908,7 +11124,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Serialize a database @@ -10982,12 +11198,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the ** [database connection] D to disconnect from database S and then -** reopen S as an in-memory database based on the serialization contained -** in P. The serialized database P is N bytes in size. M is the size of -** the buffer P, which might be larger than N. If M is larger than N, and -** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is -** permitted to add content to the in-memory database as long as the total -** size does not exceed M bytes. +** reopen S as an in-memory database based on the serialization +** contained in P. If S is a NULL pointer, the main database is +** used. The serialized database P is N bytes in size. M is the size +** of the buffer P, which might be larger than N. If M is larger than +** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then +** SQLite is permitted to add content to the in-memory database as +** long as the total size does not exceed M bytes. ** ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will ** invoke sqlite3_free() on the serialization buffer when the database @@ -11055,6 +11272,77 @@ SQLITE_API int sqlite3_deserialize( #define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ /* +** CAPI3REF: Bind array values to the CARRAY table-valued function +** +** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to +** parameter that is the first argument of the [carray() table-valued function]. +** The S parameter is a pointer to the [prepared statement] that uses the +** carray() functions. I is the parameter index to be bound. I must be the +** index of the parameter that is the first argument to the carray() +** table-valued function. P is a pointer to the array to be bound, and N +** is the number of elements in the array. The F argument is one of +** constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], +** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. +** +** If the X argument is not a NULL pointer or one of the special +** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke +** the function X with argument D when it is finished using the data in P. +** The call to X(D) is a destructor for the array P. The destructor X(D) +** is invoked even if the call to sqlite3_carray_bind_v2() fails. If the X +** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes +** that the data static and the destructor is never invoked. If the X +** parameter is the special-case value [SQLITE_TRANSIENT], then +** sqlite3_carray_bind_v2() makes its own private copy of the data prior +** to returning and never invokes the destructor X. +** +** The sqlite3_carray_bind() function works the same as sqlite3_carray_bind_v2() +** with a D parameter set to P. In other words, +** sqlite3_carray_bind(S,I,P,N,F,X) is same as +** sqlite3_carray_bind_v2(S,I,P,N,F,X,P). +*/ +SQLITE_API int sqlite3_carray_bind_v2( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*), /* Destructor for aData */ + void *pDel /* Optional argument to xDel() */ +); +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructor for aData */ +); + +/* +** CAPI3REF: Datatypes for the CARRAY table-valued function +** +** The fifth argument to the [sqlite3_carray_bind()] interface musts be +** one of the following constants, to specify the datatype of the array +** that is being bound into the [carray table-valued function]. +*/ +#define SQLITE_CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define SQLITE_CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define SQLITE_CARRAY_DOUBLE 2 /* Data is doubles */ +#define SQLITE_CARRAY_TEXT 3 /* Data is char* */ +#define SQLITE_CARRAY_BLOB 4 /* Data is struct iovec */ + +/* +** Versions of the above #defines that omit the initial SQLITE_, for +** legacy compatibility. +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + +/* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. */ @@ -12313,14 +12601,32 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** update the "main" database attached to handle db with the changes found in ** the changeset passed via the second and third arguments. ** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. Additionally, starting with version 3.51.0, +** an error code and error message that may be accessed using the +** [sqlite3_errcode()] and [sqlite3_errmsg()] APIs are left in the database +** handle. +** ** The fourth argument (xFilter) passed to these functions is the "filter -** callback". If it is not NULL, then for each table affected by at least one -** change in the changeset, the filter callback is invoked with -** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument as the first. If the "filter callback" -** returns zero, then no attempt is made to apply any changes to the table. -** Otherwise, if the return value is non-zero or the xFilter argument to -** is NULL, all changes related to the table are attempted. +** callback". This may be passed NULL, in which case all changes in the +** changeset are applied to the database. For sqlite3changeset_apply() and +** sqlite3_changeset_apply_v2(), if it is not NULL, then it is invoked once +** for each table affected by at least one change in the changeset. In this +** case the table name is passed as the second argument, and a copy of +** the context pointer passed as the sixth argument to apply() or apply_v2() +** as the first. If the "filter callback" returns zero, then no attempt is +** made to apply any changes to the table. Otherwise, if the return value is +** non-zero, all changes related to the table are attempted. +** +** For sqlite3_changeset_apply_v3(), the xFilter callback is invoked once +** per change. The second argument in this case is an sqlite3_changeset_iter +** that may be queried using the usual APIs for the details of the current +** change. If the "filter callback" returns zero in this case, then no attempt +** is made to apply the current change. If it returns non-zero, the change +** is applied. ** ** For each table that is not excluded by the filter callback, this function ** tests that the target database contains a compatible table. A table is @@ -12341,11 +12647,11 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** one such warning is issued for each table in the changeset. ** ** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for -** each type of change is below. +** to modify the table contents according to each UPDATE, INSERT or DELETE +** change that is not excluded by a filter callback. If a change cannot be +** applied cleanly, the conflict handler function passed as the fifth argument +** to sqlite3changeset_apply() may be invoked. A description of exactly when +** the conflict handler is invoked for each type of change is below. ** ** Unlike the xFilter argument, xConflict may not be passed NULL. The results ** of passing anything other than a valid function pointer as the xConflict @@ -12441,12 +12747,6 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** This can be used to further customize the application's conflict ** resolution strategy. ** -** All changes made by these functions are enclosed in a savepoint transaction. -** If any other error (aside from a constraint failure when attempting to -** write to the target database) occurs, then the savepoint transaction is -** rolled back, restoring the target database to its original state, and an -** SQLite error code returned. -** ** If the output parameters (ppRebase) and (pnRebase) are non-NULL and ** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() ** may set (*ppRebase) to point to a "rebase" that may be used with the @@ -12496,6 +12796,23 @@ SQLITE_API int sqlite3changeset_apply_v2( void **ppRebase, int *pnRebase, /* OUT: Rebase data */ int flags /* SESSION_CHANGESETAPPLY_* flags */ ); +SQLITE_API int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); /* ** CAPI3REF: Flags for sqlite3changeset_apply_v2 @@ -12915,6 +13232,23 @@ SQLITE_API int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ); +SQLITE_API int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); SQLITE_API int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, @@ -13007,6 +13341,232 @@ SQLITE_API int sqlite3session_config(int op, void *pArg); #define SQLITE_SESSION_CONFIG_STRMSIZE 1 /* +** CAPI3REF: Configure a changegroup object +** +** Configure the changegroup object passed as the first argument. +** At present the only valid value for the second parameter is +** [SQLITE_CHANGEGROUP_CONFIG_PATCHSET]. +*/ +SQLITE_API int sqlite3changegroup_config(sqlite3_changegroup*, int, void *pArg); + +/* +** CAPI3REF: Options for sqlite3changegroup_config(). +** +** The following values may be passed as the 2nd parameter to +** sqlite3changegroup_config(). +** +** <dt>SQLITE_CHANGEGROUP_CONFIG_PATCHSET <dd> +** A changegroup object generates either a changeset or patchset. Usually, +** this is determined by whether the first call to sqlite3changegroup_add() +** is passed a changeset or a patchset. Or, if the first changes are added +** to the changegroup object using the sqlite3changegroup_change_xxx() +** APIs, then this option may be used to configure whether the changegroup +** object generates a changeset or patchset. +** +** When this option is invoked, parameter pArg must point to a value of +** type int. If the changegroup currently contains zero changes, and the +** value of the int variable is zero or greater than zero, then the +** changegroup is configured to generate a changeset or patchset, +** respectively. It is a no-op, not an error, if the changegroup is not +** configured because it has already started accumulating changes. +** +** Before returning, the int variable is set to 0 if the changegroup is +** configured to generate a changeset, or 1 if it is configured to generate +** a patchset. +*/ +#define SQLITE_CHANGEGROUP_CONFIG_PATCHSET 1 + + +/* +** CAPI3REF: Begin adding a change to a changegroup +** +** This API is used, in concert with other sqlite3changegroup_change_xxx() +** APIs, to add changes to a changegroup object one at a time. To add a +** single change, the caller must: +** +** 1. Invoke sqlite3changegroup_change_begin() to indicate the type of +** change (INSERT, UPDATE or DELETE), the affected table and whether +** or not the change should be marked as indirect. +** +** 2. Invoke sqlite3changegroup_change_int64() or one of the other four +** value functions - _null(), _double(), _text() or _blob() - one or +** more times to specify old.* and new.* values for the change being +** constructed. +** +** 3. Invoke sqlite3changegroup_change_finish() to either finish adding +** the change to the group, or to discard the change altogether. +** +** The first argument to this function must be a pointer to the existing +** changegroup object that the change will be added to. The second argument +** must be SQLITE_INSERT, SQLITE_UPDATE or SQLITE_DELETE. The third is the +** name of the table that the change affects, and the fourth is a boolean +** flag specifying whether the change should be marked as "indirect" (if +** bIndirect is non-zero) or not indirect (if bIndirect is zero). +** +** Following a successful call to this function, this function may not be +** called again on the same changegroup object until after +** sqlite3changegroup_change_finish() has been called. Doing so is an +** SQLITE_MISUSE error. +** +** The changegroup object passed as the first argument must be already +** configured with schema data for the specified table. It may be configured +** either by calling sqlite3changegroup_schema() with a database that contains +** the table, or sqlite3changegroup_add() with a changeset that contains the +** table. If the changegroup object has not been configured with a schema for +** the specified table when this function is called, SQLITE_ERROR is returned. +** +** If successful, SQLITE_OK is returned. Otherwise, if an error occurs, an +** SQLite error code is returned. In this case, if argument pzErr is non-NULL, +** then (*pzErr) may be set to point to a buffer containing a utf-8 formated, +** nul-terminated, English language error message. It is the responsibility +** of the caller to eventually free this buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_begin( + sqlite3_changegroup*, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +); + +/* +** CAPI3REF: Add a 64-bit integer to a changegroup +** +** This function may only be called between a successful call to +** sqlite3changegroup_change_begin() and its matching +** sqlite3changegroup_change_finish() call. If it is called at any +** other time, it is an SQLITE_MISUSE error. Calling this function +** specifies a 64-bit integer value to be used in the change currently being +** added to the changegroup object passed as the first argument. +** +** The second parameter, bNew, specifies whether the value is to be part of +** the new.* (if bNew is non-zero) or old.* (if bNew is zero) record of +** the change under construction. If this does not match the type of change +** specified by the preceding call to sqlite3changegroup_change_begin() (i.e. +** an old.* value for an SQLITE_INSERT change, or a new.* value for an +** SQLITE_DELETE), then SQLITE_ERROR is returned. +** +** The third parameter specifies the column of the old.* or new.* record that +** the value will be a part of. If the specified table has an explicit primary +** key, then this is the index of the table column, numbered from 0 in the order +** specified within the CREATE TABLE statement. Or, if the table uses an +** implicit rowid key, then the column 0 is the rowid and the explicit columns +** are numbered starting from 1. If the iCol parameter is less than 0 or greater +** than the index of the last column in the table, SQLITE_RANGE is returned. +** +** The fourth parameter is the integer value to use as part of the old.* or +** new.* record. +** +** If this call is successful, SQLITE_OK is returned. Otherwise, if an +** error occurs, an SQLite error code is returned. +*/ +SQLITE_API int sqlite3changegroup_change_int64( + sqlite3_changegroup*, + int bNew, + int iCol, + sqlite3_int64 iVal +); + +/* +** CAPI3REF: Add a NULL to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently under construction with a NULL value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_null(sqlite3_changegroup*, int, int); + +/* +** CAPI3REF: Add an double to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently being constructed with a real value +** instead of a 64-bit integer. +*/ +SQLITE_API int sqlite3changegroup_change_double(sqlite3_changegroup*, int, int, double); + +/* +** CAPI3REF: Add a text value to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a text value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the text encoded using +** utf-8. Parameter nVal may either be the size of the text value in bytes, or +** else a negative value, in which case the buffer pVal points to is assumed to +** be nul-terminated. +*/ +SQLITE_API int sqlite3changegroup_change_text( + sqlite3_changegroup*, int, int, const char *pVal, int nVal +); + +/* +** CAPI3REF: Add a blob to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a blob value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the blob. Parameter +** nVal is the size of the blob in bytes. +*/ +SQLITE_API int sqlite3changegroup_change_blob( + sqlite3_changegroup*, int, int, const void *pVal, int nVal +); + +/* +** CAPI3REF: Finish adding one-at-at-time changes to a changegroup +** +** This function may only be called following a successful call to +** sqlite3changegroup_change_begin(). Otherwise, it is an SQLITE_MISUSE error. +** +** If parameter bDiscard is non-zero, then the current change is simply +** discarded. In this case this function is always successful and SQLITE_OK +** returned. +** +** If parameter bDiscard is zero, then an attempt is made to add the current +** change to the changegroup. Assuming the changegroup is configured to +** produce a changeset (not a patchset), this requires that: +** +** * If the change is an INSERT or DELETE, then a value must be specified +** for all columns of the new.* or old.* record, respectively. +** +** * If the change is an UPDATE record, then values must be provided for +** the PRIMARY KEY columns of the old.* record, but must not be provided +** for PRIMARY KEY columns of the new.* record. +** +** * If the change is an UPDATE record, then for each non-PRIMARY KEY +** column in the old.* record for which a value has been provided, a +** value must also be provided for the same column in the new.* record. +** Similarly, for each non-PK column in the old.* record for which +** a value is not provided, a value must not be provided for the same +** column in the new.* record. +** +** * All values specified for PRIMARY KEY columns must be non-NULL. +** +** Otherwise, it is an error. +** +** If the changegroup already contains a change for the same row (identified +** by PRIMARY KEY columns), then the current change is combined with the +** existing change in the same way as for sqlite3changegroup_add(). +** +** For a patchset, all of the above rules apply except that it doesn't matter +** whether or not values are provided for the non-PK old.* record columns +** for an UPDATE or DELETE change. This means that code used to produce +** a changeset using the sqlite3changegroup_change_xxx() APIs may also +** be used to produce patchsets. +** +** If the call is successful, SQLITE_OK is returned. Otherwise, if an error +** occurs, an SQLite error code is returned. If an error is returned and +** parameter pzErr is not NULL, then (*pzErr) may be set to point to a buffer +** containing a nul-terminated, utf-8 encoded, English language error message. +** It is the responsibility of the caller to eventually free any such error +** message buffer using sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_change_finish( + sqlite3_changegroup*, + int bDiscard, + char **pzErr +); + +/* ** Make sure we can call this stuff from C++. */ #ifdef __cplusplus diff --git a/contrib/sqlite3/sqlite3ext.h b/contrib/sqlite3/sqlite3ext.h index cf775dfbde0f..cad1a2a00160 100644 --- a/contrib/sqlite3/sqlite3ext.h +++ b/contrib/sqlite3/sqlite3ext.h @@ -368,6 +368,14 @@ struct sqlite3_api_routines { int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); /* Version 3.50.0 and later */ int (*setlk_timeout)(sqlite3*,int,int); + /* Version 3.51.0 and later */ + int (*set_errmsg)(sqlite3*,int,const char*); + int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); + /* Version 3.52.0 and later */ + void (*str_truncate)(sqlite3_str*,int); + void (*str_free)(sqlite3_str*); + int (*carray_bind)(sqlite3_stmt*,int,void*,int,int,void(*)(void*)); + int (*carray_bind_v2)(sqlite3_stmt*,int,void*,int,int,void(*)(void*),void*); }; /* @@ -703,6 +711,14 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_set_clientdata sqlite3_api->set_clientdata /* Version 3.50.0 and later */ #define sqlite3_setlk_timeout sqlite3_api->setlk_timeout +/* Version 3.51.0 and later */ +#define sqlite3_set_errmsg sqlite3_api->set_errmsg +#define sqlite3_db_status64 sqlite3_api->db_status64 +/* Version 3.52.0 and later */ +#define sqlite3_str_truncate sqlite3_api->str_truncate +#define sqlite3_str_free sqlite3_api->str_free +#define sqlite3_carray_bind sqlite3_api->carray_bind +#define sqlite3_carray_bind_v2 sqlite3_api->carray_bind_v2 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/contrib/sqlite3/sqlite3rc.h b/contrib/sqlite3/sqlite3rc.h index 42ece488e353..498fdc5ca84a 100644 --- a/contrib/sqlite3/sqlite3rc.h +++ b/contrib/sqlite3/sqlite3rc.h @@ -1,3 +1,3 @@ #ifndef SQLITE_RESOURCE_VERSION -#define SQLITE_RESOURCE_VERSION 3,50,4 +#define SQLITE_RESOURCE_VERSION 3,53,1 #endif diff --git a/contrib/sqlite3/tea/Makefile.in b/contrib/sqlite3/tea/Makefile.in index 5b2ad4c69992..04c8f87f555b 100644 --- a/contrib/sqlite3/tea/Makefile.in +++ b/contrib/sqlite3/tea/Makefile.in @@ -119,7 +119,7 @@ TCLLIBDIR = @TCLLIBDIR@ # typically come from the ./configure command-line invocation). # CFLAGS.configure = @SH_CFLAGS@ @TEAISH_CFLAGS@ @CFLAGS@ @CPPFLAGS@ $(TCL_INCLUDE_SPEC) -#CFLAGS.configure += -DUSE_TCL_STUBS=1 +CFLAGS.configure += -DUSE_TCL_STUBS=@TEAISH_USE_STUBS@ # # LDFLAGS.configure = LDFLAGS as known at configure-time. @@ -146,13 +146,14 @@ LDFLAGS.shlib = @SH_LDFLAGS@ # sources passed to [teaish-src-add], but may also be appended to by # teaish.make. # -tx.src =@TEAISH_EXT_SRC@ +tx.src = @TEAISH_EXT_SRC@ # # tx.CFLAGS is typically set by teaish.make, whereas TEAISH_CFLAGS # gets set up via the configure script. # tx.CFLAGS = +tx.CPPFLAGS = # # tx.LDFLAGS is typically set by teaish.make, whereas TEAISH_LDFLAGS @@ -167,6 +168,11 @@ tx.LDFLAGS = # tx.dist.files = @TEAISH_DIST_FILES@ +# +# The base name for a distribution tar/zip file. +# +tx.dist.basename = $(tx.name.dist)-$(tx.version) + # List of deps which may trigger an auto-reconfigure. # teaish__autogen.deps = \ @@ -199,16 +205,21 @@ $(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ @AUTODEPS@ @if TEAISH_TESTER_TCL_IN -@TEAISH_TESTER_TCL_IN@: -@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ -config.log: @TEAISH_TESTER_TCL@ +@TEAISH_TESTER_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TESTER_TCL_IN@ +@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ +@endif +@if TEAISH_TEST_TCL_IN +@TEAISH_TEST_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TEST_TCL_IN@ +@TEAISH_TEST_TCL@: @TEAISH_TEST_TCL_IN@ @endif # # CC variant for compiling Tcl-using sources. # CC.tcl = \ - $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) + $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) $(tx.CPPFLAGS) # # CC variant for linking $(tx.src) into an extension DLL. Note that @@ -217,7 +228,7 @@ CC.tcl = \ # CC.dll = \ $(CC.tcl) $(tx.src) $(LDFLAGS.shlib) \ - $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) $(TCL_STUB_LIB_SPEC) + $(tx.LDFLAGS) $(LDFLAGS.configure) $(LDFLAGS) $(TCL_STUB_LIB_SPEC) @if TEAISH_ENABLE_DLL # @@ -248,16 +259,25 @@ test-extension: # this name is reserved for use by teaish.make[.in] test-prepre: $(tx.dll) @endif @if TEAISH_TESTER_TCL -test-core.args = @TEAISH_TESTER_TCL@ +teaish.tester.tcl = @TEAISH_TESTER_TCL@ +test-core.args = $(teaish.tester.tcl) @if TEAISH_ENABLE_DLL test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' @else test-core.args += '' '' @endif test-core.args += @TEAISH_TESTUTIL_TCL@ +# Clients may pass additional args via test.args=... +# and ::argv will be rewritten before the test script loads, to +# remove $(test-core.args) +test.args ?= test-core: test-pre - $(TCLSH) $(test-core.args) -test-prepre: @TEAISH_TESTER_TCL@ + $(TCLSH) $(test-core.args) $(test.args) +test-gdb: $(teaish.tester.tcl) + gdb --args $(TCLSH) $(test-core.args) $(test.args) +test-vg.flags ?= --leak-check=full -v --show-reachable=yes --track-origins=yes +test-vg: $(teaish.tester.tcl) + valgrind $(test-vg.flags) $(TCLSH) $(test-core.args) $(test.args) @else # !TEAISH_TESTER_TCL test-prepre: @endif # TEAISH_TESTER_TCL @@ -288,7 +308,7 @@ distclean-core: distclean-pre @endif @endif @if TEAISH_TESTER_TCL_IN - rm -f @TEAISH_TESTER_TCL@ + rm -f $(teaish.tester.tcl) @endif @if TEAISH_PKGINDEX_TCL_IN rm -f @TEAISH_PKGINDEX_TCL@ @@ -355,10 +375,15 @@ install-core: install-pre @endif install-test: install-core @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ + set xtra=""; \ + if [ x != "x$(DESTDIR)" ]; then \ + xtra='set ::auto_path [linsert $$::auto_path 0 [file normalize $(DESTDIR)$(TCLLIBDIR)/..]];'; \ + fi; \ if echo \ - 'set c 0; ' \ + 'set c 0; ' $$xtra \ '@TEAISH_POSTINST_PREREQUIRE@' \ - 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)} xc]} {incr c};' \ + 'if {$$c && "" ne $$xc} {puts $$xc; puts "auto_path=$$::auto_path"};' \ 'exit $$c' \ | $(TCLSH) ; then \ echo "passed"; \ @@ -406,7 +431,7 @@ config.log: $(teaish.makefile.in) # recognized when running in --teaish-install mode, causing # the sub-configure to fail. dist.flags = --with-tclsh=$(TCLSH) -dist.reconfig = $(teaish.dir)/configure $(dist.flags) +dist.reconfig = $(teaish.dir)/configure $(tx.dist.reconfig-flags) $(dist.flags) # Temp dir for dist.zip. Must be different than dist.tgz or else # parallel builds may hose the dist. @@ -414,24 +439,23 @@ teaish__dist.tmp.zip = teaish__dist_zip # # Make a distribution zip file... # -dist.basename = $(tx.name.dist)-$(tx.version) -dist.zip = $(dist.basename).zip +dist.zip = $(tx.dist.basename).zip .PHONY: dist.zip dist.zip-core dist.zip-post #dist.zip-pre: # We apparently can't add a pre-hook here, else "make dist" rebuilds # the archive each time it's run. $(dist.zip): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.zip) - @mkdir -p $(teaish__dist.tmp.zip)/$(dist.basename) + @mkdir -p $(teaish__dist.tmp.zip)/$(tx.dist.basename) @tar cf $(teaish__dist.tmp.zip)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(dist.basename) + @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(tx.dist.basename) @if TEAISH_DIST_FULL @$(dist.reconfig) \ - --teaish-install=$(teaish__dist.tmp.zip)/$(dist.basename) \ - --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null + --teaish-install=$(teaish__dist.tmp.zip)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null @endif - @rm -f $(dist.basename)/tmp.tar $(dist.zip) - @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(dist.basename) + @rm -f $(tx.dist.basename)/tmp.tar $(dist.zip) + @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(tx.dist.basename) @rm -fr $(teaish__dist.tmp.zip) @ls -la $(dist.zip) dist.zip-core: $(dist.zip) @@ -447,23 +471,23 @@ undist: undist-zip # Make a distribution tarball... # teaish__dist.tmp.tgz = teaish__dist_tgz -dist.tgz = $(dist.basename).tar.gz +dist.tgz = $(tx.dist.basename).tar.gz .PHONY: dist.tgz dist.tgz-core dist.tgz-post # dist.tgz-pre: # see notes in dist.zip $(dist.tgz): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.tgz) - @mkdir -p $(teaish__dist.tmp.tgz)/$(dist.basename) + @mkdir -p $(teaish__dist.tmp.tgz)/$(tx.dist.basename) @tar cf $(teaish__dist.tmp.tgz)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(dist.basename) + @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(tx.dist.basename) @if TEAISH_DIST_FULL - @rm -f $(teaish__dist.tmp.tgz)/$(dist.basename)/pkgIndex.tcl.in; # kludge + @rm -f $(teaish__dist.tmp.tgz)/$(tx.dist.basename)/pkgIndex.tcl.in; # kludge @$(dist.reconfig) \ - --teaish-install=$(teaish__dist.tmp.tgz)/$(dist.basename) \ - --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null + --teaish-install=$(teaish__dist.tmp.tgz)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null @endif - @rm -f $(dist.basename)/tmp.tar $(dist.tgz) - @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(dist.basename) + @rm -f $(tx.dist.basename)/tmp.tar $(dist.tgz) + @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(tx.dist.basename) @rm -fr $(teaish__dist.tmp.tgz) @ls -la $(dist.tgz) dist.tgz-core: $(dist.tgz) diff --git a/contrib/sqlite3/tea/README.txt b/contrib/sqlite3/tea/README.txt index fb7cb1924854..122b08d32d0c 100644 --- a/contrib/sqlite3/tea/README.txt +++ b/contrib/sqlite3/tea/README.txt @@ -83,22 +83,12 @@ script and then run make. For example: $ cd sqlite-*-tea $ ./configure --with-tcl=/path/to/tcl/install/root - $ make + $ make test $ make install WINDOWS BUILD ============= -The recommended method to build extensions under windows is to use the -Msys + Mingw build process. This provides a Unix-style build while -generating native Windows binaries. Using the Msys + Mingw build tools -means that you can use the same configure script as per the Unix build -to create a Makefile. See the tcl/win/README file for the URL of -the Msys + Mingw download. -If you have VC++ then you may wish to use the files in the win -subdirectory and build the extension using just VC++. These files have -been designed to be as generic as possible but will require some -additional maintenance by the project developer to synchronise with -the TEA configure.in and Makefile.in files. Instructions for using the -VC++ makefile are written in the first part of the Makefile.vc -file. +On Windows this build is known to work on Cygwin and some Msys2 +environments. We do not currently support Microsoft makefiles for +native Windows builds. diff --git a/contrib/sqlite3/tea/_teaish.tester.tcl.in b/contrib/sqlite3/tea/_teaish.tester.tcl.in index 59d11f0a8f6e..e04d8e63e790 100644 --- a/contrib/sqlite3/tea/_teaish.tester.tcl.in +++ b/contrib/sqlite3/tea/_teaish.tester.tcl.in @@ -21,7 +21,8 @@ if {[llength [lindex $::argv 0]] > 0} { # ----^^^^^^^ needed on Haiku when argv 0 is just a filename, else # load cannot find the file. } -source -encoding utf-8 [lindex $::argv 2]; # teaish/tester.tcl +set ::argv [lassign $argv - -] +source -encoding utf-8 [lindex $::argv 0]; # teaish/tester.tcl @if TEAISH_PKGINIT_TCL apply {{file} { set dir [file dirname $::argv0] diff --git a/contrib/sqlite3/tea/configure b/contrib/sqlite3/tea/configure index 47378126f5ab..01b3abcc2fba 100755 --- a/contrib/sqlite3/tea/configure +++ b/contrib/sqlite3/tea/configure @@ -1,7 +1,20 @@ #!/bin/sh +# Look for and run autosetup... dir0="`dirname "$0"`" -dirA="$dir0/../autosetup" -# This is the case ^^^^^^^^^^^^ in the SQLite "autoconf" bundle. +dirA="$dir0" +if [ -d $dirA/autosetup ]; then + # A local copy of autosetup + dirA=$dirA/autosetup +elif [ -d $dirA/../autosetup ]; then + # SQLite "autoconf" bundle + dirA=$dirA/../autosetup +elif [ -d $dirA/../../autosetup ]; then + # SQLite canonical source tree + dirA=$dirA/../../autosetup +else + echo "$0: Cannot find autosetup" 1>&2 + exit 1 +fi WRAPPER="$0"; export WRAPPER; exec "`"$dirA/autosetup-find-tclsh"`" \ "$dirA/autosetup" --teaish-extension-dir="$dir0" \ "$@" diff --git a/contrib/sqlite3/tea/doc/sqlite3.n b/contrib/sqlite3/tea/doc/sqlite3.n deleted file mode 100644 index 3514046342da..000000000000 --- a/contrib/sqlite3/tea/doc/sqlite3.n +++ /dev/null @@ -1,15 +0,0 @@ -.TH sqlite3 n 4.1 "Tcl-Extensions" -.HS sqlite3 tcl -.BS -.SH NAME -sqlite3 \- an interface to the SQLite3 database engine -.SH SYNOPSIS -\fBsqlite3\fI command_name ?filename?\fR -.br -.SH DESCRIPTION -SQLite3 is a self-contains, zero-configuration, transactional SQL database -engine. This extension provides an easy to use interface for accessing -SQLite database files from Tcl. -.PP -For full documentation see \fIhttps://sqlite.org/\fR and -in particular \fIhttps://sqlite.org/tclsqlite.html\fR. diff --git a/contrib/sqlite3/tea/generic/tclsqlite3.c b/contrib/sqlite3/tea/generic/tclsqlite3.c index 197ce744836c..144d4399d8e5 100644 --- a/contrib/sqlite3/tea/generic/tclsqlite3.c +++ b/contrib/sqlite3/tea/generic/tclsqlite3.c @@ -54,6 +54,10 @@ # define CONST const #elif !defined(Tcl_Size) typedef int Tcl_Size; +# ifndef Tcl_BounceRefCount +# define Tcl_BounceRefCount(X) Tcl_IncrRefCount(X); Tcl_DecrRefCount(X) + /* https://www.tcl-lang.org/man/tcl9.0/TclLib/Object.html */ +# endif #endif /**** End copy of tclsqlite.h ****/ @@ -125,6 +129,15 @@ /* Forward declaration */ typedef struct SqliteDb SqliteDb; +/* Add -DSQLITE_ENABLE_QRF_IN_TCL to add the Query Result Formatter (QRF) +** into the build of the TCL extension, when building using separate +** source files. The QRF is included automatically when building from +** the tclsqlite3.c amalgamation. +*/ +#if defined(SQLITE_ENABLE_QRF_IN_TCL) +#include "qrf.h" +#endif + /* ** New SQL functions can be created as TCL scripts. Each such function ** is described by an instance of the following structure. @@ -1089,7 +1102,9 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ Tcl_DecrRefCount(pCmd); } - if( rc && rc!=TCL_RETURN ){ + if( TCL_BREAK==rc ){ + sqlite3_result_null(context); + }else if( rc && rc!=TCL_RETURN ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); }else{ Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); @@ -1107,7 +1122,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ }else if( (c=='b' && pVar->bytes==0 && strcmp(zType,"boolean")==0 ) || (c=='b' && pVar->bytes==0 && strcmp(zType,"booleanString")==0 ) || (c=='w' && strcmp(zType,"wideInt")==0) - || (c=='i' && strcmp(zType,"int")==0) + || (c=='i' && strcmp(zType,"int")==0) ){ eType = SQLITE_INTEGER; }else if( c=='d' && strcmp(zType,"double")==0 ){ @@ -1621,11 +1636,12 @@ struct DbEvalContext { SqlPreparedStmt *pPreStmt; /* Current statement */ int nCol; /* Number of columns returned by pStmt */ int evalFlags; /* Flags used */ - Tcl_Obj *pArray; /* Name of array variable */ + Tcl_Obj *pVarName; /* Name of target array/dict variable */ Tcl_Obj **apColName; /* Array of column names */ }; #define SQLITE_EVAL_WITHOUTNULLS 0x00001 /* Unset array(*) for NULL */ +#define SQLITE_EVAL_ASDICT 0x00002 /* Use dict instead of array */ /* ** Release any cache of column names currently held as part of @@ -1646,20 +1662,20 @@ static void dbReleaseColumnNames(DbEvalContext *p){ /* ** Initialize a DbEvalContext structure. ** -** If pArray is not NULL, then it contains the name of a Tcl array +** If pVarName is not NULL, then it contains the name of a Tcl array ** variable. The "*" member of this array is set to a list containing ** the names of the columns returned by the statement as part of each ** call to dbEvalStep(), in order from left to right. e.g. if the names ** of the returned columns are a, b and c, it does the equivalent of the ** tcl command: ** -** set ${pArray}(*) {a b c} +** set ${pVarName}(*) {a b c} */ static void dbEvalInit( DbEvalContext *p, /* Pointer to structure to initialize */ SqliteDb *pDb, /* Database handle */ Tcl_Obj *pSql, /* Object containing SQL script */ - Tcl_Obj *pArray, /* Name of Tcl array to set (*) element of */ + Tcl_Obj *pVarName, /* Name of Tcl array to set (*) element of */ int evalFlags /* Flags controlling evaluation */ ){ memset(p, 0, sizeof(DbEvalContext)); @@ -1667,9 +1683,9 @@ static void dbEvalInit( p->zSql = Tcl_GetString(pSql); p->pSql = pSql; Tcl_IncrRefCount(pSql); - if( pArray ){ - p->pArray = pArray; - Tcl_IncrRefCount(pArray); + if( pVarName ){ + p->pVarName = pVarName; + Tcl_IncrRefCount(pVarName); } p->evalFlags = evalFlags; addDatabaseRef(p->pDb); @@ -1692,7 +1708,7 @@ static void dbEvalRowInfo( Tcl_Obj **apColName = 0; /* Array of column names */ p->nCol = nCol = sqlite3_column_count(pStmt); - if( nCol>0 && (papColName || p->pArray) ){ + if( nCol>0 && (papColName || p->pVarName) ){ apColName = (Tcl_Obj**)Tcl_Alloc( sizeof(Tcl_Obj*)*nCol ); for(i=0; i<nCol; i++){ apColName[i] = Tcl_NewStringObj(sqlite3_column_name(pStmt,i), -1); @@ -1701,20 +1717,35 @@ static void dbEvalRowInfo( p->apColName = apColName; } - /* If results are being stored in an array variable, then create - ** the array(*) entry for that array + /* If results are being stored in a variable then create the + ** array(*) or dict(*) entry for that variable. */ - if( p->pArray ){ + if( p->pVarName ){ Tcl_Interp *interp = p->pDb->interp; Tcl_Obj *pColList = Tcl_NewObj(); Tcl_Obj *pStar = Tcl_NewStringObj("*", -1); + Tcl_IncrRefCount(pColList); + Tcl_IncrRefCount(pStar); for(i=0; i<nCol; i++){ Tcl_ListObjAppendElement(interp, pColList, apColName[i]); } - Tcl_IncrRefCount(pStar); - Tcl_ObjSetVar2(interp, p->pArray, pStar, pColList, 0); + if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + Tcl_ObjSetVar2(interp, p->pVarName, pStar, pColList, 0); + }else{ + Tcl_Obj * pDict = Tcl_ObjGetVar2(interp, p->pVarName, NULL, 0); + if( !pDict ){ + pDict = Tcl_NewDictObj(); + }else if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjPut(interp, pDict, pStar, pColList)==TCL_OK ){ + Tcl_ObjSetVar2(interp, p->pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); + } Tcl_DecrRefCount(pStar); + Tcl_DecrRefCount(pColList); } } @@ -1756,7 +1787,7 @@ static int dbEvalStep(DbEvalContext *p){ if( rcs==SQLITE_ROW ){ return TCL_OK; } - if( p->pArray ){ + if( p->pVarName ){ dbEvalRowInfo(p, 0, 0); } rcs = sqlite3_reset(pStmt); @@ -1807,9 +1838,9 @@ static void dbEvalFinalize(DbEvalContext *p){ dbReleaseStmt(p->pDb, p->pPreStmt, 0); p->pPreStmt = 0; } - if( p->pArray ){ - Tcl_DecrRefCount(p->pArray); - p->pArray = 0; + if( p->pVarName ){ + Tcl_DecrRefCount(p->pVarName); + p->pVarName = 0; } Tcl_DecrRefCount(p->pSql); dbReleaseColumnNames(p); @@ -1884,7 +1915,7 @@ static int DbUseNre(void){ /* ** This function is part of the implementation of the command: ** -** $db eval SQL ?ARRAYNAME? SCRIPT +** $db eval SQL ?TGT-NAME? SCRIPT */ static int SQLITE_TCLAPI DbEvalNextCmd( ClientData data[], /* data[0] is the (DbEvalContext*) */ @@ -1898,8 +1929,8 @@ static int SQLITE_TCLAPI DbEvalNextCmd( ** is a pointer to a Tcl_Obj containing the script to run for each row ** returned by the queries encapsulated in data[0]. */ DbEvalContext *p = (DbEvalContext *)data[0]; - Tcl_Obj *pScript = (Tcl_Obj *)data[1]; - Tcl_Obj *pArray = p->pArray; + Tcl_Obj * const pScript = (Tcl_Obj *)data[1]; + Tcl_Obj * const pVarName = p->pVarName; while( (rc==TCL_OK || rc==TCL_CONTINUE) && TCL_OK==(rc = dbEvalStep(p)) ){ int i; @@ -1907,15 +1938,46 @@ static int SQLITE_TCLAPI DbEvalNextCmd( Tcl_Obj **apColName; dbEvalRowInfo(p, &nCol, &apColName); for(i=0; i<nCol; i++){ - if( pArray==0 ){ + if( pVarName==0 ){ Tcl_ObjSetVar2(interp, apColName[i], 0, dbEvalColumnValue(p,i), 0); }else if( (p->evalFlags & SQLITE_EVAL_WITHOUTNULLS)!=0 - && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL + && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL ){ - Tcl_UnsetVar2(interp, Tcl_GetString(pArray), - Tcl_GetString(apColName[i]), 0); + /* Remove NULL-containing column from the target container... */ + if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + /* Target is an array */ + Tcl_UnsetVar2(interp, Tcl_GetString(pVarName), + Tcl_GetString(apColName[i]), 0); + }else{ + /* Target is a dict */ + Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0); + if( pDict ){ + if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjRemove(interp, pDict, apColName[i])==TCL_OK ){ + Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); + } + } + }else if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + /* Target is an array: set target(colName) = colValue */ + Tcl_ObjSetVar2(interp, pVarName, apColName[i], + dbEvalColumnValue(p,i), 0); }else{ - Tcl_ObjSetVar2(interp, pArray, apColName[i], dbEvalColumnValue(p,i), 0); + /* Target is a dict: set target(colName) = colValue */ + Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0); + if( !pDict ){ + pDict = Tcl_NewDictObj(); + }else if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjPut(interp, pDict, apColName[i], + dbEvalColumnValue(p,i))==TCL_OK ){ + Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); } } @@ -1988,6 +2050,376 @@ static void DbHookCmd( } /* +** Implementation of the "db format" command. +** +** Based on provided options, format the results of the SQL statement(s) +** provided into human-readable form using the Query Result Formatter (QRF) +** and return the resuling text. +** +** Syntax: db format OPTIONS SQL +** +** OPTIONS may be: +** +** -style ("auto"|"box"|"column"|...) Output style +** -esc ("auto"|"off"|"ascii"|"symbol") How to deal with ctrl chars +** -text ("auto"|"off"|"sql"|"csv"|...) How to escape TEXT values +** -title ("auto"|"off"|"sql"|...|"off") How to escape column names +** -blob ("auto"|"text"|"sql"|...) How to escape BLOB values +** -wordwrap ("auto"|"off"|"on") Try to wrap at word boundry? +** -textjsonb ("auto"|"off"|"on") Auto-convert JSONB to text? +** -splitcolumn ("auto"|"off"|"on") Enable split-column mode +** -defaultalign ("auto"|"left"|...) Default alignment +** -titalalign ("auto"|"left"|"right"|...) Default column name alignment +** -border ("auto"|"off"|"on") Border for box and table styles +** -wrap NUMBER Max width of any single column +** -screenwidth NUMBER Width of the display TTY +** -linelimit NUMBER Max lines for any cell +** -charlimit NUMBER Content truncated to this size +** -titlelimit NUMBER Max width of column titles +** -multiinsert NUMBER Multi-row INSERT byte size +** -align LIST-OF-ALIGNMENT Alignment of columns +** -widths LIST-OF-NUMBERS Widths for individual columns +** -columnsep TEXT Column separator text +** -rowsep TEXT Row separator text +** -tablename TEXT Table name for style "insert" +** -null TEXT Text for NULL values +** +** A mapping from TCL "format" command options to sqlite3_qrf_spec fields +** is below. Use this to reference the QRF documentation: +** +** TCL Option spec field +** ---------- ---------- +** -style eStyle +** -esc eEsc +** -text eText +** -title eTitle, bTitle +** -blob eBlob +** -wordwrap bWordWrap +** -textjsonb bTextJsonb +** -splitcolumn bSplitColumn +** -defaultalign eDfltAlign +** -titlealign eTitleAlign +** -border bBorder +** -wrap nWrap +** -screenwidth nScreenWidth +** -linelimit nLineLimit +** -charlimit nCharLimit +** -titlelimit nTitleLimit +** -multiinsert nMultiInsert +** -align nAlign, aAlign +** -widths nWidth, aWidth +** -columnsep zColumnSep +** -rowsep zRowSep +** -tablename zTableName +** -null zNull +*/ +static int dbQrf(SqliteDb *pDb, int objc, Tcl_Obj *const*objv){ +#ifndef SQLITE_QRF_H + Tcl_SetResult(pDb->interp, "QRF not available in this build", TCL_VOLATILE); + return TCL_ERROR; +#else + char *zResult = 0; /* Result to be returned */ + const char *zSql = 0; /* SQL to run */ + int i; /* Loop counter */ + int rc; /* Result code */ + sqlite3_qrf_spec qrf; /* Formatting spec */ + static const char *azAlign[] = { + "auto", "bottom", "c", + "center", "e", "left", + "middle", "n", "ne", + "nw", "right", "s", + "se", "sw", "top", + "w", 0 + }; + static const unsigned char aAlignMap[] = { + QRF_ALIGN_Auto, QRF_ALIGN_Bottom, QRF_ALIGN_C, + QRF_ALIGN_Center, QRF_ALIGN_E, QRF_ALIGN_Left, + QRF_ALIGN_Middle, QRF_ALIGN_N, QRF_ALIGN_NE, + QRF_ALIGN_NW, QRF_ALIGN_Right, QRF_ALIGN_S, + QRF_ALIGN_SE, QRF_ALIGN_SW, QRF_ALIGN_Top, + QRF_ALIGN_W + }; + + memset(&qrf, 0, sizeof(qrf)); + qrf.iVersion = 1; + qrf.pzOutput = &zResult; + for(i=2; i<objc; i++){ + const char *zArg = Tcl_GetString(objv[i]); + const char *azBool[] = { "auto", "yes", "no", "on", "off", 0 }; + const unsigned char aBoolMap[] = { 0, 2, 1, 2, 1 }; + if( zArg[0]!='-' ){ + if( zSql ){ + Tcl_AppendResult(pDb->interp, "unknown argument: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + zSql = zArg; + }else if( i==objc-1 ){ + Tcl_AppendResult(pDb->interp, "option has no argument: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + }else if( strcmp(zArg,"-style")==0 ){ + static const char *azStyles[] = { + "auto", "box", "column", + "count", "csv", "eqp", + "explain", "html", "insert", + "jobject", "json", "line", + "list", "markdown", "quote", + "stats", "stats-est", "stats-vm", + "table", 0 + }; + static unsigned char aStyleMap[] = { + QRF_STYLE_Auto, QRF_STYLE_Box, QRF_STYLE_Column, + QRF_STYLE_Count, QRF_STYLE_Csv, QRF_STYLE_Eqp, + QRF_STYLE_Explain, QRF_STYLE_Html, QRF_STYLE_Insert, + QRF_STYLE_JObject, QRF_STYLE_Json, QRF_STYLE_Line, + QRF_STYLE_List, QRF_STYLE_Markdown, QRF_STYLE_Quote, + QRF_STYLE_Stats, QRF_STYLE_StatsEst, QRF_STYLE_StatsVm, + QRF_STYLE_Table + }; + int style; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azStyles, + "format style (-style)", 0, &style); + if( rc ) goto format_failed; + qrf.eStyle = aStyleMap[style]; + i++; + }else if( strcmp(zArg,"-esc")==0 ){ + static const char *azEsc[] = { + "ascii", "auto", "off", "symbol", 0 + }; + static unsigned char aEscMap[] = { + QRF_ESC_Ascii, QRF_ESC_Auto, QRF_ESC_Off, QRF_ESC_Symbol + }; + int esc; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azEsc, + "control character escape (-esc)", 0, &esc); + if( rc ) goto format_failed; + qrf.eEsc = aEscMap[esc]; + i++; + }else if( strcmp(zArg,"-text")==0 || strcmp(zArg, "-title")==0 ){ + /* NB: --title can be "off" or "on but --text may not be. Thus we put + ** the "off" and "on" choices first and start the search on the + ** thrid element of the array when processing --text */ + static const char *azText[] = { "off", "on", + "auto", "csv", "html", + "json", "plain", "relaxed", + "sql", "tcl", 0 + }; + static unsigned char aTextMap[] = { + QRF_TEXT_Auto, QRF_TEXT_Csv, QRF_TEXT_Html, + QRF_TEXT_Json, QRF_TEXT_Plain, QRF_TEXT_Relaxed, + QRF_TEXT_Sql, QRF_TEXT_Tcl + }; + int txt; + int k = zArg[2]=='e'; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], &azText[k*2], zArg, + 0, &txt); + if( rc ) goto format_failed; + if( k ){ + qrf.eText = aTextMap[txt]; + }else if( txt<=1 ){ + qrf.bTitles = txt ? QRF_Yes : QRF_No; + qrf.eTitle = QRF_TEXT_Auto; + }else{ + qrf.bTitles = QRF_Yes; + qrf.eTitle = aTextMap[txt-2]; + } + i++; + }else if( strcmp(zArg,"-blob")==0 ){ + static const char *azBlob[] = { + "auto", "hex", "json", + "tcl", "text", "sql", + "size", 0 + }; + static unsigned char aBlobMap[] = { + QRF_BLOB_Auto, QRF_BLOB_Hex, QRF_BLOB_Json, + QRF_BLOB_Tcl, QRF_BLOB_Text, QRF_BLOB_Sql, + QRF_BLOB_Size + }; + int blob; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBlob, + "BLOB encoding (-blob)", 0, &blob); + if( rc ) goto format_failed; + qrf.eBlob = aBlobMap[blob]; + i++; + }else if( strcmp(zArg,"-wordwrap")==0 ){ + int v = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, + "-wordwrap", 0, &v); + if( rc ) goto format_failed; + qrf.bWordWrap = aBoolMap[v]; + i++; + }else if( strcmp(zArg,"-textjsonb")==0 + || strcmp(zArg,"-splitcolumn")==0 + || strcmp(zArg,"-border")==0 + ){ + int v = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, + zArg, 0, &v); + if( rc ) goto format_failed; + if( zArg[1]=='t' ){ + qrf.bTextJsonb = aBoolMap[v]; + }else if( zArg[1]=='b' ){ + qrf.bBorder = aBoolMap[v]; + }else{ + qrf.bSplitColumn = aBoolMap[v]; + } + i++; + }else if( strcmp(zArg,"-defaultalign")==0 || strcmp(zArg,"-titlealign")==0){ + int ax = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azAlign, + zArg[1]=='d' ? "default alignment (-defaultalign)" : + "title alignment (-titlealign)", + 0, &ax); + if( rc ) goto format_failed; + if( zArg[1]=='d' ){ + qrf.eDfltAlign = aAlignMap[ax]; + }else{ + qrf.eTitleAlign = aAlignMap[ax]; + } + i++; + }else if( strcmp(zArg,"-wrap")==0 + || strcmp(zArg,"-screenwidth")==0 + || strcmp(zArg,"-linelimit")==0 + || strcmp(zArg,"-titlelimit")==0 + ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<QRF_MIN_WIDTH ){ + v = QRF_MIN_WIDTH; + }else if( v>QRF_MAX_WIDTH ){ + v = QRF_MAX_WIDTH; + } + if( zArg[1]=='w' ){ + qrf.nWrap = v; + }else if( zArg[1]=='s' ){ + qrf.nScreenWidth = v; + }else if( zArg[1]=='t' ){ + qrf.nTitleLimit = v; + }else{ + qrf.nLineLimit = v; + } + i++; + }else if( strcmp(zArg,"-charlimit")==0 ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<0 ) v = 0; + qrf.nCharLimit = v; + i++; + }else if( strcmp(zArg,"-multiinsert")==0 ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<0 ) v = 0; + qrf.nMultiInsert = v; + i++; + }else if( strcmp(zArg,"-align")==0 ){ + Tcl_Size n = 0; + int jj; + rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); + if( rc ) goto format_failed; + sqlite3_free(qrf.aAlign); + qrf.aAlign = sqlite3_malloc64( (n+1)*sizeof(qrf.aAlign[0]) ); + if( qrf.aAlign==0 ){ + Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + memset(qrf.aAlign, 0, (n+1)*sizeof(qrf.aAlign[0])); + qrf.nAlign = n; + for(jj=0; jj<n; jj++){ + int x; + Tcl_Obj *pTerm; + rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); + if( rc ) goto format_failed; + rc = Tcl_GetIndexFromObj(pDb->interp, pTerm, azAlign, + "column alignment (-align)", 0, &x); + if( rc ) goto format_failed; + qrf.aAlign[jj] = aAlignMap[x]; + } + i++; + }else if( strcmp(zArg,"-widths")==0 ){ + Tcl_Size n = 0; + int jj; + rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); + if( rc ) goto format_failed; + sqlite3_free(qrf.aWidth); + qrf.aWidth = sqlite3_malloc64( (n+1)*sizeof(qrf.aWidth[0]) ); + if( qrf.aWidth==0 ){ + Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + memset(qrf.aWidth, 0, (n+1)*sizeof(qrf.aWidth[0])); + qrf.nWidth = n; + for(jj=0; jj<n; jj++){ + Tcl_Obj *pTerm; + int v; + rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); + if( rc ) goto format_failed; + rc = Tcl_GetIntFromObj(pDb->interp, pTerm, &v); + if( v<(-QRF_MAX_WIDTH) ){ + v = -QRF_MAX_WIDTH; + }else if( v>QRF_MAX_WIDTH ){ + v = QRF_MAX_WIDTH; + } + qrf.aWidth[jj] = (short int)v; + } + i++; + }else if( strcmp(zArg,"-columnsep")==0 ){ + qrf.zColumnSep = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-rowsep")==0 ){ + qrf.zRowSep = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-tablename")==0 ){ + qrf.zTableName = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-null")==0 ){ + qrf.zNull = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-version")==0 ){ + /* Undocumented. Testing use only */ + qrf.iVersion = atoi(Tcl_GetString(objv[i+1])); + i++; + }else{ + Tcl_AppendResult(pDb->interp, "unknown option: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + } + while( zSql && zSql[0] ){ + SqlPreparedStmt *pStmt = 0; /* Next statement to run */ + char *zErr = 0; /* Error message from QRF */ + + rc = dbPrepareAndBind(pDb, zSql, &zSql, &pStmt); + if( rc ) goto format_failed; + if( pStmt==0 ) continue; + rc = sqlite3_format_query_result(pStmt->pStmt, &qrf, &zErr); + dbReleaseStmt(pDb, pStmt, 0); + if( rc ){ + Tcl_SetResult(pDb->interp, zErr, TCL_VOLATILE); + sqlite3_free(zErr); + rc = TCL_ERROR; + goto format_failed; + } + } + Tcl_SetResult(pDb->interp, zResult, TCL_VOLATILE); + rc = TCL_OK; + /* Fall through...*/ + +format_failed: + sqlite3_free(qrf.aWidth); + sqlite3_free(qrf.aAlign); + sqlite3_free(zResult); + return rc; + +#endif +} + +/* ** The "sqlite" command below creates a new Tcl command for each ** connection it opens to an SQLite database. This routine is invoked ** whenever one of those connection-specific commands is executed @@ -2016,15 +2448,15 @@ static int SQLITE_TCLAPI DbObjCmd( "commit_hook", "complete", "config", "copy", "deserialize", "enable_load_extension", "errorcode", "erroroffset", "eval", - "exists", "function", "incrblob", - "interrupt", "last_insert_rowid", "nullvalue", - "onecolumn", "preupdate", "profile", - "progress", "rekey", "restore", - "rollback_hook", "serialize", "status", - "timeout", "total_changes", "trace", - "trace_v2", "transaction", "unlock_notify", - "update_hook", "version", "wal_hook", - 0 + "exists", "format", "function", + "incrblob", "interrupt", "last_insert_rowid", + "nullvalue", "onecolumn", "preupdate", + "profile", "progress", "rekey", + "restore", "rollback_hook", "serialize", + "status", "timeout", "total_changes", + "trace", "trace_v2", "transaction", + "unlock_notify", "update_hook", "version", + "wal_hook", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BIND_FALLBACK, @@ -2033,14 +2465,15 @@ static int SQLITE_TCLAPI DbObjCmd( DB_COMMIT_HOOK, DB_COMPLETE, DB_CONFIG, DB_COPY, DB_DESERIALIZE, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_ERROROFFSET, DB_EVAL, - DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, - DB_INTERRUPT, DB_LAST_INSERT_ROWID, DB_NULLVALUE, - DB_ONECOLUMN, DB_PREUPDATE, DB_PROFILE, - DB_PROGRESS, DB_REKEY, DB_RESTORE, - DB_ROLLBACK_HOOK, DB_SERIALIZE, DB_STATUS, - DB_TIMEOUT, DB_TOTAL_CHANGES, DB_TRACE, - DB_TRACE_V2, DB_TRANSACTION, DB_UNLOCK_NOTIFY, - DB_UPDATE_HOOK, DB_VERSION, DB_WAL_HOOK, + DB_EXISTS, DB_FORMAT, DB_FUNCTION, + DB_INCRBLOB, DB_INTERRUPT, DB_LAST_INSERT_ROWID, + DB_NULLVALUE, DB_ONECOLUMN, DB_PREUPDATE, + DB_PROFILE, DB_PROGRESS, DB_REKEY, + DB_RESTORE, DB_ROLLBACK_HOOK, DB_SERIALIZE, + DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, + DB_TRACE, DB_TRACE_V2, DB_TRANSACTION, + DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, DB_VERSION, + DB_WAL_HOOK }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -2858,13 +3291,15 @@ deserialize_error: } /* - ** $db eval ?options? $sql ?array? ?{ ...code... }? + ** $db eval ?options? $sql ?varName? ?{ ...code... }? ** - ** The SQL statement in $sql is evaluated. For each row, the values are - ** placed in elements of the array named "array" and ...code... is executed. - ** If "array" and "code" are omitted, then no callback is every invoked. - ** If "array" is an empty string, then the values are placed in variables - ** that have the same name as the fields extracted by the query. + ** The SQL statement in $sql is evaluated. For each row, the values + ** are placed in elements of the array or dict named $varName and + ** ...code... is executed. If $varName and $code are omitted, then + ** no callback is ever invoked. If $varName is an empty string, + ** then the values are placed in variables that have the same name + ** as the fields extracted by the query, and those variables are + ** accessible during the eval of $code. */ case DB_EVAL: { int evalFlags = 0; @@ -2872,8 +3307,9 @@ deserialize_error: while( objc>3 && (zOpt = Tcl_GetString(objv[2]))!=0 && zOpt[0]=='-' ){ if( strcmp(zOpt, "-withoutnulls")==0 ){ evalFlags |= SQLITE_EVAL_WITHOUTNULLS; - } - else{ + }else if( strcmp(zOpt, "-asdict")==0 ){ + evalFlags |= SQLITE_EVAL_ASDICT; + }else{ Tcl_AppendResult(interp, "unknown option: \"", zOpt, "\"", (void*)0); return TCL_ERROR; } @@ -2881,8 +3317,8 @@ deserialize_error: objv++; } if( objc<3 || objc>5 ){ - Tcl_WrongNumArgs(interp, 2, objv, - "?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?"); + Tcl_WrongNumArgs(interp, 2, objv, + "?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?"); return TCL_ERROR; } @@ -2908,17 +3344,17 @@ deserialize_error: }else{ ClientData cd2[2]; DbEvalContext *p; - Tcl_Obj *pArray = 0; + Tcl_Obj *pVarName = 0; Tcl_Obj *pScript; if( objc>=5 && *(char *)Tcl_GetString(objv[3]) ){ - pArray = objv[3]; + pVarName = objv[3]; } pScript = objv[objc-1]; Tcl_IncrRefCount(pScript); p = (DbEvalContext *)Tcl_Alloc(sizeof(DbEvalContext)); - dbEvalInit(p, pDb, objv[2], pArray, evalFlags); + dbEvalInit(p, pDb, objv[2], pVarName, evalFlags); cd2[0] = (void *)p; cd2[1] = (void *)pScript; @@ -2928,6 +3364,18 @@ deserialize_error: } /* + ** $db format [OPTIONS] SQL + ** + ** Run the SQL statement(s) given as the final argument. Use the + ** Query Result Formatter extension of SQLite to format the output as + ** text and return that text. + */ + case DB_FORMAT: { + rc = dbQrf(pDb, objc, objv); + break; + } + + /* ** $db function NAME [OPTIONS] SCRIPT ** ** Create a new SQL function called NAME. Whenever that function is diff --git a/contrib/sqlite3/tea/teaish.tcl b/contrib/sqlite3/tea/teaish.tcl index 9333495aa3da..47e0ea7013a1 100644 --- a/contrib/sqlite3/tea/teaish.tcl +++ b/contrib/sqlite3/tea/teaish.tcl @@ -64,12 +64,18 @@ apply {{dir} { -name.pkg sqlite3 -version $version -name.dist $distname - -vsatisfies 8.6- -libDir sqlite$version -pragmas $pragmas + -src generic/tclsqlite3.c } + # We should also have: + # -vsatisfies 8.6- + # But at least one platform is failing this vsatisfies check + # for no apparent reason: + # https://sqlite.org/forum/forumpost/fde857fb8101a4be }} [teaish-get -dir] + # # Must return either an empty string or a list in the form accepted by # autosetup's [options] function. @@ -119,8 +125,6 @@ proc teaish-options {} { proc teaish-configure {} { use teaish/feature - teaish-src-add -dist -dir generic/tclsqlite3.c - if {[proj-opt-was-provided override-sqlite-version]} { teaish-pkginfo-set -version [opt-val override-sqlite-version] proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] diff --git a/crypto/openssh/sshconnect2.c b/crypto/openssh/sshconnect2.c index 478a9a52fd38..5a48c73edbef 100644 --- a/crypto/openssh/sshconnect2.c +++ b/crypto/openssh/sshconnect2.c @@ -1,4 +1,4 @@ -/* $OpenBSD: sshconnect2.c,v 1.385 2026/04/02 07:48:13 djm Exp $ */ +/* $OpenBSD: sshconnect2.c,v 1.386 2026/05/13 05:11:02 djm Exp $ */ /* * Copyright (c) 2000 Markus Friedl. All rights reserved. * Copyright (c) 2008 Damien Miller. All rights reserved. @@ -1277,7 +1277,7 @@ identity_sign(struct identity *id, u_char **sigp, size_t *lenp, * PKCS#11 tokens may not support all signature algorithms, * so check what we get back. */ - if ((id->key->flags & SSHKEY_FLAG_EXT) != 0 && + if (id->key != NULL && (id->key->flags & SSHKEY_FLAG_EXT) != 0 && (r = sshkey_check_sigtype(*sigp, *lenp, alg)) != 0) { debug_fr(r, "sshkey_check_sigtype"); goto out; diff --git a/include/paths.h b/include/paths.h index 26924bcdba91..9157c17524c7 100644 --- a/include/paths.h +++ b/include/paths.h @@ -54,6 +54,7 @@ #define _PATH_DEFTAPE "/dev/sa0" #define _PATH_DEVGPIOC "/dev/gpioc" #define _PATH_DEVNULL "/dev/null" +#define _PATH_DEVPOWER "/dev/power" #define _PATH_DEVZERO "/dev/zero" #define _PATH_DRUM "/dev/drum" #define _PATH_ESDB "/usr/share/i18n/esdb" diff --git a/lib/clang/libllvm/Makefile b/lib/clang/libllvm/Makefile index 327724bcb52e..ea90b37f8cbd 100644 --- a/lib/clang/libllvm/Makefile +++ b/lib/clang/libllvm/Makefile @@ -562,8 +562,8 @@ SRCS_EXT+= DWARFLinker/Parallel/SyntheticTypeNameBuilder.cpp SRCS_EXT+= DWARFLinker/Utils.cpp SRCS_EXT+= DWP/DWP.cpp SRCS_EXT+= DWP/DWPError.cpp -SRCS_MIW+= DebugInfo/BTF/BTFContext.cpp -SRCS_MIW+= DebugInfo/BTF/BTFParser.cpp +SRCS_MIN+= DebugInfo/BTF/BTFContext.cpp +SRCS_MIN+= DebugInfo/BTF/BTFParser.cpp SRCS_MIN+= DebugInfo/CodeView/AppendingTypeTableBuilder.cpp SRCS_MIN+= DebugInfo/CodeView/CVSymbolVisitor.cpp SRCS_MIN+= DebugInfo/CodeView/CVTypeVisitor.cpp @@ -607,7 +607,7 @@ SRCS_MIN+= DebugInfo/CodeView/TypeTableCollection.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFAcceleratorTable.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFAddressRange.cpp -SRCS_MIW+= DebugInfo/DWARF/DWARFCFIPrinter.cpp +SRCS_MIN+= DebugInfo/DWARF/DWARFCFIPrinter.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFCompileUnit.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFContext.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFDebugAbbrev.cpp @@ -630,11 +630,11 @@ SRCS_MIN+= DebugInfo/DWARF/DWARFListTable.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFTypeUnit.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFUnit.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFUnitIndex.cpp -SRCS_MIW+= DebugInfo/DWARF/DWARFUnwindTablePrinter.cpp +SRCS_MIN+= DebugInfo/DWARF/DWARFUnwindTablePrinter.cpp SRCS_MIN+= DebugInfo/DWARF/DWARFVerifier.cpp -SRCS_MIW+= DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp +SRCS_MIN+= DebugInfo/DWARF/LowLevel/DWARFCFIProgram.cpp SRCS_MIN+= DebugInfo/DWARF/LowLevel/DWARFExpression.cpp -SRCS_MIW+= DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp +SRCS_MIN+= DebugInfo/DWARF/LowLevel/DWARFUnwindTable.cpp SRCS_MIW+= DebugInfo/GSYM/CallSiteInfo.cpp SRCS_MIW+= DebugInfo/GSYM/DwarfTransformer.cpp SRCS_MIW+= DebugInfo/GSYM/ExtractRanges.cpp @@ -748,8 +748,8 @@ SRCS_EXT+= DebugInfo/PDB/UDTLayout.cpp SRCS_MIW+= DebugInfo/Symbolize/DIPrinter.cpp SRCS_MIW+= DebugInfo/Symbolize/Markup.cpp SRCS_MIW+= DebugInfo/Symbolize/MarkupFilter.cpp -SRCS_MIW+= DebugInfo/Symbolize/SymbolizableObjectFile.cpp -SRCS_MIW+= DebugInfo/Symbolize/Symbolize.cpp +SRCS_MIN+= DebugInfo/Symbolize/SymbolizableObjectFile.cpp +SRCS_MIN+= DebugInfo/Symbolize/Symbolize.cpp SRCS_MIW+= Debuginfod/BuildIDFetcher.cpp SRCS_MIW+= Debuginfod/Debuginfod.cpp SRCS_MIW+= Debuginfod/HTTPClient.cpp @@ -1077,28 +1077,28 @@ SRCS_EXT+= MCA/Stages/RetireStage.cpp SRCS_EXT+= MCA/Stages/Stage.cpp SRCS_EXT+= MCA/Support.cpp SRCS_EXT+= MCA/View.cpp -SRCS_MIW+= ObjCopy/Archive.cpp -SRCS_MIW+= ObjCopy/COFF/COFFObjcopy.cpp -SRCS_MIW+= ObjCopy/COFF/COFFObject.cpp -SRCS_MIW+= ObjCopy/COFF/COFFReader.cpp -SRCS_MIW+= ObjCopy/COFF/COFFWriter.cpp -SRCS_MIW+= ObjCopy/CommonConfig.cpp -SRCS_MIW+= ObjCopy/ConfigManager.cpp -SRCS_MIW+= ObjCopy/ELF/ELFObjcopy.cpp -SRCS_MIW+= ObjCopy/ELF/ELFObject.cpp -SRCS_MIW+= ObjCopy/MachO/MachOLayoutBuilder.cpp -SRCS_MIW+= ObjCopy/MachO/MachOObjcopy.cpp -SRCS_MIW+= ObjCopy/MachO/MachOObject.cpp -SRCS_MIW+= ObjCopy/MachO/MachOReader.cpp -SRCS_MIW+= ObjCopy/MachO/MachOWriter.cpp -SRCS_MIW+= ObjCopy/ObjCopy.cpp -SRCS_MIW+= ObjCopy/XCOFF/XCOFFObjcopy.cpp -SRCS_MIW+= ObjCopy/XCOFF/XCOFFReader.cpp -SRCS_MIW+= ObjCopy/XCOFF/XCOFFWriter.cpp -SRCS_MIW+= ObjCopy/wasm/WasmObjcopy.cpp -SRCS_MIW+= ObjCopy/wasm/WasmObject.cpp -SRCS_MIW+= ObjCopy/wasm/WasmReader.cpp -SRCS_MIW+= ObjCopy/wasm/WasmWriter.cpp +SRCS_MIN+= ObjCopy/Archive.cpp +SRCS_MIN+= ObjCopy/COFF/COFFObjcopy.cpp +SRCS_MIN+= ObjCopy/COFF/COFFObject.cpp +SRCS_MIN+= ObjCopy/COFF/COFFReader.cpp +SRCS_MIN+= ObjCopy/COFF/COFFWriter.cpp +SRCS_MIN+= ObjCopy/CommonConfig.cpp +SRCS_MIN+= ObjCopy/ConfigManager.cpp +SRCS_MIN+= ObjCopy/ELF/ELFObjcopy.cpp +SRCS_MIN+= ObjCopy/ELF/ELFObject.cpp +SRCS_MIN+= ObjCopy/MachO/MachOLayoutBuilder.cpp +SRCS_MIN+= ObjCopy/MachO/MachOObjcopy.cpp +SRCS_MIN+= ObjCopy/MachO/MachOObject.cpp +SRCS_MIN+= ObjCopy/MachO/MachOReader.cpp +SRCS_MIN+= ObjCopy/MachO/MachOWriter.cpp +SRCS_MIN+= ObjCopy/ObjCopy.cpp +SRCS_MIN+= ObjCopy/XCOFF/XCOFFObjcopy.cpp +SRCS_MIN+= ObjCopy/XCOFF/XCOFFReader.cpp +SRCS_MIN+= ObjCopy/XCOFF/XCOFFWriter.cpp +SRCS_MIN+= ObjCopy/wasm/WasmObjcopy.cpp +SRCS_MIN+= ObjCopy/wasm/WasmObject.cpp +SRCS_MIN+= ObjCopy/wasm/WasmReader.cpp +SRCS_MIN+= ObjCopy/wasm/WasmWriter.cpp SRCS_MIN+= Object/Archive.cpp SRCS_MIN+= Object/ArchiveWriter.cpp SRCS_MIN+= Object/Binary.cpp @@ -1115,7 +1115,7 @@ SRCS_MIN+= Object/IRObjectFile.cpp SRCS_MIN+= Object/IRSymtab.cpp SRCS_MIN+= Object/MachOObjectFile.cpp SRCS_MIN+= Object/MachOUniversal.cpp -SRCS_MIW+= Object/MachOUniversalWriter.cpp +SRCS_MIN+= Object/MachOUniversalWriter.cpp SRCS_MIN+= Object/Minidump.cpp SRCS_MIN+= Object/ModuleSymbolTable.cpp SRCS_EXT+= Object/Object.cpp @@ -1124,12 +1124,12 @@ SRCS_MIN+= Object/OffloadBinary.cpp SRCS_MIW+= Object/OffloadBundle.cpp SRCS_MIN+= Object/RecordStreamer.cpp SRCS_MIN+= Object/RelocationResolver.cpp -SRCS_MIW+= Object/SymbolSize.cpp +SRCS_MIN+= Object/SymbolSize.cpp SRCS_MIN+= Object/SymbolicFile.cpp SRCS_MIN+= Object/TapiFile.cpp SRCS_MIN+= Object/TapiUniversal.cpp SRCS_MIN+= Object/WasmObjectFile.cpp -SRCS_MIW+= Object/WindowsMachineFlag.cpp +SRCS_MIN+= Object/WindowsMachineFlag.cpp SRCS_MIN+= Object/WindowsResource.cpp SRCS_MIN+= Object/XCOFFObjectFile.cpp SRCS_MIN+= ObjectYAML/COFFYAML.cpp @@ -1298,7 +1298,7 @@ SRCS_MIN+= Support/OptimizedStructLayout.cpp SRCS_MIN+= Support/OptionStrCmp.cpp SRCS_MIN+= Support/Optional.cpp SRCS_MIN+= Support/PGOOptions.cpp -SRCS_MIW+= Support/Parallel.cpp +SRCS_MIN+= Support/Parallel.cpp SRCS_MIN+= Support/Path.cpp SRCS_MIN+= Support/PluginLoader.cpp SRCS_MIN+= Support/PrettyStackTrace.cpp @@ -1849,7 +1849,7 @@ SRCS_MIN+= TextAPI/TextStubCommon.cpp SRCS_MIN+= TextAPI/TextStubV5.cpp SRCS_MIN+= TextAPI/Utils.cpp SRCS_MIN+= ToolDrivers/llvm-dlltool/DlltoolDriver.cpp -SRCS_MIW+= ToolDrivers/llvm-lib/LibDriver.cpp +SRCS_MIN+= ToolDrivers/llvm-lib/LibDriver.cpp SRCS_MIN+= Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp SRCS_MIN+= Transforms/AggressiveInstCombine/TruncInstCombine.cpp SRCS_MIN+= Transforms/CFGuard/CFGuard.cpp diff --git a/lib/libc/gen/fts.3 b/lib/libc/gen/fts.3 index 199603b5f3c7..b6dbfffe8079 100644 --- a/lib/libc/gen/fts.3 +++ b/lib/libc/gen/fts.3 @@ -25,7 +25,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd May 21, 2026 +.Dd May 29, 2026 .Dt FTS 3 .Os .Sh NAME @@ -497,6 +497,13 @@ field to and leave the contents of the .Fa statp field undefined. +The roots and any directories encountered during traversal +.Po +.Dv FTS_D , +.Dv FTS_DC , +.Dv FTS_DP +.Pc +are still fully populated. .It Dv FTS_NOSTAT_TYPE This option is similar to .Dv FTS_NOSTAT , diff --git a/lib/libc/gen/fts.c b/lib/libc/gen/fts.c index 4aa386d777cd..e8063ecb646e 100644 --- a/lib/libc/gen/fts.c +++ b/lib/libc/gen/fts.c @@ -41,6 +41,7 @@ #include <fcntl.h> #include <fts.h> #include <stdalign.h> +#include <stdint.h> #include <stdlib.h> #include <string.h> #include <unistd.h> @@ -114,6 +115,19 @@ static const char *ufslike_filesystems[] = { 0 }; +/* + * POSIX provides nlink_t but unfortunately not NLINK_MAX. + */ +#define NLINK_MAX \ + _Generic((nlink_t)0, \ + int16_t: INT16_MAX, \ + uint16_t: UINT16_MAX, \ + int32_t: INT32_MAX, \ + uint32_t: UINT32_MAX, \ + int64_t: INT64_MAX, \ + uint64_t: UINT64_MAX, \ + default: 0) + static FTS * __fts_open(FTS *sp, char * const *argv) { @@ -736,7 +750,7 @@ fts_build(FTS *sp, int type) int cderrno, descend, oflag, saved_errno, nostat, doadjust, readdir_errno; long level; - long nlinks; /* has to be signed because -1 is a magic value */ + int64_t nlinks; /* has to be signed because -1 is a magic value */ size_t dnamlen, len, maxlen, nitems; /* Set current node pointer. */ @@ -759,16 +773,36 @@ fts_build(FTS *sp, int type) } /* - * Nlinks is the number of possible entries of type directory in the - * directory if we're cheating on stat calls, 0 if we're not doing - * any stat calls at all, -1 if we're doing stats on everything. + * In the FTS_PHYSICAL | FTS_NOSTAT case, we want to avoid calling + * fstat() unnecessarily, but we still need to call it for + * subdirectories. The current directory's link count provides an + * upper bound on the number of subdirectories we may encounter + * (including . and .. in the FTS_SEEDOT case). We initialize + * nlinks to the current directory's link count, then decrement it + * every time we encounter a directory, so when we hit zero we can + * save some time by not calling fstat() on subsequent entries. + * + * If FTS_NOSTAT is not set, or the link count is less than two + * (which should not be possible) or equal to NLINK_MAX (which + * suggests that the actual value could be higher), or the current + * filesystem is not known to provide reliable link counts, we + * initialize nlinks to -1 and fstat() everything. + * + * In the rare case where we don't need to stat anything, even + * subdirectories, we initialize nlinks to 0 regardless of the + * actual link count. + * + * Note that we ignore the FTS_NOSTAT flag in the FTS_LOGICAL + * case, although we could choose to only stat symbolic links. + * Implementing this is left as an exercise for the reader. */ if (type == BNAMES) { nlinks = 0; /* Be quiet about nostat, GCC. */ nostat = 0; } else if (ISSET(FTS_NOSTAT) && ISSET(FTS_PHYSICAL)) { - if (fts_ufslinks(sp, cur)) + if (cur->fts_nlink >= 2 && cur->fts_nlink < NLINK_MAX && + cur->fts_nlink <= INT64_MAX && fts_ufslinks(sp, cur)) nlinks = cur->fts_nlink - (ISSET(FTS_SEEDOT) ? 0 : 2); else nlinks = -1; diff --git a/lib/libc/posix1e/acl_id_to_name.c b/lib/libc/posix1e/acl_id_to_name.c index 78e050a8648a..c90e6083cca1 100644 --- a/lib/libc/posix1e/acl_id_to_name.c +++ b/lib/libc/posix1e/acl_id_to_name.c @@ -67,7 +67,7 @@ _posix1e_acl_id_to_name(acl_tag_t tag, uid_t id, ssize_t buf_len, char *buf, else p = getpwuid(id); if (!p) - i = snprintf(buf, buf_len, "%d", id); + i = snprintf(buf, buf_len, "%ju", (uintmax_t)id); else i = snprintf(buf, buf_len, "%s", p->pw_name); @@ -83,7 +83,7 @@ _posix1e_acl_id_to_name(acl_tag_t tag, uid_t id, ssize_t buf_len, char *buf, else g = getgrgid(id); if (g == NULL) - i = snprintf(buf, buf_len, "%d", id); + i = snprintf(buf, buf_len, "%ju", (uintmax_t)id); else i = snprintf(buf, buf_len, "%s", g->gr_name); diff --git a/lib/libc/posix1e/acl_to_text_nfs4.c b/lib/libc/posix1e/acl_to_text_nfs4.c index 157215c9dd52..4f19f3a9a7b2 100644 --- a/lib/libc/posix1e/acl_to_text_nfs4.c +++ b/lib/libc/posix1e/acl_to_text_nfs4.c @@ -69,7 +69,7 @@ format_who(char *str, size_t size, const acl_entry_t entry, int numeric) else pwd = NULL; if (pwd == NULL) - snprintf(str, size, "user:%d", (unsigned int)*id); + snprintf(str, size, "user:%ju", (uintmax_t)*id); else snprintf(str, size, "user:%s", pwd->pw_name); acl_free(id); @@ -89,7 +89,7 @@ format_who(char *str, size_t size, const acl_entry_t entry, int numeric) else grp = NULL; if (grp == NULL) - snprintf(str, size, "group:%d", (unsigned int)*id); + snprintf(str, size, "group:%ju", (uintmax_t)*id); else snprintf(str, size, "group:%s", grp->gr_name); acl_free(id); diff --git a/lib/libc/tests/gen/fts_options_test.c b/lib/libc/tests/gen/fts_options_test.c index fc3015138a49..863c0809d16e 100644 --- a/lib/libc/tests/gen/fts_options_test.c +++ b/lib/libc/tests/gen/fts_options_test.c @@ -36,6 +36,8 @@ fts_options_prepare(const struct atf_tc *tc) ATF_REQUIRE_EQ(0, mkdir("dir", 0755)); ATF_REQUIRE_EQ(0, close(creat("file", 0644))); ATF_REQUIRE_EQ(0, close(creat("dir/file", 0644))); + ATF_REQUIRE_EQ(0, mkdir("dir/sd", 0750)); + ATF_REQUIRE_EQ(0, mkdir("dir/sd/sd", 0700)); ATF_REQUIRE_EQ(0, symlink("..", "dir/up")); ATF_REQUIRE_EQ(0, symlink("dir", "dirl")); ATF_REQUIRE_EQ(0, symlink("file", "filel")); @@ -57,6 +59,10 @@ ATF_TC_BODY(fts_options_logical, tc) { FTS_DL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_F, "file", "dir/file" }, + { FTS_D, "sd", "dir/sd" }, + { FTS_D, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd" }, { FTS_D, "up", "dir/up" }, { FTS_DL, "dead", "dir/up/dead" }, { FTS_DC, "dir", "dir/up/dir" }, @@ -67,6 +73,10 @@ ATF_TC_BODY(fts_options_logical, tc) { FTS_DP, "dir", "dir" }, { FTS_D, "dirl", "dirl" }, { FTS_F, "file", "dirl/file" }, + { FTS_D, "sd", "dirl/sd" }, + { FTS_D, "sd", "dirl/sd/sd" }, + { FTS_DP, "sd", "dirl/sd/sd" }, + { FTS_DP, "sd", "dirl/sd" }, { FTS_D, "up", "dirl/up" }, { FTS_DL, "dead", "dirl/up/dead" }, { FTS_DC, "dir", "dirl/up/dir" }, @@ -108,6 +118,10 @@ ATF_TC_BODY(fts_options_logical_nostat, tc) { FTS_DL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_NSOK, "file", "dir/file" }, + { FTS_D, "sd", "dir/sd" }, + { FTS_D, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd" }, { FTS_D, "up", "dir/up" }, { FTS_DL, "dead", "dir/up/dead" }, { FTS_DC, "dir", "dir/up/dir" }, @@ -118,6 +132,10 @@ ATF_TC_BODY(fts_options_logical_nostat, tc) { FTS_DP, "dir", "dir" }, { FTS_D, "dirl", "dirl" }, { FTS_NSOK, "file", "dirl/file" }, + { FTS_D, "sd", "dirl/sd" }, + { FTS_D, "sd", "dirl/sd/sd" }, + { FTS_DP, "sd", "dirl/sd/sd" }, + { FTS_DP, "sd", "dirl/sd" }, { FTS_D, "up", "dirl/up" }, { FTS_DL, "dead", "dirl/up/dead" }, { FTS_DC, "dir", "dirl/up/dir" }, @@ -151,6 +169,14 @@ ATF_TC_BODY(fts_options_logical_seedot, tc) { FTS_DOT, ".", "dir/." }, { FTS_DOT, "..", "dir/.." }, { FTS_F, "file", "dir/file" }, + { FTS_D, "sd", "dir/sd" }, + { FTS_DOT, ".", "dir/sd/." }, + { FTS_DOT, "..", "dir/sd/.." }, + { FTS_D, "sd", "dir/sd/sd" }, + { FTS_DOT, ".", "dir/sd/sd/." }, + { FTS_DOT, "..", "dir/sd/sd/.." }, + { FTS_DP, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd" }, { FTS_D, "up", "dir/up" }, { FTS_DOT, ".", "dir/up/." }, { FTS_DOT, "..", "dir/up/.." }, @@ -165,6 +191,14 @@ ATF_TC_BODY(fts_options_logical_seedot, tc) { FTS_DOT, ".", "dirl/." }, { FTS_DOT, "..", "dirl/.." }, { FTS_F, "file", "dirl/file" }, + { FTS_D, "sd", "dirl/sd" }, + { FTS_DOT, ".", "dirl/sd/." }, + { FTS_DOT, "..", "dirl/sd/.." }, + { FTS_D, "sd", "dirl/sd/sd" }, + { FTS_DOT, ".", "dirl/sd/sd/." }, + { FTS_DOT, "..", "dirl/sd/sd/.." }, + { FTS_DP, "sd", "dirl/sd/sd" }, + { FTS_DP, "sd", "dirl/sd" }, { FTS_D, "up", "dirl/up" }, { FTS_DOT, ".", "dirl/up/." }, { FTS_DOT, "..", "dirl/up/.." }, @@ -198,6 +232,10 @@ ATF_TC_BODY(fts_options_physical, tc) { FTS_SL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dir", "dir" }, { FTS_SL, "dirl", "dirl" }, @@ -224,6 +262,10 @@ ATF_TC_BODY(fts_options_physical_nochdir, tc) { FTS_SL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_F, "file", "dir/file" }, + { FTS_D, "sd", "dir/sd" }, + { FTS_D, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd/sd" }, + { FTS_DP, "sd", "dir/sd" }, { FTS_SL, "up", "dir/up" }, { FTS_DP, "dir", "dir" }, { FTS_SL, "dirl", "dirl" }, @@ -250,10 +292,18 @@ ATF_TC_BODY(fts_options_physical_comfollow, tc) { FTS_DL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dir", "dir" }, { FTS_D, "dirl", "dirl" }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dirl", "dirl" }, { FTS_F, "file", "file" }, @@ -279,10 +329,18 @@ ATF_TC_BODY(fts_options_physical_comfollowdir, tc) { FTS_DL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dir", "dir" }, { FTS_D, "dirl", "dirl" }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dirl", "dirl" }, { FTS_F, "file", "file" }, @@ -308,6 +366,10 @@ ATF_TC_BODY(fts_options_physical_nostat, tc) { FTS_SL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_NSOK, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_NSOK, "up", "up" }, { FTS_DP, "dir", "dir" }, { FTS_SL, "dirl", "dirl" }, @@ -334,6 +396,10 @@ ATF_TC_BODY(fts_options_physical_nostat_type, tc) { FTS_SL, "dead", "dead" }, { FTS_D, "dir", "dir" }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_D, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dir", "dir" }, { FTS_SL, "dirl", "dirl" }, @@ -362,6 +428,14 @@ ATF_TC_BODY(fts_options_physical_seedot, tc) { FTS_DOT, ".", "." }, { FTS_DOT, "..", ".." }, { FTS_F, "file", "file" }, + { FTS_D, "sd", "sd" }, + { FTS_DOT, ".", "." }, + { FTS_DOT, "..", ".." }, + { FTS_D, "sd", "sd" }, + { FTS_DOT, ".", "." }, + { FTS_DOT, "..", ".." }, + { FTS_DP, "sd", "sd" }, + { FTS_DP, "sd", "sd" }, { FTS_SL, "up", "up" }, { FTS_DP, "dir", "dir" }, { FTS_SL, "dirl", "dirl" }, diff --git a/lib/libsys/fcntl.2 b/lib/libsys/fcntl.2 index d67c38cfbc6c..b919e1b8674b 100644 --- a/lib/libsys/fcntl.2 +++ b/lib/libsys/fcntl.2 @@ -176,7 +176,9 @@ descriptor to also have the flag set. .El .It Dv F_SETFD Set flags associated with -.Fa fd . +.Fa fd +to +.Fa arg . The available flags are .Dv FD_CLOEXEC , .Dv FD_CLOFORK diff --git a/lib/libsys/getsockopt.2 b/lib/libsys/getsockopt.2 index 3867824681d7..85d94e014631 100644 --- a/lib/libsys/getsockopt.2 +++ b/lib/libsys/getsockopt.2 @@ -25,7 +25,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd November 25, 2024 +.Dd April 21, 2026 .Dt GETSOCKOPT 2 .Os .Sh NAME @@ -220,6 +220,10 @@ Incoming TCP and UDP connections are distributed among the participating listening sockets based on a hash function of local port number, and foreign IP address and port number. A maximum of 256 sockets can be bound to the same load-balancing group. +.Dv PF_DIVERT +sockets may also be bound to a group, see the +.Xr divert 4 +manual page for details. .Pp .Dv SO_KEEPALIVE enables the diff --git a/lib/libutil/login.conf.5 b/lib/libutil/login.conf.5 index d4bbc1d67780..9b136291dc1e 100644 --- a/lib/libutil/login.conf.5 +++ b/lib/libutil/login.conf.5 @@ -17,7 +17,7 @@ .\" 5. Modifications may be freely made to this file providing the above .\" conditions are met. .\" -.Dd December 15, 2025 +.Dd May 27, 2026 .Dt LOGIN.CONF 5 .Os .Sh NAME @@ -194,17 +194,18 @@ login environment. .It "cputime time CPU usage limit." .It "datasize size Maximum data size limit." .It "filesize size Maximum file size limit." +.It "kqueues number Maximum number of kernel event queues." .It "maxproc number Maximum number of processes." .It "memorylocked size Maximum locked in core memory size limit." .It "memoryuse size Maximum of core memory use size limit." .It "openfiles number Maximum number of open files per process." +.It "pipebuf size Maximum size of pipe buffers." +.It "pseudoterminals number Maximum number of pseudo-terminals." .It "sbsize size Maximum permitted socketbuffer size." -.It "vmemoryuse size Maximum permitted total VM usage per process." .It "stacksize size Maximum stack size limit." -.It "pseudoterminals number Maximum number of pseudo-terminals." .It "swapuse size Maximum swap space size limit." .It "umtxp number Maximum number of process-shared pthread locks." -.It "pipebuf size Maximum size of pipe buffers." +.It "vmemoryuse size Maximum permitted total VM usage per process." .El .Pp These resource limit entries actually specify both the maximum diff --git a/lib/libutil/login_class.c b/lib/libutil/login_class.c index 90e4e01f7c3b..8a465e2eb24e 100644 --- a/lib/libutil/login_class.c +++ b/lib/libutil/login_class.c @@ -63,9 +63,9 @@ static struct login_res { { "vmemoryuse", login_getcapsize, RLIMIT_VMEM }, { "pseudoterminals", login_getcapnum, RLIMIT_NPTS }, { "swapuse", login_getcapsize, RLIMIT_SWAP }, - { "kqueues", login_getcapsize, RLIMIT_KQUEUES }, + { "kqueues", login_getcapnum, RLIMIT_KQUEUES }, { "umtxp", login_getcapnum, RLIMIT_UMTXP }, - { "pipebuf", login_getcapnum, RLIMIT_PIPEBUF }, + { "pipebuf", login_getcapsize, RLIMIT_PIPEBUF }, { "vms", login_getcapnum, RLIMIT_VMM }, { NULL, 0, 0 } }; diff --git a/libexec/rc/rc.conf b/libexec/rc/rc.conf index 75420e42cdeb..27e8c8456b6f 100644 --- a/libexec/rc/rc.conf +++ b/libexec/rc/rc.conf @@ -736,7 +736,11 @@ newsyslog_flags="-CN" # Newsyslog flags to create marked files mixer_enable="YES" # Run the sound mixer. opensm_enable="NO" # Opensm(8) for infiniband devices defaults to off nuageinit_enable="NO" # Run nuageinit at startup -virtual_oss_enable="NO" # Run virtual_oss at startup + +virtual_oss_enable="NO" # Run virtual_oss at startup. +virtual_oss_configs="dsp" # List of configurations. +virtual_oss_default_control_device="vdsp.ctl" # Default configuration's + # control device. # rctl(8) requires kernel options RACCT and RCTL rctl_enable="YES" # Load rctl(8) rules on boot diff --git a/libexec/rc/rc.d/virtual_oss b/libexec/rc/rc.d/virtual_oss index 73a486f547a5..9861545b8bfc 100644 --- a/libexec/rc/rc.d/virtual_oss +++ b/libexec/rc/rc.d/virtual_oss @@ -22,13 +22,9 @@ status_cmd="${name}_status" required_modules="cuse" -configs= pidpath="/var/run/${name}" -default_unit=$(sysctl -n hw.snd.default_unit 2> /dev/null) - -# Default configuration's control device. -: "${virtual_oss_default_control_device:="vdsp.ctl"}" +default_unit=$(sysctl -n hw.snd.default_unit 2> /dev/null) virtual_oss_default_args="\ -S \ -C 2 \ @@ -42,12 +38,6 @@ virtual_oss_default_args="\ -l dsp.loop \ -t ${virtual_oss_default_control_device}" -# Set to NO by default. Set it to "YES" to enable virtual_oss. -: "${virtual_oss_enable:="NO"}" - -# List of configurations to use. Default is "dsp". -: "${virtual_oss_configs:="dsp"}" - # Default (dsp) virtual_oss config. : "${virtual_oss_dsp:="${virtual_oss_default_args}"}" @@ -86,10 +76,17 @@ stop_instance() if [ -z "${instance_args}" ]; then warn "no such config: ${config}" else - startmsg -n "Stopping virtual_oss config: ${config}: " - kill "$(cat "${pidpath}/${config}.pid")" - rm -f "${pidpath}/${config}.pid" - startmsg "done" + pidfile="${pidpath}/${config}.pid" + if [ ! -f "${pidfile}" ]; then + warn "not running: ${config}" + else + pid="$(cat "${pidfile}")" + startmsg -n "Stopping virtual_oss config: ${config}: " + kill "${pid}" + pwait "${pid}" + rm -f "${pidfile}" + startmsg "done" + fi fi } diff --git a/libexec/talkd/announce.c b/libexec/talkd/announce.c index 80c663ba48b4..cfe8b4eae65e 100644 --- a/libexec/talkd/announce.c +++ b/libexec/talkd/announce.c @@ -40,6 +40,7 @@ #include <errno.h> #include <paths.h> +#include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -152,7 +153,7 @@ print_mesg(const char *tty, CTL_MSG *request, * stack up processes trying to write messages to a tty * that is permanently blocked. */ - if (ttymsg(&iovec, 1, tty, RING_WAIT - 5) != NULL) + if (ttymsg(&iovec, 1, tty, RING_WAIT - 5, true) != 0) return (FAILED); return (SUCCESS); diff --git a/sbin/ping/tests/ping_test.sh b/sbin/ping/tests/ping_test.sh index af700615dc8d..ab45ad809a52 100644 --- a/sbin/ping/tests/ping_test.sh +++ b/sbin/ping/tests/ping_test.sh @@ -253,14 +253,22 @@ inject_reply_cleanup() ifconfig `cat tun.txt` destroy } -atf_test_case timestamp_origin +atf_test_case timestamp_origin cleanup timestamp_origin_head() { atf_set "descr" "ICMP Originate Timestamp" + atf_set "require.user" "root" + atf_set "require.config" "allow_sysctl_side_effects" } timestamp_origin_body() { require_ipv4 + # The kernel only replies to ICMP timestamp requests when + # net.inet.icmp.tstamprepl is enabled. Save the current value + # so the cleanup hook can restore it, then enable replies. + sysctl -n net.inet.icmp.tstamprepl > tstamprepl.txt + sysctl net.inet.icmp.tstamprepl=1 + # Run ping timestamp out=$(ping -Mt -c1 127.0.0.1) @@ -286,6 +294,12 @@ timestamp_origin_body() atf_fail "tso ($tso) differs from tsr ($tsr) by $diff seconds" fi } +timestamp_origin_cleanup() +{ + if [ -f tstamprepl.txt ]; then + sysctl net.inet.icmp.tstamprepl=`cat tstamprepl.txt` + fi +} atf_init_test_cases() { diff --git a/sbin/route/route_netlink.c b/sbin/route/route_netlink.c index 051662688047..0d4420767082 100644 --- a/sbin/route/route_netlink.c +++ b/sbin/route/route_netlink.c @@ -950,7 +950,8 @@ flushroutes_fib_nl(int fib, int af) struct snl_msg_info attrs = {}; print_nlmsg(&h, hdr, &attrs); } - if (r.rta_table != (uint32_t)fib || r.rtm_family != af) + if (r.rta_table != (uint32_t)fib || + (af != AF_UNSPEC && r.rtm_family != af)) continue; if ((r.rta_rtflags & RTF_GATEWAY) == 0) continue; diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile index 0b94158659fb..4b9740e12081 100644 --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -903,6 +903,7 @@ _ntb_hw_intel.4= ntb_hw_intel.4 _ntb_hw_plx.4= ntb_hw_plx.4 _ntb_transport.4=ntb_transport.4 _nvram.4= nvram.4 +_padlock.4= padlock.4 _pchtherm.4= pchtherm.4 _qat.4= qat.4 _qat_c2xxx.4= qat_c2xxx.4 @@ -949,10 +950,6 @@ _vmm.4= vmm.4 .endif .endif -.if ${MACHINE_CPUARCH} == "i386" -_padlock.4= padlock.4 -.endif - .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "aarch64" _hwt.4= hwt.4 .if ${MACHINE_CPUARCH} == "amd64" diff --git a/share/man/man4/acpi.4 b/share/man/man4/acpi.4 index 91193d899200..27e7d7ba414d 100644 --- a/share/man/man4/acpi.4 +++ b/share/man/man4/acpi.4 @@ -23,7 +23,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd March 21, 2026 +.Dd May 27, 2026 .Dt ACPI 4 .Os .Sh NAME @@ -123,7 +123,8 @@ Some newer systems require use of this register, while some only work with legacy rebooting support. .It Va hw.acpi.lid_switch_state Sleep type -.Pq Li awake , Li standby , Li s2mem , Li s2idle , Li hibernate , Li poweroff +.Pq Li awake , Li standby , Li fw_suspend , Li suspend_to_idle , \ +Li fw_hibernate , Li poweroff to enter when the lid switch (i.e., a notebook screen) is closed, or .Dq Li NONE .Pq do nothing . @@ -131,7 +132,8 @@ Default is .Dq Li NONE . .It Va hw.acpi.power_button_state Sleep type -.Pq Li awake , Li standby , Li s2mem , Li s2idle , Li hibernate , Li poweroff +.Pq Li awake , Li standby , Li fw_suspend , Li suspend_to_idle , \ +Li fw_hibernate , Li poweroff to enter when the power button is pressed, or .Dq Li NONE .Pq do nothing . @@ -152,13 +154,14 @@ Most current systems do not support .Li S4BIOS . .It Va hw.acpi.sleep_button_state Sleep type -.Pq Li awake , Li standby , Li s2mem , Li s2idle , Li hibernate , Li poweroff +.Pq Li awake , Li standby , Li fw_suspend , Li suspend_to_idle , \ +Li fw_hibernate , Li poweroff to enter when the sleep button is pressed. This is usually a special function button on the keyboard. Default is usually -.Li s2mem +.Li fw_suspend if supported, and -.Li s2idle +.Li suspend_to_idle if not. .It Va hw.acpi.sleep_delay Wait this number of seconds between preparing the system to suspend and diff --git a/share/man/man4/divert.4 b/share/man/man4/divert.4 index 647bb72ab49b..6292df190eb1 100644 --- a/share/man/man4/divert.4 +++ b/share/man/man4/divert.4 @@ -1,5 +1,4 @@ -.\" -.Dd January 23, 2026 +.Dd April 21, 2026 .Dt DIVERT 4 .Os .Sh NAME @@ -57,7 +56,26 @@ firewall processing at the next rule. .Pp By reading from and writing to a divert socket, matching packets can be passed through an arbitrary ``filter'' as they travel through -the host machine, special routing tricks can be done, etc. +the host machine, special routing tricks can be done, etc.. +.Pp +Multiple divert sockets may be bound to the same port if the +.Dv SO_REUSEPORT_LB +socket option is set on all of them. +In this case, the kernel will attempt to load-balance packets among +the sockets. +The implementation ensures that packets from the same flow are delivered +to the same socket. +To this end it relies on the firewall to provide a flow identifier with +each diverted packet. +When using the +.Xr pf 4 +firewall, this is the associated state ID, if one exists, otherwise all +packets are diverted to the first socket in the group. +Currently the +.Xr ipfw 4 +firewall does not provide a flow identifier, so all packets are diverted +to the first socket in the group. +At most 32 sockets can be bound to the same port. .Sh READING PACKETS Packets are diverted either as they are ``incoming'' or ``outgoing.'' Incoming packets are diverted after reception on an IP interface, diff --git a/share/man/man4/linuxkpi_wlan.4 b/share/man/man4/linuxkpi_wlan.4 index 65c77d8d7631..fa0b15b5e0b1 100644 --- a/share/man/man4/linuxkpi_wlan.4 +++ b/share/man/man4/linuxkpi_wlan.4 @@ -6,7 +6,7 @@ .\" This documentation was written by Bj\xc3\xb6rn Zeeb under sponsorship from .\" the FreeBSD Foundation. .\" -.Dd December 28, 2025 +.Dd May 23, 2026 .Dt LINUXKPI_WLAN 4 .Os .Sh NAME @@ -107,6 +107,12 @@ debug messages. See .Pa sys/compat/linuxkpi/common/src/linux_80211.h for details. +.It Va compat.linuxkpi.80211.suspend_type +For the time being this variable allows suspend/resume to be +enabled/disabled. +The default is 1 which enables normal suspend/resume. +To disable any suspend/resume set it to 0. +Other values may enable specific features in the future. .It Va compat.linuxkpi.80211.IF.dump_stas Print statistics for a given, associated .Xr wlan 4 diff --git a/share/man/man4/mac_do.4 b/share/man/man4/mac_do.4 index 8c08e072be88..7f05d5f88bf8 100644 --- a/share/man/man4/mac_do.4 +++ b/share/man/man4/mac_do.4 @@ -1,8 +1,8 @@ .\"- .\" SPDX-License-Identifier: BSD-2-Clause .\" -.\" Copyright (c) 2024 Baptiste Daroussin <bapt@FreeBSD.org> -.\" Copyright (c) 2024 The FreeBSD Foundation +.\" Copyright (c) 2024, Baptiste Daroussin <bapt@FreeBSD.org> +.\" Copyright (c) 2024, 2026, The FreeBSD Foundation .\" .\" Portions of this documentation were written by Olivier Certner .\" <olce@FreeBSD.org> at Kumacom SARL under sponsorship from the FreeBSD @@ -42,13 +42,23 @@ policy module allows unprivileged users to change process credentials according to rules configured by the administrator. It supports per-jail configuration. .Pp -Currently, the +The .Nm -policy module only produces effects to processes spawned from the +policy module only produces effects on processes spawned from specific +executables from a configurable whitelist. +By default, this whitelist only contains the .Pa /usr/bin/mdo executable, please see .Xr mdo 1 for more details on this program. +.Pp +Section +.Sx CREDENTIALS RULES +specifies the format of credentials transition rules, and section +.Sx CONFIGURATION +explains how to configure +.Nm , +including rules and authorized executables. .Sh CREDENTIALS RULES Rules specify which transitions of process credentials .Nm @@ -267,84 +277,170 @@ then converted to unsigned ones as specified in the C standard for the and .Vt gid_t types, which are both 32-bit unsigned integers. -.Sh RUNTIME CONFIGURATION -The following +.Sh CONFIGURATION +Each parameter of +.Nm +has a corresponding +.Xr sysctl 8 +knob. +.Nm +supports per-jail values for most parameters. +.Ss Sysctl Knobs +The .Xr sysctl 8 -knobs are available: +knobs presented by +.Nm +are of two types: Global ones, which influence all uses of the module, and +per-jail ones, which allow to retrieve or change the setting applicable to +a jail +.Pq or the host +from within. +They are tagged accordingly in the list below. +.Pp +The indicated default values for per-jail parameters applies to the host. +Those applying to jails are described in the +.Sx Jail Parameters +subsection below. +.Pp +The following knobs are available: .Bl -tag -width indent .It Va security.mac.do.enabled Enable the .Nm policy. -(Default: 1). -.It Va security.mac.do.rules -The list of credential rules, whose syntax is described in the -.Sx CREDENTIALS RULES -section above. -This list is specific to each jail. -Please see the -.Sx JAIL SUPPORT -section below for more details on the interaction of -.Nm -with jails. +.Pq Global. Default: 1. .It Va security.mac.do.print_parse_error Logs a message on trying to set incorrect rules via the .Va security.mac.do.rules .Xr sysctl 8 knob. +.Pq Global. Default: 1. +.It Va security.mac.do.rules +The list of credentials transition rules, whose syntax is described in the +.Sx CREDENTIALS RULES +section above. +An empty string effectively disables the policy. +.Pq Per-jail. Default: Empty. +.It Va security.mac.do.exec_paths +The list of absolute paths +.Pq relative to the current jail's root +to authorized executables, separated by colons +.Pq Ql ":" . +Only processes launched from these executables are considered by the +.Nm +policy. +An empty string effectively disables the policy. +.Po +Per-jail. Default: +.Ql "/usr/bin/mdo" . +.Pc .El -.Sh JAIL SUPPORT +.Ss Jail Parameters +Most parameters of .Nm -supports per-jail configuration of rules. +are per-jail +.Po +see the +.Sx "Sysctl Knobs" +subsection above +.Pc . +Those that are per-jail have corresponding jail parameters that can be used to +set their initial values on jail creation or modify their values in a running +jail from outside the jail. .Pp -By default, at creation, a new jail has no credentials rules, effectively -disabling +By default, as it is for the host, a new jail has .Nm -for its processes. +disabled for its processes, as if the +.Va mac.do +parameter below was explicitly set to +.Ql disable . .Pp -The following jail parameters are defined: +Each unspecified parameter other than +.Va mac.do +defaults to a copy of its value in the currently applicable configuration for +a running jail, or from the parent jail on jail creation. +.Pp +The following jail parameters are available: .Bl -tag -width indent .It Va mac.do Possible values are: +.Pp .Bl -tag -width "'disable'" -compact .It Ql new .Nm -will enforce specific credential rules in the jail. -The -.Va mac.do.rules -jail parameter must also be set in this case. +will use a specific configuration for the jail. +This case degrades to +.Ql disable +if one of the other jail parameters end up empty after applying the default +values rule +.Pq see the preamble . .It Ql disable Disables .Nm in the jail. -Strictly equivalent to jail creation's default behavior and to setting the rules -to an empty string. +This is achieved by ensuring that at least one of the +.Va mac.do.rules +and +.Va mac.do.exec_paths +jail parameters are empty. +If none of them ends up empty after applying the default values rule +.Pq see the preamble , +.Va mac.do.rules +is forced to an empty value. +An explicit +.Ql disable +is incompatible with explicit non-empty values for all other jail parameters and +will trigger an error. .It Ql inherit -The jail's credentials rules are inherited from the jail's parent +The other per-jail parameters are inherited from those applicable to the jail's +parent .Pq which may themselves have been inherited . -Modified rules propagate to all children jails configured for inheritance. +Configuration modifications immediately propagate to all descendant jails +configured for inheritance +.Po +as long as there is no intervening jail having its own specific configuration +.Pc . .El +.Pp +The default value depends on the absence or presence and value of the other jail +parameters. +As soon as one of the latter is present and not empty, the default value is +.Ql new , +else it is +.Ql disable . +Inheritance is never established implicitly, it must be explicitly requested. .It Va mac.do.rules The credentials rules for the jail. -It is always equal to the value that can be retrieved by the +See the description of the corresponding .Xr sysctl 8 knob .Va security.mac.do.rules -described in section -.Sx RUNTIME CONFIGURATION . -If set, and the jail parameter -.Va mac.do -is not so explicitly, the value of the latter will default to -.Ql disable -if empty, else to -.Ql new . +in subsection +.Sx "Sysctl Knobs" +above. +See also the preamble for the default values rule. +.It Va mac.do.exec_paths +The authorized executables. +See the description of the corresponding +.Xr sysctl 8 +knob +.Va security.mac.do.exec_paths +in subsection +.Sx "Sysctl Knobs" +above. +See also the preamble for the default values rule. .El +.Ss Consistency +Values read or set from jail parameters are always consistent with their +corresponding +.Xr sysctl 8 +knobs, effectively operating on the same internal +.Dq variable . .Pp -Each jail must have -.Xr mdo 1 -installed at path -.Pa /usr/bin/mdo , -as this path is currently not configurable. +All accesses to some parameter or some jail configuration as a whole, whether +a read or a modification, are sequentially consistent. +In other words, they appear to be atomic, and all threads see them happening in +the same order. .Sh EXAMPLES Here are several examples of single rules matching processes having a real user ID of 10001: @@ -405,6 +501,7 @@ current supplementary groups must be kept. .Sh AUTHORS .An Olivier Certner Aq Mt olce@FreeBSD.org .An Baptiste Daroussin Aq Mt bapt@FreeBSD.org +.An Kushagra Srivastava Aq Mt kushagra1403@gmail.com .Sh BUGS Currently, .Nm diff --git a/share/man/man4/pcm.4 b/share/man/man4/pcm.4 index 4e70d95d5441..9ea0f14c3930 100644 --- a/share/man/man4/pcm.4 +++ b/share/man/man4/pcm.4 @@ -197,9 +197,6 @@ The Parametric Software Equalizer (EQ) enables the use of controls (bass and treble). Commonly used for ear-candy or frequency compensation due to the vast difference in hardware quality. -EQ is disabled by default, but can be enabled with the -.Va hint.pcm.%d.eq -tunable. .Ss VCHANs Each device can optionally support more playback and recording channels than physical hardware provides by using @@ -230,12 +227,6 @@ driver. The following tunables can not be changed during runtime using .Xr sysctl 8 . .Bl -tag -width indent -.It Va hint.pcm.%d.eq -Set to 1 or 0 to explicitly enable (1) or disable (0) the equalizer. -Requires a driver reload if changed. -Enabling this will make bass and treble controls appear in mixer applications. -This tunable is undefined by default. -Equalizing is disabled by default. .It Va hint.pcm.%d.vpc Set to 1 or 0 to explicitly enable (1) or disable (0) the VPC feature. This tunable is undefined by default. @@ -416,15 +407,6 @@ When a channel is closed the channel volume will be reset to 0db. This means that any changes to the volume will be lost. Enabling this will preserve the volume, at the cost of possible confusion when applications tries to re-open the same device. -.It Va hw.snd.vpc_mixer_bypass -The recommended way to use the VPC feature is to teach applications to use the -correct -.Fn ioctl : -.Dv SNDCTL_DSP_GETPLAYVOL , SNDCTL_DSP_SETPLAYVOL , -.Dv SNDCTL_DSP_SETRECVOL , SNDCTL_DSP_SETRECVOL . -This is however not always possible. -Enable this to allow applications to use their own existing mixer logic -to control their own channel volume. .It Va hw.snd.vpc_reset Enable to restore all channel volumes back to the default value of 0db. .It Va dev.pcm.%d.bitperfect @@ -439,6 +421,17 @@ the definitive format/rate target. The recommended way to use bitperfect mode is to disable VCHANs and enable this sysctl. Default is disabled. +.It Va dev.pcm.%d.eq +Set to 1 or 0 to enable (1) or disable (0) the equalizer. +Default is disabled. +When enabled, the mixer bass and treble controls can be used, as well as the +.Va dev.pcm.%d.eq_preamp +tunable. +.It Va dev.pcm.%d.eq_preamp +Equalizer preamp (in dB). +.Va dev.pcm.%d.eq +has to be enabled, in order for this tunable to take effect. +Read the tunable's description to see the available range of values. .It Va dev.pcm.%d.[play|rec].vchans Enable (1) or disable (0) VCHANs. Default is enabled. diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile index 7044b6f1bb68..894a9971295c 100644 --- a/share/man/man9/Makefile +++ b/share/man/man9/Makefile @@ -943,6 +943,7 @@ MLINKS+=cpu_machdep.9 cpu_copy_thread.9 \ cpu_machdep.9 cpu_thread_clean.9 \ cpu_machdep.9 cpu_thread_exit.9 \ cpu_machdep.9 cpu_thread_free.9 \ + cpu_machdep.9 cpu_thread_new_kstack.9 \ cpu_machdep.9 cpu_throw.9 \ cpu_machdep.9 cpu_update_pcb.9 MLINKS+=cpuset.9 CPUSET_T_INITIALIZER.9 \ diff --git a/share/man/man9/cpu_machdep.9 b/share/man/man9/cpu_machdep.9 index 415d86a8b766..514f3f0104cc 100644 --- a/share/man/man9/cpu_machdep.9 +++ b/share/man/man9/cpu_machdep.9 @@ -8,7 +8,7 @@ .\" Technology), and Capabilities Limited under Defense Advanced Research .\" Projects Agency (DARPA) Contract No. FA8750-24-C-B047 ("DEC"). .\" -.Dd January 31, 2025 +.Dd May 27, 2026 .Dt cpu_machdep 9 .Os .Sh NAME @@ -31,6 +31,7 @@ .Nm cpu_thread_clean , .Nm cpu_thread_exit , .Nm cpu_thread_free , +.Nm cpu_thread_new_kstack , .Nm cpu_throw , .Nm cpu_update_pcb .Nd machine-dependent interfaces to handle CPU and thread state @@ -84,6 +85,8 @@ .Ft void .Fn cpu_thread_free "struct thread *td" .Ft void +.Fn cpu_thread_new_kstack "struct thread *td" +.Ft void .Fn cpu_throw "struct thread *old" "struct thread *new" .Ft void .Fn cpu_update_pcb "struct thread *td" @@ -366,19 +369,25 @@ When the process object is later reused for a new process in .Xr fork 2 , the kernel recycles that last thread object and uses it as the initial thread in the new process. -When a thread is recycled, some of the steps in the thread allocation -and free cycle are skipped as an optimization. +When a thread is recycled, a new kernel stack may be allocated if +the existing kernel stack is not suitable for the new process. .Pp .Fn cpu_thread_alloc initializes machine-dependent fields in .Fa td +when allocating a new thread object. +.Pp +.Fn cpu_thread_new_kstack +initializes kernel stack-related machine-dependent fields in +.Fa td after allocating a new kernel stack. This function typically sets the .Fa td_pcb +.Pq on architectures which store the pcb in the kernel stack and initial .Fa td_frame pointers. -.Fn cpu_thread_alloc +.Fn cpu_thread_new_kstack is called both when allocating a new thread object and when a recycled thread allocates a new kernel stack. Note that this function is @@ -386,12 +395,19 @@ Note that this function is called if a recycled thread reuses its existing kernel stack. .Pp .Fn cpu_thread_clean -releases any machine-dependent resources for the last thread in a +releases machine-dependent resources for the last thread in a process during .Xr wait 2 . -The thread is a candidate for recycling so should be reset to run as a +Since the thread is a candidate for recycling, +machine-dependent fields should be reset to run as a new thread in case it is recycled by a future .Xr fork 2 . +In particular, +if the thread reuses its existing kernel stack, +no other +.Fn cpu_thread_* +function will be invoked before the thread is reused as the main +thread of a new process. .Pp .Fn cpu_thread_exit cleans any machine-dependent state in diff --git a/share/mk/bsd.sys.mk b/share/mk/bsd.sys.mk index 93e7c8572310..4ffac401eb4c 100644 --- a/share/mk/bsd.sys.mk +++ b/share/mk/bsd.sys.mk @@ -92,8 +92,6 @@ CWARNFLAGS.clang+= -Wno-error=unused-but-set-parameter .if ${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 190000 # Similar to gcc >= 8.1 -Wno-error=cast-function-type below CWARNFLAGS.clang+= -Wno-error=cast-function-type-mismatch -.endif -.if ${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 210000 CXXWARNFLAGS.clang+= -Wno-c++20-extensions CXXWARNFLAGS.clang+= -Wno-c++23-lambda-attributes CXXWARNFLAGS.clang+= -Wno-nullability-completeness diff --git a/stand/defs.mk b/stand/defs.mk index 7f3803c7ab42..ac4f38451111 100644 --- a/stand/defs.mk +++ b/stand/defs.mk @@ -78,7 +78,7 @@ LIBSA= ${BOOTOBJ}/libsa/libsa.a .if ${MACHINE} == "i386" LIBSA32= ${LIBSA} .else -LIBSA32= ${BOOTOBJ}/libsa32/libsa32.a +LIBSA32= ${BOOTOBJ}/${"${LOADER}" == "loader_ia32":?efi/libsa32efi:libsa32}/libsa32.a .endif # Standard options: diff --git a/stand/efi/Makefile b/stand/efi/Makefile index 1887b9536a5b..5ece7eff8ef2 100644 --- a/stand/efi/Makefile +++ b/stand/efi/Makefile @@ -3,12 +3,18 @@ NO_OBJ=t .include <bsd.init.mk> SUBDIR.yes+= libefi -SUBDIR.${MK_LOADER_IA32}+= libefi32 +SUBDIR.${MK_LOADER_IA32}+= libsa32efi libefi32 SUBDIR.${MK_FDT}+= fdt SUBDIR.yes+= .WAIT SUBDIR.yes+= boot1 gptboot +.if ${LOADER_DEFAULT_INTERP} == "lua" +SUBDIR.${MK_LOADER_IA32}+= liblua32efi +.elif ${LOADER_DEFAULT_INTERP} == "4th" +SUBDIR.${MK_LOADER_IA32}+= ficl32efi +.endif + SUBDIR.${MK_FORTH}+= loader_4th SUBDIR.${MK_LOADER_LUA}+= loader_lua SUBDIR.${MK_LOADER_IA32}+= loader_ia32 diff --git a/stand/efi/Makefile.inc b/stand/efi/Makefile.inc index 6bb0af1d84f3..452820aeeb32 100644 --- a/stand/efi/Makefile.inc +++ b/stand/efi/Makefile.inc @@ -8,6 +8,11 @@ LDFLAGS+= -nostdlib .if ${MACHINE_CPUARCH} == "amd64" CFLAGS+= -fshort-wchar CFLAGS+= -mno-red-zone +.if ${DO32:U0} == 1 +# This is needed so the EDK2 EFI structs have the correct layout for +# the ia32 loader. +CFLAGS+= -malign-double +.endif .endif .if ${MACHINE_CPUARCH} == "aarch64" diff --git a/stand/efi/ficl32efi/Makefile b/stand/efi/ficl32efi/Makefile new file mode 100644 index 000000000000..63800cb0f209 --- /dev/null +++ b/stand/efi/ficl32efi/Makefile @@ -0,0 +1,3 @@ +DO32=1 + +.include "${.CURDIR}/../../ficl/Makefile" diff --git a/stand/efi/liblua32efi/Makefile b/stand/efi/liblua32efi/Makefile new file mode 100644 index 000000000000..cf391967ecf0 --- /dev/null +++ b/stand/efi/liblua32efi/Makefile @@ -0,0 +1,3 @@ +DO32=1 + +.include "${.CURDIR}/../../liblua/Makefile" diff --git a/stand/efi/libsa32efi/Makefile b/stand/efi/libsa32efi/Makefile new file mode 100644 index 000000000000..109c420bf734 --- /dev/null +++ b/stand/efi/libsa32efi/Makefile @@ -0,0 +1 @@ +.include "${.CURDIR}/../../libsa32/Makefile" diff --git a/stand/i386/pxeldr/pxeboot.8 b/stand/i386/pxeldr/pxeboot.8 index 496b244cf00f..aa6a5d5c5f7f 100644 --- a/stand/i386/pxeldr/pxeboot.8 +++ b/stand/i386/pxeldr/pxeboot.8 @@ -130,6 +130,14 @@ In all other respects, acts just like .Xr loader 8 . .Pp +.Nm +requires NFS over UDP. +Many recent distributions of Linux only serve NFS over TCP. +Enable UDP in some distributions by uncommenting +.Ql udp=y +in +.Pa /etc/nfs.conf . +.Pp For further information on Intel's PXE specifications and Wired for Management (WfM) systems, see .Li http://www.pix.net/software/pxeboot/archive/pxespec.pdf . diff --git a/stand/libsa/rpc.c b/stand/libsa/rpc.c index 6b11282a10be..bc5412f1efeb 100644 --- a/stand/libsa/rpc.c +++ b/stand/libsa/rpc.c @@ -415,6 +415,11 @@ rpc_getport(struct iodesc *d, n_long prog, n_long vers) return (-1); } port = (int)ntohl(res->port); + if (port == 0) { + printf("Portmapper returned 0. TCP-only NFS server?\n"); + free(pkt); + return (-1); + } free(pkt); rpc_pmap_putcache(d->destip, prog, vers, port); diff --git a/stand/loader.mk b/stand/loader.mk index 496252e7a534..0d94ed3e6263 100644 --- a/stand/loader.mk +++ b/stand/loader.mk @@ -162,14 +162,14 @@ LIBFICL= ${BOOTOBJ}/ficl/libficl.a .if ${MACHINE} == "i386" LIBFICL32= ${LIBFICL} .else -LIBFICL32= ${BOOTOBJ}/ficl32/libficl.a +LIBFICL32= ${BOOTOBJ}/${"${LOADER}" == "loader_ia32":?efi/ficl32efi:ficl32}/libficl.a .endif LIBLUA= ${BOOTOBJ}/liblua/liblua.a .if ${MACHINE} == "i386" LIBLUA32= ${LIBLUA} .else -LIBLUA32= ${BOOTOBJ}/liblua32/liblua.a +LIBLUA32= ${BOOTOBJ}/${"${LOADER}" == "loader_ia32":?efi/liblua32efi:liblua32}/liblua.a .endif CLEANFILES+= vers.c diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index e338db372df3..8df4868f5312 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -1367,8 +1367,9 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) thread0.td_kstack = (char *)physfree - kernphys + KERNSTART; thread0.td_kstack_pages = kstack_pages; - kstack0_sz = thread0.td_kstack_pages * PAGE_SIZE; + kstack0_sz = ptoa(kstack_pages); bzero(thread0.td_kstack, kstack0_sz); + cpu_thread_new_kstack(&thread0); physfree += kstack0_sz; /* @@ -1521,8 +1522,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) * We initialize the PCB pointer early so that exception * handlers will work. */ - cpu_max_ext_state_size = sizeof(struct savefpu); - set_top_of_stack_td(&thread0); thread0.td_pcb = get_pcb_td(&thread0); /* @@ -1828,29 +1827,53 @@ wrmsr_early_safe_start(void) { struct region_descriptor efi_idt; struct gate_descriptor *gpf_descr; + int i; sidt(&wrmsr_early_safe_orig_efi_idt); efi_idt.rd_limit = 32 * sizeof(idt0[0]); efi_idt.rd_base = (uintptr_t)idt0; lidt(&efi_idt); - gpf_descr = &idt0[IDT_GP]; - gpf_descr->gd_looffset = (uintptr_t)wrmsr_early_safe_gp_handler; - gpf_descr->gd_hioffset = (uintptr_t)wrmsr_early_safe_gp_handler >> 16; - gpf_descr->gd_selector = rcs(); - gpf_descr->gd_type = SDT_SYSTGT; - gpf_descr->gd_p = 1; + /* Setup handler for all possible exceptions. */ + for (i = 0; i < 32; i++) { + gpf_descr = &idt0[i]; + gpf_descr->gd_looffset = + (uintptr_t)wrmsr_early_safe_gp_handler; + gpf_descr->gd_hioffset = + (uintptr_t)wrmsr_early_safe_gp_handler >> 16; + gpf_descr->gd_selector = rcs(); + gpf_descr->gd_type = SDT_SYSTGT; + gpf_descr->gd_p = 1; + } } void wrmsr_early_safe_end(void) { - struct gate_descriptor *gpf_descr; + int i; lidt(&wrmsr_early_safe_orig_efi_idt); - gpf_descr = &idt0[IDT_GP]; - memset_early(gpf_descr, 0, sizeof(*gpf_descr)); + for (i = 0; i < 32; i++) + memset_early(&idt0[i], 0, sizeof(idt0[0])); +} + +int +safe_read(vm_offset_t addr, char *valp) +{ + struct uio uio; + struct iovec iov; + + iov.iov_base = valp; + iov.iov_len = 1; + uio.uio_offset = addr; + uio.uio_iov = &iov; + uio.uio_iovcnt = 1; + uio.uio_resid = 1; + uio.uio_segflg = UIO_SYSSPACE; + uio.uio_rw = UIO_READ; + uio.uio_td = NULL; + return (uiomove_mem(UIO_MEM_KMEM, &uio)); } #ifdef KDB diff --git a/sys/amd64/amd64/mem.c b/sys/amd64/amd64/mem.c index ab1e6cde6cd5..7d1f0f42d01c 100644 --- a/sys/amd64/amd64/mem.c +++ b/sys/amd64/amd64/mem.c @@ -61,10 +61,6 @@ #include <machine/specialreg.h> #include <machine/vmparam.h> -#include <vm/vm.h> -#include <vm/pmap.h> -#include <vm/vm_extern.h> - #include <machine/memdev.h> /* @@ -72,99 +68,22 @@ */ MALLOC_DEFINE(M_MEMDESC, "memdesc", "memory range descriptors"); -/* ARGSUSED */ int memrw(struct cdev *dev, struct uio *uio, int flags) { - struct iovec *iov; - void *p, *vd; - ssize_t orig_resid; - vm_prot_t prot; - u_long v; - u_int c; - int error; - - error = 0; - orig_resid = uio->uio_resid; - while (uio->uio_resid > 0 && error == 0) { - iov = uio->uio_iov; - if (iov->iov_len == 0) { - uio->uio_iov++; - uio->uio_iovcnt--; - if (uio->uio_iovcnt < 0) - panic("memrw"); - continue; - } - v = uio->uio_offset; - c = ulmin(iov->iov_len, PAGE_SIZE - (u_int)(v & PAGE_MASK)); - - switch (dev2unit(dev)) { - case CDEV_MINOR_KMEM: - /* - * Since c is clamped to be less or equal than - * PAGE_SIZE, the uiomove() call does not - * access past the end of the direct map. - */ - if (v >= kva_layout.dmap_low && - v < kva_layout.dmap_high) { - error = uiomove((void *)v, c, uio); - break; - } - - switch (uio->uio_rw) { - case UIO_READ: - prot = VM_PROT_READ; - break; - case UIO_WRITE: - prot = VM_PROT_WRITE; - break; - } + enum uiomove_mem_req req; - if (!kernacc((void *)v, c, prot)) { - error = EFAULT; - break; - } - - /* - * If the extracted address is not accessible - * through the direct map, then we make a - * private (uncached) mapping because we can't - * depend on the existing kernel mapping - * remaining valid until the completion of - * uiomove(). - * - * XXX We cannot provide access to the - * physical page 0 mapped into KVA. - */ - v = pmap_extract(kernel_pmap, v); - if (v == 0) { - error = EFAULT; - break; - } - /* FALLTHROUGH */ - case CDEV_MINOR_MEM: - if (v < dmaplimit) { - vd = PHYS_TO_DMAP(v); - error = uiomove(vd, c, uio); - break; - } - if (v > cpu_getmaxphyaddr()) { - error = EFAULT; - break; - } - p = pmap_mapdev(v, PAGE_SIZE); - error = uiomove(p, c, uio); - pmap_unmapdev(p, PAGE_SIZE); - break; - } + switch (dev2unit(dev)) { + case CDEV_MINOR_KMEM: + req = UIO_MEM_KMEM; + break; + case CDEV_MINOR_MEM: + req = UIO_MEM_MEM; + break; + default: + __unreachable(); } - /* - * Don't return error if any byte was written. Read and write - * can return error only if no i/o was performed. - */ - if (uio->uio_resid != orig_resid) - error = 0; - return (error); + return (uiomove_mem(req, uio)); } /* diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S index 09d4ef85b087..1d7d05843ba8 100644 --- a/sys/amd64/amd64/support.S +++ b/sys/amd64/amd64/support.S @@ -1566,20 +1566,19 @@ msr_onfault: ret ENTRY(wrmsr_early_safe) + movq %rsp,%r11 movl %edi,%ecx movl %esi,%eax sarq $32,%rsi movl %esi,%edx wrmsr xorl %eax,%eax -wrmsr_early_faulted: ret ENTRY(wrmsr_early_safe_gp_handler) - addq $8,%rsp + movq %r11,%rsp movl $EFAULT,%eax - movq $wrmsr_early_faulted,(%rsp) - iretq + ret /* * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index fb18b7d06f9e..df9cc44bcbb2 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -944,13 +944,9 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str) { int code; u_int type; - struct soft_segment_descriptor softseg; - struct user_segment_descriptor *gdt; code = frame->tf_err; type = frame->tf_trapno; - gdt = *PCPU_PTR(gdt); - sdtossd(&gdt[IDXSEL(frame->tf_cs)], &softseg); printf("\n%s trap %d: %s while in %s mode\n", type_str, type, type < nitems(trap_msg) ? trap_msg[type] : UNKNOWN, @@ -975,11 +971,6 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str) frame->tf_rsp); printf("frame pointer = %#hx:%#lx\n", frame->tf_ss, frame->tf_rbp); - printf("code segment = base 0x%lx, limit 0x%lx, type 0x%x\n", - softseg.ssd_base, softseg.ssd_limit, softseg.ssd_type); - printf(" = DPL %d, pres %d, long %d, def32 %d, gran %d\n", - softseg.ssd_dpl, softseg.ssd_p, softseg.ssd_long, softseg.ssd_def32, - softseg.ssd_gran); printf("processor eflags = "); if (frame->tf_rflags & PSL_T) printf("trace trap, "); @@ -990,8 +981,9 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str) if (frame->tf_rflags & PSL_RF) printf("resume, "); printf("IOPL = %ld\n", (frame->tf_rflags & PSL_IOPL) >> 12); - printf("current process = %d (%s)\n", - curproc->p_pid, curthread->td_name); + printf("current thread = %d/%d (%s/%s)\n", + curproc->p_pid, curthread->td_tid, curproc->p_comm, + curthread->td_name); printf("rdi: %016lx rsi: %016lx rdx: %016lx\n", frame->tf_rdi, frame->tf_rsi, frame->tf_rdx); @@ -1003,8 +995,6 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str) frame->tf_r11, frame->tf_r12); printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13, frame->tf_r14, frame->tf_r15); - - printf("trap number = %d\n", type); } static void diff --git a/sys/amd64/amd64/uio_machdep.c b/sys/amd64/amd64/uio_machdep.c index 16915bccf9f5..11e6ad2b1da9 100644 --- a/sys/amd64/amd64/uio_machdep.c +++ b/sys/amd64/amd64/uio_machdep.c @@ -44,9 +44,11 @@ #include <sys/uio.h> #include <vm/vm.h> +#include <vm/vm_extern.h> #include <vm/vm_page.h> #include <machine/vmparam.h> +#include <machine/md_var.h> /* * Implement uiomove(9) from physical memory using the direct map to @@ -141,3 +143,97 @@ out: td->td_pflags &= ~TDP_DEADLKTREAT; return (error); } + +int +uiomove_mem(enum uiomove_mem_req req, struct uio *uio) +{ + struct iovec *iov; + void *p, *vd; + ssize_t orig_resid; + vm_prot_t prot; + u_long v; + u_int c; + int error; + + error = 0; + orig_resid = uio->uio_resid; + while (uio->uio_resid > 0 && error == 0) { + iov = uio->uio_iov; + if (iov->iov_len == 0) { + uio->uio_iov++; + uio->uio_iovcnt--; + if (uio->uio_iovcnt < 0) + panic("memrw"); + continue; + } + v = uio->uio_offset; + c = ulmin(iov->iov_len, PAGE_SIZE - (u_int)(v & PAGE_MASK)); + + switch (req) { + case UIO_MEM_KMEM: + /* + * Since c is clamped to be less or equal than + * PAGE_SIZE, the uiomove() call does not + * access past the end of the direct map. + */ + if (v >= kva_layout.dmap_low && + v < kva_layout.dmap_high) { + error = uiomove((void *)v, c, uio); + break; + } + + switch (uio->uio_rw) { + case UIO_READ: + prot = VM_PROT_READ; + break; + case UIO_WRITE: + prot = VM_PROT_WRITE; + break; + } + + if (!kernacc((void *)v, c, prot)) { + error = EFAULT; + break; + } + + /* + * If the extracted address is not accessible + * through the direct map, then we make a + * private (uncached) mapping because we can't + * depend on the existing kernel mapping + * remaining valid until the completion of + * uiomove(). + * + * XXX We cannot provide access to the + * physical page 0 mapped into KVA. + */ + v = pmap_extract(kernel_pmap, v); + if (v == 0) { + error = EFAULT; + break; + } + /* FALLTHROUGH */ + case UIO_MEM_MEM: + if (v < dmaplimit) { + vd = PHYS_TO_DMAP(v); + error = uiomove(vd, c, uio); + break; + } + if (v > cpu_getmaxphyaddr()) { + error = EFAULT; + break; + } + p = pmap_mapdev(v, PAGE_SIZE); + error = uiomove(p, c, uio); + pmap_unmapdev(p, PAGE_SIZE); + break; + } + } + /* + * Don't return error if any byte was written. Read and write + * can return error only if no i/o was performed. + */ + if (uio->uio_resid != orig_resid) + error = 0; + return (error); +} diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 2e180003e93d..1de891680f94 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -83,13 +83,6 @@ _Static_assert(OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf), "OFFSETOF_MONITORBUF does not correspond with offset of pc_monitorbuf."); -void -set_top_of_stack_td(struct thread *td) -{ - td->td_md.md_stack_base = td->td_kstack + - td->td_kstack_pages * PAGE_SIZE; -} - struct savefpu * get_pcb_user_save_td(struct thread *td) { @@ -167,8 +160,6 @@ copy_thread(struct thread *td1, struct thread *td2) clear_pcb_flags(pcb2, PCB_TLSBASE); } - td2->td_frame = (struct trapframe *)td2->td_md.md_stack_base - 1; - /* * Set registers for trampoline to user mode. Leave space for the * return address on stack. These are the kernel mode register values. @@ -240,9 +231,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) return; } - /* Point the stack and pcb to the actual location */ - set_top_of_stack_td(td2); - td2->td_pcb = pcb2 = get_pcb_td(td2); + pcb2 = td2->td_pcb; copy_thread(td1, td2); @@ -379,18 +368,17 @@ void cpu_thread_alloc(struct thread *td) { struct pcb *pcb; - struct xstate_hdr *xhdr; - set_top_of_stack_td(td); td->td_pcb = pcb = get_pcb_td(td); - td->td_frame = (struct trapframe *)td->td_md.md_stack_base - 1; td->td_md.md_usr_fpu_save = fpu_save_area_alloc(); pcb->pcb_save = get_pcb_user_save_pcb(pcb); - if (use_xsave) { - xhdr = (struct xstate_hdr *)(pcb->pcb_save + 1); - bzero(xhdr, sizeof(*xhdr)); - xhdr->xstate_bv = xsave_mask; - } +} + +void +cpu_thread_new_kstack(struct thread *td) +{ + td->td_md.md_stack_base = td_kstack_top(td); + td->td_frame = (struct trapframe *)td->td_md.md_stack_base - 1; } void diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index c6a095f2d98a..9d76736cc46b 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -329,6 +329,7 @@ device xz # lzma decompression device bpf # Berkeley packet filter # random(4) +device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG # Disabled for now since tpm(4) breaks suspend/resume. #device tpm # Trusted Platform Module diff --git a/sys/amd64/conf/MINIMAL b/sys/amd64/conf/MINIMAL index d67ae8189a9e..ba64c39bc6b9 100644 --- a/sys/amd64/conf/MINIMAL +++ b/sys/amd64/conf/MINIMAL @@ -125,6 +125,7 @@ device ether # Ethernet support device bpf # Berkeley packet filter # random(4) +device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG # Disabled for now since tpm(4) breaks suspend/resume. #device tpm # Trusted Platform Module diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index 46a30518b212..0e8fe916490b 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -94,7 +94,6 @@ void gsbase_load_fault(void) __asm(__STRING(gsbase_load_fault)); void fpstate_drop(struct thread *td); void pagezero(void *addr); void setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int ist); -void set_top_of_stack_td(struct thread *td); struct savefpu *get_pcb_user_save_td(struct thread *td); struct savefpu *get_pcb_user_save_pcb(struct pcb *pcb); void pci_early_quirks(void); @@ -107,6 +106,12 @@ void wrmsr_early_safe_start(void); void wrmsr_early_safe_end(void); int wrmsr_early_safe(u_int msr, uint64_t data); +enum uiomove_mem_req { + UIO_MEM_KMEM = 101, + UIO_MEM_MEM, +}; +int uiomove_mem(enum uiomove_mem_req req, struct uio *uio); + #endif /* !_MACHINE_MD_VAR_H_ */ #endif /* __i386__ */ diff --git a/sys/amd64/include/stack.h b/sys/amd64/include/stack.h index 3c27266b775b..7d821348be0e 100644 --- a/sys/amd64/include/stack.h +++ b/sys/amd64/include/stack.h @@ -12,17 +12,15 @@ /* Get the current kernel thread stack usage. */ #define GET_STACK_USAGE(total, used) do { \ struct thread *td = curthread; \ - (total) = td->td_kstack_pages * PAGE_SIZE; \ - (used) = td->td_kstack + td->td_kstack_pages * PAGE_SIZE - \ - (char *)&td; \ + (total) = ptoa(td->td_kstack_pages); \ + (used) = td_kstack_top(td) - (char *)&td; \ } while (0) static __inline bool kstack_contains(struct thread *td, vm_offset_t va, size_t len) { return (va >= (vm_offset_t)td->td_kstack && va + len >= va && - va + len <= (vm_offset_t)td->td_kstack + td->td_kstack_pages * - PAGE_SIZE); + va + len <= (vm_offset_t)td_kstack_top(td)); } #endif /* _SYS_PROC_H_ */ diff --git a/sys/arm/allwinner/a64/sun50i_a64_acodec.c b/sys/arm/allwinner/a64/sun50i_a64_acodec.c index 12c9a86cf361..93b0328e99e0 100644 --- a/sys/arm/allwinner/a64/sun50i_a64_acodec.c +++ b/sys/arm/allwinner/a64/sun50i_a64_acodec.c @@ -339,19 +339,9 @@ static int a64codec_mixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct a64codec_softc *sc; - struct mtx *mixer_lock; - uint8_t do_unlock; u_int val; sc = device_get_softc(mix_getdevinfo(m)); - mixer_lock = mixer_get_lock(m); - - if (mtx_owned(mixer_lock)) { - do_unlock = 0; - } else { - do_unlock = 1; - mtx_lock(mixer_lock); - } right = left; @@ -375,10 +365,6 @@ a64codec_mixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned ri } A64CODEC_UNLOCK(sc); - if (do_unlock) { - mtx_unlock(mixer_lock); - } - return (left | (right << 8)); } diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c index a06e6773cd49..989adef3478d 100644 --- a/sys/arm/arm/machdep.c +++ b/sys/arm/arm/machdep.c @@ -378,8 +378,7 @@ init_proc0(void *kstack) proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_kstack_pages = kstack_pages; - thread0.td_pcb = (struct pcb *)(thread0.td_kstack + - thread0.td_kstack_pages * PAGE_SIZE) - 1; + thread0.td_pcb = (struct pcb *)td_kstack_top(&thread0) - 1; thread0.td_pcb->pcb_flags = 0; thread0.td_pcb->pcb_fpflags = 0; thread0.td_pcb->pcb_vfpcpu = -1; diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c index bee1c705fbbd..a8a4b6b8c7be 100644 --- a/sys/arm/arm/vm_machdep.c +++ b/sys/arm/arm/vm_machdep.c @@ -97,9 +97,6 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) if ((flags & RFPROC) == 0) return; - /* Point the pcb to the top of the stack */ - pcb2 = (struct pcb *) - (td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE) - 1; #ifdef VFP /* Store actual state of VFP */ if (curthread == td1) { @@ -107,7 +104,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) vfp_save_state(td1, td1->td_pcb); } #endif - td2->td_pcb = pcb2; + pcb2 = td2->td_pcb; /* Clone td1's pcb */ bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); @@ -116,8 +113,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) mdp2 = &p2->p_md; bcopy(&td1->td_proc->p_md, mdp2, sizeof(*mdp2)); - /* Point the frame to the stack in front of pcb and copy td1's frame */ - td2->td_frame = (struct trapframe *)pcb2 - 1; + /* Copy td1's frame */ *td2->td_frame = *td1->td_frame; /* @@ -245,8 +241,12 @@ cpu_thread_exit(struct thread *td) void cpu_thread_alloc(struct thread *td) { - td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages * - PAGE_SIZE) - 1; +} + +void +cpu_thread_new_kstack(struct thread *td) +{ + td->td_pcb = (struct pcb *)td_kstack_top(td) - 1; /* * Ensure td_frame is aligned to an 8 byte boundary as it will be * placed into the stack pointer which must be 8 byte aligned in diff --git a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c index e0c4327d8e05..6e974a1a61bb 100644 --- a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c @@ -365,6 +365,7 @@ bcm_sdhci_attach(device_t dev) return (0); fail: + bcm_dma_free(sc->sc_dma_ch); if (sc->sc_intrhand) bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intrhand); if (sc->sc_irq_res) diff --git a/sys/arm/include/stack.h b/sys/arm/include/stack.h index e80d3dc060fd..f6bc67dbe771 100644 --- a/sys/arm/include/stack.h +++ b/sys/arm/include/stack.h @@ -68,7 +68,7 @@ void unwind_module_unloaded(struct linker_file *); /* Get the current kernel thread stack usage. */ #define GET_STACK_USAGE(total, used) do { \ struct thread *td = curthread; \ - (total) = td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb); \ + (total) = ptoa(td->td_kstack_pages) - sizeof(struct pcb); \ (used) = td->td_kstack + (total) - (char *)&td; \ } while (0) @@ -76,8 +76,7 @@ static __inline bool kstack_contains(struct thread *td, vm_offset_t va, size_t len) { return (va >= (vm_offset_t)td->td_kstack && va + len >= va && - va + len <= (vm_offset_t)td->td_kstack + td->td_kstack_pages * - PAGE_SIZE - sizeof(struct pcb)); + va + len <= (vm_offset_t)td_kstack_top(td) - sizeof(struct pcb)); } #endif /* _SYS_PROC_H_ */ diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S index 5a4181348a54..5efbc4b36710 100644 --- a/sys/arm64/arm64/exception.S +++ b/sys/arm64/arm64/exception.S @@ -92,10 +92,34 @@ blr x1 1: - ldr x0, [x18, #PC_CURTHREAD] + ldr x19, [x18, #PC_CURTHREAD] + + ldr x1, [x19, #TD_MD_SCTLR] + /* + * If the upper bit in SCTLR_EL1.TCF0 is set we are either in async + * or asym modes. Either of which could set TFSRE0_EL1. + */ + tbz x1, #(SCTLR_TCF0_SHIFT + 1), 2f + /* Check for a tag fault */ + mrs x1, TFSRE0_EL1_REG + tbz x1, #TFSRE0_TF0_SHIFT, 2f + + /* + * A fault has happened, set MD_FLAG_MTE_ASYNC_FAULT. As FEAT_LSE + * is a required feature where FEAT_MTE_ASYNC could be implemented + * we can depend on it being present to set the flag. + */ + ldr w1, =MD_FLAG_MTE_ASYNC_FAULT + add x2, x19, #TD_MD_FLAGS +.arch_extension lse + stset w1, [x2] +.arch_extension nolse + +2: + mov x0, x19 bl ptrauth_exit_el0 - ldr x0, [x18, #(PC_CURTHREAD)] + mov x0, x19 bl dbg_monitor_enter /* Unmask debug and SError exceptions */ @@ -118,6 +142,14 @@ msr daifset, #(DAIF_ALL) .if \el == 0 ldr x0, [x18, #PC_CURTHREAD] + + ldr x1, [x0, #TD_MD_SCTLR] + /* See above for why we check this field */ + tbz x1, #(SCTLR_TCF0_SHIFT + 1), 1f + dsb ish + msr TFSRE0_EL1_REG, xzr +1: + mov x1, sp bl dbg_monitor_exit diff --git a/sys/arm64/arm64/exec_machdep.c b/sys/arm64/arm64/exec_machdep.c index a2e1e42249b4..d0a7302e2f7d 100644 --- a/sys/arm64/arm64/exec_machdep.c +++ b/sys/arm64/arm64/exec_machdep.c @@ -471,6 +471,7 @@ exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack) /* Generate new pointer authentication keys */ ptrauth_exec(td); + mte_exec(td); } /* Sanity check these are the same size, they will be memcpy'd to and from */ diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c index 22696796e69d..6c86f190282d 100644 --- a/sys/arm64/arm64/genassym.c +++ b/sys/arm64/arm64/genassym.c @@ -53,8 +53,6 @@ ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); ASSYM(PC_SSBD, offsetof(struct pcpu, pc_ssbd)); -/* Size of pcb, rounded to keep stack alignment */ -ASSYM(PCB_SIZE, roundup2(sizeof(struct pcb), STACKALIGNBYTES + 1)); ASSYM(PCB_SINGLE_STEP_SHIFT, PCB_SINGLE_STEP_SHIFT); ASSYM(PCB_REGS, offsetof(struct pcb, pcb_x)); ASSYM(PCB_X19, PCB_X19); @@ -76,6 +74,9 @@ ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); ASSYM(TD_MD_CANARY, offsetof(struct thread, td_md.md_canary)); ASSYM(TD_MD_EFIRT_TMP, offsetof(struct thread, td_md.md_efirt_tmp)); +ASSYM(TD_MD_FLAGS, offsetof(struct thread, td_md.md_flags)); +ASSYM(MD_FLAG_MTE_ASYNC_FAULT, MD_FLAG_MTE_ASYNC_FAULT); +ASSYM(TD_MD_SCTLR, offsetof(struct thread, td_md.md_sctlr)); ASSYM(TF_SIZE, sizeof(struct trapframe)); ASSYM(TF_SP, offsetof(struct trapframe, tf_sp)); diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S index b200aa93c281..bd61b485edf7 100644 --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -128,8 +128,7 @@ virtdone: /* Set up the stack */ adrp x25, initstack_end - add x25, x25, :lo12:initstack_end - sub sp, x25, #PCB_SIZE + add sp, x25, :lo12:initstack_end /* Zero the BSS */ ldr x15, .Lbss diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index d219c737c215..f35ec7ab2e2e 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -131,6 +131,7 @@ uintptr_t boot_canary = 0x49a2d892bc05a0b1ul; #endif static struct trapframe proc0_tf; +static struct pcb pcb0; int early_boot = 1; int cold = 1; @@ -443,14 +444,14 @@ init_proc0(void *kstack) #if defined(PERTHREAD_SSP) thread0.td_md.md_canary = boot_canary; #endif - thread0.td_pcb = (struct pcb *)(thread0.td_kstack + - thread0.td_kstack_pages * PAGE_SIZE) - 1; + thread0.td_pcb = &pcb0; thread0.td_pcb->pcb_flags = 0; thread0.td_pcb->pcb_fpflags = 0; thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; thread0.td_pcb->pcb_vfpcpu = UINT_MAX; thread0.td_frame = &proc0_tf; ptrauth_thread0(&thread0); + mte_thread0(&thread0); pcpup->pc_curpcb = thread0.td_pcb; /* diff --git a/sys/arm64/arm64/mte.c b/sys/arm64/arm64/mte.c new file mode 100644 index 000000000000..6e902858a8b9 --- /dev/null +++ b/sys/arm64/arm64/mte.c @@ -0,0 +1,191 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024-2026 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/libkern.h> +#include <sys/proc.h> + +#include <machine/cpu_feat.h> +#include <machine/pcb.h> +#include <machine/pte.h> +#include <machine/sysarch.h> +#include <vm/vm.h> +#include <vm/vm_page.h> + +/* Version of MTE implemented. 0 == unimplemented */ +static u_int __read_mostly mte_version = 0; + +/* + * FEAT_MTE (mte_version == 1) has userspace instructions, but no tag + * checking. May of the registers/fields need FEAT_MTE2 to be implemented + * before we can access them. + */ +#define MTE_HAS_TAG_CHECK (mte_version >= 2) + +struct thread *mte_switch(struct thread *); + +#define load_tags(addr) ({ \ + uint64_t __val; \ + asm volatile( \ + ".arch_extension memtag \n" \ + "ldgm %0, [%1] \n" \ + ".arch_extension nomemtag" : "=r" (__val) : "r" (addr)); \ + __val; \ +}) + +#define set_tags(tags, addr) do { \ + asm volatile( \ + ".arch_extension memtag \n" \ + "stgm %0, [%1] \n" \ + ".arch_extension nomemtag" : "=r" (tags) : "r" (addr)); \ +} while (0) + +/* Fetch the block size used by tag load and store instructions */ +static inline size_t +mte_block_size(void) +{ + return (sizeof(int) << GMID_BS_SIZE(READ_SPECIALREG(GMID_EL1_REG))); +} + +static void +mte_update_sctlr(struct thread *td, uint64_t sctlr) +{ + MPASS((sctlr & ~(SCTLR_ATA0 | SCTLR_TCF0_MASK)) == 0); + td->td_md.md_sctlr &= ~(SCTLR_ATA0 | SCTLR_TCF0_MASK); + td->td_md.md_sctlr |= sctlr; +} + +/** + * Clear/sync the allocation tags for a given page. This should be done on + * allocation of a page to ensure a tag check fault does not occur immediately + * after accessing newly tagged memory. + */ +void +mte_sync_tags(vm_page_t page) +{ + char *addr; + size_t block_size; + + if (!MTE_HAS_TAG_CHECK) + return; + + /* don't clear the tags on a page that's already setup for mte */ + if ((page->md.pv_flags & PV_MTE_TAGGED) != 0) + return; + + block_size = mte_block_size(); + addr = PHYS_TO_DMAP(page->phys_addr); + + for (size_t count = 0; count < PAGE_SIZE; + count += block_size, addr += block_size) + asm volatile( + ".arch_extension memtag \n" + "stgm xzr, [%0] \n" + ".arch_extension nomemtag" : : "r" (addr)); + + page->md.pv_flags |= PV_MTE_TAGGED; +} + +/** + * Copy the allocation tags from given target to destination page. This is called + * on a copy-on-write and anything that causes a pmap_copy_page call. + */ +void +mte_copy_tags(vm_page_t srcpage, vm_page_t dstpage, char *src, char *dst) +{ + size_t block_size; + uint64_t tags; + + MPASS((srcpage->md.pv_flags & PV_MTE_TAGGED) != 0); + + /* + * Copy the tags from the source page to the destination page, + * incrementing by the block count read from GMID_EL1 + */ + block_size = mte_block_size(); + for (size_t count = 0; count < PAGE_SIZE; + count += block_size, src += block_size, dst += block_size) { + tags = load_tags(src); + set_tags(tags, dst); + } + dstpage->md.pv_flags |= PV_MTE_TAGGED; +} + +void +mte_fork(struct thread *new_td, struct thread *orig_td) +{ + if (!MTE_HAS_TAG_CHECK) + return; + + mte_update_sctlr(new_td, + orig_td->td_md.md_sctlr & SCTLR_TCF0_MASK); + new_td->td_md.md_gcr = orig_td->td_md.md_gcr; +} + +void +mte_exec(struct thread *td) +{ + if (!MTE_HAS_TAG_CHECK) + return; + + mte_update_sctlr(td, SCTLR_TCF0_NONE); + td->td_md.md_gcr = GCR_RRND; +} + +void +mte_copy_thread(struct thread *new_td, struct thread *orig_td) +{ + if (!MTE_HAS_TAG_CHECK) + return; + + mte_update_sctlr(new_td, + orig_td->td_md.md_sctlr & SCTLR_TCF0_MASK); + new_td->td_md.md_gcr = orig_td->td_md.md_gcr; +} + +/* Only for kernel threads */ +void +mte_thread_alloc(struct thread *td) +{ +} + +/* Only for a kernel thread */ +void +mte_thread0(struct thread *td) +{ +} + + +struct thread * +mte_switch(struct thread *td) +{ + if (MTE_HAS_TAG_CHECK) { + WRITE_SPECIALREG(GCR_EL1_REG, td->td_md.md_gcr); + } + return (td); +} diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index adc583812e5b..1fb9ac2011aa 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -146,6 +146,7 @@ #include <vm/uma.h> #include <machine/asan.h> +#include <machine/cpu.h> #include <machine/cpu_feat.h> #include <machine/elf.h> #include <machine/ifunc.h> @@ -358,6 +359,7 @@ struct pv_chunks_list __exclusive_cache_line pv_chunks[PMAP_MEMDOM]; vm_paddr_t dmap_phys_base; /* The start of the dmap region */ vm_paddr_t dmap_phys_max; /* The limit of the dmap region */ vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */ +static int dmap_attr = VM_MEMATTR_WRITE_BACK; extern pt_entry_t pagetable_l0_ttbr1[]; @@ -483,7 +485,7 @@ static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); static bool pmap_activate_int(struct thread *td, pmap_t pmap); static void pmap_alloc_asid(pmap_t pmap); static int pmap_change_props_locked(void *addr, vm_size_t size, - vm_prot_t prot, int mode, bool skip_unmapped); + vm_prot_t prot, int mode, int old_mode, bool skip_unmapped); static bool pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, pt_entry_t l3e, vm_page_t ml3, struct rwlock **lockp); static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va); @@ -1586,6 +1588,7 @@ pmap_page_init(vm_page_t m) TAILQ_INIT(&m->md.pv_list); m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; + m->md.pv_flags = 0; } static void @@ -6920,6 +6923,7 @@ pmap_zero_page(vm_page_t m) void *va = VM_PAGE_TO_DMAP(m); pagezero(va); + m->md.pv_flags &= ~PV_MTE_TAGGED; } /* @@ -6951,6 +6955,15 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst) void *src = VM_PAGE_TO_DMAP(msrc); void *dst = VM_PAGE_TO_DMAP(mdst); + /* + * On a page copy, check whether the src page is tagged. If it is, + * we must copy the tags before copying the contents of the page. + */ + if ((msrc->md.pv_flags & PV_MTE_TAGGED) != 0) + mte_copy_tags(msrc, mdst, src, dst); + else + mdst->md.pv_flags &= ~PV_MTE_TAGGED; + pagecopy(src, dst); } @@ -6967,6 +6980,9 @@ pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], int cnt; while (xfersize > 0) { + KASSERT(ADDR_IS_CANONICAL(a_offset), + ("%s: Address not in canonical form: %lx", __func__, a_offset)); + a_pg_offset = a_offset & PAGE_MASK; m_a = ma[a_offset >> PAGE_SHIFT]; p_a = m_a->phys_addr; @@ -8161,7 +8177,7 @@ pmap_unmapbios(void *p, vm_size_t size) /* Ensure the attributes are as expected for the DMAP region */ PMAP_LOCK(kernel_pmap); error = pmap_change_props_locked(va, size, - PROT_READ | PROT_WRITE, VM_MEMATTR_DEFAULT, false); + PROT_READ | PROT_WRITE, VM_MEMATTR_DEFAULT, -1, false); PMAP_UNLOCK(kernel_pmap); KASSERT(error == 0, ("%s: Failed to reset DMAP attributes: %d", __func__, error)); @@ -8267,7 +8283,25 @@ pmap_change_attr(void *va, vm_size_t size, int mode) int error; PMAP_LOCK(kernel_pmap); - error = pmap_change_props_locked(va, size, PROT_NONE, mode, false); + error = pmap_change_props_locked(va, size, PROT_NONE, mode, -1, false); + PMAP_UNLOCK(kernel_pmap); + return (error); +} + +int +pmap_change_dmap_attr(int mode) +{ + int error; + + KASSERT(mode == VM_MEMATTR_WRITE_BACK || + mode == VM_MEMATTR_TAGGED, + ("%s: mode %d must be compatible with write-back", __func__, mode)); + + PMAP_LOCK(kernel_pmap); + error = pmap_change_props_locked((void *)DMAP_MIN_ADDRESS, + dmap_max_addr - DMAP_MIN_ADDRESS, PROT_NONE, mode, dmap_attr, true); + if (error == 0) + dmap_attr = mode; PMAP_UNLOCK(kernel_pmap); return (error); } @@ -8289,20 +8323,20 @@ pmap_change_prot(void *va, vm_size_t size, vm_prot_t prot) return (EINVAL); PMAP_LOCK(kernel_pmap); - error = pmap_change_props_locked(va, size, prot, -1, false); + error = pmap_change_props_locked(va, size, prot, -1, -1, false); PMAP_UNLOCK(kernel_pmap); return (error); } static int pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, - int mode, bool skip_unmapped) + int mode, int old_mode, bool skip_unmapped) { vm_offset_t base, offset, tmpva, va; vm_size_t pte_size; vm_paddr_t pa; pt_entry_t pte, *ptep, *newpte; - pt_entry_t bits, mask; + pt_entry_t bits, mask, old_mode_bits, old_mode_mask; char *tmpptep; int lvl, rv; @@ -8316,8 +8350,8 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, !(base >= VM_MIN_KERNEL_ADDRESS && base < VM_MAX_KERNEL_ADDRESS)) return (EINVAL); - bits = 0; - mask = 0; + bits = old_mode_bits = 0; + mask = old_mode_mask = 0; if (mode != -1) { bits = ATTR_S1_IDX(mode); mask = ATTR_S1_IDX_MASK; @@ -8326,6 +8360,10 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, bits |= ATTR_S1_XN; } } + if (old_mode != -1) { + old_mode_bits = ATTR_S1_IDX(old_mode); + old_mode_mask = ATTR_S1_IDX_MASK; + } if (prot != VM_PROT_NONE) { /* Don't mark the DMAP as executable. It never is on arm64. */ if (VIRT_IN_DMAP(base)) { @@ -8353,11 +8391,14 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, if (ptep == NULL && !skip_unmapped) { return (EINVAL); } else if ((ptep == NULL && skip_unmapped) || - (pmap_load(ptep) & mask) == bits) { + (pmap_load(ptep) & mask) == bits || + (pmap_load(ptep) & old_mode_mask) != old_mode_bits) { /* - * We already have the correct attribute or there - * is no memory mapped at this address and we are - * skipping unmapped memory. + * We already have one of the following meaning + * we can skip this memory region:: + * - No memory mapped at this address + * - The new attributes are already set + * - The expected attributes are incorrect */ switch (lvl) { default: @@ -8487,12 +8528,24 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, pa = PTE_TO_PHYS(pte); if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(pa)) { + int dmap_mode; + + /* + * When booting on HW with MTE enabled we may + * need to swap to a tagged type for the DMAP + * to allow tags to be set through it. + */ + if (mode == VM_MEMATTR_WRITE_BACK) + dmap_mode = dmap_attr; + else + dmap_mode = mode; + /* * Keep the DMAP memory in sync. */ rv = pmap_change_props_locked( PHYS_TO_DMAP(pa), pte_size, - prot, mode, true); + prot, dmap_mode, old_mode, true); if (rv != 0) return (rv); } diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S index b3bf88135e57..b349072c06f4 100644 --- a/sys/arm64/arm64/swtch.S +++ b/sys/arm64/arm64/swtch.S @@ -75,7 +75,7 @@ * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) - /* Of old == NULL skip disabling stepping */ + /* If old == NULL skip disabling stepping */ cbz x0, 1f /* If we were single stepping, disable it */ @@ -96,8 +96,9 @@ ENTRY(cpu_throw) mov x0, x1 #endif - /* This returns the thread pointer so no need to save it */ + /* These return the thread pointer so no need to save it */ bl ptrauth_switch + bl mte_switch #ifdef PERTHREAD_SSP mov x19, x0 #endif @@ -176,8 +177,9 @@ ENTRY(cpu_switch) mov x0, x1 #endif - /* This returns the thread pointer so no need to save it */ + /* These return the thread pointer so no need to save it */ bl ptrauth_switch + bl mte_switch /* This returns the thread pcb */ bl pmap_switch /* Move the new pcb out of the way */ @@ -276,6 +278,8 @@ ENTRY(fork_trampoline) ldp x26, x27, [sp, #TF_X + 26 * 8] ldp x28, x29, [sp, #TF_X + 28 * 8] + add sp, sp, #(TF_SIZE) + /* * No need for interrupts reenabling since PSR * will be set to the desired value anyway. diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c index 4cb87ca9856e..635bdcef7025 100644 --- a/sys/arm64/arm64/vm_machdep.c +++ b/sys/arm64/arm64/vm_machdep.c @@ -27,8 +27,8 @@ #include "opt_platform.h" -#include <sys/param.h> #include <sys/systm.h> +#include <sys/kernel.h> #include <sys/limits.h> #include <sys/proc.h> #include <sys/sf_buf.h> @@ -61,6 +61,8 @@ */ cpu_reset_hook_t cpu_reset_hook = psci_reset; +static uma_zone_t pcb_zone; + /* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child @@ -89,25 +91,21 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) #endif } - pcb2 = (struct pcb *)(td2->td_kstack + - td2->td_kstack_pages * PAGE_SIZE) - 1; - - td2->td_pcb = pcb2; + pcb2 = td2->td_pcb; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); /* Clear the debug register state. */ bzero(&pcb2->pcb_dbg_regs, sizeof(pcb2->pcb_dbg_regs)); ptrauth_fork(td2, td1); + mte_fork(td2, td1); - tf = STACKALIGN((struct trapframe *)pcb2 - 1); + tf = td2->td_frame; bcopy(td1->td_frame, tf, sizeof(*tf)); tf->tf_x[0] = 0; tf->tf_x[1] = 0; tf->tf_spsr = td1->td_frame->tf_spsr & (PSR_M_32 | PSR_DAIF); - td2->td_frame = tf; - /* Set the return value registers for fork() */ td2->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return; td2->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td2; @@ -203,6 +201,7 @@ cpu_copy_thread(struct thread *td, struct thread *td0) /* Generate new pointer authentication keys. */ ptrauth_copy_thread(td, td0); + mte_copy_thread(td, td0); } /* @@ -265,17 +264,21 @@ cpu_thread_exit(struct thread *td) void cpu_thread_alloc(struct thread *td) { - - td->td_pcb = (struct pcb *)(td->td_kstack + - td->td_kstack_pages * PAGE_SIZE) - 1; - td->td_frame = (struct trapframe *)STACKALIGN( - (struct trapframe *)td->td_pcb - 1); + td->td_pcb = uma_zalloc(pcb_zone, M_WAITOK); ptrauth_thread_alloc(td); + mte_thread_alloc(td); +} + +void +cpu_thread_new_kstack(struct thread *td) +{ + td->td_frame = (struct trapframe *)td_kstack_top(td) - 1; } void cpu_thread_free(struct thread *td) { + uma_zfree(pcb_zone, td->td_pcb); } void @@ -335,3 +338,11 @@ cpu_sync_core(void) * return from ELx is a context synchronization event. */ } + +static void +pcbinit(void *dummy __unused) +{ + pcb_zone = uma_zcreate("pcb", sizeof(struct pcb), NULL, NULL, NULL, + NULL, UMA_ALIGNOF(struct pcb), 0); +} +SYSINIT(pcbinit, SI_SUB_INTRINSIC, SI_ORDER_ANY, pcbinit, NULL); diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h index 05844ad63036..bdbc601edd26 100644 --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -277,6 +277,16 @@ void ptrauth_thread0(struct thread *); void ptrauth_mp_start(uint64_t); #endif +/* Memory Tagging Extension (MTE) support */ +void mte_fork(struct thread *, struct thread *); +void mte_exec(struct thread *); +void mte_copy_thread(struct thread *, struct thread *); +void mte_thread_alloc(struct thread *); +void mte_thread0(struct thread *); + +void mte_sync_tags(vm_page_t page); +void mte_copy_tags(vm_page_t, vm_page_t, char *, char *); + /* Functions to read the sanitised view of the special registers */ void update_special_regs(u_int); void update_special_reg_iss(u_int, uint64_t, uint64_t); diff --git a/sys/arm64/include/elf.h b/sys/arm64/include/elf.h index a5a90f8c7712..7940bb259256 100644 --- a/sys/arm64/include/elf.h +++ b/sys/arm64/include/elf.h @@ -96,6 +96,12 @@ __ElfType(Auxinfo); /* First __FreeBSD_version that supports Top Byte Ignore (TBI) */ #define TBI_VERSION 1500058 +/* + * The HWCAP values must be identical to Linux. Many userspace programs + * will define missing HWCAP values to the Linux version. To keep these + * working when we add the HWCAP it must be the same. + */ + /* HWCAP */ #define HWCAP_FP (1 << 0) #define HWCAP_ASIMD (1 << 1) diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h index 286a40e7de3d..d04f975350d8 100644 --- a/sys/arm64/include/pcpu.h +++ b/sys/arm64/include/pcpu.h @@ -55,7 +55,6 @@ struct debug_monitor_state; #ifdef _KERNEL -struct pcb; struct pcpu; static inline struct pcpu * diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h index 00b54a874e12..cf20827fa666 100644 --- a/sys/arm64/include/pmap.h +++ b/sys/arm64/include/pmap.h @@ -70,9 +70,13 @@ struct md_page { TAILQ_HEAD(,pv_entry) pv_list; int pv_gen; vm_memattr_t pv_memattr; - uint8_t pv_reserve[3]; + uint8_t pv_flags; + uint8_t pv_reserve[2]; }; +/* machine page flags */ +#define PV_MTE_TAGGED 0x01 /* page is tagged with MTE */ + enum pmap_stage { PM_INVALID, PM_STAGE1, @@ -148,6 +152,7 @@ void pmap_activate_vm(pmap_t); void pmap_bootstrap_dmap(vm_size_t); void pmap_bootstrap(void); int pmap_change_attr(void *va, vm_size_t size, int mode); +int pmap_change_dmap_attr(int); int pmap_change_prot(void *va, vm_size_t size, vm_prot_t prot); void pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode); void pmap_kenter_device(vm_offset_t, vm_size_t, vm_paddr_t); diff --git a/sys/arm64/include/proc.h b/sys/arm64/include/proc.h index d5879a794269..22ceb614413d 100644 --- a/sys/arm64/include/proc.h +++ b/sys/arm64/include/proc.h @@ -69,9 +69,11 @@ struct mdthread { uint64_t md_efirt_tmp; int md_efirt_dis_pf; - int md_reserved0; + u_int md_flags; +#define MD_FLAG_MTE_ASYNC_FAULT_SHIFT 0 +#define MD_FLAG_MTE_ASYNC_FAULT (1u << 0) uint64_t md_sctlr; - uint64_t md_reserved1; + uint64_t md_gcr; /* FEAT_MTE: Tag Control Register */ }; struct mdproc { diff --git a/sys/arm64/include/stack.h b/sys/arm64/include/stack.h index 19e9e837e3ee..23e7a5af27de 100644 --- a/sys/arm64/include/stack.h +++ b/sys/arm64/include/stack.h @@ -39,11 +39,9 @@ bool unwind_frame(struct thread *, struct unwind_state *); #ifdef _SYS_PROC_H_ -#include <machine/pcb.h> - #define GET_STACK_USAGE(total, used) do { \ struct thread *td = curthread; \ - (total) = td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb); \ + (total) = ptoa(td->td_kstack_pages); \ (used) = td->td_kstack + (total) - (char *)&td; \ } while (0) @@ -51,8 +49,7 @@ static __inline bool kstack_contains(struct thread *td, vm_offset_t va, size_t len) { return (va >= (vm_offset_t)td->td_kstack && va + len >= va && - va + len <= (vm_offset_t)td->td_kstack + td->td_kstack_pages * - PAGE_SIZE - sizeof(struct pcb)); + va + len <= (vm_offset_t)td_kstack_top(td)); } #endif /* _SYS_PROC_H_ */ diff --git a/sys/arm64/rockchip/rk3328_codec.c b/sys/arm64/rockchip/rk3328_codec.c index 22e3cde9093e..a019cab27cc9 100644 --- a/sys/arm64/rockchip/rk3328_codec.c +++ b/sys/arm64/rockchip/rk3328_codec.c @@ -416,18 +416,8 @@ static int rkcodec_mixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct rkcodec_softc *sc; - struct mtx *mixer_lock; - uint8_t do_unlock; sc = device_get_softc(mix_getdevinfo(m)); - mixer_lock = mixer_get_lock(m); - - if (mtx_owned(mixer_lock)) { - do_unlock = 0; - } else { - do_unlock = 1; - mtx_lock(mixer_lock); - } right = left; @@ -443,10 +433,6 @@ rkcodec_mixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned rig } RKCODEC_UNLOCK(sc); - if (do_unlock) { - mtx_unlock(mixer_lock); - } - return (left | (right << 8)); } diff --git a/sys/arm64/rockchip/rk_gpio.c b/sys/arm64/rockchip/rk_gpio.c index 8da37d516802..7c2071d2d178 100644 --- a/sys/arm64/rockchip/rk_gpio.c +++ b/sys/arm64/rockchip/rk_gpio.c @@ -227,8 +227,22 @@ rk_gpio_intr(void *arg) status &= ~(1 << pin); if (intr_isrc_dispatch(RK_GPIO_ISRC(sc, pin), tf)) { - device_printf(sc->sc_dev, "Interrupt pin=%d unhandled\n", - pin); + /* + * Pin asserted but no consumer is registered for it + * yet (or anymore). Level-triggered sources keep + * firing on every interrupt cycle, so a single stuck + * pin floods the console with thousands of these + * messages per second. Mask the pin's IRQ at the + * controller and disable further dispatches; if a + * consumer attaches later it will re-enable through + * pic_enable_intr / rk_gpio_pic_enable_intr. + */ + RK_GPIO_LOCK(sc); + rk_gpio_write_bit(sc, RK_GPIO_INTMASK, pin, 1); + rk_gpio_write_bit(sc, RK_GPIO_INTEN, pin, 0); + RK_GPIO_UNLOCK(sc); + device_printf(sc->sc_dev, + "Interrupt pin=%d unhandled — masked\n", pin); continue; } @@ -818,10 +832,14 @@ rk_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc, return (EINVAL); } rk_gpio_write_bit(sc, RK_GPIO_DEBOUNCE, pin, 1); - rk_gpio_write_bit(sc, RK_GPIO_INTMASK, pin, 0); - rk_gpio_write_bit(sc, RK_GPIO_INTEN, pin, 1); RK_GPIO_UNLOCK(sc); + /* + * Leave the interrupt masked + disabled here. INTRNG will call + * pic_enable_intr() next to make it live. That keeps the + * masking responsibility cleanly in enable/disable rather than + * split between setup and disable. + */ return (0); } @@ -837,14 +855,86 @@ rk_pic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, if (isrc->isrc_handlers == 0) { irqsrc->mode = GPIO_INTR_CONFORM; RK_GPIO_LOCK(sc); - rk_gpio_write_bit(sc, RK_GPIO_INTEN, irqsrc->irq, 0); - rk_gpio_write_bit(sc, RK_GPIO_INTMASK, irqsrc->irq, 0); + /* + * INTEN/INTMASK are already cleared by pic_disable_intr, + * which INTRNG calls before teardown of the last handler. + * We only need to undo what setup_intr configured -- here, + * the debounce filter. + */ rk_gpio_write_bit(sc, RK_GPIO_DEBOUNCE, irqsrc->irq, 0); RK_GPIO_UNLOCK(sc); } return (0); } +/* + * INTRNG calls pic_disable_intr() during teardown of the final handler + * for a source, OR when a consumer explicitly wants the source off. + * Clear INTEN so the controller will not raise this pin at all. + * + * The in-flight masking between FILTER_SCHEDULE_THREAD and ithread + * completion is handled by pic_pre_ithread() / pic_post_ithread() + * below, NOT by this method. + */ +static void +rk_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct rk_gpio_softc *sc = device_get_softc(dev); + struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc; + + RK_GPIO_LOCK(sc); + rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 1); + rk_gpio_write_bit(sc, RK_GPIO_INTEN, rkisrc->irq, 0); + RK_GPIO_UNLOCK(sc); +} + +/* + * INTRNG calls pic_enable_intr() to make a source live for the first + * time (after setup_intr), or to re-enable after a prior + * pic_disable_intr(). Set INTEN and unmask so the controller starts + * delivering this pin. + */ +static void +rk_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + struct rk_gpio_softc *sc = device_get_softc(dev); + struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc; + + RK_GPIO_LOCK(sc); + rk_gpio_write_bit(sc, RK_GPIO_INTEN, rkisrc->irq, 1); + rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 0); + RK_GPIO_UNLOCK(sc); +} + +/* + * Called by INTRNG before delivering to the ithread. Mask the source + * so it cannot re-fire during the ithread window -- without this, + * level-low IRQs (e.g. FUSB302 INT_N) re-trigger continuously and + * starve the ithread (~210 kHz storm observed via dtrace). + * Re-unmasked in pic_post_ithread() once the ithread acks the source. + */ +static void +rk_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + struct rk_gpio_softc *sc = device_get_softc(dev); + struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc; + + RK_GPIO_LOCK(sc); + rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 1); + RK_GPIO_UNLOCK(sc); +} + +static void +rk_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ + struct rk_gpio_softc *sc = device_get_softc(dev); + struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc; + + RK_GPIO_LOCK(sc); + rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 0); + RK_GPIO_UNLOCK(sc); +} + static device_method_t rk_gpio_methods[] = { /* Device interface */ DEVMETHOD(device_probe, rk_gpio_probe), @@ -873,6 +963,10 @@ static device_method_t rk_gpio_methods[] = { DEVMETHOD(pic_map_intr, rk_pic_map_intr), DEVMETHOD(pic_setup_intr, rk_pic_setup_intr), DEVMETHOD(pic_teardown_intr, rk_pic_teardown_intr), + DEVMETHOD(pic_disable_intr, rk_pic_disable_intr), + DEVMETHOD(pic_enable_intr, rk_pic_enable_intr), + DEVMETHOD(pic_pre_ithread, rk_pic_pre_ithread), + DEVMETHOD(pic_post_ithread, rk_pic_post_ithread), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_node, rk_gpio_get_node), diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c index 68f1cabf6d07..6a55aba2669b 100644 --- a/sys/cam/ctl/scsi_ctl.c +++ b/sys/cam/ctl/scsi_ctl.c @@ -522,7 +522,8 @@ ctlferegister(struct cam_periph *periph, void *arg) new_ccb->ccb_h.io_ptr = new_io; LIST_INSERT_HEAD(&softc->atio_list, &new_ccb->ccb_h, periph_links.le); - xpt_setup_ccb(&new_ccb->ccb_h, periph->path, CAM_PRIORITY_NONE); + xpt_setup_ccb(&new_ccb->ccb_h, periph->path, + CAM_PRIORITY_NORMAL); new_ccb->ccb_h.func_code = XPT_ACCEPT_TARGET_IO; new_ccb->ccb_h.cbfcnp = ctlfedone; new_ccb->ccb_h.flags |= CAM_UNLOCKED; @@ -569,7 +570,8 @@ ctlferegister(struct cam_periph *periph, void *arg) new_ccb->ccb_h.io_ptr = new_io; LIST_INSERT_HEAD(&softc->inot_list, &new_ccb->ccb_h, periph_links.le); - xpt_setup_ccb(&new_ccb->ccb_h, periph->path, CAM_PRIORITY_NONE); + xpt_setup_ccb(&new_ccb->ccb_h, periph->path, + CAM_PRIORITY_NORMAL); new_ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY; new_ccb->ccb_h.cbfcnp = ctlfedone; new_ccb->ccb_h.flags |= CAM_UNLOCKED; @@ -1003,7 +1005,7 @@ ctlfe_requeue_ccb(struct cam_periph *periph, union ccb *ccb, int unlock) * target/lun. Reset the target and LUN fields back to the wildcard * values before we send them back down to the SIM. */ - xpt_setup_ccb_flags(&ccb->ccb_h, periph->path, CAM_PRIORITY_NONE, + xpt_setup_ccb_flags(&ccb->ccb_h, periph->path, CAM_PRIORITY_NORMAL, ccb->ccb_h.flags); xpt_action(ccb); diff --git a/sys/compat/linux/linux_mmap.c b/sys/compat/linux/linux_mmap.c index a8e790a29da4..9fecb6ebb2ad 100644 --- a/sys/compat/linux/linux_mmap.c +++ b/sys/compat/linux/linux_mmap.c @@ -63,6 +63,10 @@ static int linux_mmap_check_fp(struct file *fp, int flags, int prot, int maxprot) { + /* Linux returns EBADF if mmap() is called on an O_PATH file descriptor */ + if (fp->f_ops == &path_fileops) + return (EBADF); + /* Linux mmap() just fails for O_WRONLY files */ if ((fp->f_flag & FREAD) == 0) return (EACCES); diff --git a/sys/compat/linuxkpi/common/src/linux_80211.c b/sys/compat/linuxkpi/common/src/linux_80211.c index b9528295ad8e..901c59702840 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211.c +++ b/sys/compat/linuxkpi/common/src/linux_80211.c @@ -105,6 +105,11 @@ SYSCTL_DECL(_compat_linuxkpi); SYSCTL_NODE(_compat_linuxkpi, OID_AUTO, 80211, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "LinuxKPI 802.11 compatibility layer"); +static int lkpi_suspend_type = 1; +SYSCTL_INT(_compat_linuxkpi_80211, OID_AUTO, suspend_type, CTLFLAG_RW, + &lkpi_suspend_type, 0, + "LinuxKPI 802.11 suspend type bitmask (0=off, 1=net80211, 2=wowlan"); + static bool lkpi_order_scanlist = false; SYSCTL_BOOL(_compat_linuxkpi_80211, OID_AUTO, order_scanlist, CTLFLAG_RW, &lkpi_order_scanlist, 0, "Enable LinuxKPI 802.11 scan list shuffeling"); @@ -1602,12 +1607,13 @@ lkpi_iv_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k) } sta = LSTA_TO_STA(lsta); - keylen = k->wk_keylen; + keylen = ieee80211_crypto_get_key_len(k); lcipher = lkpi_net80211_to_l80211_cipher_suite( - k->wk_cipher->ic_cipher, k->wk_keylen); + k->wk_cipher->ic_cipher, ieee80211_crypto_get_key_len(k)); switch (lcipher) { case WLAN_CIPHER_SUITE_TKIP: - keylen += 2 * k->wk_cipher->ic_miclen; + keylen += ieee80211_crypto_get_key_txmic_len(k); + keylen += ieee80211_crypto_get_key_rxmic_len(k); break; case WLAN_CIPHER_SUITE_CCMP: case WLAN_CIPHER_SUITE_GCMP: @@ -1638,8 +1644,9 @@ lkpi_iv_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k) kc->hw_key_idx = /* set by hw and needs to be passed for TX */; #endif atomic64_set(&kc->tx_pn, k->wk_keytsc); - kc->keylen = k->wk_keylen; - memcpy(kc->key, k->wk_key, k->wk_keylen); + kc->keylen = ieee80211_crypto_get_key_len(k); + memcpy(kc->key, ieee80211_crypto_get_key_data(k), + ieee80211_crypto_get_key_len(k)); if (k->wk_flags & (IEEE80211_KEY_XMIT | IEEE80211_KEY_RECV)) kc->flags |= IEEE80211_KEY_FLAG_PAIRWISE; @@ -1651,8 +1658,12 @@ lkpi_iv_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k) switch (kc->cipher) { case WLAN_CIPHER_SUITE_TKIP: - memcpy(kc->key + NL80211_TKIP_DATA_OFFSET_TX_MIC_KEY, k->wk_txmic, k->wk_cipher->ic_miclen); - memcpy(kc->key + NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY, k->wk_rxmic, k->wk_cipher->ic_miclen); + memcpy(kc->key + NL80211_TKIP_DATA_OFFSET_TX_MIC_KEY, + ieee80211_crypto_get_key_txmic_data(k), + ieee80211_crypto_get_key_txmic_len(k)); + memcpy(kc->key + NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY, + ieee80211_crypto_get_key_rxmic_data(k), + ieee80211_crypto_get_key_rxmic_len(k)); break; case WLAN_CIPHER_SUITE_CCMP: case WLAN_CIPHER_SUITE_GCMP: @@ -2426,6 +2437,8 @@ lkpi_set_chanctx_conf(struct ieee80211_hw *hw, struct ieee80211_vif *vif, rcu_assign_pointer(vif->bss_conf.chanctx_conf, NULL); lchanctx = CHANCTX_CONF_TO_LCHANCTX(chanctx_conf); list_del(&lchanctx->entry); + memset(lchanctx, 0, sizeof(*lchanctx)); + lchanctx->lvif = VIF_TO_LVIF(vif); list_add_rcu(&lchanctx->entry, &lhw->lchanctx_list_reserved); } @@ -2460,6 +2473,8 @@ lkpi_remove_chanctx(struct ieee80211_hw *hw, struct ieee80211_vif *vif) lchanctx = CHANCTX_CONF_TO_LCHANCTX(chanctx_conf); list_del(&lchanctx->entry); lhw = HW_TO_LHW(hw); + memset(lchanctx, 0, sizeof(*lchanctx)); + lchanctx->lvif = VIF_TO_LVIF(vif); list_add_rcu(&lchanctx->entry, &lhw->lchanctx_list_reserved); } @@ -5522,10 +5537,10 @@ lkpi_hw_crypto_prepare_tkip(struct ieee80211_key *k, * "enmic" (though we do not do that). */ /* any conditions to not apply this? */ - if (skb_tailroom(skb) < k->wk_cipher->ic_miclen) + if (skb_tailroom(skb) < ieee80211_crypto_get_key_txmic_len(k)) return (ENOBUFS); - p = skb_put(skb, k->wk_cipher->ic_miclen); + p = skb_put(skb, ieee80211_crypto_get_key_txmic_len(k)); if ((kc->flags & IEEE80211_KEY_FLAG_PUT_MIC_SPACE) != 0) goto encrypt; @@ -6806,6 +6821,7 @@ linuxkpi_ieee80211_iffree(struct ieee80211_hw *hw) lkpi_80211_mo_remove_chanctx(hw, chanctx_conf); } list_del(&lchanctx->entry); + /* No need to reset the lchanctx here as we will free it below. */ list_add_rcu(&lchanctx->entry, &lhw->lchanctx_list_reserved); } } @@ -6854,10 +6870,19 @@ linuxkpi_set_ieee80211_dev(struct ieee80211_hw *hw) /* * Set a proper name before ieee80211_ifattach() if dev is set. * ath1xk also unset the dev so we need to check. + * Also we will (ab)use this opportunity to register the + * power management sub-children if thay exist (for suspend/resume). */ dev = wiphy_dev(hw->wiphy); if (dev != NULL) { ic->ic_name = dev_name(dev); + if (dev->bsddev != NULL) { + bus_identify_children(dev->bsddev); + bus_enumerate_hinted_children(dev->bsddev); + bus_topo_lock(); + bus_attach_children(dev->bsddev); + bus_topo_unlock(); + } } else { TODO("adjust arguments to still have the old dev or go through " "the hoops of getting the bsddev from hw and detach; " @@ -9533,7 +9558,130 @@ ieee80211_emulate_switch_vif_chanctx(struct ieee80211_hw *hw, } /* -------------------------------------------------------------------------- */ +/* LinuxKPI 802.11 PM. */ +int +lkpi_80211_suspend(struct ieee80211com *ic, pm_message_t state) +{ + struct lkpi_hw *lhw; + struct ieee80211_hw *hw; + int error; + + lhw = ic->ic_softc; + hw = LHW_TO_HW(lhw); + error = 0; + + /* Check: + * - device_set_wakeup_capable() / device_can_wakeup() + * - hw->wiphy->wowlan to be non-NULL, if so contents. + * - hw->wiphy->max_sched_scan_ssids (rtw88) + */ + if ((lkpi_suspend_type & 0x2) != 0) { + struct cfg80211_wowlan wowlan; + + IMPROVE("various options for WoWLAN"); + memset(&wowlan, 0, sizeof(wowlan)); + wiphy_lock(hw->wiphy); + error = lkpi_80211_mo_suspend(hw, &wowlan); + wiphy_unlock(hw->wiphy); + if (error == EOPNOTSUPP) + error = 0; + } + if ((lkpi_suspend_type & 0x1) != 0) { + struct lkpi_vif *lvif; + + ieee80211_suspend_all(ic); + wiphy_lock(hw->wiphy); + /* + * At the end of this net80211 will run a task to call + * (*ic_parent)() which is entirely unhelpful as we do not + * know when it will happen. So deal with it here. + */ + TAILQ_FOREACH(lvif, &lhw->lvif_head, lvif_entry) { + lkpi_80211_mo_remove_interface(hw, LVIF_TO_VIF(lvif)); + } + + if ((lhw->sc_flags & LKPI_MAC80211_DRV_STARTED) != 0) + lkpi_80211_mo_stop(hw, true); + wiphy_unlock(hw->wiphy); + } + + if (error < 0) + error = -error; + + if (error != 0) + ic_printf(ic, "%s: SUSPEND FAILED: %d\n", __func__, error); + + return (error); +} + +int +lkpi_80211_resume(struct ieee80211com *ic) +{ + struct lkpi_hw *lhw; + struct ieee80211_hw *hw; + int error; + bool hw_scan_running; + + lhw = ic->ic_softc; + hw = LHW_TO_HW(lhw); + error = 0; + + /* + * Ongoing HW scans during suspend are a problem on resume. + * Be verbose about that. + */ + LKPI_80211_LHW_SCAN_LOCK(lhw); + hw_scan_running = (lhw->scan_flags & (LKPI_LHW_SCAN_RUNNING|LKPI_LHW_SCAN_HW)) != 0; + LKPI_80211_LHW_SCAN_UNLOCK(lhw); + if (hw_scan_running) + ic_printf(ic, "%s: WARNING: ongoing hw scan on resume!\n", __func__); + + if ((lkpi_suspend_type & 0x1) != 0) { + struct lkpi_vif *lvif; + + wiphy_lock(hw->wiphy); + error = lkpi_80211_mo_start(hw); + if (error != 0 && error != EEXIST) { + ic_printf(ic, "%s: mo_start failed: %d\n", + __func__, error); + wiphy_unlock(hw->wiphy); + goto err; + } + + TAILQ_FOREACH(lvif, &lhw->lvif_head, lvif_entry) { + error = lkpi_80211_mo_add_interface(hw, LVIF_TO_VIF(lvif)); + if (error != 0) { + struct ieee80211vap *vap; + + vap = LVIF_TO_VAP(lvif); + ic_printf(ic, "%s: mo_add_interface %s failed: %d\n", + __func__, if_name(vap->iv_ifp), error); + wiphy_unlock(hw->wiphy); + goto err; + } + } + wiphy_unlock(hw->wiphy); + + ieee80211_resume_all(ic); + } + + if ((lkpi_suspend_type & 0x2) != 0) { + wiphy_lock(hw->wiphy); + error = lkpi_80211_mo_resume(hw); + wiphy_unlock(hw->wiphy); + if (error == EOPNOTSUPP) + error = 0; + } + +err: + if (error < 0) + error = -error; + + return (error); +} + +/* -------------------------------------------------------------------------- */ MODULE_VERSION(linuxkpi_wlan, 1); MODULE_DEPEND(linuxkpi_wlan, linuxkpi, 1, 1, 1); MODULE_DEPEND(linuxkpi_wlan, wlan, 1, 1, 1); diff --git a/sys/compat/linuxkpi/common/src/linux_80211.h b/sys/compat/linuxkpi/common/src/linux_80211.h index 569c4f12f6d6..89416edfae73 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211.h +++ b/sys/compat/linuxkpi/common/src/linux_80211.h @@ -1,6 +1,6 @@ /*- * Copyright (c) 2020-2026 The FreeBSD Foundation - * Copyright (c) 2020-2021 Bjoern A. Zeeb + * Copyright (c) 2020-2025 Bjoern A. Zeeb * * This software was developed by Björn Zeeb under sponsorship from * the FreeBSD Foundation. @@ -44,6 +44,9 @@ #include "opt_wlan.h" +#include <linux/skbuff.h> +#include <net/mac80211.h> + #if defined(IEEE80211_DEBUG) && !defined(LINUXKPI_DEBUG_80211) #define LINUXKPI_DEBUG_80211 #endif @@ -504,5 +507,16 @@ int lkpi_80211_mo_ampdu_action(struct ieee80211_hw *, struct ieee80211_vif *, struct ieee80211_ampdu_params *); int lkpi_80211_mo_sta_statistics(struct ieee80211_hw *, struct ieee80211_vif *, struct ieee80211_sta *, struct station_info *); +int lkpi_80211_mo_suspend(struct ieee80211_hw *, struct cfg80211_wowlan *); +int lkpi_80211_mo_resume(struct ieee80211_hw *); +int lkpi_80211_mo_set_wakeup(struct ieee80211_hw *, bool); +int lkpi_80211_mo_set_rekey_data(struct ieee80211_hw *, + struct ieee80211_vif *, struct cfg80211_gtk_rekey_data *); +int lkpi_80211_mo_set_default_unicast_key(struct ieee80211_hw *, + struct ieee80211_vif *, int); + +/* LinuxKPI 802.11 PM. */ +int lkpi_80211_suspend(struct ieee80211com *, pm_message_t); +int lkpi_80211_resume(struct ieee80211com *); #endif /* _LKPI_SRC_LINUX_80211_H */ diff --git a/sys/compat/linuxkpi/common/src/linux_80211_macops.c b/sys/compat/linuxkpi/common/src/linux_80211_macops.c index 42067e36c953..aa6b158b70a7 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211_macops.c +++ b/sys/compat/linuxkpi/common/src/linux_80211_macops.c @@ -819,3 +819,119 @@ lkpi_80211_mo_sta_statistics(struct ieee80211_hw *hw, struct ieee80211_vif *vif, out: return (error); } + +int +lkpi_80211_mo_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan) +{ + struct lkpi_hw *lhw; + int error; + + might_sleep(); + lockdep_assert_wiphy(hw->wiphy); + + lhw = HW_TO_LHW(hw); + if (lhw->ops->suspend == NULL) { + error = EOPNOTSUPP; + goto out; + } + + LKPI_80211_TRACE_MO("hw %p wowlan %p", hw, wowlan); + error = lhw->ops->suspend(hw, wowlan); + +out: + return (error); +} + +int +lkpi_80211_mo_resume(struct ieee80211_hw *hw) +{ + struct lkpi_hw *lhw; + int error; + + might_sleep(); + lockdep_assert_wiphy(hw->wiphy); + + lhw = HW_TO_LHW(hw); + if (lhw->ops->resume == NULL) { + error = EOPNOTSUPP; + goto out; + } + + LKPI_80211_TRACE_MO("hw %p", hw); + error = lhw->ops->resume(hw); + +out: + return (error); +} + +int +lkpi_80211_mo_set_wakeup(struct ieee80211_hw *hw, bool enable) +{ + struct lkpi_hw *lhw; + int error; + + might_sleep(); + lockdep_assert_wiphy(hw->wiphy); + + lhw = HW_TO_LHW(hw); + if (lhw->ops->set_wakeup == NULL) { + error = EOPNOTSUPP; + goto out; + } + + LKPI_80211_TRACE_MO("hw %p enable %d", hw, enable); + lhw->ops->set_wakeup(hw, enable); + error = 0; + +out: + return (error); +} + +int +lkpi_80211_mo_set_rekey_data(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, struct cfg80211_gtk_rekey_data *grd) +{ + struct lkpi_hw *lhw; + int error; + + might_sleep(); + lockdep_assert_wiphy(hw->wiphy); + + lhw = HW_TO_LHW(hw); + if (lhw->ops->set_rekey_data == NULL) { + error = EOPNOTSUPP; + goto out; + } + + LKPI_80211_TRACE_MO("hw %p vif %p grd %p", hw, vif, grd); + lhw->ops->set_rekey_data(hw, vif, grd); + error = 0; + +out: + return (error); +} + +int +lkpi_80211_mo_set_default_unicast_key(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, int idx) +{ + struct lkpi_hw *lhw; + int error; + + might_sleep(); + lockdep_assert_wiphy(hw->wiphy); + + lhw = HW_TO_LHW(hw); + if (lhw->ops->set_default_unicast_key == NULL) { + error = EOPNOTSUPP; + goto out; + } + + LKPI_80211_TRACE_MO("hw %p vif %p idx %d", hw, vif, idx); + lhw->ops->set_default_unicast_key(hw, vif, idx); + error = 0; + +out: + return (error); +} + diff --git a/sys/compat/linuxkpi/common/src/linux_acpi.c b/sys/compat/linuxkpi/common/src/linux_acpi.c index 85a3afe5d01d..680d111d0194 100644 --- a/sys/compat/linuxkpi/common/src/linux_acpi.c +++ b/sys/compat/linuxkpi/common/src/linux_acpi.c @@ -126,7 +126,7 @@ linux_handle_power_suspend_event(void *arg __unused, enum power_stype stype) /* * XXX: obiwac Not 100% sure this is correct, but * acpi_target_sleep_state does seem to be set to - * ACPI_STATE_S3 during s2idle on Linux. + * ACPI_STATE_S3 during suspend-to-idle (aka s2idle) on Linux. */ linux_acpi_target_sleep_state = ACPI_STATE_S3; pm_suspend_target_state = PM_SUSPEND_TO_IDLE; diff --git a/sys/compat/linuxkpi/common/src/linuxkpi_80211_pm.c b/sys/compat/linuxkpi/common/src/linuxkpi_80211_pm.c new file mode 100644 index 000000000000..c69288bd5886 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linuxkpi_80211_pm.c @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2025 The FreeBSD Foundation + * + * This software was developed by Björn Zeeb under sponsorship from + * the FreeBSD Foundation. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/module.h> + +#include <linux/pci.h> +#include "linux_80211.h" + +#include <net80211/ieee80211_var.h> + +struct lkpi_80211_pm_softc { + /* PCI */ + int (*suspend) (struct pci_dev *pdev, pm_message_t state); + int (*resume) (struct pci_dev *pdev); +}; + +static int +lkpi_80211_pm_suspend(struct pci_dev *pdev, pm_message_t state) +{ + const struct dev_pm_ops *pmops; + struct lkpi_80211_pm_softc *sc; + struct ieee80211com *ic; + device_t dev; + int error; + + dev = device_find_child(pdev->dev.bsddev, "lkpi80211_pm", + DEVICE_UNIT_ANY); + if (dev == NULL) { + /* Must not happen, so abort suspend if it does. */ + device_printf(pdev->dev.bsddev, + "%s: cannot find lkpi80211_pm child for %s\n", + __func__, device_get_name(pdev->dev.bsddev)); + return (ENXIO); + } + sc = device_get_softc(dev); + error = 0; + + /* Call order: wireless then pdev. */ + + ic = ieee80211_find_com(device_get_nameunit(pdev->dev.bsddev)); + if (ic != NULL) { + error = lkpi_80211_suspend(ic, state); + } else { + device_printf(pdev->dev.bsddev, + "%s: WARNING: wireless device not found\n", __func__); + } + if (error != 0) + goto err; + + /* Logic duplicated from linux_pci_suspend(). */ + pmops = pdev->pdrv->driver.pm; + if (sc->suspend != NULL) + error = sc->suspend(pdev, state); + else if (pmops != NULL && pmops->suspend != NULL) { + error = -pmops->suspend(&pdev->dev); + if (error == 0 && pmops->suspend_late != NULL) + error = -pmops->suspend_late(&pdev->dev); + if (error == 0 && pmops->suspend_noirq != NULL) + error = -pmops->suspend_noirq(&pdev->dev); + } + +err: + if (error < 0) + error = -error; + + if (error != 0) + device_printf(pdev->dev.bsddev, + "%s: WARNING: SUSPEND FAILED: %d\n", __func__, error); + + return (error); +} + +static int +lkpi_80211_pm_resume(struct pci_dev *pdev) +{ + const struct dev_pm_ops *pmops; + struct lkpi_80211_pm_softc *sc; + struct ieee80211com *ic; + device_t dev; + int error; + + dev = device_find_child(pdev->dev.bsddev, "lkpi80211_pm", + DEVICE_UNIT_ANY); + if (dev == NULL) { + /* Must not happen, so abort suspend if it does. */ + device_printf(pdev->dev.bsddev, + "%s: cannot find lkpi80211_pm child\n", __func__); + return (ENXIO); + } + sc = device_get_softc(dev); + error = 0; + + /* Call order: pdev then wireless. */ + + /* Logic duplicated from linux_pci_resume(). */ + pmops = pdev->pdrv->driver.pm; + if (sc->resume != NULL) { + error = sc->resume(pdev); + } else if (pmops != NULL && pmops->resume != NULL) { + if (pmops->resume_early != NULL) + error = -pmops->resume_early(&pdev->dev); + if (error == 0 && pmops->resume != NULL) + error = -pmops->resume(&pdev->dev); + } + if (error != 0) + device_printf(pdev->dev.bsddev, "%s: resume failed!\n", __func__); + /* Do not error out but give wireless also a chance. */ + + ic = ieee80211_find_com(device_get_nameunit(pdev->dev.bsddev)); + if (ic != NULL) { + error = lkpi_80211_resume(ic); + } else { + device_printf(pdev->dev.bsddev, + "%s: WARNING: wireless device not found\n", __func__); + } + + if (error < 0) + error = -error; + + return (error); +} + +/* -------------------------------------------------------------------------- */ +static void +lkpi_80211_pm_identify(driver_t *driver, device_t parent) +{ + + /* Make sure we're not being doubly invoked per parent. */ + if (device_find_child(parent, driver->name, DEVICE_UNIT_ANY) != NULL) + return; + + /* Make sure this is PCI for now. */ + if (device_get_devclass(parent) == devclass_find("pci")) + return; + + if (BUS_ADD_CHILD(parent, 0, driver->name, DEVICE_UNIT_ANY) == NULL) + device_printf(parent, "%s: failed to add child\n", __func__); +} + +static int +lkpi_80211_pm_probe(device_t dev) +{ + device_set_descf(dev, "LinuxKPI 802.11 %s mac80211 PM", + device_get_nameunit(device_get_parent(dev))); + return (BUS_PROBE_DEFAULT); +} + +static int +lkpi_80211_pm_attach(device_t dev) +{ + struct lkpi_80211_pm_softc *sc; + struct pci_dev *pdev; + + sc = device_get_softc(dev); + pdev = device_get_softc(device_get_parent(dev)); + + /* Intercept the driver suspend/resume calls. */ + sc->suspend = pdev->pdrv->suspend; + pdev->pdrv->suspend = lkpi_80211_pm_suspend; + sc->resume = pdev->pdrv->resume; + pdev->pdrv->resume = lkpi_80211_pm_resume; + + return (0); +} + +static int +lkpi_80211_pm_detach(device_t dev) +{ + struct lkpi_80211_pm_softc *sc; + struct pci_dev *pdev; + + sc = device_get_softc(dev); + pdev = device_get_softc(device_get_parent(dev)); + + /* Restore the original notifications. */ + pdev->pdrv->suspend = sc->suspend; + pdev->pdrv->resume = sc->resume; + + return (0); +} + +static device_method_t lkpi_80211_pm_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, lkpi_80211_pm_identify), + DEVMETHOD(device_probe, lkpi_80211_pm_probe), + DEVMETHOD(device_attach, lkpi_80211_pm_attach), + DEVMETHOD(device_detach, lkpi_80211_pm_detach), + /* + * Do not think about device_suspend/resume here. + * We are not a PCI device and LinuxKPI PCI linux_pci_suspend/resume + * are getting the notifications so we have to hijack the + * LinuxKPI upcalls. + */ + + DEVMETHOD_END +}; + +driver_t lkpi_80211_pm_driver = { + "lkpi80211_pm", + lkpi_80211_pm_methods, + sizeof(struct lkpi_80211_pm_softc), +}; + +MODULE_DEPEND(lkpi80211_pm, linuxkpi_wlan, 1, 1, 1); +MODULE_VERSION(lkpi80211_pm, 1); diff --git a/sys/conf/files b/sys/conf/files index fac94252a362..379685d83713 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -4665,6 +4665,8 @@ compat/linuxkpi/common/src/linux_80211.c optional compat_linuxkpi wlan \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_80211_macops.c optional compat_linuxkpi wlan \ compile-with "${LINUXKPI_C}" +compat/linuxkpi/common/src/linuxkpi_80211_pm.c optional compat_linuxkpi wlan \ + compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_kmod.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_acpi.c optional compat_linuxkpi acpi \ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 172f79cc5773..2f6052da7e51 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -76,6 +76,7 @@ arm64/arm64/ofw_machdep.c optional fdt arm64/arm64/pl031_rtc.c optional fdt pl031 arm64/arm64/ptrauth.c standard \ compile-with "${NORMAL_C:N-mbranch-protection*} -mbranch-protection=bti" +arm64/arm64/mte.c standard arm64/arm64/pmap.c standard arm64/arm64/ptrace_machdep.c standard arm64/arm64/rsi.c standard diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index 0e7a1f24be7e..b65d78d1ea3c 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -33,7 +33,6 @@ dev/hyperv/vmbus/i386/hyperv_machdep.c optional hyperv dev/le/if_le_isa.c optional le isa dev/ofw/ofw_pcib.c optional fdt pci dev/pcf/pcf_isa.c optional pcf -dev/random/nehemiah.c optional padlock_rng !random_loadable dev/sbni/if_sbni.c optional sbni dev/sbni/if_sbni_isa.c optional sbni isa dev/sbni/if_sbni_pci.c optional sbni pci diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 index 8a7e0b78feb4..a9b9d8875ad1 100644 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -310,6 +310,7 @@ dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/ntb/test/ntb_tool.c optional ntb_tool dev/nvram/nvram.c optional nvram isa dev/random/ivy.c optional rdrand_rng !random_loadable +dev/random/nehemiah.c optional padlock_rng !random_loadable dev/random/rdseed.c optional rdrand_rng !random_loadable dev/qat_c2xxx/qat.c optional qat_c2xxx dev/qat_c2xxx/qat_ae.c optional qat_c2xxx diff --git a/sys/contrib/dev/iwlwifi/lkpi_iwlwifi_pm.c b/sys/contrib/dev/iwlwifi/lkpi_iwlwifi_pm.c new file mode 100644 index 000000000000..7843e27d559c --- /dev/null +++ b/sys/contrib/dev/iwlwifi/lkpi_iwlwifi_pm.c @@ -0,0 +1,8 @@ +#include <sys/types.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> + +extern driver_t lkpi_80211_pm_driver; +DRIVER_MODULE(lkpi80211_pm, iwlwifi, lkpi_80211_pm_driver, 0, 0); + diff --git a/sys/contrib/dev/rtw88/lkpi_rtw88_pm.c b/sys/contrib/dev/rtw88/lkpi_rtw88_pm.c new file mode 100644 index 000000000000..53da7b2ea715 --- /dev/null +++ b/sys/contrib/dev/rtw88/lkpi_rtw88_pm.c @@ -0,0 +1,8 @@ +#include <sys/types.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> + +extern driver_t lkpi_80211_pm_driver; +DRIVER_MODULE(lkpi80211_pm, rtw88, lkpi_80211_pm_driver, 0, 0); + diff --git a/sys/contrib/dev/rtw89/lkpi_rtw89_pm.c b/sys/contrib/dev/rtw89/lkpi_rtw89_pm.c new file mode 100644 index 000000000000..6f75557fa7ca --- /dev/null +++ b/sys/contrib/dev/rtw89/lkpi_rtw89_pm.c @@ -0,0 +1,8 @@ +#include <sys/types.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> + +extern driver_t lkpi_80211_pm_driver; +DRIVER_MODULE(lkpi80211_pm, rtw89, lkpi_80211_pm_driver, 0, 0); + diff --git a/sys/contrib/edk2/Include/Base.h b/sys/contrib/edk2/Include/Base.h index 3ae798db8429..d3342c9a6ca2 100644 --- a/sys/contrib/edk2/Include/Base.h +++ b/sys/contrib/edk2/Include/Base.h @@ -841,10 +841,8 @@ STATIC_ASSERT (ALIGNOF (INT16) == sizeof (INT16), "Alignment of INT16 does not STATIC_ASSERT (ALIGNOF (UINT16) == sizeof (UINT16), "Alignment of UINT16 does not meet UEFI Specification Data Type requirements"); STATIC_ASSERT (ALIGNOF (INT32) == sizeof (INT32), "Alignment of INT32 does not meet UEFI Specification Data Type requirements"); STATIC_ASSERT (ALIGNOF (UINT32) == sizeof (UINT32), "Alignment of UINT32 does not meet UEFI Specification Data Type requirements"); -#ifndef _STANDALONE STATIC_ASSERT (ALIGNOF (INT64) == sizeof (INT64), "Alignment of INT64 does not meet UEFI Specification Data Type requirements"); STATIC_ASSERT (ALIGNOF (UINT64) == sizeof (UINT64), "Alignment of UINT64 does not meet UEFI Specification Data Type requirements"); -#endif STATIC_ASSERT (ALIGNOF (CHAR8) == sizeof (CHAR8), "Alignment of CHAR8 does not meet UEFI Specification Data Type requirements"); STATIC_ASSERT (ALIGNOF (CHAR16) == sizeof (CHAR16), "Alignment of CHAR16 does not meet UEFI Specification Data Type requirements"); STATIC_ASSERT (ALIGNOF (INTN) == sizeof (INTN), "Alignment of INTN does not meet UEFI Specification Data Type requirements"); diff --git a/sys/ddb/db_ps.c b/sys/ddb/db_ps.c index 8e027997ea75..59dcde453b93 100644 --- a/sys/ddb/db_ps.c +++ b/sys/ddb/db_ps.c @@ -358,8 +358,7 @@ DB_SHOW_COMMAND(thread, db_show_thread) if (td->td_name[0] != '\0') db_printf(" name: %s\n", td->td_name); db_printf(" pcb: %p\n", td->td_pcb); - db_printf(" stack: %p-%p\n", td->td_kstack, - td->td_kstack + td->td_kstack_pages * PAGE_SIZE - 1); + db_printf(" stack: %p-%p\n", td->td_kstack, td_kstack_top(td) - 1); db_printf(" flags: %#x ", td->td_flags); db_printf(" pflags: %#x\n", td->td_pflags); db_printf(" state: "); diff --git a/sys/dev/acpi_support/acpi_ibm.c b/sys/dev/acpi_support/acpi_ibm.c index 693d793532c1..a5c44b1f81b9 100644 --- a/sys/dev/acpi_support/acpi_ibm.c +++ b/sys/dev/acpi_support/acpi_ibm.c @@ -1449,8 +1449,8 @@ acpi_ibm_eventhandler(struct acpi_ibm_softc *sc, int arg) switch (arg) { /* * XXX "Suspend-to-RAM" here is as opposed to suspend-to-disk, but it is - * fine if our suspend sleep state transition request puts us in s2idle - * instead of suspend-to-RAM. + * fine if our suspend sleep state transition request puts us in + * suspend-to-idle instead of actual suspend-to-RAM. */ case IBM_EVENT_SUSPEND_TO_RAM: (void)power_pm_suspend(POWER_TRANSITION_SUSPEND); diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c index 8be94db73729..5cb0afa581ca 100644 --- a/sys/dev/acpica/acpi.c +++ b/sys/dev/acpica/acpi.c @@ -666,10 +666,11 @@ acpi_attach(device_t dev) /* * Pick the first valid sleep type for the sleep button default. If that - * type was hibernate and we support s2idle, set it to that. The sleep - * button prefers s2mem instead of s2idle at the moment as s2idle may not - * yet work reliably on all machines. In the future, we should set this to - * s2idle when ACPI_FADT_LOW_POWER_S0 is set. + * type was hibernate and we support suspend_to_idle , set it to that. The + * sleep button prefers fw_suspend instead of suspend_to_idle at the moment + * as suspend_to_idle may not yet work reliably on all machines. In the + * future, we should set this to suspend_to_idle when + * ACPI_FADT_LOW_POWER_S0 is set. */ sc->acpi_sleep_button_stype = POWER_STYPE_UNKNOWN; for (stype = POWER_STYPE_STANDBY; stype <= POWER_STYPE_FW_HIBERNATE; stype++) @@ -743,7 +744,7 @@ acpi_attach(device_t dev) OID_AUTO, "lid_switch_state", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, &sc->acpi_lid_switch_stype, 0, acpi_stype_sysctl, "A", - "Lid ACPI sleep state. Set to s2idle or s2mem if you want to suspend " + "Lid ACPI sleep state. Set to suspend_to_idle or fw_suspend if you want to suspend " "your laptop when you close the lid."); SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree), OID_AUTO, "suspend_state", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, @@ -2154,9 +2155,9 @@ acpi_device_pwr_for_sleep_sxd(device_t dev, ACPI_HANDLE handle, int state, * we are currently entering (sc->acpi_stype is set in acpi_EnterSleepState * before the ACPI bus gets suspended, and thus before this function is called). * - * If entering s2idle, we will try to enter whichever D-state we would've been - * transitioning to in S3. If we are entering an ACPI S-state, we evaluate the - * relevant _SxD state instead (ACPI 7.3.16 - 7.3.19). + * If entering suspend_to_idle, we will try to enter whichever D-state we + * would've been transitioning to in S3. If we are entering an ACPI S-state, we + * evaluate the relevant _SxD state instead (ACPI 7.3.16 - 7.3.19). */ int acpi_device_pwr_for_sleep(device_t bus, device_t dev, int *dstate) @@ -5064,7 +5065,7 @@ acpi_pm_func(u_long cmd, void *arg, enum power_stype stype) error = EINVAL; goto out; } - if (ACPI_FAILURE(acpi_EnterSleepState(sc, stype))) + if (ACPI_FAILURE(acpi_ReqSleepState(sc, stype))) error = ENXIO; break; default: diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c index 9ac591c14943..d37210723680 100644 --- a/sys/dev/ath/if_ath_tx.c +++ b/sys/dev/ath/if_ath_tx.c @@ -6225,7 +6225,8 @@ ath_bar_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, * Also, don't call it if bar_tx/bar_wait are 0; something * has beaten us to the punch? (XXX figure out what?) */ - if (status == 0 || attempts == 50) { + if (status == 0 || + ieee80211_ht_check_bar_exceed_retry_count(ni, attempts)) { ATH_TX_LOCK(sc); if (atid->bar_tx == 0 || atid->bar_wait == 0) DPRINTF(sc, ATH_DEBUG_SW_TX_BAR, diff --git a/sys/dev/bnxt/bnxt_en/bnxt_sriov.c b/sys/dev/bnxt/bnxt_en/bnxt_sriov.c index 270c18165fb7..071feffbadfd 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_sriov.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_sriov.c @@ -7,6 +7,8 @@ #include "bnxt_hwrm.h" #include "bnxt_sriov.h" +#ifdef PCI_IOV + static int bnxt_set_vf_admin_mac(struct bnxt_softc *softc, struct bnxt_vf_info *vf, const uint8_t *mac) @@ -973,6 +975,43 @@ void bnxt_sriov_attach(struct bnxt_softc *softc) device_printf(dev, "Failed to initialize SR-IOV (error=%d)\n", rc); } +#else + +void +bnxt_sriov_attach(struct bnxt_softc *softc __unused) +{ +} + +int +bnxt_cfg_hw_sriov(struct bnxt_softc *softc __unused, + uint16_t *num_vfs __unused, bool reset __unused) +{ + return (0); +} + +int +bnxt_approve_mac(struct bnxt_softc *sc __unused) +{ + return (0); +} + +void +bnxt_hwrm_exec_fwd_req(struct bnxt_softc *softc __unused) +{ +} + +bool +bnxt_promisc_ok(struct bnxt_softc *softc __unused) +{ + return (true); +} + +void +bnxt_update_vf_mac(struct bnxt_softc *sc __unused) +{ +} +#endif + void bnxt_reenable_sriov(struct bnxt_softc *bp) { if (BNXT_PF(bp)) { diff --git a/sys/dev/bnxt/bnxt_en/bnxt_sriov.h b/sys/dev/bnxt/bnxt_en/bnxt_sriov.h index 176f54af0aa8..24ea11f29b83 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_sriov.h +++ b/sys/dev/bnxt/bnxt_en/bnxt_sriov.h @@ -8,10 +8,6 @@ #include "opt_global.h" #include "bnxt.h" -#ifndef PCI_IOV -#define PCI_IOV 1 -#endif - /* macro definations */ #define BNXT_MAX_VFS 4 diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c index 6618016f3932..6d82302615e1 100644 --- a/sys/dev/bnxt/bnxt_en/if_bnxt.c +++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c @@ -2875,11 +2875,9 @@ bnxt_attach_post(if_ctx_t ctx) bnxt_dcb_init(softc); bnxt_rdma_aux_device_init(softc); -#if PCI_IOV /* SR-IOV attach */ if (BNXT_PF(softc) && BNXT_CHIP_P5_PLUS(softc)) bnxt_sriov_attach(softc); -#endif failed: return rc; diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 24a482b74dfb..8c5cf052b689 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -194,6 +194,7 @@ enum { IHF_CLR_ALL_SET = (1 << 5), /* Clear all set bits */ IHF_CLR_ALL_UNIGNORED = (1 << 6), /* Clear all unignored bits */ IHF_RUN_ALL_ACTIONS = (1 << 7), /* As if all cause are set */ + IHF_CLR_DELAYED = (1 << 9), /* Cleared in a delayed call */ }; #define IS_DETACHING(vi) ((vi)->flags & VI_DETACHING) diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c index f4eef54e5c6b..41606201ad39 100644 --- a/sys/dev/cxgbe/common/t4_hw.c +++ b/sys/dev/cxgbe/common/t4_hw.c @@ -4794,6 +4794,27 @@ struct intr_info { const struct intr_action *actions; }; +/* Helper to clear interrupts that have IHF_CLR_DELAYED. */ +static void +clear_int_cause_reg(struct adapter *sc, const struct intr_info *ii, int flags) +{ + u32 cause, ucause; + + cause = ucause = t4_read_reg(sc, ii->cause_reg); + if (cause == 0) + return; + flags |= ii->flags; + if (flags & IHF_IGNORE_IF_DISABLED) + ucause &= t4_read_reg(sc, ii->enable_reg); + if (flags & IHF_CLR_ALL_SET) { + t4_write_reg(sc, ii->cause_reg, cause); + (void)t4_read_reg(sc, ii->cause_reg); + } else if (ucause != 0 && flags & IHF_CLR_ALL_UNIGNORED) { + t4_write_reg(sc, ii->cause_reg, ucause); + (void)t4_read_reg(sc, ii->cause_reg); + } +} + static inline char intr_alert_char(u32 cause, u32 enable, u32 fatal) { @@ -4869,8 +4890,8 @@ t4_handle_intr(struct adapter *sc, const struct intr_info *ii, uint32_t acause, } } - /* clear */ - if (cause != 0) { + /* Clear here unless delayed clear is requested. */ + if (cause != 0 && (flags & IHF_CLR_DELAYED) == 0) { if (flags & IHF_CLR_ALL_SET) { t4_write_reg(sc, ii->cause_reg, cause); (void)t4_read_reg(sc, ii->cause_reg); @@ -5003,22 +5024,63 @@ static bool pcie_intr_handler(struct adapter *adap, int arg, int flags) .details = NULL, .actions = NULL, }; + static const struct intr_details pcie_intr_cause_ext_details[] = { + { F_IPFORMQPERR, "PCIe IP FormQ Buffer PERR" }, + { F_IPFORMQCERR, "PCIe IP FormQ Buffer CERR" }, + { F_TRGT1GRPCERR, "TRGT1 Group FIFOs CERR" }, + { F_IPSOTCERR, "PCIe IP SOT Buffer SRAM CERR" }, + { F_IPRETRYCERR, "PCIe IP Replay Buffer CERR" }, + { F_IPRXDATAGRPCERR, "PCIe IP Rx Data Group SRAMs CERR" }, + { F_IPRXHDRGRPCERR, "PCIe IP Rx Header Group SRAMs CERR" }, + { F_A0ARBRSPORDFIFOPERR, "A0 Arbiter Response Order FIFO Parity Error" }, + { F_HRSPCERR, "Master HMA Channel Response Data SRAM CERR" }, + { F_HREQRDCERR, "Master HMA Channel Read Request SRAM CERR" }, + { F_HREQWRCERR, "Master HMA Channel Write Request SRAM CERR" }, + { F_DRSPCERR, "Master DMA Channel Response Data SRAM CERR" }, + { F_DREQRDCERR, "Master DMA Channel Read Request SRAM CERR" }, + { F_DREQWRCERR, "Master DMA Channel Write Request SRAM CERR" }, + { F_CRSPCERR, "Master CMD Channel Response Data SRAM CERR" }, + { F_ARSPPERR, "Master ARM Channel Response Data SRAM PERR" }, + { F_AREQRDPERR, "Master ARM Channel Read Request SRAM PERR" }, + { F_AREQWRPERR, "Master ARM Channel Write Request SRAM PERR" }, + { F_PIOREQGRPCERR, "PIO Request Group FIFOs CERR" }, + { F_ARSPCERR, "Master ARM Channel Response Data SRAM CERR" }, + { F_AREQRDCERR, "Master ARM Channel Read Request SRAM CERR" }, + { F_AREQWRCERR, "Master ARM Channel Write Request SRAM CERR" }, + { F_MARSPPERR, "INIC MA Ctrl and Data Rsp Perr" }, + { F_INICMAWDATAORDPERR, "INIC Ma Arb Write Ord Data Fifo Perr" }, + { F_EMUPERR, "CFG EMU SRAM PERR" }, + { F_ERRSPPERR, "CFG EMU SRAM CERR" }, + { F_MSTGRPCERR, "Master Data Path and Response Read Queue SRAM CERR" }, + { 0 } + }; struct intr_info pcie_int_cause_ext = { .name = "PCIE_INT_CAUSE_EXT", .cause_reg = A_PCIE_INT_CAUSE_EXT, .enable_reg = A_PCIE_INT_ENABLE_EXT, .fatal = 0, .flags = 0, - .details = NULL, + .details = pcie_intr_cause_ext_details, .actions = NULL, }; + static const struct intr_details pcie_intr_cause_x8_details[] = { + { F_X8TGTGRPPERR, "x8 TGT Group FIFOs parity error" }, + { F_X8IPSOTPERR, "PCIe x8 IP SOT Buffer SRAM PERR" }, + { F_X8IPRETRYPERR, "PCIe x8 IP Replay Buffer PERR" }, + { F_X8IPRXDATAGRPPERR, "PCIe x8 IP Rx Data Group SRAMs PERR" }, + { F_X8IPRXHDRGRPPERR, "PCIe x8 IP Rx Header Group SRAMs PERR" }, + { F_X8IPCORECERR, "x8 IP SOT, Retry, RxData, RxHdr SRAM CERR" }, + { F_X8MSTGRPPERR, "x8 Master Data Path and Response Read Queue SRAM PERR" }, + { F_X8MSTGRPCERR, "x8 Master Data Path and Response Read Queue SRAM CERR" }, + { 0 } + }; struct intr_info pcie_int_cause_x8 = { .name = "PCIE_INT_CAUSE_X8", .cause_reg = A_PCIE_INT_CAUSE_X8, .enable_reg = A_PCIE_INT_ENABLE_X8, .fatal = 0, .flags = 0, - .details = NULL, + .details = pcie_intr_cause_x8_details, .actions = NULL, }; bool fatal = false; @@ -5050,80 +5112,247 @@ static bool tp_intr_handler(struct adapter *adap, int arg, int flags) { F_FLMTXFLSTEMPTY, "TP out of Tx pages" }, { 0 } }; - static const struct intr_info tp_intr_info = { + static const struct intr_details t7_tp_intr_details[] = { + { F_FLMTXFLSTEMPTY, "Offload memory manager Tx free list empty" }, + { F_TPCERR, "TP modules flagged Correctable Error" }, + { F_OTHERPERR, "TP Other modules (Core, TM, FLM, MMGR, DB) Parity Error" }, + { F_TPEING1PERR, "TP-ESide Ingress1 Parity Error" }, + { F_TPEING0PERR, "TP-ESide Ingress0 Parity Error" }, + { F_TPEEGPERR, "TP-ESide Egress Parity Error" }, + { F_TPCPERR, "TP-CSide Parity Error" }, + { 0 } + }; + struct intr_info tp_intr_info = { .name = "TP_INT_CAUSE", .cause_reg = A_TP_INT_CAUSE, .enable_reg = A_TP_INT_ENABLE, .fatal = 0x7fffffff, - .flags = IHF_FATAL_IFF_ENABLED, - .details = tp_intr_details, + .flags = IHF_FATAL_IFF_ENABLED | IHF_CLR_DELAYED, + .details = NULL, .actions = NULL, }; - static const struct intr_info tp_inic_perr_cause = { - .name = "TP_INIC_PERR_CAUSE", - .cause_reg = A_TP_INIC_PERR_CAUSE, - .enable_reg = A_TP_INIC_PERR_ENABLE, + static const struct intr_details tp_cerr_cause_details[] = { + { F_TPCEGDATAFIFO, "TPCSide Egress Data FIFO" }, + { F_TPCLBKDATAFIFO, "TPCSide Loopback Data FIFO" }, + { F_RSSLKPSRAM, "RSS Lookup SRAM" }, + { F_SRQSRAM, "SRQ SRAM" }, + { F_ARPDASRAM, "ARP DA SRAM" }, + { F_ARPSASRAM, "ARP SA SRAM" }, + { F_ARPGRESRAM, "ARP GRE SRAM" }, + { F_ARPIPSECSRAM1, "ARP IPSec SRAM0" }, + { F_ARPIPSECSRAM0, "ARP IPSec SRAM1" }, + { 0 } + }; + static const struct intr_info tp_cerr_cause = { + .name = "TP_CERR_CAUSE", + .cause_reg = A_TP_CERR_CAUSE, + .enable_reg = A_TP_CERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = tp_cerr_cause_details, .actions = NULL, }; + static const struct intr_details tp_c_perr_details[] = { + { F_DMXFIFOOVFL, "Demux FIFO Overflow" }, + { F_URX2TPCDDPINTF, "ULPRX to TPC DDP Interface and FIFO" }, + { F_TPCDISPTOKENFIFO, "TPC Dispatch Token FIFO" }, + { F_TPCDISPCPLFIFO3, "TPC Dispatch CPL FIFO Ch3" }, + { F_TPCDISPCPLFIFO2, "TPC Dispatch CPL FIFO Ch2" }, + { F_TPCDISPCPLFIFO1, "TPC Dispatch CPL FIFO Ch1" }, + { F_TPCDISPCPLFIFO0, "TPC Dispatch CPL FIFO Ch0" }, + { F_URXPLDINTFCRC3, "ULPRX to TPC Payload Interface CRC Error Ch3" }, + { F_URXPLDINTFCRC2, "ULPRX to TPC Payload Interface CRC Error Ch2" }, + { F_URXPLDINTFCRC1, "ULPRX to TPC Payload Interface CRC Error Ch1" }, + { F_URXPLDINTFCRC0, "ULPRX to TPC Payload Interface CRC Error Ch0" }, + { F_DMXDBFIFO, "Demux DB FIFO" }, + { F_DMXDBSRAM, "Demux DB SRAM" }, + { F_DMXCPLFIFO, "Demux CPL FIFO" }, + { F_DMXCPLSRAM, "Demux CPL SRAM" }, + { F_DMXCSUMFIFO, "Demux Checksum FIFO" }, + { F_DMXLENFIFO, "Demux Length FIFO" }, + { F_DMXCHECKFIFO, "Demux Check CRC16 FIFO" }, + { F_DMXWINFIFO, "Demux Winner FIFO" }, + { F_EGTOKENFIFO, "Egress Token FIFO Parity Error" }, + { F_EGDATAFIFO, "Egress FIFO Parity Error" }, + { F_UTX2TPCINTF3, "ULPTX to TPC Interface Parity Error Ch3" }, + { F_UTX2TPCINTF2, "ULPTX to TPC Interface Parity Error Ch2" }, + { F_UTX2TPCINTF1, "ULPTX to TPC Interface Parity Error Ch1" }, + { F_UTX2TPCINTF0, "ULPTX to TPC Interface Parity Error Ch0" }, + { F_LBKTOKENFIFO, "Loopback Token FIFO Parity Error" }, + { F_LBKDATAFIFO, "Loopback FIFO Parity Error" }, + { 0 } + }; static const struct intr_info tp_c_perr_cause = { .name = "TP_C_PERR_CAUSE", .cause_reg = A_TP_C_PERR_CAUSE, .enable_reg = A_TP_C_PERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = tp_c_perr_details, .actions = NULL, }; + static const struct intr_details tp_e_eg_perr_details[] = { + { F_MPSLPBKTOKENFIFO, "MPS Loopback Token FIFO parity error" }, + { F_MPSMACTOKENFIFO, "MPS MAC Token FIFO parity error" }, + { F_DISPIPSECFIFO3, "Ch3 Dispatch IPSec FIFO parity error" }, + { F_DISPTCPFIFO3, "Ch3 Dispatch TCP FIFO parity error" }, + { F_DISPIPFIFO3, "Ch3 Dispatch IP FIFO parity error" }, + { F_DISPETHFIFO3, "Ch3 Dispatch ETH FIFO parity error" }, + { F_DISPGREFIFO3, "Ch3 Dispatch GRE FIFO parity error" }, + { F_DISPCPL5FIFO3, "Ch3 Dispatch CPL5 FIFO parity error" }, + { F_DISPIPSECFIFO2, "Ch2 Dispatch IPSec FIFO parity error" }, + { F_DISPTCPFIFO2, "Ch2 Dispatch TCP FIFO parity error" }, + { F_DISPIPFIFO2, "Ch2 Dispatch IP FIFO parity error" }, + { F_DISPETHFIFO2, "Ch2 Dispatch ETH FIFO parity error" }, + { F_DISPGREFIFO2, "Ch2 Dispatch GRE FIFO parity error" }, + { F_DISPCPL5FIFO2, "Ch2 Dispatch CPL5 FIFO parity error" }, + { F_DISPIPSECFIFO1, "Ch1 Dispatch IPSec FIFO parity error" }, + { F_DISPTCPFIFO1, "Ch1 Dispatch TCP FIFO parity error" }, + { F_DISPIPFIFO1, "Ch1 Dispatch IP FIFO parity error" }, + { F_DISPETHFIFO1, "Ch1 Dispatch ETH FIFO parity error" }, + { F_DISPGREFIFO1, "Ch1 Dispatch GRE FIFO parity error" }, + { F_DISPCPL5FIFO1, "Ch1 Dispatch CPL5 FIFO parity error" }, + { F_DISPIPSECFIFO0, "Ch0 Dispatch IPSec FIFO parity error" }, + { F_DISPTCPFIFO0, "Ch0 Dispatch TCP FIFO parity error" }, + { F_DISPIPFIFO0, "Ch0 Dispatch IP FIFO parity error" }, + { F_DISPETHFIFO0, "Ch0 Dispatch ETH FIFO parity error" }, + { F_DISPGREFIFO0, "Ch0 Dispatch GRE FIFO parity error" }, + { F_DISPCPL5FIFO0, "Ch0 Dispatch CPL5 FIFO parity error" }, + { 0 } + }; static const struct intr_info tp_e_eg_perr_cause = { .name = "TP_E_EG_PERR_CAUSE", .cause_reg = A_TP_E_EG_PERR_CAUSE, .enable_reg = A_TP_E_EG_PERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = tp_e_eg_perr_details, .actions = NULL, }; + static const struct intr_details tp_e_in0_perr_details[] = { + { F_DMXISSFIFO, "Demux ISS FIFO parity error" }, + { F_DMXERRFIFO, "Demux Error FIFO parity error" }, + { F_DMXATTFIFO, "Demux Attributes FIFO parity error" }, + { F_DMXTCPFIFO, "Demux TCP Fields FIFO parity error" }, + { F_DMXMPAFIFO, "Demux MPA FIFO parity error" }, + { F_DMXOPTFIFO, "Demux TCP Options FIFO parity error" }, + { F_INGTOKENFIFO, "Demux Ingress Token FIFO parity error" }, + { F_DMXPLDCHKOVFL1, "Ch1 PLD TxCheck FIFO Overflow" }, + { F_DMXPLDCHKFIFO1, "Ch1 PLD TxCheck FIFO parity error" }, + { F_DMXOPTFIFO1, "Ch1 Options buffer parity error" }, + { F_DMXMPAFIFO1, "Ch1 MPA FIFO parity error" }, + { F_DMXDBFIFO1, "Ch1 DB FIFO parity error" }, + { F_DMXATTFIFO1, "Ch1 Attribute FIFO parity error" }, + { F_DMXISSFIFO1, "Ch1 ISS FIFO parity error" }, + { F_DMXTCPFIFO1, "Ch1 TCP Fields FIFO parity error" }, + { F_DMXERRFIFO1, "Ch1 Error FIFO parity error" }, + { F_MPS2TPINTF1, "Ch1 MPS2TP Interface parity error" }, + { F_DMXPLDCHKOVFL0, "Ch0 PLD TxCheck FIFO Overflow" }, + { F_DMXPLDCHKFIFO0, "Ch0 PLD TxCheck FIFO parity error" }, + { F_DMXOPTFIFO0, "Ch0 Options buffer parity error" }, + { F_DMXMPAFIFO0, "Ch0 MPA FIFO parity error" }, + { F_DMXDBFIFO0, "Ch0 DB FIFO parity error" }, + { F_DMXATTFIFO0, "Ch0 Attribute FIFO parity error" }, + { F_DMXISSFIFO0, "Ch0 ISS FIFO parity error" }, + { F_DMXTCPFIFO0, "Ch0 TCP Fields FIFO parity error" }, + { F_DMXERRFIFO0, "Ch0 Error FIFO parity error" }, + { F_MPS2TPINTF0, "Ch0 MPS2TP Interface parity error" }, + { 0 } + }; static const struct intr_info tp_e_in0_perr_cause = { .name = "TP_E_IN0_PERR_CAUSE", .cause_reg = A_TP_E_IN0_PERR_CAUSE, .enable_reg = A_TP_E_IN0_PERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = tp_e_in0_perr_details, .actions = NULL, }; + static const struct intr_details tp_e_in1_perr_details[] = { + { F_DMXPLDCHKOVFL3, "Ch3 PLD TxCheck FIFO Overflow" }, + { F_DMXPLDCHKFIFO3, "Ch3 PLD TxCheck FIFO parity error" }, + { F_DMXOPTFIFO3, "Ch3 Options buffer parity error" }, + { F_DMXMPAFIFO3, "Ch3 MPA FIFO parity error" }, + { F_DMXDBFIFO3, "Ch3 DB FIFO parity error" }, + { F_DMXATTFIFO3, "Ch3 Attribute FIFO parity error" }, + { F_DMXISSFIFO3, "Ch3 ISS FIFO parity error" }, + { F_DMXTCPFIFO3, "Ch3 TCP Fields FIFO parity error" }, + { F_DMXERRFIFO3, "Ch3 Error FIFO parity error" }, + { F_MPS2TPINTF3, "Ch3 MPS2TP Interface parity error" }, + { F_DMXPLDCHKOVFL2, "Ch2 PLD TxCheck FIFO Overflow" }, + { F_DMXPLDCHKFIFO2, "Ch2 PLD TxCheck FIFO parity error" }, + { F_DMXOPTFIFO2, "Ch2 Options buffer parity error" }, + { F_DMXMPAFIFO2, "Ch2 MPA FIFO parity error" }, + { F_DMXDBFIFO2, "Ch2 DB FIFO parity error" }, + { F_DMXATTFIFO2, "Ch2 Attribute FIFO parity error" }, + { F_DMXISSFIFO2, "Ch2 ISS FIFO parity error" }, + { F_DMXTCPFIFO2, "Ch2 TCP Fields FIFO parity error" }, + { F_DMXERRFIFO2, "Ch2 Error FIFO parity error" }, + { F_MPS2TPINTF2, "Ch2 MPS2TP Interface parity error" }, + { 0 } + }; static const struct intr_info tp_e_in1_perr_cause = { .name = "TP_E_IN1_PERR_CAUSE", .cause_reg = A_TP_E_IN1_PERR_CAUSE, .enable_reg = A_TP_E_IN1_PERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = tp_e_in1_perr_details, .actions = NULL, }; + static const struct intr_details tp_other_perr_details[] = { + { F_DMARBTPERR, "DMARBT MA Rsp Interface parity Error" }, + { F_MMGRCACHEDATASRAM, "TP MMGR Cache Data SRAM" }, + { F_MMGRCACHETAGFIFO, "TP MMGR Cache Tag FIFO" }, + { F_DBL2TLUTPERR, "TP DB Lookup Table" }, + { F_DBTXTIDPERR, "TP DB FIFOs" }, + { F_DBEXTPERR, "TP DB Extended Opcode FIFO" }, + { F_DBOPPERR, "TP DB Opcode FIFO" }, + { F_TMCACHEPERR, "TP TM Cache SRAM" }, + { F_TPPROTOSRAM, "TP Protocol SRAM" }, + { F_HSPSRAM, "HighSpeed SRAM" }, + { F_RATEGRPSRAM, "Rate Group SRAM" }, + { F_TXFBSEQFIFO, "Tx Feedback Sequence Number FIFO" }, + { F_CMDATASRAM, "Cache Data SRAM" }, + { F_CMTAGFIFO, "Cache Tag FIFO" }, + { F_RFCOPFIFO, "RCF Opcode FIFO" }, + { F_DELINVFIFO, "Delete Invalid FIFO" }, + { F_RSSCFGSRAM, "RSS Config or Round-Robin SRAM" }, + { F_RSSKEYSRAM, "RSS Key SRAM" }, + { F_RSSLKPSRAM, "RSS Lookup SRAM" }, + { F_SRQSRAM, "SRQ SRAM" }, + { F_ARPDASRAM, "ARP DA SRAM" }, + { F_ARPSASRAM, "ARP SA SRAM" }, + { F_ARPGRESRAM, "ARP GRE SRAM" }, + { F_ARPIPSECSRAM1, "ARP IPSec SRAM0" }, + { F_ARPIPSECSRAM0, "ARP IPSec SRAM1" }, + { 0 } + }; static const struct intr_info tp_o_perr_cause = { .name = "TP_O_PERR_CAUSE", .cause_reg = A_TP_O_PERR_CAUSE, .enable_reg = A_TP_O_PERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = tp_other_perr_details, .actions = NULL, }; bool fatal; - fatal = t4_handle_intr(adap, &tp_intr_info, 0, flags); if (chip_id(adap) > CHELSIO_T6) { - fatal |= t4_handle_intr(adap, &tp_inic_perr_cause, 0, flags); + tp_intr_info.details = t7_tp_intr_details; + fatal = t4_handle_intr(adap, &tp_intr_info, 0, flags); + fatal |= t4_handle_intr(adap, &tp_cerr_cause, 0, flags); fatal |= t4_handle_intr(adap, &tp_c_perr_cause, 0, flags); fatal |= t4_handle_intr(adap, &tp_e_eg_perr_cause, 0, flags); fatal |= t4_handle_intr(adap, &tp_e_in0_perr_cause, 0, flags); fatal |= t4_handle_intr(adap, &tp_e_in1_perr_cause, 0, flags); fatal |= t4_handle_intr(adap, &tp_o_perr_cause, 0, flags); + } else { + tp_intr_info.details = tp_intr_details; + fatal = t4_handle_intr(adap, &tp_intr_info, 0, flags); } + clear_int_cause_reg(adap, &tp_intr_info, flags); return (fatal); } @@ -5133,16 +5362,86 @@ static bool tp_intr_handler(struct adapter *adap, int arg, int flags) */ static bool sge_intr_handler(struct adapter *adap, int arg, int flags) { + static const struct intr_details sge_int1_details[] = { + { F_PERR_FLM_CREDITFIFO, "SGE FLM credit FIFO parity error" }, + { F_PERR_IMSG_HINT_FIFO, "SGE IMSG hint FIFO parity error" }, + { F_PERR_HEADERSPLIT_FIFO3 | F_PERR_HEADERSPLIT_FIFO2, + "SGE header split FIFO parity error" }, + { F_PERR_PAYLOAD_FIFO3 | F_PERR_PAYLOAD_FIFO2, + "SGE payload FIFO parity error" }, + { F_PERR_PC_RSP, "SGE PC response parity error" }, + { F_PERR_PC_REQ, "SGE PC request parity error" }, + { 0x003c0000, "SGE DBP PC response FIFO parity error" }, + { F_PERR_DMARBT, "SGE DMA RBT parity error" }, + { F_PERR_FLM_DBPFIFO, "SGE FLM DBP FIFO parity error" }, + { F_PERR_FLM_MCREQ_FIFO, "SGE FLM MC request FIFO parity error" }, + { F_PERR_FLM_HINTFIFO, "SGE FLM hint FIFO parity error" }, + { 0x00003c00, "SGE align control FIFO parity error" }, + { 0x000003c0, "SGE EDMA FIFO parity error" }, + { 0x0000003c, "SGE PD FIFO parity error" }, + { F_PERR_ING_CTXT_MIFRSP, "SGE Ingress context MIF response parity error" }, + { F_PERR_EGR_CTXT_MIFRSP, "SGE Egress context MIF response parity error" }, + { 0 } + }; static const struct intr_info sge_int1_info = { .name = "SGE_INT_CAUSE1", .cause_reg = A_SGE_INT_CAUSE1, .enable_reg = A_SGE_INT_ENABLE1, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = sge_int1_details, .actions = NULL, }; - static const struct intr_info sge_int2_info = { + static const struct intr_details t7_sge_int2_details[] = { + { F_TF_FIFO_PERR, "SGE TF FIFO parity error" }, + { F_PERR_EGR_DBP_TX_COAL, "SGE egress DBP TX coal parity error" }, + { F_PERR_DBP_FL_FIFO, "SGE DBP FL FIFO parity error" }, + { F_DEQ_LL_PERR, "SGE linked list SRAM parity error" }, + { F_ENQ_PERR, "SGE enq tag SRAM parity error" }, + { F_DEQ_OUT_PERR, "SGE tbuf deq output FIFO parity error" }, + { F_BUF_PERR, "SGE tbuf main buffer parity error" }, + { F_PERR_CONM_SRAM, "SGE CONM SRAM parity error" }, + { F_PERR_ISW_IDMA3_FIFO | F_PERR_ISW_IDMA2_FIFO | + F_PERR_ISW_IDMA1_FIFO | F_PERR_ISW_IDMA0_FIFO, + "SGE ISW IDMA FIFO parity error" }, + { F_PERR_ISW_DBP_FIFO, "SGE ISW DBP FIFO parity error" }, + { F_PERR_ISW_GTS_FIFO, "SGE ISW GTS FIFO parity error" }, + { F_PERR_ITP_EVR, "SGE ITP EVR parity error" }, + { F_PERR_FLM_CNTXMEM, "SGE FLM context memory parity error" }, + { F_PERR_FLM_L1CACHE, "SGE FLM L1 cache parity error" }, + { F_SGE_IPP_FIFO_PERR, "SGE IPP FIFO parity error" }, + { F_PERR_DBP_HP_FIFO, "SGE DBP HP FIFO parity error" }, + { F_PERR_DB_FIFO, "SGE doorbell FIFO parity error" }, + { F_PERR_ING_CTXT_CACHE | F_PERR_EGR_CTXT_CACHE, + "SGE context cache parity error" }, + { F_PERR_BASE_SIZE, "SGE base size parity error" }, + { 0 } + }; + static const struct intr_details t6_sge_int2_details[] = { + { F_PERR_DBP_HINT_FL_FIFO, "SGE DBP hint FL FIFO parity error" }, + { F_PERR_EGR_DBP_TX_COAL, "SGE egress DBP TX coal parity error" }, + { F_PERR_DBP_FL_FIFO, "SGE DBP FL FIFO parity error" }, + { F_DEQ_LL_PERR, "SGE tbuf dequeue linked list SRAM parity error" }, + { F_ENQ_PERR, "SGE tbuf enqueue tag SRAM parity error" }, + { F_DEQ_OUT_PERR, "SGE tbuf dequeue output FIFO parity error" }, + { F_BUF_PERR, "SGE tbuf main buffer parity error" }, + { F_PERR_CONM_SRAM, "SGE CONM SRAM parity error" }, + { F_PERR_ISW_IDMA1_FIFO, "SGE ISW IDMA FIFO parity error" }, + { F_PERR_ISW_IDMA0_FIFO, "SGE ISW IDMA FIFO parity error" }, + { F_PERR_ISW_DBP_FIFO, "SGE ISW DBP FIFO parity error" }, + { F_PERR_ISW_GTS_FIFO, "SGE ISW GTS FIFO parity error" }, + { F_PERR_ITP_EVR, "SGE ITP EVR parity error" }, + { F_PERR_FLM_CNTXMEM, "SGE FLM context memory parity error" }, + { F_PERR_FLM_L1CACHE, "SGE FLM L1 cache parity error" }, + { F_PERR_DBP_HINT_FIFO, "SGE DBP hint FIFO parity error" }, + { F_PERR_DBP_HP_FIFO, "SGE DBP high priority FIFO parity error" }, + { F_PERR_DB_FIFO, "SGE DBP merge DB FIFO parity error" }, + { F_PERR_ING_CTXT_CACHE, "SGE ingress context cache parity error" }, + { F_PERR_EGR_CTXT_CACHE, "SGE egress context cache parity error" }, + { F_PERR_BASE_SIZE, "SGE base size parity error" }, + { 0 } + }; + struct intr_info sge_int2_info = { .name = "SGE_INT_CAUSE2", .cause_reg = A_SGE_INT_CAUSE2, .enable_reg = A_SGE_INT_ENABLE2, @@ -5231,16 +5530,105 @@ static bool sge_intr_handler(struct adapter *adap, int arg, int flags) .details = NULL, .actions = NULL, }; + static const struct intr_details sge_int4_details[] = { + { F_ERR_ISHIFT_UR1 | F_ERR_ISHIFT_UR0, "SGE ishift underrun" }, + { F_BAR2_EGRESS_LEN_OR_ADDR_ERR, "SGE BAR2 PL access length or alignment error" }, + { F_ERR_CPL_EXCEED_MAX_IQE_SIZE1 | F_ERR_CPL_EXCEED_MAX_IQE_SIZE0, + "SGE CPL exceeds max IQE size" }, + { F_ERR_WR_LEN_TOO_LARGE3 | F_ERR_WR_LEN_TOO_LARGE2 | + F_ERR_WR_LEN_TOO_LARGE1 | F_ERR_WR_LEN_TOO_LARGE0, + "SGE WR length too large" }, + { F_ERR_LARGE_MINFETCH_WITH_TXCOAL3 | F_ERR_LARGE_MINFETCH_WITH_TXCOAL2 | + F_ERR_LARGE_MINFETCH_WITH_TXCOAL1 | F_ERR_LARGE_MINFETCH_WITH_TXCOAL0, + "SGE invalid MinFetchBurst with TxCoalesce" }, + { F_COAL_WITH_HP_DISABLE_ERR, "SGE coalesce with HP disable error" }, + { F_BAR2_EGRESS_COAL0_ERR, "SGE BAR2 PL access addr offset 0" }, + { F_BAR2_EGRESS_SIZE_ERR, "SGE BAR2 illegal egress QID access" }, + { F_FLM_PC_RSP_ERR, "SGE FLM PC response error" }, + { F_ERR_TH3_MAX_FETCH | F_ERR_TH2_MAX_FETCH | + F_ERR_TH1_MAX_FETCH | F_ERR_TH0_MAX_FETCH, + "SGE max fetch violation" }, + { F_ERR_RX_CPL_PACKET_SIZE1 | F_ERR_RX_CPL_PACKET_SIZE0, + "SGE CPL length mismatch error" }, + { F_ERR_BAD_UPFL_INC_CREDIT3 | F_ERR_BAD_UPFL_INC_CREDIT2 | + F_ERR_BAD_UPFL_INC_CREDIT1 | F_ERR_BAD_UPFL_INC_CREDIT0, + "SGE upfl credit wrap error" }, + { F_ERR_PHYSADDR_LEN0_IDMA1 | F_ERR_PHYSADDR_LEN0_IDMA0, + "SGE CPL_RX_PHYS_ADDR length 0 error" }, + { F_ERR_FLM_INVALID_PKT_DROP1 | F_ERR_FLM_INVALID_PKT_DROP0, + "SGE IDMA packet drop due to invalid FLM context" }, + { F_ERR_UNEXPECTED_TIMER, "SGE unexpected timer error" }, + { 0 } + }; static const struct intr_info sge_int4_info = { .name = "SGE_INT_CAUSE4", .cause_reg = A_SGE_INT_CAUSE4, .enable_reg = A_SGE_INT_ENABLE4, .fatal = 0, .flags = 0, - .details = NULL, + .details = sge_int4_details, .actions = NULL, }; - static const struct intr_info sge_int5_info = { + static const struct intr_details t7_sge_int5_details[] = { + { F_ERR_T_RXCRC, "SGE RxCRC error" }, + { F_PERR_MC_RSPDATA, "SGE MC response data parity error" }, + { F_PERR_PC_RSPDATA, "SGE PC response data parity error" }, + { F_PERR_PD_RDRSPDATA, "SGE PD read response data parity error" }, + { F_PERR_U_RXDATA, "SGE U Rx data parity error" }, + { F_PERR_UD_RXDATA, "SGE UD Rx data parity error" }, + { F_PERR_UP_DATA, "SGE uP data parity error" }, + { F_PERR_CIM2SGE_RXDATA, "SGE CIM2SGE Rx data parity error" }, + { F_PERR_IMSG_PD_FIFO, "SGE IMSG PD FIFO parity error" }, + { F_PERR_ULPTX_FIFO1 | F_PERR_ULPTX_FIFO0, "SGE ULPTX FIFO parity error" }, + { F_PERR_IDMA2IMSG_FIFO3 | F_PERR_IDMA2IMSG_FIFO2 | + F_PERR_IDMA2IMSG_FIFO1 | F_PERR_IDMA2IMSG_FIFO0, + "SGE IDMA2IMSG FIFO parity error" }, + { F_PERR_POINTER_DATA_FIFO3 | F_PERR_POINTER_DATA_FIFO2 | + F_PERR_POINTER_DATA_FIFO1 | F_PERR_POINTER_DATA_FIFO0, + "SGE pointer data FIFO parity error" }, + { F_PERR_POINTER_HDR_FIFO3 | F_PERR_POINTER_HDR_FIFO2 | + F_PERR_POINTER_HDR_FIFO1 | F_PERR_POINTER_HDR_FIFO0, + "SGE pointer header FIFO parity error" }, + { F_PERR_PAYLOAD_FIFO1 | F_PERR_PAYLOAD_FIFO0, + "SGE payload FIFO parity error" }, + { F_PERR_MGT_BAR2_FIFO, "SGE MGT BAR2 FIFO parity error" }, + { F_PERR_HEADERSPLIT_FIFO1 | F_PERR_HEADERSPLIT_FIFO0, + "SGE header split FIFO parity error" }, + { F_PERR_HINT_DELAY_FIFO, "SGE hint delay FIFO parity error" }, + { 0 } + }; + static const struct intr_details t6_sge_int5_details[] = { + { F_ERR_T_RXCRC, "SGE T RxCRC parity error" }, + { F_PERR_MC_RSPDATA, "SGE MC response data parity error" }, + { F_PERR_PC_RSPDATA, "SGE PC response data parity error" }, + { F_PERR_U_RXDATA | F_PERR_UD_RXDATA, "SGE ULP Rx data parity error" }, + { F_PERR_UP_DATA, "SGE uP data parity error" }, + { F_PERR_CIM2SGE_RXDATA, "SGE CIM2SGE Rx data parity error" }, + { F_PERR_HINT_DELAY_FIFO1 | F_PERR_HINT_DELAY_FIFO0, + "SGE hint delay FIFO parity error" }, + { F_PERR_IMSG_PD_FIFO, "SGE IMSG PD FIFO parity error" }, + { F_PERR_ULPTX_FIFO1 | F_PERR_ULPTX_FIFO0, + "SGE ULPTX FIFO parity error" }, + { F_PERR_IDMA2IMSG_FIFO1 | F_PERR_IDMA2IMSG_FIFO0, + "SGE IDMA2IMSG FIFO parity error" }, + { F_PERR_POINTER_DATA_FIFO1 | F_PERR_POINTER_DATA_FIFO0, + "SGE pointer data FIFO parity error" }, + { F_PERR_POINTER_HDR_FIFO1 | F_PERR_POINTER_HDR_FIFO0, + "SGE pointer header FIFO parity error" }, + { F_PERR_PAYLOAD_FIFO1 | F_PERR_PAYLOAD_FIFO0, + "SGE payload FIFO parity error" }, + { F_PERR_EDMA_INPUT_FIFO3 | F_PERR_EDMA_INPUT_FIFO2 | + F_PERR_EDMA_INPUT_FIFO1 | F_PERR_EDMA_INPUT_FIFO0, + "SGE EDMA input FIFO parity error" }, + { F_PERR_MGT_BAR2_FIFO, "SGE MGT BAR2 FIFO parity error" }, + { F_PERR_HEADERSPLIT_FIFO1 | F_PERR_HEADERSPLIT_FIFO0, + "SGE header split FIFO parity error" }, + { F_PERR_CIM_FIFO1 | F_PERR_CIM_FIFO0, "SGE CIM FIFO parity error" }, + { F_PERR_IDMA_SWITCH_OUTPUT_FIFO1 | F_PERR_IDMA_SWITCH_OUTPUT_FIFO0, + "SGE IDMA switch output FIFO parity error" }, + { 0 } + }; + struct intr_info sge_int5_info = { .name = "SGE_INT_CAUSE5", .cause_reg = A_SGE_INT_CAUSE5, .enable_reg = A_SGE_INT_ENABLE5, @@ -5249,31 +5637,94 @@ static bool sge_intr_handler(struct adapter *adap, int arg, int flags) .details = NULL, .actions = NULL, }; + static const struct intr_details sge_int6_details[] = { + /* T7+ */ + { 0xe0000000, "SGE fatal DEQ0 DRDY error" }, + { 0x1c000000, "SGE fatal OUT0 DRDY error" }, + { F_IMSG_DBG3_STUCK | F_IMSG_DBG2_STUCK | + F_IMSG_DBG1_STUCK | F_IMSG_DBG0_STUCK, + "SGE IMSG stuck due to insufficient credits" }, + /* T6 + */ + { F_ERR_DB_SYNC, "SGE doorbell sync failed" }, + { F_ERR_GTS_SYNC, "SGE GTS sync failed" }, + { F_FATAL_LARGE_COAL, "SGE BAR2 payload too large" }, + { F_PL_BAR2_FRM_ERR, "SGE BAR2 framing error" }, + { F_SILENT_DROP_TX_COAL, "SGE silent drop of Tx coal WR" }, + { F_ERR_INV_CTXT4, "SGE context access for invalid queue thread 4" }, + { F_ERR_BAD_DB_PIDX4, "SGE doorbell pidx too large thread 4" }, + { F_ERR_BAD_UPFL_INC_CREDIT4, "SGE upfl credit wrap thread 4" }, + { F_FATAL_TAG_MISMATCH, "SGE doorbell tag mismatch" }, + { F_FATAL_ENQ_CTL_RDY, "SGE enq_ctl_fifo overflow" }, + { F_ERR_PC_RSP_LEN3 | F_ERR_PC_RSP_LEN2 | + F_ERR_PC_RSP_LEN1 | F_ERR_PC_RSP_LEN0, + "SGE PCIe response error for DBP threads" }, + { F_FATAL_ENQ2LL_VLD, "SGE tbuf fatal_enq2ll_vld" }, + { F_FATAL_LL_EMPTY, "SGE tbuf fatal_ll_empty" }, + { F_FATAL_OFF_WDENQ, "SGE tbuf fatal_off_wdenq" }, + { 0x00000018, "SGE tbuf fatal_deq1_drdy" }, + { 0x00000006, "SGE tbuf fatal_out1_drdy" }, + { F_FATAL_DEQ, "SGE tbuf fatal_deq" }, + { 0 } + }; static const struct intr_info sge_int6_info = { .name = "SGE_INT_CAUSE6", .cause_reg = A_SGE_INT_CAUSE6, .enable_reg = A_SGE_INT_ENABLE6, .fatal = 0, .flags = 0, - .details = NULL, + .details = sge_int6_details, .actions = NULL, }; + static const struct intr_details sge_int7_details[] = { + { F_HINT_FIFO_FULL, "SGE hint FIFO full" }, + { F_CERR_HINT_DELAY_FIFO, "SGE hint delay FIFO ECC error" }, + { F_COAL_TIMER_FIFO_PERR, "SGE coalescing timer FIFO parity error" }, + { F_CMP_FIFO_PERR, "SGE CMP FIFO parity error" }, + { F_SGE_IPP_FIFO_CERR, "SGE IPP FIFO ECC error" }, + { F_CERR_ING_CTXT_CACHE | F_CERR_EGR_CTXT_CACHE, + "SGE context cache ECC error" }, + { F_IMSG_CNTX_PERR, "SGE IMSG context parity error" }, + { F_PD_FIFO_PERR, "SGE PD FIFO parity error" }, + { F_IMSG_512_FIFO_PERR, "SGE IMSG 512 FIFO parity error" }, + { F_CPLSW_FIFO_PERR, "SGE CPLSW FIFO parity error" }, + { F_IMSG_FIFO_PERR, "SGE IMSG FIFO parity error" }, + { F_CERR_ITP_EVR, "SGE ITP EVR ECC error" }, + { F_CERR_CONM_SRAM, "SGE CONM SRAM ECC error" }, + { F_CERR_FLM_CNTXMEM, "SGE FLM context memory ECC error" }, + { F_CERR_FUNC_QBASE, "SGE function queue base ECC error" }, + { F_IMSG_CNTX_CERR, "SGE IMSG context ECC error" }, + { F_PD_FIFO_CERR, "SGE PD FIFO ECC error" }, + { F_IMSG_512_FIFO_CERR, "SGE IMSG 512 FIFO ECC error" }, + { F_CPLSW_FIFO_CERR, "SGE CPLSW FIFO ECC error" }, + { F_IMSG_FIFO_CERR, "SGE IMSG FIFO ECC error" }, + { 0x0000001e, "SGE header split FIFO ECC error" }, // Bits 4:1 + { F_CERR_FLM_L1CACHE, "SGE FLM L1 cache ECC error" }, + { 0 } + }; static const struct intr_info sge_int7_info = { .name = "SGE_INT_CAUSE7", .cause_reg = A_SGE_INT_CAUSE7, .enable_reg = A_SGE_INT_ENABLE7, .fatal = 0, .flags = 0, - .details = NULL, + .details = sge_int7_details, .actions = NULL, }; + static const struct intr_details sge_int8_details[] = { + { F_TRACE_RXPERR, "SGE trace packet parity error" }, + { F_U3_RXPERR | F_U2_RXPERR | F_U1_RXPERR | F_U0_RXPERR, + "SGE ULP interface parity error" }, + { F_T3_RXPERR | F_T2_RXPERR | F_T1_RXPERR | F_T0_RXPERR, + "SGE TP interface parity error" }, + { 0 } + }; static const struct intr_info sge_int8_info = { .name = "SGE_INT_CAUSE8", .cause_reg = A_SGE_INT_CAUSE8, .enable_reg = A_SGE_INT_ENABLE8, .fatal = 0, .flags = 0, - .details = NULL, + .details = sge_int8_details, .actions = NULL, }; bool fatal; @@ -5281,8 +5732,14 @@ static bool sge_intr_handler(struct adapter *adap, int arg, int flags) if (chip_id(adap) <= CHELSIO_T5) { sge_int3_info.details = sge_int3_details; + } else if (chip_id(adap) == CHELSIO_T6) { + sge_int3_info.details = t6_sge_int3_details; + sge_int2_info.details = t6_sge_int2_details; + sge_int5_info.details = t6_sge_int5_details; } else { sge_int3_info.details = t6_sge_int3_details; + sge_int2_info.details = t7_sge_int2_details; + sge_int5_info.details = t7_sge_int5_details; } fatal = false; @@ -5316,6 +5773,19 @@ static bool sge_intr_handler(struct adapter *adap, int arg, int flags) */ static bool cim_intr_handler(struct adapter *adap, int arg, int flags) { + static const struct intr_details cim_host_t7_intr_details[] = { + { F_CORE7ACCINT, "CIM slave core 7 access interrupt "}, + { F_CORE6ACCINT, "CIM slave core 6 access interrupt "}, + { F_CORE5ACCINT, "CIM slave core 5 access interrupt "}, + { F_CORE4ACCINT, "CIM slave core 4 access interrupt "}, + { F_CORE3ACCINT, "CIM slave core 3 access interrupt "}, + { F_CORE2ACCINT, "CIM slave core 2 access interrupt "}, + { F_CORE1ACCINT, "CIM slave core 1 access interrupt "}, + { F_TIMER1INT, "CIM TIMER0 interrupt" }, + { F_TIMER0INT, "CIM TIMER0 interrupt" }, + { F_PREFDROPINT, "CIM control register prefetch drop" }, + { 0} + }; static const struct intr_details cim_host_intr_details[] = { /* T6+ */ { F_PCIE2CIMINTFPARERR, "CIM IBQ PCIe interface parity error" }, @@ -5328,8 +5798,8 @@ static bool cim_intr_handler(struct adapter *adap, int arg, int flags) { F_SGE2CIMINTFPARERR, "CIM IBQ SGE interface parity error" }, { F_ULP2CIMINTFPARERR, "CIM IBQ ULP_TX interface parity error" }, { F_TP2CIMINTFPARERR, "CIM IBQ TP interface parity error" }, - { F_OBQSGERX1PARERR, "CIM OBQ SGE1_RX parity error" }, - { F_OBQSGERX0PARERR, "CIM OBQ SGE0_RX parity error" }, + { F_OBQSGERX1PARERR, "CIM OBQ PCIE_RX parity error" }, + { F_OBQSGERX0PARERR, "CIM OBQ SGE_RX parity error" }, /* T4+ */ { F_TIEQOUTPARERRINT, "CIM TIEQ outgoing FIFO parity error" }, @@ -5354,16 +5824,17 @@ static bool cim_intr_handler(struct adapter *adap, int arg, int flags) { F_PREFDROPINT, "CIM control register prefetch drop" }, { 0} }; - static const struct intr_info cim_host_intr_info = { + struct intr_info cim_host_intr_info = { .name = "CIM_HOST_INT_CAUSE", .cause_reg = A_CIM_HOST_INT_CAUSE, .enable_reg = A_CIM_HOST_INT_ENABLE, .fatal = 0x007fffe6, .flags = IHF_FATAL_IFF_ENABLED, - .details = cim_host_intr_details, + .details = NULL, .actions = NULL, }; static const struct intr_details cim_host_upacc_intr_details[] = { + { F_CONWRERRINT, "CIM condition write error "}, { F_EEPROMWRINT, "CIM EEPROM came out of busy state" }, { F_TIMEOUTMAINT, "CIM PIF MA timeout" }, { F_TIMEOUTINT, "CIM PIF timeout" }, @@ -5423,18 +5894,54 @@ static bool cim_intr_handler(struct adapter *adap, int arg, int flags) .details = NULL, .actions = NULL, }; + static const struct intr_details cim_perr_cause_details[] = { + { F_T7_MA_CIM_INTFPERR, "MA2CIM interface parity error" }, + { F_T7_MBHOSTPARERR, "Mailbox Host Read parity error" }, + { F_MAARBINVRSPTAG, "MA Arbiter Invalid Response Tag (Fatal)" }, + { F_MAARBFIFOPARERR, "MA Arbiter FIFO Parity Error" }, + { F_SEMSRAMPARERR, "Semaphore logic SRAM Parity Error" }, + { F_RSACPARERR, "RSA Code SRAM Parity Error" }, + { F_RSADPARERR, "RSA Data SRAM Parity Error" }, + { F_T7_PLCIM_MSTRSPDATAPARERR, "PL2CIM Master response data parity error" }, + { F_T7_PCIE2CIMINTFPARERR, "IBQ PCIE intf parity error" }, + { F_T7_NCSI2CIMINTFPARERR, "IBQ NCSI intf parity error" }, + { F_T7_SGE2CIMINTFPARERR, "IBQ SGE Intf Parity error" }, + { F_T7_ULP2CIMINTFPARERR, "IBQ ULP_TX intf parity error" }, + { F_T7_TP2CIMINTFPARERR, "IBQ TP intf parity error" }, + { F_CORE7PARERR, "Slave Core7 parity error" }, + { F_CORE6PARERR, "Slave Core6 parity error" }, + { F_CORE5PARERR, "Slave Core5 parity error" }, + { F_CORE4PARERR, "Slave Core4 parity error" }, + { F_CORE3PARERR, "Slave Core3 parity error" }, + { F_CORE2PARERR, "Slave Core2 parity error" }, + { F_CORE1PARERR, "Slave Core1 parity error" }, + { F_GFTPARERR, "GFT block Memory parity error" }, + { F_MPSRSPDATAPARERR, "MPS lookup interface Response parity error" }, + { F_ER_RSPDATAPARERR, "Expansion ROM/Flash Interface Response Parity Error" }, + { F_FLOWFIFOPARERR, "SGE FlowID Prefetch FIFO Parity Error" }, + { F_OBQSRAMPARERR, "OBQ SRAM Parity Error" }, + { F_TIEQOUTPARERR, "TIE Queue Outgoing FIFO parity error" }, + { F_TIEQINPARERR, "TIE Queue Incoming FIFO parity error" }, + { F_PIFRSPPARERR, "PIF Response interface FIFO Parity error" }, + { F_PIFREQPARERR, "PIF Request interface FIFO Parity error" }, + { 0 } + }; static const struct intr_info cim_perr_cause = { .name = "CIM_PERR_CAUSE", .cause_reg = A_CIM_PERR_CAUSE, .enable_reg = A_CIM_PERR_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = cim_perr_cause_details, .actions = NULL, }; u32 val, fw_err; bool fatal; + if (chip_id(adap) >= CHELSIO_T7) + cim_host_intr_info.details = cim_host_t7_intr_details; + else + cim_host_intr_info.details = cim_host_intr_details; /* * When the Firmware detects an internal error which normally wouldn't * raise a Host Interrupt, it forces a CIM Timer0 interrupt in order @@ -5477,62 +5984,237 @@ static bool ulprx_intr_handler(struct adapter *adap, int arg, int flags) { 0x007fffff, "ULPRX parity error" }, { 0 } }; - static const struct intr_info ulprx_intr_info = { + static const struct intr_details t6_ulprx_int_cause_details[] = { + { F_SE_CNT_MISMATCH_1, "SE count mismatch in channel1" }, + { F_SE_CNT_MISMATCH_0, "SE count mismatch in channel 0" }, + { F_CAUSE_CTX_1, "Context access error on channel 1" }, + { F_CAUSE_CTX_0, "Context access error on channel 0" }, + { F_CAUSE_FF, "filp-flop based fifos" }, + { F_CAUSE_APF_1, "Arb prefetch memory, channel 1" }, + { F_CAUSE_APF_0, "Arb prefetch memory, channel 0" }, + { F_CAUSE_AF_1, "Arb fetch memory, channel 1" }, + { F_CAUSE_AF_0, "Arb fetch memory, channel 0" }, + { F_CAUSE_DDPDF_1, "ddp_data_fifo Fifo, channel 1" }, + { F_CAUSE_DDPMF_1, "ddp_msg_fifo Fifo, channel 1" }, + { F_CAUSE_MEMRF_1, "mem_req_fifo_d Fifo, channel 1" }, + { F_CAUSE_PRSDF_1, "prsr_data_fifo Fifo, channel 1" }, + { F_CAUSE_DDPDF_0, "ddp_data_fifo Fifo, channel 0" }, + { F_CAUSE_DDPMF_0, "ddp_msg_fifo Fifo, channel 0" }, + { F_CAUSE_MEMRF_0, "mem_req_fifo_d Fifo, channel 0" }, + { F_CAUSE_PRSDF_0, "prsr_data_fifo Fifo, channel 0" }, + { F_CAUSE_PCMDF_1, "Pcmd Fifo, channel 1" }, + { F_CAUSE_TPTCF_1, "tpt_ctl_fifo Fifo, channel 1" }, + { F_CAUSE_DDPCF_1, "ddp_ctl_fifo Fifo, channel 1" }, + { F_CAUSE_MPARF_1, "mpar_ctl_fifo Fifo, channel 1" }, + { F_CAUSE_MPARC_1, "mpac_ctl_fifo Fifo, channel 1" }, + { F_CAUSE_PCMDF_0, "Pcmd Fifo, channel 0" }, + { F_CAUSE_TPTCF_0, "tpt_ctl_fifo Fifo, channel 0" }, + { F_CAUSE_DDPCF_0, "ddp_ctl_fifo Fifo, channel 0" }, + { F_CAUSE_MPARF_0, "mpar_ctl_fifo Fifo, channel 0" }, + { F_CAUSE_MPARC_0, "mpac_ctl_fifo Fifo, channel 0" }, + { 0 } + }; + static const struct intr_details t7_ulprx_int_cause_details[] = { + { F_CERR_PCMD_FIFO_3, "PCMD FIFO correctable Error3" }, + { F_CERR_PCMD_FIFO_2, "PCMD FIFO correctable Error2" }, + { F_CERR_PCMD_FIFO_1, "PCMD FIFO correctable Error1" }, + { F_CERR_PCMD_FIFO_0, "PCMD FIFO correctable Error0" }, + { F_CERR_DATA_FIFO_3, "DDP Data FIFO correctable Error3" }, + { F_CERR_DATA_FIFO_2, "DDP Data FIFO correctable Error2" }, + { F_CERR_DATA_FIFO_1, "DDP Data FIFO correctable Error1" }, + { F_CERR_DATA_FIFO_0, "DDP Data FIFO correctable Error0" }, + { F_SE_CNT_MISMATCH_3, "SE count mismatch in channel3" }, + { F_SE_CNT_MISMATCH_2, "SE count mismatch in channel2" }, + { F_T7_SE_CNT_MISMATCH_1, "SE count mismatch in channel1" }, + { F_T7_SE_CNT_MISMATCH_0, "SE count mismatch in channel 0" }, + { F_T7_ENABLE_CTX_3, "Context access error on channel 3" }, + { F_T7_ENABLE_CTX_2, "Context access error on channel 2" }, + { F_T7_ENABLE_CTX_1, "Context access error on channel 1" }, + { F_T7_ENABLE_CTX_0, "Context access error on channel 0" }, + { F_T7_ENABLE_ALN_SDC_ERR_3, "SDC error reported by aligner in channel3" }, + { F_T7_ENABLE_ALN_SDC_ERR_2, "SDC error reported by aligner in channel2" }, + { F_T7_ENABLE_ALN_SDC_ERR_1, "SDC error reported by aligner in channel1" }, + { F_T7_ENABLE_ALN_SDC_ERR_0, "SDC error reported by aligner in channel0" }, + { 0 } + }; + struct intr_info ulprx_intr_info = { .name = "ULP_RX_INT_CAUSE", .cause_reg = A_ULP_RX_INT_CAUSE, .enable_reg = A_ULP_RX_INT_ENABLE, .fatal = 0x07ffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = ulprx_intr_details, + .details = NULL, .actions = NULL, }; + static const struct intr_details ulprx_int_cause_2_details[] = { + { F_ULPRX2MA_INTFPERR, "SDC error reported by ULPRX2MA interface parity checker" }, + { F_ALN_SDC_ERR_1, "SDC error reported by aligner in channel 1" }, + { F_ALN_SDC_ERR_0, "SDC error reported by aligner in channel 0" }, + { F_PF_UNTAGGED_TPT_1, "Parity error from Untagged TPT prefetch fifo channel 1" }, + { F_PF_UNTAGGED_TPT_0, "Parity error from Untagged TPT prefetch fifo channel 0" }, + { F_PF_PBL_1, "Parity error from PBL prefetch fifo channel 1" }, + { F_PF_PBL_0, "Parity error from PBL prefetch fifo channel 0" }, + { F_DDP_HINT_1, "DDP hint fifo Perr in channel 1" }, + { F_DDP_HINT_0, "DDP hint fifo Perr in channel 0" }, + { 0 } + }; static const struct intr_info ulprx_intr2_info = { .name = "ULP_RX_INT_CAUSE_2", .cause_reg = A_ULP_RX_INT_CAUSE_2, .enable_reg = A_ULP_RX_INT_ENABLE_2, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulprx_int_cause_2_details, .actions = NULL, }; + static const struct intr_details ulprx_int_cause_pcmd_details[] = { + { F_CAUSE_PCMD_SFIFO_3, "Small FIFOs, channel 3" }, + { F_CAUSE_PCMD_FIFO_3, "pcmd_ctl_fifo, channel 3" }, + { F_CAUSE_PCMD_DDP_HINT_3, "ddp_hint_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_PCMD_TPT_3, "tpt_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_PCMD_DDP_3, "ddp_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_PCMD_MPAR_3, "mpar_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_PCMD_MPAC_3, "mpac_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_PCMD_SFIFO_2, "Small FIFOs, channel 2" }, + { F_CAUSE_PCMD_FIFO_2, "pcmd_ctl_fifo, channel 2" }, + { F_CAUSE_PCMD_DDP_HINT_2, "ddp_hint_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_PCMD_TPT_2, "tpt_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_PCMD_DDP_2, "ddp_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_PCMD_MPAR_2, "mpar_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_PCMD_MPAC_2, "mpac_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_PCMD_SFIFO_1, "Small FIFOs, channel 1" }, + { F_CAUSE_PCMD_FIFO_1, "pcmd_ctl_fifo, channel 1" }, + { F_CAUSE_PCMD_DDP_HINT_1, "ddp_hint_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_PCMD_TPT_1, "tpt_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_PCMD_DDP_1, "ddp_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_PCMD_MPAR_1, "mpar_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_PCMD_MPAC_1, "mpac_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_PCMD_SFIFO_0, "Small FIFOs, channel 0" }, + { F_CAUSE_PCMD_FIFO_0, "pcmd_ctl_fifo, channel 0" }, + { F_CAUSE_PCMD_DDP_HINT_0, "ddp_hint_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_PCMD_TPT_0, "tpt_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_PCMD_DDP_0, "ddp_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_PCMD_MPAR_0, "mpar_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_PCMD_MPAC_0, "mpac_ctl_fifo FIFO, channel 0" }, + { 0 } + }; static const struct intr_info ulprx_int_cause_pcmd = { .name = "ULP_RX_INT_CAUSE_PCMD", .cause_reg = A_ULP_RX_INT_CAUSE_PCMD, .enable_reg = A_ULP_RX_INT_ENABLE_PCMD, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulprx_int_cause_pcmd_details, .actions = NULL, }; + static const struct intr_details ulprx_int_cause_data_details[] = { + { F_CAUSE_DATA_SNOOP_3, "Snoop FIFO, channel 3" }, + { F_CAUSE_DATA_SFIFO_3, "Small FIFO, channel 3" }, + { F_CAUSE_DATA_FIFO_3, "data_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_DATA_DDP_3, "ddp_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_DATA_CTX_3, "ctx_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_DATA_PARSER_3, "parser_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_DATA_SNOOP_2, "Snoop FIFO, channel 2" }, + { F_CAUSE_DATA_SFIFO_2, "Small FIFO, channel 2" }, + { F_CAUSE_DATA_FIFO_2, "data_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_DATA_DDP_2, "ddp_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_DATA_CTX_2, "ctx_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_DATA_PARSER_2, "parser_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_DATA_SNOOP_1, "Snoop FIFO, channel 1" }, + { F_CAUSE_DATA_SFIFO_1, "Small FIFO, channel 1" }, + { F_CAUSE_DATA_FIFO_1, "data_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_DATA_DDP_1, "ddp_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_DATA_CTX_1, "ctx_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_DATA_PARSER_1, "parser_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_DATA_SNOOP_0, "Snoop FIFO, channel 0" }, + { F_CAUSE_DATA_SFIFO_0, "Small FIFO, channel 0" }, + { F_CAUSE_DATA_FIFO_0, "data_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_DATA_DDP_0, "ddp_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_DATA_CTX_0, "ctx_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_DATA_PARSER_0, "parser_ctl_fifo FIFO, channel 0" }, + { 0 } + }; static const struct intr_info ulprx_int_cause_data = { .name = "ULP_RX_INT_CAUSE_DATA", .cause_reg = A_ULP_RX_INT_CAUSE_DATA, .enable_reg = A_ULP_RX_INT_ENABLE_DATA, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulprx_int_cause_data_details, .actions = NULL, }; + static const struct intr_details ulprx_int_cause_arb_details[] = { + { F_CAUSE_ARB_PBL_PF_3, "pbl_pf_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_ARB_PF_3, "pf_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_ARB_TPT_PF_3, "tpt_pf_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_ARB_F_3, "f_ctl_fifo FIFO, channel 3" }, + { F_CAUSE_ARB_PBL_PF_2, "pbl_pf_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_ARB_PF_2, "pf_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_ARB_TPT_PF_2, "tpt_pf_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_ARB_F_2, "f_ctl_fifo FIFO, channel 2" }, + { F_CAUSE_ARB_PBL_PF_1, "pbl_pf_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_ARB_PF_1, "pf_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_ARB_TPT_PF_1, "tpt_pf_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_ARB_F_1, "f_ctl_fifo FIFO, channel 1" }, + { F_CAUSE_ARB_PBL_PF_0, "pbl_pf_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_ARB_PF_0, "pf_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_ARB_TPT_PF_0, "tpt_pf_ctl_fifo FIFO, channel 0" }, + { F_CAUSE_ARB_F_0, "f_ctl_fifo FIFO, channel 0" }, + { 0 } + }; static const struct intr_info ulprx_int_cause_arb = { .name = "ULP_RX_INT_CAUSE_ARB", .cause_reg = A_ULP_RX_INT_CAUSE_ARB, .enable_reg = A_ULP_RX_INT_ENABLE_ARB, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulprx_int_cause_arb_details, .actions = NULL, }; + static const struct intr_details ulprx_int_cause_interface_details[] = { + { F_CAUSE_ULPRX2SBT_RSPPERR, "ULPRX2SBT_RspPerr" }, + { F_CAUSE_ULPRX2MA_RSPPERR, "ULPRX2MA_RspPerr" }, + { F_CAUSE_PIO_BUS_PERR, "Pio_Bus_Perr" }, + { F_CAUSE_PM2ULP_SNOOPDATA_3, "PM2ULP_SnoopData, channel 3" }, + { F_CAUSE_PM2ULP_SNOOPDATA_2, "PM2ULP_SnoopData, channel 2" }, + { F_CAUSE_PM2ULP_SNOOPDATA_1, "PM2ULP_SnoopData, channel 1" }, + { F_CAUSE_PM2ULP_SNOOPDATA_0, "PM2ULP_SnoopData, channel 0" }, + { F_CAUSE_TLS2ULP_DATA_3, "TLS2ULP_Data, channel 3" }, + { F_CAUSE_TLS2ULP_DATA_2, "TLS2ULP_Data, channel 2" }, + { F_CAUSE_TLS2ULP_DATA_1, "TLS2ULP_Data, channel 1" }, + { F_CAUSE_TLS2ULP_DATA_0, "TLS2ULP_Data, channel 0" }, + { F_CAUSE_TLS2ULP_PLENDATA_3, "TLS2ULP_PLenData, channel 3" }, + { F_CAUSE_TLS2ULP_PLENDATA_2, "TLS2ULP_PLenData, channel 2" }, + { F_CAUSE_TLS2ULP_PLENDATA_1, "TLS2ULP_PLenData, channel 1" }, + { F_CAUSE_TLS2ULP_PLENDATA_0, "TLS2ULP_PLenData, channel 0" }, + { F_CAUSE_PM2ULP_DATA_3, "Pm2Ulp_Data, channel 3" }, + { F_CAUSE_PM2ULP_DATA_2, "Pm2Ulp_Data, channel 2" }, + { F_CAUSE_PM2ULP_DATA_1, "Pm2Ulp_Data, channel 1" }, + { F_CAUSE_PM2ULP_DATA_0, "Pm2Ulp_Data, channel 0" }, + { F_CAUSE_TP2ULP_PCMD_3, "Tp2Ulp_Pcmd, channel 3" }, + { F_CAUSE_TP2ULP_PCMD_2, "Tp2Ulp_Pcmd, channel 2" }, + { F_CAUSE_TP2ULP_PCMD_1, "Tp2Ulp_Pcmd, channel 1" }, + { F_CAUSE_TP2ULP_PCMD_0, "Tp2Ulp_Pcmd, channel 0" }, + { 0 } + }; static const struct intr_info ulprx_int_cause_intf = { .name = "ULP_RX_INT_CAUSE_INTERFACE", .cause_reg = A_ULP_RX_INT_CAUSE_INTERFACE, .enable_reg = A_ULP_RX_INT_ENABLE_INTERFACE, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulprx_int_cause_interface_details, .actions = NULL, }; bool fatal = false; + if (chip_id(adap) <= CHELSIO_T5) + ulprx_intr_info.details = ulprx_intr_details; + else if (chip_id(adap) <= CHELSIO_T6) + ulprx_intr_info.details = t6_ulprx_int_cause_details; + else + ulprx_intr_info.details = t7_ulprx_int_cause_details; + fatal |= t4_handle_intr(adap, &ulprx_intr_info, 0, flags); if (chip_id(adap) < CHELSIO_T7) fatal |= t4_handle_intr(adap, &ulprx_intr2_info, 0, flags); @@ -5559,90 +6241,298 @@ static bool ulptx_intr_handler(struct adapter *adap, int arg, int flags) { 0x0fffffff, "ULPTX parity error" }, { 0 } }; - static const struct intr_info ulptx_intr_info = { + static const struct intr_details t6_ulptx_int_cause_details[] = { + { F_PBL_BOUND_ERR_CH3 | F_PBL_BOUND_ERR_CH2 | + F_PBL_BOUND_ERR_CH1 | F_PBL_BOUND_ERR_CH0, + "PBL address out of bounds" }, + { F_SGE2ULP_FIFO_PERR_SET3 | F_SGE2ULP_FIFO_PERR_SET2 | + F_SGE2ULP_FIFO_PERR_SET1 | F_SGE2ULP_FIFO_PERR_SET0, + "SGE2ULP fifo parity error" }, + { F_CIM2ULP_FIFO_PERR_SET3 | F_CIM2ULP_FIFO_PERR_SET2 | + F_CIM2ULP_FIFO_PERR_SET1 | F_CIM2ULP_FIFO_PERR_SET0, + "CIM2ULP fifo parity error" }, + { F_CQE_FIFO_PERR_SET3 | F_CQE_FIFO_PERR_SET2 | + F_CQE_FIFO_PERR_SET1 | F_CQE_FIFO_PERR_SET0, + "CQE fifo parity error" }, + { F_PBL_FIFO_PERR_SET3 | F_PBL_FIFO_PERR_SET2 | + F_PBL_FIFO_PERR_SET1 | F_PBL_FIFO_PERR_SET0, + "PBL fifo parity error" }, + { F_CMD_FIFO_PERR_SET3 | F_CMD_FIFO_PERR_SET2 | + F_CMD_FIFO_PERR_SET1 | F_CMD_FIFO_PERR_SET0, + "Command fifo parity error" }, + { F_LSO_HDR_SRAM_PERR_SET3 | F_LSO_HDR_SRAM_PERR_SET2 | + F_LSO_HDR_SRAM_PERR_SET1 | F_LSO_HDR_SRAM_PERR_SET0, + "LSO hdr parity error" }, + { 0 } + }; + struct intr_info ulptx_intr_info = { .name = "ULP_TX_INT_CAUSE", .cause_reg = A_ULP_TX_INT_CAUSE, .enable_reg = A_ULP_TX_INT_ENABLE, .fatal = 0x0fffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = ulptx_intr_details, + .details = NULL, + .actions = NULL, + }; + static const struct intr_details ulptx_int_cause_1_details[] = { + { F_PBL_BOUND_ERR_CH3 | F_PBL_BOUND_ERR_CH2 | + F_PBL_BOUND_ERR_CH1 | F_PBL_BOUND_ERR_CH0, + "PBL address out of bounds (configured PBL_ULIMIT/LLIMIT)" }, + { F_SGE2ULP_FIFO_PERR_SET3 | F_SGE2ULP_FIFO_PERR_SET2 | + F_SGE2ULP_FIFO_PERR_SET1 | F_SGE2ULP_FIFO_PERR_SET0, + "SGE2ULP FIFO parity error" }, + { F_CIM2ULP_FIFO_PERR_SET3 | F_CIM2ULP_FIFO_PERR_SET2 | + F_CIM2ULP_FIFO_PERR_SET1 | F_CIM2ULP_FIFO_PERR_SET0, + "CIM2ULP FIFO parity error" }, + { F_CQE_FIFO_PERR_SET3 | F_CQE_FIFO_PERR_SET2 | + F_CQE_FIFO_PERR_SET1 | F_CQE_FIFO_PERR_SET0, + "CQE FIFO parity error" }, + { F_PBL_FIFO_PERR_SET3 | F_PBL_FIFO_PERR_SET2 | + F_PBL_FIFO_PERR_SET1 | F_PBL_FIFO_PERR_SET0, + "PBL FIFO parity error" }, + { F_CMD_FIFO_PERR_SET3 | F_CMD_FIFO_PERR_SET2 | + F_CMD_FIFO_PERR_SET1 | F_CMD_FIFO_PERR_SET0, + "Command FIFO parity error" }, + { F_LSO_HDR_SRAM_PERR_SET3 | F_LSO_HDR_SRAM_PERR_SET2 | + F_LSO_HDR_SRAM_PERR_SET1 | F_LSO_HDR_SRAM_PERR_SET0, + "LSO HDR parity error" }, + { F_TLS_DSGL_PARERR3 | F_TLS_DSGL_PARERR2 | + F_TLS_DSGL_PARERR1 | F_TLS_DSGL_PARERR0, + "TLS Glue DSGL FIFO parity error" }, + { 0 } + }; + static const struct intr_info ulptx_intr_info1 = { + .name = "ULP_TX_INT_CAUSE_1", + .cause_reg = A_ULP_TX_INT_CAUSE_1, + .enable_reg = A_ULP_TX_INT_ENABLE_1, + .fatal = 0x0fffffff, + .flags = IHF_FATAL_IFF_ENABLED, + .details = ulptx_int_cause_1_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_2_details[] = { + { F_EDMA_IN_FIFO_PERR_SET3 | F_EDMA_IN_FIFO_PERR_SET2 | + F_EDMA_IN_FIFO_PERR_SET1 | F_EDMA_IN_FIFO_PERR_SET0, + "EDMA input FIFO parity error" }, + { F_ALIGN_CTL_FIFO_PERR_SET3 | F_ALIGN_CTL_FIFO_PERR_SET2 | + F_ALIGN_CTL_FIFO_PERR_SET1 | F_ALIGN_CTL_FIFO_PERR_SET0, + "Align control FIFO parity error" }, + { F_SGE_FIFO_PERR_SET3 | F_SGE_FIFO_PERR_SET2 | + F_SGE_FIFO_PERR_SET1 | F_SGE_FIFO_PERR_SET0, + "SGE FIFO parity error" }, + { F_STAG_FIFO_PERR_SET3 | F_STAG_FIFO_PERR_SET2 | + F_STAG_FIFO_PERR_SET1 | F_STAG_FIFO_PERR_SET0, + "STAG FIFO parity error" }, + { F_MAP_FIFO_PERR_SET3 | F_MAP_FIFO_PERR_SET2 | + F_MAP_FIFO_PERR_SET1 | F_MAP_FIFO_PERR_SET0, + "MAP FIFO parity error" }, + { F_DMA_FIFO_PERR_SET3 | F_DMA_FIFO_PERR_SET2 | + F_DMA_FIFO_PERR_SET1 | F_DMA_FIFO_PERR_SET0, + "DMA FIFO parity error" }, + { F_FSO_HDR_SRAM_PERR_SET3 | F_FSO_HDR_SRAM_PERR_SET2 | + F_FSO_HDR_SRAM_PERR_SET1 | F_FSO_HDR_SRAM_PERR_SET0, + "FSO HDR memory parity error" }, + { F_T10_PI_SRAM_PERR_SET3 | F_T10_PI_SRAM_PERR_SET2 | + F_T10_PI_SRAM_PERR_SET1 | F_T10_PI_SRAM_PERR_SET0, + "T10 PI memory parity error" }, + { 0 } + }; static const struct intr_info ulptx_intr_info2 = { .name = "ULP_TX_INT_CAUSE_2", .cause_reg = A_ULP_TX_INT_CAUSE_2, .enable_reg = A_ULP_TX_INT_ENABLE_2, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = ulptx_int_cause_2_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_3_details[] = { + { F_GF_SGE_FIFO_PARERR3 | F_GF_SGE_FIFO_PARERR2 | + F_GF_SGE_FIFO_PARERR1 | F_GF_SGE_FIFO_PARERR0, + "GF SGE interface FIFO parity error" }, + { F_DEDUPE_SGE_FIFO_PARERR3 | F_DEDUPE_SGE_FIFO_PARERR2 | + F_DEDUPE_SGE_FIFO_PARERR1 | F_DEDUPE_SGE_FIFO_PARERR0, + "DeDupe SGE interface FIFO parity error" }, + { F_GF3_DSGL_FIFO_PARERR | F_GF2_DSGL_FIFO_PARERR | + F_GF1_DSGL_FIFO_PARERR | F_GF0_DSGL_FIFO_PARERR, + "GF DSGL FIFO parity error" }, + { F_DEDUPE3_DSGL_FIFO_PARERR | F_DEDUPE2_DSGL_FIFO_PARERR | + F_DEDUPE1_DSGL_FIFO_PARERR | F_DEDUPE0_DSGL_FIFO_PARERR, + "DeDupe DSGL FIFO parity error" }, + { F_XP10_SGE_FIFO_PARERR, "XP10 SGE FIFO parity error (Ch0)" }, + { F_DSGL_PAR_ERR, "XP10 DSGL interface parity error" }, + { F_CDDIP_INT, "XP10 decompression interrupt" }, + { F_CCEIP_INT, "XP10 compression interrupt" }, + { F_TLS_SGE_FIFO_PARERR3 | F_TLS_SGE_FIFO_PARERR2 | + F_TLS_SGE_FIFO_PARERR1 | F_TLS_SGE_FIFO_PARERR0, + "TLS Glue SGE FIFO parity error" }, + { F_ULP2SMARBT_RSP_PERR, "ULP2SMARBT response data/CTL parity error" }, + { F_ULPTX2MA_RSP_PERR, "ULP2MA response data/CTL parity error" }, + { F_PCIE2ULP_PERR3 | F_PCIE2ULP_PERR2 | + F_PCIE2ULP_PERR1 | F_PCIE2ULP_PERR0, + "PCIE2ULP EDMA response parity error" }, + { F_CIM2ULP_PERR, "CIM2ULP command parity error (all ports)" }, + { 0 } + }; static const struct intr_info ulptx_intr_info3 = { .name = "ULP_TX_INT_CAUSE_3", .cause_reg = A_ULP_TX_INT_CAUSE_3, .enable_reg = A_ULP_TX_INT_ENABLE_3, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = ulptx_int_cause_3_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_4_details[] = { + { F_XP10_2_ULP_PERR, "XP10 to ULP parity error" }, + { F_ULP_2_XP10_PERR, "ULP to XP10 parity error" }, + { F_CMD_FIFO_LB1 | F_CMD_FIFO_LB0, + "Command FIFO LB error" }, + { F_TF_TP_PERR, "TF TP parity error" }, + { F_TF_SGE_PERR, "TF SGE parity error" }, + { F_TF_MEM_PERR, "TF memory parity error" }, + { F_TF_MP_PERR, "TF MP parity error" }, + { 0 } + }; static const struct intr_info ulptx_intr_info4 = { .name = "ULP_TX_INT_CAUSE_4", .cause_reg = A_ULP_TX_INT_CAUSE_4, .enable_reg = A_ULP_TX_INT_ENABLE_4, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = ulptx_int_cause_4_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_5_details[] = { + { F_DEDUPE_PERR3 | F_DEDUPE_PERR2 | + F_DEDUPE_PERR1 | F_DEDUPE_PERR0, + "DeDupe parity error" }, + { F_GF_PERR3 | F_GF_PERR2 | + F_GF_PERR1 | F_GF_PERR0, + "GF parity error" }, + { F_SGE2ULP_INV_PERR, "SGE2ULP invalid parity error" }, + { F_T7_PL_BUSPERR, "PL bus parity error" }, + { F_TLSTX2ULPTX_PERR3 | F_TLSTX2ULPTX_PERR2 | + F_TLSTX2ULPTX_PERR1 | F_TLSTX2ULPTX_PERR0, + "TLS to ULP parity error" }, + { F_XP10_2_ULP_PL_PERR, "XP10 to ULP PL parity error" }, + { F_ULP_2_XP10_PL_PERR, "ULP to XP10 PL parity error" }, + { 0 } + }; static const struct intr_info ulptx_intr_info5 = { .name = "ULP_TX_INT_CAUSE_5", .cause_reg = A_ULP_TX_INT_CAUSE_5, .enable_reg = A_ULP_TX_INT_ENABLE_5, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = ulptx_int_cause_5_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_6_details[] = { + { F_DDR_HDR_FIFO_PERR_SET3 | F_DDR_HDR_FIFO_PERR_SET2 | + F_DDR_HDR_FIFO_PERR_SET1 | F_DDR_HDR_FIFO_PERR_SET0, + "DDR HDR FIFO parity error" }, + { F_PRE_MP_RSP_PERR_SET3 | F_PRE_MP_RSP_PERR_SET2 | + F_PRE_MP_RSP_PERR_SET1 | F_PRE_MP_RSP_PERR_SET0, + "Pre-MP response parity error" }, + { F_PRE_CQE_FIFO_PERR_SET3 | F_PRE_CQE_FIFO_PERR_SET2 | + F_PRE_CQE_FIFO_PERR_SET1 | F_PRE_CQE_FIFO_PERR_SET0, + "Pre-CQE FIFO parity error" }, + { F_RSP_FIFO_PERR_SET, "Response FIFO parity error" }, + { 0 } + }; static const struct intr_info ulptx_intr_info6 = { .name = "ULP_TX_INT_CAUSE_6", .cause_reg = A_ULP_TX_INT_CAUSE_6, .enable_reg = A_ULP_TX_INT_ENABLE_6, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = ulptx_int_cause_6_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_7_details[] = { + { F_TLS_SGE_FIFO_CORERR3 | F_TLS_SGE_FIFO_CORERR2 | + F_TLS_SGE_FIFO_CORERR1 | F_TLS_SGE_FIFO_CORERR0, + "TLS SGE FIFO correctable error" }, + { F_LSO_HDR_SRAM_CERR_SET3 | F_LSO_HDR_SRAM_CERR_SET2 | + F_LSO_HDR_SRAM_CERR_SET1 | F_LSO_HDR_SRAM_CERR_SET0, + "LSO HDR SRAM correctable error" }, + { F_CORE_CMD_FIFO_CERR_SET_CH3_LB1 | F_CORE_CMD_FIFO_CERR_SET_CH2_LB1 | + F_CORE_CMD_FIFO_CERR_SET_CH1_LB1 | F_CORE_CMD_FIFO_CERR_SET_CH0_LB1, + "Core command FIFO LB1 correctable error" }, + { F_CORE_CMD_FIFO_CERR_SET_CH3_LB0 | F_CORE_CMD_FIFO_CERR_SET_CH2_LB0 | + F_CORE_CMD_FIFO_CERR_SET_CH1_LB0 | F_CORE_CMD_FIFO_CERR_SET_CH0_LB0, + "Core command FIFO LB0 correctable error" }, + { F_CQE_FIFO_CERR_SET3 | F_CQE_FIFO_CERR_SET2 | + F_CQE_FIFO_CERR_SET1 | F_CQE_FIFO_CERR_SET0, + "CQE FIFO correctable error" }, + { F_PRE_CQE_FIFO_CERR_SET3 | F_PRE_CQE_FIFO_CERR_SET2 | + F_PRE_CQE_FIFO_CERR_SET1 | F_PRE_CQE_FIFO_CERR_SET0, + "Pre-CQE FIFO correctable error" }, + { 0 } + }; static const struct intr_info ulptx_intr_info7 = { .name = "ULP_TX_INT_CAUSE_7", .cause_reg = A_ULP_TX_INT_CAUSE_7, .enable_reg = A_ULP_TX_INT_ENABLE_7, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulptx_int_cause_7_details, .actions = NULL, }; + static const struct intr_details ulptx_int_cause_8_details[] = { + { F_MEM_RSP_FIFO_CERR_SET3 | F_MEM_RSP_FIFO_CERR_SET2 | + F_MEM_RSP_FIFO_CERR_SET1 | F_MEM_RSP_FIFO_CERR_SET0, + "Memory response FIFO correctable error" }, + { F_PI_SRAM_CERR_SET3 | F_PI_SRAM_CERR_SET2 | + F_PI_SRAM_CERR_SET1 | F_PI_SRAM_CERR_SET0, + "PI SRAM correctable error" }, + { F_PRE_MP_RSP_CERR_SET3 | F_PRE_MP_RSP_CERR_SET2 | + F_PRE_MP_RSP_CERR_SET1 | F_PRE_MP_RSP_CERR_SET0, + "Pre-MP response correctable error" }, + { F_DDR_HDR_FIFO_CERR_SET3 | F_DDR_HDR_FIFO_CERR_SET2 | + F_DDR_HDR_FIFO_CERR_SET1 | F_DDR_HDR_FIFO_CERR_SET0, + "DDR HDR FIFO correctable error" }, + { F_CMD_FIFO_CERR_SET3 | F_CMD_FIFO_CERR_SET2 | + F_CMD_FIFO_CERR_SET1 | F_CMD_FIFO_CERR_SET0, + "Command FIFO correctable error" }, + { F_GF_SGE_FIFO_CORERR3 | F_GF_SGE_FIFO_CORERR2 | + F_GF_SGE_FIFO_CORERR1 | F_GF_SGE_FIFO_CORERR0, + "GF SGE FIFO correctable error" }, + { F_DEDUPE_SGE_FIFO_CORERR3 | F_DEDUPE_SGE_FIFO_CORERR2 | + F_DEDUPE_SGE_FIFO_CORERR1 | F_DEDUPE_SGE_FIFO_CORERR0, + "DeDupe SGE FIFO correctable error" }, + { F_RSP_FIFO_CERR_SET, "Response FIFO correctable error" }, + { 0 } + }; static const struct intr_info ulptx_intr_info8 = { .name = "ULP_TX_INT_CAUSE_8", .cause_reg = A_ULP_TX_INT_CAUSE_8, .enable_reg = A_ULP_TX_INT_ENABLE_8, .fatal = 0, .flags = 0, - .details = NULL, + .details = ulptx_int_cause_8_details, .actions = NULL, }; bool fatal = false; - fatal |= t4_handle_intr(adap, &ulptx_intr_info, 0, flags); - if (chip_id(adap) > CHELSIO_T4) - fatal |= t4_handle_intr(adap, &ulptx_intr_info2, 0, flags); if (chip_id(adap) > CHELSIO_T6) { + fatal |= t4_handle_intr(adap, &ulptx_intr_info1, 0, flags); + fatal |= t4_handle_intr(adap, &ulptx_intr_info2, 0, flags); fatal |= t4_handle_intr(adap, &ulptx_intr_info3, 0, flags); fatal |= t4_handle_intr(adap, &ulptx_intr_info4, 0, flags); fatal |= t4_handle_intr(adap, &ulptx_intr_info5, 0, flags); fatal |= t4_handle_intr(adap, &ulptx_intr_info6, 0, flags); fatal |= t4_handle_intr(adap, &ulptx_intr_info7, 0, flags); fatal |= t4_handle_intr(adap, &ulptx_intr_info8, 0, flags); + } else { + if (chip_id(adap) == CHELSIO_T6) + ulptx_intr_info.details = t6_ulptx_int_cause_details; + else + ulptx_intr_info.details = ulptx_intr_details; + fatal |= t4_handle_intr(adap, &ulptx_intr_info, 0, flags); + if (chip_id(adap) > CHELSIO_T4) + fatal |= t4_handle_intr(adap, &ulptx_intr_info2, 0, flags); } return (fatal); @@ -5671,6 +6561,25 @@ static bool pmtx_dump_dbg_stats(struct adapter *adap, int arg, int flags) */ static bool pmtx_intr_handler(struct adapter *adap, int arg, int flags) { + static const struct intr_details t7_pmtx_int_cause_fields[] = { + { F_MASTER_PERR, "PM_TX master parity error" }, + { F_T7_ZERO_C_CMD_ERROR, "PM_TX PCMD with zero length error" }, + { F_OESPI_COR_ERR, " oespi FIFO Correctable Error" }, + { F_ICSPI_COR_ERR, " icspi FIFO Correctable Error" }, + { F_ICSPI_OVFL, " icspi FIFO overflow" }, + { F_T7_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large" }, + { F_T7_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large" }, + { F_T7_PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large" }, + { F_PCMD_LEN_OVFL3, "PMTX channel 2 pcmd too large" }, + { F_T7_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd" }, + { 0x00f00000, "PM_TX PCMD length larger than oespi capacity" }, + { 0x000f0000, "PM_TX icspi 2x FIFO Rx framing error" }, + { 0x0000f000, "PM_TX icspi FIFO Tx framing error" }, + { 0x00000f00, "PM_TX oespi FIFO Rx framing error" }, + { 0x000000f0, "PM_TX oespi FIFO Tx framing error" }, + { 0x0000000f, "PM_TX oespi 2x FIFO Tx framing error" }, + { 0 } + }; static const struct intr_details pmtx_int_cause_fields[] = { { F_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large" }, { F_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large" }, @@ -5692,17 +6601,58 @@ static bool pmtx_intr_handler(struct adapter *adap, int arg, int flags) { 0xffffffff, -1, pmtx_dump_dbg_stats }, { 0 }, }; - static const struct intr_info pmtx_int_cause = { + struct intr_info pmtx_int_cause = { .name = "PM_TX_INT_CAUSE", .cause_reg = A_PM_TX_INT_CAUSE, .enable_reg = A_PM_TX_INT_ENABLE, .fatal = 0xffffffff, - .flags = 0, - .details = pmtx_int_cause_fields, + .flags = IHF_CLR_DELAYED, + .details = NULL, .actions = pmtx_int_cause_actions, }; + static const struct intr_details pmtx_perr_cause_details[] = { + { F_ICSPI_OVFL, "icspi FIFO Overflow" }, + { F_OSPI_OVERFLOW3_TX, " OSPI overflow on channel 3 error." }, + { F_OSPI_OVERFLOW2_TX, " OSPI overflow on channel 2 error." }, + { F_OSPI_OVERFLOW1_TX, " OSPI overflow on channel 1 error." }, + { F_OSPI_OVERFLOW0_TX, " OSPI overflow on channel 0 error." }, + { F_T7_BUNDLE_LEN_OVFL_EN, "This bit indicates bundle_len_ovfl_err." }, + { F_T7_M_INTFPERREN, "This bit indicates Parity error from MA interfaces." }, + { F_T7_1_SDC_ERR, + "SDC Error reported by Check PCMD which carries CRC16 from TP-CSide." }, + { F_MC_WCNT_FIFO_PERR, "MC Interface Write count FIFO Parity error" }, + { F_MC_WDATA_FIFO_PERR, "MC Interface Write Data FIFO Parity error" }, + { F_MC_RCNT_FIFO_PERR, "MC Interface Read count FIFO Parity error" }, + { F_MC_RDATA_FIFO_PERR, "MC Interface Read Data FIFO Parity error" }, + { F_TOKEN_PAR_ERROR, "c_pcmd, Token FIFO par error" }, + { F_BUNDLE_LEN_PAR_ERROR, "oespi par error" }, + { F_OESPI_PAR_ERROR, "oespi par error" }, + { F_DB_OPTIONS_PAR_ERROR, "db_options par error" }, + { F_ICSPI_PAR_ERROR, "icspi par error" }, + { F_C_PCMD_TOKEN_PAR_ERROR, "c_pcmd par error" }, + { 0 } + }; + static struct intr_info pmtx_perr_cause = { + .name = "PM_TX_PERR_CAUSE", + .cause_reg = A_PM_TX_PERR_CAUSE, + .enable_reg = A_PM_TX_PERR_ENABLE, + .fatal = 0xffffffff, + .flags = 0, + .details = pmtx_perr_cause_details, + .actions = NULL, + }; + bool fatal; + + if (chip_id(adap) >= CHELSIO_T7) + pmtx_int_cause.details = t7_pmtx_int_cause_fields; + else + pmtx_int_cause.details = pmtx_int_cause_fields; + fatal = t4_handle_intr(adap, &pmtx_int_cause, 0, flags); + if (chip_id(adap) >= CHELSIO_T7) + fatal |= t4_handle_intr(adap, &pmtx_perr_cause, 0, flags); + clear_int_cause_reg(adap, &pmtx_int_cause, flags); - return (t4_handle_intr(adap, &pmtx_int_cause, 0, flags)); + return (fatal); } /* @@ -5710,6 +6660,20 @@ static bool pmtx_intr_handler(struct adapter *adap, int arg, int flags) */ static bool pmrx_intr_handler(struct adapter *adap, int arg, int flags) { + static const struct intr_details t7_pmrx_int_cause_fields[] = { + { F_MASTER_PERR, "PM_RX master parity error" }, + { 0x18000000, "PMRX ospi overflow" }, + { F_BUNDLE_LEN_OVFL, "PMRX bundle len FIFO overflow" }, + { F_SDC_ERR, "PMRX SDC error" }, + { F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd" }, + { 0x003c0000, "PMRX iespi FIFO2X Rx framing error" }, + { 0x0003c000, "PMRX iespi Rx framing error" }, + { 0x00003c00, "PMRX iespi Tx framing error" }, + { 0x00000300, "PMRX ocspi Rx framing error" }, + { 0x000000c0, "PMRX ocspi Tx framing error" }, + { 0x00000030, "PMRX ocspi FIFO2X Tx framing error" }, + { 0 } + }; static const struct intr_details pmrx_int_cause_fields[] = { /* T6+ */ { 0x18000000, "PMRX ospi overflow" }, @@ -5732,17 +6696,90 @@ static bool pmrx_intr_handler(struct adapter *adap, int arg, int flags) { F_E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error"}, { 0 } }; - static const struct intr_info pmrx_int_cause = { + struct intr_info pmrx_int_cause = { .name = "PM_RX_INT_CAUSE", .cause_reg = A_PM_RX_INT_CAUSE, .enable_reg = A_PM_RX_INT_ENABLE, .fatal = 0x1fffffff, + .flags = IHF_FATAL_IFF_ENABLED | IHF_CLR_DELAYED, + .details = NULL, + .actions = NULL, + }; + static const struct intr_details pm_rx_int_cause_2_details[] = { + { F_CACHE_SRAM_ODD_CERR, "Cache Data Odd SRAM Correctable Error" }, + { F_CACHE_SRAM_EVEN_CERR, "Cache Data Even SRAM Correctable Error" }, + { F_CACHE_LRU_LEFT_CERR, "Cache LRU Left SRAM Correctable Error" }, + { F_CACHE_LRU_RIGHT_CERR, "Cache LRU Right SRAM Correctable Error" }, + { F_CACHE_ISLAND_CERR, "Cache Island SRAM Correctable Error" }, + { F_OCSPI_CERR, "ocspi FIFO Correctable Error" }, + { F_IESPI_CERR, "iespi FIFO Correctable Error" }, + { F_OCSPI2_RX_FRAMING_ERROR, "ocspi FIFO channel 2 Rx/wr framing error" }, + { F_OCSPI3_RX_FRAMING_ERROR, "ocspi FIFO channel 3 Rx/wr framing error" }, + { F_OCSPI2_TX_FRAMING_ERROR, "ocspi FIFO channel 2 Tx/rd framing error" }, + { F_OCSPI3_TX_FRAMING_ERROR, "ocspi FIFO channel 3 Tx/rd framing error" }, + { F_OCSPI2_OFIFO2X_TX_FRAMING_ERROR, "ocspi 2x FIFO 2 Tx/rd framing error" }, + { F_OCSPI3_OFIFO2X_TX_FRAMING_ERROR, "ocspi 2x FIFO 3 Tx/rd framing error" }, + { 0 } + }; + static struct intr_info pmrx_int_cause2 = { + .name = "PM_RX_INT_CAUSE_2", + .cause_reg = A_PM_RX_INT_CAUSE_2, + .enable_reg = A_PM_RX_INT_ENABLE_2, + .fatal = 0x1fffffff, + .flags = IHF_FATAL_IFF_ENABLED, + .details = pm_rx_int_cause_2_details, + .actions = NULL, + }; + static const struct intr_details pm_rx_perr_cause_details[] = { + { F_T7_SDC_ERR, "SDC error. CRC provided by TP and PM didn't match." }, + { F_T7_MA_INTF_SDC_ERR, "MA intf SDC perr" }, + { F_E_PCMD_PERR, "ulp_rx 2 pm_rx PCMD interface parity error." }, + { F_CACHE_RSP_DFIFO_PERR, "Cache Response Data FIFO Parity error" }, + { F_CACHE_SRAM_ODD_PERR, "Cache Odd SRAM error" }, + { F_CACHE_SRAM_EVEN_PERR, "Cache Even SRAM error" }, + { F_CACHE_RSVD_PERR, "Cache Reserved Parity error" }, + { F_CACHE_LRU_LEFT_PERR, "Cache LRU Left SRAM error" }, + { F_CACHE_LRU_RIGHT_PERR, "Cache LRU Rigth SRAM error" }, + { F_CACHE_RSP_CMD_PERR, "Cache Response Command FIFO error" }, + { F_CACHE_SRAM_CMD_PERR, "Cache SRAM Command FIFO error" }, + { F_CACHE_MA_CMD_PERR, "Cache MA Command FIFO error" }, + { F_CACHE_TCAM_PERR, "Cache TCAM Parity error" }, + { F_CACHE_ISLAND_PERR, "Cache island SRAM Parity error" }, + { F_MC_WCNT_FIFO_PERR, "MC Interface Write count FIFO Parity error" }, + { F_MC_WDATA_FIFO_PERR, "MC Interface Write Data FIFO Parity error" }, + { F_MC_RCNT_FIFO_PERR, "MC Interface Read count FIFO Parity error" }, + { F_MC_RDATA_FIFO_PERR, "MC Interface Read Data FIFO Parity error" }, + { F_TOKEN_FIFO_PERR, "Token FIFO Parity error" }, + { F_T7_BUNDLE_LEN_PARERR, "Bundle len fifo had parity error." }, + { F_OCSPI_PAR_ERROR, "ocspi par error vector" }, + { F_DB_OPTIONS_PAR_ERROR, "db_options par error" }, + { F_IESPI_PAR_ERROR, "iespi par error" }, + { F_E_PCMD_PAR_ERROR, "e_pcmd par error" }, + { 0 } + }; + static struct intr_info pmrx_perr_cause = { + .name = "PM_RX_PERR_CAUSE", + .cause_reg = A_PM_RX_PERR_CAUSE, + .enable_reg = A_PM_RX_PERR_ENABLE, + .fatal = 0x1fffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = pmrx_int_cause_fields, + .details = pm_rx_perr_cause_details, .actions = NULL, }; + bool fatal; + + if (chip_id(adap) >= CHELSIO_T7) { + pmrx_int_cause.details = t7_pmrx_int_cause_fields; + fatal = t4_handle_intr(adap, &pmrx_int_cause, 0, flags); + fatal |= t4_handle_intr(adap, &pmrx_int_cause2, 0, flags); + fatal |= t4_handle_intr(adap, &pmrx_perr_cause, 0, flags); + } else { + pmrx_int_cause.details = pmrx_int_cause_fields; + fatal = t4_handle_intr(adap, &pmrx_int_cause, 0, flags); + } + clear_int_cause_reg(adap, &pmrx_int_cause, flags); - return (t4_handle_intr(adap, &pmrx_int_cause, 0, flags)); + return (fatal); } /* @@ -5751,6 +6788,9 @@ static bool pmrx_intr_handler(struct adapter *adap, int arg, int flags) static bool cplsw_intr_handler(struct adapter *adap, int arg, int flags) { static const struct intr_details cplsw_int_cause_fields[] = { + /* T7+ */ + { F_PERR_CPL_128TO128_3, "CPLSW 128TO128 FIFO3 parity error" }, + { F_PERR_CPL_128TO128_2, "CPLSW 128TO128 FIFO2 parity error" }, /* T5+ */ { F_PERR_CPL_128TO128_1, "CPLSW 128TO128 FIFO1 parity error" }, { F_PERR_CPL_128TO128_0, "CPLSW 128TO128 FIFO0 parity error" }, @@ -5803,6 +6843,8 @@ static bool le_intr_handler(struct adapter *adap, int arg, int flags) { 0 } }; static const struct intr_details t6_le_intr_details[] = { + { F_CACHEINTPERR, "Parity error in cache module" }, + { F_CACHESRAMPERR, "Parity error in data sram " }, { F_CLIPSUBERR, "LE CLIP CAM reverse substitution error" }, { F_CLCAMFIFOERR, "LE CLIP CAM internal FIFO error" }, { F_CTCAMINVLDENT, "Invalid IPv6 CLIP TCAM entry" }, @@ -5865,51 +6907,206 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) .details = mps_rx_perr_intr_details, .actions = NULL, }; + static const struct intr_details mps_rx_func_intr_details[] = { + { F_MTU_ERR3, "MTU error interrupt enable bit for loopback group 3" }, + { F_MTU_ERR2, "MTU error interrupt enable bit for loopback group 2" }, + { F_MTU_ERR1, "MTU error interrupt enable bit for loopback group 1" }, + { F_MTU_ERR0, "MTU error interrupt enable bit for loopback group 0" }, + { F_DBG_LEN_ERR, "Oring of len error in traffic transfer b/w internal modules" }, + { F_DBG_SPI_ERR, "Oring of spi error in traffic transfer b/w internal modules" }, + { F_DBG_SE_CNT_ERR, "Oring of se cnt error in traffic transfer" }, + { F_DBG_SPI_LEN_SE_CNT_ERR, "Oring of all se_cnt|len|spi errors" }, + { 0 } + }; + static const struct intr_info mps_rx_func_intr_info = { + .name = "MPS_RX_FUNC_INT_CAUSE", + .cause_reg = A_MPS_RX_FUNC_INT_CAUSE, + .enable_reg = A_MPS_RX_FUNC_INT_ENABLE, + .fatal = 0xffffffff, + .flags = IHF_FATAL_IFF_ENABLED, + .details = mps_rx_func_intr_details, + .actions = NULL, + }; + static const struct intr_details mpsrx_int_cause_2_details[] = { + { F_CRYPTO2MPS_RX0_PERR | F_CRYPTO2MPS_RX1_PERR | + F_CRYPTO2MPS_RX2_PERR | F_CRYPTO2MPS_RX3_PERR, + "Crypto to MPS RX interface parity error" }, + { F_INIC2MPS_TX1_PERR | F_INIC2MPS_TX0_PERR, + "INIC to MPS TX interface parity error" }, + { F_XGMAC2MPS_RX1_PERR | F_XGMAC2MPS_RX0_PERR, + "XGMAC to MPS RX interface parity error" }, + { F_RX_FINAL_TF_FIFO_PERR, + "Final RX token FIFO output parity error" }, + { F_MPS_DWRR_FIFO_PERR, + "MPS DWRR MTU FIFO parity error" }, + { F_MAC_TF_FIFO_PERR, + "MAC token FIFO parity error" }, + { F_MAC2MPS_PT3_PERR | F_MAC2MPS_PT2_PERR | + F_MAC2MPS_PT1_PERR | F_MAC2MPS_PT0_PERR, + "MAC to MPS interface parity error" }, + { F_TP_LPBK_FIFO_PERR, "TP loopback FIFO parity error" }, + { F_TP_LPBK_TF_PERR, "Loopback token FIFO parity error" }, + { 0 } + }; static const struct intr_info mps_rx_perr_intr_info2 = { .name = "MPS_RX_PERR_INT_CAUSE2", .cause_reg = A_MPS_RX_PERR_INT_CAUSE2, .enable_reg = A_MPS_RX_PERR_INT_ENABLE2, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = mpsrx_int_cause_2_details, .actions = NULL, }; + static const struct intr_details mpsrx_int_cause_3_details[] = { + { F_FIFO_REPL_CH3_CERR | F_FIFO_REPL_CH2_CERR | + F_FIFO_REPL_CH1_CERR | F_FIFO_REPL_CH0_CERR, + "Replication FIFO ECC error" }, + { F_VLAN_FILTER_RAM_CERR, "VLAN filter SRAM ECC error" }, + { F_MPS_RX_TD_STAT_FIFO_PERR_CH3 | F_MPS_RX_TD_STAT_FIFO_PERR_CH2 | + F_MPS_RX_TD_STAT_FIFO_PERR_CH1 | F_MPS_RX_TD_STAT_FIFO_PERR_CH0, + "MPS RX TD status descriptor FIFO parity error" }, + { F_RPLCT_HDR_FIFO_IN_PERR_CH3 | F_RPLCT_HDR_FIFO_IN_PERR_CH2 | + F_RPLCT_HDR_FIFO_IN_PERR_CH1 | F_RPLCT_HDR_FIFO_IN_PERR_CH0, + "MPS RX replication header input FIFO parity error" }, + { F_ID_FIFO_IN_PERR_CH3 | F_ID_FIFO_IN_PERR_CH2 | + F_ID_FIFO_IN_PERR_CH1 | F_ID_FIFO_IN_PERR_CH0, + "MPS RX replication ID input FIFO parity error" }, + { F_DESC_HDR2_PERR_CH3 | F_DESC_HDR2_PERR_CH2 | + F_DESC_HDR2_PERR_CH1 | F_DESC_HDR2_PERR_CH0, + "MPS RX replication descriptor/header2 FIFO parity error" }, + { F_FIFO_REPL_PERR_CH3 | F_FIFO_REPL_PERR_CH2 | + F_FIFO_REPL_PERR_CH1 | F_FIFO_REPL_PERR_CH0, + "Replication FIFO parity error" }, + { F_MPS_RX_TD_PERR_CH3 | F_MPS_RX_TD_PERR_CH2 | + F_MPS_RX_TD_PERR_CH1 | F_MPS_RX_TD_PERR_CH0, + "MPS RX TD input FIFO parity error" }, + { 0 } + }; static const struct intr_info mps_rx_perr_intr_info3 = { .name = "MPS_RX_PERR_INT_CAUSE3", .cause_reg = A_MPS_RX_PERR_INT_CAUSE3, .enable_reg = A_MPS_RX_PERR_INT_ENABLE3, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = mpsrx_int_cause_3_details, .actions = NULL, }; + static const struct intr_details mpsrx_int_cause_4_details[] = { + { F_VNI_MULTICAST_FIFO_ECC_ERR_CH3 | F_VNI_MULTICAST_FIFO_ECC_ERR_CH2, + "RX out VNI multicast SRAM ECC error" }, + { F_HASH_SRAM_CLS_ENG1 | F_HASH_SRAM_CLS_ENG0, + "Classification engine hash SRAM ECC error" }, + { F_CLS_TCAM_SRAM_CLS_ENG1 | F_CLS_TCAM_SRAM_CLS_ENG0, + "Classification engine TCAM SRAM ECC error" }, + { F_CLS_TCAM_CRC_SRAM_CLS_ENG1 | F_CLS_TCAM_CRC_SRAM_CLS_ENG0, + "Classification engine TCAM CRC SRAM ECC error" }, + { F_DWRR_CH_FIFO_ECC_ERR, "DWRR output FIFO ECC error" }, + { F_MAC_RX_FIFO_ECC_ERR, "MAC RX FIFO ECC error" }, + { F_LPBK_RX_FIFO_ECC_ERR, "Loopback RX FIFO ECC error" }, + { F_CRS_DATA_STORE_N_FWD_CH3 | F_CRS_DATA_STORE_N_FWD_CH2 | + F_CRS_DATA_STORE_N_FWD_CH1 | F_CRS_DATA_STORE_N_FWD_CH0, + "CRS store and forward FIFO ECC error" }, + { F_TRACE_FWD_FIFO_CERR_CH3 | F_TRACE_FWD_FIFO_CERR_CH2 | + F_TRACE_FWD_FIFO_CERR_CH1 | F_TRACE_FWD_FIFO_CERR_CH0, + "Trace packet forward FIFO ECC error" }, + { F_TRANSPARENT_ENCAP_FWD_FIFO_CERR_CH3 | F_TRANSPARENT_ENCAP_FWD_FIFO_CERR_CH2 | + F_TRANSPARENT_ENCAP_FWD_FIFO_CERR_CH1 | F_TRANSPARENT_ENCAP_FWD_FIFO_CERR_CH0, + "Transparent encap forward FIFO ECC error" }, + { F_PTP_TRACE_FWD_FIFO_CERR_CH3 | F_PTP_TRACE_FWD_FIFO_CERR_CH2 | + F_PTP_TRACE_FWD_FIFO_CERR_CH1 | F_PTP_TRACE_FWD_FIFO_CERR_CH0, + "PTP packet forward FIFO ECC error" }, + { 0 } + }; static const struct intr_info mps_rx_perr_intr_info4 = { .name = "MPS_RX_PERR_INT_CAUSE4", .cause_reg = A_MPS_RX_PERR_INT_CAUSE4, .enable_reg = A_MPS_RX_PERR_INT_ENABLE4, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = mpsrx_int_cause_4_details, .actions = NULL, }; + static const struct intr_details mpsrx_int_cause_5_details[] = { + { F_MPS2CRYP_RX_FIFO3_PERR | F_MPS2CRYP_RX_FIFO2_PERR | + F_MPS2CRYP_RX_FIFO1_PERR | F_MPS2CRYP_RX_FIFO0_PERR, + "MPS to Crypto RX interface FIFO parity error" }, + { F_VNI_MULTICAST_SRAM2_PERR | F_VNI_MULTICAST_SRAM1_PERR | + F_VNI_MULTICAST_SRAM0_PERR, + "VNI multicast SRAM parity error" }, + { F_MAC_MULTICAST_SRAM4_PERR | F_MAC_MULTICAST_SRAM3_PERR | + F_MAC_MULTICAST_SRAM2_PERR | F_MAC_MULTICAST_SRAM1_PERR | + F_MAC_MULTICAST_SRAM0_PERR, + "MAC multicast SRAM parity error" }, + { F_MEM_WRAP_IPSEC_HDR_UPD_FIFO3_PERR | F_MEM_WRAP_IPSEC_HDR_UPD_FIFO2_PERR | + F_MEM_WRAP_IPSEC_HDR_UPD_FIFO1_PERR | F_MEM_WRAP_IPSEC_HDR_UPD_FIFO0_PERR, + "IPsec header update storing FIFO parity error" }, + { F_MEM_WRAP_CR2MPS_RX_FIFO3_PERR | F_MEM_WRAP_CR2MPS_RX_FIFO2_PERR | + F_MEM_WRAP_CR2MPS_RX_FIFO1_PERR | F_MEM_WRAP_CR2MPS_RX_FIFO0_PERR, + "IPsec storing FIFO parity error" }, + { F_MEM_WRAP_NON_IPSEC_FIFO3_PERR | F_MEM_WRAP_NON_IPSEC_FIFO2_PERR | + F_MEM_WRAP_NON_IPSEC_FIFO1_PERR | F_MEM_WRAP_NON_IPSEC_FIFO0_PERR, + "Non-IPsec storing FIFO parity error" }, + { F_MEM_WRAP_TP_DB_REQ_FIFO3_PERR | F_MEM_WRAP_TP_DB_REQ_FIFO2_PERR | + F_MEM_WRAP_TP_DB_REQ_FIFO1_PERR | F_MEM_WRAP_TP_DB_REQ_FIFO0_PERR, + "TP DB request storing FIFO parity error" }, + { F_MEM_WRAP_CNTRL_FIFO3_PERR | F_MEM_WRAP_CNTRL_FIFO2_PERR | + F_MEM_WRAP_CNTRL_FIFO1_PERR | F_MEM_WRAP_CNTRL_FIFO0_PERR, + "Header flit storing FIFO parity error" }, + { 0 } + }; static const struct intr_info mps_rx_perr_intr_info5 = { .name = "MPS_RX_PERR_INT_CAUSE5", .cause_reg = A_MPS_RX_PERR_INT_CAUSE5, .enable_reg = A_MPS_RX_PERR_INT_ENABLE5, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = mpsrx_int_cause_5_details, .actions = NULL, }; + static const struct intr_details mpsrx_int_cause_6_details[] = { + { F_T7_MEM_WRAP_IPSEC_HDR_UPD_FIFO3_PERR | F_T7_MEM_WRAP_IPSEC_HDR_UPD_FIFO2_PERR | + F_T7_MEM_WRAP_IPSEC_HDR_UPD_FIFO1_PERR | F_T7_MEM_WRAP_IPSEC_HDR_UPD_FIFO0_PERR, + "IPsec header update storing FIFO parity error" }, + { F_MEM_WRAP_CR2MPS_UPDTD_HDR_FIFO3_PERR | F_MEM_WRAP_CR2MPS_UPDTD_HDR_FIFO2_PERR | + F_MEM_WRAP_CR2MPS_UPDTD_HDR_FIFO1_PERR | F_MEM_WRAP_CR2MPS_UPDTD_HDR_FIFO0_PERR, + "IPsec updated header only storing FIFO parity error" }, + { F_MEM_WRAP_CR2MPS_RX_FIFO3_PERR | F_MEM_WRAP_CR2MPS_RX_FIFO2_PERR | + F_MEM_WRAP_CR2MPS_RX_FIFO1_PERR | F_MEM_WRAP_CR2MPS_RX_FIFO0_PERR, + "IPsec storing FIFO parity error" }, + { F_MEM_WRAP_NON_IPSEC_FIFO3_PERR | F_MEM_WRAP_NON_IPSEC_FIFO2_PERR | + F_MEM_WRAP_NON_IPSEC_FIFO1_PERR | F_MEM_WRAP_NON_IPSEC_FIFO0_PERR, + "Non-IPsec storing FIFO parity error" }, + { F_MEM_WRAP_TP_DB_REQ_FIFO3_PERR | F_MEM_WRAP_TP_DB_REQ_FIFO2_PERR | + F_MEM_WRAP_TP_DB_REQ_FIFO1_PERR | F_MEM_WRAP_TP_DB_REQ_FIFO0_PERR, + "TP DB request storing FIFO parity error" }, + { F_MEM_WRAP_CNTRL_FIFO3_PERR | F_MEM_WRAP_CNTRL_FIFO2_PERR | + F_MEM_WRAP_CNTRL_FIFO1_PERR | F_MEM_WRAP_CNTRL_FIFO0_PERR, + "Header flit storing FIFO parity error" }, + { 0 } + }; static const struct intr_info mps_rx_perr_intr_info6 = { .name = "MPS_RX_PERR_INT_CAUSE6", .cause_reg = A_MPS_RX_PERR_INT_CAUSE6, .enable_reg = A_MPS_RX_PERR_INT_ENABLE6, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = mpsrx_int_cause_6_details, .actions = NULL, }; + static const struct intr_details t7_mpstx_int_cause_details[] = { + { F_T7_PORTERR, "Tx received a frame for TP destined to a disable port" }, + { F_T7_FRMERR, "Framing error in received Data from TP or Data to MAC" }, + { F_T7_SECNTERR, "SOP-EOP count error in received Data from TP or Data to MAC" }, + { F_T7_BUBBLE, "Valid is deasserted between SOP and EOP" }, + { F_TX_TF_FIFO_PERR, "Parity error of TX token fifo" }, + { F_TX_FIFO_PERR, "Parity error of TX MPS2MAC underrun fifo" }, + { 0x0003c000, "Parity error of fifo storing non-ipsec +1 flit ipsec pkt" }, + { 0x00003fc0, "Interface parity error on TP/Crypto to MPS TX" }, + { F_NCSI2MPS, "interface Parity Error on ncsi2mps_tx_ch3" }, + { F_NCSIFIFO, "Parity Error in mps_tx_arbiter input FIFO (from NCSI)" }, + { 0x0000000f, "Parity Error in mps_tx_arbiter input FIFO (from TP)" }, + { 0 } + }; static const struct intr_details mps_tx_intr_details[] = { { F_PORTERR, "MPS Tx destination port is disabled" }, { F_FRMERR, "MPS Tx framing error" }, @@ -5921,22 +7118,27 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) { V_TPFIFO(M_TPFIFO), "MPS Tx TP FIFO parity error" }, { 0 } }; - static const struct intr_info mps_tx_intr_info = { + struct intr_info mps_tx_intr_info = { .name = "MPS_TX_INT_CAUSE", .cause_reg = A_MPS_TX_INT_CAUSE, .enable_reg = A_MPS_TX_INT_ENABLE, .fatal = 0x1ffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = mps_tx_intr_details, + .details = NULL, .actions = NULL, }; + static const struct intr_details mpstx_int_cause_2_details[] = { + { F_TX_FIFO_PERR, "ECC error of TX MPS2MAC underrun fifo" }, + { 0x0000000f, "ECC error of fifo storing non-ipsec +1 flit ipsec pkt" }, + { 0 } + }; static const struct intr_info mps_tx_intr_info2 = { .name = "MPS_TX_INT2_CAUSE", .cause_reg = A_MPS_TX_INT2_CAUSE, .enable_reg = A_MPS_TX_INT2_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = mpstx_int_cause_2_details, .actions = NULL, }; static const struct intr_info mps_tx_intr_info3 = { @@ -5972,22 +7174,51 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) .details = mps_trc_intr_details, .actions = NULL, }; + static const struct intr_details t7_mps_trc_intr_details[] = { + { F_T7_TRCPLERRENB, "TRC PL error" }, + { F_T7_MISCPERR, "TRC header register parity error" }, + { 0x0000ff00, "TRC packet FIFO parity error" }, + { 0x000000ff, "TRC filter memory parity error" }, + { 0 } + }; static const struct intr_info t7_mps_trc_intr_info = { .name = "MPS_TRC_INT_CAUSE", .cause_reg = A_T7_MPS_TRC_INT_CAUSE, .enable_reg = A_T7_MPS_TRC_INT_ENABLE, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = mps_trc_intr_details, + .details = t7_mps_trc_intr_details, .actions = NULL, }; + static const struct intr_details t7_trc_int_cause2_details[] = { + { 0x0001e000, "TRC Tx2Rx down-converter correctable error" }, + { 0x00001800, "TRC MPS2MAC down-converter correctable error" }, + { 0x00000600, "TRC MAC2MPS down-converter correctable error" }, + { 0x000001e0, "TRC Tx2Rx down-converter parity error" }, + { 0x00000018, "TRC MAC2MPS down-converter parity error" }, + { 0x00000006, "TRC MPS2MAC down-converter parity error" }, + { 0 } + }; static const struct intr_info t7_mps_trc_intr_info2 = { .name = "MPS_TRC_INT_CAUSE2", .cause_reg = A_MPS_TRC_INT_CAUSE2, .enable_reg = A_MPS_TRC_INT_ENABLE2, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = t7_trc_int_cause2_details, + .actions = NULL, + }; + static const struct intr_details mps_stat_intr_details[] = { + { F_PLREADSYNCERR, "MPS pl read sync error" }, + { 0 } + }; + static const struct intr_info mps_stat_intr_info = { + .name = "MPS_STAT_INT_CAUSE", + .cause_reg = A_MPS_STAT_INT_CAUSE, + .enable_reg = A_MPS_STAT_INT_ENABLE, + .fatal = 0xf, + .flags = IHF_FATAL_IFF_ENABLED, + .details = mps_stat_intr_details, .actions = NULL, }; static const struct intr_details mps_stat_sram_intr_details[] = { @@ -6030,6 +7261,9 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) .actions = NULL, }; static const struct intr_details mps_cls_intr_details[] = { + { F_T7_PLERRENB, "PL error"}, + { F_CIM2MPS_INTF_PAR, "cim2mps interface parity"}, + { F_TCAM_CRC_SRAM, "tcam crc sram parity error"}, { F_HASHSRAM, "MPS hash SRAM parity error" }, { F_MATCHTCAM, "MPS match TCAM parity error" }, { F_MATCHSRAM, "MPS match SRAM parity error" }, @@ -6058,9 +7292,14 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) .actions = NULL, }; bool fatal = false; + if (chip_id(adap) >= CHELSIO_T7) + mps_tx_intr_info.details = t7_mpstx_int_cause_details; + else + mps_tx_intr_info.details = mps_tx_intr_details; fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info, 0, flags); if (chip_id(adap) > CHELSIO_T6) { + fatal |= t4_handle_intr(adap, &mps_rx_func_intr_info, 0, flags); fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info2, 0, flags); fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info3, 0, flags); fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info4, 0, flags); @@ -6076,6 +7315,7 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) fatal |= t4_handle_intr(adap, &t7_mps_trc_intr_info2, 0, flags); } else fatal |= t4_handle_intr(adap, &mps_trc_intr_info, 0, flags); + fatal |= t4_handle_intr(adap, &mps_stat_intr_info, 0, flags); fatal |= t4_handle_intr(adap, &mps_stat_sram_intr_info, 0, flags); fatal |= t4_handle_intr(adap, &mps_stat_tx_intr_info, 0, flags); fatal |= t4_handle_intr(adap, &mps_stat_rx_intr_info, 0, flags); @@ -6087,7 +7327,6 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) t4_read_reg(adap, A_MPS_INT_CAUSE); /* flush */ return (fatal); - } /* @@ -6096,7 +7335,7 @@ static bool mps_intr_handler(struct adapter *adap, int arg, int flags) static bool mem_intr_handler(struct adapter *adap, int idx, int flags) { static const char name[4][5] = { "EDC0", "EDC1", "MC0", "MC1" }; - unsigned int count_reg, v; + unsigned int count_reg = 0, v; static const struct intr_details mem_intr_details[] = { { F_ECC_UE_INT_CAUSE, "Uncorrectable ECC data error(s)" }, { F_ECC_CE_INT_CAUSE, "Correctable ECC data error(s)" }, @@ -6104,10 +7343,10 @@ static bool mem_intr_handler(struct adapter *adap, int idx, int flags) { 0 } }; static const struct intr_details t7_mem_intr_details[] = { - { F_DDRPHY_INT_CAUSE, "DDRPHY" }, - { F_DDRCTL_INT_CAUSE, "DDRCTL" }, - { F_T7_ECC_CE_INT_CAUSE, "Correctable ECC data error(s)" }, + { F_DDRPHY_INT_CAUSE, "DDR PHY" }, + { F_DDRCTL_INT_CAUSE, "DDR Controller" }, { F_T7_ECC_UE_INT_CAUSE, "Uncorrectable ECC data error(s)" }, + { F_T7_ECC_CE_INT_CAUSE, "Correctable ECC data error(s)" }, { F_PERR_INT_CAUSE, "FIFO parity error" }, { 0 } }; @@ -6115,8 +7354,8 @@ static bool mem_intr_handler(struct adapter *adap, int idx, int flags) struct intr_info ii = { .name = &rname[0], .fatal = F_PERR_INT_CAUSE | F_ECC_UE_INT_CAUSE, + .flags = IHF_CLR_DELAYED, .details = mem_intr_details, - .flags = 0, .actions = NULL, }; bool fatal = false; @@ -6137,15 +7376,6 @@ static bool mem_intr_handler(struct adapter *adap, int idx, int flags) count_reg = EDC_T5_REG(A_EDC_H_ECC_STATUS, i); } fatal |= t4_handle_intr(adap, &ii, 0, flags); - if (chip_id(adap) > CHELSIO_T6) { - snprintf(rname, sizeof(rname), "EDC%u_PAR_CAUSE", i); - ii.cause_reg = EDC_T5_REG(A_EDC_H_PAR_CAUSE, i); - ii.enable_reg = EDC_T5_REG(A_EDC_H_PAR_ENABLE, i); - ii.fatal = 0xffffffff; - ii.details = NULL; - ii.flags = IHF_FATAL_IFF_ENABLED; - fatal |= t4_handle_intr(adap, &ii, 0, flags); - } break; case MEM_MC1: if (is_t4(adap) || is_t6(adap)) @@ -6167,52 +7397,30 @@ static bool mem_intr_handler(struct adapter *adap, int idx, int flags) ii.enable_reg = MC_T7_REG(A_T7_MC_P_INT_ENABLE, i); ii.fatal = F_PERR_INT_CAUSE | F_T7_ECC_UE_INT_CAUSE; ii.details = t7_mem_intr_details; - count_reg = MC_T7_REG(A_T7_MC_P_ECC_STATUS, i); } fatal |= t4_handle_intr(adap, &ii, 0, flags); - - snprintf(rname, sizeof(rname), "MC%u_PAR_CAUSE", i); - if (is_t4(adap)) { - ii.cause_reg = A_MC_PAR_CAUSE; - ii.enable_reg = A_MC_PAR_ENABLE; - } else if (chip_id(adap) < CHELSIO_T7) { - ii.cause_reg = MC_REG(A_MC_P_PAR_CAUSE, i); - ii.enable_reg = MC_REG(A_MC_P_PAR_ENABLE, i); - } else { - ii.cause_reg = MC_T7_REG(A_T7_MC_P_PAR_CAUSE, i); - ii.enable_reg = MC_T7_REG(A_T7_MC_P_PAR_ENABLE, i); - } - ii.fatal = 0xffffffff; - ii.details = NULL; - ii.flags = IHF_FATAL_IFF_ENABLED; - fatal |= t4_handle_intr(adap, &ii, 0, flags); - - if (chip_id(adap) > CHELSIO_T6) { - snprintf(rname, sizeof(rname), "MC%u_DDRCTL_INT_CAUSE", i); - ii.cause_reg = MC_T7_REG(A_MC_P_DDRCTL_INT_CAUSE, i); - ii.enable_reg = MC_T7_REG(A_MC_P_DDRCTL_INT_ENABLE, i); - fatal |= t4_handle_intr(adap, &ii, 0, flags); - } break; } - v = t4_read_reg(adap, count_reg); - if (v != 0) { - if (G_ECC_UECNT(v) != 0 && !(flags & IHF_NO_SHOW)) { - CH_ALERT(adap, - " %s: %u uncorrectable ECC data error(s)\n", - name[idx], G_ECC_UECNT(v)); - } - if (G_ECC_CECNT(v) != 0 && !(flags & IHF_NO_SHOW)) { - if (idx <= MEM_EDC1) - t4_edc_err_read(adap, idx); - CH_WARN_RATELIMIT(adap, - " %s: %u correctable ECC data error(s)\n", - name[idx], G_ECC_CECNT(v)); + if (count_reg != 0) { + v = t4_read_reg(adap, count_reg); + if (v != 0) { + if (G_ECC_UECNT(v) != 0 && !(flags & IHF_NO_SHOW)) { + CH_ALERT(adap, + " %s: %u uncorrectable ECC data error(s)\n", + name[idx], G_ECC_UECNT(v)); + } + if (G_ECC_CECNT(v) != 0 && !(flags & IHF_NO_SHOW)) { + if (idx <= MEM_EDC1) + t4_edc_err_read(adap, idx); + CH_WARN_RATELIMIT(adap, + " %s: %u correctable ECC data error(s)\n", + name[idx], G_ECC_CECNT(v)); + } + t4_write_reg(adap, count_reg, 0xffffffff); } - t4_write_reg(adap, count_reg, 0xffffffff); } - + clear_int_cause_reg(adap, &ii, flags); return (fatal); } @@ -6231,14 +7439,13 @@ static bool ma_wrap_status(struct adapter *adap, int arg, int flags) return (false); } - /* * MA interrupt handler. */ static bool ma_intr_handler(struct adapter *adap, int arg, int flags) { static const struct intr_action ma_intr_actions[] = { - { F_MEM_WRAP_INT_CAUSE, 0, ma_wrap_status }, + { F_MEM_WRAP_INT_CAUSE, -1, ma_wrap_status }, { 0 }, }; static const struct intr_info ma_intr_info = { @@ -6284,10 +7491,29 @@ static bool ma_intr_handler(struct adapter *adap, int arg, int flags) */ static bool smb_intr_handler(struct adapter *adap, int arg, int flags) { - static const struct intr_details smb_int_cause_fields[] = { - { F_MSTTXFIFOPARINT, "SMB master Tx FIFO parity error" }, - { F_MSTRXFIFOPARINT, "SMB master Rx FIFO parity error" }, - { F_SLVFIFOPARINT, "SMB slave FIFO parity error" }, + static const struct intr_details smb_int_cause_details[] = { + { F_MSTTXFIFOPARINT, "Master has Parity Error in Tx Fifo" }, + { F_MSTRXFIFOPARINT, "Master has Parity Error in Rx Fifo" }, + { F_SLVFIFOPARINT, "Slave has Parity Error in Fifo" }, + { F_SLVUNEXPBUSSTOPINT, "Slave get Unexpected BusStop" }, + { F_SLVUNEXPBUSSTARTINT, "Slave get Unexpected BusStart" }, + { F_SLVCOMMANDCODEINVINT, "Slave get Invalid Command Code" }, + { F_SLVBYTECNTERRINT, "Slave get Erroneous ByteCount value" }, + { F_SLVUNEXPACKMSTINT, "Slave get Unexpected Ack from Master" }, + { F_SLVUNEXPNACKMSTINT, "Slave get Unexpected Nack from Master" }, + { F_SLVNOBUSSTOPINT, "Slave did not get Bus Stop" }, + { F_SLVNOREPSTARTINT, "Slave has no Repeated Start" }, + { F_SLVRXADDRINT, "Slave has Address Error" }, + { F_SLVRXPECERRINT, "Slave has Pec Error" }, + { F_SLVPREPTOARPINT, "PL has invalid request" }, + { F_SLVTIMEOUTINT, "Slave has timed out" }, + { F_SLVERRINT, "Slave detected error during the current transfer" }, + { F_SLVDONEINT, "Slave has completed the current transaction" }, + { F_SLVRXRDYINT, "Slave has received bytes to be processed by uP" }, + { F_MSTTIMEOUTINT, "Master has timed out" }, + { F_MSTNACKINT, "Master has detected a NAck on the transfer" }, + { F_MSTLOSTARBINT, "Master has lost arbitration all the timeline" }, + { F_MSTDONEINT, "Master has completed the current transaction" }, { 0 } }; static const struct intr_info smb_int_cause = { @@ -6296,9 +7522,10 @@ static bool smb_intr_handler(struct adapter *adap, int arg, int flags) .enable_reg = A_SMB_INT_ENABLE, .fatal = F_SLVFIFOPARINT | F_MSTRXFIFOPARINT | F_MSTTXFIFOPARINT, .flags = 0, - .details = smb_int_cause_fields, + .details = smb_int_cause_details, .actions = NULL, }; + return (t4_handle_intr(adap, &smb_int_cause, 0, flags)); } @@ -6308,6 +7535,7 @@ static bool smb_intr_handler(struct adapter *adap, int arg, int flags) static bool ncsi_intr_handler(struct adapter *adap, int arg, int flags) { static const struct intr_details ncsi_int_cause_fields[] = { + { F_CIM2NC_PERR, " CIM to NC parity error" }, { F_CIM_DM_PRTY_ERR, "NC-SI CIM parity error" }, { F_MPS_DM_PRTY_ERR, "NC-SI MPS parity error" }, { F_TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error" }, @@ -6324,13 +7552,31 @@ static bool ncsi_intr_handler(struct adapter *adap, int arg, int flags) .details = ncsi_int_cause_fields, .actions = NULL, }; + static const struct intr_details ncsi_xgmac0_int_cause_details[] = { + { F_XAUIPCSDECERR, "RGMII PCS DEC Error" }, + { F_RGMIIRXFIFOOVERFLOW, "RGMII receive FIFO over flow" }, + { F_RGMIIRXFIFOUNDERFLOW, "RGMII receive FIFO under flow" }, + { F_RXPKTSIZEERROR, "Receive over size packet" }, + { F_WOLPATDETECTED, "WOL pattern detected" }, + { 0x000e0000, "Tx FIFO parity error" }, + { 0x0001c000, "Rx FIFO parity error" }, + { F_TXFIFO_UNDERRUN, "Tx FIFO underrun" }, + { F_RXFIFO_OVERFLOW, "Rx FIFO overflow" }, + { 0x00000f00, "XAUI SERDES BIST error" }, + { 0x000000f0, "XAUI SERDES receive low signal change" }, + { F_XAUIPCSCTCERR, "XAUI PCS CTC FIFO error" }, + { F_XAUIPCSALIGNCHANGE, "XAUI PCS alignment change" }, + { F_RGMIILINKSTSCHANGE, "RGMII link status change" }, + { F_XGM_INT, "XGM Core embedded interrupt (2nd level)" }, + { 0 } + }; static const struct intr_info ncsi_xgmac0_int_cause = { .name = "NCSI_XGMAC0_INT_CAUSE", .cause_reg = A_NCSI_XGMAC0_INT_CAUSE, .enable_reg = A_NCSI_XGMAC0_INT_ENABLE, .fatal = 0, .flags = 0, - .details = NULL, + .details = ncsi_xgmac0_int_cause_details, .actions = NULL, }; bool fatal = false; @@ -6346,32 +7592,71 @@ static bool ncsi_intr_handler(struct adapter *adap, int arg, int flags) */ static bool mac_intr_handler(struct adapter *adap, int port, int flags) { + static const struct intr_details mac_int_cause_cmn_details[] = { + { 0x3fffc0, "HSS PLL lock error " }, + { F_FLOCK_ASSERTED, "frequency lock coming out of DPLL sub-block is asserted" }, + { F_FLOCK_LOST, "frequency lock coming out of DPLL sub-blocki is lost." }, + { F_PHASE_LOCK_ASSERTED, "PHASE LOCK from DPLL sub-block is asserted" }, + { F_PHASE_LOCK_LOST, "PHASE LOCK from DPLL sub-block is lost." }, + { F_LOCK_ASSERTED, "Lock from frac_n PLL inside t7_clk module is asserted" }, + { F_LOCK_LOST, "Lock from frac_n PLL inside t7_clk module is lost " }, + { 0 } + }; static const struct intr_info mac_int_cause_cmn = { .name = "MAC_INT_CAUSE_CMN", .cause_reg = A_MAC_INT_CAUSE_CMN, .enable_reg = A_MAC_INT_EN_CMN, .fatal = 0, .flags = 0, - .details = NULL, + .details = mac_int_cause_cmn_details, .actions = NULL, }; + static const struct intr_details mac_perr_int_cause_mtip_details[] = { + { F_PERR_MAC0_TX, "MTIP MAC TX memory for MAC 0 (the 200G MAC for port 0)" }, + { F_PERR_MAC1_TX, "MTIP MAC TX memory for MAC 1 (the 200G MAC for port 1)" }, + { F_PERR_MAC2_TX, "MTIP MAC TX memory for MAC 2 (the 10-100G MAC for port 0)" }, + { F_PERR_MAC3_TX, "MTIP MAC TX memory for MAC 3 (the 10-100G MAC for port 1)" }, + { F_PERR_MAC4_TX, "MTIP MAC TX memory for MAC 4 (the 10-100G MAC for port 2)" }, + { F_PERR_MAC5_TX, "MTIP MAC TX memory for MAC 5 (the 10-100G MAC for port 3)" }, + { F_PERR_MAC0_RX, "MTIP MAC RX memory for MAC 0 (the 200G MAC for port 0)" }, + { F_PERR_MAC1_RX, "MTIP MAC RX memory for MAC 1 (the 200G MAC for port 1)" }, + { F_PERR_MAC2_RX, "MTIP MAC RX memory for MAC 2 (the 10-100G MAC for port 0)" }, + { F_PERR_MAC3_RX, "MTIP MAC RX memory for MAC 3 (the 10-100G MAC for port 1)" }, + { F_PERR_MAC4_RX, "MTIP MAC RX memory for MAC 4 (the 10-100G MAC for port 2)" }, + { F_PERR_MAC5_RX, "MTIP MAC RX memory for MAC 5 (the 10-100G MAC for port 3)" }, + { F_PERR_MAC_STAT_RX, "MTIP MAC RX statistics memory (1 for all 4 10-100G MACs)" }, + { F_PERR_MAC_STAT_TX, "MTIP MAC TX statistics memory (1 for all 4 10-100G MACs)" }, + { F_PERR_MAC_STAT_CAP, "MTIP MAC stat capture memory (1 for all 4 100G MACs)" }, + { 0 } + }; static const struct intr_info mac_perr_cause_mtip = { .name = "MAC_PERR_INT_CAUSE_MTIP", .cause_reg = A_MAC_PERR_INT_CAUSE_MTIP, .enable_reg = A_MAC_PERR_INT_EN_MTIP, .fatal = 0xffffffff, .flags = IHF_FATAL_IFF_ENABLED | IHF_IGNORE_IF_DISABLED, - .details = NULL, + .details = mac_perr_int_cause_mtip_details, .actions = NULL, }; - static const struct intr_info mac_cerr_cause_mtip = { - .name = "MAC_CERR_INT_CAUSE_MTIP", - .cause_reg = A_MAC_CERR_INT_CAUSE_MTIP, - .enable_reg = A_MAC_CERR_INT_EN_MTIP, - .fatal = 0, - .flags = 0, - .details = NULL, - .actions = NULL, + static const struct intr_details ios_intr_cause_quad0_details[] = { + { F_Q0_MAILBOX_INT_ASSERT, "Etopus Quad0 Mailbox interrupt cause" }, + { 0x00f00000, "Etopus Quad0 training failure" }, + { 0x000f0000, "Etopus Quad0 training complete" }, + { 0x0000f000, "Etopus Quad0 AN TX interrupt" }, + { 0x00000f00, "Etopus Quad0 signal detect assertion" }, + { 0x000000f0, "Etopus Quad0 CDR LOL assertion" }, + { 0x0000000f, "Etopus Quad0 LOS signal assertion" }, + { 0 } + }; + static const struct intr_details ios_intr_cause_quad1_details[] = { + { F_Q1_MAILBOX_INT_ASSERT, "Etopus Quad1 Mailbox interrupt cause" }, + { 0x00f00000, "Etopus Quad1 training failure" }, + { 0x000f0000, "Etopus Quad1 training complete" }, + { 0x0000f000, "Etopus Quad1 AN TX interrupt" }, + { 0x00000f00, "Etopus Quad1 signal detect assertion" }, + { 0x000000f0, "Etopus Quad1 CDR LOL assertion" }, + { 0x0000000f, "Etopus Quad1 LOS signal assertion" }, + { 0 } }; static const struct intr_info mac_ios_int_cause_quad0 = { .name = "MAC_IOS_INTR_CAUSE_QUAD0", @@ -6379,7 +7664,7 @@ static bool mac_intr_handler(struct adapter *adap, int port, int flags) .enable_reg = A_MAC_IOS_INTR_EN_QUAD0, .fatal = 0, .flags = 0, - .details = NULL, + .details = ios_intr_cause_quad0_details, .actions = NULL, }; static const struct intr_info mac_ios_int_cause_quad1 = { @@ -6388,7 +7673,7 @@ static bool mac_intr_handler(struct adapter *adap, int port, int flags) .enable_reg = A_MAC_IOS_INTR_EN_QUAD1, .fatal = 0, .flags = 0, - .details = NULL, + .details = ios_intr_cause_quad1_details, .actions = NULL, }; static const struct intr_details mac_intr_details[] = { @@ -6396,6 +7681,33 @@ static bool mac_intr_handler(struct adapter *adap, int port, int flags) { F_RXFIFO_PRTY_ERR, "MAC Rx FIFO parity error" }, { 0 } }; + static const struct intr_details t7_mac_int_cause_details[] = { + { F_MAC2MPS_PERR_CAUSE, "MPS2MAC Data parity error per port" }, + { F_MAC_PPS_INT_CAUSE, "One second interrupt based on PTP timer" }, + { F_MAC_TX_TS_AVAIL_INT_CAUSE, + "Time stamp is available for the last IEEE 1588 event frame" }, + { F_MAC_PATDETWAKE_INT_CAUSE, "Wake up pattern match packet received" }, + { F_MAC_MAGIC_WAKE_INT_CAUSE, "Magic packet received" }, + { F_MAC_SIGDETCHG_INT_CAUSE, "Signal Detect Change" }, + { F_MAC_PCS_LINK_GOOD_CAUSE, "PCS link good (xaui pcsr or 1g)" }, + { F_MAC_PCS_LINK_FAIL_CAUSE, "PCS Failure (xaui pcsr or 1g)" }, + { F_RXFIFOOVERFLOW, "RX Fifo Over flow error" }, + { F_MAC_REM_FAULT_INT_CAUSE, "Remote fault received by XGMAC" }, + { F_MAC_LOC_FAULT_INT_CAUSE, "Local fault received by XGMAC" }, + { F_MAC_LINK_DOWN_INT_CAUSE, "Link is down" }, + { F_MAC_LINK_UP_INT_CAUSE, "Link is up" }, + { F_MAC_AN_DONE_INT_CAUSE, "Autonegotiation complete" }, + { F_MAC_AN_PGRD_INT_CAUSE, "An page received" }, + { F_MAC_TXFIFO_ERR_INT_CAUSE, "Tx FIFO parity error" }, + { F_MAC_RXFIFO_ERR_INT_CAUSE, "Rx FIFO parity error" }, + { 0 } + }; + static const struct intr_details mac_perr_int_cause_details[] = { + { F_T6_PERR_PKT_RAM, "WoL packet data memory" }, + { F_T6_PERR_MASK_RAM, "WoL mask memory" }, + { F_T6_PERR_CRC_RAM, "WoL CRC memory" }, + { 0 } + }; char name[32]; struct intr_info ii; bool fatal = false; @@ -6428,7 +7740,7 @@ static bool mac_intr_handler(struct adapter *adap, int port, int flags) ii.enable_reg = T7_PORT_REG(port, A_T7_MAC_PORT_INT_EN); ii.fatal = 0xffffffff; ii.flags = IHF_FATAL_IFF_ENABLED; - ii.details = NULL; + ii.details = t7_mac_int_cause_details; ii.actions = NULL; } fatal |= t4_handle_intr(adap, &ii, 0, flags); @@ -6443,7 +7755,7 @@ static bool mac_intr_handler(struct adapter *adap, int port, int flags) ii.enable_reg = T7_PORT_REG(port, A_T7_MAC_PORT_PERR_INT_EN); ii.fatal = 0xffffffff; ii.flags = IHF_FATAL_IFF_ENABLED; - ii.details = NULL; + ii.details = mac_perr_int_cause_details; ii.actions = NULL; } else { ii.name = &name[0]; @@ -6484,7 +7796,6 @@ static bool mac_intr_handler(struct adapter *adap, int port, int flags) MPASS(chip_id(adap) >= CHELSIO_T7); fatal |= t4_handle_intr(adap, &mac_int_cause_cmn, 0, flags); fatal |= t4_handle_intr(adap, &mac_perr_cause_mtip, 0, flags); - fatal |= t4_handle_intr(adap, &mac_cerr_cause_mtip, 0, flags); fatal |= t4_handle_intr(adap, &mac_ios_int_cause_quad0, 0, flags); fatal |= t4_handle_intr(adap, &mac_ios_int_cause_quad1, 0, flags); @@ -6506,28 +7817,40 @@ static bool pl_timeout_status(struct adapter *adap, int arg, int flags) static bool plpl_intr_handler(struct adapter *adap, int arg, int flags) { static const struct intr_details plpl_int_cause_fields[] = { + { F_FATALPERR, "Fatal parity error" }, + { F_PERRVFID, "VFID_MAP parity error" }, + { 0 } + }; + static const struct intr_details t5_plpl_int_cause_fields[] = { { F_PL_BUSPERR, "Bus parity error" }, { F_FATALPERR, "Fatal parity error" }, { F_INVALIDACCESS, "Global reserved memory access" }, { F_TIMEOUT, "Bus timeout" }, { F_PLERR, "Module reserved access" }, - { F_PERRVFID, "VFID_MAP parity error" }, { 0 } }; static const struct intr_action plpl_int_cause_actions[] = { { F_TIMEOUT, -1, pl_timeout_status }, { 0 }, }; - static const struct intr_info plpl_int_cause = { + struct intr_info plpl_int_cause = { .name = "PL_PL_INT_CAUSE", .cause_reg = A_PL_PL_INT_CAUSE, .enable_reg = A_PL_PL_INT_ENABLE, - .fatal = F_FATALPERR | F_PERRVFID, - .flags = IHF_FATAL_IFF_ENABLED | IHF_IGNORE_IF_DISABLED, - .details = plpl_int_cause_fields, - .actions = plpl_int_cause_actions, + .fatal = F_FATALPERR, + .flags = IHF_FATAL_IFF_ENABLED, + .details = NULL, + .actions = NULL, }; + if (is_t4(adap)) { + plpl_int_cause.fatal |= F_PERRVFID; + plpl_int_cause.details = plpl_int_cause_fields; + } else { + plpl_int_cause.fatal |= F_INVALIDACCESS; + plpl_int_cause.details = t5_plpl_int_cause_fields; + plpl_int_cause.actions = plpl_int_cause_actions; + } return (t4_handle_intr(adap, &plpl_int_cause, 0, flags)); } @@ -6587,7 +7910,7 @@ static bool hma_intr_handler(struct adapter *adap, int idx, int flags) { F_RTF_INT_CAUSE, "Region translation fault" }, { F_PCIEMST_INT_CAUSE, "PCIe master access error" }, { F_MAMST_INT_CAUSE, "MA master access error" }, - { 1, "FIFO parity error" }, + { F_PERR_INT_CAUSE, "FIFO parity error" }, { 0 } }; static const struct intr_info hma_int_cause = { @@ -6682,15 +8005,6 @@ static bool gcache_intr_handler(struct adapter *adap, int idx, int flags) { F_ILLADDRACCESS0_INT_CAUSE, "GC0 illegal address access" }, { 0 } }; - static const struct intr_info gcache_perr_cause = { - .name = "GCACHE_PAR_CAUSE", - .cause_reg = A_GCACHE_PAR_CAUSE, - .enable_reg = A_GCACHE_PAR_ENABLE, - .fatal = 0xffffffff, - .flags = IHF_FATAL_IFF_ENABLED, - .details = NULL, - .actions = NULL, - }; static const struct intr_info gcache_int_cause = { .name = "GCACHE_INT_CAUSE", .cause_reg = A_GCACHE_INT_CAUSE, @@ -6700,12 +8014,7 @@ static bool gcache_intr_handler(struct adapter *adap, int idx, int flags) .details = gcache_int_cause_fields, .actions = NULL, }; - bool fatal = false; - - fatal |= t4_handle_intr(adap, &gcache_int_cause, 0, flags); - fatal |= t4_handle_intr(adap, &gcache_perr_cause, 0, flags); - - return (fatal); + return (t4_handle_intr(adap, &gcache_int_cause, 0, flags)); } /* @@ -6713,67 +8022,218 @@ static bool gcache_intr_handler(struct adapter *adap, int idx, int flags) */ static bool arm_intr_handler(struct adapter *adap, int idx, int flags) { + static const struct intr_details arm_perr_int_cause0_details[] = { + { F_INIC_WRDATA_FIFO_PERR, "INT CAUSE for INIC Write Data Fifo Parity Error" }, + { F_INIC_RDATA_FIFO_PERR, "INT CAUSE for INIC Read Data Fifo Parity Error" }, + { F_MSI_MEM_PERR, "INT CAUSE for MSI Memory Parity Error" }, + { 0x18000000, "INT CAUSE for ARM Doorbell SRAM Parity Error" }, + { F_EMMC_FIFOPARINT, "INT CAUSE for EMMC Fifo Parity Interrupt" }, + { F_ICB_RAM_PERR, "INT CAUSE for ICB SRAM Parity Error" }, + { F_MESS2AXI4_WRFIFO_PERR, "INT CAUSE for Message2AXI4 Write FIFO Parity Error" }, + { F_RC_WFIFO_OUTPERR, "INT CAUSE for AXI2RC Write FIFO Parity Error" }, + { 0x00600000, "INT CAUSE for AXI2RC SRAM Parity Error" }, + { F_MSI_FIFO_PAR_ERR, "INT CAUSE for APB2MSI FIFO Parity Error" }, + { F_INIC2MA_INTFPERR, "INT CAUSE for INIC to MA Interface Parity Error" }, + { F_RDATAFIFO0_PERR, "INT CAUSE for AXI2MA M0 Read Data Fifo Parity Error" }, + { F_RDATAFIFO1_PERR, "INT CAUSE for AXI2MA M1 Read Data Fifo Parity Error" }, + { F_WRDATAFIFO0_PERR, "INT CAUSE for AXI2MA M0 Write Data Fifo Parity Error" }, + { F_WRDATAFIFO1_PERR, "INT CAUSE for AXI2MA M1 Write Data Fifo Parity Error" }, + { F_WR512DATAFIFO0_PERR, + "INT CAUSE for AXI2MA M0 Write Data 512b Fifo Parity Error" }, + { F_WR512DATAFIFO1_PERR, + "INT CAUSE for AXI2MA M1 Write Data 512b Fifo Parity Error" }, + { F_ROBUFF_PARERR3, "INT CAUSE for Reorder Buffer Parity Error" }, + { F_ROBUFF_PARERR2, "INT CAUSE for Reorder Buffer Parity Error" }, + { F_ROBUFF_PARERR1, "INT CAUSE for Reorder Buffer Parity Error" }, + { F_ROBUFF_PARERR0, "INT CAUSE for Reorder Buffer Parity Error" }, + { F_MA2AXI_REQDATAPARERR, "INT CAUSE for MA2AXI Request Data Parity Error" }, + { F_MA2AXI_REQCTLPARERR, "INT CAUSE for MA2AXI Request Control Parity Error" }, + { F_MA_RSPPERR, "INT CAUSE for MA Response Parity Error" }, + { F_PCIE2MA_REQCTLPARERR, "INT CAUSE for PCIe to MA Control Parity Error" }, + { F_PCIE2MA_REQDATAPARERR, "INT CAUSE for PCIe to MA Data Parity Error" }, + { F_INIC2MA_REQCTLPARERR, "INT CAUSE for INIC to MA Control Parity Error" }, + { F_INIC2MA_REQDATAPARERR, "INT CAUSE for INIC to MA Data Parity Error" }, + { F_MA_RSPUE, "INT CAUSE for MA Response Uncorrectable Error" }, + { F_APB2PL_RSPDATAPERR, "INT CAUSE for APB2PL Response Data Parity Error" }, + { 0 } + }; static const struct intr_info arm_perr_cause0 = { .name = "ARM_PERR_INT_CAUSE0", .cause_reg = A_ARM_PERR_INT_CAUSE0, .enable_reg = A_ARM_PERR_INT_ENB0, .fatal = 0xffffffff, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_perr_int_cause0_details, .actions = NULL, }; + static const struct intr_details arm_perr_int_cause1_details[] = { + { F_ARWFIFO0_PERR, "AXI2MA M0 Read-Write FIFO Parity Error" }, + { F_ARWFIFO1_PERR, "AXI2MA M1 Read-Write FIFO Parity Error" }, + { F_ARWIDFIFO0_PERR, "AXI2MA M0 Read-Write ID FIFO Parity Error" }, + { F_ARWIDFIFO1_PERR, "AXI2MA M1 Read-Write ID FIFO Parity Error" }, + { F_ARIDFIFO0_PERR, "AXI2MA M0 Read FIFO Parity Error" }, + { F_ARIDFIFO1_PERR, "AXI2MA M1 Read FIFO Parity Error" }, + { F_RRSPADDR_FIFO0_PERR, "AXI2MA M0 Read Response Address FIFO Parity Error" }, + { F_RRSPADDR_FIFO1_PERR, "AXI2MA M1 Read Response Address FIFO Parity Error" }, + { F_WRSTRB_FIFO0_PERR, "AXI2MA M0 Write Strobe FIFO Parity Error" }, + { F_WRSTRB_FIFO1_PERR, "AXI2MA M1 Write Strobe FIFO Parity Error" }, + { F_MA2AXI_RSPDATAPARERR, "MA2AXI Response FIFO Parity Error" }, + { F_MA2AXI_DATA_PAR_ERR, "MA2AXI Write Data FIFO Parity Error" }, + { F_MA2AXI_WR_ORD_FIFO_PARERR, "MA2AXI Ordered Write Data FIFO Parity Error" }, + { F_NVME_DB_EMU_TRACKER_FIFO_PERR, "NVMe DB Emulation Tracker FIFO Parity Error" }, + { F_NVME_DB_EMU_QUEUE_AW_ADDR_FIFO_PERR, + "NVMe DB Emulation Queue AW Addr Parity Error" }, + { F_NVME_DB_EMU_INTERRUPT_OFFSET_FIFO_PERR, + "NVMe DB Emulation Interrupt Offset FIFO Parity Error" }, + { F_NVME_DB_EMU_ID_FIFO0_PERR, "NVMe DB Emulation ID FIFO0 Parity Error" }, + { F_NVME_DB_EMU_ID_FIFO1_PERR, "NVMe DB Emulation ID FIFO1 Parity Error" }, + { F_RC_ARWFIFO_PERR, "AXI2RC Read-Write FIFO Parity Error" }, + { F_RC_ARIDBURSTADDRFIFO_PERR, + "AXI2RC Read ID, Burst and Address FIFO Parity Error" }, + { F_RC_CFG_FIFO_PERR, "AXI2RC Config FIFO Parity Error" }, + { F_RC_RSPFIFO_PERR, "AXI2RC Response Parity Error" }, + { F_INIC_ARIDFIFO_PERR, "CCI2INIC Read ID FIFO Parity Error" }, + { F_INIC_ARWFIFO_PERR, "CCI2INIC Read-Write FIFO ontrol Parity Error" }, + { F_AXI2MA_128_RD_ADDR_SIZE_FIFO_PERR, + "AXI2MA(CCI2INIC) Read Address Size FIFO Parity Error" }, + { F_AXI2RC_128_RD_ADDR_SIZE_FIFO_PERR, + "AXI2RC Read Address Size FIFO Parity Error" }, + { F_ARM_MA_512B_RD_ADDR_SIZE_FIFO0_PERR, + "ARM_MA_512b Read Address Size FIFO0 Parity Error" }, + { F_ARM_MA_512B_RD_ADDR_SIZE_FIFO1_PERR, + "ARM_MA_512b Read Address Size FIFO1 Parity Error" }, + { F_ARM_MA_512B_ARB_FIFO_PERR, "ARM_MA_512b Arbiter FIFO Parity Error" }, + { F_PCIE_INIC_MA_ARB_FIFO_PERR, "PCIe-INIC Arbiter FIFO Parity Error" }, + { F_PCIE_INIC_ARB_RSPPERR, "PCIe-INIC Arbiter Response Parity Error" }, + { F_ITE_CACHE_PERR, "GIC500 ITE Cache SRAM Parity Error" }, + { 0 } + }; static const struct intr_info arm_perr_cause1 = { .name = "ARM_PERR_INT_CAUSE1", .cause_reg = A_ARM_PERR_INT_CAUSE1, .enable_reg = A_ARM_PERR_INT_ENB1, .fatal = 0xffffffff, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_perr_int_cause1_details, .actions = NULL, }; + static const struct intr_details arm_perr_int_cause2_details[] = { + { F_INIC_WSTRB_FIFO_PERR, "AXI2MA_128 INIC Write Strobe FIFO Parity Error" }, + { F_INIC_BID_FIFO_PERR, "AXI2MA_128 INIC bID FIFO Parity Error" }, + { F_CC_SRAM_PKA_PERR, "CryptoCell ram_pka_wrapper FIFO Parity Error" }, + { F_CC_SRAM_SEC_PERR, "CryptoCell sec_sram_wrapper FIFO Parity Error" }, + { F_MESS2AXI4_PARERR, "Message2AXI4 IBQ I/P Interface Parity Error" }, + { F_CCI2INIC_INTF_PARERR, "CCI2INIC Response Interface Parity Error" }, + { 0 } + }; static const struct intr_info arm_perr_cause2 = { .name = "ARM_PERR_INT_CAUSE2", .cause_reg = A_ARM_PERR_INT_CAUSE2, .enable_reg = A_ARM_PERR_INT_ENB2, .fatal = 0xffffffff, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_perr_int_cause2_details, .actions = NULL, }; + static const struct intr_details arm_cerr_int_cause0_details[] = { + { F_WRDATA_FIFO0_CERR, "AXI2MA M0 Write Data FIFO Correctable Error" }, + { F_WRDATA_FIFO1_CERR, "AXI2MA M1 Write Data FIFO Correctable Error" }, + { F_WR512DATAFIFO0_CERR, "AXI2MA M0 Write Data 512b FIFO Correctable Error" }, + { F_WR512DATAFIFO1_CERR, "AXI2MA M1 Write Data 512b FIFO Correctable Error" }, + { F_RDATAFIFO0_CERR, "AXI2MA M0 Read Data FIFO Correctable Error" }, + { F_RDATAFIFO1_CERR, "AXI2MA M1 Read Data FIFO Correctable Error" }, + { F_ROBUFF_CORERR0, "Reorder Buffer Correctable Error" }, + { F_ROBUFF_CORERR1, "Reorder Buffer Correctable Error" }, + { F_ROBUFF_CORERR2, "Reorder Buffer Correctable Error" }, + { F_ROBUFF_CORERR3, "Reorder Buffer Correctable Error" }, + { F_MA2AXI_RSPDATACORERR, "MA2AXI Response FIFO Correctable Error" }, + { 0x00180000, "AXI2RC SRAM Correctable Error" }, + { F_RC_WFIFO_OUTCERR, "AXI2RC Write FIFO Correctable Error" }, + { F_RC_RSPFIFO_CERR, "AXI2RC Response Correctable Error" }, + { F_MSI_MEM_CERR, "MSI Memory FIFO Correctable Error" }, + { F_INIC_WRDATA_FIFO_CERR, "INIC Write Data FIFO Correctable Error" }, + { F_INIC_RDATAFIFO_CERR, "INIC Read Data FIFO Correctable Error" }, + { 0x00003000, "ARM Doorbell SRAM Correctable Error" }, + { F_ICB_RAM_CERR, "ICB SRAM Parity Error" }, + { F_CC_SRAM_PKA_CERR, "CryptoCell ram_pka_wrapper FIFO Correctable Error" }, + { F_CC_SRAM_SEC_CERR, "CryptoCell sec_sram_wrapper FIFO Correctable Error" }, + { 0 } + }; static const struct intr_info arm_cerr_cause0 = { - .name = "ARM_CERR_INT_CAUSE", + .name = "ARM_CERR_INT_CAUSE0", .cause_reg = A_ARM_CERR_INT_CAUSE0, .enable_reg = A_ARM_CERR_INT_ENB0, .fatal = 0, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_cerr_int_cause0_details, .actions = NULL, }; + static const struct intr_details arm_err_int_cause0_details[] = { + { F_STRB0_ERROR, "Strobe Error from AXI2MA 0" }, + { F_STRB1_ERROR, "Strobe Error from AXI2MA 1" }, + { F_PCIE_INIC_MA_ARB_INV_RSP_TAG, "Invalid Response Tag for PCIE-INIc MA ARB" }, + { F_ERROR0_NOCMD_DATA, "AXI2MA 0 No Command Data Error" }, + { F_ERROR1_NOCMD_DATA, "AXI2MA 1 No Command Data Error" }, + { F_INIC_STRB_ERROR, "AXI2MA_128b INIC Strobe Error" }, + { 0 } + }; static const struct intr_info arm_err_cause0 = { - .name = "ARM_ERR_INT_CAUSE", + .name = "ARM_ERR_INT_CAUSE0", .cause_reg = A_ARM_ERR_INT_CAUSE0, .enable_reg = A_ARM_ERR_INT_ENB0, .fatal = 0, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_err_int_cause0_details, .actions = NULL, }; + + static const struct intr_details arm_peripheral_int_cause_details[] = { + { F_TIMER_INT, "TIMER_INT" }, + { F_NVME_INT, "NVME_INT" }, + { F_EMMC_WAKEUP_INT, "EMMC_WAKEUP_INT" }, + { F_EMMC_INT, "EMMC_INT" }, + { F_USB_MC_INT, "USB_MC_INT" }, + { F_USB_DMA_INT, "USB_DMA_INT" }, + { 0 } + }; static const struct intr_info arm_periph_cause = { .name = "ARM_PERIPHERAL_INT_CAUSE", .cause_reg = A_ARM_PERIPHERAL_INT_CAUSE, .enable_reg = A_ARM_PERIPHERAL_INT_ENB, .fatal = 0, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_peripheral_int_cause_details, .actions = NULL, }; + static const struct intr_details arm_arm_uart_int_cause_details[] = { + { F_RX_FIFO_NOT_EMPTY, "intcause for uart rx fifo" }, + { F_TX_FIFO_EMPTY, "intcause for uart tx fifo" }, + { 0 } + }; + static const struct intr_info arm_uart_cause = { + .name = "ARM_ARM_UART_INT_CAUSE", + .cause_reg = A_ARM_ARM_UART_INT_CAUSE, + .enable_reg = A_ARM_ARM_UART_INT_EN, + .fatal = 0, + .flags = IHF_FATAL_IFF_ENABLED, + .details = arm_arm_uart_int_cause_details, + .actions = NULL, + }; + static const struct intr_details arm_nvme_db_emu_int_cause_details[] = { + { F_INVALID_BRESP, "Invalid CCI Write Response" }, + { F_DATA_LEN_OF, + "Incorrect Write Request to be written to incorrect Devices/Regions" }, + { F_INVALID_EMU_ADDR, "Invalid Emulation Address Range Configuration" }, + { F_INVALID_AXI_ADDR_CFG, "Invalid AXI Address Configuration" }, + { 0 } + }; static const struct intr_info arm_nvme_db_emu_cause = { .name = "ARM_NVME_DB_EMU_INT_CAUSE", .cause_reg = A_ARM_NVME_DB_EMU_INT_CAUSE, .enable_reg = A_ARM_NVME_DB_EMU_INT_ENABLE, .fatal = 0, .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, - .details = NULL, + .details = arm_nvme_db_emu_int_cause_details, .actions = NULL, }; bool fatal = false; @@ -6785,12 +8245,13 @@ static bool arm_intr_handler(struct adapter *adap, int idx, int flags) fatal |= t4_handle_intr(adap, &arm_err_cause0, 0, flags); fatal |= t4_handle_intr(adap, &arm_periph_cause, 0, flags); fatal |= t4_handle_intr(adap, &arm_nvme_db_emu_cause, 0, flags); + fatal |= t4_handle_intr(adap, &arm_uart_cause, 0, flags); return (fatal); } static inline uint32_t -get_perr_ucause(struct adapter *sc, const struct intr_info *ii) +get_ucause(struct adapter *sc, const struct intr_info *ii) { uint32_t cause; @@ -6977,7 +8438,8 @@ bool t4_slow_intr_handler(struct adapter *adap, int flags) .cause_reg = A_PL_PERR_CAUSE, .enable_reg = A_PL_PERR_ENABLE, .fatal = 0xffffffff, - .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, + .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED | + IHF_CLR_DELAYED, .details = pl_int_cause_fields, .actions = NULL, }; @@ -7117,7 +8579,8 @@ bool t4_slow_intr_handler(struct adapter *adap, int flags) .cause_reg = A_PL_PERR_CAUSE, .enable_reg = A_PL_PERR_ENABLE, .fatal = 0xffffffff, - .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED, + .flags = IHF_IGNORE_IF_DISABLED | IHF_FATAL_IFF_ENABLED | + IHF_CLR_DELAYED, .details = t7_pl_perr_cause_fields, .actions = NULL, }; @@ -7125,23 +8588,19 @@ bool t4_slow_intr_handler(struct adapter *adap, int flags) uint32_t perr; if (chip_id(adap) < CHELSIO_T7) { - perr = get_perr_ucause(adap, &pl_perr_cause); - fatal |= t4_handle_intr(adap, &pl_perr_cause, 0, - flags & ~(IHF_CLR_ALL_SET | IHF_CLR_ALL_UNIGNORED)); + perr = get_ucause(adap, &pl_perr_cause); + fatal |= t4_handle_intr(adap, &pl_perr_cause, 0, flags); fatal |= t4_handle_intr(adap, &pl_int_cause, t4_perr_to_ic(adap, perr), flags); - t4_write_reg(adap, pl_perr_cause.cause_reg, perr); - (void)t4_read_reg(adap, pl_perr_cause.cause_reg); + clear_int_cause_reg(adap, &pl_perr_cause, flags); } else { - perr = get_perr_ucause(adap, &t7_pl_perr_cause); - fatal |= t4_handle_intr(adap, &t7_pl_perr_cause, 0, - flags & ~(IHF_CLR_ALL_SET | IHF_CLR_ALL_UNIGNORED)); + perr = get_ucause(adap, &t7_pl_perr_cause); + fatal |= t4_handle_intr(adap, &t7_pl_perr_cause, 0, flags); fatal |= t4_handle_intr(adap, &t7_pl_int_cause, t7_perr_to_ic1(perr), flags); fatal |= t4_handle_intr(adap, &t7_pl_int_cause2, t7_perr_to_ic2(perr), flags); - t4_write_reg(adap, t7_pl_perr_cause.cause_reg, perr); - (void)t4_read_reg(adap, t7_pl_perr_cause.cause_reg); + clear_int_cause_reg(adap, &t7_pl_perr_cause, flags); } return (fatal); } diff --git a/sys/dev/ffec/if_ffec.c b/sys/dev/ffec/if_ffec.c index 17fab283fc81..cf171a854406 100644 --- a/sys/dev/ffec/if_ffec.c +++ b/sys/dev/ffec/if_ffec.c @@ -850,7 +850,7 @@ ffec_rxfinish_onebuf(struct ffec_softc *sc, int len) * biggest header is, instead of the whole 1530ish-byte frame. */ if (sc->fecflags & FECFLAG_RACC) { - m->m_data = mtod(m, uint8_t *) + 2; + m_adj(m, 2); } else { src = mtod(m, uint8_t*); dst = src - ETHER_ALIGN; diff --git a/sys/dev/hwpmc/hwpmc_ibs.c b/sys/dev/hwpmc/hwpmc_ibs.c index 8cfe7b2df145..ae14f2ccb14c 100644 --- a/sys/dev/hwpmc/hwpmc_ibs.c +++ b/sys/dev/hwpmc/hwpmc_ibs.c @@ -510,6 +510,9 @@ pmc_ibs_intr(struct trapframe *tf) int retval, cpu; uint64_t config; + if (ibs_pcpu == NULL) + return (0); + cpu = curcpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ibs,%d] out of range CPU %d", __LINE__, cpu)); diff --git a/sys/dev/sound/macio/onyx.c b/sys/dev/sound/macio/onyx.c index f4f825a705cc..5ba22dd7c495 100644 --- a/sys/dev/sound/macio/onyx.c +++ b/sys/dev/sound/macio/onyx.c @@ -268,38 +268,21 @@ static int onyx_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct onyx_softc *sc; - struct mtx *mixer_lock; - int locked; uint8_t l, r; sc = device_get_softc(mix_getdevinfo(m)); - mixer_lock = mixer_get_lock(m); - locked = mtx_owned(mixer_lock); switch (dev) { case SOUND_MIXER_VOLUME: - - /* - * We need to unlock the mixer lock because iicbus_transfer() - * may sleep. The mixer lock itself is unnecessary here - * because it is meant to serialize hardware access, which - * is taken care of by the I2C layer, so this is safe. - */ if (left > 100 || right > 100) return (0); l = left + 128; r = right + 128; - if (locked) - mtx_unlock(mixer_lock); - onyx_write(sc, PCM3052_REG_LEFT_ATTN, l); onyx_write(sc, PCM3052_REG_RIGHT_ATTN, r); - if (locked) - mtx_lock(mixer_lock); - return (left | (right << 8)); } diff --git a/sys/dev/sound/macio/snapper.c b/sys/dev/sound/macio/snapper.c index f14009f447a8..ed83990d563b 100644 --- a/sys/dev/sound/macio/snapper.c +++ b/sys/dev/sound/macio/snapper.c @@ -436,14 +436,10 @@ static int snapper_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct snapper_softc *sc; - struct mtx *mixer_lock; - int locked; u_int l, r; u_char reg[6]; sc = device_get_softc(mix_getdevinfo(m)); - mixer_lock = mixer_get_lock(m); - locked = mtx_owned(mixer_lock); if (left > 100 || right > 100) return (0); @@ -460,21 +456,8 @@ snapper_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) reg[4] = (r & 0x00ff00) >> 8; reg[5] = r & 0x0000ff; - /* - * We need to unlock the mixer lock because iicbus_transfer() - * may sleep. The mixer lock itself is unnecessary here - * because it is meant to serialize hardware access, which - * is taken care of by the I2C layer, so this is safe. - */ - - if (locked) - mtx_unlock(mixer_lock); - snapper_write(sc, SNAPPER_VOLUME, reg); - if (locked) - mtx_lock(mixer_lock); - return (left | (right << 8)); } diff --git a/sys/dev/sound/macio/tumbler.c b/sys/dev/sound/macio/tumbler.c index bd40ea6b4f6b..89af4434e7fe 100644 --- a/sys/dev/sound/macio/tumbler.c +++ b/sys/dev/sound/macio/tumbler.c @@ -383,14 +383,10 @@ static int tumbler_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct tumbler_softc *sc; - struct mtx *mixer_lock; - int locked; u_int l, r; u_char reg[6]; sc = device_get_softc(mix_getdevinfo(m)); - mixer_lock = mixer_get_lock(m); - locked = mtx_owned(mixer_lock); switch (dev) { case SOUND_MIXER_VOLUME: @@ -407,21 +403,8 @@ tumbler_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) reg[4] = (r & 0x00ff00) >> 8; reg[5] = r & 0x0000ff; - /* - * We need to unlock the mixer lock because iicbus_transfer() - * may sleep. The mixer lock itself is unnecessary here - * because it is meant to serialize hardware access, which - * is taken care of by the I2C layer, so this is safe. - */ - - if (locked) - mtx_unlock(mixer_lock); - tumbler_write(sc, TUMBLER_VOLUME, reg); - if (locked) - mtx_lock(mixer_lock); - return (left | (right << 8)); } diff --git a/sys/dev/sound/pci/hda/hdaa_patches.c b/sys/dev/sound/pci/hda/hdaa_patches.c index d4267aae80f8..2f3a044195c4 100644 --- a/sys/dev/sound/pci/hda/hdaa_patches.c +++ b/sys/dev/sound/pci/hda/hdaa_patches.c @@ -328,6 +328,15 @@ hdac_pin_patch(struct hdaa_widget *w) patch_str = "as=1 seq=15"; break; } + } else if (id == HDA_CODEC_ALC255 && subid == DELL_WYSE7040_SUBVENDOR) { + switch (nid) { + case 20: + patch_str = "as=1 seq=0 device=Speaker"; + break; + case 33: + patch_str = "as=1 seq=15 device=Headphones"; + break; + } } else if (id == HDA_CODEC_ALC256 && (subid == DELL_I7577_SUBVENDOR || subid == DELL_L7480_SUBVENDOR)) { switch (nid) { diff --git a/sys/dev/sound/pci/hda/hdac.h b/sys/dev/sound/pci/hda/hdac.h index 27d592242578..36fd7b6a60ab 100644 --- a/sys/dev/sound/pci/hda/hdac.h +++ b/sys/dev/sound/pci/hda/hdac.h @@ -282,6 +282,7 @@ #define DELL_164AID_SUBVENDOR HDA_MODEL_CONSTRUCT(DELL, 0x164a) #define DELL_164BID_SUBVENDOR HDA_MODEL_CONSTRUCT(DELL, 0x164b) #define DELL_I7577_SUBVENDOR HDA_MODEL_CONSTRUCT(DELL, 0x0802) +#define DELL_WYSE7040_SUBVENDOR HDA_MODEL_CONSTRUCT(DELL, 0x0727) #define DELL_ALL_SUBVENDOR HDA_MODEL_CONSTRUCT(DELL, 0xffff) /* Clevo */ diff --git a/sys/dev/sound/pcm/channel.c b/sys/dev/sound/pcm/channel.c index c1e0d8d3bc52..a0ee16a14386 100644 --- a/sys/dev/sound/pcm/channel.c +++ b/sys/dev/sound/pcm/channel.c @@ -2177,7 +2177,7 @@ chn_syncstate(struct pcm_channel *c) if (c->feederflags & (1 << FEEDER_EQ)) { struct pcm_feeder *f; - int treble, bass, state; + int treble, bass; /* CHN_UNLOCK(c); */ treble = mix_get(m, SOUND_MIXER_TREBLE); @@ -2209,15 +2209,6 @@ chn_syncstate(struct pcm_channel *c) device_printf(c->dev, "EQ: Failed to set preamp -- %d\n", d->eqpreamp); - if (d->flags & SD_F_EQ_BYPASSED) - state = FEEDEQ_BYPASS; - else if (d->flags & SD_F_EQ_ENABLED) - state = FEEDEQ_ENABLE; - else - state = FEEDEQ_DISABLE; - if (FEEDER_SET(f, FEEDEQ_STATE, state) != 0) - device_printf(c->dev, - "EQ: Failed to set state -- %d\n", state); } } } diff --git a/sys/dev/sound/pcm/dsp.c b/sys/dev/sound/pcm/dsp.c index 0f76791448ca..bc92a3fbd530 100644 --- a/sys/dev/sound/pcm/dsp.c +++ b/sys/dev/sound/pcm/dsp.c @@ -728,8 +728,7 @@ dsp_ioctl(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, if (d->mixer_dev != NULL) { PCM_ACQUIRE_QUICK(d); - ret = mixer_ioctl_cmd(d->mixer_dev, cmd, arg, -1, td, - MIXER_CMD_DIRECT); + ret = mixer_ioctl_cmd(d->mixer_dev, cmd, arg, -1, td); PCM_RELEASE_QUICK(d); } else ret = EBADF; @@ -1526,8 +1525,7 @@ dsp_ioctl(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, if (d->mixer_dev != NULL) { PCM_ACQUIRE_QUICK(d); - ret = mixer_ioctl_cmd(d->mixer_dev, xcmd, arg, -1, td, - MIXER_CMD_DIRECT); + ret = mixer_ioctl_cmd(d->mixer_dev, xcmd, arg, -1, td); PCM_RELEASE_QUICK(d); } else ret = ENOTSUP; @@ -1539,8 +1537,7 @@ dsp_ioctl(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, case SNDCTL_DSP_SET_RECSRC: if (d->mixer_dev != NULL) { PCM_ACQUIRE_QUICK(d); - ret = mixer_ioctl_cmd(d->mixer_dev, cmd, arg, -1, td, - MIXER_CMD_DIRECT); + ret = mixer_ioctl_cmd(d->mixer_dev, cmd, arg, -1, td); PCM_RELEASE_QUICK(d); } else ret = ENOTSUP; diff --git a/sys/dev/sound/pcm/feeder.h b/sys/dev/sound/pcm/feeder.h index e1e91d468455..127b479cd7c9 100644 --- a/sys/dev/sound/pcm/feeder.h +++ b/sys/dev/sound/pcm/feeder.h @@ -119,11 +119,6 @@ enum { FEEDEQ_TREBLE, FEEDEQ_BASS, FEEDEQ_PREAMP, - FEEDEQ_STATE, - FEEDEQ_DISABLE, - FEEDEQ_ENABLE, - FEEDEQ_BYPASS, - FEEDEQ_UNKNOWN }; int feeder_eq_validrate(uint32_t); diff --git a/sys/dev/sound/pcm/feeder_chain.c b/sys/dev/sound/pcm/feeder_chain.c index 4fc846f77496..35bb12a062ec 100644 --- a/sys/dev/sound/pcm/feeder_chain.c +++ b/sys/dev/sound/pcm/feeder_chain.c @@ -725,7 +725,7 @@ feeder_chain(struct pcm_channel *c) /* Soft EQ only applicable for PLAY. */ if (cdesc.dummy == 0 && - c->direction == PCMDIR_PLAY && (d->flags & SD_F_EQ) && + c->direction == PCMDIR_PLAY && (d->flags & SD_F_EQ_ENABLED) && (((d->flags & SD_F_EQ_PC) && !(c->flags & CHN_F_HAS_VCHAN)) || (!(d->flags & SD_F_EQ_PC) && !(c->flags & CHN_F_VIRTUAL)))) diff --git a/sys/dev/sound/pcm/feeder_eq.c b/sys/dev/sound/pcm/feeder_eq.c index 4cf9d4f6695f..0a28dfa1ba17 100644 --- a/sys/dev/sound/pcm/feeder_eq.c +++ b/sys/dev/sound/pcm/feeder_eq.c @@ -3,7 +3,7 @@ * * Copyright (c) 2008-2009 Ariff Abdullah <ariff@FreeBSD.org> * All rights reserved. - * Copyright (c) 2024-2025 The FreeBSD Foundation + * Copyright (c) 2024-2026 The FreeBSD Foundation * * Portions of this software were developed by Christos Margiolis * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. @@ -122,7 +122,6 @@ struct feed_eq_info { uint32_t rate; uint32_t align; int32_t preamp; - int state; }; #if !defined(_KERNEL) && defined(FEEDEQ_ERR_CLIP) @@ -148,19 +147,6 @@ feed_eq_biquad(struct feed_eq_info *info, uint8_t *dst, uint32_t count, pmul = feed_eq_preamp[info->preamp].mul; pshift = feed_eq_preamp[info->preamp].shift; - if (info->state == FEEDEQ_DISABLE) { - j = count * info->channels; - dst += j * AFMT_BPS(fmt); - do { - dst -= AFMT_BPS(fmt); - v = pcm_sample_read(dst, fmt); - v = ((intpcm64_t)pmul * v) >> pshift; - pcm_sample_write(dst, v, fmt); - } while (--j != 0); - - return; - } - treble = &(info->coeff[info->treble.gain].treble); bass = &(info->coeff[info->bass.gain].bass); @@ -290,7 +276,6 @@ feed_eq_init(struct pcm_feeder *f) info->treble.gain = FEEDEQ_L2GAIN(50); info->bass.gain = FEEDEQ_L2GAIN(50); info->preamp = FEEDEQ_PREAMP2IDX(FEEDEQ_PREAMP_DEFAULT); - info->state = FEEDEQ_UNKNOWN; f->data = info; @@ -316,8 +301,6 @@ feed_eq_set(struct pcm_feeder *f, int what, int value) if (feeder_eq_validrate(value) == 0) return (EINVAL); info->rate = (uint32_t)value; - if (info->state == FEEDEQ_UNKNOWN) - info->state = FEEDEQ_ENABLE; return (feed_eq_setup(info)); case FEEDEQ_TREBLE: case FEEDEQ_BASS: @@ -333,13 +316,6 @@ feed_eq_set(struct pcm_feeder *f, int what, int value) return (EINVAL); info->preamp = FEEDEQ_PREAMP2IDX(value); break; - case FEEDEQ_STATE: - if (!(value == FEEDEQ_BYPASS || value == FEEDEQ_ENABLE || - value == FEEDEQ_DISABLE)) - return (EINVAL); - info->state = value; - feed_eq_reset(info); - break; default: return (EINVAL); } @@ -370,15 +346,6 @@ feed_eq_feed(struct pcm_feeder *f, struct pcm_channel *c, uint8_t *b, info = f->data; - /* - * 3 major states: - * FEEDEQ_BYPASS - Bypass entirely, nothing happened. - * FEEDEQ_ENABLE - Preamp+biquad filtering. - * FEEDEQ_DISABLE - Preamp only. - */ - if (info->state == FEEDEQ_BYPASS) - return (FEEDER_FEED(f->source, c, b, count, source)); - dst = b; count = SND_FXROUND(count, info->align); @@ -472,8 +439,6 @@ static int sysctl_dev_pcm_eq(SYSCTL_HANDLER_ARGS) { struct snddev_info *d; - struct pcm_channel *c; - struct pcm_feeder *f; int err, val, oval; d = oidp->oid_arg1; @@ -482,9 +447,7 @@ sysctl_dev_pcm_eq(SYSCTL_HANDLER_ARGS) PCM_LOCK(d); PCM_WAIT(d); - if (d->flags & SD_F_EQ_BYPASSED) - val = 2; - else if (d->flags & SD_F_EQ_ENABLED) + if (d->flags & SD_F_EQ_ENABLED) val = 1; else val = 0; @@ -495,30 +458,17 @@ sysctl_dev_pcm_eq(SYSCTL_HANDLER_ARGS) err = sysctl_handle_int(oidp, &val, 0, req); if (err == 0 && req->newptr != NULL && val != oval) { - if (!(val == 0 || val == 1 || val == 2)) { + if (!(val == 0 || val == 1)) { PCM_RELEASE_QUICK(d); return (EINVAL); } PCM_LOCK(d); - d->flags &= ~(SD_F_EQ_ENABLED | SD_F_EQ_BYPASSED); - if (val == 2) { - val = FEEDEQ_BYPASS; - d->flags |= SD_F_EQ_BYPASSED; - } else if (val == 1) { - val = FEEDEQ_ENABLE; + if (val == 1) d->flags |= SD_F_EQ_ENABLED; - } else - val = FEEDEQ_DISABLE; - - CHN_FOREACH(c, d, channels.pcm.busy) { - CHN_LOCK(c); - f = feeder_find(c, FEEDER_EQ); - if (f != NULL) - (void)FEEDER_SET(f, FEEDEQ_STATE, val); - CHN_UNLOCK(c); - } + else + d->flags &= ~SD_F_EQ_ENABLED; PCM_RELEASE(d); PCM_UNLOCK(d); @@ -592,17 +542,11 @@ void feeder_eq_initsys(device_t dev) { struct snddev_info *d; - const char *preamp; char buf[64]; d = device_get_softc(dev); - if (!(resource_string_value(device_get_name(dev), device_get_unit(dev), - "eq_preamp", &preamp) == 0 && - (d->eqpreamp = feed_eq_scan_preamp_arg(preamp)) != - FEEDEQ_PREAMP_INVALID)) - d->eqpreamp = FEEDEQ_PREAMP_DEFAULT; - + d->eqpreamp = FEEDEQ_PREAMP_DEFAULT; if (d->eqpreamp < FEEDEQ_PREAMP_MIN) d->eqpreamp = FEEDEQ_PREAMP_MIN; else if (d->eqpreamp > FEEDEQ_PREAMP_MAX) @@ -612,7 +556,7 @@ feeder_eq_initsys(device_t dev) SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "eq", CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, d, sizeof(d), sysctl_dev_pcm_eq, "I", - "Bass/Treble Equalizer (0=disable, 1=enable, 2=bypass)"); + "Bass/Treble Equalizer (0=disable, 1=enable)"); (void)snprintf(buf, sizeof(buf), "Bass/Treble Equalizer Preamp " "(-/+ %d.0dB , %d.%ddB step)", diff --git a/sys/dev/sound/pcm/mixer.c b/sys/dev/sound/pcm/mixer.c index 6ed2d0c3ce5c..3ddee24417cc 100644 --- a/sys/dev/sound/pcm/mixer.c +++ b/sys/dev/sound/pcm/mixer.c @@ -5,6 +5,10 @@ * Portions Copyright (c) Ryan Beasley <ryan.beasley@gmail.com> - GSoC 2006 * Copyright (c) 1999 Cameron Grant <cg@FreeBSD.org> * All rights reserved. + * Copyright (c) 2026 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,36 +41,6 @@ #include "feeder_if.h" #include "mixer_if.h" -static MALLOC_DEFINE(M_MIXER, "mixer", "mixer"); - -static int mixer_bypass = 1; -SYSCTL_INT(_hw_snd, OID_AUTO, vpc_mixer_bypass, CTLFLAG_RWTUN, - &mixer_bypass, 0, - "control channel pcm/rec volume, bypassing real mixer device"); - -#define MIXER_NAMELEN 16 -struct snd_mixer { - KOBJ_FIELDS; - void *devinfo; - int hwvol_mixer; - int hwvol_step; - int type; - device_t dev; - u_int32_t devs; - u_int32_t mutedevs; - u_int32_t recdevs; - u_int32_t recsrc; - u_int16_t level[32]; - u_int16_t level_muted[32]; - u_int8_t parent[32]; - u_int32_t child[32]; - u_int8_t realdev[32]; - char name[MIXER_NAMELEN]; - struct mtx lock; - oss_mixer_enuminfo enuminfo; - int modify_counter; -}; - static u_int16_t snd_mixerdefaults[SOUND_MIXER_NRDEVICES] = { [SOUND_MIXER_VOLUME] = 75, [SOUND_MIXER_BASS] = 50, @@ -304,7 +278,7 @@ mixer_set(struct snd_mixer *m, u_int dev, u_int32_t muted, u_int lev) if (dev == SOUND_MIXER_PCM && (d->flags & SD_F_SOFTPCMVOL)) (void)mixer_set_softpcmvol(m, d, l, r); else if ((dev == SOUND_MIXER_TREBLE || - dev == SOUND_MIXER_BASS) && (d->flags & SD_F_EQ)) + dev == SOUND_MIXER_BASS) && (d->flags & SD_F_EQ_ENABLED)) (void)mixer_set_eq(m, d, dev, (l + r) >> 1); else if (realdev != SOUND_MIXER_NONE && MIXER_SET(m, realdev, l, r) < 0) { @@ -484,8 +458,7 @@ mix_setdevs(struct snd_mixer *m, u_int32_t v) d = device_get_softc(m->dev); if (d != NULL && (d->flags & SD_F_SOFTPCMVOL)) v |= SOUND_MASK_PCM; - if (d != NULL && (d->flags & SD_F_EQ)) - v |= SOUND_MASK_TREBLE | SOUND_MASK_BASS; + v |= SOUND_MASK_TREBLE | SOUND_MASK_BASS; for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) { if (m->parent[i] < SOUND_MIXER_NRDEVICES) v |= 1 << m->parent[i]; @@ -501,64 +474,12 @@ mix_setdevs(struct snd_mixer *m, u_int32_t v) * recording devices. This function records that value in a structure * used by the rest of the mixer code. * - * This function also populates a structure used by the SNDCTL_DSP_*RECSRC* - * family of ioctls that are part of OSSV4. All recording device labels - * are concatenated in ascending order corresponding to their routing - * numbers. (Ex: a system might have 0 => 'vol', 1 => 'cd', 2 => 'line', - * etc.) For now, these labels are just the standard recording device - * names (cd, line1, etc.), but will eventually be fully dynamic and user - * controlled. - * * @param m mixer device context container thing * @param v mask of recording devices */ void mix_setrecdevs(struct snd_mixer *m, u_int32_t v) { - oss_mixer_enuminfo *ei; - char *loc; - int i, nvalues, nwrote, nleft, ncopied; - - ei = &m->enuminfo; - - nvalues = 0; - nwrote = 0; - nleft = sizeof(ei->strings); - loc = ei->strings; - - for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) { - if ((1 << i) & v) { - ei->strindex[nvalues] = nwrote; - ncopied = strlcpy(loc, snd_mixernames[i], nleft) + 1; - /* strlcpy retval doesn't include terminator */ - - nwrote += ncopied; - nleft -= ncopied; - nvalues++; - - /* - * XXX I don't think this should ever be possible. - * Even with a move to dynamic device/channel names, - * each label is limited to ~16 characters, so that'd - * take a LOT to fill this buffer. - */ - if ((nleft <= 0) || (nvalues >= OSS_ENUM_MAXVALUE)) { - device_printf(m->dev, - "mix_setrecdevs: Not enough room to store device names--please file a bug report.\n"); - device_printf(m->dev, - "mix_setrecdevs: Please include details about your sound hardware, OS version, etc.\n"); - break; - } - - loc = &ei->strings[nwrote]; - } - } - - /* - * NB: The SNDCTL_DSP_GET_RECSRC_NAMES ioctl ignores the dev - * and ctrl fields. - */ - ei->nvalues = nvalues; m->recdevs = v; } @@ -639,7 +560,7 @@ mixer_obj_create(device_t dev, kobj_class_t cls, void *devinfo, KASSERT(type == MIXER_TYPE_PRIMARY || type == MIXER_TYPE_SECONDARY, ("invalid mixer type=%d", type)); - m = (struct snd_mixer *)kobj_create(cls, M_MIXER, M_WAITOK | M_ZERO); + m = (struct snd_mixer *)kobj_create(cls, M_DEVBUF, M_WAITOK | M_ZERO); snprintf(m->name, sizeof(m->name), "%s:mixer", device_get_nameunit(dev)); if (desc != NULL) { @@ -658,9 +579,8 @@ mixer_obj_create(device_t dev, kobj_class_t cls, void *devinfo, } if (MIXER_INIT(m)) { - mtx_lock(&m->lock); mtx_destroy(&m->lock); - kobj_delete((kobj_t)m, M_MIXER); + kobj_delete((kobj_t)m, M_DEVBUF); return (NULL); } @@ -679,7 +599,7 @@ mixer_delete(struct snd_mixer *m) MIXER_UNINIT(m); mtx_destroy(&m->lock); - kobj_delete((kobj_t)m, M_MIXER); + kobj_delete((kobj_t)m, M_DEVBUF); return (0); } @@ -706,15 +626,6 @@ mixer_init(device_t dev, kobj_class_t cls, void *devinfo) name = device_get_name(dev); unit = device_get_unit(dev); - if (resource_int_value(name, unit, "eq", &val) == 0 && - val != 0) { - snddev->flags |= SD_F_EQ; - if ((val & SD_F_EQ_MASK) == val) - snddev->flags |= val; - else - snddev->flags |= SD_F_EQ_DEFAULT; - snddev->eqpreamp = 0; - } m = mixer_obj_create(dev, cls, devinfo, MIXER_TYPE_PRIMARY, NULL); if (m == NULL) @@ -762,8 +673,7 @@ mixer_init(device_t dev, kobj_class_t cls, void *devinfo) } if (snddev->flags & SD_F_SOFTPCMVOL) device_printf(dev, "Soft PCM mixer ENABLED\n"); - if (snddev->flags & SD_F_EQ) - device_printf(dev, "EQ Treble/Bass ENABLED\n"); + device_printf(dev, "EQ Treble/Bass ENABLED\n"); } return (0); @@ -804,7 +714,7 @@ mixer_uninit(device_t dev) MIXER_UNINIT(m); mtx_destroy(&m->lock); - kobj_delete((kobj_t)m, M_MIXER); + kobj_delete((kobj_t)m, M_DEVBUF); d->mixer_dev = NULL; @@ -1046,113 +956,6 @@ mixer_close(struct cdev *i_dev, int flags, int mode, struct thread *td) } static int -mixer_ioctl_channel(struct cdev *dev, u_long cmd, caddr_t arg, int mode, - struct thread *td, int from) -{ - struct snddev_info *d; - struct snd_mixer *m; - struct pcm_channel *c, *rdch, *wrch; - pid_t pid; - int j, ret; - - if (td == NULL || td->td_proc == NULL) - return (-1); - - m = dev->si_drv1; - d = device_get_softc(m->dev); - j = cmd & 0xff; - - switch (j) { - case SOUND_MIXER_PCM: - case SOUND_MIXER_RECLEV: - case SOUND_MIXER_DEVMASK: - case SOUND_MIXER_CAPS: - case SOUND_MIXER_STEREODEVS: - break; - default: - return (-1); - } - - pid = td->td_proc->p_pid; - rdch = NULL; - wrch = NULL; - c = NULL; - ret = -1; - - /* - * This is unfair. Imagine single proc opening multiple - * instances of same direction. What we do right now - * is looking for the first matching proc/pid, and just - * that. Nothing more. Consider it done. - * - * The better approach of controlling specific channel - * pcm or rec volume is by doing mixer ioctl - * (SNDCTL_DSP_[SET|GET][PLAY|REC]VOL / SOUND_MIXER_[PCM|RECLEV] - * on its open fd, rather than cracky mixer bypassing here. - */ - CHN_FOREACH(c, d, channels.pcm.opened) { - CHN_LOCK(c); - if (c->pid != pid || - !(c->feederflags & (1 << FEEDER_VOLUME))) { - CHN_UNLOCK(c); - continue; - } - if (rdch == NULL && c->direction == PCMDIR_REC) { - rdch = c; - if (j == SOUND_MIXER_RECLEV) - goto mixer_ioctl_channel_proc; - } else if (wrch == NULL && c->direction == PCMDIR_PLAY) { - wrch = c; - if (j == SOUND_MIXER_PCM) - goto mixer_ioctl_channel_proc; - } - CHN_UNLOCK(c); - if (rdch != NULL && wrch != NULL) - break; - } - - if (rdch == NULL && wrch == NULL) - return (-1); - - if ((j == SOUND_MIXER_DEVMASK || j == SOUND_MIXER_CAPS || - j == SOUND_MIXER_STEREODEVS) && - (cmd & ~0xff) == MIXER_READ(0)) { - mtx_lock(&m->lock); - *(int *)arg = mix_getdevs(m); - mtx_unlock(&m->lock); - if (rdch != NULL) - *(int *)arg |= SOUND_MASK_RECLEV; - if (wrch != NULL) - *(int *)arg |= SOUND_MASK_PCM; - ret = 0; - } - - return (ret); - -mixer_ioctl_channel_proc: - - KASSERT(c != NULL, ("%s(): NULL channel", __func__)); - CHN_LOCKASSERT(c); - - if ((cmd & ~0xff) == MIXER_WRITE(0)) { - int left, right, center; - - left = *(int *)arg & 0x7f; - right = (*(int *)arg >> 8) & 0x7f; - center = (left + right) >> 1; - chn_setvolume_multi(c, SND_VOL_C_PCM, left, right, center); - } else if ((cmd & ~0xff) == MIXER_READ(0)) { - *(int *)arg = chn_getvolume_matrix(c, SND_VOL_C_PCM, SND_CHN_T_FL); - *(int *)arg |= - chn_getvolume_matrix(c, SND_VOL_C_PCM, SND_CHN_T_FR) << 8; - } - - CHN_UNLOCK(c); - - return (0); -} - -static int mixer_ioctl(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, struct thread *td) { @@ -1169,15 +972,7 @@ mixer_ioctl(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, PCM_GIANT_ENTER(d); PCM_ACQUIRE_QUICK(d); - ret = -1; - - if (mixer_bypass != 0 && (d->flags & SD_F_VPC)) - ret = mixer_ioctl_channel(i_dev, cmd, arg, mode, td, - MIXER_CMD_CDEV); - - if (ret == -1) - ret = mixer_ioctl_cmd(i_dev, cmd, arg, mode, td, - MIXER_CMD_CDEV); + ret = mixer_ioctl_cmd(i_dev, cmd, arg, mode, td); PCM_RELEASE_QUICK(d); PCM_GIANT_LEAVE(d); @@ -1200,7 +995,7 @@ mixer_mixerinfo(struct snd_mixer *m, mixer_info *mi) */ int mixer_ioctl_cmd(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, - struct thread *td, int from) + struct thread *td) { struct snd_mixer *m; int ret = EINVAL, *arg_i = (int *)arg; @@ -1238,10 +1033,31 @@ mixer_ioctl_cmd(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, mtx_lock(&m->lock); switch (cmd) { - case SNDCTL_DSP_GET_RECSRC_NAMES: - bcopy((void *)&m->enuminfo, arg, sizeof(oss_mixer_enuminfo)); + case SNDCTL_DSP_GET_RECSRC_NAMES: { + oss_mixer_enuminfo *ei = (oss_mixer_enuminfo *)arg; + char *loc; + int i, nvalues, nwrote, nleft, ncopied; + + nvalues = 0; + nwrote = 0; + nleft = sizeof(ei->strings); + loc = ei->strings; + + for (i = 0; i < SOUND_MIXER_NRDEVICES; i++) { + if (!((1 << i) & m->recdevs)) + continue; + ei->strindex[nvalues] = nwrote; + ncopied = strlcpy(loc, snd_mixernames[i], nleft) + 1; + nwrote += ncopied; + nleft -= ncopied; + nvalues++; + loc = &ei->strings[nwrote]; + } + ei->nvalues = nvalues; + ret = 0; goto done; + } case SNDCTL_DSP_GET_RECSRC: ret = mixer_get_recroute(m, arg_i); goto done; @@ -1515,13 +1331,3 @@ mixer_oss_mixerinfo(struct cdev *i_dev, oss_mixerinfo *mi) return (EINVAL); } - -/* - * Allow the sound driver to use the mixer lock to protect its mixer - * data: - */ -struct mtx * -mixer_get_lock(struct snd_mixer *m) -{ - return (&m->lock); -} diff --git a/sys/dev/sound/pcm/mixer.h b/sys/dev/sound/pcm/mixer.h index 3ce8a4f5adee..6f764307cfc8 100644 --- a/sys/dev/sound/pcm/mixer.h +++ b/sys/dev/sound/pcm/mixer.h @@ -4,6 +4,10 @@ * Copyright (c) 2005-2009 Ariff Abdullah <ariff@FreeBSD.org> * Copyright (c) 1999 Cameron Grant <cg@FreeBSD.org> * All rights reserved. + * Copyright (c) 2026 The FreeBSD Foundation + * + * Portions of this software were developed by Christos Margiolis + * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -30,13 +34,35 @@ #ifndef _PCM_MIXER_H_ #define _PCM_MIXER_H_ +#define MIXER_NAMELEN 16 +struct snd_mixer { + KOBJ_FIELDS; + void *devinfo; + int hwvol_mixer; + int hwvol_step; + int type; + device_t dev; + u_int32_t devs; + u_int32_t mutedevs; + u_int32_t recdevs; + u_int32_t recsrc; + u_int16_t level[32]; + u_int16_t level_muted[32]; + u_int8_t parent[32]; + u_int32_t child[32]; + u_int8_t realdev[32]; + char name[MIXER_NAMELEN]; + struct mtx lock; + int modify_counter; +}; + struct snd_mixer *mixer_create(device_t dev, kobj_class_t cls, void *devinfo, const char *desc); int mixer_delete(struct snd_mixer *m); int mixer_init(device_t dev, kobj_class_t cls, void *devinfo); int mixer_uninit(device_t dev); int mixer_reinit(device_t dev); -int mixer_ioctl_cmd(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, struct thread *td, int from); +int mixer_ioctl_cmd(struct cdev *i_dev, u_long cmd, caddr_t arg, int mode, struct thread *td); int mixer_oss_mixerinfo(struct cdev *i_dev, oss_mixerinfo *mi); int mixer_hwvol_init(device_t dev); @@ -61,21 +87,11 @@ void mix_setparentchild(struct snd_mixer *m, u_int32_t parent, u_int32_t childs) void mix_setrealdev(struct snd_mixer *m, u_int32_t dev, u_int32_t realdev); u_int32_t mix_getparent(struct snd_mixer *m, u_int32_t dev); void *mix_getdevinfo(struct snd_mixer *m); -struct mtx *mixer_get_lock(struct snd_mixer *m); - -#define MIXER_CMD_DIRECT 0 /* send command within driver */ -#define MIXER_CMD_CDEV 1 /* send command from cdev/ioctl */ #define MIXER_TYPE_PRIMARY 0 /* mixer_init() */ #define MIXER_TYPE_SECONDARY 1 /* mixer_create() */ -/* - * this is a kludge to allow hiding of the struct snd_mixer definition - * 512 should be enough for all architectures - */ -#define MIXER_SIZE (512 + sizeof(struct kobj) + \ - sizeof(oss_mixer_enuminfo)) - -#define MIXER_DECLARE(name) static DEFINE_CLASS(name, name ## _methods, MIXER_SIZE) +#define MIXER_DECLARE(name) static DEFINE_CLASS(name, name ## _methods, \ + sizeof(struct snd_mixer)) #endif /* _PCM_MIXER_H_ */ diff --git a/sys/dev/sound/pcm/sound.c b/sys/dev/sound/pcm/sound.c index 9eb2dffeb908..235142eb5209 100644 --- a/sys/dev/sound/pcm/sound.c +++ b/sys/dev/sound/pcm/sound.c @@ -77,11 +77,30 @@ snd_setup_intr(device_t dev, struct resource *res, int flags, driver_intr_t hand return bus_setup_intr(dev, res, flags, NULL, hand, param, cookiep); } +static void +pcm_hotswap(void) +{ + struct snddev_info *d; + char buf[32]; + + bus_topo_assert(); + if (snd_unit >= 0) { + d = devclass_get_softc(pcm_devclass, snd_unit); + if (!PCM_REGISTERED(d)) + return; + snprintf(buf, sizeof(buf), "cdev=dsp%d", snd_unit); + if (d->reccount > 0) + devctl_notify("SND", "CONN", "IN", buf); + if (d->playcount > 0) + devctl_notify("SND", "CONN", "OUT", buf); + } else + devctl_notify("SND", "CONN", "NODEV", NULL); +} + static int sysctl_hw_snd_default_unit(SYSCTL_HANDLER_ARGS) { struct snddev_info *d; - char buf[32]; int error, unit; unit = snd_unit; @@ -95,13 +114,8 @@ sysctl_hw_snd_default_unit(SYSCTL_HANDLER_ARGS) } snd_unit = unit; snd_unit_auto = 0; + pcm_hotswap(); bus_topo_unlock(); - - snprintf(buf, sizeof(buf), "cdev=dsp%d", snd_unit); - if (d->reccount > 0) - devctl_notify("SND", "CONN", "IN", buf); - if (d->playcount > 0) - devctl_notify("SND", "CONN", "OUT", buf); } return (error); } @@ -373,6 +387,7 @@ int pcm_register(device_t dev, char *str) { struct snddev_info *d = device_get_softc(dev); + int err; /* should only be called once */ if (d->flags & SD_F_REGISTERED) @@ -415,9 +430,15 @@ pcm_register(device_t dev, char *str) "mode (1=mixer, 2=play, 4=rec. The values are OR'ed if more than " "one mode is supported)"); vchan_initsys(dev); - if (d->flags & SD_F_EQ) - feeder_eq_initsys(dev); + feeder_eq_initsys(dev); + + sndstat_register(dev, SNDST_TYPE_PCM, d->status); + err = dsp_make_dev(dev); + if (err) + return (err); + + bus_topo_lock(); if (snd_unit_auto < 0) snd_unit_auto = (snd_unit < 0) ? 1 : 0; if (snd_unit < 0 || snd_unit_auto > 1) @@ -425,9 +446,11 @@ pcm_register(device_t dev, char *str) else if (snd_unit_auto == 1) snd_unit = pcm_best_unit(snd_unit); - sndstat_register(dev, SNDST_TYPE_PCM, d->status); + if (snd_unit == device_get_unit(dev)) + pcm_hotswap(); + bus_topo_unlock(); - return (dsp_make_dev(dev)); + return (0); } int @@ -470,13 +493,14 @@ pcm_unregister(device_t dev) cv_destroy(&d->cv); mtx_destroy(&d->lock); + bus_topo_lock(); if (snd_unit == device_get_unit(dev)) { snd_unit = pcm_best_unit(-1); if (snd_unit_auto == 0) snd_unit_auto = 1; - if (snd_unit < 0) - devctl_notify("SND", "CONN", "NODEV", NULL); + pcm_hotswap(); } + bus_topo_unlock(); return (0); } diff --git a/sys/dev/sound/pcm/sound.h b/sys/dev/sound/pcm/sound.h index 4795eb7585c5..4d527d69086a 100644 --- a/sys/dev/sound/pcm/sound.h +++ b/sys/dev/sound/pcm/sound.h @@ -105,17 +105,13 @@ struct snd_mixer; #define SD_F_REGISTERED 0x00000020 #define SD_F_BITPERFECT 0x00000040 #define SD_F_VPC 0x00000080 /* volume-per-channel */ -#define SD_F_EQ 0x00000100 /* EQ */ +/* unused 0x00000100 */ #define SD_F_EQ_ENABLED 0x00000200 /* EQ enabled */ -#define SD_F_EQ_BYPASSED 0x00000400 /* EQ bypassed */ +/* unused 0x00000400 */ #define SD_F_EQ_PC 0x00000800 /* EQ per-channel */ #define SD_F_PVCHANS 0x00001000 /* Playback vchans enabled */ #define SD_F_RVCHANS 0x00002000 /* Recording vchans enabled */ -#define SD_F_EQ_DEFAULT (SD_F_EQ | SD_F_EQ_ENABLED) -#define SD_F_EQ_MASK (SD_F_EQ | SD_F_EQ_ENABLED | \ - SD_F_EQ_BYPASSED | SD_F_EQ_PC) - #define SD_F_BITS "\020" \ "\001SIMPLEX" \ /* "\002 */ \ @@ -125,9 +121,9 @@ struct snd_mixer; "\006REGISTERED" \ "\007BITPERFECT" \ "\010VPC" \ - "\011EQ" \ + /* "\011 */ \ "\012EQ_ENABLED" \ - "\013EQ_BYPASSED" \ + /* "\013 */ \ "\014EQ_PC" \ "\015PVCHANS" \ "\016RVCHANS" diff --git a/sys/dev/sound/usb/uaudio.c b/sys/dev/sound/usb/uaudio.c index f477eb768fde..eedd52774b70 100644 --- a/sys/dev/sound/usb/uaudio.c +++ b/sys/dev/sound/usb/uaudio.c @@ -356,7 +356,7 @@ struct uaudio_hid { struct uaudio_softc_child { device_t pcm_device; - struct mtx *mixer_lock; + struct mtx mixer_lock; struct snd_mixer *mixer_dev; uint32_t mix_info; @@ -2955,12 +2955,9 @@ uaudio_mixer_sysctl_handler(SYSCTL_HANDLER_ARGS) sc = (struct uaudio_softc *)oidp->oid_arg1; hint = oidp->oid_arg2; - if (sc->sc_child[0].mixer_lock == NULL) - return (ENXIO); - /* lookup mixer node */ - mtx_lock(sc->sc_child[0].mixer_lock); + mtx_lock(&sc->sc_child[0].mixer_lock); for (pmc = sc->sc_mixer_root; pmc != NULL; pmc = pmc->next) { for (chan = 0; chan != (int)pmc->nchan; chan++) { if (pmc->wValue[chan] != -1 && @@ -2971,7 +2968,7 @@ uaudio_mixer_sysctl_handler(SYSCTL_HANDLER_ARGS) } } found: - mtx_unlock(sc->sc_child[0].mixer_lock); + mtx_unlock(&sc->sc_child[0].mixer_lock); error = sysctl_handle_int(oidp, &temp, 0, req); if (error != 0 || req->newptr == NULL) @@ -2979,7 +2976,7 @@ found: /* update mixer value */ - mtx_lock(sc->sc_child[0].mixer_lock); + mtx_lock(&sc->sc_child[0].mixer_lock); if (pmc != NULL && temp >= pmc->minval && temp <= pmc->maxval) { @@ -2989,7 +2986,7 @@ found: /* start the transfer, if not already started */ usbd_transfer_start(sc->sc_mixer_xfer[0]); } - mtx_unlock(sc->sc_child[0].mixer_lock); + mtx_unlock(&sc->sc_child[0].mixer_lock); return (0); } @@ -3220,10 +3217,7 @@ uaudio_mixer_reload_all(struct uaudio_softc *sc) struct uaudio_mixer_node *pmc; int chan; - if (sc->sc_child[0].mixer_lock == NULL) - return; - - mtx_lock(sc->sc_child[0].mixer_lock); + mtx_lock(&sc->sc_child[0].mixer_lock); for (pmc = sc->sc_mixer_root; pmc != NULL; pmc = pmc->next) { /* use reset defaults for non-oss controlled settings */ if (pmc->ctl == SOUND_MIXER_NRDEVICES) @@ -3235,7 +3229,7 @@ uaudio_mixer_reload_all(struct uaudio_softc *sc) /* start HID volume keys, if any */ usbd_transfer_start(sc->sc_hid.xfer[0]); - mtx_unlock(sc->sc_child[0].mixer_lock); + mtx_unlock(&sc->sc_child[0].mixer_lock); } static void @@ -5392,8 +5386,8 @@ uaudio_mixer_bsd2value(struct uaudio_mixer_node *mc, int val) } static void -uaudio_mixer_ctl_set(struct uaudio_softc *sc, struct uaudio_mixer_node *mc, - uint8_t chan, int val) +uaudio_mixer_ctl_set(struct uaudio_softc *sc, unsigned index, + struct uaudio_mixer_node *mc, uint8_t chan, int val) { val = uaudio_mixer_bsd2value(mc, val); @@ -5402,7 +5396,9 @@ uaudio_mixer_ctl_set(struct uaudio_softc *sc, struct uaudio_mixer_node *mc, /* start the transfer, if not already started */ + mtx_lock(&sc->sc_child[index].mixer_lock); usbd_transfer_start(sc->sc_mixer_xfer[0]); + mtx_unlock(&sc->sc_child[index].mixer_lock); } static void @@ -5439,13 +5435,13 @@ uaudio_mixer_init_sub(struct uaudio_softc *sc, struct snd_mixer *m) DPRINTF("child=%u\n", i); - sc->sc_child[i].mixer_lock = mixer_get_lock(m); + mtx_init(&sc->sc_child[i].mixer_lock, "uaudio mixer lock", NULL, MTX_DEF); sc->sc_child[i].mixer_dev = m; if (i == 0 && usbd_transfer_setup(sc->sc_udev, &sc->sc_mixer_iface_index, sc->sc_mixer_xfer, uaudio_mixer_config, 1, sc, - sc->sc_child[i].mixer_lock)) { + &sc->sc_child[i].mixer_lock)) { DPRINTFN(0, "could not allocate USB transfer for mixer!\n"); return (ENOMEM); } @@ -5470,7 +5466,7 @@ uaudio_mixer_uninit_sub(struct uaudio_softc *sc, struct snd_mixer *m) if (index == 0) usbd_transfer_unsetup(sc->sc_mixer_xfer, 1); - sc->sc_child[index].mixer_lock = NULL; + mtx_destroy(&sc->sc_child[index].mixer_lock); return (0); } @@ -5488,7 +5484,7 @@ uaudio_mixer_set(struct uaudio_softc *sc, struct snd_mixer *m, for (mc = sc->sc_mixer_root; mc != NULL; mc = mc->next) { if (mc->ctl == type) { for (chan = 0; chan < mc->nchan; chan++) { - uaudio_mixer_ctl_set(sc, mc, chan, + uaudio_mixer_ctl_set(sc, index, mc, chan, chan == 0 ? left : right); } } @@ -5529,7 +5525,7 @@ uaudio_mixer_setrecsrc(struct uaudio_softc *sc, struct snd_mixer *m, uint32_t sr for (i = mc->minval; (i > 0) && (i <= mc->maxval); i++) { if (temp != (1U << mc->slctrtype[i - 1])) continue; - uaudio_mixer_ctl_set(sc, mc, 0, i); + uaudio_mixer_ctl_set(sc, index, mc, 0, i); break; } } @@ -6186,9 +6182,6 @@ uaudio_hid_attach(struct uaudio_softc *sc, if (!(sc->sc_hid.flags & UAUDIO_HID_VALID)) return (-1); - if (sc->sc_child[0].mixer_lock == NULL) - return (-1); - /* Get HID descriptor */ error = usbd_req_get_hid_desc(uaa->device, NULL, &d_ptr, &d_len, M_TEMP, sc->sc_hid.iface_index); @@ -6247,7 +6240,7 @@ uaudio_hid_attach(struct uaudio_softc *sc, /* allocate USB transfers */ error = usbd_transfer_setup(uaa->device, &sc->sc_hid.iface_index, sc->sc_hid.xfer, uaudio_hid_config, UAUDIO_HID_N_TRANSFER, - sc, sc->sc_child[0].mixer_lock); + sc, &sc->sc_child[0].mixer_lock); if (error) { DPRINTF("error=%s\n", usbd_errstr(error)); return (-1); diff --git a/sys/dev/sound/usb/uaudio_pcm.c b/sys/dev/sound/usb/uaudio_pcm.c index c24c111f983c..4b1762cfc3ec 100644 --- a/sys/dev/sound/usb/uaudio_pcm.c +++ b/sys/dev/sound/usb/uaudio_pcm.c @@ -134,39 +134,18 @@ ua_mixer_init(struct snd_mixer *m) static int ua_mixer_set(struct snd_mixer *m, unsigned type, unsigned left, unsigned right) { - struct mtx *mtx = mixer_get_lock(m); - uint8_t do_unlock; - - if (mtx_owned(mtx)) { - do_unlock = 0; - } else { - do_unlock = 1; - mtx_lock(mtx); - } uaudio_mixer_set(mix_getdevinfo(m), m, type, left, right); - if (do_unlock) { - mtx_unlock(mtx); - } + return (left | (right << 8)); } static uint32_t ua_mixer_setrecsrc(struct snd_mixer *m, uint32_t src) { - struct mtx *mtx = mixer_get_lock(m); int retval; - uint8_t do_unlock; - if (mtx_owned(mtx)) { - do_unlock = 0; - } else { - do_unlock = 1; - mtx_lock(mtx); - } retval = uaudio_mixer_setrecsrc(mix_getdevinfo(m), m, src); - if (do_unlock) { - mtx_unlock(mtx); - } + return (retval); } diff --git a/sys/dev/uart/uart_dev_ns8250.c b/sys/dev/uart/uart_dev_ns8250.c index c13eabe9055e..d6940dc80005 100644 --- a/sys/dev/uart/uart_dev_ns8250.c +++ b/sys/dev/uart/uart_dev_ns8250.c @@ -529,40 +529,40 @@ UART_CLASS(uart_ns8250_class); */ #ifdef DEV_ACPI static struct acpi_spcr_compat_data acpi_spcr_compat_data[] = { - { &uart_ns8250_class, ACPI_DBG2_16550_COMPATIBLE }, - { &uart_ns8250_class, ACPI_DBG2_16550_SUBSET }, - { &uart_ns8250_class, ACPI_DBG2_16550_WITH_GAS }, + { &uart_ns8250_class, ACPI_DBG2_16550_COMPATIBLE }, + { &uart_ns8250_class, ACPI_DBG2_16550_SUBSET }, + { &uart_ns8250_class, ACPI_DBG2_16550_WITH_GAS }, { NULL, 0 }, }; UART_ACPI_SPCR_CLASS(acpi_spcr_compat_data); static struct acpi_uart_compat_data acpi_compat_data[] = { {"AMD0020", &uart_ns8250_class, 2, 0, 48000000, UART_F_BUSY_DETECT, "AMD / Synopsys Designware UART"}, - {"AMDI0020", &uart_ns8250_class, 2, 0, 48000000, UART_F_BUSY_DETECT, "AMD / Synopsys Designware UART"}, - {"APMC0D08", &uart_ns8250_class, 2, 4, 0, 0, "APM compatible UART"}, - {"MRVL0001", &uart_ns8250_class, 2, 0, 200000000, UART_F_BUSY_DETECT, "Marvell / Synopsys Designware UART"}, - {"SCX0006", &uart_ns8250_class, 2, 0, 62500000, UART_F_BUSY_DETECT, "SynQuacer / Synopsys Designware UART"}, - {"HISI0031", &uart_ns8250_class, 2, 0, 200000000, UART_F_BUSY_DETECT, "HiSilicon / Synopsys Designware UART"}, - {"INTC1006", &uart_ns8250_class, 2, 0, 25000000, 0, "Intel ARM64 UART"}, - {"NXP0018", &uart_ns8250_class, 0, 0, 350000000, UART_F_BUSY_DETECT, "NXP / Synopsys Designware UART"}, - {"PNP0500", &uart_ns8250_class, 0, 0, 0, 0, "Standard PC COM port"}, - {"PNP0501", &uart_ns8250_class, 0, 0, 0, 0, "16550A-compatible COM port"}, - {"PNP0502", &uart_ns8250_class, 0, 0, 0, 0, "Multiport serial device (non-intelligent 16550)"}, - {"PNP0510", &uart_ns8250_class, 0, 0, 0, 0, "Generic IRDA-compatible device"}, - {"PNP0511", &uart_ns8250_class, 0, 0, 0, 0, "Generic IRDA-compatible device"}, - {"WACF004", &uart_ns8250_class, 0, 0, 0, 0, "Wacom Tablet PC Screen"}, - {"WACF00E", &uart_ns8250_class, 0, 0, 0, 0, "Wacom Tablet PC Screen 00e"}, - {"FUJ02E5", &uart_ns8250_class, 0, 0, 0, 0, "Wacom Tablet at FuS Lifebook T"}, - {NULL, NULL, 0 , 0, 0, 0, NULL}, + {"AMDI0020", &uart_ns8250_class, 2, 0, 48000000, UART_F_BUSY_DETECT, "AMD / Synopsys Designware UART"}, + {"APMC0D08", &uart_ns8250_class, 2, 4, 0, 0, "APM compatible UART"}, + {"MRVL0001", &uart_ns8250_class, 2, 0, 200000000, UART_F_BUSY_DETECT, "Marvell / Synopsys Designware UART"}, + {"SCX0006", &uart_ns8250_class, 2, 0, 62500000, UART_F_BUSY_DETECT, "SynQuacer / Synopsys Designware UART"}, + {"HISI0031", &uart_ns8250_class, 2, 0, 200000000, UART_F_BUSY_DETECT, "HiSilicon / Synopsys Designware UART"}, + {"INTC1006", &uart_ns8250_class, 2, 0, 25000000, 0, "Intel ARM64 UART"}, + {"NXP0018", &uart_ns8250_class, 0, 0, 350000000, UART_F_BUSY_DETECT, "NXP / Synopsys Designware UART"}, + {"PNP0500", &uart_ns8250_class, 0, 0, 0, 0, "Standard PC COM port"}, + {"PNP0501", &uart_ns8250_class, 0, 0, 0, 0, "16550A-compatible COM port"}, + {"PNP0502", &uart_ns8250_class, 0, 0, 0, 0, "Multiport serial device (non-intelligent 16550)"}, + {"PNP0510", &uart_ns8250_class, 0, 0, 0, 0, "Generic IRDA-compatible device"}, + {"PNP0511", &uart_ns8250_class, 0, 0, 0, 0, "Generic IRDA-compatible device"}, + {"WACF004", &uart_ns8250_class, 0, 0, 0, 0, "Wacom Tablet PC Screen"}, + {"WACF00E", &uart_ns8250_class, 0, 0, 0, 0, "Wacom Tablet PC Screen 00e"}, + {"FUJ02E5", &uart_ns8250_class, 0, 0, 0, 0, "Wacom Tablet at FuS Lifebook T"}, + {NULL, NULL, 0, 0, 0, 0, NULL}, }; UART_ACPI_CLASS_AND_DEVICE(acpi_compat_data); #endif #ifdef FDT static struct ofw_compat_data compat_data[] = { - {"ns16550", (uintptr_t)&uart_ns8250_class}, - {"ns16550a", (uintptr_t)&uart_ns8250_class}, - {NULL, (uintptr_t)NULL}, + {"ns16550", (uintptr_t)&uart_ns8250_class}, + {"ns16550a", (uintptr_t)&uart_ns8250_class}, + {NULL, (uintptr_t)NULL}, }; UART_FDT_CLASS_AND_DEVICE(compat_data); #endif diff --git a/sys/dev/uart/uart_dev_pl011.c b/sys/dev/uart/uart_dev_pl011.c index ae3c4d3218cf..f0d7bcda1fa4 100644 --- a/sys/dev/uart/uart_dev_pl011.c +++ b/sys/dev/uart/uart_dev_pl011.c @@ -382,32 +382,32 @@ static struct uart_class uart_pl011_class = { }; UART_CLASS(uart_pl011_class); -#ifdef FDT -static struct ofw_compat_data fdt_compat_data[] = { - {"arm,pl011", (uintptr_t)&uart_pl011_class}, - {NULL, (uintptr_t)NULL}, -}; -UART_FDT_CLASS_AND_DEVICE(fdt_compat_data); -#endif - #ifdef DEV_ACPI static struct acpi_spcr_compat_data acpi_spcr_compat_data[] = { - { &uart_pl011_class, ACPI_DBG2_ARM_PL011 }, - { &uart_pl011_class, ACPI_DBG2_ARM_SBSA_GENERIC }, - { &uart_pl011_class, ACPI_DBG2_ARM_SBSA_32BIT }, + { &uart_pl011_class, ACPI_DBG2_ARM_PL011 }, + { &uart_pl011_class, ACPI_DBG2_ARM_SBSA_GENERIC }, + { &uart_pl011_class, ACPI_DBG2_ARM_SBSA_32BIT }, { NULL, 0 }, }; UART_ACPI_SPCR_CLASS(acpi_spcr_compat_data); static struct acpi_uart_compat_data acpi_compat_data[] = { - {"ARMH0011", &uart_pl011_class, 2, 0, 0, 0, "uart pl011"}, - {"ARMHB000", &uart_pl011_class, 2, 0, 0, 0, "uart pl011"}, - {"ARMHB000", &uart_pl011_class, 2, 0, 0, 0, "uart pl011"}, - {NULL, NULL, 0, 0, 0, 0, NULL}, + {"ARMH0011", &uart_pl011_class, 2, 0, 0, 0, "uart pl011"}, + {"ARMHB000", &uart_pl011_class, 2, 0, 0, 0, "uart pl011"}, + {"ARMHB000", &uart_pl011_class, 2, 0, 0, 0, "uart pl011"}, + {NULL, NULL, 0, 0, 0, 0, NULL}, }; UART_ACPI_CLASS_AND_DEVICE(acpi_compat_data); #endif +#ifdef FDT +static struct ofw_compat_data fdt_compat_data[] = { + {"arm,pl011", (uintptr_t)&uart_pl011_class}, + {NULL, (uintptr_t)NULL}, +}; +UART_FDT_CLASS_AND_DEVICE(fdt_compat_data); +#endif + static int uart_pl011_bus_attach(struct uart_softc *sc) { diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c index 67745cf49397..d41121ed3a06 100644 --- a/sys/dev/usb/usb_transfer.c +++ b/sys/dev/usb/usb_transfer.c @@ -1889,8 +1889,10 @@ usbd_transfer_submit(struct usb_xfer *xfer) */ #if USB_HAVE_BUSDMA if (xfer->flags_int.bdma_enable) { + USB_BUS_LOCK(bus); /* insert the USB transfer last in the BUS-DMA queue */ usb_command_wrapper(&xfer->xroot->dma_q, xfer); + USB_BUS_UNLOCK(bus); return; } #endif diff --git a/sys/dev/virtio/p9fs/virtio_p9fs.c b/sys/dev/virtio/p9fs/virtio_p9fs.c index 19a32fea458e..c347458b4f8e 100644 --- a/sys/dev/virtio/p9fs/virtio_p9fs.c +++ b/sys/dev/virtio/p9fs/virtio_p9fs.c @@ -112,7 +112,7 @@ SYSCTL_UINT(_vfs_9p, OID_AUTO, ackmaxidle, CTLFLAG_RW, &vt9p_ackmaxidle, 0, static int vt9p_req_wait(struct vt9p_softc *chan, struct p9_req_t *req) { - KASSERT(req->tc->tag != req->rc->tag, + KASSERT(req->tc.tag != req->rc.tag, ("%s: request %p already completed", __func__, req)); if (msleep(req, VT9P_MTX(chan), 0, "chan lock", vt9p_ackmaxidle * hz)) { @@ -124,7 +124,7 @@ vt9p_req_wait(struct vt9p_softc *chan, struct p9_req_t *req) "for an ack from host\n", vt9p_ackmaxidle); return (EIO); } - KASSERT(req->tc->tag == req->rc->tag, + KASSERT(req->tc.tag == req->rc.tag, ("%s spurious event on request %p", __func__, req)); return (0); } @@ -157,7 +157,7 @@ vt9p_request(void *handle, struct p9_req_t *req) req_retry: sglist_reset(sg); /* Handle out VirtIO ring buffers */ - error = sglist_append(sg, req->tc->sdata, req->tc->size); + error = sglist_append(sg, req->tc.sdata, req->tc.size); if (error != 0) { P9_DEBUG(ERROR, "%s: sglist append failed\n", __func__); VT9P_UNLOCK(chan); @@ -165,7 +165,7 @@ req_retry: } readable = sg->sg_nseg; - error = sglist_append(sg, req->rc->sdata, req->rc->capacity); + error = sglist_append(sg, req->rc.sdata, req->rc.capacity); if (error != 0) { P9_DEBUG(ERROR, "%s: sglist append failed\n", __func__); VT9P_UNLOCK(chan); @@ -226,7 +226,7 @@ vt9p_intr_complete(void *xsc) VT9P_LOCK(chan); again: while ((curreq = virtqueue_dequeue(vq, NULL)) != NULL) { - curreq->rc->tag = curreq->tc->tag; + curreq->rc.tag = curreq->tc.tag; wakeup_one(curreq); } if (virtqueue_enable_intr(vq) != 0) { diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c index 9ea4e5f4c9df..2d4c41994c0e 100644 --- a/sys/fs/nfs/nfs_commonkrpc.c +++ b/sys/fs/nfs/nfs_commonkrpc.c @@ -1265,8 +1265,9 @@ tryagain: goto out; } sep = NFSMNT_MDSSESSION(nmp); - if (bcmp(sep->nfsess_sessionid, nd->nd_sequence, - NFSX_V4SESSIONID) == 0) { + if (bcmp(sep->nfsess_sessionid, + nd->nd_sessionid, NFSX_V4SESSIONID) == 0 && + sep->nfsess_defunct == 0) { printf("Initiate recovery. If server " "has not rebooted, " "check NFS clients for unique " diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c index b5f83a98b307..a11b55b11c43 100644 --- a/sys/fs/nfs/nfs_commonsubs.c +++ b/sys/fs/nfs/nfs_commonsubs.c @@ -368,6 +368,7 @@ nfscl_reqstart(struct nfsrv_descript *nd, int procnum, struct nfsmount *nmp, * First, fill in some of the fields of nd. */ nd->nd_slotseq = NULL; + NFSBZERO(nd->nd_sessionid, NFSX_V4SESSIONID); if (vers == NFS_VER4) { nd->nd_flag = ND_NFSV4 | ND_NFSCL; if (minorvers == NFSV41_MINORVERSION) @@ -5348,6 +5349,7 @@ nfsv4_setsequence(struct nfsmount *nmp, struct nfsrv_descript *nd, NFSM_BUILD(tl, uint32_t *, NFSX_V4SESSIONID + 4 * NFSX_UNSIGNED); nd->nd_sequence = tl; bcopy(sessionid, tl, NFSX_V4SESSIONID); + bcopy(sessionid, nd->nd_sessionid, NFSX_V4SESSIONID); tl += NFSX_V4SESSIONID / NFSX_UNSIGNED; nd->nd_slotseq = tl; if (error == 0) { @@ -5593,6 +5595,7 @@ nfsrpc_destroysession(struct nfsmount *nmp, struct nfsclsession *tsep, 0, NULL); NFSM_BUILD(tl, uint32_t *, NFSX_V4SESSIONID); bcopy(tsep->nfsess_sessionid, tl, NFSX_V4SESSIONID); + bcopy(tsep->nfsess_sessionid, nd->nd_sessionid, NFSX_V4SESSIONID); nd->nd_flag |= ND_USEGSSNAME; error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred, NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL); diff --git a/sys/fs/p9fs/p9_client.c b/sys/fs/p9fs/p9_client.c index 547de98c4c03..d3be87ee1645 100644 --- a/sys/fs/p9fs/p9_client.c +++ b/sys/fs/p9fs/p9_client.c @@ -95,7 +95,14 @@ p9_parse_opts(struct mount *mp, struct p9_client *clnt) /* These are defaults for now */ clnt->proto_version = p9_proto_2000L; - clnt->msize = 8192; + clnt->msize = P9FS_MTU; + + vfs_scanopt(mp->mnt_optnew, "msize", "%u", &clnt->msize); + if (clnt->msize > P9FS_MTU) { + vfs_mount_error(mp, "msize %u is greater than max allowed %u", + clnt->msize, P9FS_MTU); + return (EINVAL); + } /* Get the default trans callback */ clnt->ops = p9_get_trans_by_name(trans); @@ -104,43 +111,33 @@ p9_parse_opts(struct mount *mp, struct p9_client *clnt) } /* Allocate buffer for sending request and getting responses */ -static struct p9_buffer * -p9_buffer_alloc(int alloc_msize) +static void +p9_buffer_alloc(struct p9_buffer *fc, int alloc_msize) { - struct p9_buffer *fc; - - fc = uma_zalloc(p9fs_buf_zone, M_WAITOK | M_ZERO); + bzero(fc, sizeof(*fc)); fc->capacity = alloc_msize; - fc->offset = 0; - fc->size = 0; - fc->sdata = (char *)fc + sizeof(struct p9_buffer); - - return (fc); + fc->sdata = uma_zalloc(p9fs_buf_zone, M_WAITOK); } /* Free memory used by request and response buffers */ static void -p9_buffer_free(struct p9_buffer **buf) +p9_buffer_free(struct p9_buffer *buf) { - - /* Free the sdata buffers first, then the whole structure*/ - uma_zfree(p9fs_buf_zone, *buf); - *buf = NULL; + uma_zfree(p9fs_buf_zone, buf->sdata); + buf->sdata = NULL; } /* Free the request */ static void p9_free_req(struct p9_client *clnt, struct p9_req_t *req) { + if (req == NULL) + return; - if (req->tc != NULL) { - if (req->tc->tag != P9_NOTAG) - p9_tag_destroy(clnt, req->tc->tag); - p9_buffer_free(&req->tc); - } - - if (req->rc != NULL) - p9_buffer_free(&req->rc); + if (req->tc.tag != P9_NOTAG) + p9_tag_destroy(clnt, req->tc.tag); + p9_buffer_free(&req->tc); + p9_buffer_free(&req->rc); uma_zfree(p9fs_req_zone, req); } @@ -156,17 +153,17 @@ p9_get_request(struct p9_client *clnt, int *error) alloc_msize = P9FS_MTU; req = uma_zalloc(p9fs_req_zone, M_WAITOK | M_ZERO); - req->tc = p9_buffer_alloc(alloc_msize); - req->rc = p9_buffer_alloc(alloc_msize); + p9_buffer_alloc(&req->tc, alloc_msize); + p9_buffer_alloc(&req->rc, alloc_msize); tag = p9_tag_create(clnt); if (tag == P9_NOTAG) { *error = EAGAIN; - req->tc->tag = P9_NOTAG; + req->tc.tag = P9_NOTAG; p9_free_req(clnt, req); return (NULL); } - req->tc->tag = tag; + req->tc.tag = tag; return (req); } @@ -208,7 +205,7 @@ p9_client_check_return(struct p9_client *c, struct p9_req_t *req) char *ename; /* Check what we have in the receive bufer .*/ - error = p9_parse_receive(req->rc, c); + error = p9_parse_receive(&req->rc, c); if (error != 0) goto out; @@ -216,17 +213,17 @@ p9_client_check_return(struct p9_client *c, struct p9_req_t *req) * No error, We are done with the preprocessing. Return to the caller * and process the actual data. */ - if (req->rc->id != P9PROTO_RERROR && req->rc->id != P9PROTO_RLERROR) + if (req->rc.id != P9PROTO_RERROR && req->rc.id != P9PROTO_RLERROR) return (0); /* * Interpreting the error is done in different ways for Linux and * Unix version. Make sure you interpret it right. */ - if (req->rc->id == P9PROTO_RERROR) { - error = p9_buf_readf(req->rc, c->proto_version, "s?d", &ename, &ecode); - } else if (req->rc->id == P9PROTO_RLERROR) { - error = p9_buf_readf(req->rc, c->proto_version, "d", &ecode); + if (req->rc.id == P9PROTO_RERROR) { + error = p9_buf_readf(&req->rc, c->proto_version, "s?d", &ename, &ecode); + } else if (req->rc.id == P9PROTO_RLERROR) { + error = p9_buf_readf(&req->rc, c->proto_version, "d", &ecode); } else { goto out; } @@ -241,15 +238,15 @@ p9_client_check_return(struct p9_client *c, struct p9_req_t *req) * not present can hit this and return. Hence it is made a debug print. */ if (error != 0) { - if (req->rc->id == P9PROTO_RERROR) { + if (req->rc.id == P9PROTO_RERROR) { P9_DEBUG(PROTO, "RERROR error %d ename %s\n", error, ename); - } else if (req->rc->id == P9PROTO_RLERROR) { + } else if (req->rc.id == P9PROTO_RLERROR) { P9_DEBUG(PROTO, "RLERROR error %d\n", error); } } - if (req->rc->id == P9PROTO_RERROR) { + if (req->rc.id == P9PROTO_RERROR) { free(ename, M_TEMP); } return (error); @@ -308,21 +305,21 @@ p9_client_prepare_req(struct p9_client *c, int8_t type, } /* Marshall the data according to QEMU standards */ - *error = p9_buf_prepare(req->tc, type); + *error = p9_buf_prepare(&req->tc, type); if (*error != 0) { P9_DEBUG(ERROR, "%s: p9_buf_prepare failed: %d\n", __func__, *error); goto out; } - *error = p9_buf_vwritef(req->tc, c->proto_version, fmt, ap); + *error = p9_buf_vwritef(&req->tc, c->proto_version, fmt, ap); if (*error != 0) { P9_DEBUG(ERROR, "%s: p9_buf_vwrite failed: %d\n", __func__, *error); goto out; } - *error = p9_buf_finalize(c, req->tc); + *error = p9_buf_finalize(c, &req->tc); if (*error != 0) { P9_DEBUG(ERROR, "%s: p9_buf_finalize failed: %d \n", __func__, *error); @@ -474,7 +471,7 @@ p9_client_version(struct p9_client *c) if (error != 0) return (error); - error = p9_buf_readf(req->rc, c->proto_version, "ds", &msize, &version); + error = p9_buf_readf(&req->rc, c->proto_version, "ds", &msize, &version); if (error != 0) { P9_DEBUG(ERROR, "%s: version error: %d\n", __func__, error); goto out; @@ -519,8 +516,7 @@ p9_init_zones(void) /* Create the buffer zone */ p9fs_buf_zone = uma_zcreate("p9fs buf zone", - sizeof(struct p9_buffer) + P9FS_MTU, NULL, NULL, - NULL, NULL, UMA_ALIGN_PTR, 0); + P9FS_MTU, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); } void @@ -623,7 +619,7 @@ p9_client_attach(struct p9_client *clnt, struct p9_fid *afid, if (*error != 0) goto out; - *error = p9_buf_readf(req->rc, clnt->proto_version, "Q", &qid); + *error = p9_buf_readf(&req->rc, clnt->proto_version, "Q", &qid); if (*error != 0) { P9_DEBUG(ERROR, "%s: p9_buf_readf failed: %d \n", __func__, *error); @@ -777,7 +773,7 @@ p9_client_walk(struct p9_fid *oldfid, uint16_t nwnames, char **wnames, return (NULL); } - *error = p9_buf_readf(req->rc, clnt->proto_version, "R", &nwqids, + *error = p9_buf_readf(&req->rc, clnt->proto_version, "R", &nwqids, &wqids); if (*error != 0) goto out; @@ -842,7 +838,7 @@ p9_client_open(struct p9_fid *fid, int mode) if (error != 0) return (error); - error = p9_buf_readf(req->rc, clnt->proto_version, "Qd", &fid->qid, + error = p9_buf_readf(&req->rc, clnt->proto_version, "Qd", &fid->qid, &mtu); if (error != 0) goto out; @@ -892,7 +888,7 @@ p9_client_readdir(struct p9_fid *fid, char *data, uint64_t offset, return (-error); } - error = p9_buf_readf(req->rc, clnt->proto_version, "D", &count, + error = p9_buf_readf(&req->rc, clnt->proto_version, "D", &count, &dataptr); if (error != 0) { P9_DEBUG(ERROR, "%s: p0_buf_readf failed: %d\n", @@ -945,7 +941,7 @@ p9_client_read(struct p9_fid *fid, uint64_t offset, uint32_t count, char *data) return (-error); } - error = p9_buf_readf(req->rc, clnt->proto_version, "D", &count, + error = p9_buf_readf(&req->rc, clnt->proto_version, "D", &count, &dataptr); if (error != 0) { P9_DEBUG(ERROR, "%s: p9_buf_readf failed: %d\n", @@ -1017,7 +1013,7 @@ p9_client_write(struct p9_fid *fid, uint64_t offset, uint32_t count, char *data) return (-error); } - error = p9_buf_readf(req->rc, clnt->proto_version, "d", &ret); + error = p9_buf_readf(&req->rc, clnt->proto_version, "d", &ret); if (error != 0) { P9_DEBUG(ERROR, "%s: p9_buf_readf error: %d\n", __func__, error); @@ -1069,7 +1065,7 @@ p9_client_file_create(struct p9_fid *fid, char *name, uint32_t perm, int mode, if (error != 0) return (error); - error = p9_buf_readf(req->rc, clnt->proto_version, "Qd", &qid, &mtu); + error = p9_buf_readf(&req->rc, clnt->proto_version, "Qd", &qid, &mtu); if (error != 0) goto out; @@ -1101,7 +1097,7 @@ p9_client_statfs(struct p9_fid *fid, struct p9_statfs *stat) return (error); } - error = p9_buf_readf(req->rc, clnt->proto_version, "ddqqqqqqd", + error = p9_buf_readf(&req->rc, clnt->proto_version, "ddqqqqqqd", &stat->type, &stat->bsize, &stat->blocks, &stat->bfree, &stat->bavail, &stat->files, &stat->ffree, &stat->fsid, &stat->namelen); @@ -1173,7 +1169,7 @@ p9_create_symlink(struct p9_fid *fid, char *name, char *symtgt, gid_t gid) if (error != 0) return (error); - error = p9_buf_readf(req->rc, clnt->proto_version, "Q", &qid); + error = p9_buf_readf(&req->rc, clnt->proto_version, "Q", &qid); if (error != 0) { P9_DEBUG(ERROR, "%s: buf_readf failed %d\n", __func__, error); return (error); @@ -1226,7 +1222,7 @@ p9_readlink(struct p9_fid *fid, char **target) if (error != 0) return (error); - error = p9_buf_readf(req->rc, clnt->proto_version, "s", target); + error = p9_buf_readf(&req->rc, clnt->proto_version, "s", target); if (error != 0) { P9_DEBUG(ERROR, "%s: buf_readf failed %d\n", __func__, error); return (error); @@ -1260,7 +1256,7 @@ p9_client_getattr(struct p9_fid *fid, struct p9_stat_dotl *stat_dotl, goto error; } - err = p9_buf_readf(req->rc, clnt->proto_version, "A", stat_dotl); + err = p9_buf_readf(&req->rc, clnt->proto_version, "A", stat_dotl); if (err != 0) { P9_DEBUG(ERROR, "%s: buf_readf failed %d\n", __func__, err); goto error; diff --git a/sys/fs/p9fs/p9_client.h b/sys/fs/p9fs/p9_client.h index 4eb82c0232f4..e5167f9a0f58 100644 --- a/sys/fs/p9fs/p9_client.h +++ b/sys/fs/p9fs/p9_client.h @@ -54,8 +54,8 @@ enum p9_proto_versions { /* P9 Request exchanged between Host and Guest */ struct p9_req_t { - struct p9_buffer *tc; /* request buffer */ - struct p9_buffer *rc; /* response buffer */ + struct p9_buffer tc; /* request buffer */ + struct p9_buffer rc; /* response buffer */ }; /* 9P transport status */ @@ -65,8 +65,11 @@ enum transport_status { P9FS_DISCONNECT, /* transport has been dosconnected */ }; -/* This is set by QEMU so we will oblige */ -#define P9FS_MTU 8192 +/* + * This matches the Linux 5.15 and newer default. + * Note: Linux allows larger msize values than this. + */ +#define P9FS_MTU 131072 /* * Even though we have a 8k buffer, Qemu is typically doing 8168 diff --git a/sys/fs/p9fs/p9fs_vfsops.c b/sys/fs/p9fs/p9fs_vfsops.c index a0f0a5a4e494..fe8c1b5ded31 100644 --- a/sys/fs/p9fs/p9fs_vfsops.c +++ b/sys/fs/p9fs/p9fs_vfsops.c @@ -59,7 +59,7 @@ extern struct vop_vector p9fs_vnops; /* option parsing */ static const char *p9fs_opts[] = { - "from", "trans", "access", NULL + "from", "trans", "access", "msize", NULL }; /* Dispose p9fs node, freeing it to the UMA zone */ diff --git a/sys/fs/smbfs/smbfs_node.c b/sys/fs/smbfs/smbfs_node.c index 9893987f7ccd..b2b5d63e2eaf 100644 --- a/sys/fs/smbfs/smbfs_node.c +++ b/sys/fs/smbfs/smbfs_node.c @@ -212,6 +212,10 @@ smbfs_node_alloc(struct mount *mp, struct vnode *dvp, const char *dirnm, SMBERROR("new vnode '%s' born without parent ?\n", np->n_name); error = insmntque(vp, mp); if (error) { + smbfs_name_free(np->n_name); + free(np->n_rpath, M_SMBNODENAME); + if (np->n_parent != NULL && (np->n_flag & NREFPARENT) != 0) + vrele(np->n_parent); free(np, M_SMBNODE); return (error); } diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES index 25bfb99c1fef..b21f05b685b5 100644 --- a/sys/i386/conf/NOTES +++ b/sys/i386/conf/NOTES @@ -296,13 +296,6 @@ envvar hint.pcf.0.irq="5" # device glxsb # AMD Geode LX Security Block -# -# padlock is a driver for the cryptographic functions and RNG in -# VIA C3, C7, and Eden processors. -# Requires 'device crypto'. -# -device padlock_rng # VIA Padlock RNG - ##################################################################### # ABI Emulation diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index f9afb9afe45f..71992f18e8fb 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -1492,8 +1492,8 @@ init386(int first) PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd); /* Initialize the tss (except for the final esp0) early for vm86. */ - common_tss0.tss_esp0 = (vm_offset_t)thread0.td_kstack + - thread0.td_kstack_pages * PAGE_SIZE - VM86_STACK_SPACE; + common_tss0.tss_esp0 = (vm_offset_t)td_kstack_top(&thread0) - + VM86_STACK_SPACE; common_tss0.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss0.tss_ioopt = sizeof(struct i386tss) << 16; gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 672ec9360c35..72fe83d3fdd4 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -89,7 +89,7 @@ get_pcb_user_save_td(struct thread *td) { char *p; - p = td->td_kstack + td->td_kstack_pages * PAGE_SIZE - + p = td_kstack_top(td) - roundup2(cpu_max_ext_state_size, XSAVE_AREA_ALIGN); KASSERT(__is_aligned(p, XSAVE_AREA_ALIGN), ("Unaligned pcb_user_save area")); @@ -110,7 +110,7 @@ get_pcb_td(struct thread *td) { char *p; - p = td->td_kstack + td->td_kstack_pages * PAGE_SIZE - + p = td_kstack_top(td) - roundup2(cpu_max_ext_state_size, XSAVE_AREA_ALIGN) - sizeof(struct pcb); return ((struct pcb *)p); @@ -232,9 +232,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) return; } - /* Point the pcb to the top of the stack */ - pcb2 = get_pcb_td(td2); - td2->td_pcb = pcb2; + pcb2 = td2->td_pcb; copy_thread(td1, td2); @@ -248,11 +246,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) /* * Copy the trap frame for the return to user mode as if from a * syscall. This copies most of the user mode register values. - * The -VM86_STACK_SPACE (-16) is so we can expand the trapframe - * if we go to vm86. */ - td2->td_frame = (struct trapframe *)((caddr_t)td2->td_pcb - - VM86_STACK_SPACE) - 1; bcopy(td1->td_frame, td2->td_frame, sizeof(struct trapframe)); /* Set child return values. */ @@ -378,19 +372,22 @@ cpu_thread_clean(struct thread *td) void cpu_thread_alloc(struct thread *td) { +} + +void +cpu_thread_new_kstack(struct thread *td) +{ struct pcb *pcb; - struct xstate_hdr *xhdr; + /* + * The -VM86_STACK_SPACE (-16) is so we can expand the trapframe + * if we go to vm86. + */ td->td_pcb = pcb = get_pcb_td(td); td->td_frame = (struct trapframe *)((caddr_t)pcb - VM86_STACK_SPACE) - 1; pcb->pcb_ext = NULL; pcb->pcb_save = get_pcb_user_save_pcb(pcb); - if (use_xsave) { - xhdr = (struct xstate_hdr *)(pcb->pcb_save + 1); - bzero(xhdr, sizeof(*xhdr)); - xhdr->xstate_bv = xsave_mask; - } } void diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index af0841c75549..e3969223c170 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -84,6 +84,8 @@ #define ELF_NOTE_ROUNDSIZE 4 #define OLD_EI_BRAND 8 +#define ELF_OFFPAGE_PHNUM 128 + /* * ELF_ABI_NAME is a string name of the ELF ABI. ELF_ABI_ID is used * to build variable names. @@ -93,8 +95,8 @@ static int __elfN(check_header)(const Elf_Ehdr *hdr); static const Elf_Brandinfo *__elfN(get_brandinfo)(struct image_params *imgp, - const char *interp, int32_t *osrel, uint32_t *fctl0); -static int __elfN(load_file)(struct proc *p, const char *file, u_long *addr, + const Elf_Phdr *phdr, const char *interp, int32_t *osrel, uint32_t *fctl0); +static int __elfN(load_file)(struct thread *td, const char *file, u_long *addr, u_long *entry); static int __elfN(load_section)(const struct image_params *imgp, vm_ooffset_t offset, caddr_t vmaddr, size_t memsz, size_t filsz, @@ -103,7 +105,7 @@ static int __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp); static bool __elfN(freebsd_trans_osrel)(const Elf_Note *note, int32_t *osrel); static bool kfreebsd_trans_osrel(const Elf_Note *note, int32_t *osrel); -static bool __elfN(check_note)(struct image_params *imgp, +static bool __elfN(check_note)(struct image_params *imgp, const Elf_Phdr *phdr, const Elf_Brandnote *checknote, int32_t *osrel, bool *has_fctl0, uint32_t *fctl0); static vm_prot_t __elfN(trans_prot)(Elf_Word); @@ -339,8 +341,8 @@ __elfN(brand_inuse)(const Elf_Brandinfo *entry) } static const Elf_Brandinfo * -__elfN(get_brandinfo)(struct image_params *imgp, const char *interp, - int32_t *osrel, uint32_t *fctl0) +__elfN(get_brandinfo)(struct image_params *imgp, const Elf_Phdr *phdr, + const char *interp, int32_t *osrel, uint32_t *fctl0) { const Elf_Ehdr *hdr = (const Elf_Ehdr *)imgp->image_header; const Elf_Brandinfo *bi, *bi_m; @@ -369,8 +371,8 @@ __elfN(get_brandinfo)(struct image_params *imgp, const char *interp, has_fctl0 = false; *fctl0 = 0; *osrel = 0; - ret = __elfN(check_note)(imgp, bi->brand_note, osrel, - &has_fctl0, fctl0); + ret = __elfN(check_note)(imgp, phdr, bi->brand_note, + osrel, &has_fctl0, fctl0); /* Give brand a chance to veto check_note's guess */ if (ret && bi->header_supported) { ret = bi->header_supported(imgp, osrel, @@ -780,19 +782,20 @@ __elfN(load_sections)(const struct image_params *imgp, const Elf_Ehdr *hdr, * the entry point for the loaded file. */ static int -__elfN(load_file)(struct proc *p, const char *file, u_long *addr, - u_long *entry) +__elfN(load_file)(struct thread *td, const char *file, u_long *addr, + u_long *entry) { struct { struct nameidata nd; struct vattr attr; struct image_params image_params; - } *tempdata; + } *tempdata = NULL; const Elf_Ehdr *hdr = NULL; const Elf_Phdr *phdr = NULL; struct nameidata *nd; struct vattr *attr; struct image_params *imgp; + void *m_phdrs = NULL; u_long rbase; u_long base_addr = 0; int error; @@ -802,7 +805,7 @@ __elfN(load_file)(struct proc *p, const char *file, u_long *addr, * XXXJA: This check can go away once we are sufficiently confident * that the checks in namei() are correct. */ - if (IN_CAPABILITY_MODE(curthread)) + if (IN_CAPABILITY_MODE(td)) return (ECAPMODE); #endif @@ -814,7 +817,8 @@ __elfN(load_file)(struct proc *p, const char *file, u_long *addr, /* * Initialize part of the common data */ - imgp->proc = p; + imgp->td = td; + imgp->proc = td->td_proc; imgp->attr = attr; NDINIT(nd, LOOKUP, ISOPEN | FOLLOW | LOCKSHARED | LOCKLEAF, @@ -851,24 +855,35 @@ __elfN(load_file)(struct proc *p, const char *file, u_long *addr, goto fail; } - /* Only support headers that fit within first page for now */ - if (!__elfN(phdr_in_zero_page)(hdr)) { + if (!aligned(imgp->image_header + hdr->e_phoff, Elf_Addr)) { error = ENOEXEC; goto fail; } - - phdr = (const Elf_Phdr *)(imgp->image_header + hdr->e_phoff); - if (!aligned(phdr, Elf_Addr)) { - error = ENOEXEC; - goto fail; + if (__elfN(phdr_in_zero_page)(hdr)) { + phdr = (const Elf_Phdr *)(imgp->image_header + hdr->e_phoff); + } else { + if (hdr->e_phnum > ELF_OFFPAGE_PHNUM) { + error = ENOEXEC; + goto fail; + } + VOP_UNLOCK(imgp->vp); + phdr = m_phdrs = malloc(hdr->e_phnum * sizeof(Elf_Phdr), + M_TEMP, M_WAITOK | M_ZERO); + vn_lock(imgp->vp, LK_SHARED | LK_RETRY); + error = vn_rdwr(UIO_READ, imgp->vp, m_phdrs, + hdr->e_phnum * sizeof(Elf_Phdr), hdr->e_phoff, + UIO_SYSSPACE, IO_NODELOCKED, imgp->td->td_ucred, + NOCRED, NULL, imgp->td); + if (error != 0) + goto fail; } error = __elfN(load_sections)(imgp, hdr, phdr, rbase, &base_addr); if (error != 0) goto fail; - if (p->p_sysent->sv_protect != NULL) - p->p_sysent->sv_protect(imgp, SVP_INTERP); + if (imgp->proc->p_sysent->sv_protect != NULL) + imgp->proc->p_sysent->sv_protect(imgp, SVP_INTERP); *addr = base_addr; *entry = (unsigned long)hdr->e_entry + rbase; @@ -882,6 +897,7 @@ fail: VOP_UNSET_TEXT_CHECKED(nd->ni_vp); vput(nd->ni_vp); } + free(m_phdrs, M_TEMP); free(tempdata, M_TEMP); return (error); @@ -1008,7 +1024,6 @@ static int __elfN(get_interp)(struct image_params *imgp, const Elf_Phdr *phdr, char **interpp, bool *free_interpp) { - struct thread *td; char *interp; int error, interp_name_len; @@ -1016,8 +1031,6 @@ __elfN(get_interp)(struct image_params *imgp, const Elf_Phdr *phdr, ("%s: p_type %u != PT_INTERP", __func__, phdr->p_type)); ASSERT_VOP_LOCKED(imgp->vp, __func__); - td = curthread; - /* Path to interpreter */ if (phdr->p_filesz < 2 || phdr->p_filesz > MAXPATHLEN) { uprintf("Invalid PT_INTERP\n"); @@ -1045,8 +1058,8 @@ __elfN(get_interp)(struct image_params *imgp, const Elf_Phdr *phdr, error = vn_rdwr(UIO_READ, imgp->vp, interp, interp_name_len, phdr->p_offset, - UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, - NOCRED, NULL, td); + UIO_SYSSPACE, IO_NODELOCKED, imgp->td->td_ucred, + NOCRED, NULL, imgp->td); if (error != 0) { free(interp, M_TEMP); uprintf("i/o error PT_INTERP %d\n", error); @@ -1079,13 +1092,13 @@ __elfN(load_interp)(struct image_params *imgp, const Elf_Brandinfo *brand_info, if (brand_info->interp_newpath != NULL && (brand_info->interp_path == NULL || strcmp(interp, brand_info->interp_path) == 0)) { - error = __elfN(load_file)(imgp->proc, + error = __elfN(load_file)(imgp->td, brand_info->interp_newpath, addr, entry); if (error == 0) return (0); } - error = __elfN(load_file)(imgp->proc, interp, addr, entry); + error = __elfN(load_file)(imgp->td, interp, addr, entry); if (error == 0) return (0); @@ -1102,7 +1115,6 @@ __elfN(load_interp)(struct image_params *imgp, const Elf_Brandinfo *brand_info, static int __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) { - struct thread *td; const Elf_Ehdr *hdr; const Elf_Phdr *phdr; Elf_Auxargs *elf_auxargs; @@ -1111,6 +1123,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) char *interp; const Elf_Brandinfo *brand_info; struct sysentvec *sv; + void *m_phdrs; u_long addr, baddr, entry, proghdr; u_long maxalign, maxsalign, mapsz, maxv, maxv1, anon_loc; uint32_t fctl0; @@ -1135,16 +1148,6 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) * detected an ELF file. */ - if (!__elfN(phdr_in_zero_page)(hdr)) { - uprintf("Program headers not in the first page\n"); - return (ENOEXEC); - } - phdr = (const Elf_Phdr *)(imgp->image_header + hdr->e_phoff); - if (!aligned(phdr, Elf_Addr)) { - uprintf("Unaligned program headers\n"); - return (ENOEXEC); - } - n = error = 0; baddr = 0; osrel = 0; @@ -1152,7 +1155,33 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) entry = proghdr = 0; interp = NULL; free_interp = false; - td = curthread; + m_phdrs = NULL; + + if (!aligned(imgp->image_header + hdr->e_phoff, Elf_Addr)) { + uprintf("Unaligned program headers\n"); + return (ENOEXEC); + } + if (hdr->e_phoff + hdr->e_phnum * hdr->e_phentsize < hdr->e_phoff) { + uprintf("PHDRS wrap\n"); + return (ENOEXEC); + } + if (__elfN(phdr_in_zero_page)(hdr)) { + phdr = (const Elf_Phdr *)(imgp->image_header + hdr->e_phoff); + } else if (hdr->e_phnum > ELF_OFFPAGE_PHNUM) { + uprintf("Too many program headers\n"); + return (ENOEXEC); + } else { + VOP_UNLOCK(imgp->vp); + phdr = m_phdrs = malloc(hdr->e_phnum * sizeof(Elf_Phdr), + M_TEMP, M_WAITOK | M_ZERO); + vn_lock(imgp->vp, LK_SHARED | LK_RETRY); + error = vn_rdwr(UIO_READ, imgp->vp, m_phdrs, + hdr->e_phnum * sizeof(Elf_Phdr), hdr->e_phoff, + UIO_SYSSPACE, IO_NODELOCKED, imgp->td->td_ucred, + NOCRED, NULL, imgp->td); + if (error != 0) + goto ret; + } /* * Somewhat arbitrary, limit accepted max alignment for the @@ -1234,7 +1263,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) } } - brand_info = __elfN(get_brandinfo)(imgp, interp, &osrel, &fctl0); + brand_info = __elfN(get_brandinfo)(imgp, phdr, interp, &osrel, &fctl0); if (brand_info == NULL) { uprintf("ELF binary type \"%u\" not known.\n", hdr->e_ident[EI_OSABI]); @@ -1329,7 +1358,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) map = &vmspace->vm_map; maxv = sv->sv_usrstack; if ((imgp->map_flags & MAP_ASLR_STACK) == 0) - maxv -= lim_max(td, RLIMIT_STACK); + maxv -= lim_max(imgp->td, RLIMIT_STACK); if (error == 0 && mapsz >= maxv - vm_map_min(map)) { uprintf("Excessive mapping size\n"); error = ENOEXEC; @@ -1339,7 +1368,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) KASSERT((map->flags & MAP_ASLR) != 0, ("ET_DYN_ADDR_RAND but !MAP_ASLR")); error = __CONCAT(rnd_, __elfN(base))(map, - vm_map_min(map) + mapsz + lim_max(td, RLIMIT_DATA), + vm_map_min(map) + mapsz + lim_max(imgp->td, RLIMIT_DATA), /* reserve half of the address space to interpreter */ maxv / 2, maxalign, &imgp->et_dyn_addr); } @@ -1362,7 +1391,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp) * calculation is that it leaves room for the heap to grow to * its maximum allowed size. */ - addr = round_page((vm_offset_t)vmspace->vm_daddr + lim_max(td, + addr = round_page((vm_offset_t)vmspace->vm_daddr + lim_max(imgp->td, RLIMIT_DATA)); if ((map->flags & MAP_ASLR) != 0) { maxv1 = maxv / 2 + addr / 2; @@ -1438,6 +1467,7 @@ ret: ASSERT_VOP_LOCKED(imgp->vp, "skipped relock"); if (free_interp) free(interp, M_TEMP); + free(m_phdrs, M_TEMP); return (error); } @@ -2809,7 +2839,7 @@ __elfN(parse_notes)(const struct image_params *imgp, const Elf_Note *checknote, } error = vn_rdwr(UIO_READ, imgp->vp, buf, pnote->p_filesz, pnote->p_offset, UIO_SYSSPACE, IO_NODELOCKED, - curthread->td_ucred, NOCRED, NULL, curthread); + imgp->td->td_ucred, NOCRED, NULL, imgp->td); if (error != 0) { uprintf("i/o error PT_NOTE\n"); goto retf; @@ -2918,17 +2948,16 @@ note_fctl_cb(const Elf_Note *note, void *arg0, bool *res) * as for headers. */ static bool -__elfN(check_note)(struct image_params *imgp, const Elf_Brandnote *brandnote, - int32_t *osrel, bool *has_fctl0, uint32_t *fctl0) +__elfN(check_note)(struct image_params *imgp, const Elf_Phdr *phdr, + const Elf_Brandnote *brandnote, int32_t *osrel, bool *has_fctl0, + uint32_t *fctl0) { - const Elf_Phdr *phdr; const Elf_Ehdr *hdr; struct brandnote_cb_arg b_arg; struct fctl_cb_arg f_arg; int i, j; hdr = (const Elf_Ehdr *)imgp->image_header; - phdr = (const Elf_Phdr *)(imgp->image_header + hdr->e_phoff); b_arg.brandnote = brandnote; b_arg.osrel = osrel; f_arg.has_fctl0 = has_fctl0; diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 958ec559fd8d..4066682cbcc5 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -448,6 +448,7 @@ do_execve(struct thread *td, struct image_args *args, struct mac *mac_p, */ bzero(imgp, sizeof(*imgp)); imgp->proc = p; + imgp->td = td; imgp->attr = &attr; imgp->args = args; oldcred = p->p_ucred; diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c index 9be7c82ee98b..e48997ed966a 100644 --- a/sys/kern/kern_sig.c +++ b/sys/kern/kern_sig.c @@ -2038,6 +2038,16 @@ kern_sigqueue(struct thread *td, pid_t pid, int signumf, union sigval *value) if (pid <= 0) return (EINVAL); + /* + * A process in capability mode can send signals only to itself. + */ + if (pid != td->td_proc->p_pid) { + if (CAP_TRACING(td)) + ktrcapfail(CAPFAIL_SIGNAL, &signum); + if (IN_CAPABILITY_MODE(td)) + return (ECAPMODE); + } + if ((signumf & __SIGQUEUE_TID) == 0) { if ((p = pfind_any(pid)) == NULL) return (ESRCH); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index ec65bd16dd50..15a327e66c7d 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -803,6 +803,7 @@ thread_alloc(int pages) kasan_thread_alloc(td); kmsan_thread_alloc(td); cpu_thread_alloc(td); + cpu_thread_new_kstack(td); EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td); return (td); } @@ -815,7 +816,7 @@ thread_recycle(struct thread *td, int pages) vm_thread_dispose(td); if (!vm_thread_new(td, pages)) return (ENOMEM); - cpu_thread_alloc(td); + cpu_thread_new_kstack(td); } kasan_thread_alloc(td); kmsan_thread_alloc(td); diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master index b3e1d4be9fee..1032f6cd1bf0 100644 --- a/sys/kern/syscalls.master +++ b/sys/kern/syscalls.master @@ -1430,7 +1430,7 @@ _Out_ _Contains_timet_ struct ffclock_estimate *cest ); } -244 AUE_NULL STD { +244 AUE_NULL STD|CAPENABLED { int clock_nanosleep( clockid_t clock_id, int flags, diff --git a/sys/modules/Makefile b/sys/modules/Makefile index faedb856977c..f21f22c174cd 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -838,6 +838,7 @@ _nfe= nfe _nvram= nvram .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) _padlock= padlock +_padlock_rng= padlock_rng _rdrand_rng= rdrand_rng _rdseed_rng= rdseed_rng .endif @@ -911,9 +912,6 @@ _glxsb= glxsb _pcfclock= pcfclock _pst= pst _sbni= sbni -.if ${MK_CRYPT} != "no" || defined(ALL_MODULES) -_padlock_rng= padlock_rng -.endif .endif .if ${MACHINE_ARCH} == "armv7" diff --git a/sys/modules/iwlwifi/Makefile b/sys/modules/iwlwifi/Makefile index 471509c2bb1c..0212830835df 100644 --- a/sys/modules/iwlwifi/Makefile +++ b/sys/modules/iwlwifi/Makefile @@ -4,7 +4,7 @@ DEVIWLWIFIDIR= ${SRCTOP}/sys/contrib/dev/iwlwifi .PATH: ${DEVIWLWIFIDIR} -IWLWIFI_CONFIG_PM= 0 +IWLWIFI_CONFIG_PM= 1 IWLWIFI_DEBUGFS= 0 .if ${KERN_OPTS:MDEV_ACPI} IWLWIFI_CONFIG_ACPI= 1 @@ -59,6 +59,7 @@ CFLAGS+= -DCONFIG_MAC80211_DEBUGFS .if defined(IWLWIFI_CONFIG_PM) && ${IWLWIFI_CONFIG_PM} > 0 SRCS+= mvm/d3.c SRCS+= mld/d3.c +SRCS+= lkpi_iwlwifi_pm.c CFLAGS+= -DCONFIG_PM CFLAGS+= -DCONFIG_PM_SLEEP .endif diff --git a/sys/modules/linuxkpi_wlan/Makefile b/sys/modules/linuxkpi_wlan/Makefile index bafeb2d5d22a..a8dd06f06bc0 100644 --- a/sys/modules/linuxkpi_wlan/Makefile +++ b/sys/modules/linuxkpi_wlan/Makefile @@ -3,6 +3,7 @@ KMOD= linuxkpi_wlan SRCS= linux_80211.c \ linux_80211_macops.c +SRCS+= linuxkpi_80211_pm.c # QCA ath11k support. SRCS+= linux_mhi.c diff --git a/sys/modules/rtw88/Makefile b/sys/modules/rtw88/Makefile index 1978e2392da9..d9dfd5c2efb1 100644 --- a/sys/modules/rtw88/Makefile +++ b/sys/modules/rtw88/Makefile @@ -66,6 +66,7 @@ CFLAGS+= -DCONFIG_RTW88_USB .if defined(RTW88_CONFIG_PM) && ${RTW88_CONFIG_PM} > 0 SRCS+= wow.c +SRCS+= lkpi_rtw88_pm.c CFLAGS+= -DCONFIG_PM=${RTW88_CONFIG_PM} .endif diff --git a/sys/modules/rtw89/Makefile b/sys/modules/rtw89/Makefile index b7f8dc7a2c6e..682bd2ed9b53 100644 --- a/sys/modules/rtw89/Makefile +++ b/sys/modules/rtw89/Makefile @@ -54,8 +54,9 @@ SRCS+= rtw8852cu.c .endif .if defined(RTW89_CONFIG_PM) && ${RTW89_CONFIG_PM} > 0 -CFLAGS+= -DCONFIG_PM=${RTW89_CONFIG_PM} SRCS+= wow.c +SRCS+= lkpi_rtw89_pm.c +CFLAGS+= -DCONFIG_PM=${RTW89_CONFIG_PM} .endif .if defined(RTW89_DEBUGFS) && ${RTW89_DEBUGFS} > 0 diff --git a/sys/net80211/DATAPATH_RECEIVE.md b/sys/net80211/DATAPATH_RECEIVE.md new file mode 100644 index 000000000000..a930aba0c3da --- /dev/null +++ b/sys/net80211/DATAPATH_RECEIVE.md @@ -0,0 +1,160 @@ +# net80211 Datapath - Receive + +## Overview + +This document provides an overview for receive data paths in +net80211, between the interface to the operating system, through net80211 and +into the driver. + +The details about underlying implementations (eg how A-MPDU RX aggregation +is handled) will be covered in dedicated documents. + +## Concurrency Notes + +The transmit path(s), receive path and control / ioctl paths all run +in parallel and can be scheduled on multiple concurrently running +kernel threads. It's important to keep this in mind. + +## Receive Path + +### Concurrency + +There must only be one packet receive path into net80211. The net80211 stack +has not yet been fully validated to ensure that state changes all occur under +sufficient locking. + +### Data Path + +The receive path is split into three broad categories: + + * The normal 802.11/802.3 packet receive path from drivers; + * The input path for reinjected frames (eg WDS, 802.11s, BPF); + * Various side channels for offloaded non-data path (eg explicitly + scan results, management frames, etc.) + +#### Data Path - Initial Input + +The driver receive path begins in ieee80211_input.c . The four +entry points are: + + * ieee80211_input() / ieee80211_input_mimo() and + * ieee80211_input_all() / ieee80211_input_mimo_all(). + +The first two are called when the destination MAC address is a known +(struct ieee80211_node) node. These are passed up to the +VAP via a call to ni->ni_vap->iv_input(). + +The second two are called when the destination MAC address is NOT +a known node. In this instance, the frames are treated as broadcast +and routed to each VAP BSS node via a call to ieee80211_input_mimo(). + +Each VAP vap->iv_input() method handles the behavioural specific +needs of the interface. + +#### Data Path - VAP type / behaviour + +Each VAP type will do roughly the same thing - for example see +sta_input() in ieee80211_sta.c . + + * Check the frame size and protocol ID; + * Check if the frame has been decrypted in hardware; + * Grab A-MPDU session frames and put them in the reorder queue; + * Handle control frames sent to the node, or general scan frames; + * Get the frame QoS information / TID information if present; + * If appropriate, check the 802.11 receive sequence number; + * Break the handling up into data, management and control; + * Reinject into a radiotap/BPF session via a call to + ieee80211_radiotap_rx(). + +The data paths will typically do the following: + + * Do decryption if needed; + * Do 802.11 decap if needed; + * Enforce security requirements if needed; + * Eventually deliver the frame up to the higher level network + stack via a call to ieee80211_deliver_data() which will + strip away any last bits of 802.11 / net80211, + call ieee80211_vap_deliver_data(), which will call the + network stack input interface. + +The control and management paths will call vap->iv_recv_mgmt() +and vap->iv_recv_ctl() which implement the per VAP type behaviours. +These will include participating in driving the scan engine, +the per-node state machines and the VAP state machine. + +#### Reinjected Path + +#### Side Channels + +Drivers may need a specific side channel for management/control +frames, MAC layer events (eg A-MPDU aggregation session state); +some power state communication, scan information and other +things that would normally show up as 802.11 frames. + +These will be covered in more detail in other documents. + +### Receive Status and Parameters + +Received 802.11 / 802.3 frames can come with a variety of information +that isn't strictly the data payload. These include receive timestamps +(at beginning or end of frame), receive noise floor / signal strength, +channel / frequency, channel width, received rate, aggregation frame +boundaries, decryption state, etc. + +The original paths - ieee80211_input() and ieee80211_input_all() - +took a noise floor and rssi parameter. Later drivers provide +information about all of the above by attaching a (struct ieee80211_rx_stats) +to the receive mbuf via a call to ieee80211_add_rx_params() bafore +calling ieee80211_input_mimo() and ieee80211_input_mimo_all() . + +Existing drivers should be migrated to the mimo versions of these +APIs and the existing API should eventually be deprecated and +replace the mimo versions. + +All new drivers must use the ieee80211_input_mimo() and +ieee80211_input_mimo_all() API calls. + +### Driver Receive Path Requirements + +The driver receive path has a few top level requirements: + + * Driver / stack locks must not be held during receive. This means that + drivers should dequeue their frames first into a local list, release + whatever locks are needed and then pass the frames up to net80211. + + * Drivers are responsible for doing the node lookup before + calling ieee80211_input() / ieee80211_input_mimo() or + calling ieee80211_input_all() / ieee80211_input_mimo_all(). + + * Drivers are also responsible for creating and attaching the + ieee80211_rx_stats information via a call to ieee80211_add_rx_params(). + + * Drivers are responsible for tagging a frame as a potential + A_MPDU by tagging the received mbuf with the M_AMPDU flag. + They should do this by just tagging all mbufs to a node + with ni->ni_flags & IEEE80211_NODE_HT set w/ the M_AMPDU flag. + This is a holdover from the 802.11n code which enforces that + only potential AMPDU frames can be added to an A-MPDU receive + aggregation session and may be relaxed / removed in the future. + +### Driver Receive Path Methods + +Drivers can hook into the receive path processing in a variety of ways. +There are a number of vap methods that a driver can hook into +processing. The details will be covered in the driver document. + +These include: + + * vap->iv_input - the driver can replace the iv_input method + with its own method to first handle frames before they are passed + to the VAP type receive path. + * vap->iv_recv_mgmt - the driver can hook here to handle + management frames before the VAP type management receive path. + * vap->iv_recv_ctl - the driver can hook here to handle + control frames before the VAP type control receive path. + * vap->iv_bmiss - the driver can hook here to be informed of + beacon miss frames. + +These may be called at any time and overlapping with others (eg +the beacon miss event - which may be triggered by a timer - +can be called in parallel with the various receive path methods.) diff --git a/sys/net80211/DATAPATH_TRANSMIT.md b/sys/net80211/DATAPATH_TRANSMIT.md new file mode 100644 index 000000000000..b3122129d938 --- /dev/null +++ b/sys/net80211/DATAPATH_TRANSMIT.md @@ -0,0 +1,390 @@ +# net80211 Datapath - Transmit + +## Overview + +This document provides an overview for the transmit data path in +net80211, between the interface to the operating system, through net80211 and +into the driver. + +The details about underlying implementations (eg how A-MPDU RX aggregation +is handled) will be covered in dedicated documents. + +## Concurrency Notes + +The transmit path(s), receive path and control / ioctl paths all run +in parallel and can be scheduled on multiple concurrently running +kernel threads. It's important to keep this in mind. + +## Transmit Path + +There are two paths from the operating system layer into the net80211 transmit +path - the normal data path and the BPF / radiotap raw frame path. + +It is important to note that both paths have no serialisation between +them, and multiple sending paths in the OS can and will queue frames +simultaneously across multiple concurrently executing threads/CPUs. +Please keep this in mind when reading the transmit handling and +how it interacts with 802.11 sequence numbering and encryption IV. + +### Data Path - net80211 + +This is configured at the ifnet setup in ieee80211_vap_setup() - +the output path is ieee80211_vap_transmit(). This input path +takes 802.3 ethernet frames with no attached metadata (such as +rate control, transmit power, etc) - it is left up to the stack. + +This hands the packet off to ieee80211_start_pkt() which will +perform the initial 802.11 destination lookup, query the node +state (eg whether it's in power save) and the VAP state (eg +is the vap itself in power state, or in a non-RUN state) +and drop or queue the frame appropriately. + +It is then handed over to ieee80211_vap_pkt_send_dest() with +a destination ieee80211_node reference. + +ieee80211_vap_pkt_send_dest() performs the bulk of the +net80211 transmit handling. Packets will be queued here if the +destination node is in a power saving mode. + +This includes: + + * Firstly - checking if the packet needs to be queued for + power saving operation and will pass it via ieee80211_pwrsave() + if needed; + * QoS classification via a call to ieee80211_classify(); + * BPF TX tap via a call to BPF_MTAP(); + * handling 802.11 encapsulation via ieee80211_encap() if required; + * A-MPDU TX decisions, AMSDU and Atheros Fast-Frames decisions. + +At this point the packet has been 802.11 encapsulated if required, +marked as needing encryption if required, and has been optionally +fragmented into a list of 802.11 fragments. + +Finally, the packet / fragment packet chain is sent up to the driver via a call +to ieee80211_parent_xmitpkt(). The driver is expected to queue the +packet / fragment list or discard the packet / fragment list. The specific +format of the mbuf chain and how ieee80211_node references are kept +is documented in ieee80211_parent_xmitpkt(). + +#### Notes on transmit path serialisation + +Note that by default the IEEE80211_TX_LOCK() is held across the call to +ieee80211_encap() and ieee80211_parent_xmitpkt(). Drivers can register +that they properly handle 802.11 sequence number offloading via +IEEE80211_FEXT_SEQNO_OFFLOAD. The lock is to ensure that packets +queued to the driver layer are added to the driver transmit queue +in the same order that they are 802.11 encapsulated - which sets the +802.11 sequence number. Drivers which set IEEE80211_FEXT_SEQNO_OFFLOAD +indicate that they will assign the sequence number themselves - likely +at the same time that the transmit encryption IV number is assigned, +or simply offloaded in firmware - and thus this lock is not +required. + +### Data path - Driver + +The call ieee80211_parent_xmit() will call the driver ic->ic_transmit() +method. At this point the driver can choose to queue / send the frame +(and take ownership of it), or return an error, and return it back +to net80211. Currently net80211 will just free the mbuf and node reference +and return, but drivers should not assume that. + +The mbuf passed in will be either a single 802.11/802.3 frame in an mbuf, +or a list of 802.11 fragments chained by m->m_nextpkt. If the driver +has not set IEEE80211_FEXT_SEQNO_OFFLOAD then the packet will have +a sequence number assigned which the driver can fetch via M_SEQNO_GET(). +The mbuf also holds an ieee80211_node reference. + +(Note that fragments do not have sequence numbers assigned nor node +references.) + +The driver needs to do a few things with this frame. Notably if it's +an 802.3 offload device, it will be handed an 802.3 frame with no +802.11 information. In that case, the driver just needs to queue +it for send to the hardware/firmware. + +For devices which accept 802.11 frames, a few things are needed: + + * It needs to queue them for send, in the order they're given. + * If there are any reasons the frames need to be buffered in the + driver - eg node power state, asynchronous node/key/state updates - + then they'll be buffered here until needed. + * It needs to do any local hardware/firmware setup - rate control, + transmit configuration, destination queue decisions, etc. + * Hardware/firmware typically has some way to mark a frame as a type + (control, data, management), whether RTS/CTS is needed, + * If IEEE80211_FEXT_SEQNO_OFFLOAD is set in the driver, it may need to + allocate 802.11 sequence numbers via a call to ieee80211_output_seqno_assign(). + * If the frame is part of an MPDU (m->m_flags & M_AMPDU_MPDU) then + the frame may need to be handled differently. (For example rtwn(4) + leaves sequence number assignment up to the firmware when A-MPDU is + enabled.) + * If the mbuf is marked as needing encryption (IEEE80211_FC1_PROTECTED + is set in the 802.11 header) then the frame needs to be encrypted + with the current encryption state via a call to ieee80211_crypto_encap(). + * Finally, the frame is queued to the hardware/firmware. + +Again it is critical that the 802.11 sequence number and encryption be +called together in the same order. This is typically done by the TX work +being done in a lock, or all frames being pushed into a single software +TX queue. + +### Data path vs control path and the need to buffer frames + +net80211 currently treats encryption key programming, VAP state +and other updates as synchronous calls. For example, the +transmit path will call the driver to add a node, then +set the encryption keys and then queue a frame to be transmitted. + +For devices which are programmed directly with no queued operations +(such as the ath(4) devices) the encryption key and node programming +is immediate. However, for many other devices - firmware and +USB are two examples - these operations are asynchronous. +And these code paths tend to be in the transmit paths from +upper layers that may have locks held, so sleeping is not an option. + +So for now this needs to be implemented in the driver itself. +It will need to maintain a per-node queue of transmit frames; +it will need to track asynchronous node creation/updates and +encryption key updates and buffer transmit frames for a node +until the node add/update and encryption key add/update is +completed. + +### Transmit Completion Notifications + +The net80211 stack may request a completion notification +to be called when a transmit frame has completed. +This will be done via a call to ieee80211_add_callback(). +It is used in various parts of the net80211 stack to +drive the MAC state machines - for example, being notified +once an BAR (Block-ACK request) frame has completed so +the retry timer can be cancelled. + +This requires that mbufs that are transmitted with a requested +completion callback be checked and handled appropriately. +This is covered in the next section. + +### Completing and freeing transmit path mbufs + +There are two paths to freeing mbufs - ieee80211_free_mbuf() and +ieee80211_tx_complete(). + +#### Before transmit - ieee80211_free_mbuf() + +ieee80211_free_mbuf() is used in drivers and net80211 to free +a list of mbufs as part of the transmit path setup so it can +properly account for and free an 802.11 MPDU / 802.3 frame, +or a list of mbufs representing 802.11 fragments. It doesn't +handle the ieee80211_node reference as at the early stage +of transmit there is a single ieee80211_node reference +covering all of the fragments being passed to the driver +for transmit. + +If you're not supporting 802.11 fragment transmit (and you have +to register your driver with the IEEE80211_C_TXFRAG capability +to even support this) then you can ignore all of the above +and just not call ieee80211_free_mbuf() for now. + +This must not be used for receive mbufs. Yes, this is not +well named and should likely just be renamed. + +#### After transmit queueing / attempts - ieee80211_tx_complete(). + +In the general case of an transmit mbuf being completed (either +successfully or unsuccessfully) net80211 provides a call +to handle everything - ieee80211_tx_complete(). This takes +the relevant destination node (struct ieee80211_node), +the mbuf, and a status indiciating success or failure. + +A call to ieee80211_tx_complete() handles a variety of +common functions: + + * It increments the ifnet counters as appropriate; + * If the frame has a TX completion notification callback attached + it will process said callback; + * If a node is supplied then the node reference is freed + +In the past some drivers implemented the mbuf TX callback +handling themselves, resulting in some drivers supporting +callback and some drivers not supporting callbacks. The goal +here is to implement a single way for completions to be +handled. + +Note that some hardware / firmware do not support per-frame +completion / status notification. For example, USB devices +tend to not send individual notifications for frames - you +may be able to request it for specific frames, but the +status notifications are expensive. In these cases, drivers +may just call ieee80211_tx_complete() with a status based +on whether the frame was queued to the USB endpoint successfully +or not. + +#### Atheros Fast Frames / 802.11n A-MSDU transmit + +(Note this is purposely short - a larger write-up for this will be +done on a separate page.) + +The transmit path above will call ieee80211_ff_check() and +ieee80211_amsdu_check() to see if the given node/frame should be +queued for an Atheros Fast Frames MPDU or an A-MSDU. + +If the frame should be queued it will be queued locally and NULL +will be returned; if there's already a frame queued it may be +paired with a queued frame and both returned as a single mbuf / MPDU +to send. + +As far as the driver is concerned, it will be handed a single +802.11 MPDU to send. + +#### 802.11n A-MPDU transmit + +net80211 implements the A-MPDU negotiation and block-ack request/response +handling. However currently the driver must implement A-MPDU packet +queuing, buffering, submission and retransmission. + +There are some methods that the driver can override to control the +A-MPDU transmit negotiation flow (ic->ic_addba_request, ic->ic_addba_response, +ic->ic_addba_response_timeout, ic->ic_addba_stop) and the Block-Ack +response completion or error/timeout (ic->ic_bar_response). + +#### Driver queue completion + +Currently there are two things a driver should do when its own queues +are (mostly) empty: + + * When the transmit queue is empty or mostly empty, call ieee80211_ff_flush() + to flush out any pending A-MSDU / Atheros Fast Frames to be transmitted; + * When the receive queue is being handled, call ieee80211_ff_age_all() to + flush out any frames that are older than a provided time interval. + +These calls ensure that any queued frames in Fast Frames / A-MSDU queue +don't stay in there permanently. + +### Non data frame transmission (management, control, action, beacon, etc) + +Non data frames are sent via ieee80211_raw_output(). The main exception to +this is beacon frames, which are separately initialised and pulled from +net80211 into the driver by the driver specific beacon handling routines. + +Raw frames differ from data frames in a couple of ways: + + * Transmit parameters are typically sent from userland or the caller + (struct ieee80211_bpf_params \*), and + * The input path into the driver is via ic->ic_raw_xmit(), not ic->ic_transmit(). + +The driver can combine the data and non-data paths into a single path. +The main reason for keeping these separate is to cleanly support drivers +and firmware which allow 802.3 frames to be sent and received, but still +need a side channel to send and receive management frames for various other +functions. + +The raw frame output path is used by: + + * The BPF output path - ieee80211_output() ; + * The management frame output path - ieee80211_mgmt_output() ; + * The NULL data output path - ieee80211_send_nulldata() ; + * Sending probe requests - ieee80211_send_probereq() ; + * Sending probe responses - ieee80211_send_proberesp() ; + * Sending 802.11n BAR frames - ieee80211_send_bar() ; + * .. and anywhere where the individual protocol (eg 802.11s) wishes to send raw + non-data frames. + +This path is not REALLY designed for high speed data - for example, +it should work for basic packet injection, but it does not pass through +the normal functions for encryption, power save, TX aggregation and other +data specific operations. It expects to be handed a raw, already encapsulated +802.11 frame. + +Note this is not an 802.11 MPDU - this is an 802.11 frame. For example, +non-data frames may not have sequence numbers. NULL data frames have a sequence +number but that sequence number must be 0. + +Once the driver ic->ic_raw_xmit() call is made, the driver can handle the +802.11 frame in any way it sees fit. Again, it can't assume it's an 802.11 +data frame. + +### BPF path + +Control frames are injected from userland and net80211 via a raw transmit path, +separate from the data path. This dates back to the earliest Orinoco/WaveLAN +cards, where the earlier firmware only allowed 802.3 frames to be sent/received, +but later firmware introduced raw packet transmit to allow wpa_supplicant +operation. + +Packet injection begins via the BPF/radiotap input path. The code in +ieee80211_radiotap.c attaches a BPF operator to the VAP during the +call to ieee80211_radiotap_vattach(). + +Raw frames start in BPF and are queued via bpf_ieee80211_write(), which will +send the frame into the driver via a call to the VAP ifp->if_output() and then +if provided, a copy of the feedback mbuf via the VAP ifp->if_input(). + +The ifp->if_output() method by default is ieee80211_output(). The driver +can override this. This takes care of validating that it is an 802.11 +frame, extracts the (struct ieee80211_bpf_params \*) header from the +destination sockaddr passed in via BPF, finds the relevant +struct ieee80211_node \*) tx node, grabs a reference, some further sanity +checks and then calls ieee80211_raw_output(). The rest of the raw output +path is the same as net80211 sourced raw frames. + +### Power Save Management + +By default, net80211 will track legacy power-save state between IBSS nodes +and STA <-> AP nodes (ie, full node buffering via the power management bit +in the 802.11 header; TIM/ATIM bitmaps in beacons, NULL data frames to wake up) +and PS-POLL frames being sent by stations to request individual frames. + +The transmit path will pass frames destined to asleep stations to the power +save queue via a call to ieee80211_pwrsave(). + +There are a number of VAP methods for the driver to tie into if it needs to be +informed about this state (vap->iv_set_tim, vap->iv_recv_pspoll, vap->iv_node_ps). +These allow the driver to keep its own internal state in sync with net80211 +and allows it to better maintain its own transmit queue state. + +See the ath(4) driver for a comprehensive example of how these methods are used +to correctly transmit and buffer frames from an AP to STA device without packet +loss. + +### Transmit path encryption + +The net80211 stack needs to handle a variety of transmit encryption schemes +based on all the combinations that driver and firmware interfaces may require. + +In general, the transmit encryption is done in two phases: + + * In ieee80211_encap(), the transmit key is chosen via a call + to ieee80211_crypto_getucastkey() or ieee80211_crypto_getmcastkey() - the + key index is added to the 802.11 header and space is reserved between + the 802.11 header/payload and at the end for the encryption key data to be + added; + * Then when the driver transmits the frame, it calls ieee80211_crypto_encap() + to actually do the encryption. + +Some hardware will completely offload encryption, so although the key choice +is made, various driver configuration options are set to inform net80211 not +to add all the padding. Others will offload encryption but require the +space to be provided in the frame for the hardware/firmware to add the +encryption information into. + +### What is IEEE80211_F_DATAPAD ? + +This is actually to support hardware such as the Atheros 802.11abgn chips, +which have a 4 byte alignment requirement between the 802.11 header and +the data payload (including the encryption parts.) + +Yes, it likely should be a more generic option. + +### Future work + + * It would be nice to more formally define and enforce what drivers should be + doing with mbufs during the whole transmit lifecycle of an mbuf. + * Perhaps add a function or two for the drivers to use to + query whether a given mbuf has a TX notification attached (rather + than drivers querying M_TXCB) so they can individually + register for explicit notifications so they can provide more + accurate completion information. + * The fast frames age / flush routines should really be expanded to + be required functionality in net80211 drivers rather than optional + when IEEE80211_SUPPORT_SUPERG is enabled, so further software transmit + queue management is possible in net80211. + diff --git a/sys/net80211/DEBUG.md b/sys/net80211/DEBUG.md new file mode 100644 index 000000000000..2231a0992fe6 --- /dev/null +++ b/sys/net80211/DEBUG.md @@ -0,0 +1,101 @@ +# Debugging in net80211 + +This document describes how debugging is implemented in net80211. + +## Overview + +net80211 has run-time configurable debugging available. It is configured +per-VAP. It is implemented as a bitmask which can be controlled via a +sysctl at runtime. + +Debugging is compiled in when IEEE80211_DEBUG is defined. + +There is currently no global debugging API available; top-level net80211 +code is typically using printf() or some wrapper around it (eg +net80211_printf). + +The debug API is defined in (ieee80211_var.h). This includes the +debug field definitions and exported debugging API. The actual implementation +of the debugging routines is currently in (ieee80211_input.c) - see +(ieee80211_note) for an example. + +The bitmap of available debugging sections is in (ieee80211_var.h), prefixed +with IEEE80211_MSG . See (IEEE80211_MSG_DEBUG) for an example. + +## Usage + +Calls to the debugging APIs should not include a terminating '\n' character. +This will be added by the debug call. + +The simplest example is a call to IEEE80211_DPRINTF(). This takes a vap +pointer, which debug option to log to, then the format string and optional +arguments. For example: + +``` +IEEE80211_DPRINTF(vap, IEEE80211_MSG_11N, "%s: called!", __func__); +``` + +The debug flags can be combined together using bitwise OR so they are +emitted if one or more debug options are set, for example: + +``` +IEEE80211_DPRINTF(vap, IEEE80211_MSG_11N | IEEE80211_MSG_ASSOC, + "%s: called!", __func__); +``` + +There are a number of different debugging calls that are designed to be +used in different contexts. Although they all currently end up printing +to the same debug output, keeping them separate allows for future +behavioural changes whilst minimising rototilling the whole codebase (eg +allowing non-DPRINTF to turn into event tracing.) + + * Straight up debugging should be done through IEEE80211_DPRINTF() . + * Debugging that's related to a specific ieee80211_node (eg a state + change for a specific node) should be done via a call to IEEE80211_NOTE() . + * Debugging that's related to a specific ethernet MAC address (eg + scan results) should be done via a call to IEEE80211_NOTE_MAC() . + * Debugging that should include a frame header should be done via + a call to IEEE80211_NOTE_FRAME(). Note this takes a (struct ieee80211_frame \*) + pointer. + * Debugging involving discarding frames (eg invalid frames) should be + done via a call to IEEE80211_DISCARD() . + * Debugging involving discarding frames due to an invalid / bad IE should + be done via a call to IEEE80211_DISCARD_IE(). + * Debugging involving discarding frames due to a MAC address (eg ACL failure) + should be done via a call to IEEE80211_DISCARD_MAC(). + +## Usage Notes + + * It is required that the debugging be compiled in/out purely by defining or not + defining IEEE80211_DEBUG. This can often trip up unused variable warnings + when debugging is disabled, so just double-check both configurations. + + * It is important to ensure calls to the debugging (and any other logging API) + do not change any state/variables. For example, do not call a function that + updates some counter or some state variable inside a call to IEEE80211_DPRINTF(). + It won't be called at best and it will just be compiled out entirely at worst. + +## Configuration + + * The 'vap->iv_debug' field is controlled by the OS specific module. + * In FreeBSD (ieee80211_freebsd.c) it is assigned a sysctl (net.wlan.X.debug) + during (ieee80211_sysctl_vattach). + * FreeBSD ships the wlandebug(8) tool to query and set this at runtime. + +## Implementation Details + +* The debug API goes out of its way to do the debug flag check before evaluating + function parameters and potentially assembling the logging output. See + (IEEE80211_DPRINTF) for an example. + +## Future work + + * Top-level net80211 debugging APIs and control would be nice (for things + that are not specific to a VAP.) + * Drivers end up having to implement their own debugging API; it may be nice + to provide drivers a net80211 API to do their own driver specific logging. + * The debug macros should likely be refactored out to a new header file, + separate from ieee80211_var.h, so they can be more easily referenced. + * The debug fields should likely be refactored out into a new separate header + file that is designed to be consumed both by the kernel and by userland + utilities wishing to query/set the debug bitmask. diff --git a/sys/net80211/PROTOCOL.md b/sys/net80211/PROTOCOL.md new file mode 100644 index 000000000000..6d7c128bfc89 --- /dev/null +++ b/sys/net80211/PROTOCOL.md @@ -0,0 +1,563 @@ +# 802.11 protocol overview + +This is a quick overview of the 802.11 protocol and where it intersects with +net80211. It is not intended as a comprehensive deep dive into all of 802.11. + +TODO: link to appropriate sections in 802.11-2016 / 802.11-2020 depending upon +which PDF is freely available. + +## 802.11 overview + +The 802.11 protocol / specification is a very large document which covers +everything from the raw signals going out over the air up to how devices +need to behave in different operating modes. + +The IEEE specification documents and amendments describe what devices should +and must do in order to interoperate. It's important to note that the +intersection of "what the standard says" and "what devices do" is not always +fully aligned. The 802.11 specification has evolved over twenty-five years +and for the most part this allows interoperability between the original 802.11b +hardware and modern multi-band 802.11ax devices. + +It's also important to note that 802.11 is not just limited to the IEEE +specifications. 802.11 devices are almost exclusively RF devices (if you +read the specification you may find the old infrared / IR protocol definition!) +and so need to operate inside of the radio regulatory rules defined by each +country. These define a wide variety of RF environmental behaviours +including frequencies can be used, when devices can transmit, what transmit +power is allowed, interoperability with other devices (802.11 and non-802.11) +and radar interoperability. For the purposes of this document these will +be called "regulatory concerns" and will be covered elsewhere. + +The 802.11 specification breaks things up into a handful of top level areas: + + * the PHY layer - how the device interfaces with the RF environment and + encodes/decodes RF transmissions into data streams. + * the MAC layer - defines how data is packetized into individual data frames, + exchanged with the upper layer (ethernet/bridge), deciding when and what + to transmit via the PHY layer. + * MLME - (MAC layer management entities) - defines all of the state methods + and transitions that underpin the 802.11 MAC state machine. + * Security - the cipher and key management components. + * PHY specifications - the specific implementations of PHYs - 2GHz DSSS + (spread spectrum), 2GHZ CCK, OFDM, ERP, 802.11n / HT, 802.11ac / VHT, etc. + +Most 802.11 implementations do not implement a 1:1 definition of each of these +layers - notably implementing every single MLME state would be a huge amount +of work. + +## 802.11 revisions + +There have been many revisions of the 802.11 specification. The specifications +can be found online at https://www.ieee802.org/11/. + +The latest specification being implemented in net80211 is 802.11-2020, however +net80211 is far from completely compliant. Generally new code which implements +802.11 features / protocol handling should identify the specification and +section which it is referencing. + +## 802.11 protocol and frame definitions + +net80211 keeps most 802.11 frame and protocol definitions in a single location +(ieee80211.h). +This contains descriptions of the 802.11 frame and field definitions, ranging +from the lowest definition of the frame itself up through frame types/subtypes, +individual field definitions, information elements, action frames, and +anything else that can be found in the 802.11 specifications. + +The PHY definitions can be found in (ieee80211_phy.c) and (ieee80211_phy.h). +Notably those include the frame timing information useful for rate control +and frame duration calculations. + +## 802.11 Revisions + +(TBD) + +### Legacy 802.11 + +The earliest 802.11 devices implement 1Mbit/s and 2Mbit/s direct spread spectrum +frames. These include the earliest Wavelan devices. These are grandfathered +into 802.11b. The PHY specification can be found in 802.11-2020 Section 15.) + +### 802.11b + +802.11b devices implement Section 15 (1Mbit/2Mbit) PHYs as well as the high +rate DSSS specification (802.11-2020 Section 16) to provide 5.5Mbit and 11Mbit +CCK rates. They interoperate with legacy 802.11 devices by using compatible +PHY encodings and will limit their performance if said legacy devices are +detected. + +### 802.11a + +802.11a devices implement OFDM rates from 6Mbit/s to 54Mbit/s on the 5GHz +band. Among other features, it also defines 5MHz and 10MHz wide channel +behaviour. This is covered in the OFDM PHY specification (802.11-2020 +Section 17.) + +### 802.11g + +802.11g devices implement OFDM rates from 802.11a, the CCK rates from 802.11b +and the DSSS rates from legacy 802.11. These are covered in the ERP +specification (802.11-2020 Section 18.) There are some MAC extensions for +negotiating 802.11b / 802.11g interoperability and these are documented +throughout the MAC specification. This also specifies support for 5MHz and +10MHz wide channels. + +### 802.11n (HT) + +802.11n introduced a variety of high throughput rates and feature support +(hence why it's called HT - high throughput). It introduces higher density +OFDM rate encodings, 20 and 40MHz wide channels with interoperability for +earlier devices, packet aggregation via A-MPDU and A-MSDU, MIMO (multiple input, +multiple output spatial streams), some initial beamforming support, power +saving extensions and more. + +The physical layer support is covered in the HT PHY specification (802.11-2020 +Section 19.) The rest of the MAC extensions are documented throughout the +rest of the specification. + +### 802.11ac (VHT) + +802.11ac extends the 802.11n specification (hence why it's VHT - Very +High Throughput) and boosts performance by adding higher density OFDM QAM +encoding (256-QAM), wider channels (80MHz, 160MHz), split 80+80MHz channel +support, much larger A-MSDU / A-MPDU frame sizes, support for MU-MIMO +(multi-user MIMO) allowing APs to transmit to multiple STAs at the same time +and various other extensions. + +It builds on top of the 802.11n MAC and PHY specification, so a lot of +802.11n feature and MAC negotiation happens as part of 802.11ac negotiation. + +The PHY layer is covered in the VHT PHY Specification (802.11-2020 Section +21.) Again, the rest of the MAC extensions are documented throughout the +rest of the specification. + +### Greenfield versus backwards compatibility + +The various protocols supported by 802.11 build on top of earlier protocols. +So typically you're not building a single implementation for each protocol - +for example, you can't handle 802.11ac support without implementing a large +amount of 802.11n support. + +(As a side note, the 802.11 frame has a protocol version field, and +that actually changed in 802.11ah (900MHz and longer distance bands) - +which changes a lot of what the fields do. No, net80211 currently does not +support 802.11ah and will drop frames whose 802.11 protocol ID is not +supported.) + +At the PHY layer, later model hardware can transmit data encodings which +earlier model hardware just won't recognise. All they'll see is an increase +in RF power on the channel at best and signals that will confuse the +RX decoder / cause hardware issues at worst.) + +So each of the PHY specifications will lay out a few things: + + * How frames should be encoded in the air in a way that earlier + hardware can decode them enough to know it's not for them; + * How devices can identify that earlier protocol devices are around and + change the configuration (eg STA changing its own configuration, + AP changing the configuration of the network it controls, etc) + to provide backwards compatibility. + +These come at a performance cost. For example, an 802.11g AP which +supports 802.11b and 802.11 devices needs to notice that an 802.11b +device wishes to associate, and when it sees this, change some of +its configuration (notably "long preamble" so 802.11b devices can +decode frames that are being transmitted, whether destined to it or not.) + +Various devices allow backwards compatbility to be configured. +For example, an 802.11n AP may be configured to deny non-802.11n clients. +This may improve performance but then earlier clients can't connect. + +In 802.11n deployments this was known as a "greenfield deployment". +This typically disables any and all pre-11n interoperability at both +a MAC and PHY layer. net80211 has some flags for this to specifically +inform devices that they can configure the hardware for such a setup. +Not all drivers implement it however, and in a lot of cases they will +still handle pre-11n framing, even if the net80211 code will deny +association. + +There are other components to backwards compatibility which are worth +keeping in mind when reading through the 802.11 specification and +net80211 stack / driver code. These include: + + * short/long preamble - (vap_update_preamble) + * short/long slot time configuration - (vap_update_slot) + * 802.11g protection mode (vap_update_erp_protmode) - + whether to use CTS-to-self around each transmission + * 802.11n protection mode (vap_update_ht_protmode) - + whether to use RTS/CTS around each transmission + * 802.11n 20/40MHz BSS operation (whether an 802.11n AP sees other APs that + overlap its frequency range and need to reconfigure how to protect + transmissions) + +## How 802.11 (very briefly) works over the air + +This is a very brief and not at all comprehensive overview of how 802.11 +works over the air. The goal of this section is to provide enough background +information to help de-mystify reading the net80211 stack and wireless +driver source. + +### Why there's timing requirements in the first place + +Each of the PHY sections in the 802.11 specification describe what +the PHY needs to do in order to transmit and receive data. It's not +anywhere as easy as "toggle some bits on a wire". + +An important thing to understand is that hardware isn't immediate. +All the state machines in your 802.11 devices take non-zero time +to make decisions about when to transmit, when to receive, locking +onto a signal, deciding it can be decoded, getting reset for the next +frame, etc. + +So a lot of what you'll see in 802.11 negotiation and feature support +is linked to the underlying hardware implementations and limitations +of the time. For example the 802.11b specification defines the slot time +as 20uS, but the 802.11g specification lowers it to 9uS. The "slot time" +value defines the unit of time used for contention management / backoff, and +it's defined partly by what the speed of light dictates (ie how big +of a physical area you want to be able to "hear" in determining if the +area is busy) and how quickly the hardware can guarantee to respond. +It dropped to 9uS because hardware got better, but to interoperate +with older devices without starting to transmit before they're +ready to react, 802.11g devices will fall back to 20uS slot time when +they detect an 802.11b device. + +This carries through everywhere in odd places that you're not necessarily +aware of. For example, the 802.11n A-MPDU definition includes negotiated +padding between frames and limits encryption ciphers (typically CCMP or +GCMP.) This is due to hardware support - the MAC may be able to support +much less padding when no encryption is used, but setting up / resetting +the encryption / decryption blocks may take more time and thus larger +A-MPDU padding values are negotiated. + +### Wait, the speed of light? + +Yes. The speed of light is roughly 300 metres for each microsecond of +travel time. + +### Preambles, SIGs, PLCP, sending actual data and waiting / slot times + +There are a few things that are worth understanding at a high level. + + * The first thing that a device needs to do is determine + whether the air is busy or free. There'll be some hardware + to determine the signal level versus noise floor and provide + a signal to the transmit hardware that the air is free, + and to the receiver that it may want to try start decoding + something. + + * The receiver needs to get in synchronisation with the transmitter. + This is a one way operation - the transmitter needs to transmit + enough of a signal that the receiver can "lock onto" and get itself + ready for further data. This is called the "preamble" - it's + typically a low bitrate repeating pattern of data that gives + the receiver hardware time to lock onto, figure out the signal + level and be ready for the next phase. + + * Note that the receiver may pick up the preamble at any point in its + transmission so it can't guarantee it will see exactly "x" bits of some + repeating pattern. + + * Then there's other bits and pieces - eg look for L-SIG, HT-SIG + in the PHY documentation - which is used to further synchronise + what's about to happen. + + * Finally it will start transmitting the PHY framing bits needed to + identify what the upcoming transmit rate and configuration is + (all the stuff leading up to the PLCP header, then the PLCP header.) + +Things get more complicated with MIMO, MU-MIMO, 802.11ax OFDM-A, etc but +don't worry about those for now - they build on top of all of these +ideas. + +Once the data is transmitted, there's some quiet time between frames +before the receiver can ACK (and then a period of time where an ACK +is expected.) The transmitter needs to finish transmitting, then +reset its internal state back to idle to be ready to receive - and +there's the pesky speed of light speed of 300m per microsecond - +so there's some MAC (interframe spacing) and PHY (slot time) enforcing +quiet so everyone has a chance to receive the frame and the reciver +gets ready to receive. Then if there's an ACK, the ACK happens. + +### PLCP header + +Once all of the preamble, SIG/training stuff is done, the transmitter +will send a PLCP header with information about the transmitter +type and rate (and that's very handwaving it.) net80211 has definitions +for the plcp header (ieee80211_plcp_hdr) but it's highly unlikely it will +be relevant or available in modern devices. + +### How data is encoded - encoding rates, symbols, guard intervals + +Now, once the transmitter has sent all of that, it will start to send +actual data encoded at the desired transmit rate. The data bits +that you're transmitting go through a variety of encoding schemes +before they're turned into bits that are clocked out at the 802.11 +physical layer (think "forward error correction" as an example), +but they're turned into what are known as "symbols". + +A symbol can be thought of as a group of bits encoded in one specific +RF representation. Explaining all the details isn't in scope of this +document (and I encourage interested parties to do a quick dive +into information theory!) but there are a couple of important higher +level concepts to understand here that influence what happens +later on in packet delivery. + +For OFDM encoding: + + * Each symbol is preceded by a quiet time called a guard interval, to make + sure any reflections don't interfere with the upcoming symbol; + * Each symbol is then transmitted for a specific length of time to make sure + it's received by everyone inside the desired area (again light = 300m/sec + per microsecond); + * All symbols for a given 802.11 MPDU are sent at the exact same rate; + * This is repeated until all the symbols are transmitted. + +The higher the data rate used, the higher the signal level needs to be +and lower the tolerance it has to interference. Forward error correction +can only do so much, and the higher throughput rate encodings sacrifice +FEC for throughput. + +Once an uncorrectable error occurs and the frame fails CRC, the whole MPDU is +dropped by the receiver. + +Part of why A-MPDU is so important for high throughput is that the +errors are limited to a single MPDU in the burst of MPDUs. Ie, if +the transmitter sends ten MPDUs in a single A-MPDU, and five of them +have uncorrectable errors, then five .. well, didn't. This means +the receiver can ACK some but not all of the MPDUs, and the transmitter +can re-send those with new MPDUs. + +The default guard interval is 800ns. 802.11n allows for negotiating +shorter guard interval (400ns) which can be done per device in a BSS. +An 800ns guard interval is a little short of 300 metres, and 400ns is +a little short of 150 metres - so using short guard interval means +you trade increased performance for potential decreased performance +if you have reflections or stations more than 150 metres away. + +802.11ax adds support for 1.6us and 3.2us guard intervals for physically +larger deployments. + +### MAC layer framing + +The MAC layer handles data that is encapsulated in the given transmit +rate that was established in the PHY (PLCP) header. This includes +the 802.11 MAC header, CRC trailer, any of the cipher processing that +happens in between. In the case of 802.11n, it can encapsulate +multiple frames being sent back to back in a single transmission. + +Devices which do partial / no offload will typically produce and +consume 802.11 MAC layer frames to the driver and net80211. +It's thus important to understand MAC framing and frame types. + +### MPDU versus MSDU + +An MSDU (MAC service data unit) is an individual frame (think "802.2/802.3 +ethernet") passed from the network stack into net80211. + +An MPDU (MAC protocol data unit) is one or more MSDU frames wrapped by an +(ieee80211_frame) header and CRC trailer. It is what is eventually +encapsulated inside the PHY framing (preambles, training symbols, PLCP +header, etc) and sent over the air. + +Notably an 802.11 MPDU isn't just an IPv4/IPv6 frame with an 802.11 +header/trailer - it is a full ethernet frame that is being wrapped +by 802.11 framing. + +### Tracking airtime with NAV + +802.11 devices have to interoperate in a shared medium. Earlier protocol +definitions require one transmitter at a time. Later specification +devices (MU-MIMO with 802.11ac, OFDM-A with 802.11ax, etc) introduce the +ability for devices to transmit and receive simultaneously. + +The simplest way to track this is with NAV (network access vector.) +The NAV implementation in pre-11ax devices is a single counter which +counts down to zero. Once it is zero, the air is considered "available" +to attempt to check to transmit on. The transmitter will also check +whether the air is busy (ie can it detect any signals present) before +it transmits - this is called CCA (clear channel access) and is +typically implemented in hardware. + +The duration field in (ieee80211_frame) is a microsecond field which +covers the whole duration of the frame being transmitted. Receivers +that decode the frame - even if it's not destined to them! - will listen +to the NAV and add it to their own NAV. + +All 802.11 frames have a duration field. + +### Fragmented frames + +(TBD) + +### Sequence counters and duplication detection + +(TBD) + +### EDCA and QoS + +(TBD) + +### Inter-frame spacing (IFS) + +(TBD) + +## 802.11 frame layout + +An individual 802.11 frame contains frame control (version, type, subtype), +duration, addressing, sequence number and optional QoS information. +The basic definition is available at (ieee80211_frame) but other definitions +are also possible - (ieee80211_qosframe), (ieee80211_frame_addr4), +(ieee80211_qosframe_addr4). + +It then has a 4 byte CRC32 trailer appended at the end. + +### Addressing types and traffic direction + +(TBD - 3addr, 4addr, each of the fields, etc) + +### QoS versus non-QoS frames + +(TBD) + +### RTS/CTS exchange and airtime + +(TBD) + +### CTS-to-self / OFDM protection + +CTS to self is a concept introduced in 802.11g. The general idea is that a +transmitter can send a CTS to its own MAC address for the duration that it +wishes to transmit for. Since the CTS frame is transmitted at a slower +legacy rate, it both reserves airtime in any receiver in earshot, and +it is understood by older 11b only devices which do not understand 11g. + +This ends up also being useful for 11n, 11ac etc to interoperate with +earlier devices, but they typically rely on a normal RTS exchange. + +### Data frames + +(TBD) + +### Management frames + +(TBD) + +### Control frames + +(TBD) + +Notably, control frames do not have a sequence number and so can't be +de-duplicated. + +### Action frames + +(TBD) + +## Frame combinations + +There are various ways that 802.11 frames are combined together to improve +performance. + +### ACK, Delayed Block-ACK, Immediate Block-ACK + +(TBD) + +### Atheros Fast Frames + +(TBD) + +### A-MSDU + +(TBD) + +### A-MPDU + +(TBD) + +## Security / Encryption + +This is a much larger topic, however it's worth touching on the basics here to +understand how frames are redirected into the security/encryption paths in +net80211 and what devices may do with said frames. + +### WEP, IV header and keys + +WEP is an obsolete encryption method dating back to the earliest 802.11b +specifications. It involves a 4 byte header which includes + + * a 24 bit IV (initialisation vector); + * a 2 bit field indication which of four keys to use; + * A CRC at the end. + +It's relevant today because later cipher frame formats still use the IV +header - they're just extended to include more information. Notably, the +four key indexes are typically implemented and used in hardware, and have +different meanings depending upon the kind of traffic being handled. + +### WPA/WPA2 management + +This is handled in userland. The 802.11 specification covers everything +involved in key exchange and management but it's out of scope for this 802.11 +overview documentation. + +### CCMP, GCMP, TKIP frames + +These later ciphers still use the WEP header, but they then add extra bytes +to it to include the larger sequence number space, other options needed +for said ciphers, and a larger trailer for CRC and TKIP MIC. + +### IV duplication / tracking + +net80211 tracks the received IV / sequence number for each station indexed +by QoS TID. Anything with an earlier IV is discarded as a stale packet or +potential replay attack. See the ni_txseqs[] and ni_rxseqs[] field in +(ieee80211_node). + +Note that the 802.11 layer sequence number field will apply /first/. Traffic +which the 802.11 input layer thinks is old or retransmits will be discarded +before handed to the net80211 crypto routines. + +### Unicast vs Group Keys + +WEP has four global keys which are shared between all devices wishing to +communicate. The keys are provided in the WEP header. + +However for later ciphers the four key indexes start taking on new meanings. +Notably key index 0 is the "unicast key" which handles traffic for a given +station and is unique for that station, and keys 2 and 3 are used for +group keys - shared keys for broadcast traffic that all stations need to be +able to decrypt. + +(key 1 is also used for unicast station traffic for seamless station key +updating, but net80211 currently doesn't support this extension/feature.) + +There's also upcoming work for encrypted management traffic and encrypted +beacons which reuse the key indexes for their traffic, but then don't treat +them as "global keys" - they start being treated as "global keys but only +for this traffic type." + +It's important to understand the difference between global keys (WEP) versus +group and unicast keys (everything else) when looking through the net80211 +data and encryption handling paths. + +## 802.11 Operating Modes + +(TBD) + +### Station + +(TBD) + +### Access Point + +(TBD) + +### IBSS / Ad-Hoc + +(TBD) + +### Mesh / 802.11s + +(TBD) diff --git a/sys/net80211/README.md b/sys/net80211/README.md new file mode 100644 index 000000000000..be704185f43f --- /dev/null +++ b/sys/net80211/README.md @@ -0,0 +1,139 @@ +# net80211 + +This is the 802.11 wireless stack for FreeBSD. + +## Introduction + +The net80211 subsystem implements the 802.11 protocol and support infrastructure. +It supports a variety of device types, 802.11 protocols, operating modes and +security extensions. + +net80211 handles the 802.11 state machine, interface management, node management, +virtual interfaces, packet encapsulation and de-encapsulation and basic +security key management. + +The userland ioctl() API provides control mechanisms for the above and is how +management tooling (ifconfig, libifconfig) and management services (hostapd, +wpa_supplicant) interfaces with the net80211 stack. + +The security state machine and key management (802.1x, WPA, etc) are handled +by management services. + +Drivers can implement as much or as little of the 802.11 infrastructure as +needed. net80211 support drivers from full-offload (ie, supplying ethernet +encapsulated/de-encapsulated frames with management control via driver +methods) down to fully software controlled devices (ie, the hardware +is minimal and all 802.11 packet handling, state machine, reordering, security, +etc is handled by net80211.) + +## Overview + +net80211 consists of a few top level design modules: + + * The 802.11 device representation and functions (ieee80211com), used + in conjunction with an 802.11 device driver to represent the physical device. + + * The 802.11 virtual interface representation and functions (ieee80211vap), + used to represent instances of virtual interfaces. + + * A representation of 802.11 stations/nodes (ieee80211_node), which + keep the state of each 802.11 station/node that the stack knows about. + + * Encryption handling (ieee80211_crypto), handling 802.11 frame encryption, + decryption and session/state tracking. + + * Regulatory domain (ieee80211_regdomain), which implements the 802.11 + regulatory domains, allowed frequencies, operating modes and transmit power. + + * Radar detection (ieee80211_dfs.c), tracking the state of radar detection and + interoperability in the 5GHz frequency range. + + * Transmit rate control (ieee80211_ratectl.c) implements software and + firmware based rate control for devices that don't implement full rate control + offload. + + * Power save support (ieee82011_power.c) implements various power saving + mode features and support for devices which do not fully implement offloaded + power management. + + * Operating system specific interfaces (ieee80211_freebsd.c) which implement + the bulk of the operating system specific glue (logging, memory allocation, + network interfaces, etc.) + + * The configuration interface (ieee80211_ioctl.c) which implements the ioctl + API used by userland to configure and monitor the state of the 802.11 stack + and devices. + +In addition, each operating mode (adhoc, station, AP, WDS, mesh) have their own +modules that implement the state machines and functionality required for 802.11. + +## Portability + +Although net80211 attempts to keep most OS specific components in a single file +(ieee80211_freebsd.c), it is not currently perfect. + +Notably: + + * There are still plenty of FreeBSD-isms located throughout the source code, + including BSD specific includes, network APIs, etc. + + * The interface and networking model is still very BSD, including using the + system implementation of mbufs. + +When developing for net80211 please keep in mind that other operating systems +(such as DragonflyBSD) and third parties do leverage this codebase. +Try to keep all FreeBSD specific components in ieee80211_freebsd.[ch]. + +## Protocol Overview + +A basic protocol overview is available at (@ref md_net80211_PROTOCOL). + +The most comprehensive overview is the 802.11 protocol document itself, +but it is very large and implementations do not always correspond 1:1 +with the protocol definitions. + +## Functional Overview + +(TODO) + + * Module layout + * Logging + * Debugging - (@ref md_net80211_DEBUG) + * Top-level device layout (ieee80211com) + * Data / Control Path Overview (@ref md_net80211_DATAPATH_TRANSMIT), (@ref md_net80211_DATAPATH_RECEIVE) + * Deferred work + * Regulatory + * Virtual interfaces + * Operating Modes + * Nodes + * Node tables, node table iteration + * Device and VAP states + * Node states + * Operating modes + * Cipher management + * Radar detection + * ioctl interface + * ACL support + * Scanning, Scan Modules + * Power Management + * Transmit Path + * Receive Path + * A-MSDU Fast Frames + * A-MPDU + * Radiotap + * Monitor Mode + +## Driver Overview + +(TODO) + + * Introduction + * Driver Structure + * Setup and Attach + * Virtual Interfaces + * Control Path + * Data Path + * VAP state + * Device State + * Suspend and Resume + diff --git a/sys/net80211/ieee80211_ht.c b/sys/net80211/ieee80211_ht.c index 9b2b0df33785..8ebfbbae6377 100644 --- a/sys/net80211/ieee80211_ht.c +++ b/sys/net80211/ieee80211_ht.c @@ -2899,7 +2899,7 @@ bar_timeout(void *arg) if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) == 0) return; /* XXX ? */ - if (tap->txa_attempts >= ieee80211_bar_maxtries) { + if (ieee80211_ht_check_bar_exceed_retry_count(ni, tap->txa_attempts)) { struct ieee80211com *ic = ni->ni_ic; ni->ni_vap->iv_stats.is_ampdu_bar_tx_fail++; @@ -3845,3 +3845,21 @@ ieee80211_ht_check_tx_ht40(const struct ieee80211_node *ni) IEEE80211_IS_CHAN_HT40(ni->ni_chan) && (ni->ni_chw == NET80211_STA_RX_BW_40)); } + +/** + * @brief Return whether the given BAR retry count exceeds the configured count + * + * @param ni ieee80211_node to check against + * @param count BAR retry count + * @returns true if the count has exceeded the configured count, false if not + */ +bool +ieee80211_ht_check_bar_exceed_retry_count(const struct ieee80211_node *ni __unused, + int count) +{ + /* + * Note: ni isn't used here because the BAR limit is currently + * global. It's here for future work. + */ + return (count >= ieee80211_bar_maxtries); +} diff --git a/sys/net80211/ieee80211_ht.h b/sys/net80211/ieee80211_ht.h index c31bb8700289..505b39628dcf 100644 --- a/sys/net80211/ieee80211_ht.h +++ b/sys/net80211/ieee80211_ht.h @@ -247,5 +247,7 @@ bool ieee80211_ht_check_tx_shortgi_20(const struct ieee80211_node *ni); bool ieee80211_ht_check_tx_shortgi_40(const struct ieee80211_node *ni); bool ieee80211_ht_check_tx_ht40(const struct ieee80211_node *ni); bool ieee80211_ht_check_tx_ht(const struct ieee80211_node *ht); +bool ieee80211_ht_check_bar_exceed_retry_count(const struct ieee80211_node *, + int); #endif /* _NET80211_IEEE80211_HT_H_ */ diff --git a/sys/net80211/ieee80211_var.h b/sys/net80211/ieee80211_var.h index 2f11fa7c887f..f38489174986 100644 --- a/sys/net80211/ieee80211_var.h +++ b/sys/net80211/ieee80211_var.h @@ -134,8 +134,11 @@ struct ieee80211_frame; struct net80211dump_methods; +/** + * @brief ieee80211com - the top level driver / hardware instance. + */ struct ieee80211com { - void *ic_softc; /* driver softc */ + void *ic_softc; /**< pointer to driver softc */ const char *ic_name; /* usually device name */ ieee80211_com_lock_t ic_comlock; /* state update lock */ ieee80211_tx_lock_t ic_txlock; /* ic/vap TX lock */ @@ -176,7 +179,15 @@ struct ieee80211com { uint8_t ic_allmulti; /* vap's needing all multicast*/ uint8_t ic_nrunning; /* vap's marked running */ uint8_t ic_curmode; /* current mode */ + /** + * @brief Driver assigned MAC address. + * + * Drivers are required to populate ic_macaddr at attach time with + * the MAC address of the device. It is then used as the base for + * MAC addresses created for VAP interfaces. + */ uint8_t ic_macaddr[IEEE80211_ADDR_LEN]; + uint16_t ic_bintval; /* beacon interval */ uint16_t ic_lintval; /* listen interval */ uint16_t ic_holdover; /* PM hold over duration */ @@ -283,7 +294,19 @@ struct ieee80211com { struct ieee80211_regdomain *, int, struct ieee80211_channel []); - int (*ic_set_quiet)(struct ieee80211_node *, + /** + * @brief Handle the quiet time information element configuration. + * + * This allows drivers/modules to tie into the quiet time IE + * for controlling the transmit duty cycle. This may be required + * for more accurate radar detection. + * + * @param ni The ieee80211_node which transmitted the IE (eg in a scan) + * or if unknown, the BSSID node + * @param quiet_elm the quiet time element contents to parse/handle + * @returns 0 for OK, non-zero with errno (eg ENOSYS) + */ + int (*ic_set_quiet)(struct ieee80211_node *ni, u_int8_t *quiet_elm); /* regular transmit */ @@ -296,7 +319,7 @@ struct ieee80211com { int (*ic_raw_xmit)(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); - /* update device state for 802.11 slot time change */ + /** update device state for 802.11 slot time change */ void (*ic_updateslot)(struct ieee80211com *); /* handle multicast state changes */ void (*ic_update_mcast)(struct ieee80211com *); diff --git a/sys/net80211/ieee80211_vht.c b/sys/net80211/ieee80211_vht.c index 095c4108c768..696e2e54f563 100644 --- a/sys/net80211/ieee80211_vht.c +++ b/sys/net80211/ieee80211_vht.c @@ -235,10 +235,16 @@ ieee80211_vht_node_cleanup(struct ieee80211_node *ni) bzero(&ni->ni_vht_mcsinfo, sizeof(struct ieee80211_vht_mcs_info)); } -/* - * Parse an 802.11ac VHT operation IE. +/** + * @brief Parse an 802.11ac VHT operation IE. + * + * This parses the VHT operation IE (channel width, basic MCS set) + * into the given ieee80211_node . * * 802.11-2020 9.4.2.158 (VHT Operation element) + * + * @param ni ieee80211_node to parse VHT operation IE into + * @param ie The VHT operation IE to parse, 802.11 endian */ void ieee80211_parse_vhtopmode(struct ieee80211_node *ni, const uint8_t *ie) @@ -257,10 +263,16 @@ ieee80211_parse_vhtopmode(struct ieee80211_node *ni, const uint8_t *ie) #endif } -/* - * Parse an 802.11ac VHT capability IE. +/** + * @brief Parse an 802.11ac VHT capability IE. + * + * Parse the VHT capability IE into the node vht fields + * (ni->ni_vht_mcsinfo, ni->ni_vhtcap). * * 802.11-2020 9.4.2.157 (VHT Capabilities element) + * + * @param ni ieee80211_node to parse VHT info into + * @param ie VHT capability IE to parse, 802.11 endian */ void ieee80211_parse_vhtcap(struct ieee80211_node *ni, const uint8_t *ie) @@ -371,8 +383,8 @@ ieee80211_vht_node_leave(struct ieee80211_node *ni) "%s: called", __func__); } -/* - * Calculate the VHTCAP IE for a given node. +/** + * @brief Calculate the VHTCAP IE for a given node. * * This includes calculating the capability intersection based on the * current operating mode and intersection of the TX/RX MCS maps. @@ -390,7 +402,9 @@ ieee80211_vht_node_leave(struct ieee80211_node *ni) * TODO: investigate what we should negotiate for MU-MIMO beamforming * options. * - * opmode is '1' for "vhtcap as if I'm a STA", 0 otherwise. + * @param ni ieee80211_node to check + * @param vhtcap ieee80211_vht_cap to populate (in host order). + * @param opmode is '1' for "vhtcap as if I'm a STA", 0 otherwise. */ void ieee80211_vht_get_vhtcap_ie(struct ieee80211_node *ni, @@ -715,14 +729,22 @@ ieee80211_vht_get_vhtcap_ie(struct ieee80211_node *ni, } } -/* - * Add a VHTCAP field. +/** + * @brief Add a VHTCAP field. * * If in station mode, we announce what we would like our * desired configuration to be. * * Else, we announce our capabilities based on our current * configuration. + * + * TODO: This assumes that the passed in buffer has enough space for + * the VHT capabilitity IE and that seems error prone. + * + * @param frm buffer to start populating the IE into + * @param ni ieee80211_node to fetch the VHT capability from + * @returns a pointer to the first byte in the buffer after the newly + * populated IE */ uint8_t * ieee80211_add_vhtcap(uint8_t *frm, struct ieee80211_node *ni) @@ -932,8 +954,14 @@ ieee80211_vht_get_vhtinfo_ie(struct ieee80211_node *ni, net80211_vap_printf(ni->ni_vap, "%s: called; TODO!\n", __func__); } -/* - * Return true if VHT rates can be used for the given node. +/** + * @brief Check if VHT rates can be used for the given node. + * + * This returns true if any VHT rates can be used to transmit + * to the given node. + * + * @param ni ieee80211_node to check + * @returns True if any VHT rates can be transmitted to the given node */ bool ieee80211_vht_check_tx_vht(const struct ieee80211_node *ni) @@ -954,11 +982,14 @@ ieee80211_vht_check_tx_vht(const struct ieee80211_node *ni) return (IEEE80211_IS_CHAN_VHT(ni->ni_chan)); } -/* - * Return true if VHT40 rates can be transmitted to the given node. +/** + * @brief Check if VHT40 rates can be transmitted to the given node. * * This verifies that the BSS is VHT40 capable and the current * node channel width is 40MHz. + * + * @param ni ieee80211_node to check + * @returns True if 40MHz VHT rates can be transmitted to the given node */ static bool ieee80211_vht_check_tx_vht40(const struct ieee80211_node *ni) @@ -977,11 +1008,14 @@ ieee80211_vht_check_tx_vht40(const struct ieee80211_node *ni) (ni->ni_chw == NET80211_STA_RX_BW_40)); } -/* - * Return true if VHT80 rates can be transmitted to the given node. +/** + * @brief Check if VHT80 rates can be transmitted to the given node. * * This verifies that the BSS is VHT80 capable and the current * node channel width is 80MHz. + * + * @param ni ieee80211_node to check + * @returns True if 80MHz VHT rates can be transmitted to the given node */ static bool ieee80211_vht_check_tx_vht80(const struct ieee80211_node *ni) @@ -1006,11 +1040,14 @@ ieee80211_vht_check_tx_vht80(const struct ieee80211_node *ni) (ni->ni_chw != NET80211_STA_RX_BW_20)); } -/* - * Return true if VHT 160 rates can be transmitted to the given node. +/** + * @brief Check if VHT 160 rates can be transmitted to the given node. * * This verifies that the BSS is VHT80+80 or VHT160 capable and the current * node channel width is 80+80MHz or 160MHz. + * + * @param ni ieee80211_node to check + * @returns True if 160MHz VHT rates can be transmitted to the given node */ static bool ieee80211_vht_check_tx_vht160(const struct ieee80211_node *ni) diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c index 66d2c610139f..84b175b42eec 100644 --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -2805,7 +2805,6 @@ in_pcbrehash(struct inpcb *inp) struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; struct inpcbhead *head; uint32_t hash; - bool connected; INP_WLOCK_ASSERT(inp); INP_HASH_WLOCK_ASSERT(pcbinfo); @@ -2815,34 +2814,24 @@ in_pcbrehash(struct inpcb *inp) #ifdef INET6 if (inp->inp_vflag & INP_IPV6) { + MPASS(!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)); hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask); - connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr); } else #endif { + MPASS(!in_nullhost(inp->inp_faddr)); hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask); - connected = !in_nullhost(inp->inp_faddr); } /* See the comment in in_pcbinshash(). */ - if (connected && (inp->inp_flags & INP_INLBGROUP) != 0) + if ((inp->inp_flags & INP_INLBGROUP) != 0) in_pcbremlbgrouphash(inp); - /* - * When rehashing, the caller must ensure that either the new or the old - * foreign address was unspecified. - */ - if (connected) { - CK_LIST_REMOVE(inp, inp_hash_wild); - head = &pcbinfo->ipi_hash_exact[hash]; - CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact); - } else { - CK_LIST_REMOVE(inp, inp_hash_exact); - head = &pcbinfo->ipi_hash_wild[hash]; - CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild); - } + CK_LIST_REMOVE(inp, inp_hash_wild); + head = &pcbinfo->ipi_hash_exact[hash]; + CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact); } void diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c index 39bc9de6ec9f..390fdd9368b6 100644 --- a/sys/netinet/ip_divert.c +++ b/sys/netinet/ip_divert.c @@ -36,6 +36,7 @@ #include <sys/param.h> #include <sys/ck.h> #include <sys/eventhandler.h> +#include <sys/hash.h> #include <sys/kernel.h> #include <sys/lock.h> #include <sys/malloc.h> @@ -49,12 +50,14 @@ #include <sys/socket.h> #include <sys/socketvar.h> #include <sys/sysctl.h> -#include <net/vnet.h> + +#include <machine/atomic.h> #include <net/if.h> #include <net/if_var.h> #include <net/if_private.h> #include <net/netisr.h> +#include <net/vnet.h> #include <netinet/in.h> #include <netinet/in_pcb.h> @@ -88,7 +91,7 @@ */ #define DIVHASHSIZE (1 << 3) /* 8 entries, one cache line. */ #define DIVHASH(port) (port % DIVHASHSIZE) -#define DCBHASH(dcb) ((dcb)->dcb_port % DIVHASHSIZE) +#define DCBHASH(dcb) (DIVHASH((dcb)->dcb_port)) /* * Divert sockets work in conjunction with ipfw or other packet filters, @@ -147,10 +150,22 @@ struct divcb { struct epoch_context dcb_epochctx; }; +struct divcblbgroup { + CK_SLIST_ENTRY(divcblbgroup) dl_next; + struct epoch_context dl_epochctx; + uint16_t dl_port; + int dl_count; +#define DIVCBLBGROUP_SIZE 32 + struct divcb *dl_dcb[DIVCBLBGROUP_SIZE]; +}; + CK_SLIST_HEAD(divhashhead, divcb); +CK_SLIST_HEAD(divlbgrouphashhead, divcblbgroup); -VNET_DEFINE_STATIC(struct divhashhead, divhash[DIVHASHSIZE]) = {}; +VNET_DEFINE_STATIC(struct divhashhead, divhash[DIVHASHSIZE]); #define V_divhash VNET(divhash) +VNET_DEFINE_STATIC(struct divlbgrouphashhead, divlbhash[DIVHASHSIZE]); +#define V_divlbhash VNET(divlbhash) VNET_DEFINE_STATIC(uint64_t, dcb_count) = 0; #define V_dcb_count VNET(dcb_count) VNET_DEFINE_STATIC(uint64_t, dcb_gencnt) = 0; @@ -163,10 +178,15 @@ MTX_SYSINIT(divert, &divert_mtx, "divert(4) socket pcb lists", MTX_DEF); /* * Divert a packet by passing it up to the divert socket at port 'port'. + * + * 'id' is an opaque identifier for the flow and is used to load-balance packets + * across multiple divert sockets bound to the same port. Packets with the same + * identifier will be delivered to the same socket. */ static void -divert_packet(struct mbuf *m, bool incoming) +divert_packet(struct mbuf *m, uint64_t id, bool incoming) { + struct divcblbgroup *dlb; struct divcb *dcb; u_int16_t nport; struct sockaddr_in divsrc; @@ -272,10 +292,27 @@ divert_packet(struct mbuf *m, bool incoming) sizeof(divsrc.sin_zero)); } - /* Put packet on socket queue, if any */ - CK_SLIST_FOREACH(dcb, &V_divhash[DIVHASH(nport)], dcb_next) - if (dcb->dcb_port == nport) + /* + * Look for a matching divert socket or socket group, and enqueue the + * packet. + */ + CK_SLIST_FOREACH(dlb, &V_divlbhash[DIVHASH(nport)], dl_next) { + uint16_t count; + + count = atomic_load_acq_int(&dlb->dl_count); + if (dlb->dl_port == nport && count > 0) { + uint32_t hash; + + hash = jenkins_hash(&id, sizeof(uint64_t), 0); + dcb = dlb->dl_dcb[hash % count]; break; + } + } + if (dlb == NULL) { + CK_SLIST_FOREACH(dcb, &V_divhash[DIVHASH(nport)], dcb_next) + if (dcb->dcb_port == nport) + break; + } if (dcb != NULL) { struct socket *sa = dcb->dcb_socket; @@ -597,14 +634,63 @@ div_free(epoch_context_t ctx) } static void +divlbgroup_free(epoch_context_t ctx) +{ + struct divcblbgroup *dlb = __containerof(ctx, struct divcblbgroup, + dl_epochctx); + + free(dlb, M_PCB); +} + +static void +div_lbgroup_detach(struct divcb *dcb) +{ + struct divcblbgroup *dlb; + + CK_SLIST_FOREACH(dlb, &V_divlbhash[DCBHASH(dcb)], dl_next) { + if (dlb->dl_port != dcb->dcb_port) + continue; + + /* + * Delicately remove the socket from its group, taking + * care to synchronize with lookups, which do not handle + * NULL slots in the group table. + * + * Note that the hash is not stable across different + * group sizes. + */ + for (int i = 0; i < dlb->dl_count; i++) { + unsigned int count; + + if (dlb->dl_dcb[i] != dcb) + continue; + + count = dlb->dl_count; + if (i != count - 1) + dlb->dl_dcb[i] = dlb->dl_dcb[count - 1]; + atomic_store_rel_int(&dlb->dl_count, count - 1); + if (count == 1) { + CK_SLIST_REMOVE(&V_divlbhash[DCBHASH(dcb)], dlb, + divcblbgroup, dl_next); + NET_EPOCH_CALL(divlbgroup_free, + &dlb->dl_epochctx); + } + return; + } + } +} + +static void div_detach(struct socket *so) { struct divcb *dcb = so->so_pcb; so->so_pcb = NULL; DIVERT_LOCK(); - if (dcb->dcb_bound != DCB_UNBOUND) + if (dcb->dcb_bound != DCB_UNBOUND) { CK_SLIST_REMOVE(&V_divhash[DCBHASH(dcb)], dcb, divcb, dcb_next); + div_lbgroup_detach(dcb); + } V_dcb_count--; V_dcb_gencnt++; DIVERT_UNLOCK(); @@ -614,28 +700,70 @@ div_detach(struct socket *so) static int div_bind(struct socket *so, struct sockaddr *nam, struct thread *td) { + struct divcblbgroup *dlb; struct divcb *dcb; + int error; uint16_t port; if (nam->sa_family != AF_INET) return EAFNOSUPPORT; if (nam->sa_len != sizeof(struct sockaddr_in)) return EINVAL; + + error = 0; + if ((so->so_options & SO_REUSEPORT_LB) != 0) + dlb = malloc(sizeof(*dlb), M_PCB, M_WAITOK | M_ZERO); + else + dlb = NULL; + port = ((struct sockaddr_in *)nam)->sin_port; DIVERT_LOCK(); - CK_SLIST_FOREACH(dcb, &V_divhash[DIVHASH(port)], dcb_next) - if (dcb->dcb_port == port) { - DIVERT_UNLOCK(); - return (EADDRINUSE); + if (dlb == NULL) { + CK_SLIST_FOREACH(dcb, &V_divhash[DIVHASH(port)], dcb_next) { + if (dcb->dcb_port == port) { + DIVERT_UNLOCK(); + return (EADDRINUSE); + } } + } dcb = so->so_pcb; - if (dcb->dcb_bound != DCB_UNBOUND) - CK_SLIST_REMOVE(&V_divhash[DCBHASH(dcb)], dcb, divcb, dcb_next); - dcb->dcb_port = port; - CK_SLIST_INSERT_HEAD(&V_divhash[DIVHASH(port)], dcb, dcb_next); + if (dlb != NULL) { + struct divcblbgroup *tmp; + + CK_SLIST_FOREACH(tmp, &V_divlbhash[DIVHASH(port)], dl_next) { + if (tmp->dl_port == port) + break; + } + if (tmp == NULL) { + dlb->dl_port = port; + dlb->dl_count = 1; + dlb->dl_dcb[0] = dcb; + CK_SLIST_INSERT_HEAD(&V_divlbhash[DIVHASH(port)], dlb, + dl_next); + } else if (tmp->dl_count < DIVCBLBGROUP_SIZE) { + KASSERT(tmp->dl_count > 0, + ("div_bind: lbgroup %p has count 0", tmp)); + + tmp->dl_dcb[tmp->dl_count] = dcb; + atomic_store_rel_int(&tmp->dl_count, tmp->dl_count + 1); + free(dlb, M_PCB); + } else { + error = ENOSPC; + free(dlb, M_PCB); + } + } + if (error == 0) { + if (dcb->dcb_bound != DCB_UNBOUND) { + CK_SLIST_REMOVE(&V_divhash[DCBHASH(dcb)], dcb, divcb, + dcb_next); + div_lbgroup_detach(dcb); + } + dcb->dcb_port = port; + CK_SLIST_INSERT_HEAD(&V_divhash[DIVHASH(port)], dcb, dcb_next); + } DIVERT_UNLOCK(); - return (0); + return (error); } static int diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h index 934ca80a083d..081938ec7ae4 100644 --- a/sys/netinet/ip_var.h +++ b/sys/netinet/ip_var.h @@ -324,7 +324,7 @@ VNET_DECLARE(ip_fw_ctl_ptr_t, ip_fw_ctl_ptr); #define V_ip_fw_ctl_ptr VNET(ip_fw_ctl_ptr) /* Divert hooks. */ -extern void (*ip_divert_ptr)(struct mbuf *m, bool incoming); +extern void (*ip_divert_ptr)(struct mbuf *m, uint64_t id, bool incoming); /* ng_ipfw hooks -- XXX make it the same as divert and dummynet */ extern int (*ng_ipfw_input_p)(struct mbuf **, struct ip_fw_args *, bool); extern int (*ip_dn_ctl_ptr)(struct sockopt *); diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c index 48e20df3ef9a..851f70cbb0ad 100644 --- a/sys/netinet/raw_ip.c +++ b/sys/netinet/raw_ip.c @@ -96,7 +96,7 @@ VNET_DEFINE(ip_fw_ctl_ptr_t, ip_fw_ctl_ptr) = NULL; int (*ip_dn_ctl_ptr)(struct sockopt *); int (*ip_dn_io_ptr)(struct mbuf **, struct ip_fw_args *); -void (*ip_divert_ptr)(struct mbuf *, bool); +void (*ip_divert_ptr)(struct mbuf *, uint64_t, bool); int (*ng_ipfw_input_p)(struct mbuf **, struct ip_fw_args *, bool); #ifdef INET diff --git a/sys/netinet/tcp_log_buf.c b/sys/netinet/tcp_log_buf.c index 3e5955e5db4e..35fa0b56a9c7 100644 --- a/sys/netinet/tcp_log_buf.c +++ b/sys/netinet/tcp_log_buf.c @@ -1449,6 +1449,7 @@ tcp_log_tcpcbfini(struct tcpcb *tp) int i; memset(&log, 0, sizeof(log)); + microuptime(&tv); if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { for (i = 0; i < TCP_NUM_CNT_COUNTERS; i++) { log.u_raw.u64_flex[i] = tp->tcp_cnt_counters[i]; diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c index 8132899bb0d9..0cf6be2f9b33 100644 --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -465,15 +465,6 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred, bzero(&laddr6, sizeof(laddr6)); laddr6.sin6_family = AF_INET6; - if (V_fib_hash_outbound) { - uint32_t hash_type, hash_val; - - hash_val = fib6_calc_software_hash(&inp->in6p_laddr, - &sin6->sin6_addr, 0, sin6->sin6_port, - inp->inp_socket->so_proto->pr_protocol, &hash_type); - inp->inp_flowid = hash_val; - inp->inp_flowtype = hash_type; - } /* * Call inner routine, to assign local interface address. * in6_pcbladdr() may automatically fill in sin6_scope_id. @@ -520,6 +511,16 @@ in6_pcbconnect(struct inpcb *inp, struct sockaddr_in6 *sin6, struct ucred *cred, in_pcbrehash(inp); INP_HASH_WUNLOCK(pcbinfo); + if (V_fib_hash_outbound) { + uint32_t hash_type, hash_val; + + hash_val = fib6_calc_software_hash(&inp->in6p_laddr, + &sin6->sin6_addr, 0, sin6->sin6_port, + inp->inp_socket->so_proto->pr_protocol, &hash_type); + inp->inp_flowid = hash_val; + inp->inp_flowtype = hash_type; + } + return (0); } diff --git a/sys/netlink/netlink_generic.c b/sys/netlink/netlink_generic.c index c2f82eed5656..50c12175e14e 100644 --- a/sys/netlink/netlink_generic.c +++ b/sys/netlink/netlink_generic.c @@ -93,12 +93,10 @@ static struct genl_group { static inline struct genl_family * genl_family(uint16_t family_id) { - struct genl_family *gf; - - gf = &families[family_id - GENL_MIN_ID]; KASSERT(family_id - GENL_MIN_ID < MAX_FAMILIES && - gf->family_name != NULL, ("family %u does not exist", family_id)); - return (gf); + families[family_id - GENL_MIN_ID].family_name != NULL, + ("family %u does not exist", family_id)); + return (&families[family_id - GENL_MIN_ID]); } static inline uint16_t diff --git a/sys/netlink/route/iface_drivers.c b/sys/netlink/route/iface_drivers.c index 31d2523a479b..79daa4215dba 100644 --- a/sys/netlink/route/iface_drivers.c +++ b/sys/netlink/route/iface_drivers.c @@ -69,21 +69,24 @@ _nl_modify_ifp_generic(struct ifnet *ifp, struct nl_parsed_link *lattrs, int error; if (lattrs->ifla_ifalias != NULL) { - if (nlp_has_priv(npt->nlp, PRIV_NET_SETIFDESCR)) { - int len = strlen(lattrs->ifla_ifalias) + 1; - char *buf = if_allocdescr(len, M_WAITOK); - - memcpy(buf, lattrs->ifla_ifalias, len); - if_setdescr(ifp, buf); - if_setlastchange(ifp); - } else { + if (!nlp_has_priv(npt->nlp, PRIV_NET_SETIFDESCR)) { nlmsg_report_err_msg(npt, "Not enough privileges to set descr"); return (EPERM); } + int len = strlen(lattrs->ifla_ifalias) + 1; + char *buf = if_allocdescr(len, M_WAITOK); + + memcpy(buf, lattrs->ifla_ifalias, len); + if_setdescr(ifp, buf); + if_setlastchange(ifp); } if ((lattrs->ifi_change & IFF_UP) != 0 || lattrs->ifi_change == 0) { /* Request to up or down the interface */ + if (!nlp_has_priv(npt->nlp, PRIV_NET_SETIFFLAGS)) { + nlmsg_report_err_msg(npt, "Not enough privileges to set flags"); + return (EPERM); + } if (lattrs->ifi_flags & IFF_UP) if_up(ifp); else @@ -91,22 +94,21 @@ _nl_modify_ifp_generic(struct ifnet *ifp, struct nl_parsed_link *lattrs, } if (lattrs->ifla_mtu > 0) { - if (nlp_has_priv(npt->nlp, PRIV_NET_SETIFMTU)) { - struct ifreq ifr = { .ifr_mtu = lattrs->ifla_mtu }; - error = ifhwioctl(SIOCSIFMTU, ifp, (char *)&ifr, - curthread); - if (error != 0) { - nlmsg_report_err_msg(npt, "Failed to set mtu"); - return (error); - } - } else { + if (!nlp_has_priv(npt->nlp, PRIV_NET_SETIFMTU)) { nlmsg_report_err_msg(npt, "Not enough privileges to set mtu"); return (EPERM); } + struct ifreq ifr = { .ifr_mtu = lattrs->ifla_mtu }; + error = ifhwioctl(SIOCSIFMTU, ifp, (char *)&ifr, + curthread); + if (error != 0) { + nlmsg_report_err_msg(npt, "Failed to set mtu"); + return (error); + } } if ((lattrs->ifi_change & IFF_PROMISC) != 0 || - lattrs->ifi_change == 0) + lattrs->ifi_change == 0) { /* * When asking for IFF_PROMISC, set permanent flag instead * (IFF_PPROMISC) as we have no way of doing promiscuity @@ -114,24 +116,28 @@ _nl_modify_ifp_generic(struct ifnet *ifp, struct nl_parsed_link *lattrs, * function either sets or unsets IFF_PROMISC, and ifi_change * is usually set to 0xFFFFFFFF. */ + if (!nlp_has_priv(npt->nlp, PRIV_NET_SETIFFLAGS)) { + nlmsg_report_err_msg(npt, "Not enough privileges to set promisc"); + return (EPERM); + } if_setppromisc(ifp, (lattrs->ifi_flags & IFF_PROMISC) != 0); + } if (lattrs->ifla_address != NULL) { - if (nlp_has_priv(npt->nlp, PRIV_NET_SETIFMAC)) { - error = if_setlladdr(ifp, - NLA_DATA(lattrs->ifla_address), - NLA_DATA_LEN(lattrs->ifla_address)); - if (error != 0) { - nlmsg_report_err_msg(npt, - "setting IFLA_ADDRESS failed with error code: %d", - error); - return (error); - } - } else { + if (!nlp_has_priv(npt->nlp, PRIV_NET_SETIFMAC)) { nlmsg_report_err_msg(npt, "Not enough privileges to set IFLA_ADDRESS"); return (EPERM); } + error = if_setlladdr(ifp, + NLA_DATA(lattrs->ifla_address), + NLA_DATA_LEN(lattrs->ifla_address)); + if (error != 0) { + nlmsg_report_err_msg(npt, + "setting IFLA_ADDRESS failed with error code: %d", + error); + return (error); + } } return (0); diff --git a/sys/netlink/route/rt.c b/sys/netlink/route/rt.c index 8159409c9f67..39ecb537d365 100644 --- a/sys/netlink/route/rt.c +++ b/sys/netlink/route/rt.c @@ -217,7 +217,7 @@ dump_rc_nhg(struct nl_writer *nw, const struct route_nhop_data *rnd, struct rtms return; rtnh->rtnh_flags = 0; rtnh->rtnh_ifindex = if_getindex(wn[i].nh->nh_ifp); - rtnh->rtnh_hops = wn[i].weight; + rtnh->rtnh_hops = MIN(wn[i].weight, UINT8_MAX); dump_rc_nhop_gw(nw, wn[i].nh); uint32_t rtflags = nhop_get_rtflags(wn[i].nh); if (rtflags != base_rtflags) @@ -242,7 +242,8 @@ dump_rc_nhg(struct nl_writer *nw, const struct route_nhop_data *rnd, struct rtms } nlattr_set_len(nw, off); nlattr_add_u32(nw, NL_RTA_PRIORITY, nhop_metric); - nlattr_add_u32(nw, NL_RTA_WEIGHT, nhop_weight); + if (nhop_weight != RT_DEFAULT_WEIGHT) + nlattr_add_u32(nw, NL_RTA_WEIGHT, nhop_weight); } static void @@ -726,7 +727,7 @@ handle_rtm_dump(struct nlpcb *nlp, uint32_t fibnum, int family, if (fibnum == RT_TABLE_UNSPEC) { for (int i = 0; i < V_rt_numfibs; i++) { - dump_rtable_fib(&wa, fibnum, family); + dump_rtable_fib(&wa, i, family); if (wa.error != 0) break; } @@ -977,6 +978,14 @@ create_nexthop_from_attrs(struct nl_parsed_route *attrs, return (nh); } +/* pre-2.6.19 Linux API compatibility: prefer RTA_TABLE, fall back to rtm_table */ +static inline void +old_linux_compat(struct nl_parsed_route *attrs) +{ + if (attrs->rtm_table > 0 && attrs->rta_table == 0) + attrs->rta_table = attrs->rtm_table; +} + static int rtnl_handle_newroute(struct nlmsghdr *hdr, struct nlpcb *nlp, struct nl_pstate *npt) @@ -996,9 +1005,7 @@ rtnl_handle_newroute(struct nlmsghdr *hdr, struct nlpcb *nlp, return (EINVAL); } - /* pre-2.6.19 Linux API compatibility */ - if (attrs.rtm_table > 0 && attrs.rta_table == 0) - attrs.rta_table = attrs.rtm_table; + old_linux_compat(&attrs); if (attrs.rta_table >= V_rt_numfibs || attrs.rtm_family > AF_MAX) { NLMSG_REPORT_ERR_MSG(npt, "invalid fib"); return (EINVAL); @@ -1065,6 +1072,7 @@ rtnl_handle_delroute(struct nlmsghdr *hdr, struct nlpcb *nlp, return (ESRCH); } + old_linux_compat(&attrs); if (attrs.rta_table >= V_rt_numfibs || attrs.rtm_family > AF_MAX) { NLMSG_REPORT_ERR_MSG(npt, "invalid fib"); return (EINVAL); @@ -1088,6 +1096,7 @@ rtnl_handle_getroute(struct nlmsghdr *hdr, struct nlpcb *nlp, struct nl_pstate * if (error != 0) return (error); + old_linux_compat(&attrs); if (attrs.rta_table >= V_rt_numfibs || attrs.rtm_family > AF_MAX) { NLMSG_REPORT_ERR_MSG(npt, "invalid fib"); return (EINVAL); diff --git a/sys/netpfil/ipfw/ip_fw_pfil.c b/sys/netpfil/ipfw/ip_fw_pfil.c index ddd8e00316b8..7e1c24a89edd 100644 --- a/sys/netpfil/ipfw/ip_fw_pfil.c +++ b/sys/netpfil/ipfw/ip_fw_pfil.c @@ -563,7 +563,7 @@ ipfw_divert(struct mbuf **m0, struct ip_fw_args *args, bool tee) m_tag_prepend(clone, tag); /* Do the dirty job... */ - ip_divert_ptr(clone, args->flags & IPFW_ARGS_IN); + ip_divert_ptr(clone, 0, args->flags & IPFW_ARGS_IN); return 0; } diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 27d03b688937..98b5657f7285 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -11966,7 +11966,7 @@ done: pd.m->m_flags &= ~M_FASTFWD_OURS; } } - ip_divert_ptr(*m0, dir == PF_IN); + ip_divert_ptr(*m0, s != NULL ? s->id : 0, dir == PF_IN); *m0 = NULL; return (action); } else if (mtag == NULL) { diff --git a/sys/powerpc/include/stack.h b/sys/powerpc/include/stack.h index 928256b26468..533ff0fa2056 100644 --- a/sys/powerpc/include/stack.h +++ b/sys/powerpc/include/stack.h @@ -38,7 +38,7 @@ extern int end[]; /* Get the current kernel thread stack usage. */ #define GET_STACK_USAGE(total, used) do { \ struct thread *td = curthread; \ - (total) = td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb); \ + (total) = ptoa(td->td_kstack_pages) - sizeof(struct pcb); \ (used) = td->td_kstack + (total) - (char *)&td; \ } while (0) @@ -46,8 +46,7 @@ static __inline bool kstack_contains(struct thread *td, vm_offset_t va, size_t len) { return (va >= (vm_offset_t)td->td_kstack && va + len >= va && - va + len <= (vm_offset_t)td->td_kstack + td->td_kstack_pages * - PAGE_SIZE - sizeof(struct pcb)); + va + len <= (vm_offset_t)td_kstack_top(td) - sizeof(struct pcb)); } #endif /* _SYS_PROC_H_ */ diff --git a/sys/powerpc/powerpc/exec_machdep.c b/sys/powerpc/powerpc/exec_machdep.c index 00c04b4ddbaa..18e0ba004a13 100644 --- a/sys/powerpc/powerpc/exec_machdep.c +++ b/sys/powerpc/powerpc/exec_machdep.c @@ -1080,10 +1080,15 @@ cpu_thread_clean(struct thread *td) void cpu_thread_alloc(struct thread *td) { +} + +void +cpu_thread_new_kstack(struct thread *td) +{ struct pcb *pcb; - pcb = (struct pcb *)__align_down(td->td_kstack + td->td_kstack_pages * - PAGE_SIZE - sizeof(struct pcb), 0x40); + pcb = (struct pcb *)__align_down(td_kstack_top(td) - sizeof(struct pcb), + 0x40); td->td_pcb = pcb; td->td_frame = (struct trapframe *)pcb - 1; } diff --git a/sys/powerpc/powerpc/machdep.c b/sys/powerpc/powerpc/machdep.c index a975bebebaad..e1e0885e3ecf 100644 --- a/sys/powerpc/powerpc/machdep.c +++ b/sys/powerpc/powerpc/machdep.c @@ -487,8 +487,8 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp, /* * Finish setting up thread0. */ - thread0.td_pcb = (struct pcb *)__align_down(thread0.td_kstack + - thread0.td_kstack_pages * PAGE_SIZE - sizeof(struct pcb), 16); + thread0.td_pcb = (struct pcb *)__align_down(td_kstack_top(&thread0) - + sizeof(struct pcb), 16); bzero((void *)thread0.td_pcb, sizeof(struct pcb)); pc->pc_curpcb = thread0.td_pcb; diff --git a/sys/powerpc/powerpc/vm_machdep.c b/sys/powerpc/powerpc/vm_machdep.c index 00fdc301a7e7..1dc28739ad7c 100644 --- a/sys/powerpc/powerpc/vm_machdep.c +++ b/sys/powerpc/powerpc/vm_machdep.c @@ -123,9 +123,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) if (td1 == curthread) cpu_update_pcb(td1); - pcb = (struct pcb *)__align_down(td2->td_kstack + - td2->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb), 0x40); - td2->td_pcb = pcb; + pcb = td2->td_pcb; /* Copy the pcb */ bcopy(td1->td_pcb, pcb, sizeof(struct pcb)); @@ -135,7 +133,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) * Copy the trap frame for the return to user mode as if from a * syscall. This copies most of the user mode register values. */ - tf = (struct trapframe *)pcb - 1; + tf = td2->td_frame; bcopy(td1->td_frame, tf, sizeof(*tf)); /* Set up trap frame. */ @@ -143,8 +141,6 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) tf->fixreg[FIRSTARG + 1] = 0; tf->cr &= ~0x10000000; - td2->td_frame = tf; - cf = (struct callframe *)tf - 1; memset(cf, 0, sizeof(struct callframe)); #if defined(__powerpc64__) && (!defined(_CALL_ELF) || _CALL_ELF == 1) diff --git a/sys/riscv/include/stack.h b/sys/riscv/include/stack.h index 03b5794c2b13..fac8d4f317b6 100644 --- a/sys/riscv/include/stack.h +++ b/sys/riscv/include/stack.h @@ -48,12 +48,10 @@ bool unwind_frame(struct thread *, struct unwind_state *); #ifdef _SYS_PROC_H_ -#include <machine/pcb.h> - /* Get the current kernel thread stack usage. */ #define GET_STACK_USAGE(total, used) do { \ struct thread *td = curthread; \ - (total) = td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb); \ + (total) = ptoa(td->td_kstack_pages); \ (used) = td->td_kstack + (total) - (char *)&td; \ } while (0) @@ -61,8 +59,7 @@ static __inline bool kstack_contains(struct thread *td, vm_offset_t va, size_t len) { return (va >= (vm_offset_t)td->td_kstack && va + len >= va && - va + len <= (vm_offset_t)td->td_kstack + td->td_kstack_pages * - PAGE_SIZE - sizeof(struct pcb)); + va + len <= (vm_offset_t)td_kstack_top(td)); } #endif /* _SYS_PROC_H_ */ diff --git a/sys/riscv/riscv/genassym.c b/sys/riscv/riscv/genassym.c index c216c686db9a..998183e92db6 100644 --- a/sys/riscv/riscv/genassym.c +++ b/sys/riscv/riscv/genassym.c @@ -63,7 +63,6 @@ ASSYM(PMAP_MAPDEV_EARLY_SIZE, PMAP_MAPDEV_EARLY_SIZE); ASSYM(PM_SATP, offsetof(struct pmap, pm_satp)); ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); -ASSYM(PCB_SIZE, sizeof(struct pcb)); ASSYM(PCB_RA, offsetof(struct pcb, pcb_ra)); ASSYM(PCB_SP, offsetof(struct pcb, pcb_sp)); ASSYM(PCB_GP, offsetof(struct pcb, pcb_gp)); diff --git a/sys/riscv/riscv/locore.S b/sys/riscv/riscv/locore.S index 305ed8d79f10..e2126af5cad2 100644 --- a/sys/riscv/riscv/locore.S +++ b/sys/riscv/riscv/locore.S @@ -241,8 +241,8 @@ va: /* Clear frame pointer */ mv s0, zero - /* Allocate space for thread0 PCB and riscv_bootparams */ - addi sp, sp, -(PCB_SIZE + RISCV_BOOTPARAMS_SIZE) & ~STACKALIGNBYTES + /* Allocate space for riscv_bootparams */ + addi sp, sp, -RISCV_BOOTPARAMS_SIZE & ~STACKALIGNBYTES /* Clear BSS */ la t0, _C_LABEL(__bss_start) diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c index 91219676454b..f5479c479109 100644 --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -105,6 +105,7 @@ struct pcpu __pcpu[MAXCPU]; static struct trapframe proc0_tf; +static struct pcb pcb0; int early_boot = 1; int cold = 1; @@ -296,8 +297,7 @@ init_proc0(void *kstack) proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_kstack_pages = KSTACK_PAGES; - thread0.td_pcb = (struct pcb *)(thread0.td_kstack + - thread0.td_kstack_pages * PAGE_SIZE) - 1; + thread0.td_pcb = &pcb0; thread0.td_pcb->pcb_fpflags = 0; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; diff --git a/sys/riscv/riscv/vm_machdep.c b/sys/riscv/riscv/vm_machdep.c index 206110157233..e538921c9f60 100644 --- a/sys/riscv/riscv/vm_machdep.c +++ b/sys/riscv/riscv/vm_machdep.c @@ -32,8 +32,8 @@ * SUCH DAMAGE. */ -#include <sys/param.h> #include <sys/systm.h> +#include <sys/kernel.h> #include <sys/limits.h> #include <sys/proc.h> #include <sys/sf_buf.h> @@ -58,24 +58,23 @@ #define TP_OFFSET 16 /* sizeof(struct tcb) */ #endif -static void -cpu_set_pcb_frame(struct thread *td) -{ - td->td_pcb = (struct pcb *)(td->td_kstack + - td->td_kstack_pages * PAGE_SIZE) - 1; +static uma_zone_t pcb_zone; +void +cpu_thread_new_kstack(struct thread *td) +{ /* * td->td_frame + TF_SIZE will be the saved kernel stack pointer whilst * in userspace, so keep it aligned so it's also aligned when we * subtract TF_SIZE in the trap handler (and here for the initial stack * pointer). This also keeps the struct kernframe just afterwards - * aligned no matter what's in it or struct pcb. + * aligned no matter what's in it. * * NB: TF_SIZE not sizeof(struct trapframe) as we need the rounded * value to match the trap handler. */ td->td_frame = (struct trapframe *)(STACKALIGN( - (char *)td->td_pcb - sizeof(struct kernframe)) - TF_SIZE); + td_kstack_top(td) - sizeof(struct kernframe)) - TF_SIZE); } /* @@ -100,8 +99,6 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) critical_exit(); } - cpu_set_pcb_frame(td2); - pcb2 = td2->td_pcb; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); @@ -230,12 +227,13 @@ cpu_thread_exit(struct thread *td) void cpu_thread_alloc(struct thread *td) { - cpu_set_pcb_frame(td); + td->td_pcb = uma_zalloc(pcb_zone, M_WAITOK); } void cpu_thread_free(struct thread *td) { + uma_zfree(pcb_zone, td->td_pcb); } void @@ -289,3 +287,11 @@ cpu_sync_core(void) { fence_i(); } + +static void +pcbinit(void *dummy __unused) +{ + pcb_zone = uma_zcreate("pcb", sizeof(struct pcb), NULL, NULL, NULL, + NULL, UMA_ALIGNOF(struct pcb), 0); +} +SYSINIT(pcbinit, SI_SUB_INTRINSIC, SI_ORDER_ANY, pcbinit, NULL); diff --git a/sys/security/mac_do/mac_do.c b/sys/security/mac_do/mac_do.c index ba49da22ce67..93f2084d1c93 100644 --- a/sys/security/mac_do/mac_do.c +++ b/sys/security/mac_do/mac_do.c @@ -1,8 +1,9 @@ -/*- +/* * SPDX-License-Identifier: BSD-2-Clause * - * Copyright(c) 2024 Baptiste Daroussin <bapt@FreeBSD.org> - * Copyright (c) 2024 The FreeBSD Foundation + * Copyright (c) 2024 Baptiste Daroussin <bapt@FreeBSD.org> + * Copyright (c) 2024, 2026, The FreeBSD Foundation + * Copyright (c) 2025 Kushagra Srivastava <kushagra1403@gmail.com> * * Portions of this software were developed by Olivier Certner * <olce.freebsd@certner.fr> at Kumacom SARL under sponsorship from the FreeBSD @@ -23,6 +24,7 @@ #include <sys/priv.h> #include <sys/proc.h> #include <sys/refcount.h> +#include <sys/rmlock.h> #include <sys/socket.h> #include <sys/stdarg.h> #include <sys/sx.h> @@ -32,6 +34,24 @@ #include <security/mac/mac_policy.h> + +#ifdef INVARIANTS +/* + * Should typically be moved to libkern (and perhaps libc) at some point, and be + * optimized if to be used outside of INVARIANTS. + */ +static bool +is_zeroed(const void *const buf, const size_t size) +{ + const char *const p = buf; + + for (size_t i = 0; i < size; ++i) + if (p[i] != 0) + return (false); + return (true); +} +#endif + static SYSCTL_NODE(_security_mac, OID_AUTO, do, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "mac_do policy controls"); @@ -46,7 +66,16 @@ SYSCTL_INT(_security_mac_do, OID_AUTO, print_parse_error, CTLFLAG_RWTUN, static MALLOC_DEFINE(M_MAC_DO, "mac_do", "mac_do(4) security module"); -#define MAC_RULE_STRING_LEN 1024 +#define MAX_RULE_STRING_SIZE 1024 +_Static_assert(MAX_RULE_STRING_SIZE > 0, + "MAX_RULE_STRING_SIZE: No space for the NUL terminator!"); + +#define MAX_EXEC_PATHS_SIZE 2048 +#define MAX_EXEC_PATHS 8 +_Static_assert(MAX_EXEC_PATHS_SIZE > 0, + "MAX_EXEC_PATHS_SIZE: No space for the NUL terminator!"); + +struct rmlock mac_do_rml; static unsigned osd_jail_slot; static unsigned osd_thread_slot; @@ -67,6 +96,11 @@ static const char *id_type_to_str[] = { #define PARSE_ERROR_SIZE 256 +/* + * All functions having a parse error parameter must return through it a parse + * error object if and only if they return an error value (non-zero); else, NULL + * must be returned through it. + */ struct parse_error { size_t pos; char msg[PARSE_ERROR_SIZE]; @@ -123,11 +157,15 @@ typedef uint16_t flags_t; #define MDF_MAY_REJ_SUPP (1u << 11) /* (t,gid) Some explicit ID (not MDF_CURRENT) has MDF_SUPP_MUST. */ #define MDF_EXPLICIT_SUPP_MUST (1u << 12) -/* (t,gid) Whether any target clause is about primary groups. Used during - * parsing only. */ +/* + * (t,gid) Whether any target clause is about primary groups. Used during + * parsing only. + */ #define MDF_HAS_PRIMARY_CLAUSE (1u << 13) -/* (t,gid) Whether any target clause is about supplementary groups. Used during - * parsing only. */ +/* + * (t,gid) Whether any target clause is about supplementary groups. Used during + * parsing only. + */ #define MDF_HAS_SUPP_CLAUSE (1u << 14) #define MDF_TYPE_GID_MASK (MDF_ANY_SUPP | MDF_MAY_REJ_SUPP | \ MDF_EXPLICIT_SUPP_MUST | MDF_HAS_PRIMARY_CLAUSE | MDF_HAS_SUPP_CLAUSE) @@ -144,7 +182,7 @@ struct id_spec { /* * This limits the number of target clauses per type to 65535. With the current - * value of MAC_RULE_STRING_LEN (1024), this is way more than enough anyway. + * value of MAX_RULE_STRING_SIZE (1024), this is way more than enough anyway. */ typedef uint16_t id_nb_t; /* We only have a few IT_* types. */ @@ -165,8 +203,24 @@ struct rule { STAILQ_HEAD(rulehead, rule); struct rules { - char string[MAC_RULE_STRING_LEN]; + char string[MAX_RULE_STRING_SIZE]; struct rulehead head; +}; + +struct exec_paths { + char exec_paths_str[MAX_EXEC_PATHS_SIZE]; + char exec_paths[MAX_EXEC_PATHS][PATH_MAX]; + int exec_path_count; +}; + +/* + * Once in use, i.e., being pointed to by a jail, a configuration structure MUST + * NEVER CHANGE (except for the 'use_count' field). This invariant is + * fundamental to correctness! + */ +struct conf { + struct rules rules; + struct exec_paths exec_paths; volatile u_int use_count __aligned(CACHE_LINE_SIZE); }; @@ -181,6 +235,7 @@ struct id_elem { STAILQ_HEAD(id_list, id_elem); + #ifdef INVARIANTS static void check_type(const id_type_t type) @@ -221,7 +276,7 @@ check_type_and_id_flags(const id_type_t type, const flags_t flags) } break; default: - __assert_unreachable(); + __assert_unreachable(); } return; @@ -281,6 +336,18 @@ unexpected_flags: #define check_type_and_type_flags(...) #endif /* INVARIANTS */ +static bool +has_rules(const struct rules *const rules) +{ + return (rules->string[0] != '\0'); +} + +static bool +has_exec_paths(const struct exec_paths *const exec_paths) +{ + return (exec_paths->exec_paths_str[0] != '\0'); +} + /* * Returns EALREADY if both flags have some overlap, or EINVAL if flags are * incompatible, else 0 with flags successfully merged into 'dest'. @@ -323,23 +390,36 @@ toast_rules(struct rules *const rules) free(rule->gids, M_MAC_DO); free(rule, M_MAC_DO); } - free(rules, M_MAC_DO); } -static struct rules * -alloc_rules(void) +static inline void +init_rules(struct rules *const rules) { - struct rules *const rules = malloc(sizeof(*rules), M_MAC_DO, M_WAITOK); - - _Static_assert(MAC_RULE_STRING_LEN > 0, "MAC_RULE_STRING_LEN <= 0!"); - rules->string[0] = 0; + MPASS(is_zeroed(rules, sizeof(*rules))); STAILQ_INIT(&rules->head); - rules->use_count = 0; - return (rules); +} + +static inline void +init_exec_paths(struct exec_paths *const exec_paths) +{ + MPASS(is_zeroed(exec_paths, sizeof(*exec_paths))); +} + +static struct conf * +new_conf(void) +{ + struct conf *const conf = malloc(sizeof(*conf), M_MAC_DO, + M_WAITOK | M_ZERO); + + init_rules(&conf->rules); + init_exec_paths(&conf->exec_paths); + refcount_init(&conf->use_count, 1); + + return (conf); } static bool -is_null_or_empty(const char *s) +is_null_or_empty(const char *const s) { return (s == NULL || s[0] == '\0'); } @@ -433,7 +513,8 @@ static void make_parse_error(struct parse_error **const parse_error, const size_t pos, const char *const fmt, ...) { - struct parse_error *const err = malloc(sizeof(*err), M_MAC_DO, M_WAITOK); + struct parse_error *const err = malloc(sizeof(*err), M_MAC_DO, + M_WAITOK); va_list ap; err->pos = pos; @@ -740,6 +821,7 @@ parse_target_clause(char *to, struct rule *const rule, check_type_and_finish: check_type_and_type_flags(type, *tflags); finish: + MPASS(error == 0 && *parse_error == NULL); return (0); einval: /* We must have built a parse error on error. */ @@ -817,7 +899,7 @@ pour_list_into_rule(const id_type_t type, struct id_list *const list, make_parse_error(parse_error, 0, "Incompatible flags or duplicate " "GID %u.", id); - return (EINVAL); + goto einval; } check_type_and_id_flags(type, array[ref_idx].flags); @@ -831,7 +913,7 @@ pour_list_into_rule(const id_type_t type, struct id_list *const list, */ make_parse_error(parse_error, 0, "Duplicate UID %u.", id); - return (EINVAL); + goto einval; default: __assert_unreachable(); @@ -840,7 +922,12 @@ pour_list_into_rule(const id_type_t type, struct id_list *const list, *nb = ref_idx + 1; } + MPASS(*parse_error == NULL); return (0); + +einval: + MPASS(*parse_error != NULL); + return (EINVAL); } /* @@ -966,6 +1053,7 @@ parse_single_rule(char *rule, struct rules *const rules, } STAILQ_INSERT_TAIL(&rules->head, new, r_entries); + MPASS(error == 0 && *parse_error == NULL); return (0); einval: @@ -983,13 +1071,13 @@ einval: /* * Parse rules specification and produce rule structures out of it. * - * Returns 0 on success, with '*rulesp' made to point to a 'struct rule' - * representing the rules. On error, the returned value is non-zero and - * '*rulesp' is unchanged. If 'string' has length greater or equal to - * MAC_RULE_STRING_LEN, ENAMETOOLONG is returned. If it is not in the expected - * format, EINVAL is returned. If an error is returned, '*parse_error' is set - * to point to a 'struct parse_error' giving an error message for the problem, - * else '*parse_error' is set to NULL. + * Must be called with '*parse_error' set to NULL. Returns 0 on success, with + * '*rulesp' made to point to a 'struct rule' representing the rules. On error, + * the returned value is non-zero and '*rulesp' is unchanged. If 'string' has + * length greater or equal to MAX_RULE_STRING_SIZE, ENAMETOOLONG is returned. If + * it is not in the expected format, EINVAL is returned. If an error is + * returned, '*parse_error' is set to point to a 'struct parse_error' giving an + * error message for the problem. * * Expected format: A >-colon-separated list of rules of the form * "<from>><target>" (for backwards compatibility, a semi-colon ":" is accepted @@ -1007,24 +1095,20 @@ einval: * - "gid=1010>gid=1011,gid=1012,gid=1013" */ static int -parse_rules(const char *const string, struct rules **const rulesp, +parse_rules(const char *const string, struct rules *const rules, struct parse_error **const parse_error) { const size_t len = strlen(string); char *copy, *p, *rule; - struct rules *rules; int error = 0; - *parse_error = NULL; - - if (len >= MAC_RULE_STRING_LEN) { + if (len >= MAX_RULE_STRING_SIZE) { make_parse_error(parse_error, 0, "Rule specification string is too long (%zu, max %zu)", - len, MAC_RULE_STRING_LEN - 1); + len, MAX_RULE_STRING_SIZE - 1); return (ENAMETOOLONG); } - rules = alloc_rules(); bcopy(string, rules->string, len + 1); MPASS(rules->string[len] == '\0'); /* Catch some races. */ @@ -1040,72 +1124,188 @@ parse_rules(const char *const string, struct rules **const rulesp, if (error != 0) { (*parse_error)->pos += rule - copy; toast_rules(rules); - goto out; + goto error; + } + } + + MPASS(error == 0 && *parse_error == NULL); +out: + free(copy, M_MAC_DO); + return (error); +error: + MPASS(error != 0 && *parse_error != NULL); + goto out; +} + +/* + * Similar constraints as parse_rules() (which see). + */ +static int +parse_exec_paths(const char *const string, struct exec_paths *const exec_paths, + struct parse_error **const parse_error) +{ + const size_t len = strlen(string); + char *copy, *p, *path; + int error = 0; + + if (len >= MAX_EXEC_PATHS_SIZE) { + make_parse_error(parse_error, 0, + "Exec path specification string is too long (%zu, max %u)", + len, MAX_EXEC_PATHS_SIZE - 1); + return (ENAMETOOLONG); + } + + bcopy(string, exec_paths->exec_paths_str, len + 1); + MPASS(exec_paths->exec_paths_str[len] == '\0'); + + copy = malloc(len + 1, M_MAC_DO, M_WAITOK); + bcopy(string, copy, len + 1); + MPASS(copy[len] == '\0'); + + p = copy; + while ((path = strsep(&p, ":")) != NULL) { + size_t path_len; + + if (*path == '\0') + continue; + + if (exec_paths->exec_path_count >= MAX_EXEC_PATHS) { + make_parse_error(parse_error, path - copy, + "Too many exec paths specified (max %d)", + MAX_EXEC_PATHS); + error = EINVAL; + goto error; } + + path_len = strlen(path); + if (path_len >= PATH_MAX) { + make_parse_error(parse_error, path - copy, + "Exec paths too long (%zu, max %u)", + path_len, PATH_MAX - 1); + error = ENAMETOOLONG; + goto error; + } + + strlcpy(exec_paths->exec_paths[exec_paths->exec_path_count], + path, PATH_MAX); + exec_paths->exec_path_count++; } - *rulesp = rules; + MPASS(error == 0 && *parse_error == NULL); out: free(copy, M_MAC_DO); return (error); +error: + MPASS(error != 0 && *parse_error != NULL); + goto out; +} + +static void +hold_conf(struct conf *const conf) +{ + int old_count __diagused = refcount_acquire(&conf->use_count); + + KASSERT(old_count != 0, + ("MAC/do: Trying to resurrect a destroyed configuration.")); +} + +static void +drop_conf(struct conf *const conf) +{ + if (refcount_release(&conf->use_count)) { + toast_rules(&conf->rules); + free(conf, M_MAC_DO); + } } /* - * Find rules applicable to the passed prison. + * Find configuration applicable to the passed prison. + * + * Returns the applicable configuration (which always exists), with an + * additional reference that must be freed by the caller. 'pr' must not be + * locked. + * + * The applicable configuration is that of the closest ancestor prison + * (including itself) of the passed prison that actually has a 'struct conf' + * associated to it. * - * Returns the applicable rules (and never NULL). 'pr' must be unlocked. - * 'aprp' is set to the (ancestor) prison holding these, and it must be unlocked - * once the caller is done accessing the rules. '*aprp' is equal to 'pr' if and - * only if the current jail has its own set of rules. + * If 'hpr' is not NULL, it is used to return a pointer to the (unlocked) prison + * holding the applicable configuration. + * + * The find_conf_unlocked() variant needs 'mac_do_rml' to be (read- or write-) + * locked. The find_conf() variant will take a read lock for the duration of + * the search. + * + * The configuration returned by this function is sequentially consistent with + * other concurrent reads and configuration modifications, even in the presence + * of concurrent changes of configurations higher up in the jail tree (whether + * they "change" the value of some parameters, install a new configuration where + * there wasn't any, breaking inheritance from higher up, or remove an existing + * one, establishing inheritance from higher up). */ -static struct rules * -find_rules(struct prison *const pr, struct prison **const aprp) +static struct conf * +find_conf_locked(struct prison *const pr, struct prison **const hpr) { - struct prison *cpr, *ppr; - struct rules *rules; + struct prison *cpr, *ppr; /* Current and parent. */ + struct conf *conf; + rm_assert(&mac_do_rml, RA_LOCKED); + /* + * We do not need to take any locks here to climb the prison tree as + * either the start prison ('pr') is that of the current thread (and our + * ancestors are necessarily stable), or it is a prison passed by the jail + * machinery to an OSD method, in which case the prison tree lock is + * already being held. + */ cpr = pr; for (;;) { - prison_lock(cpr); - rules = osd_jail_get(cpr, osd_jail_slot); - if (rules != NULL) + conf = osd_jail_get_unlocked(cpr, osd_jail_slot); + if (conf != NULL) break; - prison_unlock(cpr); ppr = cpr->pr_parent; - MPASS(ppr != NULL); /* prison0 always has rules. */ + /* + * 'prison0' always has a mac_do(4) configuration because we + * installed one on module load/activation and nothing can + * destroy it as 'prison0' is not a regular jail and the + * 'mac.do' parameter cannot be set to 'inherit' on it, which is + * the only way to clear an existing configuration. + */ + KASSERT(ppr != NULL, + ("MAC/do: 'prison0' must always have a configuration.")); cpr = ppr; } - *aprp = cpr; - return (rules); + hold_conf(conf); + if (hpr != NULL) + *hpr = cpr; + return (conf); } -static void -hold_rules(struct rules *const rules) +static struct conf * +find_conf(struct prison *const pr, struct prison **const hpr) { - refcount_acquire(&rules->use_count); -} + struct conf *conf; + struct rm_priotracker rmpt; -static void -drop_rules(struct rules *const rules) -{ - if (refcount_release(&rules->use_count)) - toast_rules(rules); + rm_rlock(&mac_do_rml, &rmpt); + conf = find_conf_locked(pr, hpr); + rm_runlock(&mac_do_rml, &rmpt); + return (conf); } #ifdef INVARIANTS static void -check_rules_use_count(const struct rules *const rules, u_int expected) +check_conf_use_count(const struct conf *const conf, u_int expected) { - const u_int use_count = refcount_load(&rules->use_count); + const u_int use_count = refcount_load(&conf->use_count); if (use_count != expected) - panic("MAC/do: Rules at %p: Use count is %u, expected %u", - rules, use_count, expected); + panic("MAC/do: Configuration at %p: Use count is %u, " + "expected %u", conf, use_count, expected); } #else -#define check_rules_use_count(...) +#define check_conf_use_count(...) #endif /* INVARIANTS */ /* @@ -1117,7 +1317,7 @@ check_rules_use_count(const struct rules *const rules, u_int expected) static void dealloc_jail_osd(void *const value) { - struct rules *const rules = value; + struct conf *const conf = value; /* * If called because the "holding" jail goes down, no one should be @@ -1133,124 +1333,259 @@ dealloc_jail_osd(void *const value) * we ensure that all thread's slots are freed first in mac_do_destroy() * to be able to check that only one reference remains. */ - check_rules_use_count(rules, 1); - toast_rules(rules); + check_conf_use_count(conf, 1); + drop_conf(conf); +} + +/* + * Sets a mac_do(4) configuration on a jail. + * + * 'conf' is the new conf to set (can be NULL), and an additional reference will + * be taken on it to represent the jail holding it (if not NULL). 'rsv' must + * have been allocated through osd_reserve() (if 'conf' is not NULL; else can + * be NULL). + * + * The previous configuration on the jail (or NULL) is returned (with an + * associated reference if not NULL). + */ +static struct conf * +set_conf_locked(struct prison *const pr, struct conf *const conf, + void **const rsv) +{ + struct conf *old_conf; + int error __diagused; + + KASSERT(conf == NULL || rsv != NULL, + ("MAC/do: OSD reserve needed to avoid allocating memory")); + rm_assert(&mac_do_rml, RA_WLOCKED); + + if (conf != NULL) + hold_conf(conf); + old_conf = osd_jail_get_unlocked(pr, osd_jail_slot); + error = osd_jail_set_reserved(pr, osd_jail_slot, rsv, conf); + KASSERT(error == 0, ("MAC/do: osd_jail_set_reserved() failed " + "with 'conf' = %p and 'rsv' = %p", conf, rsv)); + if (conf == NULL) + /* + * This completely frees the OSD slot, but doesn't call the + * destructor since we've just put NULL into the slot. + */ + osd_jail_del(pr, osd_jail_slot); + return (old_conf); +} + +/* + * Immediately replace the jail's configuration. + * + * To be used only if the configuration to set does not depend in any way on the + * currently applicable configuration. + * + * Takes care of write-locking 'mac_do_rml', which should be unlocked on entry + * and will be unlocked on exit. + */ +static void +set_conf(struct prison *const pr, struct conf *const conf) +{ + void **const rsv = conf != NULL ? osd_reserve(osd_jail_slot) : NULL; + struct conf *old_conf; + + rm_wlock(&mac_do_rml); + old_conf = set_conf_locked(pr, conf, rsv); + rm_wunlock(&mac_do_rml); + if (old_conf != NULL) + drop_conf(old_conf); } /* * Remove the rules specifically associated to a prison. * * In practice, this means that the rules become inherited (from the closest - * ascendant that has some). + * ancestor that has some). * * Destroys the 'osd_jail_slot' slot of the passed jail. */ -static void -remove_rules(struct prison *const pr) +static struct conf * +remove_conf_locked(struct prison *const pr) { - struct rules *old_rules; - int error __unused; + return (set_conf_locked(pr, NULL, NULL)); +} - prison_lock(pr); - /* - * We burden ourselves with extracting rules first instead of just - * letting osd_jail_del() call dealloc_jail_osd() as we want to - * decrement their use count, and possibly free them, outside of the - * prison lock. - */ - old_rules = osd_jail_get(pr, osd_jail_slot); - error = osd_jail_set(pr, osd_jail_slot, NULL); - /* osd_set() never allocates memory when 'value' is NULL, nor fails. */ - MPASS(error == 0); - /* - * This completely frees the OSD slot, but doesn't call the destructor - * since we've just put NULL in the slot. - */ - osd_jail_del(pr, osd_jail_slot); - prison_unlock(pr); +static struct conf * +new_default_conf(void) +{ + const char *const mdo_path = "/usr/bin/mdo"; + struct conf *conf = new_conf(); + + strlcpy(conf->exec_paths.exec_paths_str, mdo_path, + MAX_EXEC_PATHS_SIZE); + strlcpy(conf->exec_paths.exec_paths[0], mdo_path, + PATH_MAX); + conf->exec_paths.exec_path_count = 1; - if (old_rules != NULL) - drop_rules(old_rules); + return (conf); } -/* - * Assign already built rules to a jail. - */ static void -set_rules(struct prison *const pr, struct rules *const rules) +clone_rules(struct rules *const dst, const struct rules *const src) { - struct rules *old_rules; - void **rsv; + const struct rule *src_rule; + + strlcpy(dst->string, src->string, sizeof(dst->string)); - check_rules_use_count(rules, 0); - hold_rules(rules); - rsv = osd_reserve(osd_jail_slot); + STAILQ_FOREACH(src_rule, &src->head, r_entries) { + struct rule *const dst_rule = malloc(sizeof(*dst_rule), + M_MAC_DO, M_WAITOK); + bcopy(src_rule, dst_rule, sizeof(*dst_rule)); - prison_lock(pr); - old_rules = osd_jail_get(pr, osd_jail_slot); - osd_jail_set_reserved(pr, osd_jail_slot, rsv, rules); - prison_unlock(pr); - if (old_rules != NULL) - drop_rules(old_rules); + if (src_rule->uids_nb > 0) { + const size_t uids_size = sizeof(*dst_rule->uids) * + src_rule->uids_nb; + + dst_rule->uids = malloc(uids_size, M_MAC_DO, M_WAITOK); + bcopy(src_rule->uids, dst_rule->uids, uids_size); + } + + if (src_rule->gids_nb > 0) { + const size_t gids_size = sizeof(*dst_rule->gids) * + src_rule->gids_nb; + + dst_rule->gids = malloc(gids_size, M_MAC_DO, M_WAITOK); + bcopy(src_rule->gids, dst_rule->gids, gids_size); + } + + STAILQ_INSERT_TAIL(&dst->head, dst_rule, r_entries); + } +} + +static void +clone_exec_paths(struct exec_paths *const dst, + const struct exec_paths *const src) +{ + MPASS(is_zeroed(dst, sizeof(*dst))); + dst->exec_path_count = src->exec_path_count; + for (int i = 0; i < src->exec_path_count; i++) + strlcpy(dst->exec_paths[i], src->exec_paths[i], + sizeof(dst->exec_paths[i])); + + strlcpy(dst->exec_paths_str, src->exec_paths_str, + sizeof(dst->exec_paths_str)); } /* - * Assigns empty rules to a jail. + * Sets/modifies the MAC/do configuration for a jail. + * + * Must be called with '*parse_error' set to NULL. + * + * Supports explicitly setting all parameters or only some of them. An + * unspecified parameter must be passed as NULL. The values of unspecified + * parameters are copied from those of the passed model configuration (which is + * expected to be the currently applicable configuration, i.e., that of the + * closest ancestor jail that has one). + * + * 'mac_do_rml' needs to be write-locked (and stays so). 'old_conf' serves to + * return, on no error, the old configuration with a reference (which must be + * eventually freed). */ -static void -set_empty_rules(struct prison *const pr) +static int +parse_and_set_conf(struct prison *const pr, const char *const rules_string, + const char *const exec_paths_string, const struct conf *const model_conf, + struct conf **const old_conf, struct parse_error **const parse_error) { - struct rules *const rules = alloc_rules(); + struct conf *const conf = new_conf(); + int error = 0; + + KASSERT(model_conf != NULL || + (rules_string != NULL && exec_paths_string != NULL), + ("MAC/do: %s: Model configuration needed!", __func__)); + + if (rules_string != NULL) { + error = parse_rules(rules_string, &conf->rules, parse_error); + if (error != 0) + goto error; + } + else + clone_rules(&conf->rules, &model_conf->rules); + + if (exec_paths_string != NULL) { + error = parse_exec_paths(exec_paths_string, &conf->exec_paths, + parse_error); + if (error != 0) + goto error; + } else + clone_exec_paths(&conf->exec_paths, + &model_conf->exec_paths); - set_rules(pr, rules); + MPASS(error == 0); + *old_conf = set_conf_locked(pr, conf, osd_reserve(osd_jail_slot)); + + MPASS(error == 0 && *parse_error == NULL); +out: + drop_conf(conf); + return (error); +error: + MPASS(error != 0 && *parse_error != NULL); + goto out; } /* - * Parse a rules specification and assign them to a jail. + * Calls parse_and_set_conf() and closes the current configuration transaction. * - * Returns the same error code as parse_rules() (which see). + * Closes the transaction by unlocking 'mac_do_rml' and releasing the old + * configuration returned by parse_and_set_conf(). */ static int -parse_and_set_rules(struct prison *const pr, const char *rules_string, +parse_and_commit_conf(struct prison *const pr, const char *const rules_string, + const char *const exec_paths_string, const struct conf *const model_conf, struct parse_error **const parse_error) { - struct rules *rules; + struct conf *old_conf; int error; - error = parse_rules(rules_string, &rules, parse_error); - if (error != 0) - return (error); - set_rules(pr, rules); - return (0); + error = parse_and_set_conf(pr, rules_string, exec_paths_string, + model_conf, &old_conf, parse_error); + rm_wunlock(&mac_do_rml); + + if (error == 0 && old_conf != NULL) + drop_conf(old_conf); + return (error); } + static int mac_do_sysctl_rules(SYSCTL_HANDLER_ARGS) { - char *const buf = malloc(MAC_RULE_STRING_LEN, M_MAC_DO, M_WAITOK); - struct prison *const td_pr = req->td->td_ucred->cr_prison; - struct prison *pr; - struct rules *rules; - struct parse_error *parse_error; + char *const buf = malloc(MAX_RULE_STRING_SIZE, M_MAC_DO, M_WAITOK); + struct prison *const pr = req->td->td_ucred->cr_prison; + struct conf *conf; + struct parse_error *parse_error = NULL; int error; - rules = find_rules(td_pr, &pr); - strlcpy(buf, rules->string, MAC_RULE_STRING_LEN); - prison_unlock(pr); + if (req->newptr != NULL) { + rm_wlock(&mac_do_rml); + conf = find_conf_locked(pr, NULL); + } else + conf = find_conf(pr, NULL); + strlcpy(buf, conf->rules.string, MAX_RULE_STRING_SIZE); - error = sysctl_handle_string(oidp, buf, MAC_RULE_STRING_LEN, req); - if (error != 0 || req->newptr == NULL) + error = sysctl_handle_string(oidp, buf, MAX_RULE_STRING_SIZE, req); + if (req->newptr == NULL) goto out; + if (error != 0) { + rm_wunlock(&mac_do_rml); + goto out; + } - /* Set our prison's rules, not that of the jail we inherited from. */ - error = parse_and_set_rules(td_pr, buf, &parse_error); + /* Unlocks 'mac_do_rml'. */ + error = parse_and_commit_conf(pr, buf, NULL, conf, &parse_error); if (error != 0) { if (print_parse_error) printf("MAC/do: Parse error at index %zu: %s\n", parse_error->pos, parse_error->msg); free_parse_error(parse_error); } + out: + drop_conf(conf); free(buf, M_MAC_DO); return (error); } @@ -1262,32 +1597,71 @@ SYSCTL_PROC(_security_mac_do, OID_AUTO, rules, SYSCTL_JAIL_PARAM_SYS_SUBNODE(mac, do, CTLFLAG_RW, "Jail MAC/do parameters"); -SYSCTL_JAIL_PARAM_STRING(_mac_do, rules, CTLFLAG_RW, MAC_RULE_STRING_LEN, +SYSCTL_JAIL_PARAM_STRING(_mac_do, rules, CTLFLAG_RW, MAX_RULE_STRING_SIZE, "Jail MAC/do rules"); - static int -mac_do_jail_create(void *obj, void *data __unused) +mac_do_sysctl_exec_paths(SYSCTL_HANDLER_ARGS) { - struct prison *const pr = obj; + char *const buf = malloc(MAX_EXEC_PATHS_SIZE, M_MAC_DO, M_WAITOK); + struct prison *const pr = req->td->td_ucred->cr_prison; + struct conf *conf; + struct parse_error *parse_error = NULL; + int error; - set_empty_rules(pr); - return (0); + if (req->newptr != NULL) { + rm_wlock(&mac_do_rml); + conf = find_conf_locked(pr, NULL); + } else + conf = find_conf(pr, NULL); + strlcpy(buf, conf->exec_paths.exec_paths_str, MAX_EXEC_PATHS_SIZE); + + error = sysctl_handle_string(oidp, buf, MAX_EXEC_PATHS_SIZE, req); + if (req->newptr == NULL) + goto out; + if (error != 0) { + rm_wunlock(&mac_do_rml); + goto out; + } + + /* Unlocks 'mac_do_rml'. */ + error = parse_and_commit_conf(pr, NULL, buf, conf, &parse_error); + if (error != 0) { + if (print_parse_error) + printf("MAC/do: Parse error at index %zu: %s\n", + parse_error->pos, parse_error->msg); + free_parse_error(parse_error); + } + +out: + drop_conf(conf); + free(buf, M_MAC_DO); + return (error); } +SYSCTL_PROC(_security_mac_do, OID_AUTO, exec_paths, + CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE, + 0, 0, mac_do_sysctl_exec_paths, "A", + "Colon-separated list of allowed executables"); + +SYSCTL_JAIL_PARAM_STRING(_mac_do, exec_paths, CTLFLAG_RW, MAX_EXEC_PATHS_SIZE, + "Jail MAC/do executable paths"); + static int mac_do_jail_get(void *obj, void *data) { - struct prison *ppr, *const pr = obj; + struct prison *const pr = obj; struct vfsoptlist *const opts = data; - struct rules *rules; + struct prison *hpr_out; + struct conf *const applicable_conf = find_conf(pr, &hpr_out); + const struct prison *const hpr = hpr_out; + const struct rules *const rules = &applicable_conf->rules; + const struct exec_paths *const exec_paths = &applicable_conf->exec_paths; int jsys, error; - rules = find_rules(pr, &ppr); + jsys = hpr == pr ? (has_rules(rules) && has_exec_paths(exec_paths) ? + JAIL_SYS_NEW : JAIL_SYS_DISABLE) : JAIL_SYS_INHERIT; - jsys = pr == ppr ? - (STAILQ_EMPTY(&rules->head) ? JAIL_SYS_DISABLE : JAIL_SYS_NEW) : - JAIL_SYS_INHERIT; error = vfs_setopt(opts, "mac.do", &jsys, sizeof(jsys)); if (error != 0 && error != ENOENT) goto done; @@ -1296,9 +1670,14 @@ mac_do_jail_get(void *obj, void *data) if (error != 0 && error != ENOENT) goto done; + error = vfs_setopts(opts, "mac.do.exec_paths", + exec_paths->exec_paths_str); + if (error != 0 && error != ENOENT) + goto done; + error = 0; done: - prison_unlock(ppr); + drop_conf(applicable_conf); return (error); } @@ -1317,11 +1696,16 @@ static int mac_do_jail_check(void *obj, void *data) { struct vfsoptlist *opts = data; - char *rules_string; - int error, jsys, size; + char *rules_string, *exec_paths_string; + int error, jsys, rules_size = 0, exec_paths_size = 0; + bool absent_or_empty_rules, absent_or_empty_exec_paths; error = vfs_copyopt(opts, "mac.do", &jsys, sizeof(jsys)); if (error == ENOENT) + /* + * Mark unspecified. Will fill it up below depending on the + * other options. + */ jsys = -1; else { if (error != 0) @@ -1332,75 +1716,117 @@ mac_do_jail_check(void *obj, void *data) } /* - * We use vfs_getopt() here instead of vfs_getopts() to get the length. - * We perform the additional checks done by the latter here, even if - * jail_set() calls vfs_getopts() itself later (they becoming - * inconsistent wouldn't cause any security problem). + * We use vfs_getopt() below instead of vfs_getopts() to get the + * string's buffer size. We perform the additional checks done by the + * latter here, even if jail_set() calls vfs_getopts() itself later + * (they becoming inconsistent wouldn't cause any security problem). */ - error = vfs_getopt(opts, "mac.do.rules", (void**)&rules_string, &size); - if (error == ENOENT) { - /* - * Default (in absence of "mac.do.rules") is to disable (and, in - * particular, not inherit). - */ - if (jsys == -1) - jsys = JAIL_SYS_DISABLE; - if (jsys == JAIL_SYS_NEW) { - vfs_opterror(opts, "'mac.do.rules' must be specified " - "given 'mac.do''s value"); + /* Rules. */ + error = vfs_getopt(opts, "mac.do.rules", (void **)&rules_string, + &rules_size); + if (error == ENOENT) + rules_string = NULL; + else { + if (error != 0) + return (error); + if (rules_size == 0 || rules_string[rules_size - 1] != '\0') { + vfs_opterror(opts, + "'mac.do.rules' not a proper string"); return (EINVAL); } + if (rules_size > MAX_RULE_STRING_SIZE) { + vfs_opterror(opts, "'mac.do.rules' too long"); + return (ENAMETOOLONG); + } + } - /* Absence of "mac.do.rules" at this point is OK. */ - error = 0; - } else { + /* Executable paths. */ + error = vfs_getopt(opts, "mac.do.exec_paths", + (void **)&exec_paths_string, &exec_paths_size); + if (error == ENOENT) + exec_paths_string = NULL; + else { if (error != 0) return (error); - - /* Not a proper string. */ - if (size == 0 || rules_string[size - 1] != '\0') { - vfs_opterror(opts, "'mac.do.rules' not a proper string"); + if (exec_paths_size == 0 || + exec_paths_string[exec_paths_size - 1] != '\0') { + vfs_opterror(opts, + "'mac.do.exec_paths' not a proper string"); return (EINVAL); } - - if (size > MAC_RULE_STRING_LEN) { - vfs_opterror(opts, "'mdo.rules' too long"); + if (exec_paths_size > MAX_EXEC_PATHS_SIZE) { + vfs_opterror(opts, "'mac.do.exec_paths' too long"); return (ENAMETOOLONG); } + } - if (jsys == -1) - /* Default (if "mac.do.rules" is present). */ - jsys = rules_string[0] == '\0' ? JAIL_SYS_DISABLE : - JAIL_SYS_NEW; + absent_or_empty_rules = is_null_or_empty(rules_string); + absent_or_empty_exec_paths = is_null_or_empty(exec_paths_string); + /* If not specified, infer 'jsys' from passed options. */ + if (jsys == -1) { /* - * Be liberal and accept JAIL_SYS_DISABLE and JAIL_SYS_INHERIT - * with an explicit empty rules specification. + * Default in absence of "mac.do.rules" and "mac.do.exec_paths" + * is to disable. We never implicitly inherit, as that changes + * reasoning about configurations. */ - switch (jsys) { - case JAIL_SYS_DISABLE: - case JAIL_SYS_INHERIT: - if (rules_string[0] != '\0') { - vfs_opterror(opts, "'mac.do.rules' specified " - "but should not given 'mac.do''s value"); - return (EINVAL); - } - break; + if (!absent_or_empty_rules || !absent_or_empty_exec_paths) + jsys = JAIL_SYS_NEW; + else + jsys = JAIL_SYS_DISABLE; + } + + /* Final checks based on resolved 'jsys'. */ + switch (jsys) { + case JAIL_SYS_DISABLE: + /* + * Tolerate specified but empty rules or execution paths + * (instead of not being specified). Also, tolerate that one of + * them is not empty (but not both). Indeed, as soon as one is + * empty, mac_do(4) is effectively disabled. This allows the + * administrator to still specify a value for one of them, which + * is then used for new sub-jails that do not inherit and for + * which no value for the parameter is explicitly specified + * (because then the value passed here is copied). + */ + if (!absent_or_empty_rules && !absent_or_empty_exec_paths) { + vfs_opterror(opts, + "One of 'mac.do.rules' and 'mac_do.exec_paths' " + "should not be specified or should be empty when " + "'mac.do' is 'disabled'"); + return (EINVAL); } + break; + + case JAIL_SYS_INHERIT: + /* + * Canonically, no parameters should be specified in this case. + * However, we tolerate empty ones, and also non-empty ones + * provided they match the inherited values, so that we can + * report the *resolved* value of current parameters via + * mac_do_jail_get() and have them re-applicable to this jail in + * a similar situation. Testing that inherited values are the + * same as passed ones is more expensive than a single test and + * requires some atomicity, which is why we do not perform that + * here but only in mac_do_jail_set(). + */ + break; } - return (error); + return (0); } static int mac_do_jail_set(void *obj, void *data) { - struct prison *pr = obj; - struct vfsoptlist *opts = data; - char *rules_string; - struct parse_error *parse_error; + struct prison *const pr = obj; + struct vfsoptlist *const opts = data; + char *rules_string, *exec_paths_string; + struct parse_error *parse_error = NULL; + struct conf *model_conf; int error, jsys; + bool absent_or_empty_rules, absent_or_empty_exec_paths; /* * The invariants checks used below correspond to what has already been @@ -1414,60 +1840,147 @@ mac_do_jail_set(void *obj, void *data) rules_string = vfs_getopts(opts, "mac.do.rules", &error); MPASS(error == 0 || error == ENOENT); - if (error == 0) { - MPASS(strlen(rules_string) < MAC_RULE_STRING_LEN); - if (jsys == -1) - /* Default (if "mac.do.rules" is present). */ - jsys = rules_string[0] == '\0' ? JAIL_SYS_DISABLE : - JAIL_SYS_NEW; + exec_paths_string = vfs_getopts(opts, "mac.do.exec_paths", &error); + MPASS(error == 0 || error == ENOENT); + + absent_or_empty_rules = is_null_or_empty(rules_string); + absent_or_empty_exec_paths = is_null_or_empty(exec_paths_string); + + if (jsys == -1) { + if (!absent_or_empty_rules || !absent_or_empty_exec_paths) + jsys = JAIL_SYS_NEW; else - MPASS(jsys == JAIL_SYS_NEW || - ((jsys == JAIL_SYS_DISABLE || - jsys == JAIL_SYS_INHERIT) && - rules_string[0] == '\0')); - } else { - MPASS(jsys != JAIL_SYS_NEW); - if (jsys == -1) + jsys = JAIL_SYS_DISABLE; + } + + if (jsys == JAIL_SYS_INHERIT) { + struct conf *old_conf = NULL; + + error = 0; + rm_wlock(&mac_do_rml); + + if (!absent_or_empty_rules || !absent_or_empty_exec_paths) { /* - * Default (in absence of "mac.do.rules") is to disable - * (and, in particular, not inherit). + * Some values specified. Check that they match the + * ones we are going to inherit. */ - jsys = JAIL_SYS_DISABLE; - /* If disabled, we'll store an empty rule specification. */ - if (jsys == JAIL_SYS_DISABLE) - rules_string = ""; + model_conf = find_conf_locked(pr->pr_parent, NULL); + if (strcmp(model_conf->rules.string, rules_string) + != 0) { + error = EINVAL; + vfs_opterror(opts, + "'mac.do' is 'inherited' but 'mac.do.rules'" + " was specified with a different value " + "than the one to be inherited (\"%s\")", + model_conf->rules.string); + } + if (strcmp(model_conf->exec_paths.exec_paths_str, + exec_paths_string) != 0) { + error = EINVAL; + vfs_opterror(opts, + "'mac.do' is 'inherited' but " + "'mac.do.exec_paths' was specified with a " + "different value than the one to be " + "inherited (\"%s\")", + model_conf->exec_paths.exec_paths_str); + } + drop_conf(model_conf); + } + + if (error == 0) + old_conf = remove_conf_locked(pr); + + rm_wunlock(&mac_do_rml); + + if (old_conf != NULL) + drop_conf(old_conf); + + return (error); } + model_conf = NULL; + /* Freeze configuration accesses. */ + rm_wlock(&mac_do_rml); + switch (jsys) { - case JAIL_SYS_INHERIT: - remove_rules(pr); - error = 0; - break; case JAIL_SYS_DISABLE: - case JAIL_SYS_NEW: - error = parse_and_set_rules(pr, rules_string, &parse_error); - if (error != 0) { - vfs_opterror(opts, - "MAC/do: Parse error at index %zu: %s\n", - parse_error->pos, parse_error->msg); - free_parse_error(parse_error); + /* + * mac_do(4) is disabled iff one of the parameter's string is + * empty. The parse_and_commit_conf() call below treats passing + * NULL for a parameter as a flag to copy its value from the + * relevant ancestor jail's configuration, so we have to watch + * for the final result having an empty parameter if no + * parameter has been explicitly passed as empty. Thanks to + * mac_do_jail_check(), we know that at least one parameter is + * absent or empty (see the comment for the corresponding case + * there). + */ + MPASS(absent_or_empty_rules || absent_or_empty_exec_paths); + if (!absent_or_empty_rules) + exec_paths_string = ""; + else if (!absent_or_empty_exec_paths) + rules_string = ""; + else { + /* + * Both are either empty or absent. If at least one is + * absent, we retrieve the applicable configuration as + * it will serve as a template (provides default + * values). + */ + if (rules_string == NULL || exec_paths_string == NULL) + model_conf = find_conf_locked(pr, NULL); + /* If both are absent, we have to examine if, in the + * currently applicable configuration, one of the + * parameters, which we are going to copy, is + * effectively empty. If both of those are non-empty, + * we keep the executable paths and empty the rules, + * since we expect that this is more convenient to + * administrators that may want to enable mac_do(4) + * later by just setting new rules. + */ + if (rules_string == NULL && exec_paths_string == NULL && + has_rules(&model_conf->rules) && + has_exec_paths(&model_conf->exec_paths)) + rules_string = ""; } break; + + case JAIL_SYS_NEW: + /* See the comment before the same test above. */ + if (rules_string == NULL || exec_paths_string == NULL) + model_conf = find_conf_locked(pr, NULL); + break; + default: __assert_unreachable(); } + + /* Unlocks 'mac_do_rml'. */ + error = parse_and_commit_conf(pr, rules_string, exec_paths_string, + model_conf, &parse_error); + if (model_conf != NULL) + drop_conf(model_conf); + if (error != 0) { + vfs_opterror(opts, + "MAC/do: Parse error at index %zu: %s\n", + parse_error->pos, parse_error->msg); + free_parse_error(parse_error); + } + return (error); } /* * OSD jail methods. * - * There is no PR_METHOD_REMOVE, as OSD storage is destroyed by the common jail - * code (see prison_cleanup()), which triggers a run of our dealloc_jail_osd() - * destructor. + * There is no PR_METHOD_REMOVE method, as OSD storage is destroyed by the + * common jail code (see prison_cleanup()), which triggers a run of our + * dealloc_jail_osd() destructor. There is neither a PR_METHOD_CREATE as + * PR_METHOD_SET is called just after (or the created jail destroyed if some + * PR_METHOD_CREATE fails), and our mac_do_jail_set() will ensure a jail is + * properly configured. */ static const osd_method_t osd_methods[PR_MAXMETHOD] = { - [PR_METHOD_CREATE] = mac_do_jail_create, [PR_METHOD_GET] = mac_do_jail_get, [PR_METHOD_CHECK] = mac_do_jail_check, [PR_METHOD_SET] = mac_do_jail_set, @@ -1492,8 +2005,8 @@ struct mac_do_data_header { * indicates this header is uninitialized. */ int priv; - /* Rules to apply. */ - struct rules *rules; + /* The configuration that applies. */ + struct conf *conf; }; /* @@ -1536,7 +2049,7 @@ clear_data(void *const data) struct mac_do_data_header *const hdr = data; if (hdr != NULL) { - drop_rules(hdr->rules); + drop_conf(hdr->conf); /* We don't deallocate so as to save time on next access. */ hdr->priv = 0; } @@ -1558,7 +2071,7 @@ is_data_reusable(const void *const data, const size_t size) static void set_data_header(void *const data, const size_t size, const int priv, - struct rules *const rules) + struct conf *const conf) { struct mac_do_data_header *const hdr = data; @@ -1567,7 +2080,7 @@ set_data_header(void *const data, const size_t size, const int priv, MPASS(size <= hdr->allocated_size); hdr->size = size; hdr->priv = priv; - hdr->rules = rules; + hdr->conf = conf; } /* The proc lock (and any other non-sleepable lock) must not be held. */ @@ -1933,7 +2446,7 @@ static int mac_do_priv_grant(struct ucred *cred, int priv) { struct mac_do_setcred_data *const data = fetch_data(); - const struct rules *rules; + struct rules *rules; const struct ucred *new_cred; const struct rule *rule; u_int setcred_flags; @@ -1950,7 +2463,7 @@ mac_do_priv_grant(struct ucred *cred, int priv) /* No. */ return (EPERM); - rules = data->hdr.rules; + rules = &data->hdr.conf->rules; new_cred = data->new_cred; KASSERT(new_cred != NULL, ("priv_check*() called before mac_cred_check_setcred()")); @@ -1987,7 +2500,10 @@ mac_do_priv_grant(struct ucred *cred, int priv) static int check_proc(void) { + struct prison *const pr = curproc->p_ucred->cr_prison; char *path, *to_free; + struct conf *conf; + struct exec_paths *exec_paths; int error; /* @@ -2010,7 +2526,18 @@ check_proc(void) */ if (vn_fullpath_jail(curproc->p_textvp, &path, &to_free) != 0) return (EPERM); - error = strcmp(path, "/usr/bin/mdo") == 0 ? 0 : EPERM; + + error = EPERM; + conf = find_conf(pr, NULL); + exec_paths = &conf->exec_paths; + + for (int i = 0; i < exec_paths->exec_path_count; i++) + if (strcmp(exec_paths->exec_paths[i], path) == 0) { + error = 0; + break; + } + + drop_conf(conf); free(to_free, M_TEMP); return (error); } @@ -2018,9 +2545,9 @@ check_proc(void) static void mac_do_setcred_enter(void) { - struct rules *rules; - struct prison *pr; + struct prison *const pr = curproc->p_ucred->cr_prison; struct mac_do_setcred_data * data; + struct conf *conf; int error; /* @@ -2042,9 +2569,7 @@ mac_do_setcred_enter(void) /* * Find the currently applicable rules. */ - rules = find_rules(curproc->p_ucred->cr_prison, &pr); - hold_rules(rules); - prison_unlock(pr); + conf = find_conf(pr, NULL); /* * Setup thread data to be used by other hooks. @@ -2052,7 +2577,7 @@ mac_do_setcred_enter(void) data = fetch_data(); if (!is_data_reusable(data, sizeof(*data))) data = alloc_data(data, sizeof(*data)); - set_data_header(data, sizeof(*data), PRIV_CRED_SETCRED, rules); + set_data_header(data, sizeof(*data), PRIV_CRED_SETCRED, conf); /* Not really necessary, but helps to catch programming errors. */ data->new_cred = NULL; data->setcred_flags = 0; @@ -2099,14 +2624,18 @@ mac_do_setcred_exit(void) static void mac_do_init(struct mac_policy_conf *mpc) { + struct conf *const default_conf = new_default_conf(); struct prison *pr; + rm_init_flags(&mac_do_rml, "mac_do(4)", RM_SLEEPABLE); + osd_jail_slot = osd_jail_register(dealloc_jail_osd, osd_methods); - set_empty_rules(&prison0); + set_conf(&prison0, default_conf); sx_slock(&allprison_lock); TAILQ_FOREACH(pr, &allprison, pr_list) - set_empty_rules(pr); + set_conf(pr, default_conf); sx_sunlock(&allprison_lock); + drop_conf(default_conf); osd_thread_slot = osd_thread_register(dealloc_thread_osd); } @@ -2120,6 +2649,7 @@ mac_do_destroy(struct mac_policy_conf *mpc) */ osd_thread_deregister(osd_thread_slot); osd_jail_deregister(osd_jail_slot); + rm_destroy(&mac_do_rml); } static struct mac_policy_ops do_ops = { diff --git a/sys/sys/imgact.h b/sys/sys/imgact.h index c1c94a2eabfd..51cda4406cad 100644 --- a/sys/sys/imgact.h +++ b/sys/sys/imgact.h @@ -57,6 +57,7 @@ struct image_args { struct image_params { struct proc *proc; /* our process */ + struct thread *td; struct label *execlabel; /* optional exec label */ struct vnode *vp; /* pointer to vnode of file to exec */ struct vm_object *object; /* The vm object for this vp */ diff --git a/sys/sys/proc.h b/sys/sys/proc.h index b560ea2e8020..ed69a09422e2 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -1246,6 +1246,7 @@ void cpu_thread_alloc(struct thread *); void cpu_thread_clean(struct thread *); void cpu_thread_exit(struct thread *); void cpu_thread_free(struct thread *); +void cpu_thread_new_kstack(struct thread *); struct thread *thread_alloc(int pages); int thread_check_susp(struct thread *td, bool sleep); void thread_cow_get_proc(struct thread *newtd, struct proc *p); @@ -1326,6 +1327,12 @@ td_get_sched(struct thread *td) return ((struct td_sched *)&td[1]); } +static __inline char * +td_kstack_top(struct thread *td) +{ + return (td->td_kstack + ptoa(td->td_kstack_pages)); +} + static __inline void ruxreset(struct rusage_ext *rux) { diff --git a/sys/sys/resource.h b/sys/sys/resource.h index 9e0635cdb328..6d8d17a05e02 100644 --- a/sys/sys/resource.h +++ b/sys/sys/resource.h @@ -145,7 +145,7 @@ static const char *rlimit_ident[] = { "kqueues", "umtx", "pipebuf", - "vmm", + "vms", }; #endif diff --git a/sys/sys/signal.h b/sys/sys/signal.h index 863b981c2b7a..792087f735a1 100644 --- a/sys/sys/signal.h +++ b/sys/sys/signal.h @@ -307,8 +307,8 @@ struct __siginfo32 { #define SEGV_ACCERR 2 /* Invalid permissions for mapped */ /* object. */ #define SEGV_PKUERR 100 /* x86: PKU violation */ -#define SEGV_MTEAERR 100 /* arm64: Asynchronous Arm MTE error */ -#define SEGV_MTESERR 101 /* arm64: Synchronous Arm MTE error */ +#define SEGV_MTEAERR 200 /* arm64: Asynchronous Arm MTE error */ +#define SEGV_MTESERR 201 /* arm64: Synchronous Arm MTE error */ /* codes for SIGFPE */ #define FPE_INTOVF 1 /* Integer overflow. */ diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 7f655b48ba08..88c25f06e0cb 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -554,6 +554,14 @@ void intr_prof_stack_use(struct thread *td, struct trapframe *frame); void counted_warning(unsigned *counter, const char *msg); /* + * Safely read one byte of kernel memory at address addr, placing the + * value into *valp. Returns 0 on success, EFAULT if read was + * impossible, e.g. due to the address not being mapped or not having + * necessary permissions. + */ +int safe_read(vm_offset_t addr, char *valp); + +/* * APIs to manage deprecation and obsolescence. */ void _gone_in(int major, const char *msg, ...) __printflike(2, 3); diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c index 191711ec7765..02e1c993529e 100644 --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -739,7 +739,7 @@ intr_prof_stack_use(struct thread *td, struct trapframe *frame) if (TRAPF_USERMODE(frame)) return; - stack_top = td->td_kstack + td->td_kstack_pages * PAGE_SIZE; + stack_top = td_kstack_top(td); current = (char *)&stack_top; /* diff --git a/sys/x86/conf/NOTES b/sys/x86/conf/NOTES index 3ae80fbe45c5..ebe272769818 100644 --- a/sys/x86/conf/NOTES +++ b/sys/x86/conf/NOTES @@ -549,6 +549,7 @@ envvar hint.pbio.0.port="0x360" device smbios device vpd device tpm +device padlock_rng # VIA Padlock RNG device rdrand_rng # Intel Bull Mountain RNG device aesni # AES-NI OpenCrypto module device ossl # OpenSSL OpenCrypto module diff --git a/sys/x86/x86/ucode.c b/sys/x86/x86/ucode.c index 72133de211f8..3d7008eb30f2 100644 --- a/sys/x86/x86/ucode.c +++ b/sys/x86/x86/ucode.c @@ -80,7 +80,7 @@ static const void *ucode_data; static struct ucode_ops *ucode_loader; /* Variables used for reporting success or failure. */ -enum { +static enum { NO_ERROR, NO_MATCH, VERIFICATION_FAILED, @@ -204,7 +204,6 @@ ucode_intel_match(const uint8_t *data, size_t *len) uint64_t platformid; size_t resid; uint32_t data_size, flags, regs[4], sig, total_size; - int i; do_cpuid(1, regs); sig = regs[0]; @@ -226,19 +225,35 @@ ucode_intel_match(const uint8_t *data, size_t *len) if (total_size == 0) total_size = UCODE_INTEL_DEFAULT_DATA_SIZE + sizeof(struct ucode_intel_header); - if (data_size > total_size + sizeof(struct ucode_intel_header)) + + if (total_size > data_size + sizeof(struct ucode_intel_header)) table = (const struct ucode_intel_extsig_table *) ((const uint8_t *)(hdr + 1) + data_size); else table = NULL; - if (hdr->processor_signature == sig) { - if ((hdr->processor_flags & flags) != 0) { - *len = data_size; - return (hdr + 1); + if (hdr->processor_signature == sig && + (hdr->processor_flags & flags) != 0) { + *len = data_size; + return (hdr + 1); + } + if (table != NULL) { + size_t extsize; + + extsize = total_size - + (data_size + sizeof(struct ucode_intel_header)); + if (extsize < sizeof(struct ucode_intel_extsig_table)) { + ucode_error = VERIFICATION_FAILED; + break; } - } else if (table != NULL) { - for (i = 0; i < table->signature_count; i++) { + extsize -= sizeof(struct ucode_intel_extsig_table); + for (uint32_t i = 0; i < table->signature_count; i++) { + if (extsize < sizeof(struct ucode_intel_extsig)) { + ucode_error = VERIFICATION_FAILED; + goto out; + } + extsize -= sizeof(struct ucode_intel_extsig); + entry = &table->entries[i]; if (entry->processor_signature == sig && (entry->processor_flags & flags) != 0) { @@ -248,6 +263,7 @@ ucode_intel_match(const uint8_t *data, size_t *len) } } } +out: return (NULL); } diff --git a/tests/sys/acl/run b/tests/sys/acl/run index f8e9c8d87f71..42dbc7373f7f 100644 --- a/tests/sys/acl/run +++ b/tests/sys/acl/run @@ -105,7 +105,7 @@ if (isatty(fileno(STDOUT))) { } } print $status, "\n"; -exit $failed ? 1 : 0; +exit($failed ? 1 : 0); sub process_test($$$$) { diff --git a/tests/sys/acl/tools-posix.test b/tests/sys/acl/tools-posix.test index 2b2a27d24a0d..aa92911761a6 100644 --- a/tests/sys/acl/tools-posix.test +++ b/tests/sys/acl/tools-posix.test @@ -80,7 +80,7 @@ $ getfacl -qh lll > group::r-x > other::r-x -$ getfacl -q lll +$ getfacl -nq lll > user::rw- > user:42:r-- > group::r-- @@ -89,7 +89,7 @@ $ getfacl -q lll > other::r-- $ setfacl -hm u:44:x,g:45:w lll -$ getfacl -h lll +$ getfacl -hn lll > # file: lll > # owner: root > # group: wheel @@ -111,7 +111,7 @@ $ rm lll # Test removing entries. $ setfacl -x user:42: xxx -$ getfacl xxx +$ getfacl -n xxx > # file: xxx > # owner: root > # group: wheel @@ -369,7 +369,7 @@ $ rm ddd/xxx $ setfacl -dm u::rwx,g::rx,o::rx,mask::rwx ddd $ setfacl -dm g:42:rwx,u:43:r ddd -$ getfacl -dq ddd +$ getfacl -dnq ddd > user::rwx > user:43:r-- > group::r-x @@ -378,7 +378,7 @@ $ getfacl -dq ddd > other::r-x $ touch ddd/xxx -$ getfacl -q ddd/xxx +$ getfacl -nq ddd/xxx > user::rw- > user:43:r-- > group::r-x # effective: r-- @@ -387,7 +387,7 @@ $ getfacl -q ddd/xxx > other::r-- $ mkdir ddd/ddd -$ getfacl -q ddd/ddd +$ getfacl -nq ddd/ddd > user::rwx > user:43:r-- > group::r-x @@ -405,7 +405,7 @@ $ ls -l fff | cut -d' ' -f1 > prw-r--r-- $ setfacl -m u:42:r,g:43:w fff -$ getfacl fff +$ getfacl -n fff > # file: fff > # owner: root > # group: wheel diff --git a/tests/sys/capsicum/capmode.cc b/tests/sys/capsicum/capmode.cc index c6eef19b350f..fdc572f11b5b 100644 --- a/tests/sys/capsicum/capmode.cc +++ b/tests/sys/capsicum/capmode.cc @@ -703,8 +703,8 @@ FORK_TEST(Capmode, NewThread) { close(thread_pipe[1]); } -static volatile sig_atomic_t had_signal = 0; -static void handle_signal(int) { had_signal = 1; } +static volatile sig_atomic_t signal_cnt = 0; +static void handle_signal(int) { signal_cnt++; } FORK_TEST(Capmode, SelfKill) { pid_t me = getpid(); @@ -722,7 +722,13 @@ FORK_TEST(Capmode, SelfKill) { // Can only kill(2) to own pid. EXPECT_CAPMODE(kill(child, SIGUSR1)); EXPECT_OK(kill(me, SIGUSR1)); - EXPECT_EQ(1, had_signal); + EXPECT_EQ(1, signal_cnt); + + union sigval sv; + sv.sival_int = 0x1234; + EXPECT_CAPMODE(sigqueue(child, SIGUSR1, sv)); + EXPECT_OK(sigqueue(me, SIGUSR1, sv)); + EXPECT_EQ(2, signal_cnt); signal(SIGUSR1, original); } diff --git a/tests/sys/kern/ptrace_test.c b/tests/sys/kern/ptrace_test.c index c1a5d226e990..3a55a6f48033 100644 --- a/tests/sys/kern/ptrace_test.c +++ b/tests/sys/kern/ptrace_test.c @@ -3614,6 +3614,10 @@ ATF_TC_BODY(ptrace__PT_STEP_with_signal, tc) ATF_REQUIRE(pl.pl_flags & PL_FLAG_SI); REQUIRE_EQ(pl.pl_siginfo.si_signo, SIGABRT); +#if defined(__riscv) + atf_tc_expect_fail("PT_STEP not implemented on riscv, see sys/riscv/riscv/ptrace_machdep.c"); +#endif + /* Step the child process inserting SIGUSR1. */ REQUIRE_EQ(ptrace(PT_STEP, fpid, (caddr_t)1, SIGUSR1), 0); @@ -3731,6 +3735,10 @@ ATF_TC_BODY(ptrace__step_siginfo, tc) ATF_REQUIRE(WIFSTOPPED(status)); REQUIRE_EQ(WSTOPSIG(status), SIGSTOP); +#if defined(__riscv) + atf_tc_expect_fail("PT_STEP not implemented on riscv, see sys/riscv/riscv/ptrace_machdep.c"); +#endif + /* Step the child ignoring the SIGSTOP. */ REQUIRE_EQ(ptrace(PT_STEP, fpid, (caddr_t)1, 0), 0); diff --git a/tests/sys/mac/do/Makefile b/tests/sys/mac/do/Makefile index 980067ea56e6..0c40f65b65f6 100644 --- a/tests/sys/mac/do/Makefile +++ b/tests/sys/mac/do/Makefile @@ -2,13 +2,12 @@ PACKAGE= tests TESTSDIR= ${TESTSBASE}/sys/mac/do -ATF_TESTS_SH+= valid_configs invalid_configs +ATF_TESTS_SH+= valid_configs invalid_configs consistency ${PACKAGE}FILES+= common.sh TEST_METADATA+= execenv="jail" TEST_METADATA+= required_kmods="mac_do" TEST_METADATA+= required_user="root" -TEST_METADATA+= required_programs="sysctl" .include <bsd.test.mk> diff --git a/tests/sys/mac/do/common.sh b/tests/sys/mac/do/common.sh index 88529adcc1f3..4f0e838bbf5f 100644 --- a/tests/sys/mac/do/common.sh +++ b/tests/sys/mac/do/common.sh @@ -1,5 +1,6 @@ +# Copyright (c) 2026 The FreeBSD Foundation # -# Copyright (c) 2026, The FreeBSD Foundation +# SPDX-License-Identifier: BSD-2-Clause # # This software was developed by Olivier Certner <olce@FreeBSD.org> at # Kumacom SARL under sponsorship from the FreeBSD Foundation. @@ -9,11 +10,79 @@ rules_parameter() echo "$1".rules } +exec_paths_parameter() +{ + echo "$1".exec_paths +} + +: ${MDO:=/usr/bin/mdo} + +ROOT_KNOB=security.mac.do +RULES_KNOB=$(rules_parameter ${ROOT_KNOB}) +EXEC_PATHS_KNOB=$(exec_paths_parameter ${ROOT_KNOB}) +PPE_KNOB=${ROOT_KNOB}.print_parse_error + +ROOT_JAIL_PARAM=mac.do +RULES_JAIL_PARAM=$(rules_parameter ${ROOT_JAIL_PARAM}) +EXEC_PATHS_JAIL_PARAM=$(exec_paths_parameter ${ROOT_JAIL_PARAM}) + +# To be overridden to execute commands in a sub-jail +JEXEC= + +# Exit status: 0 iff disabled +mac_do_disabled() +{ + [ -z "$($JEXEC sysctl -n ${RULES_KNOB})" ] || + [ -z "$($JEXEC sysctl -n ${EXEC_PATHS_KNOB})" ] +} + +mac_do_check_disabled() +{ + mac_do_disabled || atf_fail "mac_do(4) expected disabled but is not." +} -CONF_ROOT_KNOB=security.mac.do -RULES_KNOB=$(rules_parameter ${CONF_ROOT_KNOB}) -PPE_KNOB=${CONF_ROOT_KNOB}.print_parse_error +mac_do_ensure_disabled() +{ + mac_do_disabled || $JEXEC sysctl ${RULES_KNOB}="" +} +sysctl_rules() +{ + $JEXEC sysctl -n ${RULES_KNOB} +} + +sysctl_exec_paths() +{ + $JEXEC sysctl -n ${EXEC_PATHS_KNOB} +} + +# $1 = sysctl func, $2 = expected value +sysctl_check() +{ + local func value + + func=$1 + value=$2 + atf_check [ "$($func)" = "$value" ] +} + +# $1 = value +sysctl_check_rules() +{ + local value + + value=$1 + sysctl_check sysctl_rules $value +} + +# $1 = value +sysctl_check_exec_paths() +{ + local value + + value=$1 + sysctl_check sysctl_exec_paths $value +} # $1 = knob name, $2 = value sysctl_set_and_check() @@ -22,8 +91,8 @@ sysctl_set_and_check() knob=$1 value=$2 - atf_check -o ignore sysctl "$knob"="$value" - atf_check -o inline:"$value\n" sysctl -n "$knob" + atf_check -o ignore $JEXEC sysctl "$knob"="$value" + atf_check -o inline:"$value\n" $JEXEC sysctl -n "$knob" } # $1 = knob name, $2 = value @@ -34,8 +103,8 @@ sysctl_set_and_check_fails() knob=$1 value=$2 orig_value=$(sysctl -n "$knob") - atf_check -s not-exit:0 -o ignore -e ignore sysctl "$knob"="$value" - atf_check -o inline:"${orig_value}\n" sysctl -n "$knob" + atf_check -s not-exit:0 -o ignore -e ignore $JEXEC sysctl "$knob"="$value" + atf_check -o inline:"${orig_value}\n" $JEXEC sysctl -n "$knob" } # $1 = sysctl function, $2 = value @@ -45,9 +114,9 @@ sysctl_set_and_check_rules_common() func=$1 value=$2 - "$func" ${RULES_KNOB} "$value" - # Same spec but using the older in-rule separator (':') + # Use older in-rule separator (':') first to have final value as specified "$func" ${RULES_KNOB} "$(echo "$value" | sed 's%>%:%')" + "$func" ${RULES_KNOB} "$value" } # $1 = value @@ -68,5 +137,40 @@ sysctl_set_and_check_fails_rules() sysctl_set_and_check_rules_common sysctl_set_and_check_fails "$value" } +# $1 = sysctl function, $2 = value +sysctl_set_and_check_exec_paths_common() +{ + local func value + + func=$1 + value=$2 + # Use older in-rule separator (':') first to have final value as specified + "$func" ${EXEC_PATHS_KNOB} "$(echo "$value" | sed 's%>%:%')" + "$func" ${EXEC_PATHS_KNOB} "$value" +} + +# $1 = value +sysctl_set_and_check_exec_paths() +{ + local value + + value=$1 + sysctl_set_and_check_exec_paths_common sysctl_set_and_check "$value" +} + +# Create a persistent subjail. Echoes its JID. +launch_subjail() +{ + ( + set -o pipefail + $JEXEC jail -c -J /dev/stdout persist=true | + sed -nE 's%^.*jid=([0-9]+).*$%\1%p' + ) || atf_fail "Cannot create a subjail (check children limits?)" +} + +atf_require_prog sysctl +atf_require_prog jail +atf_require_prog sed + # Do not pollute kernel logs with parse errors sysctl $PPE_KNOB=0 >/dev/null 2>&1 diff --git a/tests/sys/mac/do/consistency.sh b/tests/sys/mac/do/consistency.sh new file mode 100644 index 000000000000..6a64917edb6d --- /dev/null +++ b/tests/sys/mac/do/consistency.sh @@ -0,0 +1,211 @@ +# Copyright (c) 2026 The FreeBSD Foundation +# +# SPDX-License-Identifier: BSD-2-Clause +# +# This software was developed by Olivier Certner <olce@FreeBSD.org> at +# Kumacom SARL under sponsorship from the FreeBSD Foundation. + +SJ_JID_FILE=sj.jid + +atf_test_case concurrent_rules_exec_paths_changes +concurrent_rules_exec_paths_changes_head() +{ + atf_set descr "Consistency of rules and exec paths changes on same jail" +} +concurrent_rules_exec_paths_changes_body() +{ + local rules exec_paths rules_es exec_paths_es + + for I in $(jot - 1 1000); do + sysctl_set_and_check_rules "uid=$I>uid=1001" + done & + rules=$! + + for I in $(jot - 1 1000); do + sysctl_set_and_check_exec_paths /nowhere/nonexistent$I + done & + exec_paths=$! + + wait $rules + rules_es=$? + + wait $exec_paths + exec_paths_es=$? + + # atf_check called in the asynchronous AND-OR lists above causes exit of the + # subshells and also a write to the ATF result file. These writes are + # concurrent and may cause the result file to be malformed. Consequently, + # it is important that, once execution becomes sequential again, atf_fail() is + # called again (and not just exit()). + if [ $rules_es -ne 0 ] || [ $exec_paths_es -ne 0 ]; then + atf_fail "Rules exit status: $rules_es, \ +exec paths exit status: $exec_paths_es" + fi +} + +atf_test_case inheritance cleanup +inheritance_head() +{ + atf_set descr "Simple inheritance test (values propagated to child jail)" +} +inheritance_body() +{ + local sj rules exec_paths + + # For the sake of not running the test under Kyua + mac_do_ensure_disabled + + sj=$(launch_subjail) + echo $sj > "${SJ_JID_FILE}" + + jail -m jid=$sj ${ROOT_JAIL_PARAM}=inherit + JEXEC="jexec $sj" + mac_do_check_disabled + JEXEC= + + rules="uid=1001>uid=0" + sysctl_set_and_check_rules $rules + JEXEC="jexec $sj" + sysctl_check_rules $rules + JEXEC= + + rules="gid=1001>uid=0" + sysctl_set_and_check_rules $rules + JEXEC="jexec $sj" + sysctl_check_rules $rules + JEXEC= + + # Not really necessary, just to keep mac_do(4) disabled + sysctl_set_and_check_rules "" + + exec_paths="/nowhere/nonexistent" + sysctl_set_and_check_exec_paths $exec_paths + JEXEC="jexec $sj" + sysctl_check_exec_paths $exec_paths + JEXEC= + + exec_paths="$MDO" + sysctl_set_and_check_exec_paths $exec_paths + JEXEC="jexec $sj" + sysctl_check_exec_paths $exec_paths + JEXEC= +} +inheritance_cleanup() +{ + # We clean up our subjail manually just for the sake of launching this test + # with atf-sh. Kyua is informed that these tests should run in a jail, and + # kills it automatically after the test, which kills all subjails. It is + # annoying that atf-sh does not offer a more practical way to pass + # information from the body to the cleanup part than a file. + jail -r $(cat "${SJ_JID_FILE}") + rm -f "${SJ_JID_FILE}" +} + +atf_test_case inheritance_relax_parent_jail cleanup +inheritance_relax_parent_jail_head() +{ + atf_set descr \ + "Test sequential consistency in a \"relax parent rules\" scenario" +} +inheritance_relax_parent_jail_body() +{ + local sj rules exec_paths subproc + + sj=$(launch_subjail) + echo $sj > "${SJ_JID_FILE}" + + jail -m jid=$sj ${ROOT_JAIL_PARAM}=inherit + rules="uid=1001>uid=0" + sysctl_set_and_check_rules $rules + # Additional inheritance sanity check + JEXEC="jexec $sj" + sysctl_check_rules $rules + JEXEC= + exec_paths="$MDO" + sysctl_set_and_check_exec_paths $exec_paths + # Additional inheritance sanity check + JEXEC="jexec $sj" + sysctl_check_exec_paths $exec_paths + JEXEC= + + # Launch a process that tries to become 'root' from user 1002, and verify + # that this always fails. + { for I in $(jot - 1 1000); do + jexec $sj "$MDO" -u 1002 -g 1002 -G 1002 "$MDO" -i true 2>/dev/null && + exit 1 + done; true; } & + subproc=$! + + # Decouple the subjail from the parent jail, copying its parameters + jail -m jid=$sj ${ROOT_JAIL_PARAM}=new + # Allow user 1002 to become 'root' on the parent jail + sysctl_set_and_check_rules "$rules;uid=1002>uid=0" + JEXEC="jexec $sj" + # Additional sanity check (that rules of the subjail are now independent) + [ "$(sysctl_rules)" == $rules ] || atf_fail "Rules not copied" + [ "$(sysctl_exec_paths)" == $exec_paths ] || + atf_fail "Exec paths not copied" + JEXEC= + + wait $subproc || atf_fail "A transition wrongly succeeded in the subjail!" +} +inheritance_relax_parent_jail_cleanup() +{ + # See inheritance_cleanup() for explanations + jail -r $(cat "${SJ_JID_FILE}") + rm -f "${SJ_JID_FILE}" +} + +atf_test_case same_knob_and_jail_parameter cleanup +same_knob_and_jail_parameter_head() +{ + atf_set descr \ + "Corresponding sysctl knobs and jail parameters have same value" +} +same_knob_and_jail_parameter_body() +{ + local sj rules exec_paths subproc + + sj=$(launch_subjail) + echo $sj > "${SJ_JID_FILE}" + + # Set sysctl knobs, observe parameters + rules="uid=19999>uid=21700" + exec_paths="/improbable/path/he" + JEXEC="jexec $sj" + sysctl_set_and_check_rules $rules + sysctl_set_and_check_exec_paths $exec_paths + JEXEC= + atf_check -o inline:"$rules\n" jls -j $sj ${RULES_JAIL_PARAM} + atf_check -o inline:"${exec_paths}\n" jls -j $sj ${EXEC_PATHS_JAIL_PARAM} + + # Set parameters, observe knobs + rules="uid=128000>uid=-1" + exec_paths="/hello/i_ve/changed" + jail -m jid=$sj ${RULES_JAIL_PARAM}=$rules \ + ${EXEC_PATHS_JAIL_PARAM}=${exec_paths} + JEXEC="jexec $sj" + sysctl_check_rules $rules + sysctl_check_exec_paths $exec_paths + JEXEC= +} +same_knob_and_jail_parameter_cleanup() +{ + # See inheritance_cleanup() for explanations + jail -r $(cat "${SJ_JID_FILE}") + rm -f "${SJ_JID_FILE}" +} + + +atf_init_test_cases() +{ + . $(atf_get_srcdir)/common.sh + atf_require_prog jot + # Needs an absolute path for mdo(1), to set it in exec_paths + atf_require_prog "$MDO" + + atf_add_test_case concurrent_rules_exec_paths_changes + atf_add_test_case inheritance + atf_add_test_case inheritance_relax_parent_jail + atf_add_test_case same_knob_and_jail_parameter +} diff --git a/tests/sys/mac/do/invalid_configs.sh b/tests/sys/mac/do/invalid_configs.sh index f24309cb2f3b..d1a9eb8c1e96 100644 --- a/tests/sys/mac/do/invalid_configs.sh +++ b/tests/sys/mac/do/invalid_configs.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env atf-sh +# Copyright (c) 2026 The FreeBSD Foundation # -# Copyright (c) 2026, The FreeBSD Foundation +# SPDX-License-Identifier: BSD-2-Clause # # This software was developed by Olivier Certner <olce@FreeBSD.org> at # Kumacom SARL under sponsorship from the FreeBSD Foundation. @@ -75,7 +75,7 @@ rules_wrong_separator_body() atf_init_test_cases() { - . $(atf_get_srcdir)/common.sh + . "$(atf_get_srcdir)"/common.sh atf_add_test_case rule_no_target_part atf_add_test_case rule_no_match_part diff --git a/tests/sys/mac/do/valid_configs.sh b/tests/sys/mac/do/valid_configs.sh index bd5b53b5d5d8..fc1c9a370854 100644 --- a/tests/sys/mac/do/valid_configs.sh +++ b/tests/sys/mac/do/valid_configs.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env atf-sh +# Copyright (c) 2026 The FreeBSD Foundation # -# Copyright (c) 2026, The FreeBSD Foundation +# SPDX-License-Identifier: BSD-2-Clause # # This software was developed by Olivier Certner <olce@FreeBSD.org> at # Kumacom SARL under sponsorship from the FreeBSD Foundation. @@ -120,7 +120,7 @@ gid= 1001 >gid =5" atf_init_test_cases() { - . $(atf_get_srcdir)/common.sh + . "$(atf_get_srcdir)"/common.sh atf_add_test_case rule_uid_to_any atf_add_test_case rule_uid_to_uid diff --git a/tests/sys/netipsec/tunnel/Makefile b/tests/sys/netipsec/tunnel/Makefile index c6060a790cc3..49fddc403005 100644 --- a/tests/sys/netipsec/tunnel/Makefile +++ b/tests/sys/netipsec/tunnel/Makefile @@ -13,8 +13,8 @@ ATF_TESTS_SH+= empty \ aesni_aes_gcm_256 \ chacha20_poly1305 -# Each test uses the same names for its jails, so they must be run serially. -TEST_METADATA+= is_exclusive=true +TEST_METADATA+= execenv="jail" \ + execenv_jail_params="vnet allow.raw_sockets" ${PACKAGE}FILES+= utils.subr diff --git a/tools/kerneldoc/subsys/Doxyfile-net80211 b/tools/kerneldoc/subsys/Doxyfile-net80211 index f9ce81e4ca06..6f9eccc73185 100644 --- a/tools/kerneldoc/subsys/Doxyfile-net80211 +++ b/tools/kerneldoc/subsys/Doxyfile-net80211 @@ -9,7 +9,8 @@ EXTRACT_ALL = YES # for undocumented src, no warnings enabled #-------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- -INPUT = $(DOXYGEN_SRC_PATH)/net80211/ $(NOTREVIEWED) +INPUT = $(DOXYGEN_SRC_PATH)/net80211/ +USE_MDFILE_AS_MAINPAGE = $(DOXYGEN_SRC_PATH)/net80211/README.md GENERATE_TAGFILE = net80211/net80211.tag diff --git a/tools/test/stress2/misc/all.exclude b/tools/test/stress2/misc/all.exclude index 9ec5bffde0f6..ef8cb69cd241 100644 --- a/tools/test/stress2/misc/all.exclude +++ b/tools/test/stress2/misc/all.exclude @@ -39,6 +39,7 @@ mount7.sh https://people.freebsd.org/~pho/stress/log/log0549.txt 20240912 mlockall2.sh Unrecoverable OOM killing seen 20190203 mlockall6.sh https://people.freebsd.org/~pho/stress/log/log0430.txt 20230403 mlockall7.sh Needs further investigation 20210123 +msdos24.sh Waiting for fix 20260529 msetdomain.sh May change policy for random threads to domainset_fixed 20210104 newfs4.sh watchdog fired. newbuf (still seen 20240729) 20190225 nfs10.sh Double fault 20151013 diff --git a/tools/test/stress2/misc/msdos22.sh b/tools/test/stress2/misc/msdos22.sh new file mode 100755 index 000000000000..f3a2110f13f1 --- /dev/null +++ b/tools/test/stress2/misc/msdos22.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +# Test file operations using long random file names consisting only of ASCII characters + +MDUNIT=10 +FS=/mnt +LOCALE=C +FILES=1000 + +export LANG=$LOCALE + +randomfilename () { + name="" + count=$(jot -r 1 10 3) + for r in $(jot -r $count 7 0); do + r=$(( r + 0 )) + c='_' + if [ $r -gt 0 ]; then + for i in $(jot $r); do + name="$name$i" + done + fi + count=$(( count - 1 )) + if [ "$count" -gt 0 ]; then + name="$name$c" + fi + done + echo "$name" +} + +( + set -e + + mdconfig -u $MDUNIT -t malloc -s 512m + newfs_msdos -c 8 -F 32 /dev/md$MDUNIT > /dev/null 2>&1 + mkdir -p $FS + mount_msdosfs /dev/md$MDUNIT $FS + + mkdir -p $FS/test + cd $FS/test + + for i in $(jot $FILES); do + testfiles="$testfiles +$(randomfilename)" + done + + testfiles=$(echo "$testfiles" | grep "." | sort -R | uniq) + + for f in $testfiles; do + echo "$f" > $f + done + for f in $(echo "$testfiles" | sort -R); do + cp $f $f.tmp + done + for f in $(echo "$testfiles" | sort -R); do + mv $f.tmp $f + done + for f in $(echo "$testfiles" | sort -R); do + rm $f + done +) + +failed=$? + +cd + +[ "$failed" -ne 0 ] && ls $FS/test + +umount /dev/md$MDUNIT + +#[ "$failed" -ne 0 ] && hd /dev/md$MDUNIT > /tmp/msdos22.dump + +fsck_msdosfs -y /dev/md$MDUNIT + +mdconfig -d -u $MDUNIT 2>/dev/null + +exit $failed diff --git a/tools/test/stress2/misc/msdos23.sh b/tools/test/stress2/misc/msdos23.sh new file mode 100755 index 000000000000..fcaa90917235 --- /dev/null +++ b/tools/test/stress2/misc/msdos23.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +# Test file operations using random file names containing non-ASCII characters + +MDUNIT=10 +FS=/mnt +LOCALE=C.UTF-8 +FILES=1000 + +export LANG=$LOCALE + +randomfilename () { + name="" + count=$(jot -r 1 10 3) + for r in $(jot -r $count 7 0); do + r=$(( r + 0 )) + c='·' + if [ $r -gt 0 ]; then + for i in $(jot $r); do + name="$name$i" + done + fi + count=$(( count - 1 )) + if [ "$count" -gt 0 ]; then + name="$name$c" + fi + done + echo "$name" +} + +( + set -e + + mdconfig -u $MDUNIT -t malloc -s 512m + newfs_msdos -c 8 -F 32 /dev/md$MDUNIT > /dev/null 2>&1 + mkdir -p $FS + mount_msdosfs -L $LOCALE /dev/md$MDUNIT $FS + + mkdir -p $FS/test + cd $FS/test + + for i in $(jot $FILES); do + testfiles="$testfiles +$(randomfilename)" + done + + testfiles=$(echo "$testfiles" | grep "." | sort -R | uniq) + + for f in $testfiles; do + echo "$f" > $f + done + for f in $(echo "$testfiles" | sort -R); do + cp $f $f.tmp + done + for f in $(echo "$testfiles" | sort -R); do + mv $f.tmp $f + done + for f in $(echo "$testfiles" | sort -R); do + rm $f + done +) + +failed=$? + +cd + +[ "$failed" -ne 0 ] && ls $FS/test + +umount /dev/md$MDUNIT + +#[ "$failed" -ne 0 ] && hd /dev/md$MDUNIT > /tmp/msdos23.dump + +fsck_msdosfs -y /dev/md$MDUNIT + +mdconfig -d -u $MDUNIT 2>/dev/null + +exit $failed diff --git a/tools/test/stress2/misc/msdos24.sh b/tools/test/stress2/misc/msdos24.sh new file mode 100755 index 000000000000..edb524293c0c --- /dev/null +++ b/tools/test/stress2/misc/msdos24.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +# Test file operations using long random file names containing UTF-16 surrogate pairs + +MDUNIT=10 +FS=/mnt +LOCALE=C.UTF-8 +FILES=1000 + +export LANG=$LOCALE + +randomfilename () { + name="" + count=$(jot -r 1 10 3) + for r in $(jot -r $count 7 0); do + r=$(( r + 0 )) + emoji="\0360\0237\0230\020$r" + c=$(echo -e $emoji) + if [ $r -gt 0 ]; then + for i in $(jot $r); do + name="$name$i" + done + fi + count=$(( count - 1 )) + if [ "$count" -gt 0 ]; then + name="$name$c" + fi + done + echo "$name" +} + +( + set -e + + mdconfig -u $MDUNIT -t malloc -s 512m + newfs_msdos -c 8 -F 32 /dev/md$MDUNIT > /dev/null 2>&1 + mkdir -p $FS + mount_msdosfs -L $LOCALE /dev/md$MDUNIT $FS + + mkdir -p $FS/test + cd $FS/test + + for i in $(jot $FILES); do + testfiles="$testfiles +$(randomfilename)" + done + + testfiles=$(echo "$testfiles" | grep "." | sort -R | uniq) + + for f in $testfiles; do + echo "$f" > $f + done + for f in $(echo "$testfiles" | sort -R); do + cp $f $f.tmp + done + for f in $(echo "$testfiles" | sort -R); do + mv $f.tmp $f + done + for f in $(echo "$testfiles" | sort -R); do + rm $f + done +) + +failed=$? + +cd + +[ "$failed" -ne 0 ] && ls $FS/test + +umount /dev/md$MDUNIT + +#[ "$failed" -ne 0 ] && hd /dev/md$MDUNIT > /tmp/msdos24.dump + +fsck_msdosfs -y /dev/md$MDUNIT + +mdconfig -d -u $MDUNIT 2>/dev/null + +exit $failed diff --git a/tools/tools/nanobsd/defaults.sh b/tools/tools/nanobsd/defaults.sh index adeb83365050..3bbbcf472ae7 100755 --- a/tools/tools/nanobsd/defaults.sh +++ b/tools/tools/nanobsd/defaults.sh @@ -718,8 +718,8 @@ nano_makefs() { image=$4 dir=$5 - makefs ${options} -F "${metalog}" -N "${NANO_WORLDDIR}/etc" \ - -s "${size}b" -T "${NANO_TIMESTAMP}" -t ffs "${image}" "${dir}" + makefs -t ffs ${options} -F "${metalog}" -N "${NANO_WORLDDIR}/etc" \ + -R "${size}b" -T "${NANO_TIMESTAMP}" "${image}" "${dir}" } # Convenient spot to work around any umount issues that your build environment @@ -883,6 +883,7 @@ cust_comconsole() { # Tell loader to use serial console early. echo "${NANO_BOOT2CFG}" > ${NANO_WORLDDIR}/boot.config + tgt_touch boot.config } ####################################################################### diff --git a/tools/tools/nanobsd/legacy.sh b/tools/tools/nanobsd/legacy.sh index 06dce15ffa0a..467c7b845672 100644 --- a/tools/tools/nanobsd/legacy.sh +++ b/tools/tools/nanobsd/legacy.sh @@ -31,6 +31,10 @@ [ -n "$NANO_SECTS" ] || NANO_SECTS=63 [ -n "$NANO_HEADS" ] || NANO_HEADS=16 +# The first partition should start at offset 16, +# because the first 16 sectors are reserved for metadata. +METADATA_SECTS=16 + # Functions and variable definitions used by the legacy nanobsd # image building system. @@ -92,47 +96,36 @@ calculate_partitioning() { ' > ${NANO_LOG}/_.partitioning } -_xxx_adjust_code_size() -{ - # XXX adjust the CODE_SIZE value by rounding it up to - # a bsize of 32768 (DFL_BLKSIZE). - # Otherwise makefs -s will fail because of the guard - # introduced in 5ad283b3c60d. - codesize=$1 - bsize=32768 - echo $(( ((codesize + (bsize - 1)) / bsize) * bsize )) -} - create_code_slice() { pprint 2 "build code slice" pprint 3 "log: ${NANO_OBJ}/_.cs" ( - IMG=${NANO_DISKIMGDIR}/_.disk.image + IMG=${NANO_DISKIMGDIR}/${NANO_IMG1NAME} MNT=${NANO_OBJ}/_.mnt mkdir -p ${MNT} - CODE_SIZE=`head -n 1 ${NANO_LOG}/_.partitioning | awk '{ print $2 }'` + CODE_SIZE=$(awk '$3 == 1 {print $2}' "${NANO_LOG}/_.partitioning") if [ "${NANO_MD_BACKING}" = "swap" ] ; then - MD=`mdconfig -a -t swap -s ${CODE_SIZE} -x ${NANO_SECTS} \ - -y ${NANO_HEADS}` + MD=$(mdconfig -a -t swap -s ${CODE_SIZE} -x ${NANO_SECTS} \ + -y ${NANO_HEADS}) else echo "Creating md backing file..." rm -f ${IMG} dd if=/dev/zero of=${IMG} seek=${CODE_SIZE} count=0 - MD=`mdconfig -a -t vnode -f ${IMG} -x ${NANO_SECTS} \ - -y ${NANO_HEADS}` + MD=$(mdconfig -a -t vnode -f ${IMG} -x ${NANO_SECTS} \ + -y ${NANO_HEADS}) fi trap "echo 'Running exit trap code' ; df -i ${MNT} ; nano_umount ${MNT} || true ; mdconfig -d -u $MD" 1 2 15 EXIT gpart create -s bsd "${MD}" - gpart add -t freebsd-ufs -b 16 "${MD}" + gpart add -t freebsd-ufs -b "${METADATA_SECTS}" "${MD}" if [ -f ${NANO_WORLDDIR}/boot/boot ]; then - echo "Making bootable partition" - gpart bootcode -b ${NANO_WORLDDIR}/boot/boot ${MD} + echo "Making bootable partition" + gpart bootcode -b ${NANO_WORLDDIR}/boot/boot ${MD} else - echo "Partition will not be bootable" + echo "Partition will not be bootable" fi gpart list ${MD} @@ -145,8 +138,8 @@ create_code_slice() { nano_umount ${MNT} if [ "${NANO_MD_BACKING}" = "swap" ] ; then - echo "Writing out _.disk.image..." - dd conv=sparse if=/dev/${MD} of=${NANO_DISKIMGDIR}/_.disk.image bs=64k + echo "Writing out ${NANO_IMG1NAME}..." + dd conv=sparse if=/dev/${MD} of=${IMG} bs=64k fi mdconfig -d -u $MD @@ -160,9 +153,8 @@ _create_code_slice() { pprint 3 "log: ${NANO_OBJ}/_.cs" ( - IMG=${NANO_DISKIMGDIR}/_.disk.image - CODE_SIZE=$(head -n 1 "${NANO_LOG}/_.partitioning" | awk '{ print $2 }') - CODE_SIZE=$(_xxx_adjust_code_size "$CODE_SIZE") + IMG=${NANO_DISKIMGDIR}/${NANO_IMG1NAME} + CODE_SIZE=$(awk '$3 == 1 {print $2}' "${NANO_LOG}/_.partitioning") echo "Writing code image..." if [ -f "${NANO_WORLDDIR}/boot/boot" ]; then @@ -172,12 +164,12 @@ _create_code_slice() { echo "Partition will not be bootable" fi nano_makefs "-DxZ ${NANO_MAKEFS} -o minfree=0,optimization=space" \ - "${NANO_METALOG}" "${CODE_SIZE}" "${NANO_OBJ}/_.disk.part" \ - "${NANO_WORLDDIR}" - mkimg -s bsd \ + "${NANO_METALOG}" "$(( CODE_SIZE - METADATA_SECTS ))" \ + "${NANO_OBJ}/_.disk.part" "${NANO_WORLDDIR}" + mkimg -s bsd -S 512 --capacity $(( CODE_SIZE * 512 )) \ ${bootcode} \ -p freebsd-ufs:="${NANO_OBJ}/_.disk.part" \ - -o "${NANO_DISKIMGDIR}/_.disk.image" + -o "${IMG}" rm -f "${NANO_OBJ}/_.disk.part" ) > ${NANO_OBJ}/_.cs 2>&1 @@ -194,14 +186,14 @@ create_diskimage() { mkdir -p ${MNT} if [ "${NANO_MD_BACKING}" = "swap" ] ; then - MD=`mdconfig -a -t swap -s ${NANO_MEDIASIZE} -x ${NANO_SECTS} \ - -y ${NANO_HEADS}` + MD=$(mdconfig -a -t swap -s ${NANO_MEDIASIZE} -x ${NANO_SECTS} \ + -y ${NANO_HEADS}) else echo "Creating md backing file..." rm -f ${IMG} dd if=/dev/zero of=${IMG} seek=${NANO_MEDIASIZE} count=0 - MD=`mdconfig -a -t vnode -f ${IMG} -x ${NANO_SECTS} \ - -y ${NANO_HEADS}` + MD=$(mdconfig -a -t vnode -f ${IMG} -x ${NANO_SECTS} \ + -y ${NANO_HEADS}) fi awk ' @@ -231,7 +223,7 @@ create_diskimage() { fi echo "Writing code image..." - dd conv=sparse if=${NANO_DISKIMGDIR}/_.disk.image of=/dev/${MD}${NANO_SLICE_ROOT} bs=64k + dd conv=sparse if=${NANO_DISKIMGDIR}/${NANO_IMG1NAME} of=/dev/${MD}${NANO_SLICE_ROOT} bs=64k if [ $NANO_IMAGES -gt 1 -a $NANO_INIT_IMG2 -gt 0 ] ; then # Duplicate to second image (if present) @@ -290,8 +282,9 @@ _create_diskimage() { ( local altroot bootloader cfgimage dataimage diskimage - CODE_SIZE=$(head -n 1 "${NANO_LOG}/_.partitioning" | awk '{ print $2 }') - CODE_SIZE=$(_xxx_adjust_code_size "$CODE_SIZE") + CODE_SIZE=$(awk '$3 == 1 {print $2}' "${NANO_LOG}/_.partitioning") + CONF_SIZE=$(awk '$3 == 3 {print $2}' "${NANO_LOG}/_.partitioning") + DATA_SIZE=$(awk '$3 == 4 {print $2}' "${NANO_LOG}/_.partitioning") IMG=${NANO_DISKIMGDIR}/${NANO_IMGNAME} if [ -f "${NANO_WORLDDIR}/${NANO_BOOTLOADER}" ]; then @@ -300,34 +293,35 @@ _create_diskimage() { echo "Image will not be bootable" fi - diskimage="-p freebsd:=${NANO_DISKIMGDIR}/_.disk.image" + diskimage="-p freebsd:=${NANO_DISKIMGDIR}/${NANO_IMG1NAME}:$(( NANO_SECTS * 512 ))" - if [ "$NANO_IMAGES" -gt 1 ] && [ "$NANO_INIT_IMG2" -gt 0 ] ; then - echo "Duplicating to second image..." - tgt_switch_root_fstab "${NANO_SLICE_ROOT}" "${NANO_SLICE_ALTROOT}" - nano_makefs "-DxZ ${NANO_MAKEFS} -o minfree=0,optimization=space" \ - "${NANO_METALOG}" "${CODE_SIZE}" "${NANO_OBJ}/_.altroot.part" \ - "${NANO_WORLDDIR}" - tgt_switch_root_fstab "${NANO_SLICE_ALTROOT}" "${NANO_SLICE_ROOT}" - if [ -f "${NANO_WORLDDIR}/boot/boot" ]; then - bootcode="-b ${NANO_WORLDDIR}/boot/boot" + if [ "$NANO_IMAGES" -gt 1 ] ; then + if [ "$NANO_INIT_IMG2" -gt 0 ] ; then + echo "Duplicating to second image..." + tgt_switch_root_fstab "${NANO_SLICE_ROOT}" "${NANO_SLICE_ALTROOT}" + nano_makefs "-DxZ ${NANO_MAKEFS} -o minfree=0,optimization=space" \ + "${NANO_METALOG}" "$(( CODE_SIZE - METADATA_SECTS ))" \ + "${NANO_OBJ}/_.altroot.part" "${NANO_WORLDDIR}" + tgt_switch_root_fstab "${NANO_SLICE_ALTROOT}" "${NANO_SLICE_ROOT}" + if [ -f "${NANO_WORLDDIR}/boot/boot" ]; then + bootcode="-b ${NANO_WORLDDIR}/boot/boot" + fi + mkimg -s bsd -S 512 --capacity $(( CODE_SIZE * 512 )) \ + ${bootcode} \ + -p freebsd-ufs:="${NANO_OBJ}/_.altroot.part" \ + -o "${NANO_OBJ}/_.altroot.image" + altroot="-p freebsd:=${NANO_OBJ}/_.altroot.image:+$(( NANO_SECTS * 512 ))" + rm -f "${NANO_OBJ}/_.altroot.part" + else + altroot="-p freebsd::$(( CODE_SIZE * 512 )):+$(( NANO_SECTS * 512 ))" fi - mkimg -s bsd \ - ${bootcode} \ - -p freebsd-ufs:="${NANO_OBJ}/_.altroot.part" \ - -o "${NANO_OBJ}/_.altroot.image" - altroot="-p freebsd:=${NANO_OBJ}/_.altroot.image" - rm -f "${NANO_OBJ}/_.altroot.part" else altroot="-p-" fi - if [ "${NANO_INIT_IMG2}" -eq 0 ]; then - altroot="-p freebsd::${CODE_SIZE}b" - fi # Create Config slice _populate_cfg_part "${NANO_OBJ}/_.cfg.part" "${NANO_CFGDIR}" \ - "${NANO_SLICE_CFG}" "${NANO_CONFSIZE}" "${NANO_METALOG_CFG}" + "${NANO_SLICE_CFG}" "${CONF_SIZE}" "${NANO_METALOG_CFG}" cfgimage="-p freebsd:=${NANO_OBJ}/_.cfg.part" # Create Data slice, if any. @@ -339,19 +333,18 @@ _create_diskimage() { fi if [ "${NANO_DATASIZE}" -ne 0 ] && [ -n "${NANO_SLICE_DATA}" ] ; then _populate_data_part "${NANO_OBJ}/_.data.part" "${NANO_DATADIR}" \ - "${NANO_SLICE_DATA}" "${NANO_DATASIZE}" "${NANO_METALOG_DATA}" + "${NANO_SLICE_DATA}" "${DATA_SIZE}" "${NANO_METALOG_DATA}" dataimage="-p freebsd:=${NANO_OBJ}/_.data.part" fi echo "Writing out ${NANO_IMGNAME}..." - mkimg -s mbr \ + mkimg -s mbr -S 512 --capacity $(( NANO_MEDIASIZE * 512 )) \ ${bootloader} \ ${diskimage} \ ${altroot} \ ${cfgimage} \ ${dataimage} \ -o ${IMG} - exit ) > ${NANO_LOG}/_.di 2>&1 } diff --git a/usr.bin/cap_mkdb/cap_mkdb.c b/usr.bin/cap_mkdb/cap_mkdb.c index 7ee6d2ddd2b3..019dad1ee72e 100644 --- a/usr.bin/cap_mkdb/cap_mkdb.c +++ b/usr.bin/cap_mkdb/cap_mkdb.c @@ -117,6 +117,7 @@ main(int argc, char *argv[]) if (capdbp->close(capdbp) < 0) err(1, "%s", capname); + free(capname); capname = NULL; exit(0); } @@ -151,7 +152,7 @@ db_build(char **ifiles) data.data = NULL; key.data = NULL; - for (reccnt = 0, bplen = 0; (st = cgetnext(&bp, ifiles)) > 0;) { + for (reccnt = 0, bplen = 0; (st = cgetnext(&bp, ifiles)) > 0; free(bp)) { /* * Allocate enough memory to store record, terminating diff --git a/usr.bin/limits/limits.c b/usr.bin/limits/limits.c index fbb19289e6ca..a6e95e7c37a9 100644 --- a/usr.bin/limits/limits.c +++ b/usr.bin/limits/limits.c @@ -247,7 +247,7 @@ static struct { { "kqueues", login_getcapnum }, { "umtxp", login_getcapnum }, { "pipebuf", login_getcapnum }, - { "vmms", login_getcapnum }, + { "vms", login_getcapnum }, }; _Static_assert(nitems(resources) == RLIM_NLIMITS, diff --git a/usr.bin/vtfontcvt/vtfontcvt.c b/usr.bin/vtfontcvt/vtfontcvt.c index 1e388ef36276..f39076b09be6 100644 --- a/usr.bin/vtfontcvt/vtfontcvt.c +++ b/usr.bin/vtfontcvt/vtfontcvt.c @@ -987,7 +987,7 @@ done: if (rv != 0) perror(filename); fclose(fp); - return (0); + return (rv); } static void diff --git a/usr.bin/wall/ttymsg.c b/usr.bin/wall/ttymsg.c index 4f1c367c505f..3eee72d6805b 100644 --- a/usr.bin/wall/ttymsg.c +++ b/usr.bin/wall/ttymsg.c @@ -29,15 +29,16 @@ * SUCH DAMAGE. */ - - #include <sys/types.h> +#include <sys/stat.h> #include <sys/uio.h> + #include <dirent.h> #include <errno.h> #include <fcntl.h> #include <paths.h> #include <signal.h> +#include <stdbool.h> #include <stdio.h> #include <string.h> #include <stdlib.h> @@ -46,65 +47,72 @@ #include "ttymsg.h" /* - * Display the contents of a uio structure on a terminal. Used by wall(1), - * syslogd(8), and talkd(8). Forks and finishes in child if write would block, - * waiting up to tmout seconds. Returns pointer to error string on unexpected - * error; string is not newline-terminated. Various "normal" errors are - * ignored (exclusive-use, lack of permission, etc.). + * Display the contents of a uio structure on a terminal. If shout is + * non-zero, do so even if the terminal has messages disabled. Used by + * wall(1), syslogd(8), and talkd(8). Forks and finishes in child if + * write would block, waiting up to timeout seconds. Various "normal" + * errors are ignored (exclusive-use, lack of permission, etc.). */ -const char * -ttymsg(struct iovec *iov, int iovcnt, const char *line, int tmout) +int +ttymsg(struct iovec *iov, int iovcnt, const char *tty, int timeout, + bool shout) { struct iovec localiov[TTYMSG_IOV_MAX]; - ssize_t left, wret; - int cnt, fd; - char device[MAXNAMLEN] = _PATH_DEV; - static char errbuf[1024]; - char *p; + struct stat sb; + ssize_t wret; + size_t resid; + int cnt, dd, fd, serrno; int forked; forked = 0; - if (iovcnt > (int)(sizeof(localiov) / sizeof(localiov[0]))) - return ("too many iov's (change code in wall/ttymsg.c)"); - - strlcat(device, line, sizeof(device)); - p = device + sizeof(_PATH_DEV) - 1; - if (strncmp(p, "pts/", 4) == 0) - p += 4; - if (strchr(p, '/') != NULL) { - /* A slash is an attempt to break security... */ - (void) snprintf(errbuf, sizeof(errbuf), - "Too many '/' in \"%s\"", device); - return (errbuf); + if (iovcnt > (int)(sizeof(localiov) / sizeof(localiov[0]))) { + errno = EFBIG; + return (-1); } - /* - * open will fail on slip lines or exclusive-use lines - * if not running as root; not an error. - */ - if ((fd = open(device, O_WRONLY|O_NONBLOCK, 0)) < 0) { - if (errno == EBUSY || errno == EACCES) - return (NULL); - (void) snprintf(errbuf, sizeof(errbuf), "%s: %s", device, - strerror(errno)); - return (errbuf); + dd = open(_PATH_DEV, O_SEARCH | O_DIRECTORY); + if (dd < 0) + return (-1); + fd = openat(dd, tty, O_WRONLY | O_NONBLOCK | O_RESOLVE_BENEATH); + if (fd < 0) { + serrno = errno; + close(dd); + /* + * open will fail on slip lines or exclusive-use lines + * if not running as root; not an error. + */ + if (serrno == EBUSY || serrno == EACCES) + return (0); + errno = serrno; + return (-1); + } + close(dd); + if (!shout) { + if (fstat(fd, &sb) != 0) { + serrno = errno; + close(fd); + errno = serrno; + return (-1); + } + if ((sb.st_mode & S_IWGRP) == 0) { + close(fd); + return (0); + } } - for (cnt = 0, left = 0; cnt < iovcnt; ++cnt) - left += iov[cnt].iov_len; + for (cnt = 0, resid = 0; cnt < iovcnt; ++cnt) + resid += iov[cnt].iov_len; - for (;;) { + do { wret = writev(fd, iov, iovcnt); - if (wret >= left) - break; if (wret >= 0) { - left -= wret; + resid -= wret; if (iov != localiov) { - bcopy(iov, localiov, + bcopy(iov, localiov, iovcnt * sizeof(struct iovec)); iov = localiov; } - for (cnt = 0; (size_t)wret >= iov->iov_len; ++cnt) { + while ((size_t)wret >= iov->iov_len) { wret -= iov->iov_len; ++iov; --iovcnt; @@ -124,21 +132,21 @@ ttymsg(struct iovec *iov, int iovcnt, const char *line, int tmout) } cpid = fork(); if (cpid < 0) { - (void) snprintf(errbuf, sizeof(errbuf), - "fork: %s", strerror(errno)); + serrno = errno; (void) close(fd); - return (errbuf); + errno = serrno; + return (-1); } if (cpid) { /* parent */ (void) close(fd); - return (NULL); + return (0); } forked++; - /* wait at most tmout seconds */ + /* wait at most timeout seconds */ (void) signal(SIGALRM, SIG_DFL); (void) signal(SIGTERM, SIG_DFL); /* XXX */ (void) sigsetmask(0); - (void) alarm((u_int)tmout); + (void) alarm((u_int)timeout); (void) fcntl(fd, F_SETFL, 0); /* clear O_NONBLOCK */ continue; } @@ -146,18 +154,18 @@ ttymsg(struct iovec *iov, int iovcnt, const char *line, int tmout) * We get ENODEV on a slip line if we're running as root, * and EIO if the line just went away. */ - if (errno == ENODEV || errno == EIO) + serrno = errno; + if (serrno == ENODEV || serrno == EIO) break; (void) close(fd); if (forked) _exit(1); - (void) snprintf(errbuf, sizeof(errbuf), - "%s: %s", device, strerror(errno)); - return (errbuf); - } + errno = serrno; + return (-1); + } while (resid > 0); (void) close(fd); if (forked) _exit(0); - return (NULL); + return (0); } diff --git a/usr.bin/wall/ttymsg.h b/usr.bin/wall/ttymsg.h index be97592f5e1c..840a49875fbb 100644 --- a/usr.bin/wall/ttymsg.h +++ b/usr.bin/wall/ttymsg.h @@ -1,4 +1,4 @@ #define TTYMSG_IOV_MAX 32 -const char *ttymsg(struct iovec *, int, const char *, int); +int ttymsg(struct iovec *, int, const char *, int, bool); diff --git a/usr.bin/wall/wall.c b/usr.bin/wall/wall.c index fcfcdcb3fbce..e29b896f838f 100644 --- a/usr.bin/wall/wall.c +++ b/usr.bin/wall/wall.c @@ -44,6 +44,7 @@ #include <locale.h> #include <paths.h> #include <pwd.h> +#include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -90,7 +91,6 @@ main(int argc, char *argv[]) struct wallgroup *g; struct group *grp; char **np; - const char *p; struct passwd *pw; (void)setlocale(LC_CTYPE, ""); @@ -158,8 +158,8 @@ main(int argc, char *argv[]) if (ingroup == 0) continue; } - if ((p = ttymsg(&iov, 1, utmp->ut_line, 60*5)) != NULL) - warnx("%s", p); + if (ttymsg(&iov, 1, utmp->ut_line, 60 * 5, 1) != 0) + warn("%s", utmp->ut_line); } exit(0); } diff --git a/usr.bin/xinstall/Makefile b/usr.bin/xinstall/Makefile index fd95b97160fe..c05a50420c82 100644 --- a/usr.bin/xinstall/Makefile +++ b/usr.bin/xinstall/Makefile @@ -11,7 +11,7 @@ MAN= install.1 CFLAGS+= -I${SRCTOP}/contrib/mtree CFLAGS+= -I${SRCTOP}/lib/libnetbsd -LIBADD= md +LIBADD= md util CFLAGS+= -DWITH_MD5 -DWITH_RIPEMD160 .ifdef BOOTSTRAPPING diff --git a/usr.bin/xinstall/install.1 b/usr.bin/xinstall/install.1 index c923321f20fe..169bf4ef11fe 100644 --- a/usr.bin/xinstall/install.1 +++ b/usr.bin/xinstall/install.1 @@ -33,7 +33,7 @@ .Nd install binaries .Sh SYNOPSIS .Nm -.Op Fl bCcpSsUv +.Op Fl bCcpSsUvz .Op Fl B Ar suffix .Op Fl D Ar destdir .Op Fl f Ar flags @@ -45,9 +45,10 @@ .Op Fl N Ar dbdir .Op Fl o Ar owner .Op Fl T Ar tags +.Op Fl z Ar size .Ar file1 file2 .Nm -.Op Fl bCcpSsUv +.Op Fl bCcpSsUvz .Op Fl B Ar suffix .Op Fl D Ar destdir .Op Fl f Ar flags @@ -59,6 +60,7 @@ .Op Fl N Ar dbdir .Op Fl o Ar owner .Op Fl T Ar tags +.Op Fl z Ar size .Ar file1 ... fileN directory .Nm .Fl d @@ -264,6 +266,13 @@ Cause .Nm to be verbose, showing files as they are installed or backed up. +.It Fl z Ar maxsize +Limit the comparison feature of +.Fl C +to files no larger than +.Ar maxsize . +Files exceeding this limit bypass the comparison step and are directly overwritten. +The default maximum size is 128MiB. .El .Pp By default, diff --git a/usr.bin/xinstall/xinstall.c b/usr.bin/xinstall/xinstall.c index 1aed8c1b24e4..d389bdbe3737 100644 --- a/usr.bin/xinstall/xinstall.c +++ b/usr.bin/xinstall/xinstall.c @@ -57,6 +57,7 @@ #include <string.h> #include <sysexits.h> #include <unistd.h> +#include <util.h> #include <vis.h> #include "mtree.h" @@ -87,7 +88,7 @@ #define HAVE_STRUCT_STAT_ST_FLAGS 0 #endif -#define MAX_CMP_SIZE (16 * 1024 * 1024) +#define MAX_CMP_SIZE (128 * 1024 * 1024) #define LN_ABSOLUTE 0x01 #define LN_RELATIVE 0x02 @@ -137,6 +138,7 @@ static FILE *metafp; static const char *group, *owner; static const char *suffix = BACKUP_SUFFIX; static char *destdir, *digest, *fflags, *metafile, *tags; +static size_t max_compare_size = MAX_CMP_SIZE; static int compare(int, const char *, size_t, int, const char *, size_t, char **); @@ -168,12 +170,13 @@ main(int argc, char *argv[]) u_int iflags; char *p; const char *to_name; + uint64_t num; fset = 0; iflags = 0; set = NULL; group = owner = NULL; - while ((ch = getopt(argc, argv, "B:bCcD:df:g:h:l:M:m:N:o:pSsT:Uv")) != + while ((ch = getopt(argc, argv, "B:bCcD:df:g:h:l:M:m:N:o:pSsT:Uvz:")) != -1) switch((char)ch) { case 'B': @@ -270,6 +273,13 @@ main(int argc, char *argv[]) case 'v': verbose = 1; break; + case 'z': + if (expand_number(optarg, &num) != 0 || num == 0) { + errx(EX_USAGE, "invalid max compare filesize:" + " %s", optarg); + } + max_compare_size = num; + break; case '?': default: usage(); @@ -1092,7 +1102,7 @@ compare(int from_fd, const char *from_name __unused, size_t from_len, do_digest = (digesttype != DIGEST_NONE && dresp != NULL && *dresp == NULL); - if (from_len <= MAX_CMP_SIZE) { + if (from_len <= max_compare_size) { static char *buf, *buf1, *buf2; static size_t bufsize; int n1, n2; @@ -1484,11 +1494,11 @@ usage(void) { (void)fprintf(stderr, "usage: install [-bCcpSsUv] [-f flags] [-g group] [-m mode] [-o owner]\n" -" [-M log] [-D dest] [-h hash] [-T tags]\n" +" [-M log] [-D dest] [-h hash] [-T tags] [-z maxcmpsize]\n" " [-B suffix] [-l linkflags] [-N dbdir]\n" " file1 file2\n" " install [-bCcpSsUv] [-f flags] [-g group] [-m mode] [-o owner]\n" -" [-M log] [-D dest] [-h hash] [-T tags]\n" +" [-M log] [-D dest] [-h hash] [-T tags] [-z maxcmpsize]\n" " [-B suffix] [-l linkflags] [-N dbdir]\n" " file1 ... fileN directory\n" " install -dU [-vU] [-g group] [-m mode] [-N dbdir] [-o owner]\n" diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile index 25fb71b5768f..b86c56991039 100644 --- a/usr.sbin/bhyve/Makefile +++ b/usr.sbin/bhyve/Makefile @@ -53,6 +53,7 @@ SRCS= \ pci_virtio_net.c \ pci_virtio_rnd.c \ pci_virtio_scsi.c \ + pci_virtio_scsi_ctl.c \ pci_xhci.c \ qemu_fwcfg.c \ qemu_loader.c \ diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8 index 5118974baace..ec8c880b0243 100644 --- a/usr.sbin/bhyve/bhyve.8 +++ b/usr.sbin/bhyve/bhyve.8 @@ -2,6 +2,7 @@ .\" SPDX-License-Identifier: BSD-2-Clause .\" .\" Copyright (c) 2013 Peter Grehan +.\" Copyright (c) 2026 Hans Rosenfeld .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -680,23 +681,86 @@ In that case, this feature doesn't work as expected. .Bl -bullet .Sm off .It -.Pa /dev/cam/ctl Oo Ar pp Cm \&. Ar vp Oc Oo Cm \&, Ar scsi-device-options Oc +.Oo Cm target Ns = Ns Oo ID : Oc Ar path Ns Oc +.Oo Cm \&, Ar scsi-device-options Oc +.Oo Cm \&, Ar backend-specific-options Oc .Sm on .El .Pp +Multiple +.Pa target +parameters may be specified, each configuring a different +.Ar path +as a distinct SCSI target. +If the +.Pa target +.Ar ID +is not explicitly configured for a +.Pa target , +the +.Pa target +will be assigned the next sequential +.Ar ID +following the highest +.Pa target +.Ar ID +used at that point, or 0 if it is the first target configured. +All +.Pa target +.Ar ID Ns s +must be unique per instance. +The meaning of the +.Ar path +argument is specific to each backend: +.Bl -column "Backend" "/dev/cam/ctl[pp.vp]" +.It Sy Backend Ta Sy Path Ta Sy Description +.It ctl Ta Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc Ta +The path of a CAM target layer (CTL) device node. +If no target is configured, a single target backed by +.Qq /dev/cam/ctl +will be configured by default. +.El +.Pp The .Ar scsi-device-options are: .Bl -tag -width 10n -.It Cm iid= Ns Ar IID -Initiator ID to use when sending requests to specified CTL port. -The default value is 0. +.It Cm backend= Ns Ar backend +The virtio-scsi backend to use. +The backend name is case-insensitive. +There is currently only one backend +.Qq ctl , +which is also the default backend. .It Li bootindex= Ns Ar index Add the device to the boot order at .Ar index . A fw_cfg file is used to specify the boot order. The guest firmware may ignore or not support this fw_cfg file. In that case, this feature doesn't work as expected. +.It Li ctl_ringsz= Ns Ar ringsz +The ring size to use for the control queue. +.It Li evt_ringsz= Ns Ar ringsz +The ring size to use for the event queue. +.It Li req_ringsz= Ns Ar ringsz +The ring size to use for each I/O request queue. +.It Li num_queues= Ns Ar num +The number of I/O request queues to use. +.It Li seg_max= Ns Ar num +The maximum number of segments allowed in a single command. +.It Li thr_per_q= Ns Ar num +The number of parallel request processing threads per I/O request queue. +.El +.Pp +The +.Ar backend-specific-options +for the +.Sy CTL +backend are: +.Bl -tag -width 10n +.It Cm iid= Ns Ar IID +Initiator ID to use when sending requests to +.Sy CTL . +The default value is 0. .El .Ss 9P device backends .Bl -bullet diff --git a/usr.sbin/bhyve/bhyve_config.5 b/usr.sbin/bhyve/bhyve_config.5 index 429ce3e38138..8bfe9afd0b24 100644 --- a/usr.sbin/bhyve/bhyve_config.5 +++ b/usr.sbin/bhyve/bhyve_config.5 @@ -1,6 +1,7 @@ .\" SPDX-License-Identifier: BSD-2-Clause .\" .\" Copyright (c) 2021 John H. Baldwin <jhb@FreeBSD.org> +.\" Copyright (c) 2026 Hans Rosenfeld .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions @@ -761,13 +762,61 @@ If specified, it must be a unicast MAC address. The largest supported MTU advertised to the guest. .El .Ss VirtIO SCSI Settings -.Bl -column "Name" "integer" "Default" +.Bl -column "num_queues" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description -.It Va dev Ta path Ta Ta -The path of a CAM target layer (CTL) device to export: -.Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc . -.It Va iid Ta integer Ta 0 Ta -Initiator ID to use when sending requests to the CTL port. +.It Va backend Ta string Ta ctl Ta +The virtio-scsi backend to use (case-insensitive). +.It Va ctl_ringsz Ta integer Ta 64 Ta +The ring size of the control queue. +.It Va evt_ringsz Ta integer Ta 64 Ta +The ring size of the event queue. +.It Va req_ringsz Ta integer Ta 64 Ta +The ring size of each I/O request queue. +.It Va num_queues Ta integer Ta 1 Ta +The number of I/O request queues. +.It Va seg_max Ta integer Ta 64 Ta +The maximum number of segments allowed in a single command. +.It Va thr_per_q Ta integer Ta 16 Ta +The number of parallel request processing threads per I/O request queue. +.It Va target Ta Oo Va ID : Oc Ns path Ta /dev/cam/ctl Ta +The backend +.Ar path +of a target to configure. +Optionally, a numeric target +.Ar ID +in the range from 0 to 255 may be specified before the +.Ar path , +separated by a colon. +.El +.Pp +The +.Va target +variable may be specified multiple times with different +.Ar path +arguments to configure multiple distinct SCSI targets. +If not explicitly configured, the target +.Ar ID Ns s +will be assigned sequentially beginning with the highest target +.Ar ID +configured so far, or 0 for the first target configured. +The target +.Ar ID Ns s +must be unique within each virtio-scsi instance. +.Pp +The meaning of the +.Ar path +argument is specific to each backend: +.Bl -column "Backend" "/dev/cam/ctl[pp.vp]" +.It Sy Backend Ta Sy Path Ta Sy Description +.It ctl Ta Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc Ta +The path of a CAM target layer (CTL) device to configure as a target. +.El +.Pp +The following backend-specific variables are supported for VirtIO SCSI: +.Bl -column "Backend" "Name" "integer" "Default" +.It Sy Backend Ta Sy Name Ta Sy Format Ta Sy Default Ta Sy Description +.It ctl Ta Va iid Ta integer Ta 0 Ta +Initiator ID to use when sending requests to CTL. .El .Sh SEE ALSO .Xr expand_number 3 , diff --git a/usr.sbin/bhyve/pci_virtio_scsi.c b/usr.sbin/bhyve/pci_virtio_scsi.c index 5fb867e5eae7..97f1ff3c65d8 100644 --- a/usr.sbin/bhyve/pci_virtio_scsi.c +++ b/usr.sbin/bhyve/pci_virtio_scsi.c @@ -35,7 +35,6 @@ #include <sys/uio.h> #include <sys/time.h> #include <sys/queue.h> -#include <sys/sbuf.h> #include <errno.h> #include <fcntl.h> @@ -48,16 +47,6 @@ #include <pthread.h> #include <pthread_np.h> -#include <cam/scsi/scsi_all.h> -#include <cam/scsi/scsi_message.h> -#include <cam/ctl/ctl.h> -#include <cam/ctl/ctl_io.h> -#include <cam/ctl/ctl_backend.h> -#include <cam/ctl/ctl_ioctl.h> -#include <cam/ctl/ctl_util.h> -#include <cam/ctl/ctl_scsi_all.h> -#include <camlib.h> - #include "bhyverun.h" #include "config.h" #include "debug.h" @@ -65,206 +54,18 @@ #include "virtio.h" #include "iov.h" -#define VTSCSI_RINGSZ 64 -#define VTSCSI_REQUESTQ 1 -#define VTSCSI_THR_PER_Q 16 -#define VTSCSI_MAXQ (VTSCSI_REQUESTQ + 2) -#define VTSCSI_MAXSEG 64 - -#define VTSCSI_IN_HEADER_LEN(_sc) \ - (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size) - -#define VTSCSI_OUT_HEADER_LEN(_sc) \ - (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) - -#define VIRTIO_SCSI_MAX_CHANNEL 0 -#define VIRTIO_SCSI_MAX_TARGET 0 -#define VIRTIO_SCSI_MAX_LUN 16383 - -#define VIRTIO_SCSI_F_INOUT (1 << 0) -#define VIRTIO_SCSI_F_HOTPLUG (1 << 1) -#define VIRTIO_SCSI_F_CHANGE (1 << 2) - -static int pci_vtscsi_debug = 0; -#define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params) -#define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params) +#include "pci_virtio_scsi.h" -struct pci_vtscsi_config { - uint32_t num_queues; - uint32_t seg_max; - uint32_t max_sectors; - uint32_t cmd_per_lun; - uint32_t event_info_size; - uint32_t sense_size; - uint32_t cdb_size; - uint16_t max_channel; - uint16_t max_target; - uint32_t max_lun; -} __attribute__((packed)); - -/* - * I/O request state and I/O request queues - * - * In addition to the control queue and notification queues, each virtio-scsi - * device instance has at least one I/O request queue, the state of which is - * is kept in an array of struct pci_vtscsi_queue in the device softc. - * - * Currently there is only one I/O request queue, but it's trivial to support - * more than one. - * - * Each pci_vtscsi_queue has VTSCSI_RINGSZ pci_vtscsi_request structures pre- - * allocated on vsq_free_requests. For each I/O request coming in on the I/O - * virtqueue, the request queue handler will take a pci_vtscsi_request off - * vsq_free_requests, fills in the data from the I/O virtqueue, puts it on - * vsq_requests, and signals vsq_cv. - * - * There are VTSCSI_THR_PER_Q worker threads for each pci_vtscsi_queue which - * wait on vsq_cv. When signalled, they repeatedly take one pci_vtscsi_request - * off vsq_requests, construct a ctl_io for it, and hand it off to the CTL ioctl - * Interface, which processes it synchronously. After completion of the request, - * the pci_vtscsi_request is re-initialized and put back onto vsq_free_requests. - * - * The worker threads exit when vsq_cv is signalled after vsw_exiting was set. - * - * There are three mutexes to coordinate the accesses to an I/O request queue: - * - vsq_rmtx protects vsq_requests and must be held when waiting on vsq_cv - * - vsq_fmtx protects vsq_free_requests - * - vsq_qmtx must be held when operating on the underlying virtqueue, vsq_vq - */ -STAILQ_HEAD(pci_vtscsi_req_queue, pci_vtscsi_request); - -struct pci_vtscsi_queue { - struct pci_vtscsi_softc * vsq_sc; - struct vqueue_info * vsq_vq; - pthread_mutex_t vsq_rmtx; - pthread_mutex_t vsq_fmtx; - pthread_mutex_t vsq_qmtx; - pthread_cond_t vsq_cv; - struct pci_vtscsi_req_queue vsq_requests; - struct pci_vtscsi_req_queue vsq_free_requests; - LIST_HEAD(, pci_vtscsi_worker) vsq_workers; -}; - -struct pci_vtscsi_worker { - struct pci_vtscsi_queue * vsw_queue; - pthread_t vsw_thread; - bool vsw_exiting; - LIST_ENTRY(pci_vtscsi_worker) vsw_link; -}; - -struct pci_vtscsi_request { - struct pci_vtscsi_queue * vsr_queue; - struct iovec vsr_iov[VTSCSI_MAXSEG + SPLIT_IOV_ADDL_IOV]; - struct iovec * vsr_iov_in; - struct iovec * vsr_iov_out; - struct iovec * vsr_data_iov_in; - struct iovec * vsr_data_iov_out; - struct pci_vtscsi_req_cmd_rd * vsr_cmd_rd; - struct pci_vtscsi_req_cmd_wr * vsr_cmd_wr; - union ctl_io * vsr_ctl_io; - size_t vsr_niov_in; - size_t vsr_niov_out; - size_t vsr_data_niov_in; - size_t vsr_data_niov_out; - uint32_t vsr_idx; - STAILQ_ENTRY(pci_vtscsi_request) vsr_link; -}; - -/* - * Per-device softc - */ -struct pci_vtscsi_softc { - struct virtio_softc vss_vs; - struct vqueue_info vss_vq[VTSCSI_MAXQ]; - struct pci_vtscsi_queue vss_queues[VTSCSI_REQUESTQ]; - pthread_mutex_t vss_mtx; - int vss_iid; - int vss_ctl_fd; - uint32_t vss_features; - struct pci_vtscsi_config vss_config; +enum pci_vtscsi_walk { + PCI_VTSCSI_WALK_CONTINUE = 0, + PCI_VTSCSI_WALK_STOP, }; -#define VIRTIO_SCSI_T_TMF 0 -#define VIRTIO_SCSI_T_TMF_ABORT_TASK 0 -#define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1 -#define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2 -#define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3 -#define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4 -#define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5 -#define VIRTIO_SCSI_T_TMF_QUERY_TASK 6 -#define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7 - -/* command-specific response values */ -#define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0 -#define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10 -#define VIRTIO_SCSI_S_FUNCTION_REJECTED 11 - -struct pci_vtscsi_ctrl_tmf { - const uint32_t type; - const uint32_t subtype; - const uint8_t lun[8]; - const uint64_t id; - uint8_t response; -} __attribute__((packed)); - -#define VIRTIO_SCSI_T_AN_QUERY 1 -#define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2 -#define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4 -#define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8 -#define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16 -#define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32 -#define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64 - -struct pci_vtscsi_ctrl_an { - const uint32_t type; - const uint8_t lun[8]; - const uint32_t event_requested; - uint32_t event_actual; - uint8_t response; -} __attribute__((packed)); - -/* command-specific response values */ -#define VIRTIO_SCSI_S_OK 0 -#define VIRTIO_SCSI_S_OVERRUN 1 -#define VIRTIO_SCSI_S_ABORTED 2 -#define VIRTIO_SCSI_S_BAD_TARGET 3 -#define VIRTIO_SCSI_S_RESET 4 -#define VIRTIO_SCSI_S_BUSY 5 -#define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6 -#define VIRTIO_SCSI_S_TARGET_FAILURE 7 -#define VIRTIO_SCSI_S_NEXUS_FAILURE 8 -#define VIRTIO_SCSI_S_FAILURE 9 -#define VIRTIO_SCSI_S_INCORRECT_LUN 12 +typedef enum pci_vtscsi_walk pci_vtscsi_walk_t; +typedef pci_vtscsi_walk_t pci_vtscsi_walk_request_queue_cb_t( + struct pci_vtscsi_queue *, struct pci_vtscsi_request *, void *); -/* task_attr */ -#define VIRTIO_SCSI_S_SIMPLE 0 -#define VIRTIO_SCSI_S_ORDERED 1 -#define VIRTIO_SCSI_S_HEAD 2 -#define VIRTIO_SCSI_S_ACA 3 - -struct pci_vtscsi_event { - uint32_t event; - uint8_t lun[8]; - uint32_t reason; -} __attribute__((packed)); - -struct pci_vtscsi_req_cmd_rd { - const uint8_t lun[8]; - const uint64_t id; - const uint8_t task_attr; - const uint8_t prio; - const uint8_t crn; - const uint8_t cdb[]; -} __attribute__((packed)); - -struct pci_vtscsi_req_cmd_wr { - uint32_t sense_len; - uint32_t residual; - uint16_t status_qualifier; - uint8_t status; - uint8_t response; - uint8_t sense[]; -} __attribute__((packed)); +static void pci_vtscsi_print_supported_backends(void); static void *pci_vtscsi_proc(void *); static void pci_vtscsi_reset(void *); @@ -272,18 +73,29 @@ static void pci_vtscsi_neg_features(void *, uint64_t); static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); -static inline bool pci_vtscsi_check_lun(const uint8_t *); -static inline int pci_vtscsi_get_lun(const uint8_t *); +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task_set; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_aca; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_task_set; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_i_t_nexus_reset; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_lun_reset; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task; +static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task_set; + +static pci_vtscsi_walk_t pci_vtscsi_walk_request_queue( + struct pci_vtscsi_queue *, pci_vtscsi_walk_request_queue_cb_t *, void *); -static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, struct pci_vtscsi_ctrl_tmf *); static void pci_vtscsi_an_handle(struct pci_vtscsi_softc *, struct pci_vtscsi_ctrl_an *); +static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, + size_t); static struct pci_vtscsi_request *pci_vtscsi_alloc_request( struct pci_vtscsi_softc *); -static void pci_vtscsi_free_request(struct pci_vtscsi_request *); +static void pci_vtscsi_free_request(struct pci_vtscsi_softc *, + struct pci_vtscsi_request *); static struct pci_vtscsi_request *pci_vtscsi_get_request( struct pci_vtscsi_req_queue *); static void pci_vtscsi_put_request(struct pci_vtscsi_req_queue *, @@ -292,20 +104,25 @@ static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *, struct vqueue_info *); static void pci_vtscsi_return_request(struct pci_vtscsi_queue *, struct pci_vtscsi_request *, int); -static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, +static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); -static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, + +static int pci_vtscsi_add_target_config(nvlist_t *, const char *, int); +static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, struct pci_vtscsi_queue *, int); -static void pci_vtscsi_destroy_queue(struct pci_vtscsi_queue *); +static void pci_vtscsi_destroy_queue(struct pci_vtscsi_softc *, + struct pci_vtscsi_queue *); static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *); +SET_DECLARE(pci_vtscsi_backend_set, struct pci_vtscsi_backend); + static struct virtio_consts vtscsi_vi_consts = { .vc_name = "vtscsi", - .vc_nvq = VTSCSI_MAXQ, + .vc_nvq = VTSCSI_DEF_REQUESTQ + VIRTIO_SCSI_ADDL_Q, .vc_cfgsize = sizeof(struct pci_vtscsi_config), .vc_reset = pci_vtscsi_reset, .vc_cfgread = pci_vtscsi_cfgread, @@ -314,16 +131,51 @@ static struct virtio_consts vtscsi_vi_consts = { .vc_hv_caps = VIRTIO_RING_F_INDIRECT_DESC, }; +static const struct pci_vtscsi_config vtscsi_config = { + .num_queues = VTSCSI_DEF_REQUESTQ, + /* Leave room for the request and the response. */ + .seg_max = VTSCSI_DEF_MAXSEG - VIRTIO_SCSI_HDR_SEG, + .max_sectors = 0, + .cmd_per_lun = 1, + .event_info_size = sizeof(struct pci_vtscsi_event), + .sense_size = 96, + .cdb_size = 32, + .max_channel = VIRTIO_SCSI_MAX_CHANNEL, + .max_target = VIRTIO_SCSI_MAX_TARGET, + .max_lun = VIRTIO_SCSI_MAX_LUN +}; + +int pci_vtscsi_debug = 0; + + +static void +pci_vtscsi_print_supported_backends(void) +{ + struct pci_vtscsi_backend **vbpp; + + if (SET_COUNT(pci_vtscsi_backend_set) == 0) { + printf("No virtio-scsi backends available"); + return; + } + + SET_FOREACH(vbpp, pci_vtscsi_backend_set) { + struct pci_vtscsi_backend *vbp = *vbpp; + printf("%s\n", vbp->vsb_name); + } +} + static void * pci_vtscsi_proc(void *arg) { struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; struct pci_vtscsi_queue *q = worker->vsw_queue; struct pci_vtscsi_softc *sc = q->vsq_sc; - int iolen; for (;;) { struct pci_vtscsi_request *req; + uint8_t target; + int iolen; + int fd; pthread_mutex_lock(&q->vsq_rmtx); @@ -338,11 +190,15 @@ pci_vtscsi_proc(void *arg) req = pci_vtscsi_get_request(&q->vsq_requests); pthread_mutex_unlock(&q->vsq_rmtx); - DPRINTF("I/O request lun %d, data_niov_in %zu, data_niov_out " - "%zu", pci_vtscsi_get_lun(req->vsr_cmd_rd->lun), + target = pci_vtscsi_get_target(sc, req->vsr_cmd_rd->lun); + fd = sc->vss_targets[target].vst_fd; + + DPRINTF("I/O request tgt %u, lun %d, data_niov_in %zu, " + "data_niov_out %zu", target, + pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun), req->vsr_data_niov_in, req->vsr_data_niov_out); - iolen = pci_vtscsi_request_handle(sc, req); + iolen = pci_vtscsi_request_handle(sc, fd, req); pci_vtscsi_return_request(q, req, iolen); } @@ -359,20 +215,11 @@ pci_vtscsi_reset(void *vsc) vi_reset_dev(&sc->vss_vs); /* initialize config structure */ - sc->vss_config = (struct pci_vtscsi_config){ - .num_queues = VTSCSI_REQUESTQ, - /* Leave room for the request and the response. */ - .seg_max = VTSCSI_MAXSEG - 2, - /* CTL apparently doesn't have a limit here */ - .max_sectors = INT32_MAX, - .cmd_per_lun = 1, - .event_info_size = sizeof(struct pci_vtscsi_event), - .sense_size = 96, - .cdb_size = 32, - .max_channel = VIRTIO_SCSI_MAX_CHANNEL, - .max_target = VIRTIO_SCSI_MAX_TARGET, - .max_lun = VIRTIO_SCSI_MAX_LUN - }; + sc->vss_config = sc->vss_default_config; + + sc->vss_config.max_target = MAX(1, sc->vss_num_target) - 1; + + sc->vss_backend->vsb_reset(sc); } static void @@ -402,119 +249,284 @@ pci_vtscsi_cfgwrite(void *vsc __unused, int offset __unused, int size __unused, } /* - * LUN address parsing - * - * The LUN address consists of 8 bytes. While the spec describes this as 0x01, - * followed by the target byte, followed by a "single-level LUN structure", - * this is actually the same as a hierarchical LUN address as defined by SAM-5, - * consisting of four levels of addressing, where in each level the two MSB of - * byte 0 select the address mode used in the remaining bits and bytes. - * - * - * Only the first two levels are acutally used by virtio-scsi: - * - * Level 1: 0x01, 0xTT: Peripheral Device Addressing: Bus 1, Target 0-255 - * Level 2: 0xLL, 0xLL: Peripheral Device Addressing: Bus MBZ, LUN 0-255 - * or: Flat Space Addressing: LUN (0-16383) - * Level 3 and 4: not used, MBZ - * - * Currently, we only support Target 0. - * - * Alternatively, the first level may contain an extended LUN address to select - * the REPORT_LUNS well-known logical unit: + * ABORT TASK: Abort the specifed task queued for this LUN. * - * Level 1: 0xC1, 0x01: Extended LUN Adressing, Well-Known LUN 1 (REPORT_LUNS) - * Level 2, 3, and 4: not used, MBZ - * - * The virtio spec says that we SHOULD implement the REPORT_LUNS well-known - * logical unit but we currently don't. + * We can stop once we have found the specified task queued for this LUN. + */ +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_abort_task(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) +{ + struct pci_vtscsi_ctrl_tmf *tmf = arg; + + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK); + + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (tmf->id != req->vsr_cmd_rd->id) + return (PCI_VTSCSI_WALK_CONTINUE); + + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; + STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); + pci_vtscsi_return_request(q, req, 0); + + return (PCI_VTSCSI_WALK_STOP); +} + +/* + * ABORT TASK SET: Abort all tasks queued for this LUN. + */ +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_abort_task_set(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) +{ + struct pci_vtscsi_ctrl_tmf *tmf = arg; + + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK_SET); + + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; + STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); + pci_vtscsi_return_request(q, req, 0); + + return (PCI_VTSCSI_WALK_CONTINUE); +} + +/* + * CLEAR ACA: Clear ACA (auto contingent allegiance) state. + */ +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_clear_aca(struct pci_vtscsi_queue *q __unused, + struct pci_vtscsi_request *req __unused, void *arg) +{ + struct pci_vtscsi_ctrl_tmf *tmf = arg; + + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_ACA); + + /* + * We don't implement handling of NACA=1 in the CONTROL byte at all. + * + * Thus, we probably should start filtering NORMACA in INQUIRY and + * reject any command that sets NACA=1. + * + * In any case, there isn't anything we need to do with our queued + * requests, so stop right here. + */ + + return (PCI_VTSCSI_WALK_STOP); +} + +/* + * CLEAR TASK SET: Clear all tasks queued for this LUN. * - * According to the virtio spec, these are the only LUNS address formats to be - * used with virtio-scsi. + * All tasks in our queue were placed there by us, so there can be no other + * I_T nexus involved. Hence, this is handled the same as ABORT TASK SET. */ +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_clear_task_set(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) +{ + struct pci_vtscsi_ctrl_tmf *tmf = arg; + + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET); + + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; + STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); + pci_vtscsi_return_request(q, req, 0); + + return (PCI_VTSCSI_WALK_CONTINUE); +} /* - * Check that the given LUN address conforms to the virtio spec, does not - * address an unknown target, and especially does not address the REPORT_LUNS - * well-known logical unit. + * I_T NEXUS RESET: Abort all tasks queued for any LUN of this target. */ -static inline bool -pci_vtscsi_check_lun(const uint8_t *lun) +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_i_t_nexus_reset(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) { - if (lun[0] == 0xC1) - return (false); + struct pci_vtscsi_ctrl_tmf *tmf = arg; - if (lun[0] != 0x01) - return (false); + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET); - if (lun[1] != 0x00) - return (false); + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); - if (lun[2] != 0x00 && (lun[2] & 0xc0) != 0x40) - return (false); + /* + * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates + * that we should actually return ABORTED here, but other documents + * such as the VirtIO spec suggest RESET. + */ + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET; + STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); + pci_vtscsi_return_request(q, req, 0); - if (lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) - return (false); + return (PCI_VTSCSI_WALK_CONTINUE); +} - return (true); +/* + * LOGICAL UNIT RESET: Abort all tasks queued for this LUN. + */ +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_lun_reset(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) +{ + struct pci_vtscsi_ctrl_tmf *tmf = arg; + + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET); + + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + /* + * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates + * that we should actually return ABORTED here, but other documents + * such as the VirtIO spec suggest RESET. + */ + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET; + STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); + pci_vtscsi_return_request(q, req, 0); + + return (PCI_VTSCSI_WALK_CONTINUE); } /* - * Get the LUN id from a LUN address. + * QUERY TASK: Is the specified task present in this LUN? + * + * We can stop once we have found the specified task queued for this LUN. + * + * Note that this function may cause false negatives under the following + * rare circumstances: + * (1) the specified task is still in the virtqueue, not yet having been + * processed by pci_vtscsi_requestq_notify() + * (2) the specified task was actively being processed by a worker thread + * but not yet processed by the backend by the time the QUERY TASK + * request was handled by the backend * - * Every code path using this function must have called pci_vtscsi_check_lun() - * before to make sure the LUN address is valid. + * While a false negative may be confusing for a guest OS looking for the + * state of an I/O request it sent, it is not considered a fatal error of + * any kind and is easy to recover from. Also, in both of the above cases, + * the QUERY TASK TMF request would need to overtake the I/O request in + * question, which can only happen if the TMF request is sent immediately + * after the I/O request. While it is technically perfectly fine for a + * guest to do so, any normal use of QUERY TASK would involve a certain + * delay before the TMF request is sent, giving the I/O request time to + * be processed. */ -static inline int -pci_vtscsi_get_lun(const uint8_t *lun) +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_query_task(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) { - assert(lun[0] == 0x01); - assert(lun[1] == 0x00); - assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); + struct pci_vtscsi_ctrl_tmf *tmf = arg; - return (((lun[2] << 8) | lun[3]) & 0x3fff); + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK); + + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + if (tmf->id != req->vsr_cmd_rd->id) + return (PCI_VTSCSI_WALK_CONTINUE); + + tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; + return (PCI_VTSCSI_WALK_STOP); } -static void -pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf, - size_t bufsize) +/* + * QUERY TASK SET: Are there any tasks present in this LUN? + * + * We can stop as soon as we've found at least one task queued for this LUN. + */ +static pci_vtscsi_walk_t +pci_vtscsi_tmf_handle_query_task_set(struct pci_vtscsi_queue *q, + struct pci_vtscsi_request *req, void *arg) { - struct pci_vtscsi_ctrl_tmf *tmf; - struct pci_vtscsi_ctrl_an *an; - uint32_t type; + struct pci_vtscsi_ctrl_tmf *tmf = arg; - if (bufsize < sizeof(uint32_t)) { - WPRINTF("ignoring truncated control request"); - return; - } + assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK_SET); - type = *(uint32_t *)buf; + if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); - if (type == VIRTIO_SCSI_T_TMF) { - if (bufsize != sizeof(*tmf)) { - WPRINTF("ignoring tmf request with size %zu", bufsize); - return; - } - tmf = (struct pci_vtscsi_ctrl_tmf *)buf; - pci_vtscsi_tmf_handle(sc, tmf); - } else if (type == VIRTIO_SCSI_T_AN_QUERY) { - if (bufsize != sizeof(*an)) { - WPRINTF("ignoring AN request with size %zu", bufsize); - return; - } - an = (struct pci_vtscsi_ctrl_an *)buf; - pci_vtscsi_an_handle(sc, an); + if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != + pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) + return (PCI_VTSCSI_WALK_CONTINUE); + + tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; + return (PCI_VTSCSI_WALK_STOP); +} + +static pci_vtscsi_walk_t +pci_vtscsi_walk_request_queue(struct pci_vtscsi_queue *q, + pci_vtscsi_walk_request_queue_cb_t cb, void *arg) +{ + struct pci_vtscsi_request *req, *tmp; + + STAILQ_FOREACH_SAFE(req, &q->vsq_requests, vsr_link, tmp) { + if (cb(q, req, arg) == PCI_VTSCSI_WALK_STOP) + return (PCI_VTSCSI_WALK_STOP); } + + return (PCI_VTSCSI_WALK_CONTINUE); } +static pci_vtscsi_walk_request_queue_cb_t *const pci_vtscsi_tmf_handler_cb[] = { + pci_vtscsi_tmf_handle_abort_task, + pci_vtscsi_tmf_handle_abort_task_set, + pci_vtscsi_tmf_handle_clear_aca, + pci_vtscsi_tmf_handle_clear_task_set, + pci_vtscsi_tmf_handle_i_t_nexus_reset, + pci_vtscsi_tmf_handle_lun_reset, + pci_vtscsi_tmf_handle_query_task, + pci_vtscsi_tmf_handle_query_task_set +}; + static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_tmf *tmf) { - union ctl_io *io; - int err; + uint8_t target; + int fd; + + if (tmf->subtype > VIRTIO_SCSI_T_TMF_MAX_FUNC) { + WPRINTF("pci_vtscsi_tmf_handle: invalid subtype %u", + tmf->subtype); + tmf->response = VIRTIO_SCSI_S_FUNCTION_REJECTED; + return; + } - if (pci_vtscsi_check_lun(tmf->lun) == false) { + if (pci_vtscsi_check_lun(sc, tmf->lun) == false) { DPRINTF("TMF request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", tmf->lun[0], tmf->lun[1], tmf->lun[2], tmf->lun[3], tmf->lun[4], tmf->lun[5], @@ -524,78 +536,145 @@ pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc, return; } - io = ctl_scsi_alloc_io(sc->vss_iid); - if (io == NULL) { - WPRINTF("failed to allocate ctl_io: err=%d (%s)", - errno, strerror(errno)); + target = pci_vtscsi_get_target(sc, tmf->lun); - tmf->response = VIRTIO_SCSI_S_FAILURE; - return; - } + fd = sc->vss_targets[target].vst_fd; - ctl_scsi_zero_io(io); + DPRINTF("TMF request tgt %d, lun %d, subtype %d, id %lu", + target, pci_vtscsi_get_lun(sc, tmf->lun), tmf->subtype, tmf->id); - io->io_hdr.io_type = CTL_IO_TASK; - io->io_hdr.nexus.initid = sc->vss_iid; - io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(tmf->lun); - io->taskio.tag_type = CTL_TAG_SIMPLE; - io->taskio.tag_num = tmf->id; - io->io_hdr.flags |= CTL_FLAG_USER_TAG; + /* + * Lock out all the worker threads from processing any waiting requests + * while we're processing the TMF request. This also effectively blocks + * pci_vtscsi_requestq_notify() from adding any new requests to the + * request queue. This in turn means we will miss any I/O requests which + * may still be in the virtqueue. + * + * This does not prevent any requests currently being processed by the + * backend from being completed and returned, which we must guarantee to + * adhere to the ordering requirements for any TMF function which aborts + * tasks. + */ + for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) { + struct pci_vtscsi_queue *q = &sc->vss_queues[i]; - switch (tmf->subtype) { - case VIRTIO_SCSI_T_TMF_ABORT_TASK: - io->taskio.task_action = CTL_TASK_ABORT_TASK; - break; + pthread_mutex_lock(&q->vsq_rmtx); + } - case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET: - io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; - break; + /* + * The backend may set response to FAILURE for the TMF request. + * + * The default response of all TMF functions is FUNCTION COMPLETE if + * there was no error, regardless of whether it actually succeeded or + * not. The two notable exceptions are QUERY TASK and QUERY TASK SET, + * which will explicitly return FUNCTION SUCCEEDED if the specified + * task or any task was active in the target/LUN, respectively. + * + * Thus, we will call the backend first. Only if the response we get is + * FUNCTION COMPLETE we'll continue processing the TMF function on our + * queues. Note that there's a slim chance that we're racing against a + * worker thread that is actively processing an I/O request, which may + * lead to our TMF request being processed by the backend before the + * same I/O request, in which case it won't be on any queue either. + */ + sc->vss_backend->vsb_tmf_hdl(sc, fd, tmf); - case VIRTIO_SCSI_T_TMF_CLEAR_ACA: - io->taskio.task_action = CTL_TASK_CLEAR_ACA; - break; + if (tmf->response != VIRTIO_SCSI_S_FUNCTION_COMPLETE) { + /* + * If this is either a FAILURE or FUNCTION REJECTED, we must + * not continue to process the TMF function on our queued + * requests. + * + * If it is FUNCTION SUCCEEDED, we do not need to process the + * TMF function on our queued requests. + * + * If it is anything else, log a warning, but handle it the + * same as above. + */ + if (tmf->response != VIRTIO_SCSI_S_FAILURE && + tmf->response != VIRTIO_SCSI_S_FUNCTION_REJECTED && + tmf->response != VIRTIO_SCSI_S_FUNCTION_SUCCEEDED) { + WPRINTF("pci_vtscsi_tmf_hdl: unexpected response from " + "backend: %d", tmf->response); + } + } else { + pci_vtscsi_walk_t ret = PCI_VTSCSI_WALK_CONTINUE; + uint32_t i; - case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET: - io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET; - break; + for (i = 0; i < sc->vss_config.num_queues; i++) { + struct pci_vtscsi_queue *q = &sc->vss_queues[i]; - case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: - io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; - break; + ret = pci_vtscsi_walk_request_queue(q, + pci_vtscsi_tmf_handler_cb[tmf->subtype], tmf); - case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: - io->taskio.task_action = CTL_TASK_LUN_RESET; - break; + if (ret == PCI_VTSCSI_WALK_STOP) + break; + } + } - case VIRTIO_SCSI_T_TMF_QUERY_TASK: - io->taskio.task_action = CTL_TASK_QUERY_TASK; - break; + /* Unlock the request queues before we return. */ + for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) { + struct pci_vtscsi_queue *q = &sc->vss_queues[i]; - case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET: - io->taskio.task_action = CTL_TASK_QUERY_TASK_SET; - break; + pthread_mutex_unlock(&q->vsq_rmtx); } +} - if (pci_vtscsi_debug) { - struct sbuf *sb = sbuf_new_auto(); - ctl_io_sbuf(io, sb); - sbuf_finish(sb); - DPRINTF("%s", sbuf_data(sb)); - sbuf_delete(sb); +static void +pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_an *an) +{ + int target; + int fd; + + if (pci_vtscsi_check_lun(sc, an->lun) == false) { + DPRINTF("AN request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" + "%.2hhx%.2hhx-%.2hhx%.2hhx", an->lun[0], an->lun[1], + an->lun[2], an->lun[3], an->lun[4], an->lun[5], an->lun[6], + an->lun[7]); + an->response = VIRTIO_SCSI_S_BAD_TARGET; + return; } - err = ioctl(sc->vss_ctl_fd, CTL_IO, io); - if (err != 0) - WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); + target = pci_vtscsi_get_target(sc, an->lun); + + fd = sc->vss_targets[target].vst_fd; + + DPRINTF("AN request tgt %d, lun %d, event requested %x", + target, pci_vtscsi_get_lun(sc, an->lun), an->event_requested); - tmf->response = io->taskio.task_status; - ctl_scsi_free_io(io); + sc->vss_backend->vsb_an_hdl(sc, fd, an); } static void -pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc __unused, - struct pci_vtscsi_ctrl_an *an __unused) +pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf, + size_t bufsize) { + uint32_t type; + + if (bufsize < sizeof(uint32_t)) { + WPRINTF("ignoring truncated control request"); + return; + } + + type = *(uint32_t *)buf; + + if (type == VIRTIO_SCSI_T_TMF) { + if (bufsize != sizeof(struct pci_vtscsi_ctrl_tmf)) { + WPRINTF("ignoring TMF request with size %zu", bufsize); + return; + } + + pci_vtscsi_tmf_handle(sc, buf); + } else if (type == VIRTIO_SCSI_T_AN_QUERY) { + if (bufsize != sizeof(struct pci_vtscsi_ctrl_an)) { + WPRINTF("ignoring AN request with size %zu", bufsize); + return; + } + + pci_vtscsi_an_handle(sc, buf); + } else { + WPRINTF("ignoring unknown control request type = %u", type); + } } static struct pci_vtscsi_request * @@ -607,6 +686,11 @@ pci_vtscsi_alloc_request(struct pci_vtscsi_softc *sc) if (req == NULL) goto fail; + req->vsr_iov = calloc(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG + + SPLIT_IOV_ADDL_IOV, sizeof(struct iovec)); + if (req->vsr_iov == NULL) + goto fail; + req->vsr_cmd_rd = calloc(1, VTSCSI_IN_HEADER_LEN(sc)); if (req->vsr_cmd_rd == NULL) goto fail; @@ -614,10 +698,9 @@ pci_vtscsi_alloc_request(struct pci_vtscsi_softc *sc) if (req->vsr_cmd_wr == NULL) goto fail; - req->vsr_ctl_io = ctl_scsi_alloc_io(sc->vss_iid); - if (req->vsr_ctl_io == NULL) + req->vsr_backend = sc->vss_backend->vsb_req_alloc(sc); + if (req->vsr_backend == NULL) goto fail; - ctl_scsi_zero_io(req->vsr_ctl_io); return (req); @@ -625,20 +708,23 @@ fail: EPRINTLN("failed to allocate request: %s", strerror(errno)); if (req != NULL) - pci_vtscsi_free_request(req); + pci_vtscsi_free_request(sc, req); return (NULL); } static void -pci_vtscsi_free_request(struct pci_vtscsi_request *req) +pci_vtscsi_free_request(struct pci_vtscsi_softc *sc, + struct pci_vtscsi_request *req) { - if (req->vsr_ctl_io != NULL) - ctl_scsi_free_io(req->vsr_ctl_io); + if (req->vsr_backend != NULL) + sc->vss_backend->vsb_req_free(req->vsr_backend); if (req->vsr_cmd_rd != NULL) free(req->vsr_cmd_rd); if (req->vsr_cmd_wr != NULL) free(req->vsr_cmd_wr); + if (req->vsr_iov != NULL) + free(req->vsr_iov); free(req); } @@ -666,19 +752,23 @@ pci_vtscsi_put_request(struct pci_vtscsi_req_queue *req_queue, static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *sc, struct vqueue_info *vq) { - struct pci_vtscsi_queue *q = &sc->vss_queues[vq->vq_num - 2]; + struct pci_vtscsi_queue *q; struct pci_vtscsi_request *req; struct vi_req vireq; size_t res __maybe_unused; - int n; + int n, numseg; + + q = &sc->vss_queues[vq->vq_num - VIRTIO_SCSI_ADDL_Q]; pthread_mutex_lock(&q->vsq_fmtx); req = pci_vtscsi_get_request(&q->vsq_free_requests); assert(req != NULL); pthread_mutex_unlock(&q->vsq_fmtx); - n = vq_getchain(vq, req->vsr_iov, VTSCSI_MAXSEG, &vireq); - assert(n >= 1 && n <= VTSCSI_MAXSEG); + numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG); + + n = vq_getchain(vq, req->vsr_iov, numseg, &vireq); + assert(n >= 1 && n <= numseg); req->vsr_idx = vireq.idx; req->vsr_queue = q; @@ -753,7 +843,7 @@ pci_vtscsi_queue_request(struct pci_vtscsi_softc *sc, struct vqueue_info *vq) assert(res == VTSCSI_IN_HEADER_LEN(q->vsq_sc)); /* Make sure this request addresses a valid LUN. */ - if (pci_vtscsi_check_lun(req->vsr_cmd_rd->lun) == false) { + if (pci_vtscsi_check_lun(sc, req->vsr_cmd_rd->lun) == false) { DPRINTF("I/O request to invalid LUN " "%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx", req->vsr_cmd_rd->lun[0], req->vsr_cmd_rd->lun[1], @@ -777,9 +867,11 @@ static void pci_vtscsi_return_request(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, int iolen) { + struct pci_vtscsi_softc *sc = q->vsq_sc; + void *iov = req->vsr_iov; void *cmd_rd = req->vsr_cmd_rd; void *cmd_wr = req->vsr_cmd_wr; - void *ctl_io = req->vsr_ctl_io; + void *backend = req->vsr_backend; int idx = req->vsr_idx; DPRINTF("request <idx=%d> completed, response %d", idx, @@ -788,15 +880,18 @@ pci_vtscsi_return_request(struct pci_vtscsi_queue *q, iolen += buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(q->vsq_sc), req->vsr_iov_out, req->vsr_niov_out); - ctl_scsi_zero_io(req->vsr_ctl_io); + sc->vss_backend->vsb_req_clear(backend); + memset(iov, 0, sizeof(struct iovec) * (sc->vss_config.seg_max + + VIRTIO_SCSI_HDR_SEG + SPLIT_IOV_ADDL_IOV)); memset(cmd_rd, 0, VTSCSI_IN_HEADER_LEN(q->vsq_sc)); memset(cmd_wr, 0, VTSCSI_OUT_HEADER_LEN(q->vsq_sc)); memset(req, 0, sizeof(struct pci_vtscsi_request)); + req->vsr_iov = iov; req->vsr_cmd_rd = cmd_rd; req->vsr_cmd_wr = cmd_wr; - req->vsr_ctl_io = ctl_io; + req->vsr_backend = backend; pthread_mutex_lock(&q->vsq_fmtx); pci_vtscsi_put_request(&q->vsq_free_requests, req); @@ -809,100 +904,26 @@ pci_vtscsi_return_request(struct pci_vtscsi_queue *q, } static int -pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, +pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { - union ctl_io *io = req->vsr_ctl_io; - void *ext_data_ptr = NULL; - uint32_t ext_data_len = 0, ext_sg_entries = 0; - int err, nxferred; - - io->io_hdr.nexus.initid = sc->vss_iid; - io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(req->vsr_cmd_rd->lun); - - io->io_hdr.io_type = CTL_IO_SCSI; - - if (req->vsr_data_niov_in > 0) { - ext_data_ptr = (void *)req->vsr_data_iov_in; - ext_sg_entries = req->vsr_data_niov_in; - ext_data_len = count_iov(req->vsr_data_iov_in, - req->vsr_data_niov_in); - io->io_hdr.flags |= CTL_FLAG_DATA_OUT; - } else if (req->vsr_data_niov_out > 0) { - ext_data_ptr = (void *)req->vsr_data_iov_out; - ext_sg_entries = req->vsr_data_niov_out; - ext_data_len = count_iov(req->vsr_data_iov_out, - req->vsr_data_niov_out); - io->io_hdr.flags |= CTL_FLAG_DATA_IN; - } - - io->scsiio.sense_len = sc->vss_config.sense_size; - io->scsiio.tag_num = req->vsr_cmd_rd->id; - io->io_hdr.flags |= CTL_FLAG_USER_TAG; - switch (req->vsr_cmd_rd->task_attr) { - case VIRTIO_SCSI_S_ORDERED: - io->scsiio.tag_type = CTL_TAG_ORDERED; - break; - case VIRTIO_SCSI_S_HEAD: - io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; - break; - case VIRTIO_SCSI_S_ACA: - io->scsiio.tag_type = CTL_TAG_ACA; - break; - case VIRTIO_SCSI_S_SIMPLE: - default: - io->scsiio.tag_type = CTL_TAG_SIMPLE; - break; - } - io->scsiio.ext_sg_entries = ext_sg_entries; - io->scsiio.ext_data_ptr = ext_data_ptr; - io->scsiio.ext_data_len = ext_data_len; - io->scsiio.ext_data_filled = 0; - io->scsiio.cdb_len = sc->vss_config.cdb_size; - memcpy(io->scsiio.cdb, req->vsr_cmd_rd->cdb, sc->vss_config.cdb_size); - - if (pci_vtscsi_debug) { - struct sbuf *sb = sbuf_new_auto(); - ctl_io_sbuf(io, sb); - sbuf_finish(sb); - DPRINTF("%s", sbuf_data(sb)); - sbuf_delete(sb); - } - - err = ioctl(sc->vss_ctl_fd, CTL_IO, io); - if (err != 0) { - WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); - req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; - } else { - req->vsr_cmd_wr->sense_len = - MIN(io->scsiio.sense_len, sc->vss_config.sense_size); - req->vsr_cmd_wr->residual = ext_data_len - - io->scsiio.ext_data_filled; - req->vsr_cmd_wr->status = io->scsiio.scsi_status; - req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OK; - memcpy(&req->vsr_cmd_wr->sense, &io->scsiio.sense_data, - req->vsr_cmd_wr->sense_len); - } - - nxferred = io->scsiio.ext_data_filled; - return (nxferred); + return (sc->vss_backend->vsb_req_hdl(sc, fd, req)); } static void pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq) { - struct pci_vtscsi_softc *sc; - struct iovec iov[VTSCSI_MAXSEG]; + struct pci_vtscsi_softc *sc = vsc; + int numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG); + struct iovec *iov = calloc(numseg, sizeof (struct iovec)); struct vi_req req; void *buf = NULL; size_t bufsize; int n; - sc = vsc; - while (vq_has_descs(vq)) { - n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &req); - assert(n >= 1 && n <= VTSCSI_MAXSEG); + n = vq_getchain(vq, iov, numseg, &req); + assert(n >= 1 && n <= numseg); bufsize = iov_to_buf(iov, n, &buf); pci_vtscsi_control_handle(sc, buf, bufsize); @@ -915,6 +936,7 @@ pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq) } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ free(buf); + free(iov); } static void @@ -937,10 +959,10 @@ pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, { struct pci_vtscsi_worker *workers; char tname[MAXCOMLEN + 1]; - int i; + uint32_t i; queue->vsq_sc = sc; - queue->vsq_vq = &sc->vss_vq[num + 2]; + queue->vsq_vq = &sc->vss_vq[num]; pthread_mutex_init(&queue->vsq_rmtx, NULL); pthread_mutex_init(&queue->vsq_fmtx, NULL); @@ -950,7 +972,7 @@ pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, STAILQ_INIT(&queue->vsq_free_requests); LIST_INIT(&queue->vsq_workers); - for (i = 0; i < VTSCSI_RINGSZ; i++) { + for (i = 0; i < sc->vss_req_ringsz; i++) { struct pci_vtscsi_request *req; req = pci_vtscsi_alloc_request(sc); @@ -960,11 +982,11 @@ pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, pci_vtscsi_put_request(&queue->vsq_free_requests, req); } - workers = calloc(VTSCSI_THR_PER_Q, sizeof(struct pci_vtscsi_worker)); + workers = calloc(sc->vss_thr_per_q, sizeof(struct pci_vtscsi_worker)); if (workers == NULL) goto fail; - for (i = 0; i < VTSCSI_THR_PER_Q; i++) { + for (i = 0; i < sc->vss_thr_per_q; i++) { workers[i].vsw_queue = queue; pthread_create(&workers[i].vsw_thread, NULL, &pci_vtscsi_proc, @@ -978,26 +1000,26 @@ pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, return (0); fail: - pci_vtscsi_destroy_queue(queue); + pci_vtscsi_destroy_queue(sc, queue); return (-1); - } static void -pci_vtscsi_destroy_queue(struct pci_vtscsi_queue *queue) +pci_vtscsi_destroy_queue(struct pci_vtscsi_softc *sc, + struct pci_vtscsi_queue *queue) { if (queue->vsq_sc == NULL) return; - for (int i = VTSCSI_RINGSZ; i > 0; i--) { + for (int i = sc->vss_req_ringsz; i > 0; i--) { struct pci_vtscsi_request *req; if (STAILQ_EMPTY(&queue->vsq_free_requests)) break; req = pci_vtscsi_get_request(&queue->vsq_free_requests); - pci_vtscsi_free_request(req); + pci_vtscsi_free_request(queue->vsq_sc, req); } pthread_cond_destroy(&queue->vsq_cv); @@ -1006,61 +1028,385 @@ pci_vtscsi_destroy_queue(struct pci_vtscsi_queue *queue) pthread_mutex_destroy(&queue->vsq_rmtx); } +/* + * Create a target config node, return target id. If the target number isn't + * given as part of the path argument, use last_id + 1. + */ +static int +pci_vtscsi_add_target_config(nvlist_t *nvl, const char *path, int last_id) +{ + uint64_t target; + char *id; + char tmp[4]; + + if (path == NULL) { + EPRINTLN("target path must be specified"); + return (-1); + } + + if (path[0] != '/' && (id = strchr(path, ':')) != NULL) { + const char *errstr; + int len = id - path; + + id = strndup(path, len); + if (id == NULL) { + EPRINTLN("failed to get id string: %s", + strerror(errno)); + return (-1); + } + + target = strtonumx(id, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0); + if (errstr != NULL) { + EPRINTLN("invalid target %s: target ID is %s", id, + errstr); + free(id); + return (-1); + } + + free(id); + path += len + 1; + } else { + target = last_id + 1; + + if (target > VIRTIO_SCSI_MAX_TARGET) { + EPRINTLN("max target (%d) reached, can't add another", + VIRTIO_SCSI_MAX_TARGET); + return (-1); + } + } + + snprintf(tmp, sizeof(tmp), "%lu", target); + + if (get_config_value_node(nvl, tmp) != NULL) { + EPRINTLN("cannot add '%s' as target %s: already exits as '%s'", + path, tmp, get_config_value_node(nvl, tmp)); + return (-1); + } + + set_config_value_node(nvl, tmp, path); + + return (target); +} + +/* + * The following forms are accepted for legacy config options to configure a + * single target: + * + * (0) -s B:D:F,virtio-scsi + * (1) -s B:D:F,virtio-scsi,<dev> + * (2) -s B:D:F,virtio-scsi,<dev>,<name=value>,... + * (3) -s B:D:F,virtio-scsi,<name=value>,... + * (4) -s B:D:F,virtio-scsi,<name=value> + * + * To configure multiple targets, the following form is accepted: + * (5) -s B:D:F,virtio-scsi,[target=[id:]<dev>,...] + */ static int pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) { - char *cp, *devname; + int last_id = -1; + char *config, *tofree, *name, *value; + nvlist_t *targets; + size_t n; + /* Make sure no one accidentally sets "dev" anymore. */ + (void) create_relative_config_node(nvl, "dev"); + + targets = create_relative_config_node(nvl, "target"); + + /* Legacy form (0) is handled in pci_vtscsi_init(). */ if (opts == NULL) return (0); - cp = strchr(opts, ','); - if (cp == NULL) { - set_config_value_node(nvl, "dev", opts); + if (strcmp("help", opts) == 0) { + pci_vtscsi_print_supported_backends(); + exit(0); + } + + n = strcspn(opts, ",="); + + /* Handle legacy form (1) and (2). */ + if (opts[n] == ',' || opts[n] == '\0') { + char *tmp = strndup(opts, n); + + last_id = pci_vtscsi_add_target_config(targets, tmp, last_id); + free(tmp); + + if (last_id < 0) + return (-1); + + opts += n; + if (opts[0] == ',' && opts[1] != '\0') + opts++; + } + + /* If this was form (1), we're done. */ + if (opts[0] == '\0') return (0); + + /* + * For form (2), (3), (4), and (5), parse the remaining options. + * + * Contrary to other options, multiple target=<dev> options create a new + * target for each such option. + * + * For compatibility reasons we also accept dev=<dev> options for + * targets. + */ + config = tofree = strdup(opts); + while ((name = strsep(&config, ",")) != NULL) { + value = strchr(name, '='); + if (value != NULL) + *value++ = '\0'; + + if (strcmp(name, "dev") == 0 || strcmp(name, "target") == 0) { + int new_id = pci_vtscsi_add_target_config(targets, + value, last_id); + + if (new_id < 0) { + free(tofree); + return (-1); + } + + if (new_id > last_id) + last_id = new_id; + + } else if (value != NULL) { + set_config_value_node(nvl, name, value); + } else { + set_config_bool_node(nvl, name, true); + } } - devname = strndup(opts, cp - opts); - set_config_value_node(nvl, "dev", devname); - free(devname); - return (pci_parse_legacy_config(nvl, cp + 1)); + + free(tofree); + return (0); +} + +static int +pci_vtscsi_count_targets(const char *prefix __unused, + const nvlist_t *parent __unused, const char *name, int type, void *arg) +{ + struct pci_vtscsi_softc *sc = arg; + const char *errstr; + uint64_t target; + + if (type != NV_TYPE_STRING) { + EPRINTLN("invalid target \"%s\" type: not a string", name); + errno = EINVAL; + return (-1); + } + + target = strtonumx(name, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0); + if (errstr != NULL) { + EPRINTLN("invalid target %s: target ID is %s", name, errstr); + return (-1); + } + + if (target >= sc->vss_num_target) + sc->vss_num_target = target + 1; + + return (0); +} + +static int +pci_vtscsi_init_target(const char *prefix __unused, const nvlist_t *parent, + const char *name, int type, void *arg) +{ + struct pci_vtscsi_softc *sc = arg; + const char *value; + const char *errstr; + uint64_t target; + int ret; + + assert(type == NV_TYPE_STRING); + + /* + * Get the numeric value of the target id from 'name'. + */ + target = strtonumx(name, 0, sc->vss_num_target - 1, &errstr, 0); + assert(errstr == NULL); + sc->vss_targets[target].vst_target = target; + + /* + * 'value' contains the backend path. Call the backend to open it. + */ + value = nvlist_get_string(parent, name); + ret = sc->vss_backend->vsb_open(sc, value, target); + if (ret != 0) + EPRINTLN("cannot open target %lu at %s: %s", target, value, + strerror(errno)); + return (ret); +} + +static int +pci_vtscsi_get_config_num(nvlist_t *nvl, const char *name, uint32_t lim_lo, + uint32_t lim_hi, uint32_t *res) +{ + const char *value; + const char *errstr; + long long val; + + value = get_config_value_node(nvl, name); + if (value == NULL) + return (0); + + val = strtonumx(value, lim_lo, lim_hi, &errstr, 0); + if (errstr != NULL) { + EPRINTLN("Invalid value for %s: %s", name, value); + return (-1); + } + + *res = (uint32_t)val; + return (0); } static int pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtscsi_softc *sc; - const char *devname, *value; - int err; - int i; + struct pci_vtscsi_backend *backend, **vbpp; + const char *value; + uint32_t val; + size_t i; + int q, err; sc = calloc(1, sizeof(struct pci_vtscsi_softc)); if (sc == NULL) return (-1); - value = get_config_value_node(nvl, "iid"); - if (value != NULL) - sc->vss_iid = strtoul(value, NULL, 10); + sc->vss_vi_consts = vtscsi_vi_consts; + sc->vss_ctl_ringsz = VTSCSI_DEF_RINGSZ; + sc->vss_evt_ringsz = VTSCSI_DEF_RINGSZ; + sc->vss_req_ringsz = VTSCSI_DEF_RINGSZ; + sc->vss_thr_per_q = VTSCSI_DEF_THR_PER_Q; + sc->vss_default_config = vtscsi_config; value = get_config_value_node(nvl, "bootindex"); if (value != NULL) { if (pci_emul_add_boot_device(pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); + errno = EINVAL; + goto fail; + } + } + + val = vtscsi_config.seg_max; + if (pci_vtscsi_get_config_num(nvl, "seg_max", VTSCSI_MIN_MAXSEG, + VTSCSI_MAX_MAXSEG, &val) != 0) + goto fail; + sc->vss_default_config.seg_max = val; + + val = vtscsi_config.num_queues; + if (pci_vtscsi_get_config_num(nvl, "num_queues", VTSCSI_MIN_REQUESTQ, + VTSCSI_MAX_REQUESTQ, &val) != 0) + goto fail; + sc->vss_default_config.num_queues = val; + + /* + * num_queues is only the number of request queues, but nvq must + * account for the control and event queues. + */ + sc->vss_vi_consts.vc_nvq = val + VIRTIO_SCSI_ADDL_Q; + + /* + * Allocate queues early, so that they're there for the call to + * vi_softc_linkup(). + */ + sc->vss_vq = calloc(sc->vss_vi_consts.vc_nvq, + sizeof(struct vqueue_info)); + if (sc->vss_vq == NULL) { + EPRINTLN("can't allocate space for %d virtqueues", + sc->vss_vi_consts.vc_nvq); + goto fail; + } + + sc->vss_queues = calloc(sc->vss_default_config.num_queues, + sizeof(struct pci_vtscsi_queue)); + if (sc->vss_queues == NULL) { + EPRINTLN("can't allocate space for %d request queues", + sc->vss_config.num_queues); + goto fail; + } + + if (pci_vtscsi_get_config_num(nvl, "ctl_ringsz", VTSCSI_MIN_RINGSZ, + VTSCSI_MAX_RINGSZ, &sc->vss_ctl_ringsz) != 0) + goto fail; + + if (pci_vtscsi_get_config_num(nvl, "evt_ringsz", VTSCSI_MIN_RINGSZ, + VTSCSI_MAX_RINGSZ, &sc->vss_evt_ringsz) != 0) + goto fail; + + if (pci_vtscsi_get_config_num(nvl, "req_ringsz", VTSCSI_MIN_RINGSZ, + VTSCSI_MAX_RINGSZ, &sc->vss_req_ringsz) != 0) + goto fail; + + if (pci_vtscsi_get_config_num(nvl, "thr_per_q", VTSCSI_MIN_THR_PER_Q, + VTSCSI_MAX_THR_PER_Q, &sc->vss_thr_per_q) != 0) + goto fail; + + value = get_config_value_node(nvl, "backend"); + if (value == NULL) { + if (SET_COUNT(pci_vtscsi_backend_set) == 0) { + WPRINTF("No virtio-scsi backends available"); + errno = EINVAL; + goto fail; + } + backend = SET_ITEM(pci_vtscsi_backend_set, 0); + } else { + backend = NULL; + SET_FOREACH(vbpp, pci_vtscsi_backend_set) { + if (strcasecmp(value, (*vbpp)->vsb_name) == 0) { + backend = *vbpp; + break; + } + } + if (backend == NULL) { + WPRINTF("No such virtio-scsi backend: %s", value); + errno = EINVAL; goto fail; } } - devname = get_config_value_node(nvl, "dev"); - if (devname == NULL) - devname = "/dev/cam/ctl"; - sc->vss_ctl_fd = open(devname, O_RDWR); - if (sc->vss_ctl_fd < 0) { - WPRINTF("cannot open %s: %s", devname, strerror(errno)); + err = backend->vsb_init(sc, backend, nvl); + if (err != 0) { + errno = EINVAL; goto fail; } + nvl = find_relative_config_node(nvl, "target"); + if (nvl != NULL) { + err = walk_config_nodes("", nvl, sc, pci_vtscsi_count_targets); + if (err != 0) + goto fail; + } + + if (sc->vss_num_target > 0) { + sc->vss_targets = malloc(sc->vss_num_target * + sizeof(struct pci_vtscsi_target)); + if (sc->vss_targets == NULL) { + EPRINTLN("can't allocate space for %lu targets", + sc->vss_num_target); + goto fail; + } + + memset(sc->vss_targets, -1, sc->vss_num_target * + sizeof(struct pci_vtscsi_target)); + + err = walk_config_nodes("", nvl, sc, pci_vtscsi_init_target); + if (err != 0) + goto fail; + } + + /* + * All targets should be open now and have a valid fd. + */ + for (i = 0; i < sc->vss_num_target; i++) + if (sc->vss_targets[i].vst_target == i) + assert(sc->vss_targets[i].vst_fd > 0); + pthread_mutex_init(&sc->vss_mtx, NULL); - vi_softc_linkup(&sc->vss_vs, &vtscsi_vi_consts, sc, pi, sc->vss_vq); + vi_softc_linkup(&sc->vss_vs, &sc->vss_vi_consts, sc, pi, sc->vss_vq); sc->vss_vs.vs_mtx = &sc->vss_mtx; /* @@ -1076,20 +1422,22 @@ pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) pci_vtscsi_reset(sc); pthread_mutex_unlock(&sc->vss_mtx); - /* controlq */ - sc->vss_vq[0].vq_qsize = VTSCSI_RINGSZ; + /* virtqueue 0: control queue */ + sc->vss_vq[0].vq_qsize = sc->vss_ctl_ringsz; sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify; - /* eventq */ - sc->vss_vq[1].vq_qsize = VTSCSI_RINGSZ; + /* virtqueue 1: event queue */ + sc->vss_vq[1].vq_qsize = sc->vss_evt_ringsz; sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify; - /* request queues */ - for (i = 2; i < VTSCSI_MAXQ; i++) { - sc->vss_vq[i].vq_qsize = VTSCSI_RINGSZ; - sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; + /* virtqueue 2-n: request queues */ + for (q = VIRTIO_SCSI_ADDL_Q; q < sc->vss_vi_consts.vc_nvq; q++) { + int rq = q - VIRTIO_SCSI_ADDL_Q; - err = pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2); + sc->vss_vq[q].vq_qsize = sc->vss_req_ringsz; + sc->vss_vq[q].vq_notify = pci_vtscsi_requestq_notify; + + err = pci_vtscsi_init_queue(sc, &sc->vss_queues[rq], q); if (err != 0) goto fail; } @@ -1101,7 +1449,8 @@ pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); - if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix())) + err = vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix()); + if (err != 0) goto fail; vi_set_io_bar(&sc->vss_vs, 0); @@ -1109,12 +1458,29 @@ pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) return (0); fail: - for (i = 2; i < VTSCSI_MAXQ; i++) - pci_vtscsi_destroy_queue(&sc->vss_queues[i - 2]); + if (sc->vss_queues != NULL) { + for (q = VIRTIO_SCSI_ADDL_Q; + q < sc->vss_vi_consts.vc_nvq; + q++) { + int rq = q - VIRTIO_SCSI_ADDL_Q; - if (sc->vss_ctl_fd > 0) - close(sc->vss_ctl_fd); + pci_vtscsi_destroy_queue(sc, &sc->vss_queues[rq]); + } + } + + pthread_mutex_destroy(&sc->vss_mtx); + + for (i = 0; i < sc->vss_num_target; i++) { + if (sc->vss_targets[i].vst_target == i && + sc->vss_targets[i].vst_fd >= 0) { + close(sc->vss_targets[i].vst_fd); + } + } + free(sc->vss_targets); + free(sc->vss_backend); + free(sc->vss_queues); + free(sc->vss_vq); free(sc); return (-1); } diff --git a/usr.sbin/bhyve/pci_virtio_scsi.h b/usr.sbin/bhyve/pci_virtio_scsi.h new file mode 100644 index 000000000000..b0f41bbdd8c3 --- /dev/null +++ b/usr.sbin/bhyve/pci_virtio_scsi.h @@ -0,0 +1,428 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2016 Jakub Klama <jceel@FreeBSD.org>. + * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>. + * Copyright (c) 2026 Hans Rosenfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _PCI_VIRTIO_SCSI_H_ +#define _PCI_VIRTIO_SCSI_H_ + +#include "iov.h" + +extern int pci_vtscsi_debug; + +#define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params) +#define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params) + +/* Absolute limits given by the VirtIO SCSI spec */ +#define VIRTIO_SCSI_MAX_CHANNEL 0 +#define VIRTIO_SCSI_MAX_TARGET 255 +#define VIRTIO_SCSI_MAX_LUN 16383 +#define VIRTIO_SCSI_HDR_SEG 2 +#define VIRTIO_SCSI_ADDL_Q 2 + +/* Features specific to VirtIO SCSI, none of which we currently support */ +#define VIRTIO_SCSI_F_INOUT (1 << 0) +#define VIRTIO_SCSI_F_HOTPLUG (1 << 1) +#define VIRTIO_SCSI_F_CHANGE (1 << 2) + +/* Default limits which we set. All of these are configurable. */ +#define VTSCSI_DEF_RINGSZ 64 +#define VTSCSI_MIN_RINGSZ 4 +#define VTSCSI_MAX_RINGSZ 4096 + +#define VTSCSI_DEF_THR_PER_Q 16 +#define VTSCSI_MIN_THR_PER_Q 1 +#define VTSCSI_MAX_THR_PER_Q 256 + +#define VTSCSI_DEF_MAXSEG 64 +#define VTSCSI_MIN_MAXSEG (VIRTIO_SCSI_HDR_SEG + 1) +#define VTSCSI_MAX_MAXSEG \ + (4096 - VIRTIO_SCSI_HDR_SEG - SPLIT_IOV_ADDL_IOV) + +#define VTSCSI_DEF_REQUESTQ 1 +#define VTSCSI_MIN_REQUESTQ 1 +#define VTSCSI_MAX_REQUESTQ (32 - VIRTIO_SCSI_ADDL_Q) + +/* + * Device-specific config space registers + * + * The guest driver may try to modify cdb_size and sense_size by writing the + * respective config space registers. Since we currently ignore all writes to + * config space, these macros are essentially constant. + */ +#define VTSCSI_IN_HEADER_LEN(_sc) \ + (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size) + +#define VTSCSI_OUT_HEADER_LEN(_sc) \ + (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) + +struct pci_vtscsi_config { + uint32_t num_queues; + uint32_t seg_max; + uint32_t max_sectors; + uint32_t cmd_per_lun; + uint32_t event_info_size; + uint32_t sense_size; + uint32_t cdb_size; + uint16_t max_channel; + uint16_t max_target; + uint32_t max_lun; +} __attribute__((packed)); + + +/* + * I/O request state and I/O request queues + * + * In addition to the control queue and notification queues, each virtio-scsi + * device instance has at least one I/O request queue, the state of which is + * is kept in an array of struct pci_vtscsi_queue in the device softc. + * + * Each pci_vtscsi_queue has configurable number of pci_vtscsi_request + * structures pre-allocated on vsq_free_requests. For each I/O request + * coming in on the I/O virtqueue, the request queue handler will take a + * pci_vtscsi_request off vsq_free_requests, fills in the data from the + * I/O virtqueue, puts it on vsq_requests, and signals vsq_cv. + * + * Each pci_vtscsi_queue will have a configurable number of worker threads, + * which wait on vsq_cv. When signalled, they repeatedly take a single + * pci_vtscsi_request off vsq_requests and hand it to the backend, which + * processes it synchronously. After completion, the pci_vtscsi_request + * is re-initialized and put back onto vsq_free_requests. + * + * The worker threads exit when vsq_cv is signalled after vsw_exiting was set. + * + * There are three mutexes to coordinate the accesses to an I/O request queue: + * - vsq_rmtx protects vsq_requests and must be held when waiting on vsq_cv + * - vsq_fmtx protects vsq_free_requests + * - vsq_qmtx must be held when operating on the underlying virtqueue, vsq_vq + * + * The I/O vectors for each request are kept in the preallocated iovec array + * vsr_iov, and pointers to the respective header/data in/out portions are set + * up to point into the array when the request is queued for processing. + * + * The number of iovecs preallocated for vsr_iov is derived from the configured + * 'seg_max' parameter defined by the virtio spec: + * - 'seg_max' parameter specifies the maximum number of I/O data vectors + * we support in any request + * - we need 2 additional iovecs for the I/O headers (VIRTIO_SCSI_HDR_SEG) + * - we need another 2 additional iovecs for split_iov() (SPLIT_IOV_ADDL_IOV) + * + * The only time we explicitly need the full size of vsr_iov after preallocation + * is during re-initialization after completing a request, and implicitly in the + * calls to split_iov() the set up the pointers. In all other cases, we use only + * 'seg_max' + VIRTIO_SCSI_HDR_SEG, and we advertise only 'seg_max' to the guest + * in accordance to the virtio spec. + */ +STAILQ_HEAD(pci_vtscsi_req_queue, pci_vtscsi_request); + +struct pci_vtscsi_queue { + struct pci_vtscsi_softc *vsq_sc; + struct vqueue_info *vsq_vq; + pthread_mutex_t vsq_rmtx; + pthread_mutex_t vsq_fmtx; + pthread_mutex_t vsq_qmtx; + pthread_cond_t vsq_cv; + struct pci_vtscsi_req_queue vsq_requests; + struct pci_vtscsi_req_queue vsq_free_requests; + LIST_HEAD(, pci_vtscsi_worker) vsq_workers; +}; + +struct pci_vtscsi_worker { + struct pci_vtscsi_queue *vsw_queue; + pthread_t vsw_thread; + bool vsw_exiting; + LIST_ENTRY(pci_vtscsi_worker) vsw_link; +}; + +struct pci_vtscsi_request { + struct pci_vtscsi_queue *vsr_queue; + struct iovec *vsr_iov; + struct iovec *vsr_iov_in; + struct iovec *vsr_iov_out; + struct iovec *vsr_data_iov_in; + struct iovec *vsr_data_iov_out; + struct pci_vtscsi_req_cmd_rd *vsr_cmd_rd; + struct pci_vtscsi_req_cmd_wr *vsr_cmd_wr; + void *vsr_backend; + size_t vsr_niov_in; + size_t vsr_niov_out; + size_t vsr_data_niov_in; + size_t vsr_data_niov_out; + uint32_t vsr_idx; + STAILQ_ENTRY(pci_vtscsi_request) vsr_link; +}; + +/* + * Per-target state. + */ +struct pci_vtscsi_target { + uint8_t vst_target; + int vst_fd; + int vst_max_sectors; +}; + +/* + * Per-device softc + */ +struct pci_vtscsi_softc { + struct virtio_softc vss_vs; + struct virtio_consts vss_vi_consts; + struct vqueue_info *vss_vq; + struct pci_vtscsi_queue *vss_queues; + pthread_mutex_t vss_mtx; + uint32_t vss_features; + size_t vss_num_target; + uint32_t vss_ctl_ringsz; + uint32_t vss_evt_ringsz; + uint32_t vss_req_ringsz; + uint32_t vss_thr_per_q; + struct pci_vtscsi_config vss_default_config; + struct pci_vtscsi_config vss_config; + struct pci_vtscsi_target *vss_targets; + struct pci_vtscsi_backend *vss_backend; +}; + +/* + * VirtIO-SCSI Task Management Function control requests + */ +#define VIRTIO_SCSI_T_TMF 0 +#define VIRTIO_SCSI_T_TMF_ABORT_TASK 0 +#define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1 +#define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2 +#define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3 +#define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4 +#define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5 +#define VIRTIO_SCSI_T_TMF_QUERY_TASK 6 +#define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7 + +#define VIRTIO_SCSI_T_TMF_MAX_FUNC VIRTIO_SCSI_T_TMF_QUERY_TASK_SET + +/* command-specific response values */ +#define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0 +#define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10 +#define VIRTIO_SCSI_S_FUNCTION_REJECTED 11 + +struct pci_vtscsi_ctrl_tmf { + const uint32_t type; + const uint32_t subtype; + const uint8_t lun[8]; + const uint64_t id; + uint8_t response; +} __attribute__((packed)); + + +/* + * VirtIO-SCSI Asynchronous Notification control requests + */ +#define VIRTIO_SCSI_T_AN_QUERY 1 +#define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2 +#define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4 +#define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8 +#define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16 +#define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32 +#define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64 + +struct pci_vtscsi_ctrl_an { + const uint32_t type; + const uint8_t lun[8]; + const uint32_t event_requested; + uint32_t event_actual; + uint8_t response; +} __attribute__((packed)); + +/* command-specific response values */ +#define VIRTIO_SCSI_S_OK 0 +#define VIRTIO_SCSI_S_OVERRUN 1 +#define VIRTIO_SCSI_S_ABORTED 2 +#define VIRTIO_SCSI_S_BAD_TARGET 3 +#define VIRTIO_SCSI_S_RESET 4 +#define VIRTIO_SCSI_S_BUSY 5 +#define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6 +#define VIRTIO_SCSI_S_TARGET_FAILURE 7 +#define VIRTIO_SCSI_S_NEXUS_FAILURE 8 +#define VIRTIO_SCSI_S_FAILURE 9 +#define VIRTIO_SCSI_S_INCORRECT_LUN 12 + +struct pci_vtscsi_event { + uint32_t event; + uint8_t lun[8]; + uint32_t reason; +} __attribute__((packed)); + +/* + * VirtIO-SCSI I/O requests + */ +struct pci_vtscsi_req_cmd_rd { + const uint8_t lun[8]; + const uint64_t id; + const uint8_t task_attr; + const uint8_t prio; + const uint8_t crn; + const uint8_t cdb[]; +} __attribute__((packed)); + +/* task_attr */ +#define VIRTIO_SCSI_S_SIMPLE 0 +#define VIRTIO_SCSI_S_ORDERED 1 +#define VIRTIO_SCSI_S_HEAD 2 +#define VIRTIO_SCSI_S_ACA 3 + +struct pci_vtscsi_req_cmd_wr { + uint32_t sense_len; + uint32_t residual; + uint16_t status_qualifier; + uint8_t status; + uint8_t response; + uint8_t sense[]; +} __attribute__((packed)); + +/* + * Backend interface + */ +struct pci_vtscsi_backend { + const char *vsb_name; + int (*vsb_init)(struct pci_vtscsi_softc *, + struct pci_vtscsi_backend *, nvlist_t *); + int (*vsb_open)(struct pci_vtscsi_softc *, const char *, + long); + void (*vsb_reset)(struct pci_vtscsi_softc *); + + void* (*vsb_req_alloc)(struct pci_vtscsi_softc *); + void (*vsb_req_clear)(void *); + void (*vsb_req_free)(void *); + + void (*vsb_tmf_hdl)(struct pci_vtscsi_softc *, int, + struct pci_vtscsi_ctrl_tmf *); + void (*vsb_an_hdl)(struct pci_vtscsi_softc *, int, + struct pci_vtscsi_ctrl_an *); + int (*vsb_req_hdl)(struct pci_vtscsi_softc *, int, + struct pci_vtscsi_request *); +}; +#define PCI_VTSCSI_BACKEND_SET(x) DATA_SET(pci_vtscsi_backend_set, x) + +/* + * LUN address parsing + * + * The LUN address consists of 8 bytes. While the spec describes this as 0x01, + * followed by the target byte, followed by a "single-level LUN structure", + * this is actually the same as a hierarchical LUN address as defined by SAM-5, + * consisting of four levels of addressing, where in each level the two MSB of + * byte 0 select the address mode used in the remaining bits and bytes. + * + * + * Only the first two levels are acutally used by virtio-scsi: + * + * Level 1: 0x01, 0xTT: Peripheral Device Addressing: Bus 1, Target 0-255 + * Level 2: 0xLL, 0xLL: Peripheral Device Addressing: Bus MBZ, LUN 0-255 + * or: Flat Space Addressing: LUN (0-16383) + * Level 3 and 4: not used, MBZ + * + * + * Alternatively, the first level may contain an extended LUN address to select + * the REPORT_LUNS well-known logical unit: + * + * Level 1: 0xC1, 0x01: Extended LUN Adressing, Well-Known LUN 1 (REPORT_LUNS) + * Level 2, 3, and 4: not used, MBZ + * + * The virtio spec says that we SHOULD implement the REPORT_LUNS well-known + * logical unit but we currently don't. + * + * According to the virtio spec, these are the only LUNS address formats to be + * used with virtio-scsi. + */ + +/* + * Check that the given LUN address conforms to the virtio spec, does not + * address an unknown target, and especially does not address the REPORT_LUNS + * well-known logical unit. + */ +static inline bool +pci_vtscsi_check_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) +{ + if (lun[0] == 0xC1) + return (false); + + if (lun[0] != 0x01) + return (false); + + if (lun[1] >= sc->vss_num_target) + return (false); + + if (lun[1] != sc->vss_targets[lun[1]].vst_target) + return (false); + + if (sc->vss_targets[lun[1]].vst_fd < 0) + return (false); + + if (lun[2] != 0x00 && (lun[2] & 0xc0) != 0x40) + return (false); + + if (lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) + return (false); + + return (true); +} + +/* + * Get the target id from a LUN address. + * + * Every code path using this function must have called pci_vtscsi_check_lun() + * before to make sure the LUN address is valid. + */ +static inline uint8_t +pci_vtscsi_get_target(struct pci_vtscsi_softc *sc, const uint8_t *lun) +{ + assert(lun[0] == 0x01); + assert(lun[1] < sc->vss_num_target); + assert(lun[1] == sc->vss_targets[lun[1]].vst_target); + assert(sc->vss_targets[lun[1]].vst_fd >= 0); + assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); + + return (lun[1]); +} + +/* + * Get the LUN id from a LUN address. + * + * Every code path using this function must have called pci_vtscsi_check_lun() + * before to make sure the LUN address is valid. + */ +static inline uint16_t +pci_vtscsi_get_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) +{ + assert(lun[0] == 0x01); + assert(lun[1] < sc->vss_num_target); + assert(lun[1] == sc->vss_targets[lun[1]].vst_target); + assert(sc->vss_targets[lun[1]].vst_fd >= 0); + assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); + + return (((lun[2] << 8) | lun[3]) & 0x3fff); +} + +#endif /* _PCI_VIRTIO_SCSI_H_ */ diff --git a/usr.sbin/bhyve/pci_virtio_scsi_ctl.c b/usr.sbin/bhyve/pci_virtio_scsi_ctl.c new file mode 100644 index 000000000000..bfe619d8a11f --- /dev/null +++ b/usr.sbin/bhyve/pci_virtio_scsi_ctl.c @@ -0,0 +1,377 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2016 Jakub Klama <jceel@FreeBSD.org>. + * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>. + * Copyright (c) 2026 Hans Rosenfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/linker_set.h> +#include <sys/types.h> +#include <sys/uio.h> +#include <sys/time.h> +#include <sys/queue.h> +#include <sys/sbuf.h> + +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> +#include <unistd.h> +#include <assert.h> +#include <pthread.h> +#include <pthread_np.h> + +#include <cam/scsi/scsi_all.h> +#include <cam/scsi/scsi_message.h> +#include <cam/ctl/ctl.h> +#include <cam/ctl/ctl_io.h> +#include <cam/ctl/ctl_backend.h> +#include <cam/ctl/ctl_ioctl.h> +#include <cam/ctl/ctl_util.h> +#include <cam/ctl/ctl_scsi_all.h> +#include <camlib.h> + +#include "bhyverun.h" +#include "config.h" +#include "debug.h" +#include "pci_emul.h" +#include "virtio.h" +#include "iov.h" +#include "pci_virtio_scsi.h" + +struct vtscsi_ctl_backend { + struct pci_vtscsi_backend vcb_backend; + int vcb_iid; +}; + +static int vtscsi_ctl_init(struct pci_vtscsi_softc *, + struct pci_vtscsi_backend *, nvlist_t *); +static int vtscsi_ctl_open(struct pci_vtscsi_softc *, const char *, long); +static void vtscsi_ctl_reset(struct pci_vtscsi_softc *); + +static void *vtscsi_ctl_req_alloc(struct pci_vtscsi_softc *); +static void vtscsi_ctl_req_clear(void *); +static void vtscsi_ctl_req_free(void *); + +static void vtscsi_ctl_tmf_hdl(struct pci_vtscsi_softc *, int, + struct pci_vtscsi_ctrl_tmf *); +static void vtscsi_ctl_an_hdl(struct pci_vtscsi_softc *, int, + struct pci_vtscsi_ctrl_an *); +static int vtscsi_ctl_req_hdl(struct pci_vtscsi_softc *, int, + struct pci_vtscsi_request *); + +static int +vtscsi_ctl_count_targets(const char *prefix __unused, + const nvlist_t *parent __unused, const char *name __unused, int type, + void *arg) +{ + int *count = arg; + + if (type != NV_TYPE_STRING) { + EPRINTLN("invalid target \"%s\" type: not a string", name); + errno = EINVAL; + return (-1); + } + + (*count)++; + + return (0); +} + +static int +vtscsi_ctl_init(struct pci_vtscsi_softc *sc, struct pci_vtscsi_backend *backend, + nvlist_t *nvl) +{ + int count = 0; + int ret = 0; + struct vtscsi_ctl_backend *ctl_backend; + const char *value; + + ctl_backend = calloc(1, sizeof(struct vtscsi_ctl_backend)); + if (ctl_backend == NULL) { + EPRINTLN("failed to allocate backend data: %s", + strerror(errno)); + return (-1); + } + + ctl_backend->vcb_backend = *backend; + sc->vss_backend = &ctl_backend->vcb_backend; + + value = get_config_value_node(nvl, "iid"); + if (value != NULL) + ctl_backend->vcb_iid = strtoul(value, NULL, 10); + + /* + * Count configured targets. If no targets were configured, use + * /dev/cam/ctl to remain compatible with previous versions. + */ + nvl = find_relative_config_node(nvl, "target"); + assert(nvl != NULL); + + ret = walk_config_nodes("", nvl, &count, vtscsi_ctl_count_targets); + if (ret != 0) + return (ret); + + if (count == 0) + set_config_value_node(nvl, "0", "/dev/cam/ctl"); + + return (0); +} + +static int +vtscsi_ctl_open(struct pci_vtscsi_softc *sc __unused, const char *devname, + long target) +{ + struct pci_vtscsi_target *tgt = &sc->vss_targets[target]; + + tgt->vst_fd = open(devname, O_RDWR); + if (tgt->vst_fd < 0) + return (-1); + + return (0); +} + +static void +vtscsi_ctl_reset(struct pci_vtscsi_softc *sc __unused) +{ + /* + * There doesn't seem to be a limit to the maximum number of + * sectors CTL can transfer in one request. + */ + sc->vss_config.max_sectors = INT32_MAX; +} + +static void * +vtscsi_ctl_req_alloc(struct pci_vtscsi_softc *sc) +{ + struct vtscsi_ctl_backend *ctl = + (struct vtscsi_ctl_backend *)sc->vss_backend; + union ctl_io *io = ctl_scsi_alloc_io(ctl->vcb_iid); + + if (io != NULL) + ctl_scsi_zero_io(io); + + return (io); +} + +static void +vtscsi_ctl_req_clear(void *io) +{ + ctl_scsi_zero_io(io); +} + +static void +vtscsi_ctl_req_free(void *io) +{ + ctl_scsi_free_io(io); +} + +static void +vtscsi_ctl_tmf_hdl(struct pci_vtscsi_softc *sc, int fd, + struct pci_vtscsi_ctrl_tmf *tmf) +{ + struct vtscsi_ctl_backend *ctl; + union ctl_io *io; + int err; + + ctl = (struct vtscsi_ctl_backend *)sc->vss_backend; + + io = vtscsi_ctl_req_alloc(sc); + if (io == NULL) { + tmf->response = VIRTIO_SCSI_S_FAILURE; + return; + } + vtscsi_ctl_req_clear(io); + + io->io_hdr.io_type = CTL_IO_TASK; + io->io_hdr.nexus.initid = ctl->vcb_iid; + io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, tmf->lun); + io->taskio.tag_type = CTL_TAG_SIMPLE; + io->taskio.tag_num = tmf->id; + io->io_hdr.flags |= CTL_FLAG_USER_TAG; + + switch (tmf->subtype) { + case VIRTIO_SCSI_T_TMF_ABORT_TASK: + io->taskio.task_action = CTL_TASK_ABORT_TASK; + break; + + case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET: + io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; + break; + + case VIRTIO_SCSI_T_TMF_CLEAR_ACA: + io->taskio.task_action = CTL_TASK_CLEAR_ACA; + break; + + case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET: + io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET; + break; + + case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: + io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; + break; + + case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: + io->taskio.task_action = CTL_TASK_LUN_RESET; + break; + + case VIRTIO_SCSI_T_TMF_QUERY_TASK: + io->taskio.task_action = CTL_TASK_QUERY_TASK; + break; + + case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET: + io->taskio.task_action = CTL_TASK_QUERY_TASK_SET; + break; + } + + if (pci_vtscsi_debug) { + struct sbuf *sb = sbuf_new_auto(); + ctl_io_sbuf(io, sb); + sbuf_finish(sb); + DPRINTF("%s", sbuf_data(sb)); + sbuf_delete(sb); + } + + err = ioctl(fd, CTL_IO, io); + if (err != 0) { + WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); + tmf->response = VIRTIO_SCSI_S_FAILURE; + } else { + tmf->response = io->taskio.task_status; + } + vtscsi_ctl_req_free(io); +} + +static void +vtscsi_ctl_an_hdl(struct pci_vtscsi_softc *sc __unused, int fd __unused, + struct pci_vtscsi_ctrl_an *an) +{ + an->response = VIRTIO_SCSI_S_FAILURE; +} + +static int +vtscsi_ctl_req_hdl(struct pci_vtscsi_softc *sc, int fd, + struct pci_vtscsi_request *req) +{ + union ctl_io *io = req->vsr_backend; + void *ext_data_ptr = NULL; + uint32_t ext_data_len = 0, ext_sg_entries = 0; + struct vtscsi_ctl_backend *ctl; + int err, nxferred; + + ctl = (struct vtscsi_ctl_backend *)sc->vss_backend; + + io->io_hdr.nexus.initid = ctl->vcb_iid; + io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, + req->vsr_cmd_rd->lun); + + io->io_hdr.io_type = CTL_IO_SCSI; + + if (req->vsr_data_niov_in > 0) { + ext_data_ptr = (void *)req->vsr_data_iov_in; + ext_sg_entries = req->vsr_data_niov_in; + ext_data_len = count_iov(req->vsr_data_iov_in, + req->vsr_data_niov_in); + io->io_hdr.flags |= CTL_FLAG_DATA_OUT; + } else if (req->vsr_data_niov_out > 0) { + ext_data_ptr = (void *)req->vsr_data_iov_out; + ext_sg_entries = req->vsr_data_niov_out; + ext_data_len = count_iov(req->vsr_data_iov_out, + req->vsr_data_niov_out); + io->io_hdr.flags |= CTL_FLAG_DATA_IN; + } + + io->scsiio.sense_len = sc->vss_config.sense_size; + io->scsiio.tag_num = req->vsr_cmd_rd->id; + io->io_hdr.flags |= CTL_FLAG_USER_TAG; + switch (req->vsr_cmd_rd->task_attr) { + case VIRTIO_SCSI_S_ORDERED: + io->scsiio.tag_type = CTL_TAG_ORDERED; + break; + case VIRTIO_SCSI_S_HEAD: + io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; + break; + case VIRTIO_SCSI_S_ACA: + io->scsiio.tag_type = CTL_TAG_ACA; + break; + case VIRTIO_SCSI_S_SIMPLE: + default: + io->scsiio.tag_type = CTL_TAG_SIMPLE; + break; + } + io->scsiio.ext_sg_entries = ext_sg_entries; + io->scsiio.ext_data_ptr = ext_data_ptr; + io->scsiio.ext_data_len = ext_data_len; + io->scsiio.ext_data_filled = 0; + io->scsiio.cdb_len = sc->vss_config.cdb_size; + memcpy(io->scsiio.cdb, req->vsr_cmd_rd->cdb, sc->vss_config.cdb_size); + + if (pci_vtscsi_debug) { + struct sbuf *sb = sbuf_new_auto(); + ctl_io_sbuf(io, sb); + sbuf_finish(sb); + DPRINTF("%s", sbuf_data(sb)); + sbuf_delete(sb); + } + + err = ioctl(fd, CTL_IO, io); + if (err != 0) { + WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; + } else { + req->vsr_cmd_wr->sense_len = + MIN(io->scsiio.sense_len, sc->vss_config.sense_size); + req->vsr_cmd_wr->residual = ext_data_len - + io->scsiio.ext_data_filled; + req->vsr_cmd_wr->status = io->scsiio.scsi_status; + req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OK; + memcpy(&req->vsr_cmd_wr->sense, &io->scsiio.sense_data, + req->vsr_cmd_wr->sense_len); + } + + nxferred = io->scsiio.ext_data_filled; + return (nxferred); +} + +static const struct pci_vtscsi_backend vtscsi_ctl_backend = { + .vsb_name = "ctl", + .vsb_init = vtscsi_ctl_init, + .vsb_open = vtscsi_ctl_open, + .vsb_reset = vtscsi_ctl_reset, + + .vsb_req_alloc = vtscsi_ctl_req_alloc, + .vsb_req_clear = vtscsi_ctl_req_clear, + .vsb_req_free = vtscsi_ctl_req_free, + + .vsb_tmf_hdl = vtscsi_ctl_tmf_hdl, + .vsb_an_hdl = vtscsi_ctl_an_hdl, + .vsb_req_hdl = vtscsi_ctl_req_hdl +}; +PCI_VTSCSI_BACKEND_SET(vtscsi_ctl_backend); diff --git a/usr.sbin/bsdinstall/distextract/distextract.c b/usr.sbin/bsdinstall/distextract/distextract.c index 32bd9453eb80..35cf19aae33f 100644 --- a/usr.sbin/bsdinstall/distextract/distextract.c +++ b/usr.sbin/bsdinstall/distextract/distextract.c @@ -291,6 +291,8 @@ extract_files(struct bsddialog_fileminibar *file) /* If that went well, perform the extraction */ if (retval == ARCHIVE_OK) retval = archive_read_extract(archive, entry, + ARCHIVE_EXTRACT_SECURE_NODOTDOT | + ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS | ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_OWNER | ARCHIVE_EXTRACT_PERM | ARCHIVE_EXTRACT_ACL | ARCHIVE_EXTRACT_XATTR | ARCHIVE_EXTRACT_FFLAGS); diff --git a/usr.sbin/bsdinstall/scripts/script b/usr.sbin/bsdinstall/scripts/script index 93d07c7899c3..4c792d80df6a 100755 --- a/usr.sbin/bsdinstall/scripts/script +++ b/usr.sbin/bsdinstall/scripts/script @@ -50,6 +50,7 @@ f_include $BSDCFG_SHARE/variable.subr ############################################################ GLOBALS : ${TMPDIR:="/tmp"} +: ${DISTRIBUTIONS=""}; export DISTRIBUTIONS # # Strings that should be moved to an i18n file and loaded with f_include_lang() diff --git a/usr.sbin/certctl/certctl.c b/usr.sbin/certctl/certctl.c index 0a07703bf37b..462f6c1730a9 100644 --- a/usr.sbin/certctl/certctl.c +++ b/usr.sbin/certctl/certctl.c @@ -1093,6 +1093,7 @@ int main(int argc, char *argv[]) { const char *command; + unsigned int i; int opt; while ((opt = getopt(argc, argv, "BcD:d:g:lL:M:no:Uv")) != -1) @@ -1155,8 +1156,8 @@ main(int argc, char *argv[]) set_defaults(); - for (unsigned i = 0; commands[i].name != NULL; i++) + for (i = 0; commands[i].name != NULL; i++) if (strcmp(command, commands[i].name) == 0) - exit(!!commands[i].func(argc, argv)); + exit(commands[i].func(argc, argv) == 0 ? 0 : 1); usage(); } diff --git a/usr.sbin/crashinfo/crashinfo.sh b/usr.sbin/crashinfo/crashinfo.sh index 68115f09f9d4..87e5565b98e3 100755 --- a/usr.sbin/crashinfo/crashinfo.sh +++ b/usr.sbin/crashinfo/crashinfo.sh @@ -149,11 +149,13 @@ fi VMCORE=$CRASHDIR/vmcore.$DUMPNR INFO=$CRASHDIR/info.$DUMPNR FILE=$CRASHDIR/core.txt.$DUMPNR +LINK=$CRASHDIR/core.txt.last HOSTNAME=`hostname` if $BATCH; then echo "Writing crash summary to $FILE." exec > $FILE 2>&1 + ln -sf $FILE $LINK fi GDB=/usr/local/bin/gdb @@ -204,6 +206,7 @@ machine=$(gdb_command $KERNEL 'printf "%s", machine') if ! $BATCH; then echo "Writing crash summary to $FILE." exec > $FILE 2>&1 + ln -sf $FILE $LINK fi echo "$HOSTNAME dumped core - see $VMCORE" diff --git a/usr.sbin/ctld/ctld.cc b/usr.sbin/ctld/ctld.cc index 9bdf15976911..834bef4ef363 100644 --- a/usr.sbin/ctld/ctld.cc +++ b/usr.sbin/ctld/ctld.cc @@ -1175,21 +1175,12 @@ conf::add_port(struct target *target, struct pport *pp) return (false); } - pp->link(); return (true); } bool -conf::add_port(struct kports &kports, struct target *target, int pp, int vp) +conf::add_port(struct target *target, const std::string &pname, int pp, int vp) { - struct pport *pport; - - std::string pname = freebsd::stringf("ioctl/%d/%d", pp, vp); - - pport = kports.find_port(pname); - if (pport != NULL) - return (add_port(target, pport)); - std::string name = pname + "-" + target->name(); const auto &pair = conf_ports.try_emplace(name, std::make_unique<ioctl_port>(target, pp, vp)); @@ -1387,6 +1378,19 @@ target::set_auth_type(const char *type) bool target::add_physical_port(std::string_view pport) { + /* Normalize ioctl port names. */ + std::string pname; + if (pport.compare(0, strlen("ioctl/"), "ioctl/") == 0) { + int ret, pp, vp; + + pname = std::string(pport); + ret = sscanf(pname.c_str(), "ioctl/%d/%d", &pp, &vp); + if (ret == 2) { + pname = freebsd::stringf("ioctl/%d/%d", pp, vp); + pport = pname; + } + } + for (const auto &s : t_pports) { if (s == pport) { log_warnx("duplicate physical port \"%s\" for target " @@ -2636,6 +2640,7 @@ conf_new_from_file(const char *path, bool ucl) bool conf::add_pports(struct kports &kports) { + std::unordered_map<struct pport *, struct target *> linked_ports; struct pport *pp; int ret, i_pp, i_vp; @@ -2643,35 +2648,50 @@ conf::add_pports(struct kports &kports) struct target *targ = kv.second.get(); for (const auto &pport : targ->pports()) { - ret = sscanf(pport.c_str(), "ioctl/%d/%d", &i_pp, - &i_vp); - if (ret > 0) { - if (!add_port(kports, targ, i_pp, i_vp)) { - log_warnx("can't create new ioctl port " - "for %s", targ->label()); + /* + * If this port is already present in the + * kernel, reuse the existing port. + */ + pp = kports.find_port(pport); + if (pp != nullptr) { + const auto &pair = linked_ports.try_emplace(pp, + targ); + if (!pair.second) { + log_warnx("can't link port \"%s\" to " + "%s, port already linked to %s", + pport.c_str(), targ->label(), + pair.first->second->label()); return (false); } + if (!add_port(targ, pp)) { + log_warnx( + "can't link port \"%s\" to %s", + pport.c_str(), targ->label()); + return (false); + } continue; } - pp = kports.find_port(pport); - if (pp == NULL) { - log_warnx("unknown port \"%s\" for %s", - pport.c_str(), targ->label()); - return (false); - } - if (pp->linked()) { - log_warnx("can't link port \"%s\" to %s, " - "port already linked to some target", - pport.c_str(), targ->label()); - return (false); - } - if (!add_port(targ, pp)) { - log_warnx("can't link port \"%s\" to %s", - pport.c_str(), targ->label()); - return (false); + /* + * If this port is an ioctl port, create a new + * port. + */ + ret = sscanf(pport.c_str(), "ioctl/%d/%d", &i_pp, + &i_vp); + if (ret == 2) { + if (!add_port(targ, pport, i_pp, i_vp)) { + log_warnx("can't create new port %s " + "for %s", pport.c_str(), + targ->label()); + return (false); + } + continue; } + + log_warnx("unknown port \"%s\" for %s", + pport.c_str(), targ->label()); + return (false); } } return (true); diff --git a/usr.sbin/ctld/ctld.hh b/usr.sbin/ctld/ctld.hh index 7ae033804157..3bc92449e372 100644 --- a/usr.sbin/ctld/ctld.hh +++ b/usr.sbin/ctld/ctld.hh @@ -487,7 +487,7 @@ struct conf { bool add_port(struct target *target, struct portal_group *pg, uint32_t ctl_port); bool add_port(struct target *target, struct pport *pp); - bool add_port(struct kports &kports, struct target *target, int pp, + bool add_port(struct target *target, const std::string &pname, int pp, int vp); bool add_pports(struct kports &kports); @@ -567,13 +567,9 @@ struct pport { const char *name() const { return pp_name.c_str(); } uint32_t ctl_port() const { return pp_ctl_port; } - bool linked() const { return pp_linked; } - void link() { pp_linked = true; } - private: std::string pp_name; uint32_t pp_ctl_port; - bool pp_linked = false; }; struct kports { diff --git a/usr.sbin/ctld/kernel.cc b/usr.sbin/ctld/kernel.cc index a6b0a79850ec..ef7ef87af090 100644 --- a/usr.sbin/ctld/kernel.cc +++ b/usr.sbin/ctld/kernel.cc @@ -446,8 +446,8 @@ add_iscsi_port(struct kports &kports, struct conf *conf, const struct cctl_port &port, std::string &name) { if (port.cfiscsi_target.empty()) { - log_debugx("CTL port %u \"%s\" wasn't managed by ctld; ", - port.port_id, name.c_str()); + log_debugx("CTL iSCSI port %u \"%s\" is not managed by ctld; " + "ignoring", port.port_id, name.c_str()); if (!kports.has_port(name)) { if (!kports.add_port(name, port.port_id)) { log_warnx("kports::add_port failed"); @@ -498,8 +498,8 @@ add_nvmf_port(struct conf *conf, const struct cctl_port &port, std::string &name) { if (port.nqn.empty() || port.ctld_transport_group_name.empty()) { - log_debugx("CTL port %u \"%s\" wasn't managed by ctld; ", - port.port_id, name.c_str()); + log_debugx("CTL NVMeoF port %u \"%s\" is not managed by ctld; " + "ignoring", port.port_id, name.c_str()); return; } @@ -549,7 +549,10 @@ conf_new_from_kernel(struct kports &kports) continue; std::string name = port.port_name; - if (port.pp != 0) { + if (port.port_frontend == "ioctl") + name += "/" + std::to_string(port.pp) + "/" + + std::to_string(port.vp); + else if (port.pp != 0 || port.vp != 0) { name += "/" + std::to_string(port.pp); if (port.vp != 0) name += "/" + std::to_string(port.vp); @@ -567,7 +570,7 @@ conf_new_from_kernel(struct kports &kports) for (const auto &lun : devlist.lun_list) { if (lun.ctld_name.empty()) { - log_debugx("CTL lun %ju wasn't managed by ctld; " + log_debugx("CTL lun %ju is not managed by ctld; " "ignoring", (uintmax_t)lun.lun_id); continue; } diff --git a/usr.sbin/lpr/lpd/lpd.8 b/usr.sbin/lpr/lpd/lpd.8 index 9d24a450ba0e..57331df02089 100644 --- a/usr.sbin/lpr/lpd/lpd.8 +++ b/usr.sbin/lpr/lpd/lpd.8 @@ -25,7 +25,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd February 19, 2026 +.Dd May 25, 2026 .Dt LPD 8 .Os .Sh NAME @@ -33,9 +33,9 @@ .Nd line printer spooler daemon .Sh SYNOPSIS .Nm -.Op Fl cdlpsFW46 +.Op Fl cdlsFW46 .Op Fl t Ar timeout -.Op Ar port# +.Op Ar port .Sh DEPRECATION NOTICE This facility is scheduled for removal prior to the release of .Fx 16.0 . @@ -63,6 +63,12 @@ the request so the parent can continue to listen for more requests. .Pp Available options: .Bl -tag -width Ds +.It Fl 4 +Accept IPv4 connections. +The default is to accept both IPv4 and IPv6. +.It Fl 6 +Accept IPv6 connections. +The default is to accept both IPv4 and IPv6. .It Fl c By default, if some remote host has a connection error while trying to send a print request to @@ -81,29 +87,25 @@ Turn on .Dv SO_DEBUG on the Internet listening socket (see .Xr setsockopt 2 ) . -.It Fl l +.It Fl F +By default, +.Nm +will daemonize into the background. The -.Fl l +.Fl F flag causes .Nm +to remain in the foreground. +Logging is still performed with +.Xr syslog 3 . +.It Fl l +Causes +.Nm to log valid requests received from the network. This can be useful for debugging purposes. -.It Fl p -The -.Fl p -flag is a synonym for the -.Fl s -flag. -It is being deprecated, and may be removed in a -future version of -.Nm . .It Fl s -The -.Fl s -(secure) flag causes -.Nm -not to open an Internet listening socket. +Do not open an Internet listening socket. This means that .Nm will not accept any connections from any remote @@ -114,17 +116,6 @@ Set the network receive timeout for client connections to .Ar timeout seconds. The default is 120. -.It Fl F -By default, -.Nm -will daemonize into the background. -The -.Fl F -flag causes -.Nm -to remain in the foreground. -Logging is still performed with -.Xr syslog 3 . .It Fl W By default, the .Nm @@ -137,19 +128,13 @@ flag causes to accept connections coming from any port. This is can be useful when you want to accept print jobs from certain implementations of lpr written for Windows. -.It Fl 4 -Inet only. -.It Fl 6 -Inet6 only. -.It Fl 46 -Inet and inet6 (default). -.It Ar "port#" -The Internet port number used to rendezvous -with other processes is normally obtained with -.Xr getservbyname 3 -but can be changed with the -.Ar port# -argument. +.It Ar "port" +The Internet port number or service name used to rendezvous with other +processes. +Can be any number from 1 to 65,535 or any service name defined in +.Pa /etc/services . +Defaults to +.Li printer . .El .Pp Access control is provided by two means. @@ -346,6 +331,7 @@ but not under same administrative control. .Xr syslog 3 , .Xr hosts.lpd 5 , .Xr printcap 5 , +.Xr services 5 , .Xr chkprintcap 8 , .Xr lpc 8 , .Xr pac 8 diff --git a/usr.sbin/lpr/lpd/lpd.c b/usr.sbin/lpr/lpd/lpd.c index d1dcd0766a77..d01aac7a7a35 100644 --- a/usr.sbin/lpr/lpd/lpd.c +++ b/usr.sbin/lpr/lpd/lpd.c @@ -101,7 +101,7 @@ static void startup(void); static void chkhost(struct sockaddr *_f, int _ch_opts); static int ckqueue(struct printer *_pp); static void fhosterr(int _ch_opts, char *_sysmsg, char *_usermsg); -static int *socksetup(int _af, int _debuglvl); +static int *socksetup(int _af, const char *portstr, int _debuglvl); static void usage(void); /* XXX from libc/net/rcmd.c */ @@ -124,7 +124,7 @@ main(int argc, char **argv) struct sockaddr_storage frominet; socklen_t fromlen; sigset_t omask, nmask; - struct servent *sp, serv; + const char *portstr = "printer"; int inet_flag = 0, inet6_flag = 0; euid = geteuid(); /* these shouldn't be different */ @@ -153,9 +153,6 @@ main(int argc, char **argv) lflag++; break; case 'p': /* letter initially used for -s */ - /* - * This will probably be removed with 5.0-release. - */ /* FALLTHROUGH */ case 's': /* secure (no inet) */ sflag++; @@ -215,19 +212,9 @@ main(int argc, char **argv) if (errs) usage(); - if (argc == 1) { - if ((i = atoi(argv[0])) == 0) - usage(); - if (i < 0 || i > USHRT_MAX) - errx(EX_USAGE, "port # %d is invalid", i); - - serv.s_port = htons(i); - sp = &serv; + if (argc > 0) { + portstr = *argv++; argc--; - } else { - sp = getservbyname("printer", "tcp"); - if (sp == NULL) - errx(EX_OSFILE, "printer/tcp: unknown service"); } if (argc != 0) @@ -339,7 +326,7 @@ main(int argc, char **argv) FD_SET(funix, &defreadfds); listen(funix, 5); if (sflag == 0) { - finet = socksetup(family, socket_debug); + finet = socksetup(family, portstr, socket_debug); } else finet = NULL; /* pretend we couldn't open TCP socket. */ if (finet) { @@ -855,7 +842,7 @@ fhosterr(int ch_opts, char *sysmsg, char *usermsg) /* if af is PF_UNSPEC more than one socket may be returned */ /* the returned list is dynamically allocated, so caller needs to free it */ static int * -socksetup(int af, int debuglvl) +socksetup(int af, const char *portstr, int debuglvl) { struct addrinfo hints, *res, *r; int error, maxs, *s, *socks; @@ -865,7 +852,7 @@ socksetup(int af, int debuglvl) hints.ai_flags = AI_PASSIVE; hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; - error = getaddrinfo(NULL, "printer", &hints, &res); + error = getaddrinfo(NULL, portstr, &hints, &res); if (error) { syslog(LOG_ERR, "%s", gai_strerror(error)); mcleanup(0); @@ -934,9 +921,9 @@ static void usage(void) { #ifdef INET6 - fprintf(stderr, "usage: lpd [-cdlsFW46] [port#]\n"); + fprintf(stderr, "usage: lpd [-cdlsFW46] [port]\n"); #else - fprintf(stderr, "usage: lpd [-cdlsFW] [port#]\n"); + fprintf(stderr, "usage: lpd [-cdlsFW] [port]\n"); #endif exit(EX_USAGE); } diff --git a/usr.sbin/lpr/lpd/printjob.c b/usr.sbin/lpr/lpd/printjob.c index 1c6736097492..9173b69a36b9 100644 --- a/usr.sbin/lpr/lpd/printjob.c +++ b/usr.sbin/lpr/lpd/printjob.c @@ -68,8 +68,12 @@ /* * The buffer size to use when reading/writing spool files. + * This is also used to compose commands for remote servers and therefore + * needs to be able to fit the longest command we could possibly send. */ -#define SPL_BUFSIZ BUFSIZ +#define SPL_BUFSIZ (BUFSIZ * 2) +_Static_assert(SPL_BUFSIZ > sizeof("x18446744073709551615 ") + PATH_MAX + + sizeof("_c2147483647 "), "SPL_BUFSIZ is too short"); /* * Error tokens @@ -132,10 +136,10 @@ static int printit(struct printer *_pp, char *_file); static void pstatus(const struct printer *_pp, const char *_msg, ...) __printflike(2, 3); static char response(const struct printer *_pp); -static void scan_out(struct printer *_pp, int _scfd, char *_scsp, +static void scan_out(struct printer *_pp, int _scfd, char *_scsp, int _dlm); static char *scnline(int _key, char *_p, int _c); -static int sendfile(struct printer *_pp, int _type, char *_file, +static int sendfile(struct printer *_pp, int _type, char *_file, char _format, int _copyreq); static int sendit(struct printer *_pp, char *_file); static void sendmail(struct printer *_pp, char *_userid, int _bombed); @@ -146,18 +150,17 @@ void printjob(struct printer *pp) { struct stat stb; - register struct jobqueue *q, **qp; + struct jobqueue *q, **qp; struct jobqueue **queue; - register int i, nitems; off_t pidoff; pid_t printpid; - int errcnt, jobcount, statok, tempfd; + int i, errcnt, jobcount, nitems, tempfd; jobcount = 0; init(pp); /* set up capabilities */ (void) write(STDOUT_FILENO, "", 1); /* ack that daemon is started */ (void) close(STDERR_FILENO); /* set up log file */ - if (open(pp->log_file, O_WRONLY|O_APPEND, LOG_FILE_MODE) < 0) { + if (open(pp->log_file, O_WRONLY | O_APPEND, LOG_FILE_MODE) < 0) { syslog(LOG_ERR, "%s: open(%s): %m", pp->printer, pp->log_file); (void) open(_PATH_DEVNULL, O_WRONLY); @@ -186,12 +189,9 @@ printjob(struct printer *pp) pp->spool_dir); exit(1); } - statok = stat(pp->lock_file, &stb); - if (statok == 0 && (stb.st_mode & LFM_PRINT_DIS)) - exit(0); /* printing disabled */ umask(S_IWOTH); - lfd = open(pp->lock_file, O_WRONLY|O_CREAT|O_EXLOCK|O_NONBLOCK, - LOCK_FILE_MODE); + lfd = open(pp->lock_file, O_WRONLY | O_CREAT | O_EXLOCK | O_NONBLOCK, + LOCK_FILE_MODE); if (lfd < 0) { if (errno == EWOULDBLOCK) /* active daemon present */ exit(0); @@ -199,12 +199,14 @@ printjob(struct printer *pp) pp->lock_file); exit(1); } - /* - * If the initial call to stat() failed, then lock_file will have - * been created by open(). Update &stb to match that new file. - */ - if (statok != 0) - statok = stat(pp->lock_file, &stb); + if (fstat(lfd, &stb) != 0) { + syslog(LOG_ERR, "%s: fstat(%s): %m", pp->printer, + pp->lock_file); + exit(1); + } + if ((stb.st_mode & LFM_PRINT_DIS) != 0) { + exit(0); /* printing disabled */ + } /* turn off non-blocking mode (was turned on for lock effects only) */ if (fcntl(lfd, F_SETFL, 0) < 0) { syslog(LOG_ERR, "%s: fcntl(%s): %m", pp->printer, @@ -215,8 +217,7 @@ printjob(struct printer *pp) /* * write process id for others to know */ - sprintf(line, "%u\n", printpid); - pidoff = i = strlen(line); + pidoff = i = snprintf(line, sizeof(line), "%u\n", printpid); if (write(lfd, line, i) != i) { syslog(LOG_ERR, "%s: write(%s): %m", pp->printer, pp->lock_file); @@ -226,7 +227,7 @@ printjob(struct printer *pp) * search the spool directory for work and sort by queue order. */ if ((nitems = getq(pp, &queue)) < 0) { - syslog(LOG_ERR, "%s: can't scan %s", pp->printer, + syslog(LOG_ERR, "%s: can't scan %s", pp->printer, pp->spool_dir); exit(1); } @@ -244,9 +245,10 @@ printjob(struct printer *pp) tempstderr); exit(1); } - if ((i = fchmod(tempfd, 0664)) == -1) { + if (fchmod(tempfd, 0664) == -1) { syslog(LOG_ERR, "%s: fchmod(%s): %m", pp->printer, tempstderr); + (void) unlink(tempstderr); exit(1); } /* lpd doesn't need it to be open, it just needs it to exist */ @@ -259,7 +261,7 @@ again: * write the name of the current control file into the lock file * so the spool queue program can tell what we're working on */ - for (qp = queue; nitems--; free((char *) q)) { + for (qp = queue; nitems--; free(q)) { q = *qp++; if (stat(q->job_cfname, &stb) < 0) continue; @@ -312,12 +314,12 @@ again: } (void) close(pfd); /* close printer */ if (ftruncate(lfd, pidoff) < 0) - syslog(LOG_WARNING, "%s: ftruncate(%s): %m", + syslog(LOG_WARNING, "%s: ftruncate(%s): %m", pp->printer, pp->lock_file); openpr(pp); /* try to reopen printer */ goto restart; } else { - syslog(LOG_WARNING, "%s: job could not be %s (%s)", + syslog(LOG_WARNING, "%s: job could not be %s (%s)", pp->printer, pp->remote ? "sent to remote host" : "printed", q->job_cfname); @@ -358,15 +360,18 @@ again: goto again; } -char fonts[4][50]; /* fonts for troff */ +char fonts[4][64]; /* fonts for troff */ -char ifonts[4][40] = { +char ifonts[4][64] = { _PATH_VFONTR, _PATH_VFONTI, _PATH_VFONTB, _PATH_VFONTS, }; +_Static_assert(sizeof(fonts) == sizeof(ifonts), "fonts != ifonts"); +_Static_assert(sizeof(fonts[0]) == sizeof(ifonts[0]), "fonts[0] != ifonts[0]"); + /* * The remaining part is the reading of the control file (cf) * and performing the various actions. @@ -374,9 +379,8 @@ char ifonts[4][40] = { static int printit(struct printer *pp, char *file) { - register int i; char *cp; - int bombed, didignorehdr; + int bombed, didignorehdr, i; bombed = OK; didignorehdr = 0; @@ -390,14 +394,13 @@ printit(struct printer *pp, char *file) /* * Reset troff fonts. */ - for (i = 0; i < 4; i++) - strcpy(fonts[i], ifonts[i]); + memcpy(fonts, ifonts, sizeof(fonts)); sprintf(&width[2], "%ld", pp->page_width); - strcpy(indent+2, "0"); + strcpy(indent, "-i0"); /* initialize job-specific count of datafiles processed */ job_dfcnt = 0; - + /* * read the control file for work to do * @@ -445,7 +448,7 @@ printit(struct printer *pp, char *file) case 'H': strlcpy(origin_host, line + 1, sizeof(origin_host)); if (class[0] == '\0') { - strlcpy(class, line+1, sizeof(class)); + strlcpy(class, line + 1, sizeof(class)); } continue; @@ -454,14 +457,14 @@ printit(struct printer *pp, char *file) if (pp->restricted) { /* restricted */ if (getpwnam(logname) == NULL) { bombed = NOACCT; - sendmail(pp, line+1, bombed); + sendmail(pp, line + 1, bombed); goto pass2; } } continue; case 'S': - cp = line+1; + cp = line + 1; i = 0; while (*cp >= '0' && *cp <= '9') i = i * 10 + (*cp++ - '0'); @@ -497,7 +500,7 @@ printit(struct printer *pp, char *file) case 'L': /* identification line */ if (!pp->no_header && !pp->header_last) - banner(pp, line+1, jobname); + banner(pp, line + 1, jobname); continue; case '1': /* troff fonts */ @@ -511,11 +514,11 @@ printit(struct printer *pp, char *file) continue; case 'W': /* page width */ - strlcpy(width+2, line + 1, sizeof(width) - 2); + strlcpy(width + 2, line + 1, sizeof(width) - 2); continue; case 'I': /* indent amount */ - strlcpy(indent+2, line + 1, sizeof(indent) - 2); + strlcpy(indent + 2, line + 1, sizeof(indent) - 2); continue; case 'Z': /* locale for pr */ @@ -537,7 +540,7 @@ printit(struct printer *pp, char *file) pp->printer, line[0], &line[1]); continue; } - i = print(pp, line[0], line+1); + i = print(pp, line[0], line + 1); switch (i) { case ERROR: if (bombed == OK) @@ -569,18 +572,18 @@ pass2: switch (line[0]) { case 'L': /* identification line */ if (!pp->no_header && pp->header_last) - banner(pp, line+1, jobname); + banner(pp, line + 1, jobname); continue; case 'M': if (bombed < NOACCT) /* already sent if >= NOACCT */ - sendmail(pp, line+1, bombed); + sendmail(pp, line + 1, bombed); continue; case 'U': - if (strchr(line+1, '/')) + if (strchr(line + 1, '/')) continue; - (void) unlink(line+1); + (void) unlink(line + 1); } /* * clean-up in case another control file exists @@ -603,14 +606,13 @@ pass2: static int print(struct printer *pp, int format, char *file) { - register int n, i; - register char *prog; - int fi, fo; + char buf[SPL_BUFSIZ]; + struct stat stb; + char *av[15]; + char *prog; FILE *fp; - char *av[15], buf[SPL_BUFSIZ]; pid_t wpid; - int p[2], retcode, stopped, wstatus, wstatus_set; - struct stat stb; + int fi, fo, i, n, p[2], retcode, stopped, wstatus, wstatus_set; /* Make sure the entire data file has arrived. */ wait4data(pp, file); @@ -640,7 +642,7 @@ print(struct printer *pp, int format, char *file) if (pp->filters[LPF_INPUT] == NULL && (format == 'f' || format == 'l' || format == 'o')) { pp->tof = 0; - while ((n = read(fi, buf, SPL_BUFSIZ)) > 0) + while ((n = read(fi, buf, sizeof(buf))) > 0) if (write(ofd, buf, n) != n) { (void) close(fi); return (REPRINT); @@ -674,7 +676,7 @@ print(struct printer *pp, int format, char *file) execl(_PATH_PR, "pr", width, length, "-h", *title ? title : " ", "-L", *locale ? locale : "C", - "-F", (char *)0); + "-F", NULL); syslog(LOG_ERR, "cannot execl %s", _PATH_PR); exit(2); } @@ -723,7 +725,7 @@ print(struct printer *pp, int format, char *file) case 'd': /* print tex output */ (void) unlink(".railmag"); if ((fo = creat(".railmag", FILMOD)) < 0) { - syslog(LOG_ERR, "%s: cannot create .railmag", + syslog(LOG_ERR, "%s: cannot create .railmag", pp->printer); (void) unlink(".railmag"); } else { @@ -736,7 +738,7 @@ print(struct printer *pp, int format, char *file) } (void) close(fo); } - prog = (format == 't') ? pp->filters[LPF_TROFF] + prog = (format == 't') ? pp->filters[LPF_TROFF] : ((format == 'n') ? pp->filters[LPF_DITROFF] : pp->filters[LPF_DVI]); av[1] = pxwidth; @@ -810,7 +812,7 @@ start: dup2(fo, STDOUT_FILENO); /* setup stderr for the filter (child process) * so it goes to our temporary errors file */ - n = open(tempstderr, O_WRONLY|O_TRUNC, 0664); + n = open(tempstderr, O_WRONLY | O_TRUNC, 0664); if (n >= 0) dup2(n, STDERR_FILENO); closelog(); @@ -882,8 +884,9 @@ start: static int sendit(struct printer *pp, char *file) { + char last[sizeof(line)]; + char *cp; int dfcopies, err, i; - char *cp, last[sizeof(line)]; /* * open control file @@ -913,7 +916,7 @@ sendit(struct printer *pp, char *file) while (get_line(cfp)) { again: if (line[0] == 'S') { - cp = line+1; + cp = line + 1; i = 0; while (*cp >= '0' && *cp <= '9') i = i * 10 + (*cp++ - '0'); @@ -932,22 +935,22 @@ sendit(struct printer *pp, char *file) strlcpy(logname, line + 1, sizeof(logname)); if (pp->restricted) { /* restricted */ if (getpwnam(logname) == NULL) { - sendmail(pp, line+1, NOACCT); + sendmail(pp, line + 1, NOACCT); err = ERROR; break; } } } else if (line[0] == 'I') { - strlcpy(indent+2, line + 1, sizeof(indent) - 2); + strlcpy(indent + 2, line + 1, sizeof(indent) - 2); } else if (line[0] >= 'a' && line[0] <= 'z') { dfcopies = 1; - strcpy(last, line); + memcpy(last, line, sizeof(last)); while ((i = get_line(cfp)) != 0) { if (strcmp(last, line) != 0) break; dfcopies++; } - switch (sendfile(pp, '\3', last+1, *last, dfcopies)) { + switch (sendfile(pp, '\3', last + 1, *last, dfcopies)) { case OK: if (i) goto again; @@ -972,8 +975,8 @@ sendit(struct printer *pp, char *file) */ fseek(cfp, 0L, 0); while (get_line(cfp)) - if (line[0] == 'U' && !strchr(line+1, '/')) - (void) unlink(line+1); + if (line[0] == 'U' && !strchr(line + 1, '/')) + (void) unlink(line + 1); /* * clean-up in case another control file exists */ @@ -989,11 +992,10 @@ sendit(struct printer *pp, char *file) static int sendfile(struct printer *pp, int type, char *file, char format, int copyreq) { - int i, amt; + char buf[SPL_BUFSIZ], opt_c[4], opt_h[4], opt_n[4]; struct stat stb; char *av[15], *filtcmd; - char buf[SPL_BUFSIZ], opt_c[4], opt_h[4], opt_n[4]; - int copycnt, filtstat, narg, resp, sfd, sfres, sizerr, statrc; + int amt, copycnt, filtstat, i, narg, resp, sfd, sfres, sizerr, statrc; /* Make sure the entire data file has arrived. */ wait4data(pp, file); @@ -1127,13 +1129,18 @@ sendfile(struct printer *pp, int type, char *file, char format, int copyreq) sendagain: copycnt++; - if (copycnt < 2) - (void) sprintf(buf, "%c%" PRId64 " %s\n", type, stb.st_size, - file); - else - (void) sprintf(buf, "%c%" PRId64 " %s_c%d\n", type, stb.st_size, - file, copycnt); - amt = strlen(buf); + if (copycnt < 2) { + amt = snprintf(buf, sizeof(buf), "%c%" PRId64 " %s\n", + type, stb.st_size, file); + } else { + amt = snprintf(buf, sizeof(buf), "%c%" PRId64 " %s_c%d\n", + type, stb.st_size, file, copycnt); + } + if (amt >= sizeof(buf)) { + syslog(LOG_ERR, "%s: file name too long", file); + sfres = ERROR; + goto return_sfres; + } for (i = 0; ; i++) { if (write(pfd, buf, amt) != amt || (resp = response(pp)) < 0 || resp == '\1') { @@ -1157,8 +1164,8 @@ sendagain: */ if (type == '\3') trstat_init(pp, file, job_dfcnt); - for (i = 0; i < stb.st_size; i += SPL_BUFSIZ) { - amt = SPL_BUFSIZ; + for (i = 0; i < stb.st_size; i += sizeof(buf)) { + amt = sizeof(buf); if (i + amt > stb.st_size) amt = stb.st_size - i; if (sizerr == 0 && read(sfd, buf, amt) != amt) @@ -1248,12 +1255,12 @@ return_sfres: static void wait4data(struct printer *pp, const char *dfile) { + struct stat statdf; const char *cp; - int statres; - u_int sleepreq; size_t dlen, hlen; time_t amtslept, cur_time, prev_mtime; - struct stat statdf; + unsigned int sleepreq; + int statres; /* Skip these checks if the print job is from the local host. */ dlen = strlen(dfile); @@ -1338,10 +1345,11 @@ wait4data(struct printer *pp, const char *dfile) static int execfilter(struct printer *pp, char *f_cmd, char *f_av[], int infd, int outfd) { + char buf[BUFSIZ]; + char *slash; + FILE *errfp; pid_t fpid, wpid; int errfd, retcode, wstatus; - FILE *errfp; - char buf[BUFSIZ], *slash; fpid = dofork(pp, DORETURN); if (fpid != 0) { @@ -1403,7 +1411,7 @@ execfilter(struct printer *pp, char *f_cmd, char *f_av[], int infd, int outfd) */ dup2(infd, STDIN_FILENO); dup2(outfd, STDOUT_FILENO); - errfd = open(tempstderr, O_WRONLY|O_TRUNC, 0664); + errfd = open(tempstderr, O_WRONLY | O_TRUNC, 0664); if (errfd >= 0) dup2(errfd, STDERR_FILENO); closelog(); @@ -1476,7 +1484,7 @@ banner(struct printer *pp, char *name1, char *name2) static char * scnline(int key, char *p, int c) { - register int scnwidth; + int scnwidth; for (scnwidth = WIDTH; --scnwidth;) { key <<= 1; @@ -1490,12 +1498,12 @@ scnline(int key, char *p, int c) static void scan_out(struct printer *pp, int scfd, char *scsp, int dlm) { - register char *strp; - register int nchrs, j; - char outbuf[LINELEN+1], *sp, c, cc; - int d, scnhgt; + char outbuf[LINELEN + 1]; + char *sp, *strp; + int d, j, nchrs, scnhgt; + char c, cc; - for (scnhgt = 0; scnhgt++ < HEIGHT+DROP; ) { + for (scnhgt = 0; scnhgt++ < HEIGHT + DROP; ) { strp = &outbuf[0]; sp = scsp; for (nchrs = 0; ; ) { @@ -1504,9 +1512,10 @@ scan_out(struct printer *pp, int scfd, char *scsp, int dlm) for (j = WIDTH; --j;) *strp++ = BACKGND; else - strp = scnline(scnkey[(int)c][scnhgt-1-d], strp, cc); - if (*sp == dlm || *sp == '\0' || - nchrs++ >= pp->page_width/(WIDTH+1)-1) + strp = scnline(scnkey[(int)c][scnhgt - 1 - d], + strp, cc); + if (*sp == dlm || *sp == '\0' || + nchrs++ >= pp->page_width / (WIDTH + 1) - 1) break; *strp++ = BACKGND; *strp++ = BACKGND; @@ -1522,8 +1531,7 @@ scan_out(struct printer *pp, int scfd, char *scsp, int dlm) static int dropit(int c) { - switch(c) { - + switch (c) { case TRC('_'): case TRC(';'): case TRC(','): @@ -1533,7 +1541,6 @@ dropit(int c) case TRC('q'): case TRC('y'): return (DROP); - default: return (0); } @@ -1546,11 +1553,10 @@ dropit(int c) static void sendmail(struct printer *pp, char *userid, int bombed) { - register int i; - int p[2], s; - register const char *cp; struct stat stb; FILE *fp; + const char *cp; + int i, p[2], s; pipe(p); if ((s = dofork(pp, DORETURN)) == 0) { /* child */ @@ -1561,7 +1567,7 @@ sendmail(struct printer *pp, char *userid, int bombed) cp++; else cp = _PATH_SENDMAIL; - execl(_PATH_SENDMAIL, cp, "-t", (char *)0); + execl(_PATH_SENDMAIL, cp, "-t", NULL); _exit(0); } else if (s > 0) { /* parent */ dup2(p[1], STDOUT_FILENO); @@ -1590,8 +1596,8 @@ sendmail(struct printer *pp, char *userid, int bombed) break; case FILTERERR: cp = "FILTERERR"; - if (stat(tempstderr, &stb) < 0 || stb.st_size == 0 - || (fp = fopen(tempstderr, "r")) == NULL) { + if (stat(tempstderr, &stb) < 0 || stb.st_size == 0 || + (fp = fopen(tempstderr, "r")) == NULL) { printf("\nhad some errors and may not have printed\n"); break; } @@ -1623,9 +1629,9 @@ sendmail(struct printer *pp, char *userid, int bombed) static int dofork(const struct printer *pp, int action) { + struct passwd *pwd; pid_t forkpid; int i, fail; - struct passwd *pwd; forkpid = -1; if (daemon_uname == NULL) { @@ -1704,7 +1710,6 @@ error_ret: static void abortpr(int signo __unused) { - (void) unlink(tempstderr); kill(0, SIGINT); if (of_pid > 0) @@ -1734,12 +1739,13 @@ init(struct printer *pp) void startprinting(const char *printer) { - struct printer myprinter, *pp = &myprinter; + struct printer myprinter; + struct printer *pp = &myprinter; int status; init_printer(pp); status = getprintcap(printer, pp); - switch(status) { + switch (status) { case PCAPERR_OSERR: syslog(LOG_ERR, "can't open printer description file: %m"); exit(1); @@ -1760,8 +1766,8 @@ startprinting(const char *printer) static void openpr(const struct printer *pp) { - int p[2]; char *cp; + int p[2]; if (pp->remote) { openrem(pp); @@ -1772,7 +1778,7 @@ openpr(const struct printer *pp) * local print queues. For remote machines, all 'of=' * filter processing is handled in sendfile(), and that * does not use these global "output filter" variables. - */ + */ ofd = -1; of_pid = 0; return; @@ -1806,8 +1812,7 @@ openpr(const struct printer *pp) cp = pp->filters[LPF_OUTPUT]; else cp++; - execl(pp->filters[LPF_OUTPUT], cp, width, length, - (char *)0); + execl(pp->filters[LPF_OUTPUT], cp, width, length, NULL); syslog(LOG_ERR, "%s: execl(%s): %m", pp->printer, pp->filters[LPF_OUTPUT]); exit(1); @@ -1827,11 +1832,10 @@ openpr(const struct printer *pp) static void opennet(const struct printer *pp) { - register int i; - int resp; - u_long port; - char *ep; void (*savealrm)(int); + char *ep; + unsigned long port; + int i, resp; port = strtoul(pp->lp, &ep, 0); if (*ep != '@' || port > 65535) { @@ -1864,7 +1868,7 @@ opennet(const struct printer *pp) pstatus(pp, "waiting for %s to come up", pp->lp); else - pstatus(pp, + pstatus(pp, "waiting for access to printer on %s", pp->lp); } @@ -1879,7 +1883,7 @@ opennet(const struct printer *pp) static void opentty(const struct printer *pp) { - register int i; + int i; for (i = 1; ; i = i < 32 ? i << 1 : i) { pfd = open(pp->lp, pp->rw ? O_RDWR : O_WRONLY); @@ -1892,7 +1896,7 @@ opentty(const struct printer *pp) exit(1); } if (i == 1) - pstatus(pp, + pstatus(pp, "waiting for %s to become ready (offline?)", pp->printer); sleep(i); @@ -1908,9 +1912,8 @@ opentty(const struct printer *pp) static void openrem(const struct printer *pp) { - register int i; - int resp; void (*savealrm)(int); + int i, resp; for (i = 1; ; i = i < 256 ? i << 1 : i) { resp = -1; @@ -1920,16 +1923,15 @@ openrem(const struct printer *pp) alarm(0); (void)signal(SIGALRM, savealrm); if (pfd >= 0) { - if ((writel(pfd, "\2", pp->remote_queue, "\n", - (char *)0) - == 2 + strlen(pp->remote_queue)) - && (resp = response(pp)) == 0) + if (writel(pfd, "\2", pp->remote_queue, "\n", NULL) == + 2 + strlen(pp->remote_queue) && + (resp = response(pp)) == 0) break; (void) close(pfd); } if (i == 1) { if (resp < 0) - pstatus(pp, "waiting for %s to come up", + pstatus(pp, "waiting for %s to come up", pp->remote_host); else { pstatus(pp, @@ -1951,7 +1953,7 @@ setty(const struct printer *pp) { struct termios ttybuf; - if (ioctl(pfd, TIOCEXCL, (char *)0) < 0) { + if (ioctl(pfd, TIOCEXCL, NULL) < 0) { syslog(LOG_ERR, "%s: ioctl(TIOCEXCL): %m", pp->printer); exit(1); } @@ -1980,22 +1982,23 @@ setty(const struct printer *pp) static void pstatus(const struct printer *pp, const char *msg, ...) { - int fd; - char *buf; va_list ap; - va_start(ap, msg); + char *buf; + int fd; umask(S_IWOTH); - fd = open(pp->status_file, O_WRONLY|O_CREAT|O_EXLOCK, STAT_FILE_MODE); + fd = open(pp->status_file, O_WRONLY | O_CREAT | O_EXLOCK, + STAT_FILE_MODE); if (fd < 0) { syslog(LOG_ERR, "%s: open(%s): %m", pp->printer, pp->status_file); exit(1); } ftruncate(fd, 0); + va_start(ap, msg); vasprintf(&buf, msg, ap); va_end(ap); - writel(fd, buf, "\n", (char *)0); + writel(fd, buf, "\n", NULL); close(fd); free(buf); } diff --git a/usr.sbin/sndctl/sndctl.8 b/usr.sbin/sndctl/sndctl.8 index 73414bd95325..44dcb791ca51 100644 --- a/usr.sbin/sndctl/sndctl.8 +++ b/usr.sbin/sndctl/sndctl.8 @@ -27,7 +27,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd December 2, 2025 +.Dd April 17, 2026 .Dt SNDCTL 8 .Os .Sh NAME @@ -82,6 +82,8 @@ The device controls are as follows: .It bitperfect Ta Boolean Ta Read/Write Ta Bit-perfect mode enabled .It autoconv Ta Boolean Ta Read/Write Ta Auto-conversions enabled .It realtime Ta Boolean Ta Read/Write Ta Real-time mode enabled +.It eq Ta Boolean Ta Read/Write Ta Equalization enabled +.It eq_preamp Ta String Ta Read/Write Ta Equalization preamp value (in dB) .It play Ta Group Ta Read Ta Playback properties .It play.format Ta String Ta Read/Write Ta Playback format .It play.rate Ta Number Ta Read/Write Ta Playback sample rate diff --git a/usr.sbin/sndctl/sndctl.c b/usr.sbin/sndctl/sndctl.c index bbc2da6a4ab9..778643a2a978 100644 --- a/usr.sbin/sndctl/sndctl.c +++ b/usr.sbin/sndctl/sndctl.c @@ -97,6 +97,8 @@ struct snd_dev { int bitperfect; int realtime; int autoconv; + int eq; + char eq_preamp[BUFSIZ]; struct { char format[FMTSTR_LEN]; int rate; @@ -130,6 +132,8 @@ struct map { static int mod_bitperfect(struct snd_dev *, void *); static int mod_autoconv(struct snd_dev *, void *); static int mod_realtime(struct snd_dev *, void *); +static int mod_eq(struct snd_dev *, void *); +static int mod_eq_preamp(struct snd_dev *, void *); static int mod_play_vchans(struct snd_dev *, void *); static int mod_play_rate(struct snd_dev *, void *); static int mod_play_format(struct snd_dev *, void *); @@ -149,6 +153,8 @@ static struct snd_ctl dev_ctls[] = { { "bitperfect", F(bitperfect), NUM, mod_bitperfect }, { "autoconv", F(autoconv), NUM, mod_autoconv }, { "realtime", F(realtime), NUM, mod_realtime }, + { "eq", F(eq), NUM, mod_eq }, + { "eq_preamp", F(eq_preamp), STR, mod_eq_preamp }, { "play", F(play), GRP, NULL }, { "play.format", F(play.format), STR, mod_play_format }, { "play.rate", F(play.rate), NUM, mod_play_rate }, @@ -436,6 +442,7 @@ read_dev(char *path) struct sndstioc_nv_arg arg; struct snd_dev *dp = NULL; struct snd_chan *ch; + char buf[64]; size_t nitems, nchans, i, j; int fd, caps, unit, t1, t2, t3; @@ -557,6 +564,14 @@ read_dev(char *path) if (t1 == 0 && t2 == 0 && t3 == 0) dp->realtime = 1; + snprintf(buf, sizeof(buf), "dev.pcm.%d.eq", dp->unit); + if (sysctl_int(buf, NULL, &dp->eq)) + xo_err(1, "%s: sysctl", dp->name); + + snprintf(buf, sizeof(buf), "dev.pcm.%d.eq_preamp", dp->unit); + if (sysctl_str(buf, NULL, dp->eq_preamp, sizeof(dp->eq_preamp))) + xo_err(1, "%s: sysctl", dp->name); + if (!nvlist_exists(nvlist_get_nvlist(di[i], SNDST_DSPS_PROVIDER_INFO), SNDST_DSPS_SOUND4_CHAN_INFO)) xo_errx(1, "%s: channel info list empty", dp->name); @@ -843,6 +858,32 @@ mod_realtime(struct snd_dev *dp, void *arg) } static int +mod_eq(struct snd_dev *dp, void *arg) +{ + char buf[64]; + + if (dp->from_user) + return (-1); + + snprintf(buf, sizeof(buf), "dev.pcm.%d.eq", dp->unit); + + return (sysctl_int(buf, arg, &dp->eq)); +} + +static int +mod_eq_preamp(struct snd_dev *dp, void *arg) +{ + char buf[64]; + + if (dp->from_user) + return (-1); + + snprintf(buf, sizeof(buf), "dev.pcm.%d.eq_preamp", dp->unit); + + return (sysctl_str(buf, arg, dp->eq_preamp, sizeof(dp->eq_preamp))); +} + +static int mod_play_vchans(struct snd_dev *dp, void *arg) { char buf[64]; diff --git a/usr.sbin/spi/spi.c b/usr.sbin/spi/spi.c index 1ced2371e3d0..bbbc3d322b00 100644 --- a/usr.sbin/spi/spi.c +++ b/usr.sbin/spi/spi.c @@ -30,6 +30,7 @@ #include <sys/spigenio.h> #include <sys/sysctl.h> +#include <assert.h> #include <errno.h> #include <fcntl.h> #include <inttypes.h> @@ -364,30 +365,24 @@ main(int argc, char *argv[], char *envp[] __unused) /* do data transfer */ - if (stream) { - while (!err && !feof(stdin)) { - if (fdir == DIR_READ) { - err = perform_read(hdev, &opt); - } - else if (fdir == DIR_WRITE) { - err = perform_write(hdev, &opt); - } - else if (fdir == DIR_READWRITE) { - err = perform_readwrite(hdev, &opt); - } - } - } - else { - if (fdir == DIR_READ) { + assert(fdir != DIR_NONE); + do { + switch (fdir) { + case DIR_READ: err = perform_read(hdev, &opt); - } - else if (fdir == DIR_WRITE) { + break; + case DIR_WRITE: err = perform_write(hdev, &opt); - } - else if (fdir == DIR_READWRITE) { + break; + case DIR_READWRITE: err = perform_readwrite(hdev, &opt); + break; + default: + fprintf(stderr, "Invalid state (%d)\n", fdir); + err = EINVAL; + break; } - } + } while (stream && !err && !feof(stdin)); the_end: diff --git a/usr.sbin/syslogd/syslogd.c b/usr.sbin/syslogd/syslogd.c index 59cb56fd5970..70cb705c0cb7 100644 --- a/usr.sbin/syslogd/syslogd.c +++ b/usr.sbin/syslogd/syslogd.c @@ -360,7 +360,6 @@ static int evaluate_prop_filter(const struct prop_filter *filter, static nvlist_t *prop_filter_compile(const char *); static void parsemsg(const char *, char *); static void printsys(char *); -static const char *ttymsg_check(struct iovec *, int, char *, int); static void usage(void); static bool validate(struct sockaddr *, const char *); static void unmapped(struct sockaddr *); @@ -1756,8 +1755,6 @@ iovlist_truncate(struct iovlist *il, size_t size) static void fprintlog_write(struct filed *f, struct iovlist *il, int flags) { - const char *msgret; - switch (f->f_type) { case F_FORW: { ssize_t lsent; @@ -1910,10 +1907,10 @@ fprintlog_write(struct filed *f, struct iovlist *il, int flags) dprintf(" %s%s\n", _PATH_DEV, f->f_fname); iovlist_append(il, "\r\n"); errno = 0; /* ttymsg() only sometimes returns an errno */ - if ((msgret = cap_ttymsg(cap_syslogd, il->iov, il->iovcnt, - f->f_fname, 10))) { + if (cap_ttymsg(cap_syslogd, il->iov, il->iovcnt, f->f_fname, 10, + true) != 0) { f->f_type = F_UNUSED; - logerror(msgret); + logerror(f->f_fname); } break; @@ -2143,7 +2140,6 @@ wallmsg(const struct filed *f, struct iovec *iov, const int iovlen) static int reenter; /* avoid calling ourselves */ struct utmpx *ut; int i; - const char *p; if (reenter++) return; @@ -2153,19 +2149,19 @@ wallmsg(const struct filed *f, struct iovec *iov, const int iovlen) if (ut->ut_type != USER_PROCESS) continue; if (f->f_type == F_WALL) { - if ((p = ttymsg(iov, iovlen, ut->ut_line, - TTYMSGTIME)) != NULL) - dprintf("%s\n", p); + if (ttymsg(iov, iovlen, ut->ut_line, TTYMSGTIME, + false) != 0 && errno != ENOENT) + dprintf("%s: %m\n", ut->ut_line); continue; } /* should we send the message to this user? */ for (i = 0; i < MAXUNAMES; i++) { if (!f->f_uname[i][0]) break; - if (!strcmp(f->f_uname[i], ut->ut_user)) { - if ((p = ttymsg_check(iov, iovlen, ut->ut_line, - TTYMSGTIME)) != NULL) - dprintf("%s\n", p); + if (strcmp(f->f_uname[i], ut->ut_user) == 0) { + if (ttymsg(iov, iovlen, ut->ut_line, TTYMSGTIME, + true) != 0 && errno != ENOENT) + dprintf("%s: %m\n", ut->ut_line); break; } } @@ -2175,29 +2171,6 @@ wallmsg(const struct filed *f, struct iovec *iov, const int iovlen) } /* - * Wrapper routine for ttymsg() that checks the terminal for messages enabled. - */ -static const char * -ttymsg_check(struct iovec *iov, int iovcnt, char *line, int tmout) -{ - static char device[1024]; - static char errbuf[1024]; - struct stat sb; - - (void) snprintf(device, sizeof(device), "%s%s", _PATH_DEV, line); - - if (stat(device, &sb) < 0) { - (void) snprintf(errbuf, sizeof(errbuf), - "%s: %s", device, strerror(errno)); - return (errbuf); - } - if ((sb.st_mode & S_IWGRP) == 0) - /* Messages disabled. */ - return (NULL); - return (ttymsg(iov, iovcnt, line, tmout)); -} - -/* * Return a printable representation of a host address. */ static const char * diff --git a/usr.sbin/syslogd/syslogd_cap.c b/usr.sbin/syslogd/syslogd_cap.c index 7539e6b8661b..0149d09dab99 100644 --- a/usr.sbin/syslogd/syslogd_cap.c +++ b/usr.sbin/syslogd/syslogd_cap.c @@ -50,7 +50,7 @@ casper_command(const char *cmd, const nvlist_t *limits __unused, else if (strcmp(cmd, "readconfigfile") == 0) error = casper_readconfigfile(nvlin, nvlout); else if (strcmp(cmd, "ttymsg") == 0) - error = casper_ttymsg(nvlin, nvlout); + error = casper_ttymsg(nvlin); else if (strcmp(cmd, "wallmsg") == 0) error = casper_wallmsg(nvlin); diff --git a/usr.sbin/syslogd/syslogd_cap.h b/usr.sbin/syslogd/syslogd_cap.h index 2e52c57bcdf8..60106d7862bd 100644 --- a/usr.sbin/syslogd/syslogd_cap.h +++ b/usr.sbin/syslogd/syslogd_cap.h @@ -62,13 +62,13 @@ extern SLIST_HEAD(cfiled_list, cap_filed) cfiled_head; int cap_p_open(cap_channel_t *, size_t, const char *, int *); nvlist_t *cap_readconfigfile(cap_channel_t *, const char *); -const char *cap_ttymsg(cap_channel_t *, struct iovec *, int, const char *, int); +int cap_ttymsg(cap_channel_t *, struct iovec *, int, const char *, int, bool); void cap_wallmsg(cap_channel_t *, const struct filed *, struct iovec *, const int); int casper_p_open(nvlist_t *, nvlist_t *); int casper_readconfigfile(nvlist_t *, nvlist_t *); -int casper_ttymsg(nvlist_t *, nvlist_t *); +int casper_ttymsg(nvlist_t *); int casper_wallmsg(nvlist_t *); nvlist_t *filed_to_nvlist(const struct filed *); @@ -83,8 +83,8 @@ struct prop_filter *nvlist_to_prop_filter(const nvlist_t *nvl_prop_filter); p_open(prog, rpd) #define cap_readconfigfile(chan, cf) \ readconfigfile(cf) -#define cap_ttymsg(chan, iov, iovcnt, line, tmout) \ - ttymsg(iov, iovcnt, line, tmout) +#define cap_ttymsg(chan, iov, iovcnt, line, timeout) \ + ttymsg(iov, iovcnt, line, timeout) #define cap_wallmsg(chan, f, iov, iovcnt) \ wallmsg(f, iov, iovcnt) diff --git a/usr.sbin/syslogd/syslogd_cap_log.c b/usr.sbin/syslogd/syslogd_cap_log.c index 5e2034abd9eb..901349c4dc5a 100644 --- a/usr.sbin/syslogd/syslogd_cap_log.c +++ b/usr.sbin/syslogd/syslogd_cap_log.c @@ -52,14 +52,12 @@ cap_p_open(cap_channel_t *chan, size_t filed_idx, const char *prog, exit(1); } error = nvlist_get_number(nvl, "error"); - if (error != 0) { - errno = error; - logerror("Failed to open piped command"); - } pipedesc_w = dnvlist_take_descriptor(nvl, "pipedesc_w", -1); *procdesc = dnvlist_take_descriptor(nvl, "procdesc", -1); nvlist_destroy(nvl); + if (error != 0) + errno = error; return (pipedesc_w); } @@ -81,29 +79,27 @@ casper_p_open(nvlist_t *nvlin, nvlist_t *nvlout) pipedesc_w = p_open(prog, &procdesc); if (pipedesc_w == -1) - return (-1); + return (errno); nvlist_move_descriptor(nvlout, "pipedesc_w", pipedesc_w); nvlist_move_descriptor(nvlout, "procdesc", procdesc); return (0); } - - return (-1); + return (ECAPMODE); } -const char * +int cap_ttymsg(cap_channel_t *chan, struct iovec *iov, int iovcnt, - const char *line, int tmout) + const char *line, int timeout, bool shout) { nvlist_t *nvl = nvlist_create(0); - int error; - static char errbuf[1024]; - char *ret = NULL; + int error = 0; nvlist_add_string(nvl, "cmd", "ttymsg"); for (int i = 0; i < iovcnt; ++i) nvlist_append_string_array(nvl, "iov_strs", iov[i].iov_base); nvlist_add_string(nvl, "line", line); - nvlist_add_number(nvl, "tmout", tmout); + nvlist_add_number(nvl, "timeout", timeout); + nvlist_add_bool(nvl, "shout", shout); nvl = cap_xfer_nvlist(chan, nvl); if (nvl == NULL) { @@ -111,28 +107,23 @@ cap_ttymsg(cap_channel_t *chan, struct iovec *iov, int iovcnt, exit(1); } error = nvlist_get_number(nvl, "error"); + nvlist_destroy(nvl); if (error != 0) { errno = error; - logerror("Failed to ttymsg"); - } - if (nvlist_exists_string(nvl, "errstr")) { - const char *errstr = nvlist_get_string(nvl, "errstr"); - (void)strlcpy(errbuf, errstr, sizeof(errbuf)); - ret = errbuf; + return (-1); } - - nvlist_destroy(nvl); - return (ret); + return (0); } int -casper_ttymsg(nvlist_t *nvlin, nvlist_t *nvlout) +casper_ttymsg(nvlist_t *nvlin) { const char * const *nvlstrs; struct iovec *iov; - size_t iovcnt; - int tmout; const char *line; + size_t iovcnt; + int ret, timeout; + bool shout; nvlstrs = nvlist_get_string_array(nvlin, "iov_strs", &iovcnt); assert(iovcnt <= TTYMSG_IOV_MAX); @@ -144,13 +135,12 @@ casper_ttymsg(nvlist_t *nvlin, nvlist_t *nvlout) iov[i].iov_len = strlen(nvlstrs[i]); } line = nvlist_get_string(nvlin, "line"); - tmout = nvlist_get_number(nvlin, "tmout"); - line = ttymsg(iov, iovcnt, line, tmout); - if (line != NULL) - nvlist_add_string(nvlout, "errstr", line); - + timeout = nvlist_get_number(nvlin, "timeout"); + shout = nvlist_get_bool(nvlin, "shout"); + if ((ret = ttymsg(iov, iovcnt, line, timeout, shout)) != 0) + ret = errno; free(iov); - return (0); + return (ret); } void @@ -158,7 +148,6 @@ cap_wallmsg(cap_channel_t *chan, const struct filed *f, struct iovec *iov, int iovcnt) { nvlist_t *nvl = nvlist_create(0); - int error; nvlist_add_string(nvl, "cmd", "wallmsg"); /* @@ -175,11 +164,6 @@ cap_wallmsg(cap_channel_t *chan, const struct filed *f, struct iovec *iov, logerror("Failed to xfer wallmsg nvlist"); exit(1); } - error = nvlist_get_number(nvl, "error"); - if (error != 0) { - errno = error; - logerror("Failed to wallmsg"); - } nvlist_destroy(nvl); } diff --git a/usr.sbin/virtual_oss/virtual_oss/virtual_oss.c b/usr.sbin/virtual_oss/virtual_oss/virtual_oss.c index 1d95c87d57b1..42a748a5e3bd 100644 --- a/usr.sbin/virtual_oss/virtual_oss/virtual_oss.c +++ b/usr.sbin/virtual_oss/virtual_oss/virtual_oss.c @@ -783,8 +783,8 @@ virtual_oss_process(void *arg __unused) /* check if compressor should be applied */ voss_compressor(buffer_temp, pvp->rx_compressor_gain, - &pvp->rx_compressor_param, samples, - samples * src_chans, (1ULL << (pvp->bits - 1)) - 1ULL); + &pvp->rx_compressor_param, samples * src_chans, + src_chans, (1ULL << (pvp->bits - 1)) - 1ULL); TAILQ_FOREACH(pvc, &pvp->head, entry) { diff --git a/usr.sbin/zzz/Makefile b/usr.sbin/zzz/Makefile index 4d1a2dd9dc34..6ce3e03fa6cf 100644 --- a/usr.sbin/zzz/Makefile +++ b/usr.sbin/zzz/Makefile @@ -1,4 +1,6 @@ -SCRIPTS=zzz.sh +.include <src.opts.mk> + +PROG= zzz MAN= zzz.8 .include <bsd.prog.mk> diff --git a/usr.sbin/zzz/zzz.8 b/usr.sbin/zzz/zzz.8 index c2799c28e177..e7df72c1e830 100644 --- a/usr.sbin/zzz/zzz.8 +++ b/usr.sbin/zzz/zzz.8 @@ -1,63 +1,59 @@ -.\" Copyright (c) 2003 Nate Lawson -.\" All rights reserved. +.\"- +.\" SPDX-License-Identifier: BSD-2-Clause .\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted provided that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. +.\" Copyright (c) 2026 The FreeBSD Foundation .\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -.\" SUCH DAMAGE. +.\" This documentation was written by Aymeric Wibo <obiwac@freebsd.org> under +.\" sponsorship from the FreeBSD Foundation. .\" -.Dd July 13, 2003 +.Dd May 26, 2026 .Dt ZZZ 8 .Os .Sh NAME .Nm zzz -.Nd suspend an ACPI or APM system +.Nd suspend the system .Sh SYNOPSIS .Nm .Sh DESCRIPTION The .Nm -utility -checks for -.Tn ACPI -or -.Tn APM -support and then suspends the system appropriately. -For -.Tn APM , -.Pp -.Dl apm -z -.Pp -will be issued. -For -.Tn ACPI, -the configured suspend state will be looked up, checked to see -if it is supported and, +utility suspends the system by writing a suspend transition request to +.Pa /dev/power . .Pp -.Dl acpiconf -s <state> -.Pp -will be issued. +The specific sleep method used +.Pq e.g.\& suspend-to-idle or firmware suspend +is determined by the +.Va kern.power.suspend +sysctl. .Sh SEE ALSO .Xr acpi 4 , -.Xr apm 4 , -.Xr acpiconf 8 , +.Xr acpiconf 8 +.Sh HISTORY +The +.Nm +utility first appeared as a shell script in +.Fx 5.1 . +It suspended the system by invoking +.Xr acpiconf 8 +with the suspend state configured via +.Va hw.acpi.suspend_state , +or .Xr apm 8 +on systems without ACPI support. +.Pp +It was rewritten in C in +.Fx 16.0 +to use the generic power management interface provided by +.Pa /dev/power . .Sh AUTHORS -This manual page was written by +The original +.Nm +script was written by +.An Mark Santcroos Aq Mt marks@ripe.net . +This manual page was originally written by .An Nate Lawson Aq Mt njl@FreeBSD.org . +Both the current program and manual page were written by +.An Aymeric Wibo Aq Mt obiwac@FreeBSD.org , +under sponsorship from the +.Fx +Foundation. diff --git a/usr.sbin/zzz/zzz.c b/usr.sbin/zzz/zzz.c new file mode 100644 index 000000000000..a014537720cf --- /dev/null +++ b/usr.sbin/zzz/zzz.c @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 The FreeBSD Foundation + * + * This software was developed by Aymeric Wibo <obiwac@freebsd.org> + * under sponsorship from the FreeBSD Foundation. + */ + +#include <sys/power.h> + +#include <err.h> +#include <errno.h> +#include <fcntl.h> +#include <paths.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sysexits.h> + +static void +usage(void) +{ + (void)fprintf(stderr, "usage: zzz\n"); + exit(1); +} + +int +main(int argc, char *argv[]) +{ + int powerfd; + enum power_transition trans; + + (void)argv; + if (argc > 1) + usage(); + + powerfd = open(_PATH_DEVPOWER, O_RDWR); + if (powerfd < 0) + err(EX_OSFILE, "could not open power device"); + + trans = POWER_TRANSITION_SUSPEND; + if (ioctl(powerfd, PIOTRANSITION, &trans) != 0) + err(EX_IOERR, "could not request suspend transition"); + + return (EXIT_SUCCESS); +} diff --git a/usr.sbin/zzz/zzz.sh b/usr.sbin/zzz/zzz.sh deleted file mode 100644 index dde36e7984c6..000000000000 --- a/usr.sbin/zzz/zzz.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# -# Suspend the system using either ACPI or APM. -# For APM, "apm -z" will be issued. -# For ACPI, the configured suspend state will be looked up, checked to see -# if it is supported, and "acpiconf -s <state>" will be issued. -# -# Mark Santcroos <marks@ripe.net> -# - -PATH=/sbin:/usr/sbin:/usr/bin:/bin - -ACPI_SUSPEND_STATE=hw.acpi.suspend_state -ACPI_SUPPORTED_STATES=hw.acpi.supported_sleep_state -APM_SUSPEND_DELAY=machdep.apm_suspend_delay - -# Check for ACPI support -if sysctl $ACPI_SUSPEND_STATE >/dev/null 2>&1; then - # Get configured suspend state - SUSPEND_STATE=$(sysctl -n $ACPI_SUSPEND_STATE) - - # Get list of supported suspend states - SUPPORTED_STATES=$(sysctl -n $ACPI_SUPPORTED_STATES) - - # Check if the configured suspend state is supported by the system - if echo "$SUPPORTED_STATES" | grep "$SUSPEND_STATE" >/dev/null; then - # execute ACPI style suspend command - exec acpiconf -s "$SUSPEND_STATE" - else - echo "Requested suspend state $SUSPEND_STATE is not supported." - echo "Supported states: $SUPPORTED_STATES" - fi -# Check for APM support -elif sysctl $APM_SUSPEND_DELAY >/dev/null 2>&1; then - # Execute APM style suspend command - exec apm -z -else - echo "Error: no ACPI or APM suspend support found." -fi - -exit 1 |