FTLDNS (`pihole-FTL`) provides an interactive API and also generates statistics for Pi-hole[®](https://pi-hole.net/trademark-rules-and-brand-guidelines/)'s Web interface.
@@ -13,15 +26,14 @@ FTLDNS (`pihole-FTL`) provides an interactive API and also generates statistics
- **Interactive**: our API can be used to interface with your projects
- **Insightful**: stats normally reserved inside of `dnsmasq` are made available so you can see what's really happening on your network
-
-# Official documentation
+## Official documentation
The official *FTL*DNS documentation can be found [here](https://docs.pi-hole.net/ftldns/).
-# Installation
+## Installation
-FTLDNS (`pihole-FTL`) is installed by default when you choose to enable the Web interface when installing Pi-hole.
+FTLDNS (`pihole-FTL`) is automatically installed when installing Pi-hole.
-> IMPORTANT!
+### IMPORTANT
->FTLDNS will _disable_ any existing installations of `dnsmasq`. This is because FTLDNS _is_ `dnsmasq` + Pi-hole's code, so both cannot run simultaneously.
+>FTLDNS will *disable* any existing installations of `dnsmasq`. This is because FTLDNS *is* `dnsmasq` + Pi-hole's code, so both cannot run simultaneously.
diff --git a/build.sh b/build.sh
index 78e5b423..56bce08d 100755
--- a/build.sh
+++ b/build.sh
@@ -34,6 +34,14 @@ fi
# Remove possibly generated api/docs elements
rm -rf src/api/docs/hex
+# Remove compiled LUA scripts if older than the plain ones
+for scriptname in src/lua/scripts/*.lua; do
+ if [ -f "${scriptname}.hex" ] && [ "${scriptname}.hex" -ot "${scriptname}" ]; then
+ echo "INFO: ${scriptname} is outdated and will be recompiled"
+ rm "${scriptname}.hex"
+ fi
+done
+
# Configure build, pass CMake CACHE entries if present
# Wrap multiple options in "" as first argument to ./build.sh:
# ./build.sh "-DA=1 -DB=2" install
@@ -52,7 +60,7 @@ cmake --build . -- -j $(nproc)
# Otherwise, we simply copy the binary one level up
if [[ -n "${install}" ]]; then
echo "Installing pihole-FTL"
- SUDO=$(which sudo)
+ SUDO=$(command -v sudo)
${SUDO} cmake --install .
else
echo "Copying compiled pihole-FTL binary to repository root"
diff --git a/deploy.sh b/deploy.sh
new file mode 100755
index 00000000..fb7c3a4b
--- /dev/null
+++ b/deploy.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2022 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine
+# Deploy script for FTL
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+
+# Transfer Builds to Pi-hole server for pihole checkout
+# We use sftp for secure transfer and use the branch name as dir on the server.
+# The branch name could contain slashes, creating hierarchical dirs. However,
+# this is not supported by sftp's `mkdir` (option -p) is not available. Therefore,
+# we need to loop over each dir level and create them one by one.
+
+
+# Safeguard: do not deploy if TARGET_DIR is empty
+if [[ -z ${TARGET_DIR} ]]; then
+ echo "Error: Empty target dir."
+ exit 1
+fi
+
+IFS='/'
+read -r -a path <<<"${TARGET_DIR}"
+
+# Safeguard: do not deploy if more than one subdir (eg. /tweak/feature/subfeature) needs to be created
+if [[ "${#path[@]}" -gt 2 ]]; then
+ echo "Error: Your branch name contains more then one subdir. We won't deploy that."
+ exit 1
+fi
+
+unset IFS
+
+old_path="."
+
+for dir in "${path[@]}"; do
+ mapfile -t dir_content <<< "$(
+ sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path}
+ ls -1"
+ )"
+
+ # Only try to create the subdir if does not already exist
+ if [[ "${dir_content[*]}" =~ "${dir}" ]]; then
+ echo "Dir: ${old_path}/${dir} already exists"
+ else
+ echo "Creating dir: ${old_path}/${dir}"
+ sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path}
+ -mkdir ${dir}"
+ fi
+
+ old_path="${old_path}/${dir}"
+done
+
+sftp -r -b - "${USER}"@"${HOST}" <<< "cd ${old_path}
+put ${SOURCE_DIR}/* ./"
diff --git a/patch/lua.sh b/patch/lua.sh
new file mode 100644
index 00000000..478d4581
--- /dev/null
+++ b/patch/lua.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+
+patch -p1 < patch/lua/0001-add-pihole-library.patch
diff --git a/patch/lua/0001-add-pihole-library.patch b/patch/lua/0001-add-pihole-library.patch
new file mode 100644
index 00000000..30a0f592
--- /dev/null
+++ b/patch/lua/0001-add-pihole-library.patch
@@ -0,0 +1,94 @@
+diff --git a/src/lua/linit.c b/src/lua/linit.c
+index 69808f84..83b89555 100644
+--- a/src/lua/linit.c
++++ b/src/lua/linit.c
+@@ -50,6 +50,9 @@ static const luaL_Reg loadedlibs[] = {
+ {LUA_MATHLIBNAME, luaopen_math},
+ {LUA_UTF8LIBNAME, luaopen_utf8},
+ {LUA_DBLIBNAME, luaopen_debug},
++ /****** Pi-hole modification ******/
++ {LUA_PIHOLELIBNAME, luaopen_pihole},
++ /**********************************/
+ {NULL, NULL}
+ };
+
+diff --git a/src/lua/lua.c b/src/lua/lua.c
+index 454ce12f..a363925c 100644
+--- a/src/lua/lua.c
++++ b/src/lua/lua.c
+@@ -20,6 +20,10 @@
+ #include "lauxlib.h"
+ #include "lualib.h"
+
++/** Pi-hole modification **/
++#include "ftl_lua.h"
++/**************************/
++
+
+ #if !defined(LUA_PROGNAME)
+ #define LUA_PROGNAME "lua"
+@@ -190,7 +194,9 @@ static int dostring (lua_State *L, const char *s, const char *name) {
+ /*
+ ** Receives 'globname[=modname]' and runs 'globname = require(modname)'.
+ */
+-static int dolibrary (lua_State *L, char *globname) {
++/************** Pi-hole modification ***************/
++int dolibrary (lua_State *L, char *globname) {
++/***************************************************/
+ int status;
+ char *modname = strchr(globname, '=');
+ if (modname == NULL) /* no explicit name? */
+@@ -597,6 +599,12 @@ static int pmain (lua_State *L) {
+ if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */
+ return 0; /* error running LUA_INIT */
+ }
++
++ /************** Pi-hole modification ***************/
++ // Load and enable libraries bundled with Pi-hole
++ ftl_lua_init(L);
++ /***************************************************/
++
+ if (!runargs(L, argv, script)) /* execute arguments -e and -l */
+ return 0; /* something failed */
+ if (script < argc && /* execute main script (if there is one) */
+@@ -616,7 +622,9 @@ static int pmain (lua_State *L) {
+ }
+
+
+-int main (int argc, char **argv) {
++/******* Pi-hole modification ********/
++int lua_main (int argc, char **argv) {
++/*************************************/
+ int status, result;
+ lua_State *L = luaL_newstate(); /* create state */
+ if (L == NULL) {
+diff --git a/src/lua/luac.c b/src/lua/luac.c
+index 56ddc414..d7d219e4 100644
+--- a/src/lua/luac.c
++++ b/src/lua/luac.c
+@@ -194,7 +194,9 @@ static int pmain(lua_State* L)
+ return 0;
+ }
+
+-int main(int argc, char* argv[])
++/******* Pi-hole modification ********/
++int luac_main(int argc, char* argv[])
++/*************************************/
+ {
+ lua_State* L;
+ int i=doargs(argc,argv);
+diff --git a/src/lua/lualib.h b/src/lua/lualib.h
+index eb08b530..e6dfbf6c 100644
+--- a/src/lua/lualib.h
++++ b/src/lua/lualib.h
+@@ -44,6 +44,10 @@ LUAMOD_API int (luaopen_debug) (lua_State *L);
+ #define LUA_LOADLIBNAME "package"
+ LUAMOD_API int (luaopen_package) (lua_State *L);
+
++/************ Pi-hole modification *************/
++#define LUA_PIHOLELIBNAME "pihole"
++LUAMOD_API int (luaopen_pihole) (lua_State *L);
++/***********************************************/
+
+ /* open all previous libraries */
+ LUALIB_API void (luaL_openlibs) (lua_State *L);
diff --git a/patch/sqlite3.sh b/patch/sqlite3.sh
new file mode 100644
index 00000000..9b447217
--- /dev/null
+++ b/patch/sqlite3.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+patch -p1 < patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
+patch -p1 < patch/sqlite3/0002-make-sqlite3ErrName-public.patch
diff --git a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
new file mode 100644
index 00000000..02ed43bf
--- /dev/null
+++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
@@ -0,0 +1,30 @@
+diff --git a/src/database/shell.c b/src/database/shell.c
+index 6280ebf6..a5e82f70 100644
+--- a/src/database/shell.c
++++ b/src/database/shell.c
+@@ -126,6 +126,8 @@ typedef unsigned char u8;
+ #endif
+ #include
+ #include
++// print_FTL_version()
++#include "../log.h"
+
+ #if !defined(_WIN32) && !defined(WIN32)
+ # include
+@@ -25855,7 +25857,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){
+ #endif
+
+ #if SQLITE_SHELL_IS_UTF8
+-int SQLITE_CDECL main(int argc, char **argv){
++int SQLITE_CDECL sqlite3_shell_main(int argc, char **argv){
+ #else
+ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
+ char **argv;
+@@ -26377,6 +26379,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
+ char *zHome;
+ char *zHistory;
+ int nHistory;
++ print_FTL_version();
+ printf(
+ "SQLite version %s %.19s\n" /*extra-version-info*/
+ "Enter \".help\" for usage hints.\n",
diff --git a/patch/sqlite3/0002-make-sqlite3ErrName-public.patch b/patch/sqlite3/0002-make-sqlite3ErrName-public.patch
new file mode 100644
index 00000000..92cf9a72
--- /dev/null
+++ b/patch/sqlite3/0002-make-sqlite3ErrName-public.patch
@@ -0,0 +1,22 @@
+diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c
+index 2763b1b4..55f88efb 100644
+--- a/src/database/sqlite3.c
++++ b/src/database/sqlite3.c
+@@ -173885,8 +173885,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){
+ ** Return a static string containing the name corresponding to the error code
+ ** specified in the argument.
+ */
+-#if defined(SQLITE_NEED_ERR_NAME)
+-SQLITE_PRIVATE const char *sqlite3ErrName(int rc){
++SQLITE_API const char *sqlite3ErrName(int rc){
+ const char *zName = 0;
+ int i, origRc = rc;
+ for(i=0; i<2 && zName==0; i++, rc &= 0xff){
+@@ -173991,7 +173990,6 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){
+ }
+ return zName;
+ }
+-#endif
+
+ /*
+ ** Return a static string that describes the kind of error specified in the
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b6668655..43f8ce99 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -32,7 +32,8 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
# SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection.
# SQLITE_OMIT_DESERIALIZE: This option causes the the sqlite3_serialize() and sqlite3_deserialize() interfaces to be omitted from the build (was the default before 3.36.0)
# HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion
-set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=-8000 -DSQLITE_USE_URI=1 -DSQLITE_OMIT_DESERIALIZE -DHAVE_READLINE")
+# SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB)
+set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=-8000 -DSQLITE_USE_URI=1 -DSQLITE_OMIT_DESERIALIZE -DHAVE_READLINE_CACHE_SIZE=-16384")
# Code hardening and debugging improvements
# -fstack-protector-strong: The program will be resistant to having its stack overflowed
@@ -228,6 +229,24 @@ else()
message(STATUS "Building FTL with readline support: NO")
endif()
+# Do we want to compile an all-in FTL version?
+if(DEFINED ENV{CI_ARCH})
+ if($ENV{CI_ARCH} STREQUAL "x86_64_full")
+ add_definitions(-DDNSMASQ_ALL_OPTS)
+ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR})
+ find_package(DBus REQUIRED)
+ # Use results of find_package() call.
+ include_directories(${DBUS_INCLUDE_DIRS})
+ target_link_libraries(pihole-FTL ${DBUS_LIBRARIES})
+ find_library(LIBMNL mnl)
+ find_library(LIBNFTNL nftnl)
+ find_library(LIBNFTABLES nftables)
+ find_library(LIBNFNETLINK nfnetlink)
+ find_library(LIBNETFILTER_CONNTRACK netfilter_conntrack)
+ target_link_libraries(pihole-FTL ${LIBMNL} ${LIBNFTABLES} ${LIBNFTNL} ${LIBNFNETLINK} ${LIBNETFILTER_CONNTRACK})
+ endif()
+endif()
+
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE)
endif()
@@ -246,6 +265,7 @@ add_subdirectory(ph7)
add_subdirectory(database)
add_subdirectory(dnsmasq)
add_subdirectory(lua)
+add_subdirectory(lua/scripts)
add_subdirectory(tre-regex)
add_subdirectory(syscalls)
add_subdirectory(config)
diff --git a/src/FTL.h b/src/FTL.h
index feabc8a3..5e332e97 100644
--- a/src/FTL.h
+++ b/src/FTL.h
@@ -129,8 +129,8 @@
// DELAY_STARTUP should only delay the startup of the resolver during a starting up system
// This setting control how long after boot we consider a system to be in starting-up mode
-// Default: 60 [seconds]
-#define DELAY_UPTIME 60
+// Default: 180 [seconds]
+#define DELAY_UPTIME 180
// DB_QUERY_MAX_ITER defines how many queries we check periodically for updates to be added
// to the in-memory database. This value may need to be increased on *very* busy systems.
diff --git a/src/FindDBus.cmake b/src/FindDBus.cmake
new file mode 100644
index 00000000..4a1a1805
--- /dev/null
+++ b/src/FindDBus.cmake
@@ -0,0 +1,59 @@
+# - Try to find DBus
+# Once done, this will define
+#
+# DBUS_FOUND - system has DBus
+# DBUS_INCLUDE_DIRS - the DBus include directories
+# DBUS_LIBRARIES - link these to use DBus
+#
+# Copyright (C) 2012 Raphael Kubo da Costa
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND ITS CONTRIBUTORS ``AS
+# IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR ITS
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FIND_PACKAGE(PkgConfig)
+PKG_CHECK_MODULES(PC_DBUS QUIET dbus-1)
+
+FIND_LIBRARY(DBUS_LIBRARIES
+ NAMES dbus-1
+ HINTS ${PC_DBUS_LIBDIR}
+ ${PC_DBUS_LIBRARY_DIRS}
+)
+
+FIND_PATH(DBUS_INCLUDE_DIR
+ NAMES dbus/dbus.h
+ HINTS ${PC_DBUS_INCLUDEDIR}
+ ${PC_DBUS_INCLUDE_DIRS}
+)
+
+GET_FILENAME_COMPONENT(_DBUS_LIBRARY_DIR ${DBUS_LIBRARIES} PATH)
+FIND_PATH(DBUS_ARCH_INCLUDE_DIR
+ NAMES dbus/dbus-arch-deps.h
+ HINTS ${PC_DBUS_INCLUDEDIR}
+ ${PC_DBUS_INCLUDE_DIRS}
+ ${_DBUS_LIBRARY_DIR}
+ ${DBUS_INCLUDE_DIR}
+ PATH_SUFFIXES include
+)
+
+SET(DBUS_INCLUDE_DIRS ${DBUS_INCLUDE_DIR} ${DBUS_ARCH_INCLUDE_DIR})
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(DBUS REQUIRED_VARS DBUS_INCLUDE_DIRS DBUS_LIBRARIES)
diff --git a/src/args.c b/src/args.c
index c357ab2f..ce14440b 100644
--- a/src/args.c
+++ b/src/args.c
@@ -13,6 +13,12 @@
#include "dnsmasq/dnsmasq.h"
#undef __USE_XOPEN
+#include
+#if !defined(NETTLE_VERSION_MAJOR)
+# define NETTLE_VERSION_MAJOR 2
+# define NETTLE_VERSION_MINOR 0
+#endif
+
#include "FTL.h"
#include "args.h"
#include "version.h"
@@ -28,8 +34,6 @@
#include "lua/ftl_lua.h"
// run_dhcp_discover()
#include "dhcp-discover.h"
-// defined in dnsmasq.c
-extern void print_dnsmasq_version(void);
// mg_version()
#include "civetweb/civetweb.h"
// cJSON_Version()
@@ -37,6 +41,9 @@ extern void print_dnsmasq_version(void);
// ph7_lib_version()
#include "ph7/ph7.h"
+// defined in dnsmasq.c
+extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal);
+
// defined in database/shell.c
extern int sqlite3_shell_main(int argc, char **argv);
@@ -45,6 +52,86 @@ bool daemonmode = true, cli_mode = false;
int argc_dnsmasq = 0;
const char** argv_dnsmasq = NULL;
+// Extended SGR sequence:
+//
+// "\x1b[%dm"
+//
+// where %d is one of the following values for commonly supported colors:
+//
+// 0: reset colors/style
+// 1: bold
+// 4: underline
+// 30 - 37: black, red, green, yellow, blue, magenta, cyan, and white text
+// 40 - 47: black, red, green, yellow, blue, magenta, cyan, and white background
+//
+// https://en.wikipedia.org/wiki/ANSI_escape_code#SGR
+//
+#define COL_NC "\x1b[0m" // normal font
+#define COL_BOLD "\x1b[1m" // bold font
+#define COL_ITALIC "\x1b[3m" // italic font
+#define COL_ULINE "\x1b[4m" // underline font
+#define COL_GREEN "\x1b[32m" // normal foreground color
+#define COL_YELLOW "\x1b[33m" // normal foreground color
+#define COL_GRAY "\x1b[90m" // bright foreground color
+#define COL_RED "\x1b[91m" // bright foreground color
+#define COL_BLUE "\x1b[94m" // bright foreground color
+#define COL_PURPLE "\x1b[95m" // bright foreground color
+#define COL_CYAN "\x1b[96m" // bright foreground color
+
+static inline bool __attribute__ ((pure)) is_term(void)
+{
+ // test whether STDOUT refers to a terminal
+ return isatty(fileno(stdout)) == 1;
+}
+
+// Returns green [✓]
+const char __attribute__ ((pure)) *cli_tick(void)
+{
+ return is_term() ? "["COL_GREEN"✓"COL_NC"]" : "[✓]";
+}
+
+// Returns red [✗]
+const char __attribute__ ((pure)) *cli_cross(void)
+{
+ return is_term() ? "["COL_RED"✗"COL_NC"]" : "[✗]";
+}
+
+// Returns [i]
+const char __attribute__ ((pure)) *cli_info(void)
+{
+ return is_term() ? COL_BOLD"[i]"COL_NC : "[i]";
+}
+
+// Returns [?]
+const char __attribute__ ((const)) *cli_qst(void)
+{
+ return "[?]";
+}
+
+// Returns green "done!""
+const char __attribute__ ((pure)) *cli_done(void)
+{
+ return is_term() ? COL_GREEN"done!"COL_NC : "done!";
+}
+
+// Sets font to bold
+const char __attribute__ ((pure)) *cli_bold(void)
+{
+ return is_term() ? COL_BOLD : "";
+}
+
+// Resets font to normal
+const char __attribute__ ((pure)) *cli_normal(void)
+{
+ return is_term() ? COL_NC : "";
+}
+
+// Set color if STDOUT is a terminal
+static const char __attribute__ ((pure)) *cli_color(const char *color)
+{
+ return is_term() ? color : "";
+}
+
static inline bool strEndsWith(const char *input, const char *end){
return strcmp(input + strlen(input) - strlen(end), end) == 0;
}
@@ -137,8 +224,7 @@ void parse_args(int argc, char* argv[])
const char *arg[2];
arg[0] = "";
arg[1] = "--test";
- main_dnsmasq(2, arg);
- ok = true;
+ exit(main_dnsmasq(2, arg));
}
// If we find "--" we collect everything behind that for dnsmasq
@@ -147,23 +233,23 @@ void parse_args(int argc, char* argv[])
// Remember that the rest is for dnsmasq ...
consume_for_dnsmasq = true;
- // Special command interpretation for "pihole-FTL -- --help dhcp"
- if(argc > 1 && strcmp(argv[argc-2], "--help") == 0 && strcmp(argv[argc-1], "dhcp") == 0)
- {
- display_opts();
- exit(EXIT_SUCCESS);
- }
- // and "pihole-FTL -- --help dhcp6"
- if(argc > 1 && strcmp(argv[argc-2], "--help") == 0 && strcmp(argv[argc-1], "dhcp6") == 0)
- {
- display_opts6();
- exit(EXIT_SUCCESS);
- }
-
// ... and skip the current argument ("--")
continue;
}
+ // List available DHCPv4 config options
+ if(strcmp(argv[i], "--list-dhcp") == 0 || strcmp(argv[i], "--list-dhcp4") == 0)
+ {
+ display_opts();
+ exit(EXIT_SUCCESS);
+ }
+ // List available DHCPv6 config options
+ if(strcmp(argv[i], "--list-dhcp6") == 0)
+ {
+ display_opts6();
+ exit(EXIT_SUCCESS);
+ }
+
// If consume_for_dnsmasq is true, we collect all remaining options for
// dnsmasq
if(consume_for_dnsmasq)
@@ -238,21 +324,30 @@ void parse_args(int argc, char* argv[])
// Extended version output
if(strcmp(argv[i], "-vv") == 0)
{
+ const char *bold = cli_bold();
+ const char *normal = cli_normal();
+ const char *green = cli_color(COL_GREEN);
+ const char *yellow = cli_color(COL_YELLOW);
+
// Print FTL version
- printf("****************************** FTL **********************************\n");
- printf("Version: %s\n", get_FTL_version());
- printf("Branch: %s\n", GIT_BRANCH);
- printf("Commit: %s (%s)\n", GIT_HASH, GIT_DATE);
- printf("Architecture: %s\n", FTL_ARCH);
- printf("Compiler: %s\n\n", FTL_CC);
+ printf("****************************** %s%sFTL%s **********************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s%s%s\n",
+ green, bold, get_FTL_version(), normal);
+ printf("Branch: " GIT_BRANCH "\n");
+ printf("Commit: " GIT_HASH " (" GIT_DATE ")\n");
+ printf("Architecture: " FTL_ARCH "\n");
+ printf("Compiler: " FTL_CC "\n\n");
// Print dnsmasq version and compile time options
- print_dnsmasq_version();
+ print_dnsmasq_version(yellow, green, bold, normal);
// Print SQLite3 version and compile time options
- printf("****************************** SQLite3 ******************************\n");
- printf("Version: %s\n", sqlite3_libversion());
- printf("Compile options: ");
+ printf("****************************** %s%sSQLite3%s ******************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s%s%s\n",
+ green, bold, sqlite3_libversion(), normal);
+ printf("Features: ");
unsigned int o = 0;
const char *opt = NULL;
while((opt = sqlite3_compileoption_get(o++)) != NULL)
@@ -262,17 +357,30 @@ void parse_args(int argc, char* argv[])
printf("%s", opt);
}
printf("\n\n");
- printf("****************************** LUA **********************************\n");
- printf("Version: %s\n", LUA_COPYRIGHT);
+ printf("******************************** %s%sLUA%s ********************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s" LUA_VERSION_MAJOR "." LUA_VERSION_MINOR"%s\n",
+ green, bold, normal);
+ printf("Libraries: ");
+ print_embedded_scripts();
+ printf("\n\n");
+ printf("***************************** %s%sLIBNETTLE%s *****************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s" xstr(NETTLE_VERSION_MAJOR) "." xstr(NETTLE_VERSION_MINOR) "%s\n",
+ green, bold, normal);
+ printf("GMP: %s\n", NETTLE_USE_MINI_GMP ? "Mini" : "Full");
printf("\n");
- printf("****************************** CivetWeb *****************************\n");
- printf("Version: %s\n", mg_version());
+ printf("****************************** %s%sCivetWeb%s *****************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s%s%s\n", green, bold, mg_version(), normal);
printf("\n");
- printf("****************************** cJSON ********************************\n");
- printf("Version: %s\n", cJSON_Version());
+ printf("****************************** %s%scJSON%s ********************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s%s%s\n", green, bold, cJSON_Version(), normal);
printf("\n");
- printf("****************************** PH7 **********************************\n");
- printf("Version: %s\n", ph7_lib_version());
+ printf("****************************** %s%sPH7%s **********************************\n",
+ yellow, bold, normal);
+ printf("Version: %s%s%s%s\n", green, bold, ph7_lib_version(), normal);
exit(EXIT_SUCCESS);
}
@@ -290,6 +398,12 @@ void parse_args(int argc, char* argv[])
exit(EXIT_SUCCESS);
}
+ if(strcmp(argv[i], "--hash") == 0)
+ {
+ printf("%s\n",GIT_HASH);
+ exit(EXIT_SUCCESS);
+ }
+
// Don't go into background
if(strcmp(argv[i], "-f") == 0 ||
strcmp(argv[i], "no-daemon") == 0)
@@ -332,33 +446,84 @@ void parse_args(int argc, char* argv[])
// List of implemented arguments
if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0)
{
- printf("pihole-FTL - The Pi-hole FTL engine\n\n");
- printf("Usage: sudo service pihole-FTL \n");
- printf("where '' is one of start / stop / restart\n\n");
- printf("Available arguments:\n");
- printf("\t debug More verbose logging,\n");
- printf("\t don't go into daemon mode\n");
- printf("\t test Don't start pihole-FTL but\n");
- printf("\t instead quit immediately\n");
- printf("\t-v, version Return FTL version\n");
- printf("\t-vv Return more version information\n");
- printf("\t-t, tag Return git tag\n");
- printf("\t-b, branch Return git branch\n");
- printf("\t-f, no-daemon Don't go into daemon mode\n");
- printf("\t-h, help Display this help and exit\n");
- printf("\tdnsmasq-test Test syntax of dnsmasq's\n");
- printf("\t config files and exit\n");
- printf("\tregex-test str Test str against all regular\n");
+ const char *bold = cli_bold();
+ const char *normal = cli_normal();
+ const char *blue = cli_color(COL_BLUE);
+ const char *cyan = cli_color(COL_CYAN);
+ const char *gray = cli_color(COL_GRAY);
+ const char *green = cli_color(COL_GREEN);
+ const char *yellow = cli_color(COL_YELLOW);
+ const char *purple = cli_color(COL_PURPLE);
+
+ printf("%sThe Pi-hole FTL engine - %s%s\n\n", bold, get_FTL_version(), normal);
+ printf("Typically, pihole-FTL runs as a system service and is controlled\n");
+ printf("by %ssudo service pihole-FTL %s%s where %s%s is one out\n", green, purple, normal, purple, normal);
+ printf("of %sstart%s, %sstop%s, or %srestart%s.\n\n", green, normal, green, normal, green, normal);
+ printf("pihole-FTL exposes some features going beyond the standard\n");
+ printf("%sservice pihole-FTL%s command. These are:\n\n", green, normal);
+
+ printf("%sVersion information:%s\n", yellow, normal);
+ printf("\t%s-v%s, %sversion%s Return FTL version\n", green, normal, green, normal);
+ printf("\t%s-vv%s Return verbose version information\n", green, normal);
+ printf("\t%s-t%s, %stag%s Return git tag\n", green, normal, green, normal);
+ printf("\t%s-b%s, %sbranch%s Return git branch\n", green, normal, green, normal);
+ printf("\t%s--hash%s Return git commit hash\n\n", green, normal);
+
+ printf("%sRegular expression testing:%s\n", yellow, normal);
+ printf("\t%sregex-test %sstr%s Test %sstr%s against all regular\n", green, blue, normal, blue, normal);
printf("\t expressions in the database\n");
- printf("\tregex-test str rgx Test str against regular expression\n");
- printf("\t given by rgx\n");
- printf("\t--lua, lua FTL's lua interpreter\n");
- printf("\t--luac, luac FTL's lua compiler\n");
- printf("\tdhcp-discover Discover DHCP servers in the local\n");
+ printf("\t%sregex-test %sstr %srgx%s Test %sstr%s against regular expression\n", green, blue, cyan, normal, blue, normal);
+ printf("\t given by regular expression %srgx%s\n\n", cyan, normal);
+
+ printf(" Example: %spihole-FTL regex-test %ssomebad.domain %sbad%s\n", green, blue, cyan, normal);
+ printf(" to test %ssomebad.domain%s against %sbad%s\n\n", blue, normal, cyan, normal);
+ printf(" An optional %s-q%s prevents any output (exit code testing):\n", gray, normal);
+ printf(" %spihole-FTL %s-q%s regex-test %ssomebad.domain %sbad%s\n\n", green, gray, green, blue, cyan, normal);
+
+ printf("%sEmbedded Lua engine:%s\n", yellow, normal);
+ printf("\t%s--lua%s, %slua%s FTL's lua interpreter\n", green, normal, green, normal);
+ printf("\t%s--luac%s, %sluac%s FTL's lua compiler\n\n", green, normal, green, normal);
+
+ printf(" Usage: %spihole-FTL lua %s[OPTIONS] [SCRIPT [ARGS]]%s\n\n", green, cyan, normal);
+ printf(" Options:\n\n");
+ printf(" - %s[OPTIONS]%s is an optional set of options. All available\n", cyan, normal);
+ printf(" options can be seen by running %spihole-FTL lua --help%s\n", green, normal);
+ printf(" - %s[SCRIPT]%s is the optional name of a Lua script.\n", cyan, normal);
+ printf(" If this script does not exist, an interactive shell is\n");
+ printf(" started instead.\n");
+ printf(" - %s[SCRIPT [ARGS]]%s can be used to pass optional args to\n", cyan, normal);
+ printf(" the script.\n\n");
+
+ printf("%sEmbedded SQLite3 shell:%s\n", yellow, normal);
+ printf("\t%ssql %s[-h]%s, %ssqlite3 %s[-h]%s FTL's SQLite3 shell\n", green, gray, normal, green, gray, normal);
+ printf("\t%s-h%s starts a special %shuman-readable mode%s\n\n", gray, normal, bold, normal);
+
+ printf(" Usage: %spihole-FTL sqlite3 %s[-h] %s[OPTIONS] [FILENAME] [SQL]%s\n\n", green, gray, cyan, normal);
+ printf(" Options:\n\n");
+ printf(" - %s[OPTIONS]%s is an optional set of options. All available\n", cyan, normal);
+ printf(" options can be found in %spihole-FTL sqlite3 --help%s\n", green, normal);
+ printf(" - %s[FILENAME]%s is the optional name of an SQLite database.\n", cyan, normal);
+ printf(" A new database is created if the file does not previously\n");
+ printf(" exist. If this argument is omitted, SQLite3 will use a\n");
+ printf(" transient in-memory database instead.\n");
+ printf(" - %s[SQL]%s is an optional SQL statement to be executed. If\n", cyan, normal);
+ printf(" omitted, an interactive shell is started instead.\n\n");
+
+ printf("%sEmbedded dnsmasq options:%s\n", yellow, normal);
+ printf("\t%sdnsmasq-test%s Test syntax of dnsmasq's config\n", green, normal);
+ printf("\t%s--list-dhcp4%s List known DHCPv4 config options\n", green, normal);
+ printf("\t%s--list-dhcp6%s List known DHCPv6 config options\n\n", green, normal);
+
+ printf("%sDebugging and special use:%s\n", yellow, normal);
+ printf("\t%sd%s, %sdebug%s Enter debugging mode\n", green, normal, green, normal);
+ printf("\t%stest%s Don't start pihole-FTL but\n", green, normal);
+ printf("\t instead quit immediately\n");
+ printf("\t%s-f%s, %sno-daemon%s Don't go into daemon mode\n\n", green, normal, green, normal);
+
+ printf("%sOther:%s\n", yellow, normal);
+ printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal);
printf("\t network\n");
- printf("\tsql, sqlite3 FTL's SQLite3 shell\n");
- printf("\tsql -h, sqlite3 -h FTL's SQLite3 shell (human-readable mode)\n");
- printf("\n\nOnline help: https://github.com/pi-hole/FTL\n");
+ printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal);
exit(EXIT_SUCCESS);
}
@@ -391,77 +556,3 @@ void parse_args(int argc, char* argv[])
}
}
}
-
-// Extended SGR sequence:
-//
-// "\x1b[%dm"
-//
-// where %d is one of the following values for commonly supported colors:
-//
-// 0: reset colors/style
-// 1: bold
-// 4: underline
-// 30 - 37: black, red, green, yellow, blue, magenta, cyan, and white text
-// 40 - 47: black, red, green, yellow, blue, magenta, cyan, and white background
-//
-// https://en.wikipedia.org/wiki/ANSI_escape_code#SGR
-//
-#define COL_NC "\x1b[0m" // normal font
-#define COL_BOLD "\x1b[1m" // bold font
-#define COL_ITALIC "\x1b[3m" // italic font
-#define COL_ULINE "\x1b[4m" // underline font
-#define COL_GREEN "\x1b[32m" // normal foreground color
-#define COL_YELLOW "\x1b[33m" // normal foreground color
-#define COL_GRAY "\x1b[90m" // bright foreground color
-#define COL_RED "\x1b[91m" // bright foreground color
-#define COL_BLUE "\x1b[94m" // bright foreground color
-#define COL_PURPLE "\x1b[95m" // bright foreground color
-#define COL_CYAN "\x1b[96m" // bright foreground color
-
-static inline bool __attribute__ ((const)) is_term(void)
-{
- // test whether STDOUT refers to a terminal
- return isatty(fileno(stdout)) == 1;
-}
-
-// Returns green [✓]
-const char __attribute__ ((const)) *cli_tick(void)
-{
- return is_term() ? "["COL_GREEN"✓"COL_NC"]" : "[✓]";
-}
-
-// Returns red [✗]
-const char __attribute__ ((const)) *cli_cross(void)
-{
- return is_term() ? "["COL_RED"✗"COL_NC"]" : "[✗]";
-}
-
-// Returns [i]
-const char __attribute__ ((const)) *cli_info(void)
-{
- return is_term() ? COL_BOLD"[i]"COL_NC : "[i]";
-}
-
-// Returns [?]
-const char __attribute__ ((const)) *cli_qst(void)
-{
- return "[?]";
-}
-
-// Returns green "done!""
-const char __attribute__ ((const)) *cli_done(void)
-{
- return is_term() ? COL_GREEN"done!"COL_NC : "done!";
-}
-
-// Sets font to bold
-const char __attribute__ ((const)) *cli_bold(void)
-{
- return is_term() ? COL_BOLD : "";
-}
-
-// Resets font to normal
-const char __attribute__ ((const)) *cli_normal(void)
-{
- return is_term() ? COL_NC : "";
-}
diff --git a/src/args.h b/src/args.h
index fe031f9e..57371e4e 100644
--- a/src/args.h
+++ b/src/args.h
@@ -16,13 +16,13 @@ extern bool daemonmode, cli_mode, dnsmasq_debug;
extern int argc_dnsmasq;
extern const char ** argv_dnsmasq;
-const char *cli_tick(void) __attribute__ ((const));
-const char *cli_cross(void) __attribute__ ((const));
-const char *cli_info(void) __attribute__ ((const));
+const char *cli_tick(void) __attribute__ ((pure));
+const char *cli_cross(void) __attribute__ ((pure));
+const char *cli_info(void) __attribute__ ((pure));
const char *cli_qst(void) __attribute__ ((const));
-const char *cli_done(void) __attribute__ ((const));
-const char *cli_bold(void) __attribute__ ((const));
-const char *cli_normal(void) __attribute__ ((const));
+const char *cli_done(void) __attribute__ ((pure));
+const char *cli_bold(void) __attribute__ ((pure));
+const char *cli_normal(void) __attribute__ ((pure));
// defined in dnsmasq_interface.c
int check_struct_sizes(void);
diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c
index 8723b274..1cd480b7 100644
--- a/src/config/toml_helper.c
+++ b/src/config/toml_helper.c
@@ -12,7 +12,7 @@
#include "toml_helper.h"
#include "../config/config.h"
-FILE *openFTLtoml(const char *mode)
+FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode)
{
FILE *fp;
// If reading: first check if there is a local file
diff --git a/src/config/toml_helper.h b/src/config/toml_helper.h
index db53d485..a99b457d 100644
--- a/src/config/toml_helper.h
+++ b/src/config/toml_helper.h
@@ -12,7 +12,7 @@
#include "../FTL.h"
-FILE *openFTLtoml(const char *mode);
+FILE *openFTLtoml(const char *mode) __attribute((malloc)) __attribute((nonnull(1)));
void catTOMLsection(FILE *fp, const unsigned int indent, const char *key);
void catTOMLextrainfo(FILE *fp, const unsigned int indent, const char *infostr);
void catTOMLstring(FILE *fp, const unsigned int indent, const char *key, const char *description, const char *values, const char *val, const char *dptr);
diff --git a/src/daemon.c b/src/daemon.c
index 66c2b87a..92118ea6 100644
--- a/src/daemon.c
+++ b/src/daemon.c
@@ -28,7 +28,6 @@
pthread_t threads[THREADS_MAX] = { 0 };
pthread_t api_threads[MAX_API_THREADS] = { 0 };
-pid_t api_tids[MAX_API_THREADS] = { 0 };
bool resolver_ready = false;
void go_daemon(void)
@@ -121,17 +120,6 @@ static void removepid(void)
return;
}
fclose(f);
-
- // We also try to remove the file. We still empty the file above
- // to ensure it is at least empty when it cannot be removed.
- // because removing files on Linux is actually unlinking them.
- // If any processes still have the file open, it will remain
- // in existence until the last file descriptor referring to
- // it is closed.
- if(remove(config.files.pid) != 0)
- {
- log_warn("Unable to remove PID file: %s", strerror(errno));
- }
}
char *getUserName(void)
@@ -201,7 +189,12 @@ void delay_startup(void)
// Sleep if requested by DELAY_STARTUP
log_info("Sleeping for %d seconds as requested by configuration ...", config.delay_startup);
- sleep(config.delay_startup);
+ if(sleep(config.delay_startup) != 0)
+ {
+ log_crit("Sleeping was interrupted by an external signal");
+ cleanup(EXIT_FAILURE);
+ exit(EXIT_FAILURE);
+ }
log_info("Done sleeping, continuing startup of resolver...");
}
@@ -266,15 +259,12 @@ void cleanup(const int ret)
// Do proper cleanup only if FTL started successfully
if(resolver_ready)
{
+ // Terminate threads
terminate_threads();
// Cancel and join possibly still running API worker threads
for(unsigned int tid = 0; tid < MAX_API_THREADS; tid++)
{
- // Skip if this is an unused slot
- if(api_threads[tid] == 0)
- continue;
-
// Otherwise, cancel and join the thread
log_notice("Joining API worker thread %d", tid);
pthread_cancel(api_threads[tid]);
diff --git a/src/daemon.h b/src/daemon.h
index 12b49942..cc91dd4d 100644
--- a/src/daemon.h
+++ b/src/daemon.h
@@ -12,9 +12,8 @@
#include "enums.h"
extern pthread_t threads[THREADS_MAX];
-#define MAX_API_THREADS 40
+#define MAX_API_THREADS 5
extern pthread_t api_threads[MAX_API_THREADS];
-extern pid_t api_tids[MAX_API_THREADS];
void go_daemon(void);
void savepid(void);
diff --git a/src/database/common.c b/src/database/common.c
index 9eaf752b..e01f52b9 100644
--- a/src/database/common.c
+++ b/src/database/common.c
@@ -144,8 +144,8 @@ int dbquery(sqlite3* db, const char *format, ...)
int rc = sqlite3_exec(db, query, NULL, NULL, NULL);
if( rc != SQLITE_OK ){
- log_err("SQL query \"%s\" failed: %s",
- query, sqlite3_errstr(rc));
+ log_err("ERROR: SQL query \"%s\" failed: %s (%s)",
+ query, sqlite3_errstr(rc), sqlite3ErrName(sqlite3_extended_errcode(db)));
sqlite3_free(query);
checkFTLDBrc(rc);
return rc;
diff --git a/src/database/common.h b/src/database/common.h
index de3e740d..950fab85 100644
--- a/src/database/common.h
+++ b/src/database/common.h
@@ -54,12 +54,15 @@ const char *get_sqlite3_version(void);
extern bool DBdeleteoldqueries;
// Return if FTL's database is known to be broken
-// We abort execution of all database-related activitites in this case
+// We abort execution of all database-related activities in this case
bool FTLDBerror(void) __attribute__ ((pure));
// Check SQLite3 non-success return codes for possible database corruption
bool checkFTLDBrc(const int rc);
+// Get human-readable *extended* error codes (defined in sqlite3.c)
+extern const char *sqlite3ErrName(int rc);
+
// Database macros
#define SQL_bool(db, ...) {\
int ret;\
diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c
index 09d7702c..240cbb7b 100644
--- a/src/database/gravity-db.c
+++ b/src/database/gravity-db.c
@@ -15,13 +15,12 @@
#include "../config/config.h"
// logging routines
#include "../log.h"
-// match_regex()
-#include "../regex_r.h"
// getstr()
#include "../shmem.h"
// SQLite3 prepared statement vectors
#include "../vector.h"
// log_subnet_warning()
+// logg_inaccessible_adlist
#include "message-table.h"
// getMACfromIP()
#include "network-table.h"
@@ -149,17 +148,16 @@ bool gravityDB_open(void)
// BUT NOT google.de itself
// Example 3: *google.de
// matches 'google.de' and all of its subdomains but
- // also other domains starting in google.de, like
+ // also other domains ending in google.de, like
// abcgoogle.de
rc = sqlite3_prepare_v3(gravity_db,
- "SELECT EXISTS("
- "SELECT domain, "
- "CASE WHEN substr(domain, 1, 1) = '*' " // Does the database string start in '*' ?
- "THEN '*' || substr(:input, - length(domain) + 1) " // If so: Crop the input domain and prepend '*'
- "ELSE :input " // If not: Use input domain directly for comparison
- "END matcher "
- "FROM domain_audit WHERE matcher = domain" // Match where (modified) domain equals the database domain
- ");", -1, SQLITE_PREPARE_PERSISTENT, &auditlist_stmt, NULL);
+ "SELECT domain, "
+ "CASE WHEN substr(domain, 1, 1) = '*' " // Does the database string start in '*' ?
+ "THEN '*' || substr(:input, - length(domain) + 1) " // If so: Crop the input domain and prepend '*'
+ "ELSE :input " // If not: Use input domain directly for comparison
+ "END matcher "
+ "FROM domain_audit WHERE matcher = domain" // Match where (modified) domain equals the database domain
+ ";", -1, SQLITE_PREPARE_PERSISTENT, &auditlist_stmt, NULL);
if( rc != SQLITE_OK )
{
@@ -201,11 +199,11 @@ bool gravityDB_reopen(void)
return gravityDB_open();
}
-static char* get_client_querystr(const char* table, const char* groups)
+static char* get_client_querystr(const char *table, const char *column, const char *groups)
{
// Build query string with group filtering
char *querystr = NULL;
- if(asprintf(&querystr, "SELECT EXISTS(SELECT domain from %s WHERE domain = ? AND group_id IN (%s));", table, groups) < 1)
+ if(asprintf(&querystr, "SELECT %s from %s WHERE domain = ? AND group_id IN (%s);", column, table, groups) < 1)
{
log_err("get_client_querystr(%s, %s) - asprintf() error", table, groups);
return NULL;
@@ -833,12 +831,12 @@ bool gravityDB_prepare_client_statements(clientsData *client)
// returns true as soon as it sees the first row from the query inside
// of EXISTS().
log_debug(DEBUG_DATABASE, "gravityDB_open(): Preparing vw_whitelist statement for client %s", clientip);
- querystr = get_client_querystr("vw_whitelist", getstr(client->groupspos));
+ querystr = get_client_querystr("vw_whitelist", "id", getstr(client->groupspos));
sqlite3_stmt* stmt = NULL;
int rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
if( rc != SQLITE_OK )
{
- log_err("gravityDB_open(\"SELECT EXISTS(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
+ log_err("gravityDB_open(\"SELECT(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
gravityDB_close();
return false;
}
@@ -847,11 +845,11 @@ bool gravityDB_prepare_client_statements(clientsData *client)
// Prepare gravity statement
log_debug(DEBUG_DATABASE, "gravityDB_open(): Preparing vw_gravity statement for client %s", clientip);
- querystr = get_client_querystr("vw_gravity", getstr(client->groupspos));
+ querystr = get_client_querystr("vw_gravity", "domain", getstr(client->groupspos));
rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
if( rc != SQLITE_OK )
{
- log_err("gravityDB_open(\"SELECT EXISTS(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
+ log_err("gravityDB_open(\"SELECT(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
gravityDB_close();
return false;
}
@@ -860,11 +858,11 @@ bool gravityDB_prepare_client_statements(clientsData *client)
// Prepare blacklist statement
log_debug(DEBUG_DATABASE, "gravityDB_open(): Preparing vw_blacklist statement for client %s", clientip);
- querystr = get_client_querystr("vw_blacklist", getstr(client->groupspos));
+ querystr = get_client_querystr("vw_blacklist", "id", getstr(client->groupspos));
rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
if( rc != SQLITE_OK )
{
- log_err("gravityDB_open(\"SELECT EXISTS(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
+ log_err("gravityDB_open(\"SELECT(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
gravityDB_close();
return false;
}
@@ -978,10 +976,11 @@ bool gravityDB_getTable(const unsigned char list)
return true;
}
-// Get a single domain from a running SELECT operation This function returns a
-// pointer to a string as long as there are domains available. Once we reached
-// the end of the table, it returns NULL. It also returns NULL when it
-// encounters an error (e.g., on reading errors). Errors are logged to FTL.log
+// Get a single domain from a running SELECT operation
+// This function returns a pointer to a string as long as there are domains
+// available. Once we reached the end of the table, it returns NULL. It also
+// returns NULL when it encounters an error (e.g., on reading errors). Errors
+// are logged to FTL.log
// This function is performance critical as it might be called millions of times
// for large blocking lists
inline const char* gravityDB_getDomain(int *rowid)
@@ -1112,7 +1111,7 @@ int gravityDB_count(const enum gravity_tables list)
return result;
}
-static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname)
+static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname, int *domain_id)
{
// Do not try to bind text to statement when database is not available
if(!gravityDB_opened && !gravityDB_open())
@@ -1148,7 +1147,7 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
sqlite3_clear_bindings(stmt);
return LIST_NOT_AVAILABLE;
}
- else if(rc != SQLITE_ROW)
+ else if(rc != SQLITE_ROW && rc != SQLITE_DONE)
{
// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
// is a real error we should log
@@ -1159,8 +1158,10 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
return LIST_NOT_AVAILABLE;
}
- // Get result of query "SELECT EXISTS(...)"
- const int result = sqlite3_column_int(stmt, 0);
+ // Get result of query (if available)
+ const int result = (rc == SQLITE_ROW) ? sqlite3_column_int(stmt, 0) : -1;
+ if(domain_id != NULL)
+ *domain_id = result;
log_debug(DEBUG_DATABASE, "domain_in_list(\"%s\", %p, %s): %d", domain, stmt, listname, result);
@@ -1176,8 +1177,7 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
sqlite3_clear_bindings(stmt);
// Return if domain was found in current table
- // SELECT EXISTS(...) either returns 0 (false) or 1 (true).
- return (result == 1) ? FOUND : NOT_FOUND;
+ return (rc == SQLITE_ROW) ? FOUND : NOT_FOUND;
}
void gravityDB_reload_groups(clientsData* client)
@@ -1234,17 +1234,7 @@ enum db_result in_allowlist(const char *domain, DNSCacheData *dns_cache, clients
// We have to check both the exact whitelist (using a prepared database statement)
// as well the compiled regex whitelist filters to check if the current domain is
// whitelisted.
- enum db_result allowed = domain_in_list(domain, stmt, "allow");
-
- // For performance reasons, the regex evaluations is executed only if the
- // exact whitelist lookup does not deliver a positive match. This is an
- // optimization as the database lookup will most likely hit (a) more domains
- // and (b) will be faster (given a sufficiently large number of regex
- // whitelisting filters).
- if(allowed == NOT_FOUND)
- allowed = match_regex(domain, dns_cache, client->id, REGEX_ALLOW, false) != -1;
-
- return allowed;
+ return domain_in_list(domain, stmt, "whitelist", &dns_cache->domainlist_id);
}
enum db_result in_gravity(const char *domain, clientsData *client)
@@ -1272,10 +1262,10 @@ enum db_result in_gravity(const char *domain, clientsData *client)
if(stmt == NULL)
stmt = gravity_stmt->get(gravity_stmt, client->id);
- return domain_in_list(domain, stmt, "gravity");
+ return domain_in_list(domain, stmt, "gravity", NULL);
}
-enum db_result in_denylist(const char *domain, clientsData *client)
+enum db_result in_denylist(const char *domain, DNSCacheData *dns_cache, clientsData *client)
{
// If list statement is not ready and cannot be initialized (e.g. no
// access to the database), we return false to prevent an FTL crash
@@ -1300,7 +1290,7 @@ enum db_result in_denylist(const char *domain, clientsData *client)
if(stmt == NULL)
stmt = blacklist_stmt->get(blacklist_stmt, client->id);
- return domain_in_list(domain, stmt, "deny");
+ return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id);
}
bool in_auditlist(const char *domain)
@@ -1311,7 +1301,7 @@ bool in_auditlist(const char *domain)
return false;
// We check the domain_audit table for the given domain
- return domain_in_list(domain, auditlist_stmt, "auditlist") == FOUND;
+ return domain_in_list(domain, auditlist_stmt, "auditlist", NULL) == FOUND;
}
bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex,
@@ -2240,3 +2230,42 @@ bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups,
return okay;
}
+
+void check_inaccessible_adlists(void)
+{
+ // Check if any adlist was inaccessible in the last gravity run
+ // If so, gravity stored `status` in the adlist table with
+ // "3": List unavailable, Pi-hole used a local copy
+ // "4": List unavailable, there is no local copy available
+
+ // Do not proceed when database is not available
+ if(!gravityDB_opened && !gravityDB_open())
+ {
+ log_err("check_inaccessible_adlists(): Gravity database not available");
+ return;
+ }
+
+ const char *querystr = "SELECT id, address FROM adlist WHERE status IN (3,4) AND enabled=1";
+
+ // Prepare query
+ sqlite3_stmt *query_stmt;
+ int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &query_stmt, NULL);
+ if(rc != SQLITE_OK){
+ log_err("check_inaccessible_adlists(): %s - SQL error prepare: %s", querystr, sqlite3_errstr(rc));
+ gravityDB_close();
+ return;
+ }
+
+ // Perform query
+ while((rc = sqlite3_step(query_stmt)) == SQLITE_ROW)
+ {
+ int id = sqlite3_column_int(query_stmt, 0);
+ const char *address = (const char*)sqlite3_column_text(query_stmt, 1);
+
+ // log to the message table
+ logg_inaccessible_adlist(id, address);
+ }
+
+ // Finalize statement
+ sqlite3_finalize(query_stmt);
+}
diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h
index 47996488..2d593e47 100644
--- a/src/database/gravity-db.h
+++ b/src/database/gravity-db.h
@@ -50,9 +50,10 @@ const char* gravityDB_getDomain(int *rowid);
char* get_client_names_from_ids(const char *group_ids) __attribute__ ((malloc));
void gravityDB_finalizeTable(void);
int gravityDB_count(const enum gravity_tables list);
+void check_inaccessible_adlists(void);
enum db_result in_gravity(const char *domain, clientsData *client);
-enum db_result in_denylist(const char *domain, clientsData *client);
+enum db_result in_denylist(const char *domain, DNSCacheData *dns_cache, clientsData *client);
enum db_result in_allowlist(const char *domain, DNSCacheData *dns_cache, clientsData *client);
bool in_auditlist(const char *domain);
diff --git a/src/database/message-table.c b/src/database/message-table.c
index 16f8f9e4..022518f1 100644
--- a/src/database/message-table.c
+++ b/src/database/message-table.c
@@ -27,7 +27,7 @@
#include "../gc.h"
static const char *message_types[MAX_MESSAGE] =
- { "REGEX", "SUBNET", "HOSTNAME", "DNSMASQ_CONFIG", "RATE_LIMIT", "DNSMASQ_WARN", "LOAD", "SHMEM", "DISK" };
+ { "REGEX", "SUBNET", "HOSTNAME", "DNSMASQ_CONFIG", "RATE_LIMIT", "DNSMASQ_WARN", "LOAD", "SHMEM", "DISK", "ADLIST" };
static unsigned char message_blob_types[MAX_MESSAGE][5] =
{
@@ -94,6 +94,13 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] =
SQLITE_NULL, // Not used
SQLITE_NULL // Not used
},
+ { // INACCESSIBLE_ADLIST_MESSAGE: The message column contains the corresponding adlist URL
+ SQLITE_INTEGER, // database index of the adlist (so the dashboard can show a link)
+ SQLITE_NULL, // not used
+ SQLITE_NULL, // not used
+ SQLITE_NULL, // not used
+ SQLITE_NULL // not used
+ },
};
// Create message table in the database
bool create_message_table(sqlite3 *db)
@@ -143,7 +150,7 @@ bool flush_message_table(void)
return true;
}
-static bool add_message(const enum message_type type, const bool unique,
+static bool add_message(const enum message_type type,
const char *message, const int count,...)
{
bool okay = false;
@@ -159,55 +166,52 @@ static bool add_message(const enum message_type type, const bool unique,
return false;
}
- // Ensure there are no duplicates when adding host name or rate-limiting messages
- if(unique)
- {
- sqlite3_stmt* stmt = NULL;
- const char *querystr = "DELETE FROM message WHERE type = ?1 AND message = ?2";
- int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
- if( rc != SQLITE_OK ){
- log_err("add_message(type=%u, message=%s) - SQL error prepare DELETE: %s",
- type, message, sqlite3_errstr(rc));
- goto end_of_add_message;
- }
-
- // Bind type to prepared statement
- if((rc = sqlite3_bind_text(stmt, 1, message_types[type], -1, SQLITE_STATIC)) != SQLITE_OK)
- {
- log_err("add_message(type=%u, message=%s) - Failed to bind type DELETE: %s",
- type, message, sqlite3_errstr(rc));
- sqlite3_reset(stmt);
- sqlite3_finalize(stmt);
- goto end_of_add_message;
- }
-
- // Bind message to prepared statement
- if((rc = sqlite3_bind_text(stmt, 2, message, -1, SQLITE_STATIC)) != SQLITE_OK)
- {
- log_err("add_message(type=%u, message=%s) - Failed to bind message DELETE: %s",
- type, message, sqlite3_errstr(rc));
- sqlite3_reset(stmt);
- sqlite3_finalize(stmt);
- goto end_of_add_message;
- }
-
- // Execute and finalize
- if((rc = sqlite3_step(stmt)) != SQLITE_OK && rc != SQLITE_DONE)
- {
- log_err("add_message(type=%u, message=%s) - SQL error step DELETE: %s",
- type, message, sqlite3_errstr(rc));
- goto end_of_add_message;
- }
- sqlite3_clear_bindings(stmt);
- sqlite3_reset(stmt);
- sqlite3_finalize(stmt);
+ // Ensure there are no duplicates when adding messages
+ sqlite3_stmt* stmt = NULL;
+ const char *querystr = "DELETE FROM message WHERE type = ?1 AND message = ?2";
+ int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+ if( rc != SQLITE_OK ){
+ log_err("add_message(type=%u, message=%s) - SQL error prepare DELETE: %s",
+ type, message, sqlite3_errstr(rc));
+ goto end_of_add_message;
}
+ // Bind type to prepared statement
+ if((rc = sqlite3_bind_text(stmt, 1, message_types[type], -1, SQLITE_STATIC)) != SQLITE_OK)
+ {
+ log_err("add_message(type=%u, message=%s) - Failed to bind type DELETE: %s",
+ type, message, sqlite3_errstr(rc));
+ sqlite3_reset(stmt);
+ sqlite3_finalize(stmt);
+ goto end_of_add_message;
+ }
+
+ // Bind message to prepared statement
+ if((rc = sqlite3_bind_text(stmt, 2, message, -1, SQLITE_STATIC)) != SQLITE_OK)
+ {
+ log_err("add_message(type=%u, message=%s) - Failed to bind message DELETE: %s",
+ type, message, sqlite3_errstr(rc));
+ sqlite3_reset(stmt);
+ sqlite3_finalize(stmt);
+ goto end_of_add_message;
+ }
+
+ // Execute and finalize
+ if((rc = sqlite3_step(stmt)) != SQLITE_OK && rc != SQLITE_DONE)
+ {
+ log_err("add_message(type=%u, message=%s) - SQL error step DELETE: %s",
+ type, message, sqlite3_errstr(rc));
+ goto end_of_add_message;
+ }
+ sqlite3_clear_bindings(stmt);
+ sqlite3_reset(stmt);
+ sqlite3_finalize(stmt);
+ stmt = NULL;
+
// Prepare SQLite statement
- sqlite3_stmt* stmt = NULL;
- const char *querystr = "INSERT INTO message (timestamp,type,message,blob1,blob2,blob3,blob4,blob5) "
- "VALUES ((cast(strftime('%s', 'now') as int)),?,?,?,?,?,?,?);";
- int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+ querystr = "INSERT INTO message (timestamp,type,message,blob1,blob2,blob3,blob4,blob5) "
+ "VALUES ((cast(strftime('%s', 'now') as int)),?,?,?,?,?,?,?);";
+ rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
log_err("add_message(type=%u, message=%s) - SQL error prepare: %s",
@@ -313,7 +317,7 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex
// Log to database only if not in CLI mode
if(!cli_mode)
- add_message(REGEX_MESSAGE, false, warning, 3, type, regex, dbindex);
+ add_message(REGEX_MESSAGE, warning, 3, type, regex, dbindex);
}
void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids,
@@ -328,7 +332,7 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m
// Log to database
char *names = get_client_names_from_ids(matching_ids);
- add_message(SUBNET_MESSAGE, false, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id);
+ add_message(SUBNET_MESSAGE, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id);
free(names);
}
@@ -339,7 +343,7 @@ void logg_hostname_warning(const char *ip, const char *name, const unsigned int
ip, name, pos);
// Log to database
- add_message(HOSTNAME_MESSAGE, true, ip, 2, name, (const int)pos);
+ add_message(HOSTNAME_MESSAGE, ip, 2, name, (const int)pos);
}
void logg_fatal_dnsmasq_message(const char *message)
@@ -348,7 +352,7 @@ void logg_fatal_dnsmasq_message(const char *message)
log_crit("Error in dnsmasq core: %s", message);
// Log to database
- add_message(DNSMASQ_CONFIG_MESSAGE, false, message, 0);
+ add_message(DNSMASQ_CONFIG_MESSAGE, message, 0);
// FTL will dies after this point, so we should make sure to clean up
// behind ourselves
@@ -364,7 +368,7 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit
clientIP, turnaround, turnaround == 1 ? "" : "s");
// Log to database
- add_message(RATE_LIMIT_MESSAGE, true, clientIP, 2, config.rate_limit.count, config.rate_limit.interval);
+ add_message(RATE_LIMIT_MESSAGE, clientIP, 2, config.rate_limit.count, config.rate_limit.interval);
}
void logg_warn_dnsmasq_message(char *message)
@@ -373,7 +377,7 @@ void logg_warn_dnsmasq_message(char *message)
log_warn("WARNING in dnsmasq core: %s", message);
// Log to database
- add_message(DNSMASQ_WARN_MESSAGE, false, message, 0);
+ add_message(DNSMASQ_WARN_MESSAGE, message, 0);
}
void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg)
@@ -381,16 +385,25 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem,
if(load > 0.0)
{
log_warn("Long-term load (15min avg) larger than number of processors: %.1f > %d", load, nprocs);
- add_message(LOAD_MESSAGE, true, "excessive load", 2, load, nprocs);
+ add_message(LOAD_MESSAGE, "excessive load", 2, load, nprocs);
}
else if(shmem > -1)
{
log_warn("RAM shortage (%s) ahead: %d%% is used (%s)", path, shmem, msg);
- add_message(SHMEM_MESSAGE, true, path, 2, shmem, msg);
+ add_message(SHMEM_MESSAGE, path, 2, shmem, msg);
}
else if(disk > -1)
{
log_warn("Disk shortage (%s) ahead: %d%% is used (%s)", path, disk, msg);
- add_message(DISK_MESSAGE, true, path, 2, disk, msg);
+ add_message(DISK_MESSAGE, path, 2, disk, msg);
}
}
+
+void logg_inaccessible_adlist(const int dbindex, const char *address)
+{
+ // Log to FTL.log
+ log_warn("Adlist with ID %d (%s) was inaccessible during last gravity run", dbindex, address);
+
+ // Log to database
+ add_message(INACCESSIBLE_ADLIST_MESSAGE, address, 1, dbindex);
+}
diff --git a/src/database/message-table.h b/src/database/message-table.h
index ece5b752..7d006a28 100644
--- a/src/database/message-table.h
+++ b/src/database/message-table.h
@@ -23,5 +23,6 @@ void logg_fatal_dnsmasq_message(const char *message);
void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit_count);
void logg_warn_dnsmasq_message(char *message);
void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg);
+void logg_inaccessible_adlist(const int dbindex, const char *address);
#endif //MESSAGETABLE_H
diff --git a/src/database/query-table.c b/src/database/query-table.c
index 654d9b77..73316d2e 100644
--- a/src/database/query-table.c
+++ b/src/database/query-table.c
@@ -404,8 +404,7 @@ bool export_queries_to_disk(bool final)
last_disk_db_idx, last_mem_db_idx, time);
// Start database timer
- if(config.debug & DEBUG_DATABASE)
- timer_start(DATABASE_WRITE_TIMER);
+ timer_start(DATABASE_WRITE_TIMER);
// Attach disk database
if(!attach_disk_database(NULL))
@@ -761,7 +760,7 @@ void DB_read_queries(void)
}
const int status_int = sqlite3_column_int(stmt, 3);
- if(status_int < QUERY_UNKNOWN || status_int > QUERY_STATUS_MAX)
+ if(status_int < QUERY_UNKNOWN || status_int >= QUERY_STATUS_MAX)
{
log_warn("Database: STATUS should be within [%i,%i] but is %i",
QUERY_UNKNOWN, QUERY_STATUS_MAX-1, status_int);
@@ -925,16 +924,18 @@ void DB_read_queries(void)
query->CNAME_domainID = CNAMEdomainID;
}
}
- else if(status == QUERY_REGEX)
+ else if(sqlite3_column_bytes(stmt, 7) != 0)
{
- // QUERY_REGEX: Set ID regex which was the reason for blocking
- const int cacheID = findCacheID(query->domainID, query->clientID, query->type);
+ // Set ID of the domainlist entry that was the reason for permitting/blocking this query
+ // We assume the value in this field is said ID when it is not a CNAME-related domain
+ // (checked above) and the value of additional_info is not NULL (0 bytes storage size)
+ const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true);
DNSCacheData *cache = getDNSCache(cacheID, true);
// Only load if
// a) we have a cache entry
// b) the value of additional_info is not NULL (0 bytes storage size)
if(cache != NULL && sqlite3_column_bytes(stmt, 7) != 0)
- cache->deny_regex_id = sqlite3_column_int(stmt, 7);
+ cache->domainlist_id = sqlite3_column_int(stmt, 7);
}
// Increment status counters, we first have to add one to the count of
@@ -985,6 +986,7 @@ void DB_read_queries(void)
break;
case QUERY_CACHE: // Cached or local config
+ case QUERY_CACHE_STALE:
// Nothing to be done here
break;
@@ -1280,16 +1282,16 @@ bool queries_to_database(void)
else if(query->status == QUERY_REGEX)
{
// Restore regex ID if applicable
- const int cacheID = findCacheID(query->domainID, query->clientID, query->type);
+ const int cacheID = findCacheID(query->domainID, query->clientID, query->type, false);
DNSCacheData *cache = getDNSCache(cacheID, true);
if(cache != NULL)
{
sqlite3_bind_int(query_stmt, 9, ADDINFO_REGEX_ID);
- sqlite3_bind_int(query_stmt, 10, cache->deny_regex_id);
+ sqlite3_bind_int(query_stmt, 10, cache->domainlist_id);
// Execute prepared addinfo statement and check if successful
sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID);
- sqlite3_bind_int(addinfo_stmt, 2, cache->deny_regex_id);
+ sqlite3_bind_int(addinfo_stmt, 2, cache->domainlist_id);
if(sqlite3_step(addinfo_stmt) != SQLITE_DONE)
{
log_err("Encountered error while trying to store addinfo");
diff --git a/src/database/shell.c b/src/database/shell.c
index 00266127..a5e82f70 100644
--- a/src/database/shell.c
+++ b/src/database/shell.c
@@ -34,6 +34,8 @@
/* This needs to come before any includes for MSVC compiler */
#define _CRT_SECURE_NO_WARNINGS
#endif
+typedef unsigned int u32;
+typedef unsigned short int u16;
/*
** Optionally #include a user-defined header, whereby compilation options
@@ -55,6 +57,15 @@
# define SQLITE_OS_WINRT 0
#endif
+/*
+** If SQLITE_SHELL_FIDDLE is defined then the shell is modified
+** somewhat for use as a WASM module in a web browser. This flag
+** should only be used when building the "fiddle" web application, as
+** the browser-mode build has much different user input requirements
+** and this build mode rewires the user input subsystem to account for
+** that.
+*/
+
/*
** Warning pragmas copied from msvc.h in the core.
*/
@@ -94,6 +105,14 @@
# define _LARGEFILE_SOURCE 1
#endif
+#if defined(SQLITE_SHELL_FIDDLE) && !defined(_POSIX_SOURCE)
+/*
+** emcc requires _POSIX_SOURCE (or one of several similar defines)
+** to expose strdup().
+*/
+# define _POSIX_SOURCE
+#endif
+
#include
#include
#include
@@ -249,20 +268,21 @@ static void setTextMode(FILE *file, int isOutput){
# define setTextMode(X,Y)
#endif
-/*
-** When compiling with emcc (a.k.a. emscripten), we're building a
-** WebAssembly (WASM) bundle and need to disable and rewire a few
-** things.
-*/
-#ifdef __EMSCRIPTEN__
-#define SQLITE_SHELL_WASM_MODE
-#else
-#undef SQLITE_SHELL_WASM_MODE
-#endif
-
/* True if the timer is enabled */
static int enableTimer = 0;
+/* A version of strcmp() that works with NULL values */
+static int cli_strcmp(const char *a, const char *b){
+ if( a==0 ) a = "";
+ if( b==0 ) b = "";
+ return strcmp(a,b);
+}
+static int cli_strncmp(const char *a, const char *b, size_t n){
+ if( a==0 ) a = "";
+ if( b==0 ) b = "";
+ return strncmp(a,b,n);
+}
+
/* Return the current wall-clock time */
static sqlite3_int64 timeOfDay(void){
static sqlite3_vfs *clockVfs = 0;
@@ -551,6 +571,7 @@ static void utf8_width_print(FILE *pOut, int w, const char *zUtf){
int i;
int n;
int aw = w<0 ? -w : w;
+ if( zUtf==0 ) zUtf = "";
for(i=n=0; zUtf[i]; i++){
if( (zUtf[i]&0xc0)!=0x80 ){
n++;
@@ -694,7 +715,7 @@ static char *local_getline(char *zLine, FILE *in){
if( stdin_is_interactive && in==stdin ){
char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
if( zTrans ){
- int nTrans = strlen30(zTrans)+1;
+ i64 nTrans = strlen(zTrans)+1;
if( nTrans>nLine ){
zLine = realloc(zLine, nTrans);
shell_check_oom(zLine);
@@ -721,7 +742,7 @@ static char *local_getline(char *zLine, FILE *in){
** be freed by the caller or else passed back into this routine via the
** zPrior argument for reuse.
*/
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
char *zPrompt;
char *zResult;
@@ -741,7 +762,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
}
return zResult;
}
-#endif /* !SQLITE_SHELL_WASM_MODE */
+#endif /* !SQLITE_SHELL_FIDDLE */
/*
** Return the value of a hexadecimal digit. Return -1 if the input
@@ -830,9 +851,9 @@ static void freeText(ShellText *p){
** quote character for zAppend.
*/
static void appendText(ShellText *p, const char *zAppend, char quote){
- int len;
- int i;
- int nAppend = strlen30(zAppend);
+ i64 len;
+ i64 i;
+ i64 nAppend = strlen30(zAppend);
len = nAppend+p->n+1;
if( quote ){
@@ -991,10 +1012,10 @@ static void shellAddSchemaName(
const char *zName = (const char*)sqlite3_value_text(apVal[2]);
sqlite3 *db = sqlite3_context_db_handle(pCtx);
UNUSED_PARAMETER(nVal);
- if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){
+ if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){
for(i=0; inInit>in.mx ) return 0;
+ c = RE_START-1;
}
if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){
@@ -3973,6 +4023,10 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){
if( pRe->aArg[x]==c ) re_add_state(pNext, x+1);
break;
}
+ case RE_OP_ATSTART: {
+ if( cPrev==RE_START ) re_add_state(pThis, x+1);
+ break;
+ }
case RE_OP_ANY: {
if( c!=0 ) re_add_state(pNext, x+1);
break;
@@ -4054,7 +4108,9 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){
}
}
for(i=0; inState; i++){
- if( pRe->aOp[pNext->aState[i]]==RE_OP_ACCEPT ){ rc = 1; break; }
+ int x = pNext->aState[i];
+ while( pRe->aOp[x]==RE_OP_GOTO ) x += pRe->aArg[x];
+ if( pRe->aOp[x]==RE_OP_ACCEPT ){ rc = 1; break; }
}
re_match_end:
sqlite3_free(pToFree);
@@ -4209,7 +4265,6 @@ static const char *re_subcompile_string(ReCompiled *p){
iStart = p->nState;
switch( c ){
case '|':
- case '$':
case ')': {
p->sIn.i--;
return 0;
@@ -4246,6 +4301,14 @@ static const char *re_subcompile_string(ReCompiled *p){
re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1);
break;
}
+ case '$': {
+ re_append(p, RE_OP_MATCH, RE_EOF);
+ break;
+ }
+ case '^': {
+ re_append(p, RE_OP_ATSTART, 0);
+ break;
+ }
case '{': {
int m = 0, n = 0;
int sz, j;
@@ -4264,6 +4327,7 @@ static const char *re_subcompile_string(ReCompiled *p){
if( m==0 ){
if( n==0 ) return "both m and n are zero in '{m,n}'";
re_insert(p, iPrev, RE_OP_FORK, sz+1);
+ iPrev++;
n--;
}else{
for(j=1; jsIn.i+1>=pRe->sIn.mx ){
- re_append(pRe, RE_OP_MATCH, RE_EOF);
- re_append(pRe, RE_OP_ACCEPT, 0);
- *ppRe = pRe;
- }else if( pRe->sIn.i>=pRe->sIn.mx ){
+ if( pRe->sIn.i>=pRe->sIn.mx ){
re_append(pRe, RE_OP_ACCEPT, 0);
*ppRe = pRe;
}else{
@@ -4411,7 +4471,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){
pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6));
pRe->zInit[j++] = 0x80 | (x&0x3f);
}else if( x<=0xffff ){
- pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12));
+ pRe->zInit[j++] = (unsigned char)(0xe0 | (x>>12));
pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f);
pRe->zInit[j++] = 0x80 | (x&0x3f);
}else{
@@ -4470,6 +4530,67 @@ static void re_sql_func(
}
}
+#if defined(SQLITE_DEBUG)
+/*
+** This function is used for testing and debugging only. It is only available
+** if the SQLITE_DEBUG compile-time option is used.
+**
+** Compile a regular expression and then convert the compiled expression into
+** text and return that text.
+*/
+static void re_bytecode_func(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ const char *zPattern;
+ const char *zErr;
+ ReCompiled *pRe;
+ sqlite3_str *pStr;
+ int i;
+ int n;
+ char *z;
+
+ zPattern = (const char*)sqlite3_value_text(argv[0]);
+ if( zPattern==0 ) return;
+ zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0);
+ if( zErr ){
+ re_free(pRe);
+ sqlite3_result_error(context, zErr, -1);
+ return;
+ }
+ if( pRe==0 ){
+ sqlite3_result_error_nomem(context);
+ return;
+ }
+ pStr = sqlite3_str_new(0);
+ if( pStr==0 ) goto re_bytecode_func_err;
+ if( pRe->nInit>0 ){
+ sqlite3_str_appendf(pStr, "INIT ");
+ for(i=0; inInit; i++){
+ sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]);
+ }
+ sqlite3_str_appendf(pStr, "\n");
+ }
+ for(i=0; (unsigned)inState; i++){
+ sqlite3_str_appendf(pStr, "%-8s %4d\n",
+ ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]);
+ }
+ n = sqlite3_str_length(pStr);
+ z = sqlite3_str_finish(pStr);
+ if( n==0 ){
+ sqlite3_free(z);
+ }else{
+ sqlite3_result_text(context, z, n-1, sqlite3_free);
+ }
+
+re_bytecode_func_err:
+ re_free(pRe);
+}
+
+#endif /* SQLITE_DEBUG */
+
+
/*
** Invoke this routine to register the regexp() function with the
** SQLite database connection.
@@ -4494,12 +4615,19 @@ int sqlite3_regexp_init(
rc = sqlite3_create_function(db, "regexpi", 2,
SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
(void*)db, re_sql_func, 0, 0);
+#if defined(SQLITE_DEBUG)
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_function(db, "regexp_bytecode", 1,
+ SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
+ 0, re_bytecode_func, 0, 0);
+ }
+#endif /* SQLITE_DEBUG */
}
return rc;
}
/************************* End ../ext/misc/regexp.c ********************/
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
/************************* Begin ../ext/misc/fileio.c ******************/
/*
** 2014-06-13
@@ -6769,8 +6897,8 @@ SQLITE_EXTENSION_INIT1
#endif
/* typedef sqlite3_int64 i64; */
/* typedef unsigned char u8; */
-typedef UINT32_TYPE u32; /* 4-byte unsigned integer */
-typedef UINT16_TYPE u16; /* 2-byte unsigned integer */
+/* typedef UINT32_TYPE u32; // 4-byte unsigned integer // */
+/* typedef UINT16_TYPE u16; // 2-byte unsigned integer // */
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
@@ -10051,6 +10179,10 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
*/
static int idxIdentifierRequiresQuotes(const char *zId){
int i;
+ int nId = STRLEN(zId);
+
+ if( sqlite3_keyword_check(zId, nId) ) return 1;
+
for(i=0; zId[i]; i++){
if( !(zId[i]=='_')
&& !(zId[i]>='0' && zId[i]<='9')
@@ -11277,7 +11409,12 @@ void sqlite3_expert_destroy(sqlite3expert *p){
/************************* End ../ext/expert/sqlite3expert.c ********************/
#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
-/************************* Begin ../ext/misc/dbdata.c ******************/
+#define SQLITE_SHELL_HAVE_RECOVER 1
+#else
+#define SQLITE_SHELL_HAVE_RECOVER 0
+#endif
+#if SQLITE_SHELL_HAVE_RECOVER
+/************************* Begin ../ext/recover/dbdata.c ******************/
/*
** 2019-04-17
**
@@ -11351,16 +11488,20 @@ void sqlite3_expert_destroy(sqlite3expert *p){
** It contains one entry for each b-tree pointer between a parent and
** child page in the database.
*/
+
#if !defined(SQLITEINT_H)
/* #include "sqlite3ext.h" */
/* typedef unsigned char u8; */
+/* typedef unsigned int u32; */
#endif
SQLITE_EXTENSION_INIT1
#include
#include
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
#define DBDATA_PADDING_BYTES 100
typedef struct DbdataTable DbdataTable;
@@ -11382,11 +11523,12 @@ struct DbdataCursor {
/* Only for the sqlite_dbdata table */
u8 *pRec; /* Buffer containing current record */
- int nRec; /* Size of pRec[] in bytes */
- int nHdr; /* Size of header in bytes */
+ sqlite3_int64 nRec; /* Size of pRec[] in bytes */
+ sqlite3_int64 nHdr; /* Size of header in bytes */
int iField; /* Current field number */
u8 *pHdrPtr;
u8 *pPtr;
+ u32 enc; /* Text encoding */
sqlite3_int64 iIntkey; /* Integer key value */
};
@@ -11579,14 +11721,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){
/*
** Utility methods to decode 16 and 32-bit big-endian unsigned integers.
*/
-static unsigned int get_uint16(unsigned char *a){
+static u32 get_uint16(unsigned char *a){
return (a[0]<<8)|a[1];
}
-static unsigned int get_uint32(unsigned char *a){
- return ((unsigned int)a[0]<<24)
- | ((unsigned int)a[1]<<16)
- | ((unsigned int)a[2]<<8)
- | ((unsigned int)a[3]);
+static u32 get_uint32(unsigned char *a){
+ return ((u32)a[0]<<24)
+ | ((u32)a[1]<<16)
+ | ((u32)a[2]<<8)
+ | ((u32)a[3]);
}
/*
@@ -11601,7 +11743,7 @@ static unsigned int get_uint32(unsigned char *a){
*/
static int dbdataLoadPage(
DbdataCursor *pCsr, /* Cursor object */
- unsigned int pgno, /* Page number of page to load */
+ u32 pgno, /* Page number of page to load */
u8 **ppPage, /* OUT: pointer to page buffer */
int *pnPage /* OUT: Size of (*ppPage) in bytes */
){
@@ -11611,25 +11753,27 @@ static int dbdataLoadPage(
*ppPage = 0;
*pnPage = 0;
- sqlite3_bind_int64(pStmt, 2, pgno);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- int nCopy = sqlite3_column_bytes(pStmt, 0);
- if( nCopy>0 ){
- u8 *pPage;
- pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
- if( pPage==0 ){
- rc = SQLITE_NOMEM;
- }else{
- const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
- memcpy(pPage, pCopy, nCopy);
- memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
+ if( pgno>0 ){
+ sqlite3_bind_int64(pStmt, 2, pgno);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ int nCopy = sqlite3_column_bytes(pStmt, 0);
+ if( nCopy>0 ){
+ u8 *pPage;
+ pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
+ if( pPage==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
+ memcpy(pPage, pCopy, nCopy);
+ memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
+ }
+ *ppPage = pPage;
+ *pnPage = nCopy;
}
- *ppPage = pPage;
- *pnPage = nCopy;
}
+ rc2 = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = rc2;
}
- rc2 = sqlite3_reset(pStmt);
- if( rc==SQLITE_OK ) rc = rc2;
return rc;
}
@@ -11638,17 +11782,30 @@ static int dbdataLoadPage(
** Read a varint. Put the value in *pVal and return the number of bytes.
*/
static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){
- sqlite3_int64 v = 0;
+ sqlite3_uint64 u = 0;
int i;
for(i=0; i<8; i++){
- v = (v<<7) + (z[i]&0x7f);
- if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
+ u = (u<<7) + (z[i]&0x7f);
+ if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
}
- v = (v<<8) + (z[i]&0xff);
- *pVal = v;
+ u = (u<<8) + (z[i]&0xff);
+ *pVal = (sqlite3_int64)u;
return 9;
}
+/*
+** Like dbdataGetVarint(), but set the output to 0 if it is less than 0
+** or greater than 0xFFFFFFFF. This can be used for all varints in an
+** SQLite database except for key values in intkey tables.
+*/
+static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){
+ sqlite3_int64 val;
+ int nRet = dbdataGetVarint(z, &val);
+ if( val<0 || val>0xFFFFFFFF ) val = 0;
+ *pVal = val;
+ return nRet;
+}
+
/*
** Return the number of bytes of space used by an SQLite value of type
** eType.
@@ -11685,9 +11842,10 @@ static int dbdataValueBytes(int eType){
*/
static void dbdataValue(
sqlite3_context *pCtx,
+ u32 enc,
int eType,
u8 *pData,
- int nData
+ sqlite3_int64 nData
){
if( eType>=0 && dbdataValueBytes(eType)<=nData ){
switch( eType ){
@@ -11729,7 +11887,19 @@ static void dbdataValue(
default: {
int n = ((eType-12) / 2);
if( eType % 2 ){
- sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT);
+ switch( enc ){
+#ifndef SQLITE_OMIT_UTF16
+ case SQLITE_UTF16BE:
+ sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
+ break;
+ case SQLITE_UTF16LE:
+ sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
+ break;
+#endif
+ default:
+ sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
+ break;
+ }
}else{
sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
}
@@ -11757,6 +11927,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage);
if( rc!=SQLITE_OK ) return rc;
if( pCsr->aPage ) break;
+ if( pCsr->bOnePage ) return SQLITE_OK;
pCsr->iPgno++;
}
pCsr->iCell = pTab->bPtr ? -2 : 0;
@@ -11820,7 +11991,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
if( bNextPage || iOff>pCsr->nPage ){
bNextPage = 1;
}else{
- iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload);
+ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
}
/* If this is a leaf intkey cell, load the rowid */
@@ -11867,7 +12038,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
/* Load content from overflow pages */
if( nPayload>nLocal ){
sqlite3_int64 nRem = nPayload - nLocal;
- unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
+ u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
while( nRem>0 ){
u8 *aOvfl = 0;
int nOvfl = 0;
@@ -11887,7 +12058,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
}
}
- iHdr = dbdataGetVarint(pCsr->pRec, &nHdr);
+ iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr);
+ if( nHdr>nPayload ) nHdr = 0;
pCsr->nHdr = nHdr;
pCsr->pHdrPtr = &pCsr->pRec[iHdr];
pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
@@ -11901,7 +12073,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){
bNextPage = 1;
}else{
- pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType);
+ pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
pCsr->pPtr += dbdataValueBytes(iType);
}
}
@@ -11940,6 +12112,18 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){
return pCsr->aPage==0;
}
+/*
+** Return true if nul-terminated string zSchema ends in "()". Or false
+** otherwise.
+*/
+static int dbdataIsFunction(const char *zSchema){
+ size_t n = strlen(zSchema);
+ if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){
+ return (int)n-2;
+ }
+ return 0;
+}
+
/*
** Determine the size in pages of database zSchema (where zSchema is
** "main", "temp" or the name of an attached database) and set
@@ -11950,10 +12134,16 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab;
char *zSql = 0;
int rc, rc2;
+ int nFunc = 0;
sqlite3_stmt *pStmt = 0;
- zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
+ if( (nFunc = dbdataIsFunction(zSchema))>0 ){
+ zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema);
+ }else{
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
+ }
if( zSql==0 ) return SQLITE_NOMEM;
+
rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
sqlite3_free(zSql);
if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
@@ -11964,6 +12154,25 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
return rc;
}
+/*
+** Attempt to figure out the encoding of the database by retrieving page 1
+** and inspecting the header field. If successful, set the pCsr->enc variable
+** and return SQLITE_OK. Otherwise, return an SQLite error code.
+*/
+static int dbdataGetEncoding(DbdataCursor *pCsr){
+ int rc = SQLITE_OK;
+ int nPg1 = 0;
+ u8 *aPg1 = 0;
+ rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
+ assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 );
+ if( rc==SQLITE_OK && nPg1>0 ){
+ pCsr->enc = get_uint32(&aPg1[56]);
+ }
+ sqlite3_free(aPg1);
+ return rc;
+}
+
+
/*
** xFilter method for sqlite_dbdata and sqlite_dbptr.
*/
@@ -11981,19 +12190,28 @@ static int dbdataFilter(
assert( pCsr->iPgno==1 );
if( idxNum & 0x01 ){
zSchema = (const char*)sqlite3_value_text(argv[0]);
+ if( zSchema==0 ) zSchema = "";
}
if( idxNum & 0x02 ){
pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
pCsr->bOnePage = 1;
}else{
- pCsr->nPage = dbdataDbsize(pCsr, zSchema);
rc = dbdataDbsize(pCsr, zSchema);
}
if( rc==SQLITE_OK ){
+ int nFunc = 0;
if( pTab->pStmt ){
pCsr->pStmt = pTab->pStmt;
pTab->pStmt = 0;
+ }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){
+ char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema);
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ }
}else{
rc = sqlite3_prepare_v2(pTab->db,
"SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
@@ -12006,13 +12224,20 @@ static int dbdataFilter(
}else{
pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
}
+
+ /* Try to determine the encoding of the db by inspecting the header
+ ** field on page 1. */
+ if( rc==SQLITE_OK ){
+ rc = dbdataGetEncoding(pCsr);
+ }
+
if( rc==SQLITE_OK ){
rc = dbdataNext(pCursor);
}
return rc;
}
-/*
+/*
** Return a column for the sqlite_dbdata or sqlite_dbptr table.
*/
static int dbdataColumn(
@@ -12056,11 +12281,12 @@ static int dbdataColumn(
case DBDATA_COLUMN_VALUE: {
if( pCsr->iField<0 ){
sqlite3_result_int64(ctx, pCsr->iIntkey);
- }else{
+ }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){
sqlite3_int64 iType;
- dbdataGetVarint(pCsr->pHdrPtr, &iType);
+ dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
dbdataValue(
- ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
+ ctx, pCsr->enc, iType, pCsr->pPtr,
+ &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
);
}
break;
@@ -12130,7 +12356,3121 @@ int sqlite3_dbdata_init(
return sqlite3DbdataRegister(db);
}
-/************************* End ../ext/misc/dbdata.c ********************/
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
+/************************* End ../ext/recover/dbdata.c ********************/
+/************************* Begin ../ext/recover/sqlite3recover.h ******************/
+/*
+** 2022-08-27
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains the public interface to the "recover" extension -
+** an SQLite extension designed to recover data from corrupted database
+** files.
+*/
+
+/*
+** OVERVIEW:
+**
+** To use the API to recover data from a corrupted database, an
+** application:
+**
+** 1) Creates an sqlite3_recover handle by calling either
+** sqlite3_recover_init() or sqlite3_recover_init_sql().
+**
+** 2) Configures the new handle using one or more calls to
+** sqlite3_recover_config().
+**
+** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on
+** the handle until it returns something other than SQLITE_OK. If it
+** returns SQLITE_DONE, then the recovery operation completed without
+** error. If it returns some other non-SQLITE_OK value, then an error
+** has occurred.
+**
+** 4) Retrieves any error code and English language error message using the
+** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs,
+** respectively.
+**
+** 5) Destroys the sqlite3_recover handle and frees all resources
+** using sqlite3_recover_finish().
+**
+** The application may abandon the recovery operation at any point
+** before it is finished by passing the sqlite3_recover handle to
+** sqlite3_recover_finish(). This is not an error, but the final state
+** of the output database, or the results of running the partial script
+** delivered to the SQL callback, are undefined.
+*/
+
+#ifndef _SQLITE_RECOVER_H
+#define _SQLITE_RECOVER_H
+
+/* #include "sqlite3.h" */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** An instance of the sqlite3_recover object represents a recovery
+** operation in progress.
+**
+** Constructors:
+**
+** sqlite3_recover_init()
+** sqlite3_recover_init_sql()
+**
+** Destructor:
+**
+** sqlite3_recover_finish()
+**
+** Methods:
+**
+** sqlite3_recover_config()
+** sqlite3_recover_errcode()
+** sqlite3_recover_errmsg()
+** sqlite3_recover_run()
+** sqlite3_recover_step()
+*/
+typedef struct sqlite3_recover sqlite3_recover;
+
+/*
+** These two APIs attempt to create and return a new sqlite3_recover object.
+** In both cases the first two arguments identify the (possibly
+** corrupt) database to recover data from. The first argument is an open
+** database handle and the second the name of a database attached to that
+** handle (i.e. "main", "temp" or the name of an attached database).
+**
+** If sqlite3_recover_init() is used to create the new sqlite3_recover
+** handle, then data is recovered into a new database, identified by
+** string parameter zUri. zUri may be an absolute or relative file path,
+** or may be an SQLite URI. If the identified database file already exists,
+** it is overwritten.
+**
+** If sqlite3_recover_init_sql() is invoked, then any recovered data will
+** be returned to the user as a series of SQL statements. Executing these
+** SQL statements results in the same database as would have been created
+** had sqlite3_recover_init() been used. For each SQL statement in the
+** output, the callback function passed as the third argument (xSql) is
+** invoked once. The first parameter is a passed a copy of the fourth argument
+** to this function (pCtx) as its first parameter, and a pointer to a
+** nul-terminated buffer containing the SQL statement formated as UTF-8 as
+** the second. If the xSql callback returns any value other than SQLITE_OK,
+** then processing is immediately abandoned and the value returned used as
+** the recover handle error code (see below).
+**
+** If an out-of-memory error occurs, NULL may be returned instead of
+** a valid handle. In all other cases, it is the responsibility of the
+** application to avoid resource leaks by ensuring that
+** sqlite3_recover_finish() is called on all allocated handles.
+*/
+sqlite3_recover *sqlite3_recover_init(
+ sqlite3* db,
+ const char *zDb,
+ const char *zUri
+);
+sqlite3_recover *sqlite3_recover_init_sql(
+ sqlite3* db,
+ const char *zDb,
+ int (*xSql)(void*, const char*),
+ void *pCtx
+);
+
+/*
+** Configure an sqlite3_recover object that has just been created using
+** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function
+** may only be called before the first call to sqlite3_recover_step()
+** or sqlite3_recover_run() on the object.
+**
+** The second argument passed to this function must be one of the
+** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument
+** depend on the specific SQLITE_RECOVER_* symbol in use.
+**
+** SQLITE_OK is returned if the configuration operation was successful,
+** or an SQLite error code otherwise.
+*/
+int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg);
+
+/*
+** SQLITE_RECOVER_LOST_AND_FOUND:
+** The pArg argument points to a string buffer containing the name
+** of a "lost-and-found" table in the output database, or NULL. If
+** the argument is non-NULL and the database contains seemingly
+** valid pages that cannot be associated with any table in the
+** recovered part of the schema, data is extracted from these
+** pages to add to the lost-and-found table.
+**
+** SQLITE_RECOVER_FREELIST_CORRUPT:
+** The pArg value must actually be a pointer to a value of type
+** int containing value 0 or 1 cast as a (void*). If this option is set
+** (argument is 1) and a lost-and-found table has been configured using
+** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is
+** corrupt and an attempt is made to recover records from pages that
+** appear to be linked into the freelist. Otherwise, pages on the freelist
+** are ignored. Setting this option can recover more data from the
+** database, but often ends up "recovering" deleted records. The default
+** value is 0 (clear).
+**
+** SQLITE_RECOVER_ROWIDS:
+** The pArg value must actually be a pointer to a value of type
+** int containing value 0 or 1 cast as a (void*). If this option is set
+** (argument is 1), then an attempt is made to recover rowid values
+** that are not also INTEGER PRIMARY KEY values. If this option is
+** clear, then new rowids are assigned to all recovered rows. The
+** default value is 1 (set).
+**
+** SQLITE_RECOVER_SLOWINDEXES:
+** The pArg value must actually be a pointer to a value of type
+** int containing value 0 or 1 cast as a (void*). If this option is clear
+** (argument is 0), then when creating an output database, the recover
+** module creates and populates non-UNIQUE indexes right at the end of the
+** recovery operation - after all recoverable data has been inserted
+** into the new database. This is faster overall, but means that the
+** final call to sqlite3_recover_step() for a recovery operation may
+** be need to create a large number of indexes, which may be very slow.
+**
+** Or, if this option is set (argument is 1), then non-UNIQUE indexes
+** are created in the output database before it is populated with
+** recovered data. This is slower overall, but avoids the slow call
+** to sqlite3_recover_step() at the end of the recovery operation.
+**
+** The default option value is 0.
+*/
+#define SQLITE_RECOVER_LOST_AND_FOUND 1
+#define SQLITE_RECOVER_FREELIST_CORRUPT 2
+#define SQLITE_RECOVER_ROWIDS 3
+#define SQLITE_RECOVER_SLOWINDEXES 4
+
+/*
+** Perform a unit of work towards the recovery operation. This function
+** must normally be called multiple times to complete database recovery.
+**
+** If no error occurs but the recovery operation is not completed, this
+** function returns SQLITE_OK. If recovery has been completed successfully
+** then SQLITE_DONE is returned. If an error has occurred, then an SQLite
+** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not
+** considered an error if some or all of the data cannot be recovered
+** due to database corruption.
+**
+** Once sqlite3_recover_step() has returned a value other than SQLITE_OK,
+** all further such calls on the same recover handle are no-ops that return
+** the same non-SQLITE_OK value.
+*/
+int sqlite3_recover_step(sqlite3_recover*);
+
+/*
+** Run the recovery operation to completion. Return SQLITE_OK if successful,
+** or an SQLite error code otherwise. Calling this function is the same
+** as executing:
+**
+** while( SQLITE_OK==sqlite3_recover_step(p) );
+** return sqlite3_recover_errcode(p);
+*/
+int sqlite3_recover_run(sqlite3_recover*);
+
+/*
+** If an error has been encountered during a prior call to
+** sqlite3_recover_step(), then this function attempts to return a
+** pointer to a buffer containing an English language explanation of
+** the error. If no error message is available, or if an out-of memory
+** error occurs while attempting to allocate a buffer in which to format
+** the error message, NULL is returned.
+**
+** The returned buffer remains valid until the sqlite3_recover handle is
+** destroyed using sqlite3_recover_finish().
+*/
+const char *sqlite3_recover_errmsg(sqlite3_recover*);
+
+/*
+** If this function is called on an sqlite3_recover handle after
+** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK.
+*/
+int sqlite3_recover_errcode(sqlite3_recover*);
+
+/*
+** Clean up a recovery object created by a call to sqlite3_recover_init().
+** The results of using a recovery object with any API after it has been
+** passed to this function are undefined.
+**
+** This function returns the same value as sqlite3_recover_errcode().
+*/
+int sqlite3_recover_finish(sqlite3_recover*);
+
+
+#ifdef __cplusplus
+} /* end of the 'extern "C"' block */
+#endif
+
+#endif /* ifndef _SQLITE_RECOVER_H */
+
+/************************* End ../ext/recover/sqlite3recover.h ********************/
+/************************* Begin ../ext/recover/sqlite3recover.c ******************/
+/*
+** 2022-08-27
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+*/
+
+
+/* #include "sqlite3recover.h" */
+#include
+#include
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Declaration for public API function in file dbdata.c. This may be called
+** with NULL as the final two arguments to register the sqlite_dbptr and
+** sqlite_dbdata virtual tables with a database handle.
+*/
+#ifdef _WIN32
+
+#endif
+int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*);
+
+/* typedef unsigned int u32; */
+/* typedef unsigned char u8; */
+/* typedef sqlite3_int64 i64; */
+
+typedef struct RecoverTable RecoverTable;
+typedef struct RecoverColumn RecoverColumn;
+
+/*
+** When recovering rows of data that can be associated with table
+** definitions recovered from the sqlite_schema table, each table is
+** represented by an instance of the following object.
+**
+** iRoot:
+** The root page in the original database. Not necessarily (and usually
+** not) the same in the recovered database.
+**
+** zTab:
+** Name of the table.
+**
+** nCol/aCol[]:
+** aCol[] is an array of nCol columns. In the order in which they appear
+** in the table.
+**
+** bIntkey:
+** Set to true for intkey tables, false for WITHOUT ROWID.
+**
+** iRowidBind:
+** Each column in the aCol[] array has associated with it the index of
+** the bind parameter its values will be bound to in the INSERT statement
+** used to construct the output database. If the table does has a rowid
+** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the
+** index of the bind paramater to which the rowid value should be bound.
+** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY
+** KEY column, then the rowid value should be bound to the index associated
+** with the column.
+**
+** pNext:
+** All RecoverTable objects used by the recovery operation are allocated
+** and populated as part of creating the recovered database schema in
+** the output database, before any non-schema data are recovered. They
+** are then stored in a singly-linked list linked by this variable beginning
+** at sqlite3_recover.pTblList.
+*/
+struct RecoverTable {
+ u32 iRoot; /* Root page in original database */
+ char *zTab; /* Name of table */
+ int nCol; /* Number of columns in table */
+ RecoverColumn *aCol; /* Array of columns */
+ int bIntkey; /* True for intkey, false for without rowid */
+ int iRowidBind; /* If >0, bind rowid to INSERT here */
+ RecoverTable *pNext;
+};
+
+/*
+** Each database column is represented by an instance of the following object
+** stored in the RecoverTable.aCol[] array of the associated table.
+**
+** iField:
+** The index of the associated field within database records. Or -1 if
+** there is no associated field (e.g. for virtual generated columns).
+**
+** iBind:
+** The bind index of the INSERT statement to bind this columns values
+** to. Or 0 if there is no such index (iff (iField<0)).
+**
+** bIPK:
+** True if this is the INTEGER PRIMARY KEY column.
+**
+** zCol:
+** Name of column.
+**
+** eHidden:
+** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each).
+*/
+struct RecoverColumn {
+ int iField; /* Field in record on disk */
+ int iBind; /* Binding to use in INSERT */
+ int bIPK; /* True for IPK column */
+ char *zCol;
+ int eHidden;
+};
+
+#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */
+#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */
+#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */
+#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */
+
+/*
+** Bitmap object used to track pages in the input database. Allocated
+** and manipulated only by the following functions:
+**
+** recoverBitmapAlloc()
+** recoverBitmapFree()
+** recoverBitmapSet()
+** recoverBitmapQuery()
+**
+** nPg:
+** Largest page number that may be stored in the bitmap. The range
+** of valid keys is 1 to nPg, inclusive.
+**
+** aElem[]:
+** Array large enough to contain a bit for each key. For key value
+** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32].
+** In other words, the following is true if bit iKey is set, or
+** false if it is clear:
+**
+** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0
+*/
+typedef struct RecoverBitmap RecoverBitmap;
+struct RecoverBitmap {
+ i64 nPg; /* Size of bitmap */
+ u32 aElem[1]; /* Array of 32-bit bitmasks */
+};
+
+/*
+** State variables (part of the sqlite3_recover structure) used while
+** recovering data for tables identified in the recovered schema (state
+** RECOVER_STATE_WRITING).
+*/
+typedef struct RecoverStateW1 RecoverStateW1;
+struct RecoverStateW1 {
+ sqlite3_stmt *pTbls;
+ sqlite3_stmt *pSel;
+ sqlite3_stmt *pInsert;
+ int nInsert;
+
+ RecoverTable *pTab; /* Table currently being written */
+ int nMax; /* Max column count in any schema table */
+ sqlite3_value **apVal; /* Array of nMax values */
+ int nVal; /* Number of valid entries in apVal[] */
+ int bHaveRowid;
+ i64 iRowid;
+ i64 iPrevPage;
+ int iPrevCell;
+};
+
+/*
+** State variables (part of the sqlite3_recover structure) used while
+** recovering data destined for the lost and found table (states
+** RECOVER_STATE_LOSTANDFOUND[123]).
+*/
+typedef struct RecoverStateLAF RecoverStateLAF;
+struct RecoverStateLAF {
+ RecoverBitmap *pUsed;
+ i64 nPg; /* Size of db in pages */
+ sqlite3_stmt *pAllAndParent;
+ sqlite3_stmt *pMapInsert;
+ sqlite3_stmt *pMaxField;
+ sqlite3_stmt *pUsedPages;
+ sqlite3_stmt *pFindRoot;
+ sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */
+ sqlite3_stmt *pAllPage;
+ sqlite3_stmt *pPageData;
+ sqlite3_value **apVal;
+ int nMaxField;
+};
+
+/*
+** Main recover handle structure.
+*/
+struct sqlite3_recover {
+ /* Copies of sqlite3_recover_init[_sql]() parameters */
+ sqlite3 *dbIn; /* Input database */
+ char *zDb; /* Name of input db ("main" etc.) */
+ char *zUri; /* URI for output database */
+ void *pSqlCtx; /* SQL callback context */
+ int (*xSql)(void*,const char*); /* Pointer to SQL callback function */
+
+ /* Values configured by sqlite3_recover_config() */
+ char *zStateDb; /* State database to use (or NULL) */
+ char *zLostAndFound; /* Name of lost-and-found table (or NULL) */
+ int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */
+ int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */
+ int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */
+
+ int pgsz;
+ int detected_pgsz;
+ int nReserve;
+ u8 *pPage1Disk;
+ u8 *pPage1Cache;
+
+ /* Error code and error message */
+ int errCode; /* For sqlite3_recover_errcode() */
+ char *zErrMsg; /* For sqlite3_recover_errmsg() */
+
+ int eState;
+ int bCloseTransaction;
+
+ /* Variables used with eState==RECOVER_STATE_WRITING */
+ RecoverStateW1 w1;
+
+ /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */
+ RecoverStateLAF laf;
+
+ /* Fields used within sqlite3_recover_run() */
+ sqlite3 *dbOut; /* Output database */
+ sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */
+ RecoverTable *pTblList; /* List of tables recovered from schema */
+};
+
+/*
+** The various states in which an sqlite3_recover object may exist:
+**
+** RECOVER_STATE_INIT:
+** The object is initially created in this state. sqlite3_recover_step()
+** has yet to be called. This is the only state in which it is permitted
+** to call sqlite3_recover_config().
+**
+** RECOVER_STATE_WRITING:
+**
+** RECOVER_STATE_LOSTANDFOUND1:
+** State to populate the bitmap of pages used by other tables or the
+** database freelist.
+**
+** RECOVER_STATE_LOSTANDFOUND2:
+** Populate the recovery.map table - used to figure out a "root" page
+** for each lost page from in the database from which records are
+** extracted.
+**
+** RECOVER_STATE_LOSTANDFOUND3:
+** Populate the lost-and-found table itself.
+*/
+#define RECOVER_STATE_INIT 0
+#define RECOVER_STATE_WRITING 1
+#define RECOVER_STATE_LOSTANDFOUND1 2
+#define RECOVER_STATE_LOSTANDFOUND2 3
+#define RECOVER_STATE_LOSTANDFOUND3 4
+#define RECOVER_STATE_SCHEMA2 5
+#define RECOVER_STATE_DONE 6
+
+
+/*
+** Global variables used by this extension.
+*/
+typedef struct RecoverGlobal RecoverGlobal;
+struct RecoverGlobal {
+ const sqlite3_io_methods *pMethods;
+ sqlite3_recover *p;
+};
+static RecoverGlobal recover_g;
+
+/*
+** Use this static SQLite mutex to protect the globals during the
+** first call to sqlite3_recover_step().
+*/
+#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2
+
+
+/*
+** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid).
+*/
+#define RECOVER_ROWID_DEFAULT 1
+
+/*
+** Mutex handling:
+**
+** recoverEnterMutex() - Enter the recovery mutex
+** recoverLeaveMutex() - Leave the recovery mutex
+** recoverAssertMutexHeld() - Assert that the recovery mutex is held
+*/
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0
+# define recoverEnterMutex()
+# define recoverLeaveMutex()
+#else
+static void recoverEnterMutex(void){
+ sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID));
+}
+static void recoverLeaveMutex(void){
+ sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID));
+}
+#endif
+#if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG)
+static void recoverAssertMutexHeld(void){
+ assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) );
+}
+#else
+# define recoverAssertMutexHeld()
+#endif
+
+
+/*
+** Like strlen(). But handles NULL pointer arguments.
+*/
+static int recoverStrlen(const char *zStr){
+ if( zStr==0 ) return 0;
+ return (int)(strlen(zStr)&0x7fffffff);
+}
+
+/*
+** This function is a no-op if the recover handle passed as the first
+** argument already contains an error (if p->errCode!=SQLITE_OK).
+**
+** Otherwise, an attempt is made to allocate, zero and return a buffer nByte
+** bytes in size. If successful, a pointer to the new buffer is returned. Or,
+** if an OOM error occurs, NULL is returned and the handle error code
+** (p->errCode) set to SQLITE_NOMEM.
+*/
+static void *recoverMalloc(sqlite3_recover *p, i64 nByte){
+ void *pRet = 0;
+ assert( nByte>0 );
+ if( p->errCode==SQLITE_OK ){
+ pRet = sqlite3_malloc64(nByte);
+ if( pRet ){
+ memset(pRet, 0, nByte);
+ }else{
+ p->errCode = SQLITE_NOMEM;
+ }
+ }
+ return pRet;
+}
+
+/*
+** Set the error code and error message for the recover handle passed as
+** the first argument. The error code is set to the value of parameter
+** errCode.
+**
+** Parameter zFmt must be a printf() style formatting string. The handle
+** error message is set to the result of using any trailing arguments for
+** parameter substitutions in the formatting string.
+**
+** For example:
+**
+** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename);
+*/
+static int recoverError(
+ sqlite3_recover *p,
+ int errCode,
+ const char *zFmt, ...
+){
+ char *z = 0;
+ va_list ap;
+ va_start(ap, zFmt);
+ if( zFmt ){
+ z = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+ }
+ sqlite3_free(p->zErrMsg);
+ p->zErrMsg = z;
+ p->errCode = errCode;
+ return errCode;
+}
+
+
+/*
+** This function is a no-op if p->errCode is initially other than SQLITE_OK.
+** In this case it returns NULL.
+**
+** Otherwise, an attempt is made to allocate and return a bitmap object
+** large enough to store a bit for all page numbers between 1 and nPg,
+** inclusive. The bitmap is initially zeroed.
+*/
+static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){
+ int nElem = (nPg+1+31) / 32;
+ int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32);
+ RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte);
+
+ if( pRet ){
+ pRet->nPg = nPg;
+ }
+ return pRet;
+}
+
+/*
+** Free a bitmap object allocated by recoverBitmapAlloc().
+*/
+static void recoverBitmapFree(RecoverBitmap *pMap){
+ sqlite3_free(pMap);
+}
+
+/*
+** Set the bit associated with page iPg in bitvec pMap.
+*/
+static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){
+ if( iPg<=pMap->nPg ){
+ int iElem = (iPg / 32);
+ int iBit = (iPg % 32);
+ pMap->aElem[iElem] |= (((u32)1) << iBit);
+ }
+}
+
+/*
+** Query bitmap object pMap for the state of the bit associated with page
+** iPg. Return 1 if it is set, or 0 otherwise.
+*/
+static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){
+ int ret = 1;
+ if( iPg<=pMap->nPg && iPg>0 ){
+ int iElem = (iPg / 32);
+ int iBit = (iPg % 32);
+ ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0;
+ }
+ return ret;
+}
+
+/*
+** Set the recover handle error to the error code and message returned by
+** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database
+** handle db.
+*/
+static int recoverDbError(sqlite3_recover *p, sqlite3 *db){
+ return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db));
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK).
+**
+** Otherwise, it attempts to prepare the SQL statement in zSql against
+** database handle db. If successful, the statement handle is returned.
+** Or, if an error occurs, NULL is returned and an error left in the
+** recover handle.
+*/
+static sqlite3_stmt *recoverPrepare(
+ sqlite3_recover *p,
+ sqlite3 *db,
+ const char *zSql
+){
+ sqlite3_stmt *pStmt = 0;
+ if( p->errCode==SQLITE_OK ){
+ if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){
+ recoverDbError(p, db);
+ }
+ }
+ return pStmt;
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK).
+**
+** Otherwise, argument zFmt is used as a printf() style format string,
+** along with any trailing arguments, to create an SQL statement. This
+** SQL statement is prepared against database handle db and, if successful,
+** the statment handle returned. Or, if an error occurs - either during
+** the printf() formatting or when preparing the resulting SQL - an
+** error code and message are left in the recover handle.
+*/
+static sqlite3_stmt *recoverPreparePrintf(
+ sqlite3_recover *p,
+ sqlite3 *db,
+ const char *zFmt, ...
+){
+ sqlite3_stmt *pStmt = 0;
+ if( p->errCode==SQLITE_OK ){
+ va_list ap;
+ char *z;
+ va_start(ap, zFmt);
+ z = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+ if( z==0 ){
+ p->errCode = SQLITE_NOMEM;
+ }else{
+ pStmt = recoverPrepare(p, db, z);
+ sqlite3_free(z);
+ }
+ }
+ return pStmt;
+}
+
+/*
+** Reset SQLite statement handle pStmt. If the call to sqlite3_reset()
+** indicates that an error occurred, and there is not already an error
+** in the recover handle passed as the first argument, set the error
+** code and error message appropriately.
+**
+** This function returns a copy of the statement handle pointer passed
+** as the second argument.
+*/
+static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){
+ int rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){
+ recoverDbError(p, sqlite3_db_handle(pStmt));
+ }
+ return pStmt;
+}
+
+/*
+** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset()
+** indicates that an error occurred, and there is not already an error
+** in the recover handle passed as the first argument, set the error
+** code and error message appropriately.
+*/
+static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){
+ sqlite3 *db = sqlite3_db_handle(pStmt);
+ int rc = sqlite3_finalize(pStmt);
+ if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){
+ recoverDbError(p, db);
+ }
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this
+** case.
+**
+** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK.
+** Or, if an error occurs, leave an error code and message in the recover
+** handle and return a copy of the error code.
+*/
+static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){
+ if( p->errCode==SQLITE_OK ){
+ int rc = sqlite3_exec(db, zSql, 0, 0, 0);
+ if( rc ){
+ recoverDbError(p, db);
+ }
+ }
+ return p->errCode;
+}
+
+/*
+** Bind the value pVal to parameter iBind of statement pStmt. Leave an
+** error in the recover handle passed as the first argument if an error
+** (e.g. an OOM) occurs.
+*/
+static void recoverBindValue(
+ sqlite3_recover *p,
+ sqlite3_stmt *pStmt,
+ int iBind,
+ sqlite3_value *pVal
+){
+ if( p->errCode==SQLITE_OK ){
+ int rc = sqlite3_bind_value(pStmt, iBind, pVal);
+ if( rc ) recoverError(p, rc, 0);
+ }
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). NULL is returned in this case.
+**
+** Otherwise, an attempt is made to interpret zFmt as a printf() style
+** formatting string and the result of using the trailing arguments for
+** parameter substitution with it written into a buffer obtained from
+** sqlite3_malloc(). If successful, a pointer to the buffer is returned.
+** It is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** Or, if an error occurs, an error code and message is left in the recover
+** handle and NULL returned.
+*/
+static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){
+ va_list ap;
+ char *z;
+ va_start(ap, zFmt);
+ z = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+ if( p->errCode==SQLITE_OK ){
+ if( z==0 ) p->errCode = SQLITE_NOMEM;
+ }else{
+ sqlite3_free(z);
+ z = 0;
+ }
+ return z;
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). Zero is returned in this case.
+**
+** Otherwise, execute "PRAGMA page_count" against the input database. If
+** successful, return the integer result. Or, if an error occurs, leave an
+** error code and error message in the sqlite3_recover handle and return
+** zero.
+*/
+static i64 recoverPageCount(sqlite3_recover *p){
+ i64 nPg = 0;
+ if( p->errCode==SQLITE_OK ){
+ sqlite3_stmt *pStmt = 0;
+ pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb);
+ if( pStmt ){
+ sqlite3_step(pStmt);
+ nPg = sqlite3_column_int64(pStmt, 0);
+ }
+ recoverFinalize(p, pStmt);
+ }
+ return nPg;
+}
+
+/*
+** Implementation of SQL scalar function "read_i32". The first argument to
+** this function must be a blob. The second a non-negative integer. This
+** function reads and returns a 32-bit big-endian integer from byte
+** offset (4*) of the blob.
+**
+** SELECT read_i32(, )
+*/
+static void recoverReadI32(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ const unsigned char *pBlob;
+ int nBlob;
+ int iInt;
+
+ assert( argc==2 );
+ nBlob = sqlite3_value_bytes(argv[0]);
+ pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]);
+ iInt = sqlite3_value_int(argv[1]) & 0xFFFF;
+
+ if( (iInt+1)*4<=nBlob ){
+ const unsigned char *a = &pBlob[iInt*4];
+ i64 iVal = ((i64)a[0]<<24)
+ + ((i64)a[1]<<16)
+ + ((i64)a[2]<< 8)
+ + ((i64)a[3]<< 0);
+ sqlite3_result_int64(context, iVal);
+ }
+}
+
+/*
+** Implementation of SQL scalar function "page_is_used". This function
+** is used as part of the procedure for locating orphan rows for the
+** lost-and-found table, and it depends on those routines having populated
+** the sqlite3_recover.laf.pUsed variable.
+**
+** The only argument to this function is a page-number. It returns true
+** if the page has already been used somehow during data recovery, or false
+** otherwise.
+**
+** SELECT page_is_used();
+*/
+static void recoverPageIsUsed(
+ sqlite3_context *pCtx,
+ int nArg,
+ sqlite3_value **apArg
+){
+ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
+ i64 pgno = sqlite3_value_int64(apArg[0]);
+ assert( nArg==1 );
+ sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno));
+}
+
+/*
+** The implementation of a user-defined SQL function invoked by the
+** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages
+** of the database being recovered.
+**
+** This function always takes a single integer argument. If the argument
+** is zero, then the value returned is the number of pages in the db being
+** recovered. If the argument is greater than zero, it is a page number.
+** The value returned in this case is an SQL blob containing the data for
+** the identified page of the db being recovered. e.g.
+**
+** SELECT getpage(0); -- return number of pages in db
+** SELECT getpage(4); -- return page 4 of db as a blob of data
+*/
+static void recoverGetPage(
+ sqlite3_context *pCtx,
+ int nArg,
+ sqlite3_value **apArg
+){
+ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
+ i64 pgno = sqlite3_value_int64(apArg[0]);
+ sqlite3_stmt *pStmt = 0;
+
+ assert( nArg==1 );
+ if( pgno==0 ){
+ i64 nPg = recoverPageCount(p);
+ sqlite3_result_int64(pCtx, nPg);
+ return;
+ }else{
+ if( p->pGetPage==0 ){
+ pStmt = p->pGetPage = recoverPreparePrintf(
+ p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb
+ );
+ }else if( p->errCode==SQLITE_OK ){
+ pStmt = p->pGetPage;
+ }
+
+ if( pStmt ){
+ sqlite3_bind_int64(pStmt, 1, pgno);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ const u8 *aPg;
+ int nPg;
+ assert( p->errCode==SQLITE_OK );
+ aPg = sqlite3_column_blob(pStmt, 0);
+ nPg = sqlite3_column_bytes(pStmt, 0);
+ if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){
+ aPg = p->pPage1Disk;
+ }
+ sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT);
+ }
+ recoverReset(p, pStmt);
+ }
+ }
+
+ if( p->errCode ){
+ if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1);
+ sqlite3_result_error_code(pCtx, p->errCode);
+ }
+}
+
+/*
+** Find a string that is not found anywhere in z[]. Return a pointer
+** to that string.
+**
+** Try to use zA and zB first. If both of those are already found in z[]
+** then make up some string and store it in the buffer zBuf.
+*/
+static const char *recoverUnusedString(
+ const char *z, /* Result must not appear anywhere in z */
+ const char *zA, const char *zB, /* Try these first */
+ char *zBuf /* Space to store a generated string */
+){
+ unsigned i = 0;
+ if( strstr(z, zA)==0 ) return zA;
+ if( strstr(z, zB)==0 ) return zB;
+ do{
+ sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++);
+ }while( strstr(z,zBuf)!=0 );
+ return zBuf;
+}
+
+/*
+** Implementation of scalar SQL function "escape_crnl". The argument passed to
+** this function is the output of built-in function quote(). If the first
+** character of the input is "'", indicating that the value passed to quote()
+** was a text value, then this function searches the input for "\n" and "\r"
+** characters and adds a wrapper similar to the following:
+**
+** replace(replace(, '\n', char(10), '\r', char(13));
+**
+** Or, if the first character of the input is not "'", then a copy of the input
+** is returned.
+*/
+static void recoverEscapeCrnl(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ const char *zText = (const char*)sqlite3_value_text(argv[0]);
+ if( zText && zText[0]=='\'' ){
+ int nText = sqlite3_value_bytes(argv[0]);
+ int i;
+ char zBuf1[20];
+ char zBuf2[20];
+ const char *zNL = 0;
+ const char *zCR = 0;
+ int nCR = 0;
+ int nNL = 0;
+
+ for(i=0; zText[i]; i++){
+ if( zNL==0 && zText[i]=='\n' ){
+ zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1);
+ nNL = (int)strlen(zNL);
+ }
+ if( zCR==0 && zText[i]=='\r' ){
+ zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2);
+ nCR = (int)strlen(zCR);
+ }
+ }
+
+ if( zNL || zCR ){
+ int iOut = 0;
+ i64 nMax = (nNL > nCR) ? nNL : nCR;
+ i64 nAlloc = nMax * nText + (nMax+64)*2;
+ char *zOut = (char*)sqlite3_malloc64(nAlloc);
+ if( zOut==0 ){
+ sqlite3_result_error_nomem(context);
+ return;
+ }
+
+ if( zNL && zCR ){
+ memcpy(&zOut[iOut], "replace(replace(", 16);
+ iOut += 16;
+ }else{
+ memcpy(&zOut[iOut], "replace(", 8);
+ iOut += 8;
+ }
+ for(i=0; zText[i]; i++){
+ if( zText[i]=='\n' ){
+ memcpy(&zOut[iOut], zNL, nNL);
+ iOut += nNL;
+ }else if( zText[i]=='\r' ){
+ memcpy(&zOut[iOut], zCR, nCR);
+ iOut += nCR;
+ }else{
+ zOut[iOut] = zText[i];
+ iOut++;
+ }
+ }
+
+ if( zNL ){
+ memcpy(&zOut[iOut], ",'", 2); iOut += 2;
+ memcpy(&zOut[iOut], zNL, nNL); iOut += nNL;
+ memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12;
+ }
+ if( zCR ){
+ memcpy(&zOut[iOut], ",'", 2); iOut += 2;
+ memcpy(&zOut[iOut], zCR, nCR); iOut += nCR;
+ memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12;
+ }
+
+ sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT);
+ sqlite3_free(zOut);
+ return;
+ }
+ }
+
+ sqlite3_result_value(context, argv[0]);
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in
+** this case.
+**
+** Otherwise, attempt to populate temporary table "recovery.schema" with the
+** parts of the database schema that can be extracted from the input database.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
+** and error message are left in the recover handle and a copy of the
+** error code returned. It is not considered an error if part of all of
+** the database schema cannot be recovered due to corruption.
+*/
+static int recoverCacheSchema(sqlite3_recover *p){
+ return recoverExec(p, p->dbOut,
+ "WITH RECURSIVE pages(p) AS ("
+ " SELECT 1"
+ " UNION"
+ " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p"
+ ")"
+ "INSERT INTO recovery.schema SELECT"
+ " max(CASE WHEN field=0 THEN value ELSE NULL END),"
+ " max(CASE WHEN field=1 THEN value ELSE NULL END),"
+ " max(CASE WHEN field=2 THEN value ELSE NULL END),"
+ " max(CASE WHEN field=3 THEN value ELSE NULL END),"
+ " max(CASE WHEN field=4 THEN value ELSE NULL END)"
+ "FROM sqlite_dbdata('getpage()') WHERE pgno IN ("
+ " SELECT p FROM pages"
+ ") GROUP BY pgno, cell"
+ );
+}
+
+/*
+** If this recover handle is not in SQL callback mode (i.e. was not created
+** using sqlite3_recover_init_sql()) of if an error has already occurred,
+** this function is a no-op. Otherwise, issue a callback with SQL statement
+** zSql as the parameter.
+**
+** If the callback returns non-zero, set the recover handle error code to
+** the value returned (so that the caller will abandon processing).
+*/
+static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){
+ if( p->errCode==SQLITE_OK && p->xSql ){
+ int res = p->xSql(p->pSqlCtx, zSql);
+ if( res ){
+ recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res);
+ }
+ }
+}
+
+/*
+** Transfer the following settings from the input database to the output
+** database:
+**
+** + page-size,
+** + auto-vacuum settings,
+** + database encoding,
+** + user-version (PRAGMA user_version), and
+** + application-id (PRAGMA application_id), and
+*/
+static void recoverTransferSettings(sqlite3_recover *p){
+ const char *aPragma[] = {
+ "encoding",
+ "page_size",
+ "auto_vacuum",
+ "user_version",
+ "application_id"
+ };
+ int ii;
+
+ /* Truncate the output database to 0 pages in size. This is done by
+ ** opening a new, empty, temp db, then using the backup API to clobber
+ ** any existing output db with a copy of it. */
+ if( p->errCode==SQLITE_OK ){
+ sqlite3 *db2 = 0;
+ int rc = sqlite3_open("", &db2);
+ if( rc!=SQLITE_OK ){
+ recoverDbError(p, db2);
+ return;
+ }
+
+ for(ii=0; iidbIn, "PRAGMA %Q.%s", p->zDb, zPrag);
+ if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){
+ const char *zArg = (const char*)sqlite3_column_text(p1, 0);
+ char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg);
+ recoverSqlCallback(p, z2);
+ recoverExec(p, db2, z2);
+ sqlite3_free(z2);
+ if( zArg==0 ){
+ recoverError(p, SQLITE_NOMEM, 0);
+ }
+ }
+ recoverFinalize(p, p1);
+ }
+ recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;");
+
+ if( p->errCode==SQLITE_OK ){
+ sqlite3 *db = p->dbOut;
+ sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main");
+ if( pBackup ){
+ sqlite3_backup_step(pBackup, -1);
+ p->errCode = sqlite3_backup_finish(pBackup);
+ }else{
+ recoverDbError(p, db);
+ }
+ }
+
+ sqlite3_close(db2);
+ }
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in
+** this case.
+**
+** Otherwise, an attempt is made to open the output database, attach
+** and create the schema of the temporary database used to store
+** intermediate data, and to register all required user functions and
+** virtual table modules with the output handle.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
+** and error message are left in the recover handle and a copy of the
+** error code returned.
+*/
+static int recoverOpenOutput(sqlite3_recover *p){
+ struct Func {
+ const char *zName;
+ int nArg;
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value **);
+ } aFunc[] = {
+ { "getpage", 1, recoverGetPage },
+ { "page_is_used", 1, recoverPageIsUsed },
+ { "read_i32", 2, recoverReadI32 },
+ { "escape_crnl", 1, recoverEscapeCrnl },
+ };
+
+ const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE;
+ sqlite3 *db = 0; /* New database handle */
+ int ii; /* For iterating through aFunc[] */
+
+ assert( p->dbOut==0 );
+
+ if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){
+ recoverDbError(p, db);
+ }
+
+ /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules.
+ ** These two are registered with the output database handle - this
+ ** module depends on the input handle supporting the sqlite_dbpage
+ ** virtual table only. */
+ if( p->errCode==SQLITE_OK ){
+ p->errCode = sqlite3_dbdata_init(db, 0, 0);
+ }
+
+ /* Register the custom user-functions with the output handle. */
+ for(ii=0; p->errCode==SQLITE_OK && iierrCode = sqlite3_create_function(db, aFunc[ii].zName,
+ aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0
+ );
+ }
+
+ p->dbOut = db;
+ return p->errCode;
+}
+
+/*
+** Attach the auxiliary database 'recovery' to the output database handle.
+** This temporary database is used during the recovery process and then
+** discarded.
+*/
+static void recoverOpenRecovery(sqlite3_recover *p){
+ char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb);
+ recoverExec(p, p->dbOut, zSql);
+ recoverExec(p, p->dbOut,
+ "PRAGMA writable_schema = 1;"
+ "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);"
+ "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);"
+ );
+ sqlite3_free(zSql);
+}
+
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK).
+**
+** Otherwise, argument zName must be the name of a table that has just been
+** created in the output database. This function queries the output db
+** for the schema of said table, and creates a RecoverTable object to
+** store the schema in memory. The new RecoverTable object is linked into
+** the list at sqlite3_recover.pTblList.
+**
+** Parameter iRoot must be the root page of table zName in the INPUT
+** database.
+*/
+static void recoverAddTable(
+ sqlite3_recover *p,
+ const char *zName, /* Name of table created in output db */
+ i64 iRoot /* Root page of same table in INPUT db */
+){
+ sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut,
+ "PRAGMA table_xinfo(%Q)", zName
+ );
+
+ if( pStmt ){
+ int iPk = -1;
+ int iBind = 1;
+ RecoverTable *pNew = 0;
+ int nCol = 0;
+ int nName = recoverStrlen(zName);
+ int nByte = 0;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ nCol++;
+ nByte += (sqlite3_column_bytes(pStmt, 1)+1);
+ }
+ nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1;
+ recoverReset(p, pStmt);
+
+ pNew = recoverMalloc(p, nByte);
+ if( pNew ){
+ int i = 0;
+ int iField = 0;
+ char *csr = 0;
+ pNew->aCol = (RecoverColumn*)&pNew[1];
+ pNew->zTab = csr = (char*)&pNew->aCol[nCol];
+ pNew->nCol = nCol;
+ pNew->iRoot = iRoot;
+ memcpy(csr, zName, nName);
+ csr += nName+1;
+
+ for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){
+ int iPKF = sqlite3_column_int(pStmt, 5);
+ int n = sqlite3_column_bytes(pStmt, 1);
+ const char *z = (const char*)sqlite3_column_text(pStmt, 1);
+ const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
+ int eHidden = sqlite3_column_int(pStmt, 6);
+
+ if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i;
+ if( iPKF>1 ) iPk = -2;
+ pNew->aCol[i].zCol = csr;
+ pNew->aCol[i].eHidden = eHidden;
+ if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){
+ pNew->aCol[i].iField = -1;
+ }else{
+ pNew->aCol[i].iField = iField++;
+ }
+ if( eHidden!=RECOVER_EHIDDEN_VIRTUAL
+ && eHidden!=RECOVER_EHIDDEN_STORED
+ ){
+ pNew->aCol[i].iBind = iBind++;
+ }
+ memcpy(csr, z, n);
+ csr += (n+1);
+ }
+
+ pNew->pNext = p->pTblList;
+ p->pTblList = pNew;
+ pNew->bIntkey = 1;
+ }
+
+ recoverFinalize(p, pStmt);
+
+ pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName);
+ while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
+ int iField = sqlite3_column_int(pStmt, 0);
+ int iCol = sqlite3_column_int(pStmt, 1);
+
+ assert( iFieldnCol && iColnCol );
+ pNew->aCol[iCol].iField = iField;
+
+ pNew->bIntkey = 0;
+ iPk = -2;
+ }
+ recoverFinalize(p, pStmt);
+
+ if( p->errCode==SQLITE_OK ){
+ if( iPk>=0 ){
+ pNew->aCol[iPk].bIPK = 1;
+ }else if( pNew->bIntkey ){
+ pNew->iRowidBind = iBind++;
+ }
+ }
+ }
+}
+
+/*
+** This function is called after recoverCacheSchema() has cached those parts
+** of the input database schema that could be recovered in temporary table
+** "recovery.schema". This function creates in the output database copies
+** of all parts of that schema that must be created before the tables can
+** be populated. Specifically, this means:
+**
+** * all tables that are not VIRTUAL, and
+** * UNIQUE indexes.
+**
+** If the recovery handle uses SQL callbacks, then callbacks containing
+** the associated "CREATE TABLE" and "CREATE INDEX" statements are made.
+**
+** Additionally, records are added to the sqlite_schema table of the
+** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE
+** records are written directly to sqlite_schema, not actually executed.
+** If the handle is in SQL callback mode, then callbacks are invoked
+** with equivalent SQL statements.
+*/
+static int recoverWriteSchema1(sqlite3_recover *p){
+ sqlite3_stmt *pSelect = 0;
+ sqlite3_stmt *pTblname = 0;
+
+ pSelect = recoverPrepare(p, p->dbOut,
+ "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS ("
+ " SELECT rootpage, name, sql, "
+ " type='table', "
+ " sql LIKE 'create virtual%',"
+ " (type='index' AND (sql LIKE '%unique%' OR ?1))"
+ " FROM recovery.schema"
+ ")"
+ "SELECT rootpage, tbl, isVirtual, name, sql"
+ " FROM dbschema "
+ " WHERE tbl OR isIndex"
+ " ORDER BY tbl DESC, name=='sqlite_sequence' DESC"
+ );
+
+ pTblname = recoverPrepare(p, p->dbOut,
+ "SELECT name FROM sqlite_schema "
+ "WHERE type='table' ORDER BY rowid DESC LIMIT 1"
+ );
+
+ if( pSelect ){
+ sqlite3_bind_int(pSelect, 1, p->bSlowIndexes);
+ while( sqlite3_step(pSelect)==SQLITE_ROW ){
+ i64 iRoot = sqlite3_column_int64(pSelect, 0);
+ int bTable = sqlite3_column_int(pSelect, 1);
+ int bVirtual = sqlite3_column_int(pSelect, 2);
+ const char *zName = (const char*)sqlite3_column_text(pSelect, 3);
+ const char *zSql = (const char*)sqlite3_column_text(pSelect, 4);
+ char *zFree = 0;
+ int rc = SQLITE_OK;
+
+ if( bVirtual ){
+ zSql = (const char*)(zFree = recoverMPrintf(p,
+ "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)",
+ zName, zName, zSql
+ ));
+ }
+ rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0);
+ if( rc==SQLITE_OK ){
+ recoverSqlCallback(p, zSql);
+ if( bTable && !bVirtual ){
+ if( SQLITE_ROW==sqlite3_step(pTblname) ){
+ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0);
+ recoverAddTable(p, zTbl, iRoot);
+ }
+ recoverReset(p, pTblname);
+ }
+ }else if( rc!=SQLITE_ERROR ){
+ recoverDbError(p, p->dbOut);
+ }
+ sqlite3_free(zFree);
+ }
+ }
+ recoverFinalize(p, pSelect);
+ recoverFinalize(p, pTblname);
+
+ return p->errCode;
+}
+
+/*
+** This function is called after the output database has been populated. It
+** adds all recovered schema elements that were not created in the output
+** database by recoverWriteSchema1() - everything except for tables and
+** UNIQUE indexes. Specifically:
+**
+** * views,
+** * triggers,
+** * non-UNIQUE indexes.
+**
+** If the recover handle is in SQL callback mode, then equivalent callbacks
+** are issued to create the schema elements.
+*/
+static int recoverWriteSchema2(sqlite3_recover *p){
+ sqlite3_stmt *pSelect = 0;
+
+ pSelect = recoverPrepare(p, p->dbOut,
+ p->bSlowIndexes ?
+ "SELECT rootpage, sql FROM recovery.schema "
+ " WHERE type!='table' AND type!='index'"
+ :
+ "SELECT rootpage, sql FROM recovery.schema "
+ " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')"
+ );
+
+ if( pSelect ){
+ while( sqlite3_step(pSelect)==SQLITE_ROW ){
+ const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
+ int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0);
+ if( rc==SQLITE_OK ){
+ recoverSqlCallback(p, zSql);
+ }else if( rc!=SQLITE_ERROR ){
+ recoverDbError(p, p->dbOut);
+ }
+ }
+ }
+ recoverFinalize(p, pSelect);
+
+ return p->errCode;
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). In this case it returns NULL.
+**
+** Otherwise, if the recover handle is configured to create an output
+** database (was created by sqlite3_recover_init()), then this function
+** prepares and returns an SQL statement to INSERT a new record into table
+** pTab, assuming the first nField fields of a record extracted from disk
+** are valid.
+**
+** For example, if table pTab is:
+**
+** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e);
+**
+** And nField is 4, then the SQL statement prepared and returned is:
+**
+** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3);
+**
+** In this case even though 4 values were extracted from the input db,
+** only 3 are written to the output, as the generated STORED column
+** cannot be written.
+**
+** If the recover handle is in SQL callback mode, then the SQL statement
+** prepared is such that evaluating it returns a single row containing
+** a single text value - itself an SQL statement similar to the above,
+** except with SQL literals in place of the variables. For example:
+**
+** SELECT 'INSERT INTO (a, c, d) VALUES ('
+** || quote(?1) || ', '
+** || quote(?2) || ', '
+** || quote(?3) || ')';
+**
+** In either case, it is the responsibility of the caller to eventually
+** free the statement handle using sqlite3_finalize().
+*/
+static sqlite3_stmt *recoverInsertStmt(
+ sqlite3_recover *p,
+ RecoverTable *pTab,
+ int nField
+){
+ sqlite3_stmt *pRet = 0;
+ const char *zSep = "";
+ const char *zSqlSep = "";
+ char *zSql = 0;
+ char *zFinal = 0;
+ char *zBind = 0;
+ int ii;
+ int bSql = p->xSql ? 1 : 0;
+
+ if( nField<=0 ) return 0;
+
+ assert( nField<=pTab->nCol );
+
+ zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab);
+
+ if( pTab->iRowidBind ){
+ assert( pTab->bIntkey );
+ zSql = recoverMPrintf(p, "%z_rowid_", zSql);
+ if( bSql ){
+ zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind);
+ }else{
+ zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind);
+ }
+ zSqlSep = "||', '||";
+ zSep = ", ";
+ }
+
+ for(ii=0; iiaCol[ii].eHidden;
+ if( eHidden!=RECOVER_EHIDDEN_VIRTUAL
+ && eHidden!=RECOVER_EHIDDEN_STORED
+ ){
+ assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 );
+ zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol);
+
+ if( bSql ){
+ zBind = recoverMPrintf(p,
+ "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind
+ );
+ zSqlSep = "||', '||";
+ }else{
+ zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind);
+ }
+ zSep = ", ";
+ }
+ }
+
+ if( bSql ){
+ zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'",
+ zSql, zBind
+ );
+ }else{
+ zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind);
+ }
+
+ pRet = recoverPrepare(p, p->dbOut, zFinal);
+ sqlite3_free(zSql);
+ sqlite3_free(zBind);
+ sqlite3_free(zFinal);
+
+ return pRet;
+}
+
+
+/*
+** Search the list of RecoverTable objects at p->pTblList for one that
+** has root page iRoot in the input database. If such an object is found,
+** return a pointer to it. Otherwise, return NULL.
+*/
+static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){
+ RecoverTable *pRet = 0;
+ for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext);
+ return pRet;
+}
+
+/*
+** This function attempts to create a lost and found table within the
+** output db. If successful, it returns a pointer to a buffer containing
+** the name of the new table. It is the responsibility of the caller to
+** eventually free this buffer using sqlite3_free().
+**
+** If an error occurs, NULL is returned and an error code and error
+** message left in the recover handle.
+*/
+static char *recoverLostAndFoundCreate(
+ sqlite3_recover *p, /* Recover object */
+ int nField /* Number of column fields in new table */
+){
+ char *zTbl = 0;
+ sqlite3_stmt *pProbe = 0;
+ int ii = 0;
+
+ pProbe = recoverPrepare(p, p->dbOut,
+ "SELECT 1 FROM sqlite_schema WHERE name=?"
+ );
+ for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){
+ int bFail = 0;
+ if( ii<0 ){
+ zTbl = recoverMPrintf(p, "%s", p->zLostAndFound);
+ }else{
+ zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii);
+ }
+
+ if( p->errCode==SQLITE_OK ){
+ sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pProbe) ){
+ bFail = 1;
+ }
+ recoverReset(p, pProbe);
+ }
+
+ if( bFail ){
+ sqlite3_clear_bindings(pProbe);
+ sqlite3_free(zTbl);
+ zTbl = 0;
+ }
+ }
+ recoverFinalize(p, pProbe);
+
+ if( zTbl ){
+ const char *zSep = 0;
+ char *zField = 0;
+ char *zSql = 0;
+
+ zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, ";
+ for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql);
+ recoverSqlCallback(p, zSql);
+ sqlite3_free(zSql);
+ }else if( p->errCode==SQLITE_OK ){
+ recoverError(
+ p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound
+ );
+ }
+
+ return zTbl;
+}
+
+/*
+** Synthesize and prepare an INSERT statement to write to the lost_and_found
+** table in the output database. The name of the table is zTab, and it has
+** nField c* fields.
+*/
+static sqlite3_stmt *recoverLostAndFoundInsert(
+ sqlite3_recover *p,
+ const char *zTab,
+ int nField
+){
+ int nTotal = nField + 4;
+ int ii;
+ char *zBind = 0;
+ sqlite3_stmt *pRet = 0;
+
+ if( p->xSql==0 ){
+ for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind
+ );
+ }else{
+ const char *zSep = "";
+ for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind
+ );
+ }
+
+ sqlite3_free(zBind);
+ return pRet;
+}
+
+/*
+** Input database page iPg contains data that will be written to the
+** lost-and-found table of the output database. This function attempts
+** to identify the root page of the tree that page iPg belonged to.
+** If successful, it sets output variable (*piRoot) to the page number
+** of the root page and returns SQLITE_OK. Otherwise, if an error occurs,
+** an SQLite error code is returned and the final value of *piRoot
+** undefined.
+*/
+static int recoverLostAndFoundFindRoot(
+ sqlite3_recover *p,
+ i64 iPg,
+ i64 *piRoot
+){
+ RecoverStateLAF *pLaf = &p->laf;
+
+ if( pLaf->pFindRoot==0 ){
+ pLaf->pFindRoot = recoverPrepare(p, p->dbOut,
+ "WITH RECURSIVE p(pgno) AS ("
+ " SELECT ?"
+ " UNION"
+ " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno"
+ ") "
+ "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno "
+ " AND m.parent IS NULL"
+ );
+ }
+ if( p->errCode==SQLITE_OK ){
+ sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg);
+ if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){
+ *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0);
+ }else{
+ *piRoot = iPg;
+ }
+ recoverReset(p, pLaf->pFindRoot);
+ }
+ return p->errCode;
+}
+
+/*
+** Recover data from page iPage of the input database and write it to
+** the lost-and-found table in the output database.
+*/
+static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){
+ RecoverStateLAF *pLaf = &p->laf;
+ sqlite3_value **apVal = pLaf->apVal;
+ sqlite3_stmt *pPageData = pLaf->pPageData;
+ sqlite3_stmt *pInsert = pLaf->pInsert;
+
+ int nVal = -1;
+ int iPrevCell = 0;
+ i64 iRoot = 0;
+ int bHaveRowid = 0;
+ i64 iRowid = 0;
+ int ii = 0;
+
+ if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return;
+ sqlite3_bind_int64(pPageData, 1, iPage);
+ while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){
+ int iCell = sqlite3_column_int64(pPageData, 0);
+ int iField = sqlite3_column_int64(pPageData, 1);
+
+ if( iPrevCell!=iCell && nVal>=0 ){
+ /* Insert the new row */
+ sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */
+ sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */
+ sqlite3_bind_int(pInsert, 3, nVal); /* nfield */
+ if( bHaveRowid ){
+ sqlite3_bind_int64(pInsert, 4, iRowid); /* id */
+ }
+ for(ii=0; iinMaxField ){
+ sqlite3_value *pVal = sqlite3_column_value(pPageData, 2);
+ apVal[iField] = sqlite3_value_dup(pVal);
+ assert( iField==nVal || (nVal==-1 && iField==0) );
+ nVal = iField+1;
+ if( apVal[iField]==0 ){
+ recoverError(p, SQLITE_NOMEM, 0);
+ }
+ }
+
+ iPrevCell = iCell;
+ }
+ recoverReset(p, pPageData);
+
+ for(ii=0; iilaf;
+ if( p->errCode==SQLITE_OK ){
+ if( pLaf->pInsert==0 ){
+ return SQLITE_DONE;
+ }else{
+ if( p->errCode==SQLITE_OK ){
+ int res = sqlite3_step(pLaf->pAllPage);
+ if( res==SQLITE_ROW ){
+ i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0);
+ if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){
+ recoverLostAndFoundOnePage(p, iPage);
+ }
+ }else{
+ recoverReset(p, pLaf->pAllPage);
+ return SQLITE_DONE;
+ }
+ }
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3
+** state - during which the lost-and-found table of the output database
+** is populated with recovered data that can not be assigned to any
+** recovered schema object.
+*/
+static void recoverLostAndFound3Init(sqlite3_recover *p){
+ RecoverStateLAF *pLaf = &p->laf;
+
+ if( pLaf->nMaxField>0 ){
+ char *zTab = 0; /* Name of lost_and_found table */
+
+ zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField);
+ pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField);
+ sqlite3_free(zTab);
+
+ pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut,
+ "WITH RECURSIVE seq(ii) AS ("
+ " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
+ ")"
+ "SELECT ii FROM seq" , p->laf.nPg
+ );
+ pLaf->pPageData = recoverPrepare(p, p->dbOut,
+ "SELECT cell, field, value "
+ "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? "
+ "UNION ALL "
+ "SELECT -1, -1, -1"
+ );
+
+ pLaf->apVal = (sqlite3_value**)recoverMalloc(p,
+ pLaf->nMaxField*sizeof(sqlite3_value*)
+ );
+ }
+}
+
+/*
+** Initialize resources required in RECOVER_STATE_WRITING state - during which
+** tables recovered from the schema of the input database are populated with
+** recovered data.
+*/
+static int recoverWriteDataInit(sqlite3_recover *p){
+ RecoverStateW1 *p1 = &p->w1;
+ RecoverTable *pTbl = 0;
+ int nByte = 0;
+
+ /* Figure out the maximum number of columns for any table in the schema */
+ assert( p1->nMax==0 );
+ for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){
+ if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol;
+ }
+
+ /* Allocate an array of (sqlite3_value*) in which to accumulate the values
+ ** that will be written to the output database in a single row. */
+ nByte = sizeof(sqlite3_value*) * (p1->nMax+1);
+ p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte);
+ if( p1->apVal==0 ) return p->errCode;
+
+ /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT
+ ** to loop through cells that appear to belong to a single table (pSel). */
+ p1->pTbls = recoverPrepare(p, p->dbOut,
+ "SELECT rootpage FROM recovery.schema "
+ " WHERE type='table' AND (sql NOT LIKE 'create virtual%')"
+ " ORDER BY (tbl_name='sqlite_sequence') ASC"
+ );
+ p1->pSel = recoverPrepare(p, p->dbOut,
+ "WITH RECURSIVE pages(page) AS ("
+ " SELECT ?1"
+ " UNION"
+ " SELECT child FROM sqlite_dbptr('getpage()'), pages "
+ " WHERE pgno=page"
+ ") "
+ "SELECT page, cell, field, value "
+ "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno "
+ "UNION ALL "
+ "SELECT 0, 0, 0, 0"
+ );
+
+ return p->errCode;
+}
+
+/*
+** Clean up resources allocated by recoverWriteDataInit() (stuff in
+** sqlite3_recover.w1).
+*/
+static void recoverWriteDataCleanup(sqlite3_recover *p){
+ RecoverStateW1 *p1 = &p->w1;
+ int ii;
+ for(ii=0; iinVal; ii++){
+ sqlite3_value_free(p1->apVal[ii]);
+ }
+ sqlite3_free(p1->apVal);
+ recoverFinalize(p, p1->pInsert);
+ recoverFinalize(p, p1->pTbls);
+ recoverFinalize(p, p1->pSel);
+ memset(p1, 0, sizeof(*p1));
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_WRITING state - during which tables recovered from the
+** schema of the input database are populated with recovered data.
+*/
+static int recoverWriteDataStep(sqlite3_recover *p){
+ RecoverStateW1 *p1 = &p->w1;
+ sqlite3_stmt *pSel = p1->pSel;
+ sqlite3_value **apVal = p1->apVal;
+
+ if( p->errCode==SQLITE_OK && p1->pTab==0 ){
+ if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){
+ i64 iRoot = sqlite3_column_int64(p1->pTbls, 0);
+ p1->pTab = recoverFindTable(p, iRoot);
+
+ recoverFinalize(p, p1->pInsert);
+ p1->pInsert = 0;
+
+ /* If this table is unknown, return early. The caller will invoke this
+ ** function again and it will move on to the next table. */
+ if( p1->pTab==0 ) return p->errCode;
+
+ /* If this is the sqlite_sequence table, delete any rows added by
+ ** earlier INSERT statements on tables with AUTOINCREMENT primary
+ ** keys before recovering its contents. The p1->pTbls SELECT statement
+ ** is rigged to deliver "sqlite_sequence" last of all, so we don't
+ ** worry about it being modified after it is recovered. */
+ if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){
+ recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence");
+ recoverSqlCallback(p, "DELETE FROM sqlite_sequence");
+ }
+
+ /* Bind the root page of this table within the original database to
+ ** SELECT statement p1->pSel. The SELECT statement will then iterate
+ ** through cells that look like they belong to table pTab. */
+ sqlite3_bind_int64(pSel, 1, iRoot);
+
+ p1->nVal = 0;
+ p1->bHaveRowid = 0;
+ p1->iPrevPage = -1;
+ p1->iPrevCell = -1;
+ }else{
+ return SQLITE_DONE;
+ }
+ }
+ assert( p->errCode!=SQLITE_OK || p1->pTab );
+
+ if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){
+ RecoverTable *pTab = p1->pTab;
+
+ i64 iPage = sqlite3_column_int64(pSel, 0);
+ int iCell = sqlite3_column_int(pSel, 1);
+ int iField = sqlite3_column_int(pSel, 2);
+ sqlite3_value *pVal = sqlite3_column_value(pSel, 3);
+ int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell);
+
+ assert( bNewCell==0 || (iField==-1 || iField==0) );
+ assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol );
+
+ if( bNewCell ){
+ int ii = 0;
+ if( p1->nVal>=0 ){
+ if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){
+ recoverFinalize(p, p1->pInsert);
+ p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal);
+ p1->nInsert = p1->nVal;
+ }
+ if( p1->nVal>0 ){
+ sqlite3_stmt *pInsert = p1->pInsert;
+ for(ii=0; iinCol; ii++){
+ RecoverColumn *pCol = &pTab->aCol[ii];
+ int iBind = pCol->iBind;
+ if( iBind>0 ){
+ if( pCol->bIPK ){
+ sqlite3_bind_int64(pInsert, iBind, p1->iRowid);
+ }else if( pCol->iFieldnVal ){
+ recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]);
+ }
+ }
+ }
+ if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){
+ sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid);
+ }
+ if( SQLITE_ROW==sqlite3_step(pInsert) ){
+ const char *z = (const char*)sqlite3_column_text(pInsert, 0);
+ recoverSqlCallback(p, z);
+ }
+ recoverReset(p, pInsert);
+ assert( p->errCode || pInsert );
+ if( pInsert ) sqlite3_clear_bindings(pInsert);
+ }
+ }
+
+ for(ii=0; iinVal; ii++){
+ sqlite3_value_free(apVal[ii]);
+ apVal[ii] = 0;
+ }
+ p1->nVal = -1;
+ p1->bHaveRowid = 0;
+ }
+
+ if( iPage!=0 ){
+ if( iField<0 ){
+ p1->iRowid = sqlite3_column_int64(pSel, 3);
+ assert( p1->nVal==-1 );
+ p1->nVal = 0;
+ p1->bHaveRowid = 1;
+ }else if( iFieldnCol ){
+ assert( apVal[iField]==0 );
+ apVal[iField] = sqlite3_value_dup( pVal );
+ if( apVal[iField]==0 ){
+ recoverError(p, SQLITE_NOMEM, 0);
+ }
+ p1->nVal = iField+1;
+ }
+ p1->iPrevCell = iCell;
+ p1->iPrevPage = iPage;
+ }
+ }else{
+ recoverReset(p, pSel);
+ p1->pTab = 0;
+ }
+
+ return p->errCode;
+}
+
+/*
+** Initialize resources required by sqlite3_recover_step() in
+** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not
+** already allocated to a recovered schema element is determined.
+*/
+static void recoverLostAndFound1Init(sqlite3_recover *p){
+ RecoverStateLAF *pLaf = &p->laf;
+ sqlite3_stmt *pStmt = 0;
+
+ assert( p->laf.pUsed==0 );
+ pLaf->nPg = recoverPageCount(p);
+ pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg);
+
+ /* Prepare a statement to iterate through all pages that are part of any tree
+ ** in the recoverable part of the input database schema to the bitmap. And,
+ ** if !p->bFreelistCorrupt, add all pages that appear to be part of the
+ ** freelist. */
+ pStmt = recoverPrepare(
+ p, p->dbOut,
+ "WITH trunk(pgno) AS ("
+ " SELECT read_i32(getpage(1), 8) AS x WHERE x>0"
+ " UNION"
+ " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0"
+ "),"
+ "trunkdata(pgno, data) AS ("
+ " SELECT pgno, getpage(pgno) FROM trunk"
+ "),"
+ "freelist(data, n, freepgno) AS ("
+ " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata"
+ " UNION ALL"
+ " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0"
+ "),"
+ ""
+ "roots(r) AS ("
+ " SELECT 1 UNION ALL"
+ " SELECT rootpage FROM recovery.schema WHERE rootpage>0"
+ "),"
+ "used(page) AS ("
+ " SELECT r FROM roots"
+ " UNION"
+ " SELECT child FROM sqlite_dbptr('getpage()'), used "
+ " WHERE pgno=page"
+ ") "
+ "SELECT page FROM used"
+ " UNION ALL "
+ "SELECT freepgno FROM freelist WHERE NOT ?"
+ );
+ if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt);
+ pLaf->pUsedPages = pStmt;
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not
+** already allocated to a recovered schema element is determined.
+*/
+static int recoverLostAndFound1Step(sqlite3_recover *p){
+ RecoverStateLAF *pLaf = &p->laf;
+ int rc = p->errCode;
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_step(pLaf->pUsedPages);
+ if( rc==SQLITE_ROW ){
+ i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0);
+ recoverBitmapSet(pLaf->pUsed, iPg);
+ rc = SQLITE_OK;
+ }else{
+ recoverFinalize(p, pLaf->pUsedPages);
+ pLaf->pUsedPages = 0;
+ }
+ }
+ return rc;
+}
+
+/*
+** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2
+** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1
+** are sorted into sets that likely belonged to the same database tree.
+*/
+static void recoverLostAndFound2Init(sqlite3_recover *p){
+ RecoverStateLAF *pLaf = &p->laf;
+
+ assert( p->laf.pAllAndParent==0 );
+ assert( p->laf.pMapInsert==0 );
+ assert( p->laf.pMaxField==0 );
+ assert( p->laf.nMaxField==0 );
+
+ pLaf->pMapInsert = recoverPrepare(p, p->dbOut,
+ "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)"
+ );
+ pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut,
+ "WITH RECURSIVE seq(ii) AS ("
+ " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
+ ")"
+ "SELECT pgno, child FROM sqlite_dbptr('getpage()') "
+ " UNION ALL "
+ "SELECT NULL, ii FROM seq", p->laf.nPg
+ );
+ pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut,
+ "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?"
+ );
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified
+** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged
+** to the same database tree.
+*/
+static int recoverLostAndFound2Step(sqlite3_recover *p){
+ RecoverStateLAF *pLaf = &p->laf;
+ if( p->errCode==SQLITE_OK ){
+ int res = sqlite3_step(pLaf->pAllAndParent);
+ if( res==SQLITE_ROW ){
+ i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1);
+ if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){
+ sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild);
+ sqlite3_bind_value(pLaf->pMapInsert, 2,
+ sqlite3_column_value(pLaf->pAllAndParent, 0)
+ );
+ sqlite3_step(pLaf->pMapInsert);
+ recoverReset(p, pLaf->pMapInsert);
+ sqlite3_bind_int64(pLaf->pMaxField, 1, iChild);
+ if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){
+ int nMax = sqlite3_column_int(pLaf->pMaxField, 0);
+ if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax;
+ }
+ recoverReset(p, pLaf->pMaxField);
+ }
+ }else{
+ recoverFinalize(p, pLaf->pAllAndParent);
+ pLaf->pAllAndParent =0;
+ return SQLITE_DONE;
+ }
+ }
+ return p->errCode;
+}
+
+/*
+** Free all resources allocated as part of sqlite3_recover_step() calls
+** in one of the RECOVER_STATE_LOSTANDFOUND[123] states.
+*/
+static void recoverLostAndFoundCleanup(sqlite3_recover *p){
+ recoverBitmapFree(p->laf.pUsed);
+ p->laf.pUsed = 0;
+ sqlite3_finalize(p->laf.pUsedPages);
+ sqlite3_finalize(p->laf.pAllAndParent);
+ sqlite3_finalize(p->laf.pMapInsert);
+ sqlite3_finalize(p->laf.pMaxField);
+ sqlite3_finalize(p->laf.pFindRoot);
+ sqlite3_finalize(p->laf.pInsert);
+ sqlite3_finalize(p->laf.pAllPage);
+ sqlite3_finalize(p->laf.pPageData);
+ p->laf.pUsedPages = 0;
+ p->laf.pAllAndParent = 0;
+ p->laf.pMapInsert = 0;
+ p->laf.pMaxField = 0;
+ p->laf.pFindRoot = 0;
+ p->laf.pInsert = 0;
+ p->laf.pAllPage = 0;
+ p->laf.pPageData = 0;
+ sqlite3_free(p->laf.apVal);
+ p->laf.apVal = 0;
+}
+
+/*
+** Free all resources allocated as part of sqlite3_recover_step() calls.
+*/
+static void recoverFinalCleanup(sqlite3_recover *p){
+ RecoverTable *pTab = 0;
+ RecoverTable *pNext = 0;
+
+ recoverWriteDataCleanup(p);
+ recoverLostAndFoundCleanup(p);
+
+ for(pTab=p->pTblList; pTab; pTab=pNext){
+ pNext = pTab->pNext;
+ sqlite3_free(pTab);
+ }
+ p->pTblList = 0;
+ sqlite3_finalize(p->pGetPage);
+ p->pGetPage = 0;
+
+ {
+#ifndef NDEBUG
+ int res =
+#endif
+ sqlite3_close(p->dbOut);
+ assert( res==SQLITE_OK );
+ }
+ p->dbOut = 0;
+}
+
+/*
+** Decode and return an unsigned 16-bit big-endian integer value from
+** buffer a[].
+*/
+static u32 recoverGetU16(const u8 *a){
+ return (((u32)a[0])<<8) + ((u32)a[1]);
+}
+
+/*
+** Decode and return an unsigned 32-bit big-endian integer value from
+** buffer a[].
+*/
+static u32 recoverGetU32(const u8 *a){
+ return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]);
+}
+
+/*
+** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal)
+** and return the number of bytes consumed.
+*/
+static int recoverGetVarint(const u8 *a, i64 *pVal){
+ sqlite3_uint64 u = 0;
+ int i;
+ for(i=0; i<8; i++){
+ u = (u<<7) + (a[i]&0x7f);
+ if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
+ }
+ u = (u<<8) + (a[i]&0xff);
+ *pVal = (sqlite3_int64)u;
+ return 9;
+}
+
+/*
+** The second argument points to a buffer n bytes in size. If this buffer
+** or a prefix thereof appears to contain a well-formed SQLite b-tree page,
+** return the page-size in bytes. Otherwise, if the buffer does not
+** appear to contain a well-formed b-tree page, return 0.
+*/
+static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){
+ u8 *aUsed = aTmp;
+ int nFrag = 0;
+ int nActual = 0;
+ int iFree = 0;
+ int nCell = 0; /* Number of cells on page */
+ int iCellOff = 0; /* Offset of cell array in page */
+ int iContent = 0;
+ int eType = 0;
+ int ii = 0;
+
+ eType = (int)a[0];
+ if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0;
+
+ iFree = (int)recoverGetU16(&a[1]);
+ nCell = (int)recoverGetU16(&a[3]);
+ iContent = (int)recoverGetU16(&a[5]);
+ if( iContent==0 ) iContent = 65536;
+ nFrag = (int)a[7];
+
+ if( iContent>n ) return 0;
+
+ memset(aUsed, 0, n);
+ memset(aUsed, 0xFF, iContent);
+
+ /* Follow the free-list. This is the same format for all b-tree pages. */
+ if( iFree && iFree<=iContent ) return 0;
+ while( iFree ){
+ int iNext = 0;
+ int nByte = 0;
+ if( iFree>(n-4) ) return 0;
+ iNext = recoverGetU16(&a[iFree]);
+ nByte = recoverGetU16(&a[iFree+2]);
+ if( iFree+nByte>n ) return 0;
+ if( iNext && iNextiContent ) return 0;
+ for(ii=0; iin ){
+ return 0;
+ }
+ if( eType==0x05 || eType==0x02 ) nByte += 4;
+ nByte += recoverGetVarint(&a[iOff+nByte], &nPayload);
+ if( eType==0x0D ){
+ i64 dummy = 0;
+ nByte += recoverGetVarint(&a[iOff+nByte], &dummy);
+ }
+ if( eType!=0x05 ){
+ int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23);
+ int M = ((n-12)*32/255)-23;
+ int K = M+((nPayload-M)%(n-4));
+
+ if( nPayloadn ){
+ return 0;
+ }
+ for(iByte=iOff; iByte<(iOff+nByte); iByte++){
+ if( aUsed[iByte]!=0 ){
+ return 0;
+ }
+ aUsed[iByte] = 0xFF;
+ }
+ }
+
+ nActual = 0;
+ for(ii=0; iipMethods!=&recover_methods );
+ return pFd->pMethods->xClose(pFd);
+}
+
+/*
+** Write value v to buffer a[] as a 16-bit big-endian unsigned integer.
+*/
+static void recoverPutU16(u8 *a, u32 v){
+ a[0] = (v>>8) & 0x00FF;
+ a[1] = (v>>0) & 0x00FF;
+}
+
+/*
+** Write value v to buffer a[] as a 32-bit big-endian unsigned integer.
+*/
+static void recoverPutU32(u8 *a, u32 v){
+ a[0] = (v>>24) & 0x00FF;
+ a[1] = (v>>16) & 0x00FF;
+ a[2] = (v>>8) & 0x00FF;
+ a[3] = (v>>0) & 0x00FF;
+}
+
+/*
+** Detect the page-size of the database opened by file-handle pFd by
+** searching the first part of the file for a well-formed SQLite b-tree
+** page. If parameter nReserve is non-zero, then as well as searching for
+** a b-tree page with zero reserved bytes, this function searches for one
+** with nReserve reserved bytes at the end of it.
+**
+** If successful, set variable p->detected_pgsz to the detected page-size
+** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page
+** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or,
+** if an error occurs (e.g. an IO or OOM error), then an SQLite error code
+** is returned. The final value of p->detected_pgsz is undefined in this
+** case.
+*/
+static int recoverVfsDetectPagesize(
+ sqlite3_recover *p, /* Recover handle */
+ sqlite3_file *pFd, /* File-handle open on input database */
+ u32 nReserve, /* Possible nReserve value */
+ i64 nSz /* Size of database file in bytes */
+){
+ int rc = SQLITE_OK;
+ const int nMin = 512;
+ const int nMax = 65536;
+ const int nMaxBlk = 4;
+ u32 pgsz = 0;
+ int iBlk = 0;
+ u8 *aPg = 0;
+ u8 *aTmp = 0;
+ int nBlk = 0;
+
+ aPg = (u8*)sqlite3_malloc(2*nMax);
+ if( aPg==0 ) return SQLITE_NOMEM;
+ aTmp = &aPg[nMax];
+
+ nBlk = (nSz+nMax-1)/nMax;
+ if( nBlk>nMaxBlk ) nBlk = nMaxBlk;
+
+ do {
+ for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax);
+ memset(aPg, 0, nMax);
+ rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax);
+ if( rc==SQLITE_OK ){
+ int pgsz2;
+ for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){
+ int iOff;
+ for(iOff=0; iOff(u32)p->detected_pgsz ){
+ p->detected_pgsz = pgsz;
+ p->nReserve = nReserve;
+ }
+ if( nReserve==0 ) break;
+ nReserve = 0;
+ }while( 1 );
+
+ p->detected_pgsz = pgsz;
+ sqlite3_free(aPg);
+ return rc;
+}
+
+/*
+** The xRead() method of the wrapper VFS. This is used to intercept calls
+** to read page 1 of the input database.
+*/
+static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
+ int rc = SQLITE_OK;
+ if( pFd->pMethods==&recover_methods ){
+ pFd->pMethods = recover_g.pMethods;
+ rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
+ if( nByte==16 ){
+ sqlite3_randomness(16, aBuf);
+ }else
+ if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){
+ /* Ensure that the database has a valid header file. The only fields
+ ** that really matter to recovery are:
+ **
+ ** + Database page size (16-bits at offset 16)
+ ** + Size of db in pages (32-bits at offset 28)
+ ** + Database encoding (32-bits at offset 56)
+ **
+ ** Also preserved are:
+ **
+ ** + first freelist page (32-bits at offset 32)
+ ** + size of freelist (32-bits at offset 36)
+ **
+ ** We also try to preserve the auto-vacuum, incr-value, user-version
+ ** and application-id fields - all 32 bit quantities at offsets
+ ** 52, 60, 64 and 68. All other fields are set to known good values.
+ **
+ ** Byte offset 105 should also contain the page-size as a 16-bit
+ ** integer.
+ */
+ const int aPreserve[] = {32, 36, 52, 60, 64, 68};
+ u8 aHdr[108] = {
+ 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66,
+ 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
+ 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20,
+ 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x2e, 0x5b, 0x30,
+
+ 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00
+ };
+ u8 *a = (u8*)aBuf;
+
+ u32 pgsz = recoverGetU16(&a[16]);
+ u32 nReserve = a[20];
+ u32 enc = recoverGetU32(&a[56]);
+ u32 dbsz = 0;
+ i64 dbFileSize = 0;
+ int ii;
+ sqlite3_recover *p = recover_g.p;
+
+ if( pgsz==0x01 ) pgsz = 65536;
+ rc = pFd->pMethods->xFileSize(pFd, &dbFileSize);
+
+ if( rc==SQLITE_OK && p->detected_pgsz==0 ){
+ rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize);
+ }
+ if( p->detected_pgsz ){
+ pgsz = p->detected_pgsz;
+ nReserve = p->nReserve;
+ }
+
+ if( pgsz ){
+ dbsz = dbFileSize / pgsz;
+ }
+ if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){
+ enc = SQLITE_UTF8;
+ }
+
+ sqlite3_free(p->pPage1Cache);
+ p->pPage1Cache = 0;
+ p->pPage1Disk = 0;
+
+ p->pgsz = nByte;
+ p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2);
+ if( p->pPage1Cache ){
+ p->pPage1Disk = &p->pPage1Cache[nByte];
+ memcpy(p->pPage1Disk, aBuf, nByte);
+
+ recoverPutU32(&aHdr[28], dbsz);
+ recoverPutU32(&aHdr[56], enc);
+ recoverPutU16(&aHdr[105], pgsz-nReserve);
+ if( pgsz==65536 ) pgsz = 1;
+ recoverPutU16(&aHdr[16], pgsz);
+ aHdr[20] = nReserve;
+ for(ii=0; iipPage1Cache, aBuf, nByte);
+ }else{
+ rc = p->errCode;
+ }
+
+ }
+ pFd->pMethods = &recover_methods;
+ }else{
+ rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
+ }
+ return rc;
+}
+
+/*
+** Used to make sqlite3_io_methods wrapper methods less verbose.
+*/
+#define RECOVER_VFS_WRAPPER(code) \
+ int rc = SQLITE_OK; \
+ if( pFd->pMethods==&recover_methods ){ \
+ pFd->pMethods = recover_g.pMethods; \
+ rc = code; \
+ pFd->pMethods = &recover_methods; \
+ }else{ \
+ rc = code; \
+ } \
+ return rc;
+
+/*
+** Methods of the wrapper VFS. All methods except for xRead() and xClose()
+** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent
+** method on the lower level VFS, then reinstall the wrapper before returning.
+** Those that return an integer value use the RECOVER_VFS_WRAPPER macro.
+*/
+static int recoverVfsWrite(
+ sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff
+){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff)
+ );
+}
+static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xTruncate(pFd, size)
+ );
+}
+static int recoverVfsSync(sqlite3_file *pFd, int flags){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xSync(pFd, flags)
+ );
+}
+static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xFileSize(pFd, pSize)
+ );
+}
+static int recoverVfsLock(sqlite3_file *pFd, int eLock){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xLock(pFd, eLock)
+ );
+}
+static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xUnlock(pFd, eLock)
+ );
+}
+static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xCheckReservedLock(pFd, pResOut)
+ );
+}
+static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){
+ RECOVER_VFS_WRAPPER (
+ (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND)
+ );
+}
+static int recoverVfsSectorSize(sqlite3_file *pFd){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xSectorSize(pFd)
+ );
+}
+static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xDeviceCharacteristics(pFd)
+ );
+}
+static int recoverVfsShmMap(
+ sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp
+){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp)
+ );
+}
+static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xShmLock(pFd, offset, n, flags)
+ );
+}
+static void recoverVfsShmBarrier(sqlite3_file *pFd){
+ if( pFd->pMethods==&recover_methods ){
+ pFd->pMethods = recover_g.pMethods;
+ pFd->pMethods->xShmBarrier(pFd);
+ pFd->pMethods = &recover_methods;
+ }else{
+ pFd->pMethods->xShmBarrier(pFd);
+ }
+}
+static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){
+ RECOVER_VFS_WRAPPER (
+ pFd->pMethods->xShmUnmap(pFd, deleteFlag)
+ );
+}
+
+static int recoverVfsFetch(
+ sqlite3_file *pFd,
+ sqlite3_int64 iOff,
+ int iAmt,
+ void **pp
+){
+ *pp = 0;
+ return SQLITE_OK;
+}
+static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){
+ return SQLITE_OK;
+}
+
+/*
+** Install the VFS wrapper around the file-descriptor open on the input
+** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held
+** when this function is called.
+*/
+static void recoverInstallWrapper(sqlite3_recover *p){
+ sqlite3_file *pFd = 0;
+ assert( recover_g.pMethods==0 );
+ recoverAssertMutexHeld();
+ sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
+ assert( pFd==0 || pFd->pMethods!=&recover_methods );
+ if( pFd && pFd->pMethods ){
+ int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0);
+ recover_g.pMethods = pFd->pMethods;
+ recover_g.p = p;
+ recover_methods.iVersion = iVersion;
+ pFd->pMethods = &recover_methods;
+ }
+}
+
+/*
+** Uninstall the VFS wrapper that was installed around the file-descriptor open
+** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be
+** held when this function is called.
+*/
+static void recoverUninstallWrapper(sqlite3_recover *p){
+ sqlite3_file *pFd = 0;
+ recoverAssertMutexHeld();
+ sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd);
+ if( pFd && pFd->pMethods ){
+ pFd->pMethods = recover_g.pMethods;
+ recover_g.pMethods = 0;
+ recover_g.p = 0;
+ }
+}
+
+/*
+** This function does the work of a single sqlite3_recover_step() call. It
+** is guaranteed that the handle is not in an error state when this
+** function is called.
+*/
+static void recoverStep(sqlite3_recover *p){
+ assert( p && p->errCode==SQLITE_OK );
+ switch( p->eState ){
+ case RECOVER_STATE_INIT:
+ /* This is the very first call to sqlite3_recover_step() on this object.
+ */
+ recoverSqlCallback(p, "BEGIN");
+ recoverSqlCallback(p, "PRAGMA writable_schema = on");
+
+ recoverEnterMutex();
+ recoverInstallWrapper(p);
+
+ /* Open the output database. And register required virtual tables and
+ ** user functions with the new handle. */
+ recoverOpenOutput(p);
+
+ /* Open transactions on both the input and output databases. */
+ recoverExec(p, p->dbIn, "PRAGMA writable_schema = on");
+ recoverExec(p, p->dbIn, "BEGIN");
+ if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1;
+ recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema");
+ recoverTransferSettings(p);
+ recoverOpenRecovery(p);
+ recoverCacheSchema(p);
+
+ recoverUninstallWrapper(p);
+ recoverLeaveMutex();
+
+ recoverExec(p, p->dbOut, "BEGIN");
+
+ recoverWriteSchema1(p);
+ p->eState = RECOVER_STATE_WRITING;
+ break;
+
+ case RECOVER_STATE_WRITING: {
+ if( p->w1.pTbls==0 ){
+ recoverWriteDataInit(p);
+ }
+ if( SQLITE_DONE==recoverWriteDataStep(p) ){
+ recoverWriteDataCleanup(p);
+ if( p->zLostAndFound ){
+ p->eState = RECOVER_STATE_LOSTANDFOUND1;
+ }else{
+ p->eState = RECOVER_STATE_SCHEMA2;
+ }
+ }
+ break;
+ }
+
+ case RECOVER_STATE_LOSTANDFOUND1: {
+ if( p->laf.pUsed==0 ){
+ recoverLostAndFound1Init(p);
+ }
+ if( SQLITE_DONE==recoverLostAndFound1Step(p) ){
+ p->eState = RECOVER_STATE_LOSTANDFOUND2;
+ }
+ break;
+ }
+ case RECOVER_STATE_LOSTANDFOUND2: {
+ if( p->laf.pAllAndParent==0 ){
+ recoverLostAndFound2Init(p);
+ }
+ if( SQLITE_DONE==recoverLostAndFound2Step(p) ){
+ p->eState = RECOVER_STATE_LOSTANDFOUND3;
+ }
+ break;
+ }
+
+ case RECOVER_STATE_LOSTANDFOUND3: {
+ if( p->laf.pInsert==0 ){
+ recoverLostAndFound3Init(p);
+ }
+ if( SQLITE_DONE==recoverLostAndFound3Step(p) ){
+ p->eState = RECOVER_STATE_SCHEMA2;
+ }
+ break;
+ }
+
+ case RECOVER_STATE_SCHEMA2: {
+ int rc = SQLITE_OK;
+
+ recoverWriteSchema2(p);
+ p->eState = RECOVER_STATE_DONE;
+
+ /* If no error has occurred, commit the write transaction on the output
+ ** database. Regardless of whether or not an error has occurred, make
+ ** an attempt to end the read transaction on the input database. */
+ recoverExec(p, p->dbOut, "COMMIT");
+ rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0);
+ if( p->errCode==SQLITE_OK ) p->errCode = rc;
+
+ recoverSqlCallback(p, "PRAGMA writable_schema = off");
+ recoverSqlCallback(p, "COMMIT");
+ p->eState = RECOVER_STATE_DONE;
+ recoverFinalCleanup(p);
+ break;
+ };
+
+ case RECOVER_STATE_DONE: {
+ /* no-op */
+ break;
+ };
+ }
+}
+
+
+/*
+** This is a worker function that does the heavy lifting for both init
+** functions:
+**
+** sqlite3_recover_init()
+** sqlite3_recover_init_sql()
+**
+** All this function does is allocate space for the recover handle and
+** take copies of the input parameters. All the real work is done within
+** sqlite3_recover_run().
+*/
+sqlite3_recover *recoverInit(
+ sqlite3* db,
+ const char *zDb,
+ const char *zUri, /* Output URI for _recover_init() */
+ int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */
+ void *pSqlCtx /* Context arg for _recover_init_sql() */
+){
+ sqlite3_recover *pRet = 0;
+ int nDb = 0;
+ int nUri = 0;
+ int nByte = 0;
+
+ if( zDb==0 ){ zDb = "main"; }
+
+ nDb = recoverStrlen(zDb);
+ nUri = recoverStrlen(zUri);
+
+ nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1;
+ pRet = (sqlite3_recover*)sqlite3_malloc(nByte);
+ if( pRet ){
+ memset(pRet, 0, nByte);
+ pRet->dbIn = db;
+ pRet->zDb = (char*)&pRet[1];
+ pRet->zUri = &pRet->zDb[nDb+1];
+ memcpy(pRet->zDb, zDb, nDb);
+ if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri);
+ pRet->xSql = xSql;
+ pRet->pSqlCtx = pSqlCtx;
+ pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT;
+ }
+
+ return pRet;
+}
+
+/*
+** Initialize a recovery handle that creates a new database containing
+** the recovered data.
+*/
+sqlite3_recover *sqlite3_recover_init(
+ sqlite3* db,
+ const char *zDb,
+ const char *zUri
+){
+ return recoverInit(db, zDb, zUri, 0, 0);
+}
+
+/*
+** Initialize a recovery handle that returns recovered data in the
+** form of SQL statements via a callback.
+*/
+sqlite3_recover *sqlite3_recover_init_sql(
+ sqlite3* db,
+ const char *zDb,
+ int (*xSql)(void*, const char*),
+ void *pSqlCtx
+){
+ return recoverInit(db, zDb, 0, xSql, pSqlCtx);
+}
+
+/*
+** Return the handle error message, if any.
+*/
+const char *sqlite3_recover_errmsg(sqlite3_recover *p){
+ return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory";
+}
+
+/*
+** Return the handle error code.
+*/
+int sqlite3_recover_errcode(sqlite3_recover *p){
+ return p ? p->errCode : SQLITE_NOMEM;
+}
+
+/*
+** Configure the handle.
+*/
+int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){
+ int rc = SQLITE_OK;
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ }else if( p->eState!=RECOVER_STATE_INIT ){
+ rc = SQLITE_MISUSE;
+ }else{
+ switch( op ){
+ case 789:
+ /* This undocumented magic configuration option is used to set the
+ ** name of the auxiliary database that is ATTACH-ed to the database
+ ** connection and used to hold state information during the
+ ** recovery process. This option is for debugging use only and
+ ** is subject to change or removal at any time. */
+ sqlite3_free(p->zStateDb);
+ p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg);
+ break;
+
+ case SQLITE_RECOVER_LOST_AND_FOUND: {
+ const char *zArg = (const char*)pArg;
+ sqlite3_free(p->zLostAndFound);
+ if( zArg ){
+ p->zLostAndFound = recoverMPrintf(p, "%s", zArg);
+ }else{
+ p->zLostAndFound = 0;
+ }
+ break;
+ }
+
+ case SQLITE_RECOVER_FREELIST_CORRUPT:
+ p->bFreelistCorrupt = *(int*)pArg;
+ break;
+
+ case SQLITE_RECOVER_ROWIDS:
+ p->bRecoverRowid = *(int*)pArg;
+ break;
+
+ case SQLITE_RECOVER_SLOWINDEXES:
+ p->bSlowIndexes = *(int*)pArg;
+ break;
+
+ default:
+ rc = SQLITE_NOTFOUND;
+ break;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Do a unit of work towards the recovery job. Return SQLITE_OK if
+** no error has occurred but database recovery is not finished, SQLITE_DONE
+** if database recovery has been successfully completed, or an SQLite
+** error code if an error has occurred.
+*/
+int sqlite3_recover_step(sqlite3_recover *p){
+ if( p==0 ) return SQLITE_NOMEM;
+ if( p->errCode==SQLITE_OK ) recoverStep(p);
+ if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){
+ return SQLITE_DONE;
+ }
+ return p->errCode;
+}
+
+/*
+** Do the configured recovery operation. Return SQLITE_OK if successful, or
+** else an SQLite error code.
+*/
+int sqlite3_recover_run(sqlite3_recover *p){
+ while( SQLITE_OK==sqlite3_recover_step(p) );
+ return sqlite3_recover_errcode(p);
+}
+
+
+/*
+** Free all resources associated with the recover handle passed as the only
+** argument. The results of using a handle with any sqlite3_recover_**
+** API function after it has been passed to this function are undefined.
+**
+** A copy of the value returned by the first call made to sqlite3_recover_run()
+** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has
+** not been called on this handle.
+*/
+int sqlite3_recover_finish(sqlite3_recover *p){
+ int rc;
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ recoverFinalCleanup(p);
+ if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){
+ rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0);
+ if( p->errCode==SQLITE_OK ) p->errCode = rc;
+ }
+ rc = p->errCode;
+ sqlite3_free(p->zErrMsg);
+ sqlite3_free(p->zStateDb);
+ sqlite3_free(p->zLostAndFound);
+ sqlite3_free(p->pPage1Cache);
+ sqlite3_free(p);
+ }
+ return rc;
+}
+
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
+/************************* End ../ext/recover/sqlite3recover.c ********************/
#endif
#if defined(SQLITE_ENABLE_SESSION)
@@ -12252,15 +15592,16 @@ struct ShellState {
char *zNonce; /* Nonce for temporary safe-mode excapes */
EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */
ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
struct {
const char * zInput; /* Input string from wasm/JS proxy */
const char * zPos; /* Cursor pos into zInput */
+ const char * zDefaultDbName; /* Default name for db file */
} wasm;
#endif
};
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
static ShellState shellState;
#endif
@@ -12592,10 +15933,23 @@ static void outputModePop(ShellState *p){
*/
static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){
int i;
- char *zBlob = (char *)pBlob;
- raw_printf(out,"X'");
- for(i=0; i> 4) ];
+ zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ];
+ }
+ zStr[i*2] = '\0';
+
+ raw_printf(out,"X'%s'", zStr);
+ sqlite3_free(zStr);
}
/*
@@ -12755,9 +16109,9 @@ static void output_c_string(FILE *out, const char *z){
/*
** Output the given string as a quoted according to JSON quoting rules.
*/
-static void output_json_string(FILE *out, const char *z, int n){
+static void output_json_string(FILE *out, const char *z, i64 n){
unsigned int c;
- if( n<0 ) n = (int)strlen(z);
+ if( n<0 ) n = strlen(z);
fputc('"', out);
while( n-- ){
c = *(z++);
@@ -12928,7 +16282,7 @@ static int safeModeAuth(
UNUSED_PARAMETER(zA4);
switch( op ){
case SQLITE_ATTACH: {
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
/* In WASM builds the filesystem is a virtual sandbox, so
** there's no harm in using ATTACH. */
failIfSafeMode(p, "cannot run ATTACH in safe mode");
@@ -13001,15 +16355,37 @@ static int shellAuth(
**
** This routine converts some CREATE TABLE statements for shadow tables
** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements.
+**
+** If the schema statement in z[] contains a start-of-comment and if
+** sqlite3_complete() returns false, try to terminate the comment before
+** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c
*/
static void printSchemaLine(FILE *out, const char *z, const char *zTail){
+ char *zToFree = 0;
if( z==0 ) return;
if( zTail==0 ) return;
+ if( zTail[0]==';' && (strstr(z, "/*")!=0 || strstr(z,"--")!=0) ){
+ const char *zOrig = z;
+ static const char *azTerm[] = { "", "*/", "\n" };
+ int i;
+ for(i=0; iautoEQPtest ){
utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText);
}
@@ -13083,14 +16461,14 @@ static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){
*/
static void eqp_render_level(ShellState *p, int iEqpId){
EQPGraphRow *pRow, *pNext;
- int n = strlen30(p->sGraph.zPrefix);
+ i64 n = strlen(p->sGraph.zPrefix);
char *z;
for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){
pNext = eqp_next_row(p, iEqpId, pRow);
z = pRow->zText;
utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix,
pNext ? "|--" : "`--", z);
- if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){
+ if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){
memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4);
eqp_render_level(p, pRow->iEqpId);
p->sGraph.zPrefix[n] = 0;
@@ -13690,6 +17068,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){
while( (zSql[len]&0xc0)==0x80 ) len--;
}
zCode = sqlite3_mprintf("%.*s", len, zSql);
+ shell_check_oom(zCode);
for(i=0; zCode[i]; i++){ if( IsSpace(zSql[i]) ) zCode[i] = ' '; }
if( iOffset<25 ){
zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode, iOffset, "");
@@ -13806,7 +17185,7 @@ static void displayLinuxIoStats(FILE *out){
int i;
for(i=0; icMode = p->mode;
sqlite3_reset(pSql);
return;
@@ -14844,10 +18223,10 @@ static int expertDotCommand(
int n;
if( z[0]=='-' && z[1]=='-' ) z++;
n = strlen30(z);
- if( n>=2 && 0==strncmp(z, "-verbose", n) ){
+ if( n>=2 && 0==cli_strncmp(z, "-verbose", n) ){
pState->expert.bVerbose = 1;
}
- else if( n>=2 && 0==strncmp(z, "-sample", n) ){
+ else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){
if( i==(nArg-1) ){
raw_printf(stderr, "option requires an argument: %s\n", z);
rc = SQLITE_ERROR;
@@ -15195,18 +18574,20 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
zTable = azArg[0];
zType = azArg[1];
zSql = azArg[2];
+ if( zTable==0 ) return 0;
+ if( zType==0 ) return 0;
dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0;
noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0;
- if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){
+ if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){
if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n");
}else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){
if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n");
- }else if( strncmp(zTable, "sqlite_", 7)==0 ){
+ }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){
return 0;
}else if( dataOnly ){
/* no-op */
- }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
+ }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
char *zIns;
if( !p->writableSchema ){
raw_printf(p->out, "PRAGMA writable_schema=ON;\n");
@@ -15224,7 +18605,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
printSchemaLine(p->out, zSql, ";\n");
}
- if( strcmp(zType, "table")==0 ){
+ if( cli_strcmp(zType, "table")==0 ){
ShellText sSelect;
ShellText sTable;
char **azCol;
@@ -15342,7 +18723,7 @@ static int run_schema_dump_query(
*/
static const char *(azHelp[]) = {
#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) \
- && !defined(SQLITE_SHELL_WASM_MODE)
+ && !defined(SQLITE_SHELL_FIDDLE)
".archive ... Manage SQL archives",
" Each command must have exactly one of the following options:",
" -c, --create Create a new archive",
@@ -15368,7 +18749,7 @@ static const char *(azHelp[]) = {
#ifndef SQLITE_OMIT_AUTHORIZATION
".auth ON|OFF Show authorizer callbacks",
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".backup ?DB? FILE Backup DB (default \"main\") to FILE",
" Options:",
" --append Use the appendvfs",
@@ -15376,18 +18757,18 @@ static const char *(azHelp[]) = {
#endif
".bail on|off Stop after hitting an error. Default OFF",
".binary on|off Turn binary output on or off. Default OFF",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".cd DIRECTORY Change the working directory to DIRECTORY",
#endif
".changes on|off Show number of rows changed by SQL",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".check GLOB Fail if output since .testcase does not match",
".clone NEWDB Clone data into NEWDB from the existing database",
#endif
".connection [close] [#] Open or close an auxiliary database connection",
".databases List names and files of attached databases",
".dbconfig ?op? ?val? List or change sqlite3_db_config() options",
-#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
+#if SQLITE_SHELL_HAVE_RECOVER
".dbinfo ?DB? Show status information about the database",
#endif
".dump ?OBJECTS? Render database content as SQL",
@@ -15406,11 +18787,11 @@ static const char *(azHelp[]) = {
" trace Like \"full\" but enable \"PRAGMA vdbe_trace\"",
#endif
" trigger Like \"full\" but also show trigger bytecode",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".excel Display the output of next command in spreadsheet",
" --bom Put a UTF8 byte-order mark on intermediate file",
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".exit ?CODE? Exit this program with return-code CODE",
#endif
".expert EXPERIMENTAL. Suggest indexes for queries",
@@ -15421,7 +18802,7 @@ static const char *(azHelp[]) = {
".fullschema ?--indent? Show schema and the content of sqlite_stat tables",
".headers on|off Turn display of headers on or off",
".help ?-all? ?PATTERN? Show help text for PATTERN",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".import FILE TABLE Import data from FILE into TABLE",
" Options:",
" --ascii Use \\037 and \\036 as column and row separators",
@@ -15450,10 +18831,10 @@ static const char *(azHelp[]) = {
".lint OPTIONS Report potential schema issues.",
" Options:",
" fkey-indexes Find missing foreign key indexes",
-#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_MODE)
+#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE)
".load FILE ?ENTRY? Load an extension library",
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".log FILE|off Turn logging on or off. FILE can be stderr/stdout",
#endif
".mode MODE ?OPTIONS? Set output mode",
@@ -15468,7 +18849,7 @@ static const char *(azHelp[]) = {
" line One value per line",
" list Values delimited by \"|\"",
" markdown Markdown table format",
- " qbox Shorthand for \"box --width 60 --quote\"",
+ " qbox Shorthand for \"box --wrap 60 --quote\"",
" quote Escape answers as for SQL",
" table ASCII-art table",
" tabs Tab-separated values",
@@ -15480,11 +18861,11 @@ static const char *(azHelp[]) = {
" --quote Quote output text as SQL literals",
" --noquote Do not quote output text",
" TABLE The name of SQL table used for \"insert\" mode",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".nonce STRING Suspend safe mode for one command if nonce matches",
#endif
".nullvalue STRING Use STRING in place of NULL values",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE",
" If FILE begins with '|' then open as a pipe",
" --bom Put a UTF8 byte-order mark at the beginning",
@@ -15506,7 +18887,7 @@ static const char *(azHelp[]) = {
" --nofollow Do not follow symbolic links",
" --readonly Open FILE readonly",
" --zip FILE is a ZIP archive",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".output ?FILE? Send output to FILE or stdout if FILE is omitted",
" If FILE begins with '|' then open it as a pipe.",
" Options:",
@@ -15530,20 +18911,19 @@ static const char *(azHelp[]) = {
" --reset Reset the count for each input and interrupt",
#endif
".prompt MAIN CONTINUE Replace the standard prompts",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".quit Exit this program",
".read FILE Read input from FILE or command output",
" If FILE begins with \"|\", it is a command that generates the input.",
#endif
-#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
+#if SQLITE_SHELL_HAVE_RECOVER
".recover Recover as much data as possible from corrupt db.",
- " --freelist-corrupt Assume the freelist is corrupt",
- " --recovery-db NAME Store recovery metadata in database file NAME",
+ " --ignore-freelist Ignore pages that appear to be on db freelist",
" --lost-and-found TABLE Alternative name for the lost-and-found table",
" --no-rowids Do not attempt to recover rowid values",
" that are not also INTEGER PRIMARY KEYs",
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".restore ?DB? FILE Restore content of DB (default \"main\") from FILE",
".save ?OPTIONS? FILE Write database to FILE (an alias for .backup ...)",
#endif
@@ -15580,7 +18960,7 @@ static const char *(azHelp[]) = {
" --sha3-384 Use the sha3-384 algorithm",
" --sha3-512 Use the sha3-512 algorithm",
" Any other argument is a LIKE pattern for tables to hash",
-#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE)
+#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
".shell CMD ARGS... Run CMD ARGS... in a system shell",
#endif
".show Show the current values for various settings",
@@ -15589,11 +18969,11 @@ static const char *(azHelp[]) = {
" on Turn on automatic stat display",
" stmt Show statement stats",
" vmstep Show the virtual machine step count only",
-#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE)
+#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
".system CMD ARGS... Run CMD ARGS... in a system shell",
#endif
".tables ?TABLE? List names of tables matching LIKE pattern TABLE",
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
".testcase NAME Begin redirecting output to 'testcase-out.txt'",
#endif
".testctrl CMD ... Run various sqlite3_test_control() operations",
@@ -15643,9 +19023,9 @@ static int showHelp(FILE *out, const char *zPattern){
char *zPat;
if( zPattern==0
|| zPattern[0]=='0'
- || strcmp(zPattern,"-a")==0
- || strcmp(zPattern,"-all")==0
- || strcmp(zPattern,"--all")==0
+ || cli_strcmp(zPattern,"-a")==0
+ || cli_strcmp(zPattern,"-all")==0
+ || cli_strcmp(zPattern,"--all")==0
){
/* Show all commands, but only one line per command */
if( zPattern==0 ) zPattern = "";
@@ -15882,7 +19262,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
iOffset = k;
continue;
}
- if( strncmp(zLine, "| end ", 6)==0 ){
+ if( cli_strncmp(zLine, "| end ", 6)==0 ){
break;
}
rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
@@ -15910,7 +19290,7 @@ readHexDb_error:
}else{
while( fgets(zLine, sizeof(zLine), p->in)!=0 ){
nLine++;
- if(strncmp(zLine, "| end ", 6)==0 ) break;
+ if(cli_strncmp(zLine, "| end ", 6)==0 ) break;
}
p->lineno = nLine;
}
@@ -16002,28 +19382,28 @@ static void shellEscapeCrnl(
const char *zText = (const char*)sqlite3_value_text(argv[0]);
UNUSED_PARAMETER(argc);
if( zText && zText[0]=='\'' ){
- int nText = sqlite3_value_bytes(argv[0]);
- int i;
+ i64 nText = sqlite3_value_bytes(argv[0]);
+ i64 i;
char zBuf1[20];
char zBuf2[20];
const char *zNL = 0;
const char *zCR = 0;
- int nCR = 0;
- int nNL = 0;
+ i64 nCR = 0;
+ i64 nNL = 0;
for(i=0; zText[i]; i++){
if( zNL==0 && zText[i]=='\n' ){
zNL = unused_string(zText, "\\n", "\\012", zBuf1);
- nNL = (int)strlen(zNL);
+ nNL = strlen(zNL);
}
if( zCR==0 && zText[i]=='\r' ){
zCR = unused_string(zText, "\\r", "\\015", zBuf2);
- nCR = (int)strlen(zCR);
+ nCR = strlen(zCR);
}
}
if( zNL || zCR ){
- int iOut = 0;
+ i64 iOut = 0;
i64 nMax = (nNL > nCR) ? nNL : nCR;
i64 nAlloc = nMax * nText + (nMax+64)*2;
char *zOut = (char*)sqlite3_malloc64(nAlloc);
@@ -16146,11 +19526,11 @@ static void open_db(ShellState *p, int openFlags){
sqlite3_regexp_init(p->db, 0, 0);
sqlite3_ieee_init(p->db, 0, 0);
sqlite3_series_init(p->db, 0, 0);
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
sqlite3_fileio_init(p->db, 0, 0);
sqlite3_completion_init(p->db, 0, 0);
#endif
-#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
+#if SQLITE_SHELL_HAVE_RECOVER
sqlite3_dbdata_init(p->db, 0, 0);
#endif
#ifdef SQLITE_HAVE_ZLIB
@@ -16264,8 +19644,8 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){
** Linenoise completion callback
*/
static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){
- int nLine = strlen30(zLine);
- int i, iStart;
+ i64 nLine = strlen(zLine);
+ i64 i, iStart;
sqlite3_stmt *pStmt = 0;
char *zSql;
char zBuf[1000];
@@ -16403,11 +19783,11 @@ static void output_file_close(FILE *f){
*/
static FILE *output_file_open(const char *zFile, int bTextMode){
FILE *f;
- if( strcmp(zFile,"stdout")==0 ){
+ if( cli_strcmp(zFile,"stdout")==0 ){
f = stdout;
- }else if( strcmp(zFile, "stderr")==0 ){
+ }else if( cli_strcmp(zFile, "stderr")==0 ){
f = stderr;
- }else if( strcmp(zFile, "off")==0 ){
+ }else if( cli_strcmp(zFile, "off")==0 ){
f = 0;
}else{
f = fopen(zFile, bTextMode ? "w" : "wb");
@@ -16431,7 +19811,7 @@ static int sql_trace_callback(
ShellState *p = (ShellState*)pArg;
sqlite3_stmt *pStmt;
const char *zSql;
- int nSql;
+ i64 nSql;
if( p->traceOut==0 ) return 0;
if( mType==SQLITE_TRACE_CLOSE ){
utf8_printf(p->traceOut, "-- closing database connection\n");
@@ -16459,17 +19839,18 @@ static int sql_trace_callback(
}
}
if( zSql==0 ) return 0;
- nSql = strlen30(zSql);
+ nSql = strlen(zSql);
+ if( nSql>1000000000 ) nSql = 1000000000;
while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
switch( mType ){
case SQLITE_TRACE_ROW:
case SQLITE_TRACE_STMT: {
- utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
+ utf8_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
break;
}
case SQLITE_TRACE_PROFILE: {
sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
- utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
+ utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
break;
}
}
@@ -17018,7 +20399,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
}
if( zDb==0 ){
zSchemaTab = sqlite3_mprintf("main.sqlite_schema");
- }else if( strcmp(zDb,"temp")==0 ){
+ }else if( cli_strcmp(zDb,"temp")==0 ){
zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema");
}else{
zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb);
@@ -17034,8 +20415,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion);
return 0;
}
-#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE)
- && defined(SQLITE_ENABLE_DBPAGE_VTAB) */
+#endif /* SQLITE_SHELL_HAVE_RECOVER */
/*
** Print the current sqlite3_errmsg() value to stderr and return 1.
@@ -17152,7 +20532,7 @@ static int optionMatch(const char *zStr, const char *zOpt){
if( zStr[0]!='-' ) return 0;
zStr++;
if( zStr[0]=='-' ) zStr++;
- return strcmp(zStr, zOpt)==0;
+ return cli_strcmp(zStr, zOpt)==0;
}
/*
@@ -18308,364 +21688,16 @@ end_ar_command:
*******************************************************************************/
#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */
-#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
-/*
-** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op.
-** Otherwise, the SQL statement or statements in zSql are executed using
-** database connection db and the error code written to *pRc before
-** this function returns.
-*/
-static void shellExec(sqlite3 *db, int *pRc, const char *zSql){
- int rc = *pRc;
- if( rc==SQLITE_OK ){
- char *zErr = 0;
- rc = sqlite3_exec(db, zSql, 0, 0, &zErr);
- if( rc!=SQLITE_OK ){
- raw_printf(stderr, "SQL error: %s\n", zErr);
- }
- sqlite3_free(zErr);
- *pRc = rc;
- }
-}
+#if SQLITE_SHELL_HAVE_RECOVER
/*
-** Like shellExec(), except that zFmt is a printf() style format string.
+** This function is used as a callback by the recover extension. Simply
+** print the supplied SQL statement to stdout.
*/
-static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){
- char *z = 0;
- if( *pRc==SQLITE_OK ){
- va_list ap;
- va_start(ap, zFmt);
- z = sqlite3_vmprintf(zFmt, ap);
- va_end(ap);
- if( z==0 ){
- *pRc = SQLITE_NOMEM;
- }else{
- shellExec(db, pRc, z);
- }
- sqlite3_free(z);
- }
-}
-
-/*
-** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
-** Otherwise, an attempt is made to allocate, zero and return a pointer
-** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set
-** to SQLITE_NOMEM and NULL returned.
-*/
-static void *shellMalloc(int *pRc, sqlite3_int64 nByte){
- void *pRet = 0;
- if( *pRc==SQLITE_OK ){
- pRet = sqlite3_malloc64(nByte);
- if( pRet==0 ){
- *pRc = SQLITE_NOMEM;
- }else{
- memset(pRet, 0, nByte);
- }
- }
- return pRet;
-}
-
-/*
-** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
-** Otherwise, zFmt is treated as a printf() style string. The result of
-** formatting it along with any trailing arguments is written into a
-** buffer obtained from sqlite3_malloc(), and pointer to which is returned.
-** It is the responsibility of the caller to eventually free this buffer
-** using a call to sqlite3_free().
-**
-** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL
-** pointer returned.
-*/
-static char *shellMPrintf(int *pRc, const char *zFmt, ...){
- char *z = 0;
- if( *pRc==SQLITE_OK ){
- va_list ap;
- va_start(ap, zFmt);
- z = sqlite3_vmprintf(zFmt, ap);
- va_end(ap);
- if( z==0 ){
- *pRc = SQLITE_NOMEM;
- }
- }
- return z;
-}
-
-
-/*
-** When running the ".recover" command, each output table, and the special
-** orphaned row table if it is required, is represented by an instance
-** of the following struct.
-*/
-typedef struct RecoverTable RecoverTable;
-struct RecoverTable {
- char *zQuoted; /* Quoted version of table name */
- int nCol; /* Number of columns in table */
- char **azlCol; /* Array of column lists */
- int iPk; /* Index of IPK column */
-};
-
-/*
-** Free a RecoverTable object allocated by recoverFindTable() or
-** recoverOrphanTable().
-*/
-static void recoverFreeTable(RecoverTable *pTab){
- if( pTab ){
- sqlite3_free(pTab->zQuoted);
- if( pTab->azlCol ){
- int i;
- for(i=0; i<=pTab->nCol; i++){
- sqlite3_free(pTab->azlCol[i]);
- }
- sqlite3_free(pTab->azlCol);
- }
- sqlite3_free(pTab);
- }
-}
-
-/*
-** This function is a no-op if (*pRc) is not SQLITE_OK when it is called.
-** Otherwise, it allocates and returns a RecoverTable object based on the
-** final four arguments passed to this function. It is the responsibility
-** of the caller to eventually free the returned object using
-** recoverFreeTable().
-*/
-static RecoverTable *recoverNewTable(
- int *pRc, /* IN/OUT: Error code */
- const char *zName, /* Name of table */
- const char *zSql, /* CREATE TABLE statement */
- int bIntkey,
- int nCol
-){
- sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */
- int rc = *pRc;
- RecoverTable *pTab = 0;
-
- pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable));
- if( rc==SQLITE_OK ){
- int nSqlCol = 0;
- int bSqlIntkey = 0;
- sqlite3_stmt *pStmt = 0;
-
- rc = sqlite3_open("", &dbtmp);
- if( rc==SQLITE_OK ){
- sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0,
- shellIdQuote, 0, 0);
- }
- if( rc==SQLITE_OK ){
- rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0);
- }
- if( rc==SQLITE_OK ){
- rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0);
- if( rc==SQLITE_ERROR ){
- rc = SQLITE_OK;
- goto finished;
- }
- }
- shellPreparePrintf(dbtmp, &rc, &pStmt,
- "SELECT count(*) FROM pragma_table_info(%Q)", zName
- );
- if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
- nSqlCol = sqlite3_column_int(pStmt, 0);
- }
- shellFinalize(&rc, pStmt);
-
- if( rc!=SQLITE_OK || nSqlColiPk to the index
- ** of the column, where columns are 0-numbered from left to right.
- ** Or, if this is a WITHOUT ROWID table or if there is no IPK column,
- ** leave zPk as "_rowid_" and pTab->iPk at -2. */
- pTab->iPk = -2;
- if( bIntkey ){
- shellPreparePrintf(dbtmp, &rc, &pPkFinder,
- "SELECT cid, name FROM pragma_table_info(%Q) "
- " WHERE pk=1 AND type='integer' COLLATE nocase"
- " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)"
- , zName, zName
- );
- if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){
- pTab->iPk = sqlite3_column_int(pPkFinder, 0);
- zPk = (const char*)sqlite3_column_text(pPkFinder, 1);
- if( zPk==0 ){ zPk = "_"; /* Defensive. Should never happen */ }
- }
- }
-
- pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName);
- pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1));
- pTab->nCol = nSqlCol;
-
- if( bIntkey ){
- pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk);
- }else{
- pTab->azlCol[0] = shellMPrintf(&rc, "");
- }
- i = 1;
- shellPreparePrintf(dbtmp, &rc, &pStmt,
- "SELECT %Q || group_concat(shell_idquote(name), ', ') "
- " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) "
- "FROM pragma_table_info(%Q)",
- bIntkey ? ", " : "", pTab->iPk,
- bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ",
- zName
- );
- while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
- const char *zText = (const char*)sqlite3_column_text(pStmt, 0);
- pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText);
- i++;
- }
- shellFinalize(&rc, pStmt);
-
- shellFinalize(&rc, pPkFinder);
- }
- }
-
- finished:
- sqlite3_close(dbtmp);
- *pRc = rc;
- if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){
- recoverFreeTable(pTab);
- pTab = 0;
- }
- return pTab;
-}
-
-/*
-** This function is called to search the schema recovered from the
-** sqlite_schema table of the (possibly) corrupt database as part
-** of a ".recover" command. Specifically, for a table with root page
-** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the
-** table must be a WITHOUT ROWID table, or if non-zero, not one of
-** those.
-**
-** If a table is found, a (RecoverTable*) object is returned. Or, if
-** no such table is found, but bIntkey is false and iRoot is the
-** root page of an index in the recovered schema, then (*pbNoop) is
-** set to true and NULL returned. Or, if there is no such table or
-** index, NULL is returned and (*pbNoop) set to 0, indicating that
-** the caller should write data to the orphans table.
-*/
-static RecoverTable *recoverFindTable(
- ShellState *pState, /* Shell state object */
- int *pRc, /* IN/OUT: Error code */
- int iRoot, /* Root page of table */
- int bIntkey, /* True for an intkey table */
- int nCol, /* Number of columns in table */
- int *pbNoop /* OUT: True if iRoot is root of index */
-){
- sqlite3_stmt *pStmt = 0;
- RecoverTable *pRet = 0;
- int bNoop = 0;
- const char *zSql = 0;
- const char *zName = 0;
-
- /* Search the recovered schema for an object with root page iRoot. */
- shellPreparePrintf(pState->db, pRc, &pStmt,
- "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot
- );
- while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
- const char *zType = (const char*)sqlite3_column_text(pStmt, 0);
- if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){
- bNoop = 1;
- break;
- }
- if( sqlite3_stricmp(zType, "table")==0 ){
- zName = (const char*)sqlite3_column_text(pStmt, 1);
- zSql = (const char*)sqlite3_column_text(pStmt, 2);
- if( zName!=0 && zSql!=0 ){
- pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol);
- break;
- }
- }
- }
-
- shellFinalize(pRc, pStmt);
- *pbNoop = bNoop;
- return pRet;
-}
-
-/*
-** Return a RecoverTable object representing the orphans table.
-*/
-static RecoverTable *recoverOrphanTable(
- ShellState *pState, /* Shell state object */
- int *pRc, /* IN/OUT: Error code */
- const char *zLostAndFound, /* Base name for orphans table */
- int nCol /* Number of user data columns */
-){
- RecoverTable *pTab = 0;
- if( nCol>=0 && *pRc==SQLITE_OK ){
- int i;
-
- /* This block determines the name of the orphan table. The prefered
- ** name is zLostAndFound. But if that clashes with another name
- ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1
- ** and so on until a non-clashing name is found. */
- int iTab = 0;
- char *zTab = shellMPrintf(pRc, "%s", zLostAndFound);
- sqlite3_stmt *pTest = 0;
- shellPrepare(pState->db, pRc,
- "SELECT 1 FROM recovery.schema WHERE name=?", &pTest
- );
- if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT);
- while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){
- shellReset(pRc, pTest);
- sqlite3_free(zTab);
- zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++);
- sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT);
- }
- shellFinalize(pRc, pTest);
-
- pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable));
- if( pTab ){
- pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab);
- pTab->nCol = nCol;
- pTab->iPk = -2;
- if( nCol>0 ){
- pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1));
- if( pTab->azlCol ){
- pTab->azlCol[nCol] = shellMPrintf(pRc, "");
- for(i=nCol-1; i>=0; i--){
- pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]);
- }
- }
- }
-
- if( *pRc!=SQLITE_OK ){
- recoverFreeTable(pTab);
- pTab = 0;
- }else{
- raw_printf(pState->out,
- "CREATE TABLE %s(rootpgno INTEGER, "
- "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted
- );
- for(i=0; iout, ", c%d", i);
- }
- raw_printf(pState->out, ");\n");
- }
- }
- sqlite3_free(zTab);
- }
- return pTab;
+static int recoverSqlCb(void *pCtx, const char *zSql){
+ ShellState *pState = (ShellState*)pCtx;
+ utf8_printf(pState->out, "%s;\n", zSql);
+ return SQLITE_OK;
}
/*
@@ -18675,32 +21707,33 @@ static RecoverTable *recoverOrphanTable(
*/
static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
int rc = SQLITE_OK;
- sqlite3_stmt *pLoop = 0; /* Loop through all root pages */
- sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */
- sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */
- const char *zRecoveryDb = ""; /* Name of "recovery" database */
- const char *zLostAndFound = "lost_and_found";
- int i;
- int nOrphan = -1;
- RecoverTable *pOrphan = 0;
-
- int bFreelist = 1; /* 0 if --freelist-corrupt is specified */
+ const char *zRecoveryDb = ""; /* Name of "recovery" database. Debug only */
+ const char *zLAF = "lost_and_found";
+ int bFreelist = 1; /* 0 if --ignore-freelist is specified */
int bRowids = 1; /* 0 if --no-rowids */
+ sqlite3_recover *p = 0;
+ int i = 0;
+
for(i=1; idb, &rc,
- /* Attach an in-memory database named 'recovery'. Create an indexed
- ** cache of the sqlite_dbptr virtual table. */
- "PRAGMA writable_schema = on;"
- "ATTACH %Q AS recovery;"
- "DROP TABLE IF EXISTS recovery.dbptr;"
- "DROP TABLE IF EXISTS recovery.freelist;"
- "DROP TABLE IF EXISTS recovery.map;"
- "DROP TABLE IF EXISTS recovery.schema;"
- "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb
+ p = sqlite3_recover_init_sql(
+ pState->db, "main", recoverSqlCb, (void*)pState
);
- if( bFreelist ){
- shellExec(pState->db, &rc,
- "WITH trunk(pgno) AS ("
- " SELECT shell_int32("
- " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x "
- " WHERE x>0"
- " UNION"
- " SELECT shell_int32("
- " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x "
- " FROM trunk WHERE x>0"
- "),"
- "freelist(data, n, freepgno) AS ("
- " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno "
- " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno"
- " UNION ALL"
- " SELECT data, n-1, shell_int32(data, 2+n) "
- " FROM freelist WHERE n>=0"
- ")"
- "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;"
- );
+ sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */
+ sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF);
+ sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids);
+ sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist);
+
+ sqlite3_recover_run(p);
+ if( sqlite3_recover_errcode(p)!=SQLITE_OK ){
+ const char *zErr = sqlite3_recover_errmsg(p);
+ int errCode = sqlite3_recover_errcode(p);
+ raw_printf(stderr, "sql error: %s (%d)\n", zErr, errCode);
}
-
- /* If this is an auto-vacuum database, add all pointer-map pages to
- ** the freelist table. Do this regardless of whether or not
- ** --freelist-corrupt was specified. */
- shellExec(pState->db, &rc,
- "WITH ptrmap(pgno) AS ("
- " SELECT 2 WHERE shell_int32("
- " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13"
- " )"
- " UNION ALL "
- " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp "
- " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)"
- ")"
- "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap"
- );
-
- shellExec(pState->db, &rc,
- "CREATE TABLE recovery.dbptr("
- " pgno, child, PRIMARY KEY(child, pgno)"
- ") WITHOUT ROWID;"
- "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) "
- " SELECT * FROM sqlite_dbptr"
- " WHERE pgno NOT IN freelist AND child NOT IN freelist;"
-
- /* Delete any pointer to page 1. This ensures that page 1 is considered
- ** a root page, regardless of how corrupt the db is. */
- "DELETE FROM recovery.dbptr WHERE child = 1;"
-
- /* Delete all pointers to any pages that have more than one pointer
- ** to them. Such pages will be treated as root pages when recovering
- ** data. */
- "DELETE FROM recovery.dbptr WHERE child IN ("
- " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1"
- ");"
-
- /* Create the "map" table that will (eventually) contain instructions
- ** for dealing with each page in the db that contains one or more
- ** records. */
- "CREATE TABLE recovery.map("
- "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT"
- ");"
-
- /* Populate table [map]. If there are circular loops of pages in the
- ** database, the following adds all pages in such a loop to the map
- ** as individual root pages. This could be handled better. */
- "WITH pages(i, maxlen) AS ("
- " SELECT page_count, ("
- " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count"
- " ) FROM pragma_page_count WHERE page_count>0"
- " UNION ALL"
- " SELECT i-1, ("
- " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1"
- " ) FROM pages WHERE i>=2"
- ")"
- "INSERT INTO recovery.map(pgno, maxlen, intkey, root) "
- " SELECT i, maxlen, NULL, ("
- " WITH p(orig, pgno, parent) AS ("
- " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)"
- " UNION "
- " SELECT i, p.parent, "
- " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p"
- " )"
- " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)"
- ") "
- "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;"
- "UPDATE recovery.map AS o SET intkey = ("
- " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno"
- ");"
-
- /* Extract data from page 1 and any linked pages into table
- ** recovery.schema. With the same schema as an sqlite_schema table. */
- "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);"
- "INSERT INTO recovery.schema SELECT "
- " max(CASE WHEN field=0 THEN value ELSE NULL END),"
- " max(CASE WHEN field=1 THEN value ELSE NULL END),"
- " max(CASE WHEN field=2 THEN value ELSE NULL END),"
- " max(CASE WHEN field=3 THEN value ELSE NULL END),"
- " max(CASE WHEN field=4 THEN value ELSE NULL END)"
- "FROM sqlite_dbdata WHERE pgno IN ("
- " SELECT pgno FROM recovery.map WHERE root=1"
- ")"
- "GROUP BY pgno, cell;"
- "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);"
- );
-
- /* Open a transaction, then print out all non-virtual, non-"sqlite_%"
- ** CREATE TABLE statements that extracted from the existing schema. */
- if( rc==SQLITE_OK ){
- sqlite3_stmt *pStmt = 0;
- /* ".recover" might output content in an order which causes immediate
- ** foreign key constraints to be violated. So disable foreign-key
- ** constraint enforcement to prevent problems when running the output
- ** script. */
- raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n");
- raw_printf(pState->out, "BEGIN;\n");
- raw_printf(pState->out, "PRAGMA writable_schema = on;\n");
- shellPrepare(pState->db, &rc,
- "SELECT sql FROM recovery.schema "
- "WHERE type='table' AND sql LIKE 'create table%'", &pStmt
- );
- while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
- const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0);
- raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n",
- &zCreateTable[12]
- );
- }
- shellFinalize(&rc, pStmt);
- }
-
- /* Figure out if an orphan table will be required. And if so, how many
- ** user columns it should contain */
- shellPrepare(pState->db, &rc,
- "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1"
- , &pLoop
- );
- if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){
- nOrphan = sqlite3_column_int(pLoop, 0);
- }
- shellFinalize(&rc, pLoop);
- pLoop = 0;
-
- shellPrepare(pState->db, &rc,
- "SELECT pgno FROM recovery.map WHERE root=?", &pPages
- );
-
- shellPrepare(pState->db, &rc,
- "SELECT max(field), group_concat(shell_escape_crnl(quote"
- "(case when (? AND field<0) then NULL else value end)"
- "), ', ')"
- ", min(field) "
- "FROM sqlite_dbdata WHERE pgno = ? AND field != ?"
- "GROUP BY cell", &pCells
- );
-
- /* Loop through each root page. */
- shellPrepare(pState->db, &rc,
- "SELECT root, intkey, max(maxlen) FROM recovery.map"
- " WHERE root>1 GROUP BY root, intkey ORDER BY root=("
- " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'"
- ")", &pLoop
- );
- while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){
- int iRoot = sqlite3_column_int(pLoop, 0);
- int bIntkey = sqlite3_column_int(pLoop, 1);
- int nCol = sqlite3_column_int(pLoop, 2);
- int bNoop = 0;
- RecoverTable *pTab;
-
- assert( bIntkey==0 || bIntkey==1 );
- pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop);
- if( bNoop || rc ) continue;
- if( pTab==0 ){
- if( pOrphan==0 ){
- pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan);
- }
- pTab = pOrphan;
- if( pTab==0 ) break;
- }
-
- if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){
- raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n");
- }
- sqlite3_bind_int(pPages, 1, iRoot);
- if( bRowids==0 && pTab->iPk<0 ){
- sqlite3_bind_int(pCells, 1, 1);
- }else{
- sqlite3_bind_int(pCells, 1, 0);
- }
- sqlite3_bind_int(pCells, 3, pTab->iPk);
-
- while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){
- int iPgno = sqlite3_column_int(pPages, 0);
- sqlite3_bind_int(pCells, 2, iPgno);
- while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){
- int nField = sqlite3_column_int(pCells, 0);
- int iMin = sqlite3_column_int(pCells, 2);
- const char *zVal = (const char*)sqlite3_column_text(pCells, 1);
-
- RecoverTable *pTab2 = pTab;
- if( pTab!=pOrphan && (iMin<0)!=bIntkey ){
- if( pOrphan==0 ){
- pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan);
- }
- pTab2 = pOrphan;
- if( pTab2==0 ) break;
- }
-
- nField = nField+1;
- if( pTab2==pOrphan ){
- raw_printf(pState->out,
- "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n",
- pTab2->zQuoted, iRoot, iPgno, nField,
- iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField]
- );
- }else{
- raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n",
- pTab2->zQuoted, pTab2->azlCol[nField], zVal
- );
- }
- }
- shellReset(&rc, pCells);
- }
- shellReset(&rc, pPages);
- if( pTab!=pOrphan ) recoverFreeTable(pTab);
- }
- shellFinalize(&rc, pLoop);
- shellFinalize(&rc, pPages);
- shellFinalize(&rc, pCells);
- recoverFreeTable(pOrphan);
-
- /* The rest of the schema */
- if( rc==SQLITE_OK ){
- sqlite3_stmt *pStmt = 0;
- shellPrepare(pState->db, &rc,
- "SELECT sql, name FROM recovery.schema "
- "WHERE sql NOT LIKE 'create table%'", &pStmt
- );
- while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
- const char *zSql = (const char*)sqlite3_column_text(pStmt, 0);
- if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){
- const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
- char *zPrint = shellMPrintf(&rc,
- "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)",
- zName, zName, zSql
- );
- raw_printf(pState->out, "%s;\n", zPrint);
- sqlite3_free(zPrint);
- }else{
- raw_printf(pState->out, "%s;\n", zSql);
- }
- }
- shellFinalize(&rc, pStmt);
- }
-
- if( rc==SQLITE_OK ){
- raw_printf(pState->out, "PRAGMA writable_schema = off;\n");
- raw_printf(pState->out, "COMMIT;\n");
- }
- sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0);
+ rc = sqlite3_recover_finish(p);
return rc;
}
-#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */
+#endif /* SQLITE_SHELL_HAVE_RECOVER */
/*
@@ -19258,7 +22035,7 @@ static int do_meta_command(char *zLine, ShellState *p){
clearTempFile(p);
#ifndef SQLITE_OMIT_AUTHORIZATION
- if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){
+ if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){
if( nArg!=2 ){
raw_printf(stderr, "Usage: .auth ON|OFF\n");
rc = 1;
@@ -19276,17 +22053,17 @@ static int do_meta_command(char *zLine, ShellState *p){
#endif
#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \
- && !defined(SQLITE_SHELL_WASM_MODE)
- if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){
+ && !defined(SQLITE_SHELL_FIDDLE)
+ if( c=='a' && cli_strncmp(azArg[0], "archive", n)==0 ){
open_db(p, 0);
failIfSafeMode(p, "cannot run .archive in safe mode");
rc = arDotCommand(p, 0, azArg, nArg);
}else
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
- if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0)
- || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
+#ifndef SQLITE_SHELL_FIDDLE
+ if( (c=='b' && n>=3 && cli_strncmp(azArg[0], "backup", n)==0)
+ || (c=='s' && n>=3 && cli_strncmp(azArg[0], "save", n)==0)
){
const char *zDestFile = 0;
const char *zDb = 0;
@@ -19300,10 +22077,10 @@ static int do_meta_command(char *zLine, ShellState *p){
const char *z = azArg[j];
if( z[0]=='-' ){
if( z[1]=='-' ) z++;
- if( strcmp(z, "-append")==0 ){
+ if( cli_strcmp(z, "-append")==0 ){
zVfs = "apndvfs";
}else
- if( strcmp(z, "-async")==0 ){
+ if( cli_strcmp(z, "-async")==0 ){
bAsync = 1;
}else
{
@@ -19353,9 +22130,9 @@ static int do_meta_command(char *zLine, ShellState *p){
}
close_db(pDest);
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){
+ if( c=='b' && n>=3 && cli_strncmp(azArg[0], "bail", n)==0 ){
if( nArg==2 ){
bail_on_error = booleanValue(azArg[1]);
}else{
@@ -19364,7 +22141,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){
+ if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){
if( nArg==2 ){
if( booleanValue(azArg[1]) ){
setBinaryMode(p->out, 1);
@@ -19380,12 +22157,12 @@ static int do_meta_command(char *zLine, ShellState *p){
/* The undocumented ".breakpoint" command causes a call to the no-op
** routine named test_breakpoint().
*/
- if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){
+ if( c=='b' && n>=3 && cli_strncmp(azArg[0], "breakpoint", n)==0 ){
test_breakpoint();
}else
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='c' && strcmp(azArg[0],"cd")==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='c' && cli_strcmp(azArg[0],"cd")==0 ){
failIfSafeMode(p, "cannot run .cd in safe mode");
if( nArg==2 ){
#if defined(_WIN32) || defined(WIN32)
@@ -19404,9 +22181,9 @@ static int do_meta_command(char *zLine, ShellState *p){
rc = 1;
}
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){
+ if( c=='c' && n>=3 && cli_strncmp(azArg[0], "changes", n)==0 ){
if( nArg==2 ){
setOrClearFlag(p, SHFLG_CountChanges, azArg[1]);
}else{
@@ -19415,12 +22192,12 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
/* Cancel output redirection, if it is currently set (by .testcase)
** Then read the content of the testcase-out.txt file and compare against
** azArg[1]. If there are differences, report an error and exit.
*/
- if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){
+ if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){
char *zRes = 0;
output_reset(p);
if( nArg!=2 ){
@@ -19440,10 +22217,10 @@ static int do_meta_command(char *zLine, ShellState *p){
}
sqlite3_free(zRes);
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){
failIfSafeMode(p, "cannot run .clone in safe mode");
if( nArg==2 ){
tryToClone(p, azArg[1]);
@@ -19452,9 +22229,9 @@ static int do_meta_command(char *zLine, ShellState *p){
rc = 1;
}
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='c' && strncmp(azArg[0], "connection", n)==0 ){
+ if( c=='c' && cli_strncmp(azArg[0], "connection", n)==0 ){
if( nArg==1 ){
/* List available connections */
int i;
@@ -19481,7 +22258,7 @@ static int do_meta_command(char *zLine, ShellState *p){
globalDb = p->db = p->pAuxDb->db;
p->pAuxDb->db = 0;
}
- }else if( nArg==3 && strcmp(azArg[1], "close")==0
+ }else if( nArg==3 && cli_strcmp(azArg[1], "close")==0
&& IsDigit(azArg[2][0]) && azArg[2][1]==0 ){
int i = azArg[2][0] - '0';
if( i<0 || i>=ArraySize(p->aAuxDb) ){
@@ -19500,7 +22277,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){
+ if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){
char **azName = 0;
int nName = 0;
sqlite3_stmt *pStmt;
@@ -19539,7 +22316,7 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(azName);
}else
- if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){
+ if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbconfig", n)==0 ){
static const struct DbConfigChoices {
const char *zName;
int op;
@@ -19564,7 +22341,7 @@ static int do_meta_command(char *zLine, ShellState *p){
int ii, v;
open_db(p, 0);
for(ii=0; ii1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue;
+ if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue;
if( nArg>=3 ){
sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0);
}
@@ -19578,18 +22355,18 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
-#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
- if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){
+#if SQLITE_SHELL_HAVE_RECOVER
+ if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbinfo", n)==0 ){
rc = shell_dbinfo_command(p, nArg, azArg);
}else
- if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){
+ if( c=='r' && cli_strncmp(azArg[0], "recover", n)==0 ){
open_db(p, 0);
rc = recoverDatabaseCmd(p, nArg, azArg);
}else
-#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */
+#endif /* SQLITE_SHELL_HAVE_RECOVER */
- if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){
+ if( c=='d' && cli_strncmp(azArg[0], "dump", n)==0 ){
char *zLike = 0;
char *zSql;
int i;
@@ -19602,7 +22379,7 @@ static int do_meta_command(char *zLine, ShellState *p){
if( azArg[i][0]=='-' ){
const char *z = azArg[i]+1;
if( z[0]=='-' ) z++;
- if( strcmp(z,"preserve-rowids")==0 ){
+ if( cli_strcmp(z,"preserve-rowids")==0 ){
#ifdef SQLITE_OMIT_VIRTUALTABLE
raw_printf(stderr, "The --preserve-rowids option is not compatible"
" with SQLITE_OMIT_VIRTUALTABLE\n");
@@ -19613,13 +22390,13 @@ static int do_meta_command(char *zLine, ShellState *p){
ShellSetFlag(p, SHFLG_PreserveRowid);
#endif
}else
- if( strcmp(z,"newlines")==0 ){
+ if( cli_strcmp(z,"newlines")==0 ){
ShellSetFlag(p, SHFLG_Newlines);
}else
- if( strcmp(z,"data-only")==0 ){
+ if( cli_strcmp(z,"data-only")==0 ){
ShellSetFlag(p, SHFLG_DumpDataOnly);
}else
- if( strcmp(z,"nosys")==0 ){
+ if( cli_strcmp(z,"nosys")==0 ){
ShellSetFlag(p, SHFLG_DumpNoSys);
}else
{
@@ -19701,7 +22478,7 @@ static int do_meta_command(char *zLine, ShellState *p){
p->shellFlgs = savedShellFlags;
}else
- if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){
+ if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){
if( nArg==2 ){
setOrClearFlag(p, SHFLG_Echo, azArg[1]);
}else{
@@ -19710,22 +22487,22 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){
+ if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){
if( nArg==2 ){
p->autoEQPtest = 0;
if( p->autoEQPtrace ){
if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0);
p->autoEQPtrace = 0;
}
- if( strcmp(azArg[1],"full")==0 ){
+ if( cli_strcmp(azArg[1],"full")==0 ){
p->autoEQP = AUTOEQP_full;
- }else if( strcmp(azArg[1],"trigger")==0 ){
+ }else if( cli_strcmp(azArg[1],"trigger")==0 ){
p->autoEQP = AUTOEQP_trigger;
#ifdef SQLITE_DEBUG
- }else if( strcmp(azArg[1],"test")==0 ){
+ }else if( cli_strcmp(azArg[1],"test")==0 ){
p->autoEQP = AUTOEQP_on;
p->autoEQPtest = 1;
- }else if( strcmp(azArg[1],"trace")==0 ){
+ }else if( cli_strcmp(azArg[1],"trace")==0 ){
p->autoEQP = AUTOEQP_full;
p->autoEQPtrace = 1;
open_db(p, 0);
@@ -19741,8 +22518,8 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){
if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc);
rc = 2;
}else
@@ -19750,10 +22527,10 @@ static int do_meta_command(char *zLine, ShellState *p){
/* The ".explain" command is automatic now. It is largely pointless. It
** retained purely for backwards compatibility */
- if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){
+ if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){
int val = 1;
if( nArg>=2 ){
- if( strcmp(azArg[1],"auto")==0 ){
+ if( cli_strcmp(azArg[1],"auto")==0 ){
val = 99;
}else{
val = booleanValue(azArg[1]);
@@ -19773,7 +22550,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#ifndef SQLITE_OMIT_VIRTUALTABLE
- if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){
+ if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){
if( p->bSafeMode ){
raw_printf(stderr,
"Cannot run experimental commands such as \"%s\" in safe mode\n",
@@ -19786,7 +22563,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#endif
- if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){
+ if( c=='f' && cli_strncmp(azArg[0], "filectrl", n)==0 ){
static const struct {
const char *zCtrlName; /* Name of a test-control option */
int ctrlCode; /* Integer code for that option */
@@ -19816,7 +22593,7 @@ static int do_meta_command(char *zLine, ShellState *p){
zCmd = nArg>=2 ? azArg[1] : "help";
if( zCmd[0]=='-'
- && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0)
+ && (cli_strcmp(zCmd,"--schema")==0 || cli_strcmp(zCmd,"-schema")==0)
&& nArg>=4
){
zSchema = azArg[2];
@@ -19832,7 +22609,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
/* --help lists all file-controls */
- if( strcmp(zCmd,"help")==0 ){
+ if( cli_strcmp(zCmd,"help")==0 ){
utf8_printf(p->out, "Available file-controls:\n");
for(i=0; iout, " .filectrl %s %s\n",
@@ -19846,7 +22623,7 @@ static int do_meta_command(char *zLine, ShellState *p){
** of the option name, or a numerical value. */
n2 = strlen30(zCmd);
for(i=0; ishowHeader = booleanValue(azArg[1]);
p->shellFlgs |= SHFLG_HeaderSet;
@@ -19990,7 +22767,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='h' && strncmp(azArg[0], "help", n)==0 ){
+ if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){
if( nArg>=2 ){
n = showHelp(p->out, azArg[1]);
if( n==0 ){
@@ -20001,8 +22778,8 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='i' && strncmp(azArg[0], "import", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){
char *zTable = 0; /* Insert data into this table */
char *zSchema = 0; /* within this schema (may default to "main") */
char *zFile = 0; /* Name of file to extra content from */
@@ -20042,18 +22819,18 @@ static int do_meta_command(char *zLine, ShellState *p){
showHelp(p->out, "import");
goto meta_command_exit;
}
- }else if( strcmp(z,"-v")==0 ){
+ }else if( cli_strcmp(z,"-v")==0 ){
eVerbose++;
- }else if( strcmp(z,"-schema")==0 && imode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){
+ if( nSep==2 && p->mode==MODE_Csv
+ && cli_strcmp(p->rowSeparator,SEP_CrLf)==0
+ ){
/* When importing CSV (only), if the row separator is set to the
** default output row separator, change it to the default input
** row separator. This avoids having to maintain different input
@@ -20292,10 +23071,10 @@ static int do_meta_command(char *zLine, ShellState *p){
sCtx.nRow, sCtx.nErr, sCtx.nLine-1);
}
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
#ifndef SQLITE_UNTESTABLE
- if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){
+ if( c=='i' && cli_strncmp(azArg[0], "imposter", n)==0 ){
char *zSql;
char *zCollist = 0;
sqlite3_stmt *pStmt;
@@ -20396,13 +23175,13 @@ static int do_meta_command(char *zLine, ShellState *p){
#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */
#ifdef SQLITE_ENABLE_IOTRACE
- if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){
+ if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){
SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...);
if( iotrace && iotrace!=stdout ) fclose(iotrace);
iotrace = 0;
if( nArg<2 ){
sqlite3IoTrace = 0;
- }else if( strcmp(azArg[1], "-")==0 ){
+ }else if( cli_strcmp(azArg[1], "-")==0 ){
sqlite3IoTrace = iotracePrintf;
iotrace = stdout;
}else{
@@ -20418,7 +23197,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#endif
- if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){
+ if( c=='l' && n>=5 && cli_strncmp(azArg[0], "limits", n)==0 ){
static const struct {
const char *zLimitName; /* Name of a limit */
int limitCode; /* Integer code for that limit */
@@ -20477,13 +23256,13 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){
+ if( c=='l' && n>2 && cli_strncmp(azArg[0], "lint", n)==0 ){
open_db(p, 0);
lintDotCommand(p, azArg, nArg);
}else
-#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_MODE)
- if( c=='l' && strncmp(azArg[0], "load", n)==0 ){
+#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE)
+ if( c=='l' && cli_strncmp(azArg[0], "load", n)==0 ){
const char *zFile, *zProc;
char *zErrMsg = 0;
failIfSafeMode(p, "cannot run .load in safe mode");
@@ -20504,8 +23283,8 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='l' && strncmp(azArg[0], "log", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){
failIfSafeMode(p, "cannot run .log in safe mode");
if( nArg!=2 ){
raw_printf(stderr, "Usage: .log FILENAME\n");
@@ -20518,7 +23297,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#endif
- if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){
+ if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){
const char *zMode = 0;
const char *zTabname = 0;
int i, n2;
@@ -20537,10 +23316,10 @@ static int do_meta_command(char *zLine, ShellState *p){
cmOpts.bQuote = 0;
}else if( zMode==0 ){
zMode = z;
- /* Apply defaults for qbox pseudo-mods. If that
+ /* Apply defaults for qbox pseudo-mode. If that
* overwrites already-set values, user was informed of this.
*/
- if( strcmp(z, "qbox")==0 ){
+ if( cli_strcmp(z, "qbox")==0 ){
ColModeOpts cmo = ColModeOpts_default_qbox;
zMode = "box";
cmOpts = cmo;
@@ -20579,58 +23358,58 @@ static int do_meta_command(char *zLine, ShellState *p){
zMode = modeDescr[p->mode];
}
n2 = strlen30(zMode);
- if( strncmp(zMode,"lines",n2)==0 ){
+ if( cli_strncmp(zMode,"lines",n2)==0 ){
p->mode = MODE_Line;
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
- }else if( strncmp(zMode,"columns",n2)==0 ){
+ }else if( cli_strncmp(zMode,"columns",n2)==0 ){
p->mode = MODE_Column;
if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){
p->showHeader = 1;
}
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
p->cmOpts = cmOpts;
- }else if( strncmp(zMode,"list",n2)==0 ){
+ }else if( cli_strncmp(zMode,"list",n2)==0 ){
p->mode = MODE_List;
sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column);
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
- }else if( strncmp(zMode,"html",n2)==0 ){
+ }else if( cli_strncmp(zMode,"html",n2)==0 ){
p->mode = MODE_Html;
- }else if( strncmp(zMode,"tcl",n2)==0 ){
+ }else if( cli_strncmp(zMode,"tcl",n2)==0 ){
p->mode = MODE_Tcl;
sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space);
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
- }else if( strncmp(zMode,"csv",n2)==0 ){
+ }else if( cli_strncmp(zMode,"csv",n2)==0 ){
p->mode = MODE_Csv;
sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
- }else if( strncmp(zMode,"tabs",n2)==0 ){
+ }else if( cli_strncmp(zMode,"tabs",n2)==0 ){
p->mode = MODE_List;
sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab);
- }else if( strncmp(zMode,"insert",n2)==0 ){
+ }else if( cli_strncmp(zMode,"insert",n2)==0 ){
p->mode = MODE_Insert;
set_table_name(p, zTabname ? zTabname : "table");
- }else if( strncmp(zMode,"quote",n2)==0 ){
+ }else if( cli_strncmp(zMode,"quote",n2)==0 ){
p->mode = MODE_Quote;
sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
- }else if( strncmp(zMode,"ascii",n2)==0 ){
+ }else if( cli_strncmp(zMode,"ascii",n2)==0 ){
p->mode = MODE_Ascii;
sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit);
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record);
- }else if( strncmp(zMode,"markdown",n2)==0 ){
+ }else if( cli_strncmp(zMode,"markdown",n2)==0 ){
p->mode = MODE_Markdown;
p->cmOpts = cmOpts;
- }else if( strncmp(zMode,"table",n2)==0 ){
+ }else if( cli_strncmp(zMode,"table",n2)==0 ){
p->mode = MODE_Table;
p->cmOpts = cmOpts;
- }else if( strncmp(zMode,"box",n2)==0 ){
+ }else if( cli_strncmp(zMode,"box",n2)==0 ){
p->mode = MODE_Box;
p->cmOpts = cmOpts;
- }else if( strncmp(zMode,"count",n2)==0 ){
+ }else if( cli_strncmp(zMode,"count",n2)==0 ){
p->mode = MODE_Count;
- }else if( strncmp(zMode,"off",n2)==0 ){
+ }else if( cli_strncmp(zMode,"off",n2)==0 ){
p->mode = MODE_Off;
- }else if( strncmp(zMode,"json",n2)==0 ){
+ }else if( cli_strncmp(zMode,"json",n2)==0 ){
p->mode = MODE_Json;
}else{
raw_printf(stderr, "Error: mode should be one of: "
@@ -20641,12 +23420,12 @@ static int do_meta_command(char *zLine, ShellState *p){
p->cMode = p->mode;
}else
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='n' && strcmp(azArg[0], "nonce")==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){
if( nArg!=2 ){
raw_printf(stderr, "Usage: .nonce NONCE\n");
rc = 1;
- }else if( p->zNonce==0 || strcmp(azArg[1],p->zNonce)!=0 ){
+ }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){
raw_printf(stderr, "line %d: incorrect nonce: \"%s\"\n",
p->lineno, azArg[1]);
exit(1);
@@ -20656,9 +23435,9 @@ static int do_meta_command(char *zLine, ShellState *p){
** at the end of this procedure */
}
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){
+ if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){
if( nArg==2 ){
sqlite3_snprintf(sizeof(p->nullValue), p->nullValue,
"%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]);
@@ -20668,7 +23447,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){
+ if( c=='o' && cli_strncmp(azArg[0], "open", n)==0 && n>=2 ){
const char *zFN = 0; /* Pointer to constant filename */
char *zNewFilename = 0; /* Name of the database file to open */
int iName = 1; /* Index in azArg[] of the filename */
@@ -20678,7 +23457,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* Check for command-line arguments */
for(iName=1; iNameszMax = integerValue(azArg[++iName]);
#endif /* SQLITE_OMIT_DESERIALIZE */
}else
-#endif /* !SQLITE_SHELL_WASM_MODE */
+#endif /* !SQLITE_SHELL_FIDDLE */
if( z[0]=='-' ){
utf8_printf(stderr, "unknown option: %s\n", z);
rc = 1;
@@ -20728,11 +23507,11 @@ static int do_meta_command(char *zLine, ShellState *p){
/* If a filename is specified, try to open it first */
if( zFN || p->openMode==SHELL_OPEN_HEXDB ){
if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN);
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
if( p->bSafeMode
&& p->openMode!=SHELL_OPEN_HEXDB
&& zFN
- && strcmp(zFN,":memory:")!=0
+ && cli_strcmp(zFN,":memory:")!=0
){
failIfSafeMode(p, "cannot open disk-based database files in safe mode");
}
@@ -20761,10 +23540,11 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
if( (c=='o'
- && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0))
- || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0)
+ && (cli_strncmp(azArg[0], "output", n)==0
+ || cli_strncmp(azArg[0], "once", n)==0))
+ || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0)
){
char *zFile = 0;
int bTxtMode = 0;
@@ -20778,21 +23558,21 @@ static int do_meta_command(char *zLine, ShellState *p){
if( c=='e' ){
eMode = 'x';
bOnce = 2;
- }else if( strncmp(azArg[0],"once",n)==0 ){
+ }else if( cli_strncmp(azArg[0],"once",n)==0 ){
bOnce = 1;
}
for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n",
@@ -20865,7 +23645,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else{
p->out = output_file_open(zFile, bTxtMode);
if( p->out==0 ){
- if( strcmp(zFile,"off")!=0 ){
+ if( cli_strcmp(zFile,"off")!=0 ){
utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile);
}
p->out = stdout;
@@ -20877,16 +23657,16 @@ static int do_meta_command(char *zLine, ShellState *p){
}
sqlite3_free(zFile);
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){
+ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "parameter", n)==0 ){
open_db(p,0);
if( nArg<=1 ) goto parameter_syntax_error;
/* .parameter clear
** Clear all bind parameters by dropping the TEMP table that holds them.
*/
- if( nArg==2 && strcmp(azArg[1],"clear")==0 ){
+ if( nArg==2 && cli_strcmp(azArg[1],"clear")==0 ){
sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;",
0, 0, 0);
}else
@@ -20894,7 +23674,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .parameter list
** List all bind parameters.
*/
- if( nArg==2 && strcmp(azArg[1],"list")==0 ){
+ if( nArg==2 && cli_strcmp(azArg[1],"list")==0 ){
sqlite3_stmt *pStmt = 0;
int rx;
int len = 0;
@@ -20923,7 +23703,7 @@ static int do_meta_command(char *zLine, ShellState *p){
** Make sure the TEMP table used to hold bind parameters exists.
** Create it if necessary.
*/
- if( nArg==2 && strcmp(azArg[1],"init")==0 ){
+ if( nArg==2 && cli_strcmp(azArg[1],"init")==0 ){
bind_table_init(p);
}else
@@ -20933,7 +23713,7 @@ static int do_meta_command(char *zLine, ShellState *p){
** VALUE can be in either SQL literal notation, or if not it will be
** understood to be a text string.
*/
- if( nArg==4 && strcmp(azArg[1],"set")==0 ){
+ if( nArg==4 && cli_strcmp(azArg[1],"set")==0 ){
int rx;
char *zSql;
sqlite3_stmt *pStmt;
@@ -20971,7 +23751,7 @@ static int do_meta_command(char *zLine, ShellState *p){
** Remove the NAME binding from the parameter binding table, if it
** exists.
*/
- if( nArg==3 && strcmp(azArg[1],"unset")==0 ){
+ if( nArg==3 && cli_strcmp(azArg[1],"unset")==0 ){
char *zSql = sqlite3_mprintf(
"DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]);
shell_check_oom(zSql);
@@ -20983,7 +23763,7 @@ static int do_meta_command(char *zLine, ShellState *p){
showHelp(p->out, "parameter");
}else
- if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){
+ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){
int i;
for(i=1; i1 ) raw_printf(p->out, " ");
@@ -20993,7 +23773,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
- if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){
+ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "progress", n)==0 ){
int i;
int nn = 0;
p->flgProgress = 0;
@@ -21004,19 +23784,19 @@ static int do_meta_command(char *zLine, ShellState *p){
if( z[0]=='-' ){
z++;
if( z[0]=='-' ) z++;
- if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){
+ if( cli_strcmp(z,"quiet")==0 || cli_strcmp(z,"q")==0 ){
p->flgProgress |= SHELL_PROGRESS_QUIET;
continue;
}
- if( strcmp(z,"reset")==0 ){
+ if( cli_strcmp(z,"reset")==0 ){
p->flgProgress |= SHELL_PROGRESS_RESET;
continue;
}
- if( strcmp(z,"once")==0 ){
+ if( cli_strcmp(z,"once")==0 ){
p->flgProgress |= SHELL_PROGRESS_ONCE;
continue;
}
- if( strcmp(z,"limit")==0 ){
+ if( cli_strcmp(z,"limit")==0 ){
if( i+1>=nArg ){
utf8_printf(stderr, "Error: missing argument on --limit\n");
rc = 1;
@@ -21038,7 +23818,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */
- if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){
+ if( c=='p' && cli_strncmp(azArg[0], "prompt", n)==0 ){
if( nArg >= 2) {
strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1);
}
@@ -21047,14 +23827,14 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='q' && cli_strncmp(azArg[0], "quit", n)==0 ){
rc = 2;
}else
#endif
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){
FILE *inSaved = p->in;
int savedLineno = p->lineno;
failIfSafeMode(p, "cannot run .read in safe mode");
@@ -21088,10 +23868,10 @@ static int do_meta_command(char *zLine, ShellState *p){
p->in = inSaved;
p->lineno = savedLineno;
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
-#ifndef SQLITE_SHELL_WASM_MODE
- if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){
+#ifndef SQLITE_SHELL_FIDDLE
+ if( c=='r' && n>=3 && cli_strncmp(azArg[0], "restore", n)==0 ){
const char *zSrcFile;
const char *zDb;
sqlite3 *pSrc;
@@ -21142,9 +23922,9 @@ static int do_meta_command(char *zLine, ShellState *p){
}
close_db(pSrc);
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){
+ if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){
if( nArg==2 ){
p->scanstatsOn = (u8)booleanValue(azArg[1]);
#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -21156,7 +23936,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){
+ if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){
ShellText sSelect;
ShellState data;
char *zErrMsg = 0;
@@ -21299,15 +24079,15 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( (c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0)
- || (c=='t' && n==9 && strncmp(azArg[0], "treetrace", n)==0)
+ if( (c=='s' && n==11 && cli_strncmp(azArg[0], "selecttrace", n)==0)
+ || (c=='t' && n==9 && cli_strncmp(azArg[0], "treetrace", n)==0)
){
unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff;
sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x);
}else
#if defined(SQLITE_ENABLE_SESSION)
- if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){
+ if( c=='s' && cli_strncmp(azArg[0],"session",n)==0 && n>=3 ){
struct AuxDb *pAuxDb = p->pAuxDb;
OpenSession *pSession = &pAuxDb->aSession[0];
char **azCmd = &azArg[1];
@@ -21318,7 +24098,7 @@ static int do_meta_command(char *zLine, ShellState *p){
open_db(p, 0);
if( nArg>=3 ){
for(iSes=0; iSesnSession; iSes++){
- if( strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break;
+ if( cli_strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break;
}
if( iSesnSession ){
pSession = &pAuxDb->aSession[iSes];
@@ -21334,7 +24114,7 @@ static int do_meta_command(char *zLine, ShellState *p){
** Invoke the sqlite3session_attach() interface to attach a particular
** table so that it is never filtered.
*/
- if( strcmp(azCmd[0],"attach")==0 ){
+ if( cli_strcmp(azCmd[0],"attach")==0 ){
if( nCmd!=2 ) goto session_syntax_error;
if( pSession->p==0 ){
session_not_open:
@@ -21352,7 +24132,9 @@ static int do_meta_command(char *zLine, ShellState *p){
** .session patchset FILE
** Write a changeset or patchset into a file. The file is overwritten.
*/
- if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){
+ if( cli_strcmp(azCmd[0],"changeset")==0
+ || cli_strcmp(azCmd[0],"patchset")==0
+ ){
FILE *out = 0;
failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]);
if( nCmd!=2 ) goto session_syntax_error;
@@ -21386,7 +24168,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .session close
** Close the identified session
*/
- if( strcmp(azCmd[0], "close")==0 ){
+ if( cli_strcmp(azCmd[0], "close")==0 ){
if( nCmd!=1 ) goto session_syntax_error;
if( pAuxDb->nSession ){
session_close(pSession);
@@ -21397,7 +24179,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .session enable ?BOOLEAN?
** Query or set the enable flag
*/
- if( strcmp(azCmd[0], "enable")==0 ){
+ if( cli_strcmp(azCmd[0], "enable")==0 ){
int ii;
if( nCmd>2 ) goto session_syntax_error;
ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
@@ -21411,7 +24193,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .session filter GLOB ....
** Set a list of GLOB patterns of table names to be excluded.
*/
- if( strcmp(azCmd[0], "filter")==0 ){
+ if( cli_strcmp(azCmd[0], "filter")==0 ){
int ii, nByte;
if( nCmd<2 ) goto session_syntax_error;
if( pAuxDb->nSession ){
@@ -21436,7 +24218,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .session indirect ?BOOLEAN?
** Query or set the indirect flag
*/
- if( strcmp(azCmd[0], "indirect")==0 ){
+ if( cli_strcmp(azCmd[0], "indirect")==0 ){
int ii;
if( nCmd>2 ) goto session_syntax_error;
ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
@@ -21450,7 +24232,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .session isempty
** Determine if the session is empty
*/
- if( strcmp(azCmd[0], "isempty")==0 ){
+ if( cli_strcmp(azCmd[0], "isempty")==0 ){
int ii;
if( nCmd!=1 ) goto session_syntax_error;
if( pAuxDb->nSession ){
@@ -21463,7 +24245,7 @@ static int do_meta_command(char *zLine, ShellState *p){
/* .session list
** List all currently open sessions
*/
- if( strcmp(azCmd[0],"list")==0 ){
+ if( cli_strcmp(azCmd[0],"list")==0 ){
for(i=0; inSession; i++){
utf8_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName);
}
@@ -21473,13 +24255,13 @@ static int do_meta_command(char *zLine, ShellState *p){
** Open a new session called NAME on the attached database DB.
** DB is normally "main".
*/
- if( strcmp(azCmd[0],"open")==0 ){
+ if( cli_strcmp(azCmd[0],"open")==0 ){
char *zName;
if( nCmd!=3 ) goto session_syntax_error;
zName = azCmd[2];
if( zName[0]==0 ) goto session_syntax_error;
for(i=0; inSession; i++){
- if( strcmp(pAuxDb->aSession[i].zName,zName)==0 ){
+ if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){
utf8_printf(stderr, "Session \"%s\" already exists\n", zName);
goto meta_command_exit;
}
@@ -21510,15 +24292,15 @@ static int do_meta_command(char *zLine, ShellState *p){
#ifdef SQLITE_DEBUG
/* Undocumented commands for internal testing. Subject to change
** without notice. */
- if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){
- if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){
+ if( c=='s' && n>=10 && cli_strncmp(azArg[0], "selftest-", 9)==0 ){
+ if( cli_strncmp(azArg[0]+9, "boolean", n-9)==0 ){
int i, v;
for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v);
}
}
- if( strncmp(azArg[0]+9, "integer", n-9)==0 ){
+ if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){
int i; sqlite3_int64 v;
for(i=1; i=4 && strncmp(azArg[0],"selftest",n)==0 ){
+ if( c=='s' && n>=4 && cli_strncmp(azArg[0],"selftest",n)==0 ){
int bIsInit = 0; /* True to initialize the SELFTEST table */
int bVerbose = 0; /* Verbose output */
int bSelftestExists; /* True if SELFTEST already exists */
@@ -21544,10 +24326,10 @@ static int do_meta_command(char *zLine, ShellState *p){
for(i=1; i0 ){
printf("%d: %s %s\n", tno, zOp, zSql);
}
- if( strcmp(zOp,"memo")==0 ){
+ if( cli_strcmp(zOp,"memo")==0 ){
utf8_printf(p->out, "%s\n", zSql);
}else
- if( strcmp(zOp,"run")==0 ){
+ if( cli_strcmp(zOp,"run")==0 ){
char *zErrMsg = 0;
str.n = 0;
str.z[0] = 0;
@@ -21617,7 +24399,7 @@ static int do_meta_command(char *zLine, ShellState *p){
rc = 1;
utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg);
sqlite3_free(zErrMsg);
- }else if( strcmp(zAns,str.z)!=0 ){
+ }else if( cli_strcmp(zAns,str.z)!=0 ){
nErr++;
rc = 1;
utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns);
@@ -21637,7 +24419,7 @@ static int do_meta_command(char *zLine, ShellState *p){
utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest);
}else
- if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){
+ if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){
if( nArg<2 || nArg>3 ){
raw_printf(stderr, "Usage: .separator COL ?ROW?\n");
rc = 1;
@@ -21652,7 +24434,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){
+ if( c=='s' && n>=4 && cli_strncmp(azArg[0],"sha3sum",n)==0 ){
const char *zLike = 0; /* Which table to checksum. 0 means everything */
int i; /* Loop counter */
int bSchema = 0; /* Also hash the schema */
@@ -21670,15 +24452,15 @@ static int do_meta_command(char *zLine, ShellState *p){
if( z[0]=='-' ){
z++;
if( z[0]=='-' ) z++;
- if( strcmp(z,"schema")==0 ){
+ if( cli_strcmp(z,"schema")==0 ){
bSchema = 1;
}else
- if( strcmp(z,"sha3-224")==0 || strcmp(z,"sha3-256")==0
- || strcmp(z,"sha3-384")==0 || strcmp(z,"sha3-512")==0
+ if( cli_strcmp(z,"sha3-224")==0 || cli_strcmp(z,"sha3-256")==0
+ || cli_strcmp(z,"sha3-384")==0 || cli_strcmp(z,"sha3-512")==0
){
iSize = atoi(&z[5]);
}else
- if( strcmp(z,"debug")==0 ){
+ if( cli_strcmp(z,"debug")==0 ){
bDebug = 1;
}else
{
@@ -21718,20 +24500,20 @@ static int do_meta_command(char *zLine, ShellState *p){
const char *zTab = (const char*)sqlite3_column_text(pStmt,0);
if( zTab==0 ) continue;
if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue;
- if( strncmp(zTab, "sqlite_",7)!=0 ){
+ if( cli_strncmp(zTab, "sqlite_",7)!=0 ){
appendText(&sQuery,"SELECT * FROM ", 0);
appendText(&sQuery,zTab,'"');
appendText(&sQuery," NOT INDEXED;", 0);
- }else if( strcmp(zTab, "sqlite_schema")==0 ){
+ }else if( cli_strcmp(zTab, "sqlite_schema")==0 ){
appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema"
" ORDER BY name;", 0);
- }else if( strcmp(zTab, "sqlite_sequence")==0 ){
+ }else if( cli_strcmp(zTab, "sqlite_sequence")==0 ){
appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence"
" ORDER BY name;", 0);
- }else if( strcmp(zTab, "sqlite_stat1")==0 ){
+ }else if( cli_strcmp(zTab, "sqlite_stat1")==0 ){
appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1"
" ORDER BY tbl,idx;", 0);
- }else if( strcmp(zTab, "sqlite_stat4")==0 ){
+ }else if( cli_strcmp(zTab, "sqlite_stat4")==0 ){
appendText(&sQuery, "SELECT * FROM ", 0);
appendText(&sQuery, zTab, 0);
appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0);
@@ -21768,9 +24550,10 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(zSql);
}else
-#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE)
+#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
if( c=='s'
- && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0)
+ && (cli_strncmp(azArg[0], "shell", n)==0
+ || cli_strncmp(azArg[0],"system",n)==0)
){
char *zCmd;
int i, x;
@@ -21789,9 +24572,9 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(zCmd);
if( x ) raw_printf(stderr, "System command returns %d\n", x);
}else
-#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */
- if( c=='s' && strncmp(azArg[0], "show", n)==0 ){
+ if( c=='s' && cli_strncmp(azArg[0], "show", n)==0 ){
static const char *azBool[] = { "off", "on", "trigger", "full"};
const char *zOut;
int i;
@@ -21844,11 +24627,11 @@ static int do_meta_command(char *zLine, ShellState *p){
p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : "");
}else
- if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){
+ if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){
if( nArg==2 ){
- if( strcmp(azArg[1],"stmt")==0 ){
+ if( cli_strcmp(azArg[1],"stmt")==0 ){
p->statsOn = 2;
- }else if( strcmp(azArg[1],"vmstep")==0 ){
+ }else if( cli_strcmp(azArg[1],"vmstep")==0 ){
p->statsOn = 3;
}else{
p->statsOn = (u8)booleanValue(azArg[1]);
@@ -21861,9 +24644,9 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0)
- || (c=='i' && (strncmp(azArg[0], "indices", n)==0
- || strncmp(azArg[0], "indexes", n)==0) )
+ if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0)
+ || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0
+ || cli_strncmp(azArg[0], "indexes", n)==0) )
){
sqlite3_stmt *pStmt;
char **azResult;
@@ -21969,9 +24752,9 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(azResult);
}else
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
/* Begin redirecting output to the file "testcase-out.txt" */
- if( c=='t' && strcmp(azArg[0],"testcase")==0 ){
+ if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){
output_reset(p);
p->out = output_file_open("testcase-out.txt", 0);
if( p->out==0 ){
@@ -21983,10 +24766,10 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?");
}
}else
-#endif /* !defined(SQLITE_SHELL_WASM_MODE) */
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
#ifndef SQLITE_UNTESTABLE
- if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){
+ if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){
static const struct {
const char *zCtrlName; /* Name of a test-control option */
int ctrlCode; /* Integer code for that option */
@@ -22033,7 +24816,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
/* --help lists all test-controls */
- if( strcmp(zCmd,"help")==0 ){
+ if( cli_strcmp(zCmd,"help")==0 ){
utf8_printf(p->out, "Available test-controls:\n");
for(i=0; iout, " .testctrl %s %s\n",
@@ -22047,7 +24830,7 @@ static int do_meta_command(char *zLine, ShellState *p){
** of the option name, or a numerical value. */
n2 = strlen30(zCmd);
for(i=0; i4 && strncmp(azArg[0], "timeout", n)==0 ){
+ if( c=='t' && n>4 && cli_strncmp(azArg[0], "timeout", n)==0 ){
open_db(p, 0);
sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0);
}else
- if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){
+ if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){
if( nArg==2 ){
enableTimer = booleanValue(azArg[1]);
if( enableTimer && !HAS_TIMER ){
@@ -22238,7 +25021,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#ifndef SQLITE_OMIT_TRACE
- if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
+ if( c=='t' && cli_strncmp(azArg[0], "trace", n)==0 ){
int mType = 0;
int jj;
open_db(p, 0);
@@ -22288,7 +25071,7 @@ static int do_meta_command(char *zLine, ShellState *p){
#endif /* !defined(SQLITE_OMIT_TRACE) */
#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE)
- if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){
+ if( c=='u' && cli_strncmp(azArg[0], "unmodule", n)==0 ){
int ii;
int lenOpt;
char *zOpt;
@@ -22301,7 +25084,7 @@ static int do_meta_command(char *zLine, ShellState *p){
zOpt = azArg[1];
if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++;
lenOpt = (int)strlen(zOpt);
- if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){
+ if( lenOpt>=3 && cli_strncmp(zOpt, "-allexcept",lenOpt)==0 ){
assert( azArg[nArg]==0 );
sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0);
}else{
@@ -22313,14 +25096,14 @@ static int do_meta_command(char *zLine, ShellState *p){
#endif
#if SQLITE_USER_AUTHENTICATION
- if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
+ if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){
if( nArg<2 ){
raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
rc = 1;
goto meta_command_exit;
}
open_db(p, 0);
- if( strcmp(azArg[1],"login")==0 ){
+ if( cli_strcmp(azArg[1],"login")==0 ){
if( nArg!=4 ){
raw_printf(stderr, "Usage: .user login USER PASSWORD\n");
rc = 1;
@@ -22332,7 +25115,7 @@ static int do_meta_command(char *zLine, ShellState *p){
utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]);
rc = 1;
}
- }else if( strcmp(azArg[1],"add")==0 ){
+ }else if( cli_strcmp(azArg[1],"add")==0 ){
if( nArg!=5 ){
raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n");
rc = 1;
@@ -22344,7 +25127,7 @@ static int do_meta_command(char *zLine, ShellState *p){
raw_printf(stderr, "User-Add failed: %d\n", rc);
rc = 1;
}
- }else if( strcmp(azArg[1],"edit")==0 ){
+ }else if( cli_strcmp(azArg[1],"edit")==0 ){
if( nArg!=5 ){
raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n");
rc = 1;
@@ -22356,7 +25139,7 @@ static int do_meta_command(char *zLine, ShellState *p){
raw_printf(stderr, "User-Edit failed: %d\n", rc);
rc = 1;
}
- }else if( strcmp(azArg[1],"delete")==0 ){
+ }else if( cli_strcmp(azArg[1],"delete")==0 ){
if( nArg!=3 ){
raw_printf(stderr, "Usage: .user delete USER\n");
rc = 1;
@@ -22375,7 +25158,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
#endif /* SQLITE_USER_AUTHENTICATION */
- if( c=='v' && strncmp(azArg[0], "version", n)==0 ){
+ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){
utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
sqlite3_libversion(), sqlite3_sourceid());
#if SQLITE_HAVE_ZLIB
@@ -22394,7 +25177,7 @@ static int do_meta_command(char *zLine, ShellState *p){
#endif
}else
- if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){
+ if( c=='v' && cli_strncmp(azArg[0], "vfsinfo", n)==0 ){
const char *zDbName = nArg==2 ? azArg[1] : "main";
sqlite3_vfs *pVfs = 0;
if( p->db ){
@@ -22408,7 +25191,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){
+ if( c=='v' && cli_strncmp(azArg[0], "vfslist", n)==0 ){
sqlite3_vfs *pVfs;
sqlite3_vfs *pCurrent = 0;
if( p->db ){
@@ -22426,7 +25209,7 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){
+ if( c=='v' && cli_strncmp(azArg[0], "vfsname", n)==0 ){
const char *zDbName = nArg==2 ? azArg[1] : "main";
char *zVfsName = 0;
if( p->db ){
@@ -22438,12 +25221,12 @@ static int do_meta_command(char *zLine, ShellState *p){
}
}else
- if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){
+ if( c=='w' && cli_strncmp(azArg[0], "wheretrace", n)==0 ){
unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff;
sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x);
}else
- if( c=='w' && strncmp(azArg[0], "width", n)==0 ){
+ if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){
int j;
assert( nArg<=ArraySize(azArg) );
p->nWidth = nArg-1;
@@ -22621,10 +25404,10 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
if( zErrMsg==0 ){
zErrorType = "Error";
zErrorTail = sqlite3_errmsg(p->db);
- }else if( strncmp(zErrMsg, "in prepare, ",12)==0 ){
+ }else if( cli_strncmp(zErrMsg, "in prepare, ",12)==0 ){
zErrorType = "Parse error";
zErrorTail = &zErrMsg[12];
- }else if( strncmp(zErrMsg, "stepping, ", 10)==0 ){
+ }else if( cli_strncmp(zErrMsg, "stepping, ", 10)==0 ){
zErrorType = "Runtime error";
zErrorTail = &zErrMsg[10];
}else{
@@ -22655,7 +25438,7 @@ static void echo_group_input(ShellState *p, const char *zDo){
if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo);
}
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
/*
** Alternate one_input_line() impl for wasm mode. This is not in the primary impl
** because we need the global shellState and cannot access it from that function
@@ -22666,7 +25449,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
const char *zBegin = shellState.wasm.zPos;
const char *z = zBegin;
char *zLine = 0;
- int nZ = 0;
+ i64 nZ = 0;
UNUSED_PARAMETER(in);
UNUSED_PARAMETER(isContinuation);
@@ -22682,11 +25465,11 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
shellState.wasm.zPos = z;
zLine = realloc(zPrior, nZ+1);
shell_check_oom(zLine);
- memcpy(zLine, zBegin, (size_t)nZ);
+ memcpy(zLine, zBegin, nZ);
zLine[nZ] = 0;
return zLine;
}
-#endif /* SQLITE_SHELL_WASM_MODE */
+#endif /* SQLITE_SHELL_FIDDLE */
/*
** Read input from *in and process it. If *in==0 then input
@@ -22700,12 +25483,12 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
static int process_input(ShellState *p){
char *zLine = 0; /* A single input line */
char *zSql = 0; /* Accumulated SQL text */
- int nLine; /* Length of current line */
- int nSql = 0; /* Bytes of zSql[] used */
- int nAlloc = 0; /* Allocated zSql[] space */
+ i64 nLine; /* Length of current line */
+ i64 nSql = 0; /* Bytes of zSql[] used */
+ i64 nAlloc = 0; /* Allocated zSql[] space */
int rc; /* Error code */
int errCnt = 0; /* Number of errors seen */
- int startline = 0; /* Line number for start of current input */
+ i64 startline = 0; /* Line number for start of current input */
QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */
if( p->inputNesting==MAX_INPUT_NESTING ){
@@ -22755,7 +25538,7 @@ static int process_input(ShellState *p){
continue;
}
/* No single-line dispositions remain; accumulate line(s). */
- nLine = strlen30(zLine);
+ nLine = strlen(zLine);
if( nSql+nLine+2>=nAlloc ){
/* Grow buffer by half-again increments when big. */
nAlloc = nSql+(nSql>>1)+nLine+100;
@@ -22763,7 +25546,7 @@ static int process_input(ShellState *p){
shell_check_oom(zSql);
}
if( nSql==0 ){
- int i;
+ i64 i;
for(i=0; zLine[i] && IsSpace(zLine[i]); i++){}
assert( nAlloc>0 && zSql!=0 );
memcpy(zSql, zLine+i, nLine+1-i);
@@ -22863,7 +25646,7 @@ static char *find_home_dir(int clearFlag){
#endif /* !_WIN32_WCE */
if( home_dir ){
- int n = strlen30(home_dir) + 1;
+ i64 n = strlen(home_dir) + 1;
char *z = malloc( n );
if( z ) memcpy(z, home_dir, n);
home_dir = z;
@@ -23069,7 +25852,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){
# endif
#endif
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
# define main fiddle_main
#endif
@@ -23080,10 +25863,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
char **argv;
#endif
#ifdef SQLITE_DEBUG
- sqlite3_uint64 mem_main_enter = sqlite3_memory_used();
+ sqlite3_int64 mem_main_enter = sqlite3_memory_used();
#endif
char *zErrMsg = 0;
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
# define data shellState
#else
ShellState data;
@@ -23103,9 +25886,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
setBinaryMode(stdin, 0);
setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
stdin_is_interactive = 0;
stdout_is_console = 1;
+ data.wasm.zDefaultDbName = "/fiddle.sqlite3";
#else
stdin_is_interactive = isatty(0);
stdout_is_console = isatty(1);
@@ -23133,7 +25917,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
#endif
#if USE_SYSTEM_SQLITE+0!=1
- if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
+ if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
sqlite3_sourceid(), SQLITE_SOURCE_ID);
exit(1);
@@ -23155,9 +25939,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
argv = argvToFree + argc;
for(i=0; i70000 ) sz = 70000;
@@ -23252,7 +26036,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
sqlite3_config(SQLITE_CONFIG_PAGECACHE,
(n>0 && sz>0) ? malloc(n*sz) : 0, sz, n);
data.shellFlgs |= SHFLG_Pagecache;
- }else if( strcmp(z,"-lookaside")==0 ){
+ }else if( cli_strcmp(z,"-lookaside")==0 ){
int n, sz;
sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
if( sz<0 ) sz = 0;
@@ -23260,7 +26044,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
if( n<0 ) n = 0;
sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n);
if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside;
- }else if( strcmp(z,"-threadsafe")==0 ){
+ }else if( cli_strcmp(z,"-threadsafe")==0 ){
int n;
n = (int)integerValue(cmdline_option_value(argc,argv,++i));
switch( n ){
@@ -23269,7 +26053,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
default: sqlite3_config(SQLITE_CONFIG_SERIALIZED); break;
}
#ifdef SQLITE_ENABLE_VFSTRACE
- }else if( strcmp(z,"-vfstrace")==0 ){
+ }else if( cli_strcmp(z,"-vfstrace")==0 ){
extern int vfstrace_register(
const char *zTraceName,
const char *zOldVfsName,
@@ -23280,50 +26064,50 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1);
#endif
#ifdef SQLITE_ENABLE_MULTIPLEX
- }else if( strcmp(z,"-multiplex")==0 ){
+ }else if( cli_strcmp(z,"-multiplex")==0 ){
extern int sqlite3_multiple_initialize(const char*,int);
sqlite3_multiplex_initialize(0, 1);
#endif
- }else if( strcmp(z,"-mmap")==0 ){
+ }else if( cli_strcmp(z,"-mmap")==0 ){
sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz);
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
- }else if( strcmp(z,"-sorterref")==0 ){
+ }else if( cli_strcmp(z,"-sorterref")==0 ){
sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz);
#endif
- }else if( strcmp(z,"-vfs")==0 ){
+ }else if( cli_strcmp(z,"-vfs")==0 ){
zVfs = cmdline_option_value(argc, argv, ++i);
#ifdef SQLITE_HAVE_ZLIB
- }else if( strcmp(z,"-zip")==0 ){
+ }else if( cli_strcmp(z,"-zip")==0 ){
data.openMode = SHELL_OPEN_ZIPFILE;
#endif
- }else if( strcmp(z,"-append")==0 ){
+ }else if( cli_strcmp(z,"-append")==0 ){
data.openMode = SHELL_OPEN_APPENDVFS;
#ifndef SQLITE_OMIT_DESERIALIZE
- }else if( strcmp(z,"-deserialize")==0 ){
+ }else if( cli_strcmp(z,"-deserialize")==0 ){
data.openMode = SHELL_OPEN_DESERIALIZE;
- }else if( strcmp(z,"-maxsize")==0 && i+10 ){
utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands"
" with \"%s\"\n", z);
@@ -23551,7 +26335,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
readStdin = 0;
break;
#endif
- }else if( strcmp(z,"-safe")==0 ){
+ }else if( cli_strcmp(z,"-safe")==0 ){
data.bSafeMode = data.bSafeModePersist = 1;
}else{
utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
@@ -23634,7 +26418,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
rc = process_input(&data);
}
}
-#ifndef SQLITE_SHELL_WASM_MODE
+#ifndef SQLITE_SHELL_FIDDLE
/* In WASM mode we have to leave the db state in place so that
** client code can "push" SQL into it after this call returns. */
free(azCmd);
@@ -23669,38 +26453,38 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
(unsigned int)(sqlite3_memory_used()-mem_main_enter));
}
#endif
-#endif /* !SQLITE_SHELL_WASM_MODE */
+#endif /* !SQLITE_SHELL_FIDDLE */
return rc;
}
-#ifdef SQLITE_SHELL_WASM_MODE
+#ifdef SQLITE_SHELL_FIDDLE
/* Only for emcc experimentation purposes. */
int fiddle_experiment(int a,int b){
- return a + b;
+ return a + b;
}
-/* Only for emcc experimentation purposes.
-
- Define this function in JS using:
-
- emcc ... --js-library somefile.js
-
- containing:
-
-mergeInto(LibraryManager.library, {
- my_foo: function(){
- console.debug("my_foo()",arguments);
- }
-});
+/*
+** Returns a pointer to the current DB handle.
*/
-/*extern void my_foo(sqlite3 *);*/
-/* Only for emcc experimentation purposes. */
-sqlite3 * fiddle_the_db(){
- printf("fiddle_the_db(%p)\n", (const void*)globalDb);
- /*my_foo(globalDb);*/
- return globalDb;
+sqlite3 * fiddle_db_handle(){
+ return globalDb;
}
+
+/*
+** Returns a pointer to the given DB name's VFS. If zDbName is 0 then
+** "main" is assumed. Returns 0 if no db with the given name is
+** open.
+*/
+sqlite3_vfs * fiddle_db_vfs(const char *zDbName){
+ sqlite3_vfs * pVfs = 0;
+ if(globalDb){
+ sqlite3_file_control(globalDb, zDbName ? zDbName : "main",
+ SQLITE_FCNTL_VFS_POINTER, &pVfs);
+ }
+ return pVfs;
+}
+
/* Only for emcc experimentation purposes. */
sqlite3 * fiddle_db_arg(sqlite3 *arg){
printf("fiddle_db_arg(%p)\n", (const void*)arg);
@@ -23714,7 +26498,7 @@ sqlite3 * fiddle_db_arg(sqlite3 *arg){
** portable enough to make real use of.
*/
void fiddle_interrupt(void){
- if(globalDb) sqlite3_interrupt(globalDb);
+ if( globalDb ) sqlite3_interrupt(globalDb);
}
/*
@@ -23728,71 +26512,72 @@ const char * fiddle_db_filename(const char * zDbName){
}
/*
-** Closes, unlinks, and reopens the db using its current filename (or
-** the default if the db is currently closed). It is assumed, for
-** purposes of the fiddle build, that the file is in a transient
-** virtual filesystem within the browser.
+** Completely wipes out the contents of the currently-opened database
+** but leaves its storage intact for reuse.
*/
void fiddle_reset_db(void){
- char *zFilename = 0;
- if(0==globalDb){
- shellState.pAuxDb->zDbFilename = "/fiddle.sqlite3";
- }else{
- zFilename =
- sqlite3_mprintf("%s", sqlite3_db_filename(globalDb, "main"));
- shell_check_oom(zFilename);
- close_db(globalDb);
- shellDeleteFile(zFilename);
- shellState.db = 0;
- shellState.pAuxDb->zDbFilename = zFilename;
+ if( globalDb ){
+ int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
+ if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0);
+ sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0);
}
- open_db(&shellState, 0);
- sqlite3_free(zFilename);
}
/*
-** Trivial exportable function for emscripten. Needs to be exported using:
-**
-** emcc ..flags... -sEXPORTED_FUNCTIONS=_fiddle_exec -sEXPORTED_RUNTIME_METHODS=ccall,cwrap
-**
-** (Note the underscore before the function name.) It processes zSql
-** as if it were input to the sqlite3 shell and redirects all output
-** to the wasm binding.
+** Uses the current database's VFS xRead to stream the db file's
+** contents out to the given callback. The callback gets a single
+** chunk of size n (its 2nd argument) on each call and must return 0
+** on success, non-0 on error. This function returns 0 on success,
+** SQLITE_NOTFOUND if no db is open, or propagates any other non-0
+** code from the callback. Note that this is not thread-friendly: it
+** expects that it will be the only thread reading the db file and
+** takes no measures to ensure that is the case.
+*/
+int fiddle_export_db( int (*xCallback)(unsigned const char *zOut, int n) ){
+ sqlite3_int64 nSize = 0;
+ sqlite3_int64 nPos = 0;
+ sqlite3_file * pFile = 0;
+ unsigned char buf[1024 * 8];
+ int nBuf = (int)sizeof(buf);
+ int rc = shellState.db
+ ? sqlite3_file_control(shellState.db, "main",
+ SQLITE_FCNTL_FILE_POINTER, &pFile)
+ : SQLITE_NOTFOUND;
+ if( rc ) return rc;
+ rc = pFile->pMethods->xFileSize(pFile, &nSize);
+ if( rc ) return rc;
+ if(nSize % nBuf){
+ /* DB size is not an even multiple of the buffer size. Reduce
+ ** buffer size so that we do not unduly inflate the db size when
+ ** exporting. */
+ if(0 == nSize % 4096) nBuf = 4096;
+ else if(0 == nSize % 2048) nBuf = 2048;
+ else if(0 == nSize % 1024) nBuf = 1024;
+ else nBuf = 512;
+ }
+ for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos);
+ if(SQLITE_IOERR_SHORT_READ == rc){
+ rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/;
+ }
+ if( 0==rc ) rc = xCallback(buf, nBuf);
+ }
+ return rc;
+}
+
+/*
+** Trivial exportable function for emscripten. It processes zSql as if
+** it were input to the sqlite3 shell and redirects all output to the
+** wasm binding. fiddle_main() must have been called before this
+** is called, or results are undefined.
*/
void fiddle_exec(const char * zSql){
- static int once = 0;
- int rc = 0;
- if(!once){
- /* Simulate an argv array for main() */
- static char * argv[] = {"fiddle",
- "-bail",
- "-safe"};
- rc = fiddle_main((int)(sizeof(argv)/sizeof(argv[0])), argv);
- once = rc ? -1 : 1;
- memset(&shellState.wasm, 0, sizeof(shellState.wasm));
- printf(
- "SQLite version %s %.19s\n" /*extra-version-info*/,
- sqlite3_libversion(), sqlite3_sourceid()
- );
- puts("WASM shell");
- puts("Enter \".help\" for usage hints.");
- if(once>0){
- fiddle_reset_db();
- }
- if(shellState.db){
- printf("Connected to %s.\n", fiddle_db_filename(NULL));
- }else{
- fprintf(stderr,"ERROR initializing db!\n");
- return;
- }
- }
- if(once<0){
- puts("DB init failed. Not executing SQL.");
- }else if(zSql && *zSql){
+ if(zSql && *zSql){
+ if('.'==*zSql) puts(zSql);
shellState.wasm.zInput = zSql;
shellState.wasm.zPos = zSql;
process_input(&shellState);
- memset(&shellState.wasm, 0, sizeof(shellState.wasm));
+ shellState.wasm.zInput = shellState.wasm.zPos = 0;
}
}
-#endif /* SQLITE_SHELL_WASM_MODE */
+#endif /* SQLITE_SHELL_FIDDLE */
diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c
index 6a7b52d9..55f88efb 100644
--- a/src/database/sqlite3.c
+++ b/src/database/sqlite3.c
@@ -1,6 +1,6 @@
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
-** version 3.39.0. By combining all the individual C code files into this
+** version 3.40.0. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a single translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
@@ -452,9 +452,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.39.0"
-#define SQLITE_VERSION_NUMBER 3039000
-#define SQLITE_SOURCE_ID "2022-06-25 14:57:57 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918"
+#define SQLITE_VERSION "3.40.0"
+#define SQLITE_VERSION_NUMBER 3040000
+#define SQLITE_SOURCE_ID "2022-11-16 12:10:08 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -976,13 +976,17 @@ SQLITE_API int sqlite3_exec(
**
** SQLite uses one of these integer values as the second
** argument to calls it makes to the xLock() and xUnlock() methods
-** of an [sqlite3_io_methods] object.
+** of an [sqlite3_io_methods] object. These values are ordered from
+** lest restrictive to most restrictive.
+**
+** The argument to xLock() is always SHARED or higher. The argument to
+** xUnlock is either SHARED or NONE.
*/
-#define SQLITE_LOCK_NONE 0
-#define SQLITE_LOCK_SHARED 1
-#define SQLITE_LOCK_RESERVED 2
-#define SQLITE_LOCK_PENDING 3
-#define SQLITE_LOCK_EXCLUSIVE 4
+#define SQLITE_LOCK_NONE 0 /* xUnlock() only */
+#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */
+#define SQLITE_LOCK_RESERVED 2 /* xLock() only */
+#define SQLITE_LOCK_PENDING 3 /* xLock() only */
+#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */
/*
** CAPI3REF: Synchronization Type Flags
@@ -1060,7 +1064,14 @@ struct sqlite3_file {
**
[SQLITE_LOCK_PENDING], or
**
[SQLITE_LOCK_EXCLUSIVE].
**
-** xLock() increases the lock. xUnlock() decreases the lock.
+** xLock() upgrades the database file lock. In other words, xLock() moves the
+** database file lock in the direction NONE toward EXCLUSIVE. The argument to
+** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never
+** SQLITE_LOCK_NONE. If the database file lock is already at or above the
+** requested lock, then the call to xLock() is a no-op.
+** xUnlock() downgrades the database file lock to either SHARED or NONE.
+* If the lock is already at or below the requested lock state, then the call
+** to xUnlock() is a no-op.
** The xCheckReservedLock() method checks whether any database connection,
** either in this process or in some other process, is holding a RESERVED,
** PENDING, or EXCLUSIVE lock on the file. It returns true
@@ -1165,9 +1176,8 @@ struct sqlite3_io_methods {
** opcode causes the xFileControl method to write the current state of
** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED],
** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE])
-** into an integer that the pArg argument points to. This capability
-** is used during testing and is only available when the SQLITE_TEST
-** compile-time option is used.
+** into an integer that the pArg argument points to.
+** This capability is only available if SQLite is compiled with [SQLITE_DEBUG].
**
**
[[SQLITE_FCNTL_SIZE_HINT]]
** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS
@@ -1559,6 +1569,26 @@ typedef struct sqlite3_mutex sqlite3_mutex;
*/
typedef struct sqlite3_api_routines sqlite3_api_routines;
+/*
+** CAPI3REF: File Name
+**
+** Type [sqlite3_filename] is used by SQLite to pass filenames to the
+** xOpen method of a [VFS]. It may be cast to (const char*) and treated
+** as a normal, nul-terminated, UTF-8 buffer containing the filename, but
+** may also be passed to special APIs such as:
+**
+**
+**
sqlite3_filename_database()
+**
sqlite3_filename_journal()
+**
sqlite3_filename_wal()
+**
sqlite3_uri_parameter()
+**
sqlite3_uri_boolean()
+**
sqlite3_uri_int64()
+**
sqlite3_uri_key()
+**
+*/
+typedef const char *sqlite3_filename;
+
/*
** CAPI3REF: OS Interface Object
**
@@ -1737,7 +1767,7 @@ struct sqlite3_vfs {
sqlite3_vfs *pNext; /* Next registered VFS */
const char *zName; /* Name of this virtual file system */
void *pAppData; /* Pointer to application-specific data */
- int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*,
+ int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*,
int flags, int *pOutFlags);
int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir);
int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut);
@@ -2615,6 +2645,7 @@ struct sqlite3_mem_methods {
**
The database is opened [shared cache] enabled, overriding
** the default shared cache setting provided by
** [sqlite3_enable_shared_cache()].)^
+** The [use of shared cache mode is discouraged] and hence shared cache
+** capabilities may be omitted from many builds of SQLite. In such cases,
+** this option is a no-op.
**
** ^(
[SQLITE_OPEN_PRIVATECACHE]
**
The database is opened [shared cache] disabled, overriding
@@ -3745,7 +3779,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
** to return an extended result code.
**
** [[OPEN_NOFOLLOW]] ^(
[SQLITE_OPEN_NOFOLLOW]
-**
The database filename is not allowed to be a symbolic link
+**
The database filename is not allowed to contain a symbolic link
** )^
**
** If the 3rd parameter to sqlite3_open_v2() is not one of the
@@ -4004,10 +4038,10 @@ SQLITE_API int sqlite3_open_v2(
**
** See the [URI filename] documentation for additional information.
*/
-SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam);
-SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault);
-SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64);
-SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N);
+SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam);
+SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault);
+SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64);
+SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N);
/*
** CAPI3REF: Translate filenames
@@ -4036,9 +4070,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N);
** return value from [sqlite3_db_filename()], then the result is
** undefined and is likely a memory access violation.
*/
-SQLITE_API const char *sqlite3_filename_database(const char*);
-SQLITE_API const char *sqlite3_filename_journal(const char*);
-SQLITE_API const char *sqlite3_filename_wal(const char*);
+SQLITE_API const char *sqlite3_filename_database(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename);
/*
** CAPI3REF: Database File Corresponding To A Journal
@@ -4104,14 +4138,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*);
** then the corresponding [sqlite3_module.xClose() method should also be
** invoked prior to calling sqlite3_free_filename(Y).
*/
-SQLITE_API char *sqlite3_create_filename(
+SQLITE_API sqlite3_filename sqlite3_create_filename(
const char *zDatabase,
const char *zJournal,
const char *zWal,
int nParam,
const char **azParam
);
-SQLITE_API void sqlite3_free_filename(char*);
+SQLITE_API void sqlite3_free_filename(sqlite3_filename);
/*
** CAPI3REF: Error Codes And Messages
@@ -5814,6 +5848,16 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6
** then the conversion is performed. Otherwise no conversion occurs.
** The [SQLITE_INTEGER | datatype] after conversion is returned.)^
**
+** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8],
+** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding
+** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X)
+** returns something other than SQLITE_TEXT, then the return value from
+** sqlite3_value_encoding(X) is meaningless. ^Calls to
+** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X),
+** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or
+** sqlite3_value_bytes16(X) might change the encoding of the value X and
+** thus change the return from subsequent calls to sqlite3_value_encoding(X).
+**
** ^Within the [xUpdate] method of a [virtual table], the
** sqlite3_value_nochange(X) interface returns true if and only if
** the column corresponding to X is unchanged by the UPDATE operation
@@ -5878,6 +5922,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value*);
SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*);
SQLITE_API int sqlite3_value_nochange(sqlite3_value*);
SQLITE_API int sqlite3_value_frombind(sqlite3_value*);
+SQLITE_API int sqlite3_value_encoding(sqlite3_value*);
/*
** CAPI3REF: Finding The Subtype Of SQL Values
@@ -5931,7 +5976,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*);
**
** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer
** when first called if N is less than or equal to zero or if a memory
-** allocate error occurs.
+** allocation error occurs.
**
** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is
** determined by the N parameter on first successful call. Changing the
@@ -6136,9 +6181,10 @@ typedef void (*sqlite3_destructor_type)(void*);
** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE].
** ^SQLite takes the text result from the application from
** the 2nd parameter of the sqlite3_result_text* interfaces.
-** ^If the 3rd parameter to the sqlite3_result_text* interfaces
-** is negative, then SQLite takes result text from the 2nd parameter
-** through the first zero character.
+** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces
+** other than sqlite3_result_text64() is negative, then SQLite computes
+** the string length itself by searching the 2nd parameter for the first
+** zero character.
** ^If the 3rd parameter to the sqlite3_result_text* interfaces
** is non-negative, then as many bytes (not characters) of the text
** pointed to by the 2nd parameter are taken as the application-defined
@@ -6588,7 +6634,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*);
**
** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name
** for the N-th database on database connection D, or a NULL pointer of N is
-** out of range. An N alue of 0 means the main database file. An N of 1 is
+** out of range. An N value of 0 means the main database file. An N of 1 is
** the "temp" schema. Larger values of N correspond to various ATTACH-ed
** databases.
**
@@ -6634,7 +6680,7 @@ SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N);
**
[sqlite3_filename_wal()]
**
*/
-SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName);
+SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName);
/*
** CAPI3REF: Determine if a database is read-only
@@ -6771,7 +6817,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
** function C that is invoked prior to each autovacuum of the database
** file. ^The callback is passed a copy of the generic data pointer (P),
** the schema-name of the attached database that is being autovacuumed,
-** the the size of the database file in pages, the number of free pages,
+** the size of the database file in pages, the number of free pages,
** and the number of bytes per page, respectively. The callback should
** return the number of free pages that should be removed by the
** autovacuum. ^If the callback returns zero, then no autovacuum happens.
@@ -6892,6 +6938,11 @@ SQLITE_API void *sqlite3_update_hook(
** to the same database. Sharing is enabled if the argument is true
** and disabled if the argument is false.)^
**
+** This interface is omitted if SQLite is compiled with
+** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE]
+** compile-time option is recommended because the
+** [use of shared cache mode is discouraged].
+**
** ^Cache sharing is enabled and disabled for an entire process.
** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]).
** In prior versions of SQLite,
@@ -6990,7 +7041,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*);
** ^The soft heap limit may not be greater than the hard heap limit.
** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N)
** is invoked with a value of N that is greater than the hard heap limit,
-** the the soft heap limit is set to the value of the hard heap limit.
+** the soft heap limit is set to the value of the hard heap limit.
** ^The soft heap limit is automatically enabled whenever the hard heap
** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and
** the soft heap limit is outside the range of 1..N, then the soft heap
@@ -9285,7 +9336,7 @@ typedef struct sqlite3_backup sqlite3_backup;
** if the application incorrectly accesses the destination [database connection]
** and so no error code is reported, but the operations may malfunction
** nevertheless. Use of the destination database connection while a
-** backup is in progress might also also cause a mutex deadlock.
+** backup is in progress might also cause a mutex deadlock.
**
** If running in [shared cache mode], the application must
** guarantee that the shared cache used by the destination database
@@ -9713,7 +9764,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2(
*/
#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */
#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */
-#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */
+#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */
#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */
/*
@@ -13144,12 +13195,17 @@ struct fts5_api {
/************** End of sqlite3.h *********************************************/
/************** Continuing where we left off in sqliteInt.h ******************/
+/*
+** Reuse the STATIC_LRU for mutex access to sqlite3_temp_directory.
+*/
+#define SQLITE_MUTEX_STATIC_TEMPDIR SQLITE_MUTEX_STATIC_VFS1
+
/*
** Include the configuration header output by 'configure' if we're using the
** autoconf-based build
*/
#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H)
-#include "config.h"
+#include "sqlite_cfg.h"
#define SQLITECONFIG_H 1
#endif
@@ -14632,6 +14688,7 @@ typedef struct FuncDef FuncDef;
typedef struct FuncDefHash FuncDefHash;
typedef struct IdList IdList;
typedef struct Index Index;
+typedef struct IndexedExpr IndexedExpr;
typedef struct IndexSample IndexSample;
typedef struct KeyClass KeyClass;
typedef struct KeyInfo KeyInfo;
@@ -14697,6 +14754,7 @@ typedef struct With With;
#define MASKBIT32(n) (((unsigned int)1)<<(n))
#define SMASKBIT32(n) ((n)<=31?((unsigned int)1)<<(n):0)
#define ALLBITS ((Bitmask)-1)
+#define TOPBIT (((Bitmask)1)<<(BMS-1))
/* A VList object records a mapping between parameters/variables/wildcards
** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer
@@ -14711,6 +14769,331 @@ typedef int VList;
** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
** pointer types (i.e. FuncDef) defined above.
*/
+/************** Include os.h in the middle of sqliteInt.h ********************/
+/************** Begin file os.h **********************************************/
+/*
+** 2001 September 16
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file (together with is companion C source-code file
+** "os.c") attempt to abstract the underlying operating system so that
+** the SQLite library will work on both POSIX and windows systems.
+**
+** This header file is #include-ed by sqliteInt.h and thus ends up
+** being included by every source file.
+*/
+#ifndef _SQLITE_OS_H_
+#define _SQLITE_OS_H_
+
+/*
+** Attempt to automatically detect the operating system and setup the
+** necessary pre-processor macros for it.
+*/
+/************** Include os_setup.h in the middle of os.h *********************/
+/************** Begin file os_setup.h ****************************************/
+/*
+** 2013 November 25
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains pre-processor directives related to operating system
+** detection and/or setup.
+*/
+#ifndef SQLITE_OS_SETUP_H
+#define SQLITE_OS_SETUP_H
+
+/*
+** Figure out if we are dealing with Unix, Windows, or some other operating
+** system.
+**
+** After the following block of preprocess macros, all of
+**
+** SQLITE_OS_KV
+** SQLITE_OS_OTHER
+** SQLITE_OS_UNIX
+** SQLITE_OS_WIN
+**
+** will defined to either 1 or 0. One of them will be 1. The others will be 0.
+** If none of the macros are initially defined, then select either
+** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform.
+**
+** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application
+** must provide its own VFS implementation together with sqlite3_os_init()
+** and sqlite3_os_end() routines.
+*/
+#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \
+ !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN)
+# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \
+ defined(__MINGW32__) || defined(__BORLANDC__)
+# define SQLITE_OS_WIN 1
+# define SQLITE_OS_UNIX 0
+# else
+# define SQLITE_OS_WIN 0
+# define SQLITE_OS_UNIX 1
+# endif
+#endif
+#if SQLITE_OS_OTHER+1>1
+# undef SQLITE_OS_KV
+# define SQLITE_OS_KV 0
+# undef SQLITE_OS_UNIX
+# define SQLITE_OS_UNIX 0
+# undef SQLITE_OS_WIN
+# define SQLITE_OS_WIN 0
+#endif
+#if SQLITE_OS_KV+1>1
+# undef SQLITE_OS_OTHER
+# define SQLITE_OS_OTHER 0
+# undef SQLITE_OS_UNIX
+# define SQLITE_OS_UNIX 0
+# undef SQLITE_OS_WIN
+# define SQLITE_OS_WIN 0
+# define SQLITE_OMIT_LOAD_EXTENSION 1
+# define SQLITE_OMIT_WAL 1
+# define SQLITE_OMIT_DEPRECATED 1
+# undef SQLITE_TEMP_STORE
+# define SQLITE_TEMP_STORE 3 /* Always use memory for temporary storage */
+# define SQLITE_DQS 0
+# define SQLITE_OMIT_SHARED_CACHE 1
+# define SQLITE_OMIT_AUTOINIT 1
+#endif
+#if SQLITE_OS_UNIX+1>1
+# undef SQLITE_OS_KV
+# define SQLITE_OS_KV 0
+# undef SQLITE_OS_OTHER
+# define SQLITE_OS_OTHER 0
+# undef SQLITE_OS_WIN
+# define SQLITE_OS_WIN 0
+#endif
+#if SQLITE_OS_WIN+1>1
+# undef SQLITE_OS_KV
+# define SQLITE_OS_KV 0
+# undef SQLITE_OS_OTHER
+# define SQLITE_OS_OTHER 0
+# undef SQLITE_OS_UNIX
+# define SQLITE_OS_UNIX 0
+#endif
+
+
+#endif /* SQLITE_OS_SETUP_H */
+
+/************** End of os_setup.h ********************************************/
+/************** Continuing where we left off in os.h *************************/
+
+/* If the SET_FULLSYNC macro is not defined above, then make it
+** a no-op
+*/
+#ifndef SET_FULLSYNC
+# define SET_FULLSYNC(x,y)
+#endif
+
+/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h
+*/
+#ifndef SQLITE_MAX_PATHLEN
+# define SQLITE_MAX_PATHLEN FILENAME_MAX
+#endif
+
+/* Maximum number of symlinks that will be resolved while trying to
+** expand a filename in xFullPathname() in the VFS.
+*/
+#ifndef SQLITE_MAX_SYMLINK
+# define SQLITE_MAX_SYMLINK 200
+#endif
+
+/*
+** The default size of a disk sector
+*/
+#ifndef SQLITE_DEFAULT_SECTOR_SIZE
+# define SQLITE_DEFAULT_SECTOR_SIZE 4096
+#endif
+
+/*
+** Temporary files are named starting with this prefix followed by 16 random
+** alphanumeric characters, and no file extension. They are stored in the
+** OS's standard temporary file directory, and are deleted prior to exit.
+** If sqlite is being embedded in another program, you may wish to change the
+** prefix to reflect your program's name, so that if your program exits
+** prematurely, old temporary files can be easily identified. This can be done
+** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line.
+**
+** 2006-10-31: The default prefix used to be "sqlite_". But then
+** Mcafee started using SQLite in their anti-virus product and it
+** started putting files with the "sqlite" name in the c:/temp folder.
+** This annoyed many windows users. Those users would then do a
+** Google search for "sqlite", find the telephone numbers of the
+** developers and call to wake them up at night and complain.
+** For this reason, the default name prefix is changed to be "sqlite"
+** spelled backwards. So the temp files are still identified, but
+** anybody smart enough to figure out the code is also likely smart
+** enough to know that calling the developer will not help get rid
+** of the file.
+*/
+#ifndef SQLITE_TEMP_FILE_PREFIX
+# define SQLITE_TEMP_FILE_PREFIX "etilqs_"
+#endif
+
+/*
+** The following values may be passed as the second argument to
+** sqlite3OsLock(). The various locks exhibit the following semantics:
+**
+** SHARED: Any number of processes may hold a SHARED lock simultaneously.
+** RESERVED: A single process may hold a RESERVED lock on a file at
+** any time. Other processes may hold and obtain new SHARED locks.
+** PENDING: A single process may hold a PENDING lock on a file at
+** any one time. Existing SHARED locks may persist, but no new
+** SHARED locks may be obtained by other processes.
+** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks.
+**
+** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a
+** process that requests an EXCLUSIVE lock may actually obtain a PENDING
+** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to
+** sqlite3OsLock().
+*/
+#define NO_LOCK 0
+#define SHARED_LOCK 1
+#define RESERVED_LOCK 2
+#define PENDING_LOCK 3
+#define EXCLUSIVE_LOCK 4
+
+/*
+** File Locking Notes: (Mostly about windows but also some info for Unix)
+**
+** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because
+** those functions are not available. So we use only LockFile() and
+** UnlockFile().
+**
+** LockFile() prevents not just writing but also reading by other processes.
+** A SHARED_LOCK is obtained by locking a single randomly-chosen
+** byte out of a specific range of bytes. The lock byte is obtained at
+** random so two separate readers can probably access the file at the
+** same time, unless they are unlucky and choose the same lock byte.
+** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range.
+** There can only be one writer. A RESERVED_LOCK is obtained by locking
+** a single byte of the file that is designated as the reserved lock byte.
+** A PENDING_LOCK is obtained by locking a designated byte different from
+** the RESERVED_LOCK byte.
+**
+** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available,
+** which means we can use reader/writer locks. When reader/writer locks
+** are used, the lock is placed on the same range of bytes that is used
+** for probabilistic locking in Win95/98/ME. Hence, the locking scheme
+** will support two or more Win95 readers or two or more WinNT readers.
+** But a single Win95 reader will lock out all WinNT readers and a single
+** WinNT reader will lock out all other Win95 readers.
+**
+** The following #defines specify the range of bytes used for locking.
+** SHARED_SIZE is the number of bytes available in the pool from which
+** a random byte is selected for a shared lock. The pool of bytes for
+** shared locks begins at SHARED_FIRST.
+**
+** The same locking strategy and
+** byte ranges are used for Unix. This leaves open the possibility of having
+** clients on win95, winNT, and unix all talking to the same shared file
+** and all locking correctly. To do so would require that samba (or whatever
+** tool is being used for file sharing) implements locks correctly between
+** windows and unix. I'm guessing that isn't likely to happen, but by
+** using the same locking range we are at least open to the possibility.
+**
+** Locking in windows is manditory. For this reason, we cannot store
+** actual data in the bytes used for locking. The pager never allocates
+** the pages involved in locking therefore. SHARED_SIZE is selected so
+** that all locks will fit on a single page even at the minimum page size.
+** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE
+** is set high so that we don't have to allocate an unused page except
+** for very large databases. But one should test the page skipping logic
+** by setting PENDING_BYTE low and running the entire regression suite.
+**
+** Changing the value of PENDING_BYTE results in a subtly incompatible
+** file format. Depending on how it is changed, you might not notice
+** the incompatibility right away, even running a full regression test.
+** The default location of PENDING_BYTE is the first byte past the
+** 1GB boundary.
+**
+*/
+#ifdef SQLITE_OMIT_WSD
+# define PENDING_BYTE (0x40000000)
+#else
+# define PENDING_BYTE sqlite3PendingByte
+#endif
+#define RESERVED_BYTE (PENDING_BYTE+1)
+#define SHARED_FIRST (PENDING_BYTE+2)
+#define SHARED_SIZE 510
+
+/*
+** Wrapper around OS specific sqlite3_os_init() function.
+*/
+SQLITE_PRIVATE int sqlite3OsInit(void);
+
+/*
+** Functions for accessing sqlite3_file methods
+*/
+SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*);
+SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset);
+SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset);
+SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size);
+SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int);
+SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize);
+SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int);
+SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int);
+SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut);
+SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*);
+SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*);
+#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0
+SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id);
+SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
+#ifndef SQLITE_OMIT_WAL
+SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **);
+SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int);
+SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id);
+SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int);
+#endif /* SQLITE_OMIT_WAL */
+SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **);
+SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *);
+
+
+/*
+** Functions for accessing sqlite3_vfs methods
+*/
+SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *);
+SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int);
+SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut);
+SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *);
+#ifndef SQLITE_OMIT_LOAD_EXTENSION
+SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *);
+SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *);
+SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void);
+SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *);
+#endif /* SQLITE_OMIT_LOAD_EXTENSION */
+SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *);
+SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int);
+SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*);
+SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*);
+
+/*
+** Convenience functions for opening and closing files using
+** sqlite3_malloc() to obtain space for the file-handle structure.
+*/
+SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*);
+SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *);
+
+#endif /* _SQLITE_OS_H_ */
+
+/************** End of os.h **************************************************/
+/************** Continuing where we left off in sqliteInt.h ******************/
/************** Include pager.h in the middle of sqliteInt.h *****************/
/************** Begin file pager.h *******************************************/
/*
@@ -15555,48 +15938,48 @@ typedef struct VdbeOpList VdbeOpList;
#define OP_Vacuum 5
#define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */
#define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */
-#define OP_Goto 8 /* jump */
-#define OP_Gosub 9 /* jump */
-#define OP_InitCoroutine 10 /* jump */
-#define OP_Yield 11 /* jump */
-#define OP_MustBeInt 12 /* jump */
-#define OP_Jump 13 /* jump */
-#define OP_Once 14 /* jump */
-#define OP_If 15 /* jump */
-#define OP_IfNot 16 /* jump */
-#define OP_IsNullOrType 17 /* jump, synopsis: if typeof(r[P1]) IN (P3,5) goto P2 */
-#define OP_IfNullRow 18 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */
+#define OP_Init 8 /* jump, synopsis: Start at P2 */
+#define OP_Goto 9 /* jump */
+#define OP_Gosub 10 /* jump */
+#define OP_InitCoroutine 11 /* jump */
+#define OP_Yield 12 /* jump */
+#define OP_MustBeInt 13 /* jump */
+#define OP_Jump 14 /* jump */
+#define OP_Once 15 /* jump */
+#define OP_If 16 /* jump */
+#define OP_IfNot 17 /* jump */
+#define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */
#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */
-#define OP_SeekLT 20 /* jump, synopsis: key=r[P3@P4] */
-#define OP_SeekLE 21 /* jump, synopsis: key=r[P3@P4] */
-#define OP_SeekGE 22 /* jump, synopsis: key=r[P3@P4] */
-#define OP_SeekGT 23 /* jump, synopsis: key=r[P3@P4] */
-#define OP_IfNotOpen 24 /* jump, synopsis: if( !csr[P1] ) goto P2 */
-#define OP_IfNoHope 25 /* jump, synopsis: key=r[P3@P4] */
-#define OP_NoConflict 26 /* jump, synopsis: key=r[P3@P4] */
-#define OP_NotFound 27 /* jump, synopsis: key=r[P3@P4] */
-#define OP_Found 28 /* jump, synopsis: key=r[P3@P4] */
-#define OP_SeekRowid 29 /* jump, synopsis: intkey=r[P3] */
-#define OP_NotExists 30 /* jump, synopsis: intkey=r[P3] */
-#define OP_Last 31 /* jump */
-#define OP_IfSmaller 32 /* jump */
-#define OP_SorterSort 33 /* jump */
-#define OP_Sort 34 /* jump */
-#define OP_Rewind 35 /* jump */
-#define OP_SorterNext 36 /* jump */
-#define OP_Prev 37 /* jump */
-#define OP_Next 38 /* jump */
-#define OP_IdxLE 39 /* jump, synopsis: key=r[P3@P4] */
-#define OP_IdxGT 40 /* jump, synopsis: key=r[P3@P4] */
-#define OP_IdxLT 41 /* jump, synopsis: key=r[P3@P4] */
-#define OP_IdxGE 42 /* jump, synopsis: key=r[P3@P4] */
+#define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */
+#define OP_SeekLT 21 /* jump, synopsis: key=r[P3@P4] */
+#define OP_SeekLE 22 /* jump, synopsis: key=r[P3@P4] */
+#define OP_SeekGE 23 /* jump, synopsis: key=r[P3@P4] */
+#define OP_SeekGT 24 /* jump, synopsis: key=r[P3@P4] */
+#define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */
+#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */
+#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */
+#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */
+#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */
+#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */
+#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */
+#define OP_Last 32 /* jump */
+#define OP_IfSmaller 33 /* jump */
+#define OP_SorterSort 34 /* jump */
+#define OP_Sort 35 /* jump */
+#define OP_Rewind 36 /* jump */
+#define OP_SorterNext 37 /* jump */
+#define OP_Prev 38 /* jump */
+#define OP_Next 39 /* jump */
+#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */
+#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */
+#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */
#define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */
#define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */
-#define OP_RowSetRead 45 /* jump, synopsis: r[P3]=rowset(P1) */
-#define OP_RowSetTest 46 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */
-#define OP_Program 47 /* jump */
-#define OP_FkIfZero 48 /* jump, synopsis: if fkctr[P1]==0 goto P2 */
-#define OP_IfPos 49 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */
+#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */
+#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */
+#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */
+#define OP_Program 48 /* jump */
+#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */
#define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */
#define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */
#define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */
@@ -15606,12 +15989,12 @@ typedef struct VdbeOpList VdbeOpList;
#define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */
#define OP_ElseEq 58 /* jump, same as TK_ESCAPE */
-#define OP_IfNotZero 59 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */
-#define OP_DecrJumpZero 60 /* jump, synopsis: if (--r[P1])==0 goto P2 */
-#define OP_IncrVacuum 61 /* jump */
-#define OP_VNext 62 /* jump */
-#define OP_Filter 63 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */
-#define OP_Init 64 /* jump, synopsis: Start at P2 */
+#define OP_IfPos 59 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */
+#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */
+#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */
+#define OP_IncrVacuum 62 /* jump */
+#define OP_VNext 63 /* jump */
+#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */
#define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */
#define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */
#define OP_Return 67
@@ -15747,13 +16130,13 @@ typedef struct VdbeOpList VdbeOpList;
#define OPFLG_OUT3 0x20 /* out3: P3 is an output */
#define OPFLG_INITIALIZER {\
/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x01, 0x00,\
-/* 8 */ 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01, 0x03,\
-/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x09, 0x09, 0x09, 0x09,\
-/* 24 */ 0x01, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x01,\
+/* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\
+/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x09, 0x09, 0x09,\
+/* 24 */ 0x09, 0x01, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09,\
/* 32 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\
-/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x23, 0x0b, 0x01,\
-/* 48 */ 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\
-/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x01, 0x01,\
+/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x01, 0x23, 0x0b,\
+/* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\
+/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x01,\
/* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\
/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\
/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\
@@ -15845,6 +16228,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5);
+SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe*, int);
SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr);
SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr);
SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
@@ -15859,6 +16243,7 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type);
SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*);
SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int);
SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int);
+SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe*);
SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*);
SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*);
SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*);
@@ -16207,297 +16592,6 @@ SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache);
/************** End of pcache.h **********************************************/
/************** Continuing where we left off in sqliteInt.h ******************/
-/************** Include os.h in the middle of sqliteInt.h ********************/
-/************** Begin file os.h **********************************************/
-/*
-** 2001 September 16
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This header file (together with is companion C source-code file
-** "os.c") attempt to abstract the underlying operating system so that
-** the SQLite library will work on both POSIX and windows systems.
-**
-** This header file is #include-ed by sqliteInt.h and thus ends up
-** being included by every source file.
-*/
-#ifndef _SQLITE_OS_H_
-#define _SQLITE_OS_H_
-
-/*
-** Attempt to automatically detect the operating system and setup the
-** necessary pre-processor macros for it.
-*/
-/************** Include os_setup.h in the middle of os.h *********************/
-/************** Begin file os_setup.h ****************************************/
-/*
-** 2013 November 25
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This file contains pre-processor directives related to operating system
-** detection and/or setup.
-*/
-#ifndef SQLITE_OS_SETUP_H
-#define SQLITE_OS_SETUP_H
-
-/*
-** Figure out if we are dealing with Unix, Windows, or some other operating
-** system.
-**
-** After the following block of preprocess macros, all of SQLITE_OS_UNIX,
-** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of
-** the three will be 1. The other two will be 0.
-*/
-#if defined(SQLITE_OS_OTHER)
-# if SQLITE_OS_OTHER==1
-# undef SQLITE_OS_UNIX
-# define SQLITE_OS_UNIX 0
-# undef SQLITE_OS_WIN
-# define SQLITE_OS_WIN 0
-# else
-# undef SQLITE_OS_OTHER
-# endif
-#endif
-#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER)
-# define SQLITE_OS_OTHER 0
-# ifndef SQLITE_OS_WIN
-# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \
- defined(__MINGW32__) || defined(__BORLANDC__)
-# define SQLITE_OS_WIN 1
-# define SQLITE_OS_UNIX 0
-# else
-# define SQLITE_OS_WIN 0
-# define SQLITE_OS_UNIX 1
-# endif
-# else
-# define SQLITE_OS_UNIX 0
-# endif
-#else
-# ifndef SQLITE_OS_WIN
-# define SQLITE_OS_WIN 0
-# endif
-#endif
-
-#endif /* SQLITE_OS_SETUP_H */
-
-/************** End of os_setup.h ********************************************/
-/************** Continuing where we left off in os.h *************************/
-
-/* If the SET_FULLSYNC macro is not defined above, then make it
-** a no-op
-*/
-#ifndef SET_FULLSYNC
-# define SET_FULLSYNC(x,y)
-#endif
-
-/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h
-*/
-#ifndef SQLITE_MAX_PATHLEN
-# define SQLITE_MAX_PATHLEN FILENAME_MAX
-#endif
-
-/* Maximum number of symlinks that will be resolved while trying to
-** expand a filename in xFullPathname() in the VFS.
-*/
-#ifndef SQLITE_MAX_SYMLINK
-# define SQLITE_MAX_SYMLINK 200
-#endif
-
-/*
-** The default size of a disk sector
-*/
-#ifndef SQLITE_DEFAULT_SECTOR_SIZE
-# define SQLITE_DEFAULT_SECTOR_SIZE 4096
-#endif
-
-/*
-** Temporary files are named starting with this prefix followed by 16 random
-** alphanumeric characters, and no file extension. They are stored in the
-** OS's standard temporary file directory, and are deleted prior to exit.
-** If sqlite is being embedded in another program, you may wish to change the
-** prefix to reflect your program's name, so that if your program exits
-** prematurely, old temporary files can be easily identified. This can be done
-** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line.
-**
-** 2006-10-31: The default prefix used to be "sqlite_". But then
-** Mcafee started using SQLite in their anti-virus product and it
-** started putting files with the "sqlite" name in the c:/temp folder.
-** This annoyed many windows users. Those users would then do a
-** Google search for "sqlite", find the telephone numbers of the
-** developers and call to wake them up at night and complain.
-** For this reason, the default name prefix is changed to be "sqlite"
-** spelled backwards. So the temp files are still identified, but
-** anybody smart enough to figure out the code is also likely smart
-** enough to know that calling the developer will not help get rid
-** of the file.
-*/
-#ifndef SQLITE_TEMP_FILE_PREFIX
-# define SQLITE_TEMP_FILE_PREFIX "etilqs_"
-#endif
-
-/*
-** The following values may be passed as the second argument to
-** sqlite3OsLock(). The various locks exhibit the following semantics:
-**
-** SHARED: Any number of processes may hold a SHARED lock simultaneously.
-** RESERVED: A single process may hold a RESERVED lock on a file at
-** any time. Other processes may hold and obtain new SHARED locks.
-** PENDING: A single process may hold a PENDING lock on a file at
-** any one time. Existing SHARED locks may persist, but no new
-** SHARED locks may be obtained by other processes.
-** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks.
-**
-** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a
-** process that requests an EXCLUSIVE lock may actually obtain a PENDING
-** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to
-** sqlite3OsLock().
-*/
-#define NO_LOCK 0
-#define SHARED_LOCK 1
-#define RESERVED_LOCK 2
-#define PENDING_LOCK 3
-#define EXCLUSIVE_LOCK 4
-
-/*
-** File Locking Notes: (Mostly about windows but also some info for Unix)
-**
-** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because
-** those functions are not available. So we use only LockFile() and
-** UnlockFile().
-**
-** LockFile() prevents not just writing but also reading by other processes.
-** A SHARED_LOCK is obtained by locking a single randomly-chosen
-** byte out of a specific range of bytes. The lock byte is obtained at
-** random so two separate readers can probably access the file at the
-** same time, unless they are unlucky and choose the same lock byte.
-** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range.
-** There can only be one writer. A RESERVED_LOCK is obtained by locking
-** a single byte of the file that is designated as the reserved lock byte.
-** A PENDING_LOCK is obtained by locking a designated byte different from
-** the RESERVED_LOCK byte.
-**
-** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available,
-** which means we can use reader/writer locks. When reader/writer locks
-** are used, the lock is placed on the same range of bytes that is used
-** for probabilistic locking in Win95/98/ME. Hence, the locking scheme
-** will support two or more Win95 readers or two or more WinNT readers.
-** But a single Win95 reader will lock out all WinNT readers and a single
-** WinNT reader will lock out all other Win95 readers.
-**
-** The following #defines specify the range of bytes used for locking.
-** SHARED_SIZE is the number of bytes available in the pool from which
-** a random byte is selected for a shared lock. The pool of bytes for
-** shared locks begins at SHARED_FIRST.
-**
-** The same locking strategy and
-** byte ranges are used for Unix. This leaves open the possibility of having
-** clients on win95, winNT, and unix all talking to the same shared file
-** and all locking correctly. To do so would require that samba (or whatever
-** tool is being used for file sharing) implements locks correctly between
-** windows and unix. I'm guessing that isn't likely to happen, but by
-** using the same locking range we are at least open to the possibility.
-**
-** Locking in windows is manditory. For this reason, we cannot store
-** actual data in the bytes used for locking. The pager never allocates
-** the pages involved in locking therefore. SHARED_SIZE is selected so
-** that all locks will fit on a single page even at the minimum page size.
-** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE
-** is set high so that we don't have to allocate an unused page except
-** for very large databases. But one should test the page skipping logic
-** by setting PENDING_BYTE low and running the entire regression suite.
-**
-** Changing the value of PENDING_BYTE results in a subtly incompatible
-** file format. Depending on how it is changed, you might not notice
-** the incompatibility right away, even running a full regression test.
-** The default location of PENDING_BYTE is the first byte past the
-** 1GB boundary.
-**
-*/
-#ifdef SQLITE_OMIT_WSD
-# define PENDING_BYTE (0x40000000)
-#else
-# define PENDING_BYTE sqlite3PendingByte
-#endif
-#define RESERVED_BYTE (PENDING_BYTE+1)
-#define SHARED_FIRST (PENDING_BYTE+2)
-#define SHARED_SIZE 510
-
-/*
-** Wrapper around OS specific sqlite3_os_init() function.
-*/
-SQLITE_PRIVATE int sqlite3OsInit(void);
-
-/*
-** Functions for accessing sqlite3_file methods
-*/
-SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*);
-SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset);
-SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset);
-SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size);
-SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int);
-SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize);
-SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int);
-SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int);
-SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut);
-SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*);
-SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*);
-#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0
-SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id);
-SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
-#ifndef SQLITE_OMIT_WAL
-SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **);
-SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int);
-SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id);
-SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int);
-#endif /* SQLITE_OMIT_WAL */
-SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **);
-SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *);
-
-
-/*
-** Functions for accessing sqlite3_vfs methods
-*/
-SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *);
-SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int);
-SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut);
-SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *);
-#ifndef SQLITE_OMIT_LOAD_EXTENSION
-SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *);
-SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *);
-SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void);
-SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *);
-#endif /* SQLITE_OMIT_LOAD_EXTENSION */
-SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *);
-SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int);
-SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*);
-SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*);
-
-/*
-** Convenience functions for opening and closing files using
-** sqlite3_malloc() to obtain space for the file-handle structure.
-*/
-SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*);
-SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *);
-
-#endif /* _SQLITE_OS_H_ */
-
-/************** End of os.h **************************************************/
-/************** Continuing where we left off in sqliteInt.h ******************/
/************** Include mutex.h in the middle of sqliteInt.h *****************/
/************** Begin file mutex.h *******************************************/
/*
@@ -16743,6 +16837,7 @@ struct Lookaside {
#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */
void *pStart; /* First byte of available memory space */
void *pEnd; /* First byte past end of available space */
+ void *pTrueEnd; /* True value of pEnd, when db->pnBytesFreed!=0 */
};
struct LookasideSlot {
LookasideSlot *pNext; /* Next buffer in the list of free buffers */
@@ -17087,6 +17182,7 @@ struct sqlite3 {
#define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */
#define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */
/* TH3 expects this value ^^^^^^^^^^ See flatten04.test */
+#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */
#define SQLITE_AllOpts 0xffffffff /* All optimizations */
/*
@@ -17659,7 +17755,7 @@ struct Table {
#ifndef SQLITE_OMIT_VIRTUALTABLE
# define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB)
# define ExprIsVtab(X) \
- ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->eTabType==TABTYP_VTAB)
+ ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB)
#else
# define IsVirtual(X) 0
# define ExprIsVtab(X) 0
@@ -17876,10 +17972,22 @@ struct UnpackedRecord {
** The Index.onError field determines whether or not the indexed columns
** must be unique and what to do if they are not. When Index.onError=OE_None,
** it means this is not a unique index. Otherwise it is a unique index
-** and the value of Index.onError indicate the which conflict resolution
-** algorithm to employ whenever an attempt is made to insert a non-unique
+** and the value of Index.onError indicates which conflict resolution
+** algorithm to employ when an attempt is made to insert a non-unique
** element.
**
+** The colNotIdxed bitmask is used in combination with SrcItem.colUsed
+** for a fast test to see if an index can serve as a covering index.
+** colNotIdxed has a 1 bit for every column of the original table that
+** is *not* available in the index. Thus the expression
+** "colUsed & colNotIdxed" will be non-zero if the index is not a
+** covering index. The most significant bit of of colNotIdxed will always
+** be true (note-20221022-a). If a column beyond the 63rd column of the
+** table is used, the "colUsed & colNotIdxed" test will always be non-zero
+** and we have to assume either that the index is not covering, or use
+** an alternative (slower) algorithm to determine whether or not
+** the index is covering.
+**
** While parsing a CREATE TABLE or CREATE INDEX statement in order to
** generate VDBE code (as opposed to parsing one read from an sqlite_schema
** table as part of parsing an existing database schema), transient instances
@@ -17915,6 +18023,8 @@ struct Index {
unsigned bNoQuery:1; /* Do not use this index to optimize queries */
unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */
unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */
+ unsigned bHasExpr:1; /* Index contains an expression, either a literal
+ ** expression, or a reference to a VIRTUAL column */
#ifdef SQLITE_ENABLE_STAT4
int nSample; /* Number of elements in aSample[] */
int nSampleCol; /* Size of IndexSample.anEq[] and so on */
@@ -17923,7 +18033,7 @@ struct Index {
tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */
tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */
#endif
- Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */
+ Bitmask colNotIdxed; /* Unindexed columns in pTab */
};
/*
@@ -18191,7 +18301,7 @@ struct Expr {
#define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */
#define EP_Win 0x008000 /* Contains window functions */
#define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */
-#define EP_MemToken 0x020000 /* Need to sqlite3DbFree() Expr.zToken */
+ /* 0x020000 // Available for reuse */
#define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */
#define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */
#define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */
@@ -18376,6 +18486,14 @@ struct IdList {
** The SrcItem object represents a single term in the FROM clause of a query.
** The SrcList object is mostly an array of SrcItems.
**
+** The jointype starts out showing the join type between the current table
+** and the next table on the list. The parser builds the list this way.
+** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each
+** jointype expresses the join between the table and the previous table.
+**
+** In the colUsed field, the high-order bit (bit 63) is set if the table
+** contains more than 63 columns and the 64-th or later column is used.
+**
** Union member validity:
**
** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc
@@ -18415,14 +18533,14 @@ struct SrcItem {
Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */
IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */
} u3;
- Bitmask colUsed; /* Bit N (1<62 */
union {
char *zIndexedBy; /* Identifier from "INDEXED BY " clause */
ExprList *pFuncArg; /* Arguments to table-valued-function */
} u1;
union {
Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */
- CteUse *pCteUse; /* CTE Usage info info fg.isCte is true */
+ CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */
} u2;
};
@@ -18436,23 +18554,11 @@ struct OnOrUsing {
};
/*
-** The following structure describes the FROM clause of a SELECT statement.
-** Each table or subquery in the FROM clause is a separate element of
-** the SrcList.a[] array.
+** This object represents one or more tables that are the source of
+** content for an SQL statement. For example, a single SrcList object
+** is used to hold the FROM clause of a SELECT statement. SrcList also
+** represents the target tables for DELETE, INSERT, and UPDATE statements.
**
-** With the addition of multiple database support, the following structure
-** can also be used to describe a particular table such as the table that
-** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL,
-** such a table must be a simple name: ID. But in SQLite, the table can
-** now be identified by a database name, a dot, then the table name: ID.ID.
-**
-** The jointype starts out showing the join type between the current table
-** and the next table on the list. The parser builds the list this way.
-** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each
-** jointype expresses the join between the table and the previous table.
-**
-** In the colUsed field, the high-order bit (bit 63) is set if the table
-** contains more than 63 columns and the 64-th or later column is used.
*/
struct SrcList {
int nSrc; /* Number of tables or subqueries in the FROM clause */
@@ -18797,7 +18903,7 @@ struct SelectDest {
int iSDParm2; /* A second parameter for the eDest disposal method */
int iSdst; /* Base register where results are written */
int nSdst; /* Number of registers allocated */
- char *zAffSdst; /* Affinity used when eDest==SRT_Set */
+ char *zAffSdst; /* Affinity used for SRT_Set, SRT_Table, and similar */
ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */
};
@@ -18862,6 +18968,28 @@ struct TriggerPrg {
# define DbMaskNonZero(M) (M)!=0
#endif
+/*
+** For each index X that has as one of its arguments either an expression
+** or the name of a virtual generated column, and if X is in scope such that
+** the value of the expression can simply be read from the index, then
+** there is an instance of this object on the Parse.pIdxExpr list.
+**
+** During code generation, while generating code to evaluate expressions,
+** this list is consulted and if a matching expression is found, the value
+** is read from the index rather than being recomputed.
+*/
+struct IndexedExpr {
+ Expr *pExpr; /* The expression contained in the index */
+ int iDataCur; /* The data cursor associated with the index */
+ int iIdxCur; /* The index cursor */
+ int iIdxCol; /* The index column that contains value of pExpr */
+ u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */
+ IndexedExpr *pIENext; /* Next in a list of all indexed expressions */
+#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
+ const char *zIdxName; /* Name of index, used only for bytecode comments */
+#endif
+};
+
/*
** An instance of the ParseCleanup object specifies an operation that
** should be performed after parsing to deallocation resources obtained
@@ -18903,7 +19031,7 @@ struct Parse {
u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */
u8 okConstFactor; /* OK to factor out constants */
u8 disableLookaside; /* Number of times lookaside has been disabled */
- u8 disableVtab; /* Disable all virtual tables for this parse */
+ u8 prepFlags; /* SQLITE_PREPARE_* flags */
u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */
@@ -18920,6 +19048,7 @@ struct Parse {
int nLabelAlloc; /* Number of slots in aLabel */
int *aLabel; /* Space to hold the labels */
ExprList *pConstExpr;/* Constant expressions */
+ IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */
Token constraintName;/* Name of the constraint currently being parsed */
yDbMask writeMask; /* Start a write transaction on these databases */
yDbMask cookieMask; /* Bitmask of schema verified databases */
@@ -19355,15 +19484,15 @@ struct Walker {
struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */
int *aiCol; /* array of column indexes */
struct IdxCover *pIdxCover; /* Check for index coverage */
- struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */
ExprList *pGroupBy; /* GROUP BY clause */
Select *pSelect; /* HAVING to WHERE clause ctx */
struct WindowRewrite *pRewrite; /* Window rewrite context */
struct WhereConst *pConst; /* WHERE clause constants */
struct RenameCtx *pRename; /* RENAME COLUMN context */
struct Table *pTab; /* Table of generated column */
+ struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */
SrcItem *pSrcItem; /* A single FROM clause item */
- DbFixer *pFix;
+ DbFixer *pFix; /* See sqlite3FixSelect() */
} u;
};
@@ -19669,6 +19798,7 @@ SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64);
SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64);
SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*);
SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3*, void*);
+SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3*, void*);
SQLITE_PRIVATE int sqlite3MallocSize(const void*);
SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, const void*);
SQLITE_PRIVATE void *sqlite3PageMalloc(int);
@@ -19689,12 +19819,16 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void);
*/
#ifdef SQLITE_USE_ALLOCA
# define sqlite3StackAllocRaw(D,N) alloca(N)
+# define sqlite3StackAllocRawNN(D,N) alloca(N)
# define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N)
# define sqlite3StackFree(D,P)
+# define sqlite3StackFreeNN(D,P)
#else
# define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N)
+# define sqlite3StackAllocRawNN(D,N) sqlite3DbMallocRawNN(D,N)
# define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N)
# define sqlite3StackFree(D,P) sqlite3DbFree(D,P)
+# define sqlite3StackFreeNN(D,P) sqlite3DbFreeNN(D,P)
#endif
/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they
@@ -19779,6 +19913,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*);
SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8);
SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8);
SQLITE_PRIVATE void sqlite3TreeViewUpsert(TreeView*, const Upsert*, u8);
+#if TREETRACE_ENABLED
SQLITE_PRIVATE void sqlite3TreeViewDelete(const With*, const SrcList*, const Expr*,
const ExprList*,const Expr*, const Trigger*);
SQLITE_PRIVATE void sqlite3TreeViewInsert(const With*, const SrcList*,
@@ -19787,6 +19922,7 @@ SQLITE_PRIVATE void sqlite3TreeViewInsert(const With*, const SrcList*,
SQLITE_PRIVATE void sqlite3TreeViewUpdate(const With*, const SrcList*, const ExprList*,
const Expr*, int, const ExprList*, const Expr*,
const Upsert*, const Trigger*);
+#endif
#ifndef SQLITE_OMIT_TRIGGER
SQLITE_PRIVATE void sqlite3TreeViewTriggerStep(TreeView*, const TriggerStep*, u8, u8);
SQLITE_PRIVATE void sqlite3TreeViewTrigger(TreeView*, const Trigger*, u8, u8);
@@ -20191,6 +20327,7 @@ SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*);
SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*);
SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*);
SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64);
+SQLITE_PRIVATE i64 sqlite3RealToI64(double);
SQLITE_PRIVATE void sqlite3Int64ToText(i64,char*);
SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
@@ -20236,6 +20373,7 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v);
SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*);
+SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3*,const Table*);
SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int);
SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2);
SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity);
@@ -20307,7 +20445,6 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[];
SQLITE_PRIVATE const char sqlite3StrBINARY[];
SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[];
SQLITE_PRIVATE const char sqlite3StdTypeAffinity[];
-SQLITE_PRIVATE const char sqlite3StdTypeMap[];
SQLITE_PRIVATE const char *sqlite3StdType[];
SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[];
SQLITE_PRIVATE const unsigned char *sqlite3aLTb;
@@ -20751,6 +20888,10 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse*, Expr*);
SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt);
#endif
+#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)
+SQLITE_PRIVATE int sqlite3KvvfsInit(void);
+#endif
+
#endif /* SQLITEINT_H */
/************** End of sqliteInt.h *******************************************/
@@ -20982,7 +21123,7 @@ SQLITE_API extern int sqlite3_open_file_count;
** autoconf-based build
*/
#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H)
-/* #include "config.h" */
+/* #include "sqlite_cfg.h" */
#define SQLITECONFIG_H 1
#endif
@@ -21147,6 +21288,9 @@ static const char * const sqlite3azCompileOpt[] = {
#ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
"DISABLE_SKIPAHEAD_DISTINCT",
#endif
+#ifdef SQLITE_DQS
+ "DQS=" CTIMEOPT_VAL(SQLITE_DQS),
+#endif
#ifdef SQLITE_ENABLE_8_3_NAMES
"ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES),
#endif
@@ -21637,9 +21781,6 @@ static const char * const sqlite3azCompileOpt[] = {
#ifdef SQLITE_OMIT_XFER_OPT
"OMIT_XFER_OPT",
#endif
-#ifdef SQLITE_PCACHE_SEPARATE_HEADER
- "PCACHE_SEPARATE_HEADER",
-#endif
#ifdef SQLITE_PERFORMANCE_TRACE
"PERFORMANCE_TRACE",
#endif
@@ -22119,10 +22260,6 @@ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY";
**
** sqlite3StdTypeAffinity[] The affinity associated with each entry
** in sqlite3StdType[].
-**
-** sqlite3StdTypeMap[] The type value (as returned from
-** sqlite3_column_type() or sqlite3_value_type())
-** for each entry in sqlite3StdType[].
*/
SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 };
SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = {
@@ -22133,14 +22270,6 @@ SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = {
SQLITE_AFF_REAL,
SQLITE_AFF_TEXT
};
-SQLITE_PRIVATE const char sqlite3StdTypeMap[] = {
- 0,
- SQLITE_BLOB,
- SQLITE_INTEGER,
- SQLITE_INTEGER,
- SQLITE_FLOAT,
- SQLITE_TEXT
-};
SQLITE_PRIVATE const char *sqlite3StdType[] = {
"ANY",
"BLOB",
@@ -22592,7 +22721,7 @@ struct DblquoteStr {
*/
struct Vdbe {
sqlite3 *db; /* The database connection that owns this statement */
- Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */
+ Vdbe **ppVPrev,*pVNext; /* Linked list of VDBEs with the same Vdbe.db */
Parse *pParse; /* Parsing context used to create this Vdbe */
ynVar nVar; /* Number of entries in aVar[] */
int nMem; /* Number of memory locations currently allocated */
@@ -23150,6 +23279,8 @@ SQLITE_API int sqlite3_db_status(
sqlite3BtreeEnterAll(db);
db->pnBytesFreed = &nByte;
+ assert( db->lookaside.pEnd==db->lookaside.pTrueEnd );
+ db->lookaside.pEnd = db->lookaside.pStart;
for(i=0; inDb; i++){
Schema *pSchema = db->aDb[i].pSchema;
if( ALWAYS(pSchema!=0) ){
@@ -23175,6 +23306,7 @@ SQLITE_API int sqlite3_db_status(
}
}
db->pnBytesFreed = 0;
+ db->lookaside.pEnd = db->lookaside.pTrueEnd;
sqlite3BtreeLeaveAll(db);
*pHighwater = 0;
@@ -23192,9 +23324,12 @@ SQLITE_API int sqlite3_db_status(
int nByte = 0; /* Used to accumulate return value */
db->pnBytesFreed = &nByte;
- for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){
+ assert( db->lookaside.pEnd==db->lookaside.pTrueEnd );
+ db->lookaside.pEnd = db->lookaside.pStart;
+ for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pVNext){
sqlite3VdbeDelete(pVdbe);
}
+ db->lookaside.pEnd = db->lookaside.pTrueEnd;
db->pnBytesFreed = 0;
*pHighwater = 0; /* IMP: R-64479-57858 */
@@ -23530,7 +23665,7 @@ static void computeJD(DateTime *p){
p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000);
p->validJD = 1;
if( p->validHMS ){
- p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000);
+ p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5);
if( p->validTZ ){
p->iJD -= p->tz*60000;
p->validYMD = 0;
@@ -24039,7 +24174,7 @@ static int parseModifier(
*/
if( sqlite3_strnicmp(z, "weekday ", 8)==0
&& sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0
- && (n=(int)r)==r && n>=0 && r<7 ){
+ && r>=0.0 && r<7.0 && (n=(int)r)==r ){
sqlite3_int64 Z;
computeYMD_HMS(p);
p->validTZ = 0;
@@ -24720,9 +24855,11 @@ SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){
}
SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){
DO_OS_MALLOC_TEST(id);
+ assert( lockType>=SQLITE_LOCK_SHARED && lockType<=SQLITE_LOCK_EXCLUSIVE );
return id->pMethods->xLock(id, lockType);
}
SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){
+ assert( lockType==SQLITE_LOCK_NONE || lockType==SQLITE_LOCK_SHARED );
return id->pMethods->xUnlock(id, lockType);
}
SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){
@@ -24837,6 +24974,7 @@ SQLITE_PRIVATE int sqlite3OsOpen(
** down into the VFS layer. Some SQLITE_OPEN_ flags (for example,
** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before
** reaching the VFS. */
+ assert( zPath || (flags & SQLITE_OPEN_EXCLUSIVE) );
rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut);
assert( rc==SQLITE_OK || pFile->pMethods==0 );
return rc;
@@ -29055,18 +29193,34 @@ static void mallocWithAlarm(int n, void **pp){
*pp = p;
}
+/*
+** Maximum size of any single memory allocation.
+**
+** This is not a limit on the total amount of memory used. This is
+** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc().
+**
+** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391
+** This provides a 256-byte safety margin for defense against 32-bit
+** signed integer overflow bugs when computing memory allocation sizes.
+** Parnoid applications might want to reduce the maximum allocation size
+** further for an even larger safety margin. 0x3fffffff or 0x0fffffff
+** or even smaller would be reasonable upper bounds on the size of a memory
+** allocations for most applications.
+*/
+#ifndef SQLITE_MAX_ALLOCATION_SIZE
+# define SQLITE_MAX_ALLOCATION_SIZE 2147483391
+#endif
+#if SQLITE_MAX_ALLOCATION_SIZE>2147483391
+# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391
+#endif
+
/*
** Allocate memory. This routine is like sqlite3_malloc() except that it
** assumes the memory subsystem has already been initialized.
*/
SQLITE_PRIVATE void *sqlite3Malloc(u64 n){
void *p;
- if( n==0 || n>=0x7fffff00 ){
- /* A memory allocation of a number of bytes which is near the maximum
- ** signed integer value might cause an integer overflow inside of the
- ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving
- ** 255 bytes of overhead. SQLite itself will never use anything near
- ** this amount. The only way to reach the limit is with sqlite3_malloc() */
+ if( n==0 || n>SQLITE_MAX_ALLOCATION_SIZE ){
p = 0;
}else if( sqlite3GlobalConfig.bMemstat ){
sqlite3_mutex_enter(mem0.mutex);
@@ -29102,7 +29256,7 @@ SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){
*/
#ifndef SQLITE_OMIT_LOOKASIDE
static int isLookaside(sqlite3 *db, const void *p){
- return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd);
+ return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pTrueEnd);
}
#else
#define isLookaside(A,B) 0
@@ -29126,18 +29280,16 @@ static int lookasideMallocSize(sqlite3 *db, const void *p){
SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, const void *p){
assert( p!=0 );
#ifdef SQLITE_DEBUG
- if( db==0 || !isLookaside(db,p) ){
- if( db==0 ){
- assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
- assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
- }else{
- assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
- assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
- }
+ if( db==0 ){
+ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
+ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
+ }else if( !isLookaside(db,p) ){
+ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+ assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
}
#endif
if( db ){
- if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
+ if( ((uptr)p)<(uptr)(db->lookaside.pTrueEnd) ){
#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
assert( sqlite3_mutex_held(db->mutex) );
@@ -29193,14 +29345,11 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){
assert( db==0 || sqlite3_mutex_held(db->mutex) );
assert( p!=0 );
if( db ){
- if( db->pnBytesFreed ){
- measureAllocationSize(db, p);
- return;
- }
if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
LookasideSlot *pBuf = (LookasideSlot*)p;
+ assert( db->pnBytesFreed==0 );
#ifdef SQLITE_DEBUG
memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */
#endif
@@ -29211,6 +29360,7 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){
#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */
if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){
LookasideSlot *pBuf = (LookasideSlot*)p;
+ assert( db->pnBytesFreed==0 );
#ifdef SQLITE_DEBUG
memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */
#endif
@@ -29219,6 +29369,10 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){
return;
}
}
+ if( db->pnBytesFreed ){
+ measureAllocationSize(db, p);
+ return;
+ }
}
assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
@@ -29226,6 +29380,43 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){
sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
sqlite3_free(p);
}
+SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3 *db, void *p){
+ assert( db!=0 );
+ assert( sqlite3_mutex_held(db->mutex) );
+ assert( p!=0 );
+ if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
+#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
+ if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
+ LookasideSlot *pBuf = (LookasideSlot*)p;
+ assert( db->pnBytesFreed==0 );
+#ifdef SQLITE_DEBUG
+ memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */
+#endif
+ pBuf->pNext = db->lookaside.pSmallFree;
+ db->lookaside.pSmallFree = pBuf;
+ return;
+ }
+#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */
+ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){
+ LookasideSlot *pBuf = (LookasideSlot*)p;
+ assert( db->pnBytesFreed==0 );
+#ifdef SQLITE_DEBUG
+ memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */
+#endif
+ pBuf->pNext = db->lookaside.pFree;
+ db->lookaside.pFree = pBuf;
+ return;
+ }
+ }
+ if( db->pnBytesFreed ){
+ measureAllocationSize(db, p);
+ return;
+ }
+ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+ assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+ sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
+ sqlite3_free(p);
+}
SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){
assert( db==0 || sqlite3_mutex_held(db->mutex) );
if( p ) sqlite3DbFreeNN(db, p);
@@ -29561,8 +29752,13 @@ SQLITE_PRIVATE void *sqlite3OomFault(sqlite3 *db){
}
DisableLookaside;
if( db->pParse ){
+ Parse *pParse;
sqlite3ErrorMsg(db->pParse, "out of memory");
db->pParse->rc = SQLITE_NOMEM_BKPT;
+ for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){
+ pParse->nErr++;
+ pParse->rc = SQLITE_NOMEM;
+ }
}
}
return 0;
@@ -30428,8 +30624,8 @@ SQLITE_API void sqlite3_str_vappendf(
case etSQLESCAPE: /* %q: Escape ' characters */
case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */
case etSQLESCAPE3: { /* %w: Escape " characters */
- int i, j, k, n, isnull;
- int needQuote;
+ i64 i, j, k, n;
+ int needQuote, isnull;
char ch;
char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */
char *escarg;
@@ -31117,8 +31313,8 @@ SQLITE_PRIVATE void sqlite3TreeViewColumnList(
sqlite3TreeViewLine(pView, "COLUMNS");
for(i=0; ipTab;
sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1);
}
- assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
+ assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0);
}
if( pItem->fg.isTabFunc ){
@@ -32016,6 +32212,7 @@ SQLITE_PRIVATE void sqlite3TreeViewUpsert(
sqlite3TreeViewPop(&pView);
}
+#if TREETRACE_ENABLED
/*
** Generate a human-readable diagram of the data structure that go
** into generating an DELETE statement.
@@ -32069,7 +32266,9 @@ SQLITE_PRIVATE void sqlite3TreeViewDelete(
}
sqlite3TreeViewPop(&pView);
}
+#endif /* TREETRACE_ENABLED */
+#if TREETRACE_ENABLED
/*
** Generate a human-readable diagram of the data structure that go
** into generating an INSERT statement.
@@ -32137,7 +32336,9 @@ SQLITE_PRIVATE void sqlite3TreeViewInsert(
}
sqlite3TreeViewPop(&pView);
}
+#endif /* TREETRACE_ENABLED */
+#if TREETRACE_ENABLED
/*
** Generate a human-readable diagram of the data structure that go
** into generating an UPDATE statement.
@@ -32213,6 +32414,7 @@ SQLITE_PRIVATE void sqlite3TreeViewUpdate(
}
sqlite3TreeViewPop(&pView);
}
+#endif /* TREETRACE_ENABLED */
#ifndef SQLITE_OMIT_TRIGGER
/*
@@ -32326,16 +32528,41 @@ SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window *p){ sqlite3TreeViewWinFunc(
** This structure is the current state of the generator.
*/
static SQLITE_WSD struct sqlite3PrngType {
- unsigned char isInit; /* True if initialized */
- unsigned char i, j; /* State variables */
- unsigned char s[256]; /* State variables */
+ u32 s[16]; /* 64 bytes of chacha20 state */
+ u8 out[64]; /* Output bytes */
+ u8 n; /* Output bytes remaining */
} sqlite3Prng;
+
+/* The RFC-7539 ChaCha20 block function
+*/
+#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b))))
+#define QR(a, b, c, d) ( \
+ a += b, d ^= a, d = ROTL(d,16), \
+ c += d, b ^= c, b = ROTL(b,12), \
+ a += b, d ^= a, d = ROTL(d, 8), \
+ c += d, b ^= c, b = ROTL(b, 7))
+static void chacha_block(u32 *out, const u32 *in){
+ int i;
+ u32 x[16];
+ memcpy(x, in, 64);
+ for(i=0; i<10; i++){
+ QR(x[0], x[4], x[ 8], x[12]);
+ QR(x[1], x[5], x[ 9], x[13]);
+ QR(x[2], x[6], x[10], x[14]);
+ QR(x[3], x[7], x[11], x[15]);
+ QR(x[0], x[5], x[10], x[15]);
+ QR(x[1], x[6], x[11], x[12]);
+ QR(x[2], x[7], x[ 8], x[13]);
+ QR(x[3], x[4], x[ 9], x[14]);
+ }
+ for(i=0; i<16; i++) out[i] = x[i]+in[i];
+}
+
/*
** Return N random bytes.
*/
SQLITE_API void sqlite3_randomness(int N, void *pBuf){
- unsigned char t;
unsigned char *zBuf = pBuf;
/* The "wsdPrng" macro will resolve to the pseudo-random number generator
@@ -32365,53 +32592,46 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){
sqlite3_mutex_enter(mutex);
if( N<=0 || pBuf==0 ){
- wsdPrng.isInit = 0;
+ wsdPrng.s[0] = 0;
sqlite3_mutex_leave(mutex);
return;
}
/* Initialize the state of the random number generator once,
- ** the first time this routine is called. The seed value does
- ** not need to contain a lot of randomness since we are not
- ** trying to do secure encryption or anything like that...
- **
- ** Nothing in this file or anywhere else in SQLite does any kind of
- ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random
- ** number generator) not as an encryption device.
+ ** the first time this routine is called.
*/
- if( !wsdPrng.isInit ){
+ if( wsdPrng.s[0]==0 ){
sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
- int i;
- char k[256];
- wsdPrng.j = 0;
- wsdPrng.i = 0;
+ static const u32 chacha20_init[] = {
+ 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574
+ };
+ memcpy(&wsdPrng.s[0], chacha20_init, 16);
if( NEVER(pVfs==0) ){
- memset(k, 0, sizeof(k));
+ memset(&wsdPrng.s[4], 0, 44);
}else{
- sqlite3OsRandomness(pVfs, 256, k);
+ sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]);
}
- for(i=0; i<256; i++){
- wsdPrng.s[i] = (u8)i;
- }
- for(i=0; i<256; i++){
- wsdPrng.j += wsdPrng.s[i] + k[i];
- t = wsdPrng.s[wsdPrng.j];
- wsdPrng.s[wsdPrng.j] = wsdPrng.s[i];
- wsdPrng.s[i] = t;
- }
- wsdPrng.isInit = 1;
+ wsdPrng.s[15] = wsdPrng.s[12];
+ wsdPrng.s[12] = 0;
+ wsdPrng.n = 0;
}
assert( N>0 );
- do{
- wsdPrng.i++;
- t = wsdPrng.s[wsdPrng.i];
- wsdPrng.j += t;
- wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j];
- wsdPrng.s[wsdPrng.j] = t;
- t += wsdPrng.s[wsdPrng.i];
- *(zBuf++) = wsdPrng.s[t];
- }while( --N );
+ while( 1 /* exit by break */ ){
+ if( N<=wsdPrng.n ){
+ memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N);
+ wsdPrng.n -= N;
+ break;
+ }
+ if( wsdPrng.n>0 ){
+ memcpy(zBuf, wsdPrng.out, wsdPrng.n);
+ N -= wsdPrng.n;
+ zBuf += wsdPrng.n;
+ }
+ wsdPrng.s[12]++;
+ chacha_block((u32*)wsdPrng.out, wsdPrng.s);
+ wsdPrng.n = 64;
+ }
sqlite3_mutex_leave(mutex);
}
@@ -33451,7 +33671,7 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){
va_list ap;
sqlite3 *db = pParse->db;
assert( db!=0 );
- assert( db->pParse==pParse );
+ assert( db->pParse==pParse || db->pParse->pToplevel==pParse );
db->errByteOffset = -2;
va_start(ap, zFormat);
zMsg = sqlite3VMPrintf(db, zFormat, ap);
@@ -35269,48 +35489,48 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){
/* 5 */ "Vacuum" OpHelp(""),
/* 6 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"),
/* 7 */ "VUpdate" OpHelp("data=r[P3@P2]"),
- /* 8 */ "Goto" OpHelp(""),
- /* 9 */ "Gosub" OpHelp(""),
- /* 10 */ "InitCoroutine" OpHelp(""),
- /* 11 */ "Yield" OpHelp(""),
- /* 12 */ "MustBeInt" OpHelp(""),
- /* 13 */ "Jump" OpHelp(""),
- /* 14 */ "Once" OpHelp(""),
- /* 15 */ "If" OpHelp(""),
- /* 16 */ "IfNot" OpHelp(""),
- /* 17 */ "IsNullOrType" OpHelp("if typeof(r[P1]) IN (P3,5) goto P2"),
- /* 18 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"),
+ /* 8 */ "Init" OpHelp("Start at P2"),
+ /* 9 */ "Goto" OpHelp(""),
+ /* 10 */ "Gosub" OpHelp(""),
+ /* 11 */ "InitCoroutine" OpHelp(""),
+ /* 12 */ "Yield" OpHelp(""),
+ /* 13 */ "MustBeInt" OpHelp(""),
+ /* 14 */ "Jump" OpHelp(""),
+ /* 15 */ "Once" OpHelp(""),
+ /* 16 */ "If" OpHelp(""),
+ /* 17 */ "IfNot" OpHelp(""),
+ /* 18 */ "IsType" OpHelp("if typeof(P1.P3) in P5 goto P2"),
/* 19 */ "Not" OpHelp("r[P2]= !r[P1]"),
- /* 20 */ "SeekLT" OpHelp("key=r[P3@P4]"),
- /* 21 */ "SeekLE" OpHelp("key=r[P3@P4]"),
- /* 22 */ "SeekGE" OpHelp("key=r[P3@P4]"),
- /* 23 */ "SeekGT" OpHelp("key=r[P3@P4]"),
- /* 24 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"),
- /* 25 */ "IfNoHope" OpHelp("key=r[P3@P4]"),
- /* 26 */ "NoConflict" OpHelp("key=r[P3@P4]"),
- /* 27 */ "NotFound" OpHelp("key=r[P3@P4]"),
- /* 28 */ "Found" OpHelp("key=r[P3@P4]"),
- /* 29 */ "SeekRowid" OpHelp("intkey=r[P3]"),
- /* 30 */ "NotExists" OpHelp("intkey=r[P3]"),
- /* 31 */ "Last" OpHelp(""),
- /* 32 */ "IfSmaller" OpHelp(""),
- /* 33 */ "SorterSort" OpHelp(""),
- /* 34 */ "Sort" OpHelp(""),
- /* 35 */ "Rewind" OpHelp(""),
- /* 36 */ "SorterNext" OpHelp(""),
- /* 37 */ "Prev" OpHelp(""),
- /* 38 */ "Next" OpHelp(""),
- /* 39 */ "IdxLE" OpHelp("key=r[P3@P4]"),
- /* 40 */ "IdxGT" OpHelp("key=r[P3@P4]"),
- /* 41 */ "IdxLT" OpHelp("key=r[P3@P4]"),
- /* 42 */ "IdxGE" OpHelp("key=r[P3@P4]"),
+ /* 20 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"),
+ /* 21 */ "SeekLT" OpHelp("key=r[P3@P4]"),
+ /* 22 */ "SeekLE" OpHelp("key=r[P3@P4]"),
+ /* 23 */ "SeekGE" OpHelp("key=r[P3@P4]"),
+ /* 24 */ "SeekGT" OpHelp("key=r[P3@P4]"),
+ /* 25 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"),
+ /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"),
+ /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"),
+ /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"),
+ /* 29 */ "Found" OpHelp("key=r[P3@P4]"),
+ /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"),
+ /* 31 */ "NotExists" OpHelp("intkey=r[P3]"),
+ /* 32 */ "Last" OpHelp(""),
+ /* 33 */ "IfSmaller" OpHelp(""),
+ /* 34 */ "SorterSort" OpHelp(""),
+ /* 35 */ "Sort" OpHelp(""),
+ /* 36 */ "Rewind" OpHelp(""),
+ /* 37 */ "SorterNext" OpHelp(""),
+ /* 38 */ "Prev" OpHelp(""),
+ /* 39 */ "Next" OpHelp(""),
+ /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"),
+ /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"),
+ /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"),
/* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"),
/* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"),
- /* 45 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"),
- /* 46 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"),
- /* 47 */ "Program" OpHelp(""),
- /* 48 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"),
- /* 49 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"),
+ /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"),
+ /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"),
+ /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"),
+ /* 48 */ "Program" OpHelp(""),
+ /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"),
/* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"),
/* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"),
/* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"),
@@ -35320,12 +35540,12 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){
/* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"),
/* 58 */ "ElseEq" OpHelp(""),
- /* 59 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"),
- /* 60 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"),
- /* 61 */ "IncrVacuum" OpHelp(""),
- /* 62 */ "VNext" OpHelp(""),
- /* 63 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"),
- /* 64 */ "Init" OpHelp("Start at P2"),
+ /* 59 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"),
+ /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"),
+ /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"),
+ /* 62 */ "IncrVacuum" OpHelp(""),
+ /* 63 */ "VNext" OpHelp(""),
+ /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"),
/* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"),
/* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"),
/* 67 */ "Return" OpHelp(""),
@@ -35454,6 +35674,981 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){
#endif
/************** End of opcodes.c *********************************************/
+/************** Begin file os_kv.c *******************************************/
+/*
+** 2022-09-06
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an experimental VFS layer that operates on a
+** Key/Value storage engine where both keys and values must be pure
+** text.
+*/
+/* #include */
+#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL))
+
+/*****************************************************************************
+** Debugging logic
+*/
+
+/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */
+#if 0
+#define SQLITE_KV_TRACE(X) printf X
+#else
+#define SQLITE_KV_TRACE(X)
+#endif
+
+/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */
+#if 0
+#define SQLITE_KV_LOG(X) printf X
+#else
+#define SQLITE_KV_LOG(X)
+#endif
+
+
+/*
+** Forward declaration of objects used by this VFS implementation
+*/
+typedef struct KVVfsFile KVVfsFile;
+
+/* A single open file. There are only two files represented by this
+** VFS - the database and the rollback journal.
+*/
+struct KVVfsFile {
+ sqlite3_file base; /* IO methods */
+ const char *zClass; /* Storage class */
+ int isJournal; /* True if this is a journal file */
+ unsigned int nJrnl; /* Space allocated for aJrnl[] */
+ char *aJrnl; /* Journal content */
+ int szPage; /* Last known page size */
+ sqlite3_int64 szDb; /* Database file size. -1 means unknown */
+};
+
+/*
+** Methods for KVVfsFile
+*/
+static int kvvfsClose(sqlite3_file*);
+static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64);
+static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64);
+static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size);
+static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size);
+static int kvvfsSyncDb(sqlite3_file*, int flags);
+static int kvvfsSyncJrnl(sqlite3_file*, int flags);
+static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize);
+static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize);
+static int kvvfsLock(sqlite3_file*, int);
+static int kvvfsUnlock(sqlite3_file*, int);
+static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut);
+static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg);
+static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg);
+static int kvvfsSectorSize(sqlite3_file*);
+static int kvvfsDeviceCharacteristics(sqlite3_file*);
+
+/*
+** Methods for sqlite3_vfs
+*/
+static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename);
+static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int kvvfsSleep(sqlite3_vfs*, int microseconds);
+static int kvvfsCurrentTime(sqlite3_vfs*, double*);
+static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+
+static sqlite3_vfs sqlite3OsKvvfsObject = {
+ 1, /* iVersion */
+ sizeof(KVVfsFile), /* szOsFile */
+ 1024, /* mxPathname */
+ 0, /* pNext */
+ "kvvfs", /* zName */
+ 0, /* pAppData */
+ kvvfsOpen, /* xOpen */
+ kvvfsDelete, /* xDelete */
+ kvvfsAccess, /* xAccess */
+ kvvfsFullPathname, /* xFullPathname */
+ kvvfsDlOpen, /* xDlOpen */
+ 0, /* xDlError */
+ 0, /* xDlSym */
+ 0, /* xDlClose */
+ kvvfsRandomness, /* xRandomness */
+ kvvfsSleep, /* xSleep */
+ kvvfsCurrentTime, /* xCurrentTime */
+ 0, /* xGetLastError */
+ kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */
+};
+
+/* Methods for sqlite3_file objects referencing a database file
+*/
+static sqlite3_io_methods kvvfs_db_io_methods = {
+ 1, /* iVersion */
+ kvvfsClose, /* xClose */
+ kvvfsReadDb, /* xRead */
+ kvvfsWriteDb, /* xWrite */
+ kvvfsTruncateDb, /* xTruncate */
+ kvvfsSyncDb, /* xSync */
+ kvvfsFileSizeDb, /* xFileSize */
+ kvvfsLock, /* xLock */
+ kvvfsUnlock, /* xUnlock */
+ kvvfsCheckReservedLock, /* xCheckReservedLock */
+ kvvfsFileControlDb, /* xFileControl */
+ kvvfsSectorSize, /* xSectorSize */
+ kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */
+ 0, /* xShmMap */
+ 0, /* xShmLock */
+ 0, /* xShmBarrier */
+ 0, /* xShmUnmap */
+ 0, /* xFetch */
+ 0 /* xUnfetch */
+};
+
+/* Methods for sqlite3_file objects referencing a rollback journal
+*/
+static sqlite3_io_methods kvvfs_jrnl_io_methods = {
+ 1, /* iVersion */
+ kvvfsClose, /* xClose */
+ kvvfsReadJrnl, /* xRead */
+ kvvfsWriteJrnl, /* xWrite */
+ kvvfsTruncateJrnl, /* xTruncate */
+ kvvfsSyncJrnl, /* xSync */
+ kvvfsFileSizeJrnl, /* xFileSize */
+ kvvfsLock, /* xLock */
+ kvvfsUnlock, /* xUnlock */
+ kvvfsCheckReservedLock, /* xCheckReservedLock */
+ kvvfsFileControlJrnl, /* xFileControl */
+ kvvfsSectorSize, /* xSectorSize */
+ kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */
+ 0, /* xShmMap */
+ 0, /* xShmLock */
+ 0, /* xShmBarrier */
+ 0, /* xShmUnmap */
+ 0, /* xFetch */
+ 0 /* xUnfetch */
+};
+
+/****** Storage subsystem **************************************************/
+#include
+#include
+#include
+
+/* Forward declarations for the low-level storage engine
+*/
+static int kvstorageWrite(const char*, const char *zKey, const char *zData);
+static int kvstorageDelete(const char*, const char *zKey);
+static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf);
+#define KVSTORAGE_KEY_SZ 32
+
+/* Expand the key name with an appropriate prefix and put the result
+** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least
+** KVSTORAGE_KEY_SZ bytes.
+*/
+static void kvstorageMakeKey(
+ const char *zClass,
+ const char *zKeyIn,
+ char *zKeyOut
+){
+ sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn);
+}
+
+/* Write content into a key. zClass is the particular namespace of the
+** underlying key/value store to use - either "local" or "session".
+**
+** Both zKey and zData are zero-terminated pure text strings.
+**
+** Return the number of errors.
+*/
+static int kvstorageWrite(
+ const char *zClass,
+ const char *zKey,
+ const char *zData
+){
+ FILE *fd;
+ char zXKey[KVSTORAGE_KEY_SZ];
+ kvstorageMakeKey(zClass, zKey, zXKey);
+ fd = fopen(zXKey, "wb");
+ if( fd ){
+ SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey,
+ (int)strlen(zData), zData,
+ strlen(zData)>50 ? "..." : ""));
+ fputs(zData, fd);
+ fclose(fd);
+ return 0;
+ }else{
+ return 1;
+ }
+}
+
+/* Delete a key (with its corresponding data) from the key/value
+** namespace given by zClass. If the key does not previously exist,
+** this routine is a no-op.
+*/
+static int kvstorageDelete(const char *zClass, const char *zKey){
+ char zXKey[KVSTORAGE_KEY_SZ];
+ kvstorageMakeKey(zClass, zKey, zXKey);
+ unlink(zXKey);
+ SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey));
+ return 0;
+}
+
+/* Read the value associated with a zKey from the key/value namespace given
+** by zClass and put the text data associated with that key in the first
+** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large
+** enough to hold it all. The value put into zBuf must always be zero
+** terminated, even if it gets truncated because nBuf is not large enough.
+**
+** Return the total number of bytes in the data, without truncation, and
+** not counting the final zero terminator. Return -1 if the key does
+** not exist.
+**
+** If nBuf<=0 then this routine simply returns the size of the data without
+** actually reading it.
+*/
+static int kvstorageRead(
+ const char *zClass,
+ const char *zKey,
+ char *zBuf,
+ int nBuf
+){
+ FILE *fd;
+ struct stat buf;
+ char zXKey[KVSTORAGE_KEY_SZ];
+ kvstorageMakeKey(zClass, zKey, zXKey);
+ if( access(zXKey, R_OK)!=0
+ || stat(zXKey, &buf)!=0
+ || !S_ISREG(buf.st_mode)
+ ){
+ SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey));
+ return -1;
+ }
+ if( nBuf<=0 ){
+ return (int)buf.st_size;
+ }else if( nBuf==1 ){
+ zBuf[0] = 0;
+ SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey,
+ (int)buf.st_size));
+ return (int)buf.st_size;
+ }
+ if( nBuf > buf.st_size + 1 ){
+ nBuf = buf.st_size + 1;
+ }
+ fd = fopen(zXKey, "rb");
+ if( fd==0 ){
+ SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey));
+ return -1;
+ }else{
+ sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd);
+ fclose(fd);
+ zBuf[n] = 0;
+ SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey,
+ n, zBuf, n>50 ? "..." : ""));
+ return (int)n;
+ }
+}
+
+/*
+** An internal level of indirection which enables us to replace the
+** kvvfs i/o methods with JavaScript implementations in WASM builds.
+** Maintenance reminder: if this struct changes in any way, the JSON
+** rendering of its structure must be updated in
+** sqlite3_wasm_enum_json(). There are no binary compatibility
+** concerns, so it does not need an iVersion member. This file is
+** necessarily always compiled together with sqlite3_wasm_enum_json(),
+** and JS code dynamically creates the mapping of members based on
+** that JSON description.
+*/
+typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods;
+struct sqlite3_kvvfs_methods {
+ int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf);
+ int (*xWrite)(const char *zClass, const char *zKey, const char *zData);
+ int (*xDelete)(const char *zClass, const char *zKey);
+ const int nKeySize;
+};
+
+/*
+** This object holds the kvvfs I/O methods which may be swapped out
+** for JavaScript-side implementations in WASM builds. In such builds
+** it cannot be const, but in native builds it should be so that
+** the compiler can hopefully optimize this level of indirection out.
+** That said, kvvfs is intended primarily for use in WASM builds.
+**
+** Note that this is not explicitly flagged as static because the
+** amalgamation build will tag it with SQLITE_PRIVATE.
+*/
+#ifndef SQLITE_WASM
+const
+#endif
+SQLITE_PRIVATE sqlite3_kvvfs_methods sqlite3KvvfsMethods = {
+kvstorageRead,
+kvstorageWrite,
+kvstorageDelete,
+KVSTORAGE_KEY_SZ
+};
+
+/****** Utility subroutines ************************************************/
+
+/*
+** Encode binary into the text encoded used to persist on disk.
+** The output text is stored in aOut[], which must be at least
+** nData+1 bytes in length.
+**
+** Return the actual length of the encoded text, not counting the
+** zero terminator at the end.
+**
+** Encoding format
+** ---------------
+**
+** * Non-zero bytes are encoded as upper-case hexadecimal
+**
+** * A sequence of one or more zero-bytes that are not at the
+** beginning of the buffer are encoded as a little-endian
+** base-26 number using a..z. "a" means 0. "b" means 1,
+** "z" means 25. "ab" means 26. "ac" means 52. And so forth.
+**
+** * Because there is no overlap between the encoding characters
+** of hexadecimal and base-26 numbers, it is always clear where
+** one stops and the next begins.
+*/
+static int kvvfsEncode(const char *aData, int nData, char *aOut){
+ int i, j;
+ const unsigned char *a = (const unsigned char*)aData;
+ for(i=j=0; i>4];
+ aOut[j++] = "0123456789ABCDEF"[c&0xf];
+ }else{
+ /* A sequence of 1 or more zeros is stored as a little-endian
+ ** base-26 number using a..z as the digits. So one zero is "b".
+ ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb",
+ ** and so forth.
+ */
+ int k;
+ for(k=1; i+k0 ){
+ aOut[j++] = 'a'+(k%26);
+ k /= 26;
+ }
+ }
+ }
+ aOut[j] = 0;
+ return j;
+}
+
+static const signed char kvvfsHexValue[256] = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
+ -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
+};
+
+/*
+** Decode the text encoding back to binary. The binary content is
+** written into pOut, which must be at least nOut bytes in length.
+**
+** The return value is the number of bytes actually written into aOut[].
+*/
+static int kvvfsDecode(const char *a, char *aOut, int nOut){
+ int i, j;
+ int c;
+ const unsigned char *aIn = (const unsigned char*)a;
+ i = 0;
+ j = 0;
+ while( 1 ){
+ c = kvvfsHexValue[aIn[i]];
+ if( c<0 ){
+ int n = 0;
+ int mult = 1;
+ c = aIn[i];
+ if( c==0 ) break;
+ while( c>='a' && c<='z' ){
+ n += (c - 'a')*mult;
+ mult *= 26;
+ c = aIn[++i];
+ }
+ if( j+n>nOut ) return -1;
+ memset(&aOut[j], 0, n);
+ j += n;
+ c = aIn[i];
+ if( c==0 ) break;
+ }else{
+ aOut[j] = c<<4;
+ c = kvvfsHexValue[aIn[++i]];
+ if( c<0 ) break;
+ aOut[j++] += c;
+ i++;
+ }
+ }
+ return j;
+}
+
+/*
+** Decode a complete journal file. Allocate space in pFile->aJrnl
+** and store the decoding there. Or leave pFile->aJrnl set to NULL
+** if an error is encountered.
+**
+** The first few characters of the text encoding will be a little-endian
+** base-26 number (digits a..z) that is the total number of bytes
+** in the decoded journal file image. This base-26 number is followed
+** by a single space, then the encoding of the journal. The space
+** separator is required to act as a terminator for the base-26 number.
+*/
+static void kvvfsDecodeJournal(
+ KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */
+ const char *zTxt, /* Text encoding. Zero-terminated */
+ int nTxt /* Bytes in zTxt, excluding zero terminator */
+){
+ unsigned int n = 0;
+ int c, i, mult;
+ i = 0;
+ mult = 1;
+ while( (c = zTxt[i++])>='a' && c<='z' ){
+ n += (zTxt[i] - 'a')*mult;
+ mult *= 26;
+ }
+ sqlite3_free(pFile->aJrnl);
+ pFile->aJrnl = sqlite3_malloc64( n );
+ if( pFile->aJrnl==0 ){
+ pFile->nJrnl = 0;
+ return;
+ }
+ pFile->nJrnl = n;
+ n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl);
+ if( nnJrnl ){
+ sqlite3_free(pFile->aJrnl);
+ pFile->aJrnl = 0;
+ pFile->nJrnl = 0;
+ }
+}
+
+/*
+** Read or write the "sz" element, containing the database file size.
+*/
+static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){
+ char zData[50];
+ zData[0] = 0;
+ sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1);
+ return strtoll(zData, 0, 0);
+}
+static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){
+ char zData[50];
+ sqlite3_snprintf(sizeof(zData), zData, "%lld", sz);
+ return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData);
+}
+
+/****** sqlite3_io_methods methods ******************************************/
+
+/*
+** Close an kvvfs-file.
+*/
+static int kvvfsClose(sqlite3_file *pProtoFile){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+
+ SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass,
+ pFile->isJournal ? "journal" : "db"));
+ sqlite3_free(pFile->aJrnl);
+ return SQLITE_OK;
+}
+
+/*
+** Read from the -journal file.
+*/
+static int kvvfsReadJrnl(
+ sqlite3_file *pProtoFile,
+ void *zBuf,
+ int iAmt,
+ sqlite_int64 iOfst
+){
+ KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+ assert( pFile->isJournal );
+ SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+ if( pFile->aJrnl==0 ){
+ int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0);
+ char *aTxt;
+ if( szTxt<=4 ){
+ return SQLITE_IOERR;
+ }
+ aTxt = sqlite3_malloc64( szTxt+1 );
+ if( aTxt==0 ) return SQLITE_NOMEM;
+ kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1);
+ kvvfsDecodeJournal(pFile, aTxt, szTxt);
+ sqlite3_free(aTxt);
+ if( pFile->aJrnl==0 ) return SQLITE_IOERR;
+ }
+ if( iOfst+iAmt>pFile->nJrnl ){
+ return SQLITE_IOERR_SHORT_READ;
+ }
+ memcpy(zBuf, pFile->aJrnl+iOfst, iAmt);
+ return SQLITE_OK;
+}
+
+/*
+** Read from the database file.
+*/
+static int kvvfsReadDb(
+ sqlite3_file *pProtoFile,
+ void *zBuf,
+ int iAmt,
+ sqlite_int64 iOfst
+){
+ KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+ unsigned int pgno;
+ int got, n;
+ char zKey[30];
+ char aData[133073];
+ assert( iOfst>=0 );
+ assert( iAmt>=0 );
+ SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+ if( iOfst+iAmt>=512 ){
+ if( (iOfst % iAmt)!=0 ){
+ return SQLITE_IOERR_READ;
+ }
+ if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){
+ return SQLITE_IOERR_READ;
+ }
+ pFile->szPage = iAmt;
+ pgno = 1 + iOfst/iAmt;
+ }else{
+ pgno = 1;
+ }
+ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
+ got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1);
+ if( got<0 ){
+ n = 0;
+ }else{
+ aData[got] = 0;
+ if( iOfst+iAmt<512 ){
+ int k = iOfst+iAmt;
+ aData[k*2] = 0;
+ n = kvvfsDecode(aData, &aData[2000], sizeof(aData)-2000);
+ if( n>=iOfst+iAmt ){
+ memcpy(zBuf, &aData[2000+iOfst], iAmt);
+ n = iAmt;
+ }else{
+ n = 0;
+ }
+ }else{
+ n = kvvfsDecode(aData, zBuf, iAmt);
+ }
+ }
+ if( nzClass, iAmt, iOfst));
+ if( iEnd>=0x10000000 ) return SQLITE_FULL;
+ if( pFile->aJrnl==0 || pFile->nJrnlaJrnl, iEnd);
+ if( aNew==0 ){
+ return SQLITE_IOERR_NOMEM;
+ }
+ pFile->aJrnl = aNew;
+ if( pFile->nJrnlaJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl);
+ }
+ pFile->nJrnl = iEnd;
+ }
+ memcpy(pFile->aJrnl+iOfst, zBuf, iAmt);
+ return SQLITE_OK;
+}
+
+/*
+** Write into the database file.
+*/
+static int kvvfsWriteDb(
+ sqlite3_file *pProtoFile,
+ const void *zBuf,
+ int iAmt,
+ sqlite_int64 iOfst
+){
+ KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+ unsigned int pgno;
+ char zKey[30];
+ char aData[131073];
+ SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst));
+ assert( iAmt>=512 && iAmt<=65536 );
+ assert( (iAmt & (iAmt-1))==0 );
+ assert( pFile->szPage<0 || pFile->szPage==iAmt );
+ pFile->szPage = iAmt;
+ pgno = 1 + iOfst/iAmt;
+ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
+ kvvfsEncode(zBuf, iAmt, aData);
+ if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){
+ return SQLITE_IOERR;
+ }
+ if( iOfst+iAmt > pFile->szDb ){
+ pFile->szDb = iOfst + iAmt;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Truncate an kvvfs-file.
+*/
+static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size));
+ assert( size==0 );
+ sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl");
+ sqlite3_free(pFile->aJrnl);
+ pFile->aJrnl = 0;
+ pFile->nJrnl = 0;
+ return SQLITE_OK;
+}
+static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ if( pFile->szDb>size
+ && pFile->szPage>0
+ && (size % pFile->szPage)==0
+ ){
+ char zKey[50];
+ unsigned int pgno, pgnoMax;
+ SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size));
+ pgno = 1 + size/pFile->szPage;
+ pgnoMax = 2 + pFile->szDb/pFile->szPage;
+ while( pgno<=pgnoMax ){
+ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
+ sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey);
+ pgno++;
+ }
+ pFile->szDb = size;
+ return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK;
+ }
+ return SQLITE_IOERR;
+}
+
+/*
+** Sync an kvvfs-file.
+*/
+static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){
+ int i, n;
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ char *zOut;
+ SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass));
+ if( pFile->nJrnl<=0 ){
+ return kvvfsTruncateJrnl(pProtoFile, 0);
+ }
+ zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 );
+ if( zOut==0 ){
+ return SQLITE_IOERR_NOMEM;
+ }
+ n = pFile->nJrnl;
+ i = 0;
+ do{
+ zOut[i++] = 'a' + (n%26);
+ n /= 26;
+ }while( n>0 );
+ zOut[i++] = ' ';
+ kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]);
+ i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut);
+ sqlite3_free(zOut);
+ return i ? SQLITE_IOERR : SQLITE_OK;
+}
+static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){
+ return SQLITE_OK;
+}
+
+/*
+** Return the current file-size of an kvvfs-file.
+*/
+static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass));
+ *pSize = pFile->nJrnl;
+ return SQLITE_OK;
+}
+static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass));
+ if( pFile->szDb>=0 ){
+ *pSize = pFile->szDb;
+ }else{
+ *pSize = kvvfsReadFileSize(pFile);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Lock an kvvfs-file.
+*/
+static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ assert( !pFile->isJournal );
+ SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock));
+
+ if( eLock!=SQLITE_LOCK_NONE ){
+ pFile->szDb = kvvfsReadFileSize(pFile);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Unlock an kvvfs-file.
+*/
+static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ assert( !pFile->isJournal );
+ SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock));
+ if( eLock==SQLITE_LOCK_NONE ){
+ pFile->szDb = -1;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an kvvfs-file.
+*/
+static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){
+ SQLITE_KV_LOG(("xCheckReservedLock\n"));
+ *pResOut = 0;
+ return SQLITE_OK;
+}
+
+/*
+** File control method. For custom operations on an kvvfs-file.
+*/
+static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){
+ SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op));
+ return SQLITE_NOTFOUND;
+}
+static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){
+ SQLITE_KV_LOG(("xFileControl(%d) on database\n", op));
+ if( op==SQLITE_FCNTL_SYNC ){
+ KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
+ int rc = SQLITE_OK;
+ SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass));
+ if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){
+ rc = SQLITE_IOERR;
+ }
+ return rc;
+ }
+ return SQLITE_NOTFOUND;
+}
+
+/*
+** Return the sector-size in bytes for an kvvfs-file.
+*/
+static int kvvfsSectorSize(sqlite3_file *pFile){
+ return 512;
+}
+
+/*
+** Return the device characteristic flags supported by an kvvfs-file.
+*/
+static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){
+ return 0;
+}
+
+/****** sqlite3_vfs methods *************************************************/
+
+/*
+** Open an kvvfs file handle.
+*/
+static int kvvfsOpen(
+ sqlite3_vfs *pProtoVfs,
+ const char *zName,
+ sqlite3_file *pProtoFile,
+ int flags,
+ int *pOutFlags
+){
+ KVVfsFile *pFile = (KVVfsFile*)pProtoFile;
+ if( zName==0 ) zName = "";
+ SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName));
+ if( strcmp(zName, "local")==0
+ || strcmp(zName, "session")==0
+ ){
+ pFile->isJournal = 0;
+ pFile->base.pMethods = &kvvfs_db_io_methods;
+ }else
+ if( strcmp(zName, "local-journal")==0
+ || strcmp(zName, "session-journal")==0
+ ){
+ pFile->isJournal = 1;
+ pFile->base.pMethods = &kvvfs_jrnl_io_methods;
+ }else{
+ return SQLITE_CANTOPEN;
+ }
+ if( zName[0]=='s' ){
+ pFile->zClass = "session";
+ }else{
+ pFile->zClass = "local";
+ }
+ pFile->aJrnl = 0;
+ pFile->nJrnl = 0;
+ pFile->szPage = -1;
+ pFile->szDb = -1;
+ return SQLITE_OK;
+}
+
+/*
+** Delete the file located at zPath. If the dirSync argument is true,
+** ensure the file-system modifications are synced to disk before
+** returning.
+*/
+static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+ if( strcmp(zPath, "local-journal")==0 ){
+ sqlite3KvvfsMethods.xDelete("local", "jrnl");
+ }else
+ if( strcmp(zPath, "session-journal")==0 ){
+ sqlite3KvvfsMethods.xDelete("session", "jrnl");
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Test for access permissions. Return true if the requested permission
+** is available, or false otherwise.
+*/
+static int kvvfsAccess(
+ sqlite3_vfs *pProtoVfs,
+ const char *zPath,
+ int flags,
+ int *pResOut
+){
+ SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath));
+ if( strcmp(zPath, "local-journal")==0 ){
+ *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0;
+ }else
+ if( strcmp(zPath, "session-journal")==0 ){
+ *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0;
+ }else
+ if( strcmp(zPath, "local")==0 ){
+ *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0;
+ }else
+ if( strcmp(zPath, "session")==0 ){
+ *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0;
+ }else
+ {
+ *pResOut = 0;
+ }
+ SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut));
+ return SQLITE_OK;
+}
+
+/*
+** Populate buffer zOut with the full canonical pathname corresponding
+** to the pathname in zPath. zOut is guaranteed to point to a buffer
+** of at least (INST_MAX_PATHNAME+1) bytes.
+*/
+static int kvvfsFullPathname(
+ sqlite3_vfs *pVfs,
+ const char *zPath,
+ int nOut,
+ char *zOut
+){
+ size_t nPath;
+#ifdef SQLITE_OS_KV_ALWAYS_LOCAL
+ zPath = "local";
+#endif
+ nPath = strlen(zPath);
+ SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath));
+ if( nOut
+static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){
+ static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000;
+ struct timeval sNow;
+ (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */
+ *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000;
+ return SQLITE_OK;
+}
+#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */
+
+#if SQLITE_OS_KV
+/*
+** This routine is called initialize the KV-vfs as the default VFS.
+*/
+SQLITE_API int sqlite3_os_init(void){
+ return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1);
+}
+SQLITE_API int sqlite3_os_end(void){
+ return SQLITE_OK;
+}
+#endif /* SQLITE_OS_KV */
+
+#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)
+SQLITE_PRIVATE int sqlite3KvvfsInit(void){
+ return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0);
+}
+#endif
+
+/************** End of os_kv.c ***********************************************/
/************** Begin file os_unix.c *****************************************/
/*
** 2004 May 22
@@ -35544,13 +36739,13 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){
/*
** standard include files.
*/
-#include
-#include
+#include /* amalgamator: keep */
+#include /* amalgamator: keep */
#include
#include
-#include
+#include /* amalgamator: keep */
/* #include */
-#include
+#include /* amalgamator: keep */
#include
#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
# include
@@ -41312,6 +42507,7 @@ static const char *unixTempFileDir(void){
static int unixGetTempname(int nBuf, char *zBuf){
const char *zDir;
int iLimit = 0;
+ int rc = SQLITE_OK;
/* It's odd to simulate an io-error here, but really this is just
** using the io-error infrastructure to test that SQLite handles this
@@ -41320,18 +42516,26 @@ static int unixGetTempname(int nBuf, char *zBuf){
zBuf[0] = 0;
SimulateIOError( return SQLITE_IOERR );
+ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
zDir = unixTempFileDir();
- if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH;
- do{
- u64 r;
- sqlite3_randomness(sizeof(r), &r);
- assert( nBuf>2 );
- zBuf[nBuf-2] = 0;
- sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c",
- zDir, r, 0);
- if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR;
- }while( osAccess(zBuf,0)==0 );
- return SQLITE_OK;
+ if( zDir==0 ){
+ rc = SQLITE_IOERR_GETTEMPPATH;
+ }else{
+ do{
+ u64 r;
+ sqlite3_randomness(sizeof(r), &r);
+ assert( nBuf>2 );
+ zBuf[nBuf-2] = 0;
+ sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c",
+ zDir, r, 0);
+ if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ){
+ rc = SQLITE_ERROR;
+ break;
+ }
+ }while( osAccess(zBuf,0)==0 );
+ }
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
+ return rc;
}
#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__)
@@ -43506,8 +44710,16 @@ SQLITE_API int sqlite3_os_init(void){
/* Register all VFSes defined in the aVfs[] array */
for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){
+#ifdef SQLITE_DEFAULT_UNIX_VFS
+ sqlite3_vfs_register(&aVfs[i],
+ 0==strcmp(aVfs[i].zName,SQLITE_DEFAULT_UNIX_VFS));
+#else
sqlite3_vfs_register(&aVfs[i], i==0);
+#endif
}
+#ifdef SQLITE_OS_KV_OPTIONAL
+ sqlite3KvvfsInit();
+#endif
unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1);
#ifndef SQLITE_OMIT_WAL
@@ -45470,10 +46682,12 @@ SQLITE_API int sqlite3_win32_set_directory8(
const char *zValue /* New value for directory being set or reset */
){
char **ppDirectory = 0;
+ int rc;
#ifndef SQLITE_OMIT_AUTOINIT
- int rc = sqlite3_initialize();
+ rc = sqlite3_initialize();
if( rc ) return rc;
#endif
+ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){
ppDirectory = &sqlite3_data_directory;
}else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){
@@ -45488,14 +46702,19 @@ SQLITE_API int sqlite3_win32_set_directory8(
if( zValue && zValue[0] ){
zCopy = sqlite3_mprintf("%s", zValue);
if ( zCopy==0 ){
- return SQLITE_NOMEM_BKPT;
+ rc = SQLITE_NOMEM_BKPT;
+ goto set_directory8_done;
}
}
sqlite3_free(*ppDirectory);
*ppDirectory = zCopy;
- return SQLITE_OK;
+ rc = SQLITE_OK;
+ }else{
+ rc = SQLITE_ERROR;
}
- return SQLITE_ERROR;
+set_directory8_done:
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
+ return rc;
}
/*
@@ -48269,6 +49488,19 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){
return 0;
}
+/*
+** If sqlite3_temp_directory is defined, take the mutex and return true.
+**
+** If sqlite3_temp_directory is NULL (undefined), omit the mutex and
+** return false.
+*/
+static int winTempDirDefined(void){
+ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
+ if( sqlite3_temp_directory!=0 ) return 1;
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
+ return 0;
+}
+
/*
** Create a temporary file name and store the resulting pointer into pzBuf.
** The pointer returned in pzBuf must be freed via sqlite3_free().
@@ -48305,20 +49537,23 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){
*/
nDir = nMax - (nPre + 15);
assert( nDir>0 );
- if( sqlite3_temp_directory ){
+ if( winTempDirDefined() ){
int nDirLen = sqlite3Strlen30(sqlite3_temp_directory);
if( nDirLen>0 ){
if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){
nDirLen++;
}
if( nDirLen>nDir ){
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
sqlite3_free(zBuf);
OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n"));
return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0);
}
sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory);
}
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
}
+
#if defined(__CYGWIN__)
else{
static const char *azDirs[] = {
@@ -49107,7 +50342,7 @@ static BOOL winIsVerbatimPathname(
** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname
** bytes in size.
*/
-static int winFullPathname(
+static int winFullPathnameNoMutex(
sqlite3_vfs *pVfs, /* Pointer to vfs object */
const char *zRelative, /* Possibly relative input path */
int nFull, /* Size of output buffer in bytes */
@@ -49286,6 +50521,20 @@ static int winFullPathname(
}
#endif
}
+static int winFullPathname(
+ sqlite3_vfs *pVfs, /* Pointer to vfs object */
+ const char *zRelative, /* Possibly relative input path */
+ int nFull, /* Size of output buffer in bytes */
+ char *zFull /* Output buffer */
+){
+ int rc;
+ MUTEX_LOGIC( sqlite3_mutex *pMutex; )
+ MUTEX_LOGIC( pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); )
+ sqlite3_mutex_enter(pMutex);
+ rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull);
+ sqlite3_mutex_leave(pMutex);
+ return rc;
+}
#ifndef SQLITE_OMIT_LOAD_EXTENSION
/*
@@ -51074,12 +52323,20 @@ struct PCache {
int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */
int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */
# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;}
- void pcacheDump(PCache *pCache){
- int N;
- int i, j;
- sqlite3_pcache_page *pLower;
+ static void pcachePageTrace(int i, sqlite3_pcache_page *pLower){
PgHdr *pPg;
unsigned char *a;
+ int j;
+ pPg = (PgHdr*)pLower->pExtra;
+ printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags);
+ a = (unsigned char *)pLower->pBuf;
+ for(j=0; j<12; j++) printf("%02x", a[j]);
+ printf(" ptr %p\n", pPg);
+ }
+ static void pcacheDump(PCache *pCache){
+ int N;
+ int i;
+ sqlite3_pcache_page *pLower;
if( sqlite3PcacheTrace<2 ) return;
if( pCache->pCache==0 ) return;
@@ -51088,21 +52345,32 @@ struct PCache {
for(i=1; i<=N; i++){
pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0);
if( pLower==0 ) continue;
- pPg = (PgHdr*)pLower->pExtra;
- printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags);
- a = (unsigned char *)pLower->pBuf;
- for(j=0; j<12; j++) printf("%02x", a[j]);
- printf("\n");
- if( pPg->pPage==0 ){
+ pcachePageTrace(i, pLower);
+ if( ((PgHdr*)pLower)->pPage==0 ){
sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0);
}
}
}
- #else
+#else
# define pcacheTrace(X)
+# define pcachePageTrace(PGNO, X)
# define pcacheDump(X)
#endif
+/*
+** Return 1 if pPg is on the dirty list for pCache. Return 0 if not.
+** This routine runs inside of assert() statements only.
+*/
+#ifdef SQLITE_DEBUG
+static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){
+ PgHdr *p;
+ for(p=pCache->pDirty; p; p=p->pDirtyNext){
+ if( p==pPg ) return 1;
+ }
+ return 0;
+}
+#endif
+
/*
** Check invariants on a PgHdr entry. Return true if everything is OK.
** Return false if any invariant is violated.
@@ -51121,8 +52389,13 @@ SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){
assert( pCache!=0 ); /* Every page has an associated PCache */
if( pPg->flags & PGHDR_CLEAN ){
assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */
- assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */
- assert( pCache->pDirtyTail!=pPg );
+ assert( !pageOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirty list */
+ }else{
+ assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */
+ assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg );
+ assert( pPg->pDirtyPrev==0 || pPg->pDirtyPrev->pDirtyNext==pPg );
+ assert( pPg->pDirtyPrev!=0 || pCache->pDirty==pPg );
+ assert( pageOnDirtyList(pCache, pPg) );
}
/* WRITEABLE pages must also be DIRTY */
if( pPg->flags & PGHDR_WRITEABLE ){
@@ -51396,8 +52669,9 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch(
assert( createFlag==0 || pCache->eCreate==eCreate );
assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) );
pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate);
- pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno,
+ pcacheTrace(("%p.FETCH %d%s (result: %p) ",pCache,pgno,
createFlag?" create":"",pRes));
+ pcachePageTrace(pgno, pRes);
return pRes;
}
@@ -51525,6 +52799,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){
pcacheUnpin(p);
}else{
pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT);
+ assert( sqlite3PcachePageSanity(p) );
}
}
}
@@ -51568,6 +52843,7 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){
pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno));
assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY );
pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD);
+ assert( sqlite3PcachePageSanity(p) );
}
assert( sqlite3PcachePageSanity(p) );
}
@@ -51630,14 +52906,24 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){
*/
SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){
PCache *pCache = p->pCache;
+ sqlite3_pcache_page *pOther;
assert( p->nRef>0 );
assert( newPgno>0 );
assert( sqlite3PcachePageSanity(p) );
pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno));
+ pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0);
+ if( pOther ){
+ PgHdr *pXPage = (PgHdr*)pOther->pExtra;
+ assert( pXPage->nRef==0 );
+ pXPage->nRef++;
+ pCache->nRefSum++;
+ sqlite3PcacheDrop(pXPage);
+ }
sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno);
p->pgno = newPgno;
if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){
pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT);
+ assert( sqlite3PcachePageSanity(p) );
}
}
@@ -51935,12 +53221,13 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd
** size can vary according to architecture, compile-time options, and
** SQLite library version number.
**
-** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained
-** using a separate memory allocation from the database page content. This
-** seeks to overcome the "clownshoe" problem (also called "internal
-** fragmentation" in academic literature) of allocating a few bytes more
-** than a power of two with the memory allocator rounding up to the next
-** power of two, and leaving the rounded-up space unused.
+** Historical note: It used to be that if the SQLITE_PCACHE_SEPARATE_HEADER
+** was defined, then the page content would be held in a separate memory
+** allocation from the PgHdr1. This was intended to avoid clownshoe memory
+** allocations. However, the btree layer needs a small (16-byte) overrun
+** area after the page content buffer. The header serves as that overrun
+** area. Therefore SQLITE_PCACHE_SEPARATE_HEADER was discontinued to avoid
+** any possibility of a memory error.
**
** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates
** with this module. Information is passed back and forth as PgHdr1 pointers.
@@ -51985,30 +53272,40 @@ typedef struct PGroup PGroup;
/*
** Each cache entry is represented by an instance of the following
-** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of
-** PgHdr1.pCache->szPage bytes is allocated directly before this structure
-** in memory.
+** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated
+** directly before this structure and is used to cache the page content.
**
-** Note: Variables isBulkLocal and isAnchor were once type "u8". That works,
+** When reading a corrupt database file, it is possible that SQLite might
+** read a few bytes (no more than 16 bytes) past the end of the page buffer.
+** It will only read past the end of the page buffer, never write. This
+** object is positioned immediately after the page buffer to serve as an
+** overrun area, so that overreads are harmless.
+**
+** Variables isBulkLocal and isAnchor were once type "u8". That works,
** but causes a 2-byte gap in the structure for most architectures (since
** pointers must be either 4 or 8-byte aligned). As this structure is located
** in memory directly after the associated page data, if the database is
** corrupt, code at the b-tree layer may overread the page buffer and
** read part of this structure before the corruption is detected. This
** can cause a valgrind error if the unitialized gap is accessed. Using u16
-** ensures there is no such gap, and therefore no bytes of unitialized memory
-** in the structure.
+** ensures there is no such gap, and therefore no bytes of uninitialized
+** memory in the structure.
+**
+** The pLruNext and pLruPrev pointers form a double-linked circular list
+** of all pages that are unpinned. The PGroup.lru element (which should be
+** the only element on the list with PgHdr1.isAnchor set to 1) forms the
+** beginning and the end of the list.
*/
struct PgHdr1 {
- sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */
- unsigned int iKey; /* Key value (page number) */
- u16 isBulkLocal; /* This page from bulk local storage */
- u16 isAnchor; /* This is the PGroup.lru element */
- PgHdr1 *pNext; /* Next in hash table chain */
- PCache1 *pCache; /* Cache that currently owns this page */
- PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */
- PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */
- /* NB: pLruPrev is only valid if pLruNext!=0 */
+ sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */
+ unsigned int iKey; /* Key value (page number) */
+ u16 isBulkLocal; /* This page from bulk local storage */
+ u16 isAnchor; /* This is the PGroup.lru element */
+ PgHdr1 *pNext; /* Next in hash table chain */
+ PCache1 *pCache; /* Cache that currently owns this page */
+ PgHdr1 *pLruNext; /* Next in circular LRU list of unpinned pages */
+ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */
+ /* NB: pLruPrev is only valid if pLruNext!=0 */
};
/*
@@ -52334,25 +53631,13 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){
pcache1LeaveMutex(pCache->pGroup);
#endif
if( benignMalloc ){ sqlite3BeginBenignMalloc(); }
-#ifdef SQLITE_PCACHE_SEPARATE_HEADER
- pPg = pcache1Alloc(pCache->szPage);
- p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra);
- if( !pPg || !p ){
- pcache1Free(pPg);
- sqlite3_free(p);
- pPg = 0;
- }
-#else
pPg = pcache1Alloc(pCache->szAlloc);
-#endif
if( benignMalloc ){ sqlite3EndBenignMalloc(); }
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
pcache1EnterMutex(pCache->pGroup);
#endif
if( pPg==0 ) return 0;
-#ifndef SQLITE_PCACHE_SEPARATE_HEADER
p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage];
-#endif
p->page.pBuf = pPg;
p->page.pExtra = &p[1];
p->isBulkLocal = 0;
@@ -52376,9 +53661,6 @@ static void pcache1FreePage(PgHdr1 *p){
pCache->pFree = p;
}else{
pcache1Free(p->page.pBuf);
-#ifdef SQLITE_PCACHE_SEPARATE_HEADER
- sqlite3_free(p);
-#endif
}
(*pCache->pnPurgeable)--;
}
@@ -53019,23 +54301,26 @@ static void pcache1Rekey(
PCache1 *pCache = (PCache1 *)p;
PgHdr1 *pPage = (PgHdr1 *)pPg;
PgHdr1 **pp;
- unsigned int h;
+ unsigned int hOld, hNew;
assert( pPage->iKey==iOld );
assert( pPage->pCache==pCache );
+ assert( iOld!=iNew ); /* The page number really is changing */
pcache1EnterMutex(pCache->pGroup);
- h = iOld%pCache->nHash;
- pp = &pCache->apHash[h];
+ assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */
+ hOld = iOld%pCache->nHash;
+ pp = &pCache->apHash[hOld];
while( (*pp)!=pPage ){
pp = &(*pp)->pNext;
}
*pp = pPage->pNext;
- h = iNew%pCache->nHash;
+ assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */
+ hNew = iNew%pCache->nHash;
pPage->iKey = iNew;
- pPage->pNext = pCache->apHash[h];
- pCache->apHash[h] = pPage;
+ pPage->pNext = pCache->apHash[hNew];
+ pCache->apHash[hNew] = pPage;
if( iNew>pCache->iMaxKey ){
pCache->iMaxKey = iNew;
}
@@ -53142,9 +54427,6 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){
&& p->isAnchor==0
){
nFree += pcache1MemSize(p->page.pBuf);
-#ifdef SQLITE_PCACHE_SEPARATE_HEADER
- nFree += sqlite3MemSize(p);
-#endif
assert( PAGE_IS_UNPINNED(p) );
pcache1PinPage(p);
pcache1RemoveFromHash(p, 1);
@@ -59633,6 +60915,7 @@ static int pager_open_journal(Pager *pPager){
if( pPager->tempFile ){
flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL);
+ flags |= SQLITE_OPEN_EXCLUSIVE;
nSpill = sqlite3Config.nStmtSpill;
}else{
flags |= SQLITE_OPEN_MAIN_JOURNAL;
@@ -59668,6 +60951,7 @@ static int pager_open_journal(Pager *pPager){
if( rc!=SQLITE_OK ){
sqlite3BitvecDestroy(pPager->pInJournal);
pPager->pInJournal = 0;
+ pPager->journalOff = 0;
}else{
assert( pPager->eState==PAGER_WRITER_LOCKED );
pPager->eState = PAGER_WRITER_CACHEMOD;
@@ -66731,6 +68015,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){
SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){
Btree *p;
assert( db!=0 );
+ if( db->pVfs==0 && db->nDb==0 ) return 1;
if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema);
assert( iDb>=0 && iDbnDb );
if( !sqlite3_mutex_held(db->mutex) ) return 0;
@@ -68303,8 +69588,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE );
assert( pPage->nOverflow==0 );
assert( sqlite3_mutex_held(pPage->pBt->mutex) );
- temp = 0;
- src = data = pPage->aData;
+ data = pPage->aData;
hdr = pPage->hdrOffset;
cellOffset = pPage->cellOffset;
nCell = pPage->nCell;
@@ -68338,7 +69622,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage);
memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz));
sz += sz2;
- }else if( NEVER(iFree+sz>usableSize) ){
+ }else if( iFree+sz>usableSize ){
return SQLITE_CORRUPT_PAGE(pPage);
}
@@ -68358,39 +69642,38 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
cbrk = usableSize;
iCellLast = usableSize - 4;
iCellStart = get2byte(&data[hdr+5]);
- for(i=0; iiCellLast ){
- return SQLITE_CORRUPT_PAGE(pPage);
+ if( nCell>0 ){
+ temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
+ memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart);
+ src = temp;
+ for(i=0; iiCellLast ){
+ return SQLITE_CORRUPT_PAGE(pPage);
+ }
+ assert( pc>=iCellStart && pc<=iCellLast );
+ size = pPage->xCellSize(pPage, &src[pc]);
+ cbrk -= size;
+ if( cbrkusableSize ){
+ return SQLITE_CORRUPT_PAGE(pPage);
+ }
+ assert( cbrk+size<=usableSize && cbrk>=iCellStart );
+ testcase( cbrk+size==usableSize );
+ testcase( pc+size==usableSize );
+ put2byte(pAddr, cbrk);
+ memcpy(&data[cbrk], &src[pc], size);
}
- assert( pc>=iCellStart && pc<=iCellLast );
- size = pPage->xCellSize(pPage, &src[pc]);
- cbrk -= size;
- if( cbrkusableSize ){
- return SQLITE_CORRUPT_PAGE(pPage);
- }
- assert( cbrk+size<=usableSize && cbrk>=iCellStart );
- testcase( cbrk+size==usableSize );
- testcase( pc+size==usableSize );
- put2byte(pAddr, cbrk);
- if( temp==0 ){
- if( cbrk==pc ) continue;
- temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
- memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart);
- src = temp;
- }
- memcpy(&data[cbrk], &src[pc], size);
}
data[hdr+7] = 0;
- defragment_out:
+defragment_out:
assert( pPage->nFree>=0 );
if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){
return SQLITE_CORRUPT_PAGE(pPage);
@@ -68447,7 +69730,6 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
** fragmented bytes within the page. */
memcpy(&aData[iAddr], &aData[pc], 2);
aData[hdr+7] += (u8)x;
- testcase( pc+x>maxPC );
return &aData[pc];
}else if( x+pc > maxPC ){
/* This slot extends off the end of the usable part of the page */
@@ -68463,9 +69745,9 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
iAddr = pc;
pTmp = &aData[pc];
pc = get2byte(pTmp);
- if( pc<=iAddr+size ){
+ if( pc<=iAddr ){
if( pc ){
- /* The next slot in the chain is not past the end of the current slot */
+ /* The next slot in the chain comes before the current slot */
*pRc = SQLITE_CORRUPT_PAGE(pPg);
}
return 0;
@@ -68617,7 +69899,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */
}else{
while( (iFreeBlk = get2byte(&data[iPtr]))pPage = pCur->apPage[pCur->iPage];
}
testcase( pgno==0 );
- assert( pgno!=0 || rc==SQLITE_CORRUPT
- || rc==SQLITE_IOERR_NOMEM
- || rc==SQLITE_NOMEM );
+ assert( pgno!=0 || rc!=SQLITE_OK );
return rc;
}
@@ -70537,6 +71817,9 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
}
}
}else{
+ if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){
+ return SQLITE_CORRUPT_PAGE(pPage);
+ }
if( get4byte(pCell)==iFrom ){
put4byte(pCell, iTo);
break;
@@ -72043,8 +73326,6 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
** vice-versa).
*/
static int moveToChild(BtCursor *pCur, u32 newPgno){
- BtShared *pBt = pCur->pBt;
-
assert( cursorOwnsBtShared(pCur) );
assert( pCur->eState==CURSOR_VALID );
assert( pCur->iPageapPage[pCur->iPage] = pCur->pPage;
pCur->ix = 0;
pCur->iPage++;
- return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags);
+ return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur,
+ pCur->curPagerFlags);
}
#ifdef SQLITE_DEBUG
@@ -72164,7 +73446,7 @@ static int moveToRoot(BtCursor *pCur){
}
sqlite3BtreeClearCursor(pCur);
}
- rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage,
+ rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage,
0, pCur->curPagerFlags);
if( rc!=SQLITE_OK ){
pCur->eState = CURSOR_INVALID;
@@ -72872,14 +74154,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){
pPage = pCur->pPage;
idx = ++pCur->ix;
- if( !pPage->isInit || sqlite3FaultSim(412) ){
- /* The only known way for this to happen is for there to be a
- ** recursive SQL function that does a DELETE operation as part of a
- ** SELECT which deletes content out from under an active cursor
- ** in a corrupt database file where the table being DELETE-ed from
- ** has pages in common with the table being queried. See TH3
- ** module cov1/btree78.test testcase 220 (2018-06-08) for an
- ** example. */
+ if( NEVER(!pPage->isInit) || sqlite3FaultSim(412) ){
return SQLITE_CORRUPT_BKPT;
}
@@ -73055,8 +74330,8 @@ static int allocateBtreePage(
assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) );
pPage1 = pBt->pPage1;
mxPage = btreePagecount(pBt);
- /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
- ** stores stores the total number of pages on the freelist. */
+ /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36
+ ** stores the total number of pages on the freelist. */
n = get4byte(&pPage1->aData[36]);
testcase( n==mxPage-1 );
if( n>=mxPage ){
@@ -73805,12 +75080,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){
assert( pPage->pBt->usableSize > (u32)(ptr-data) );
pc = get2byte(ptr);
hdr = pPage->hdrOffset;
-#if 0 /* Not required. Omit for efficiency */
- if( pcnCell*2 ){
- *pRC = SQLITE_CORRUPT_BKPT;
- return;
- }
-#endif
testcase( pc==(u32)get2byte(&data[hdr+5]) );
testcase( pc+sz==pPage->pBt->usableSize );
if( pc+sz > pPage->pBt->usableSize ){
@@ -74694,8 +75963,6 @@ static int balance_nonroot(
Pgno pgno; /* Temp var to store a page number in */
u8 abDone[NB+2]; /* True after i'th new page is populated */
Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */
- Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */
- u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */
CellArray b; /* Parsed information on cells being balanced */
memset(abDone, 0, sizeof(abDone));
@@ -75119,42 +76386,39 @@ static int balance_nonroot(
** of the table is closer to a linear scan through the file. That in turn
** helps the operating system to deliver pages from the disk more rapidly.
**
- ** An O(n^2) insertion sort algorithm is used, but since n is never more
- ** than (NB+2) (a small constant), that should not be a problem.
+ ** An O(N*N) sort algorithm is used, but since N is never more than NB+2
+ ** (5), that is not a performance concern.
**
** When NB==3, this one optimization makes the database about 25% faster
** for large insertions and deletions.
*/
for(i=0; ipgno;
- aPgFlags[i] = apNew[i]->pDbPage->flags;
- for(j=0; jpgno;
+ assert( apNew[i]->pDbPage->flags & PGHDR_WRITEABLE );
+ assert( apNew[i]->pDbPage->flags & PGHDR_DIRTY );
}
- for(i=0; ipgno < apNew[iB]->pgno ) iB = j;
}
- pgno = aPgOrder[iBest];
- aPgOrder[iBest] = 0xffffffff;
- if( iBest!=i ){
- if( iBest>i ){
- sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0);
- }
- sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]);
- apNew[i]->pgno = pgno;
+
+ /* If apNew[i] has a page number that is bigger than any of the
+ ** subsequence apNew[i] entries, then swap apNew[i] with the subsequent
+ ** entry that has the smallest page number (which we know to be
+ ** entry apNew[iB]).
+ */
+ if( iB!=i ){
+ Pgno pgnoA = apNew[i]->pgno;
+ Pgno pgnoB = apNew[iB]->pgno;
+ Pgno pgnoTemp = (PENDING_BYTE/pBt->pageSize)+1;
+ u16 fgA = apNew[i]->pDbPage->flags;
+ u16 fgB = apNew[iB]->pDbPage->flags;
+ sqlite3PagerRekey(apNew[i]->pDbPage, pgnoTemp, fgB);
+ sqlite3PagerRekey(apNew[iB]->pDbPage, pgnoA, fgA);
+ sqlite3PagerRekey(apNew[i]->pDbPage, pgnoB, fgB);
+ apNew[i]->pgno = pgnoB;
+ apNew[iB]->pgno = pgnoA;
}
}
@@ -75580,6 +76844,11 @@ static int balance(BtCursor *pCur){
}else{
break;
}
+ }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){
+ /* The page being written is not a root page, and there is currently
+ ** more than one reference to it. This only happens if the page is one
+ ** of its own ancestor pages. Corruption. */
+ rc = SQLITE_CORRUPT_BKPT;
}else{
MemPage * const pParent = pCur->apPage[iPage-1];
int const iIdx = pCur->aiIdx[iPage-1];
@@ -79423,6 +80692,16 @@ SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){
&& i >= -2251799813685248LL && i < 2251799813685248LL);
}
+/* Convert a floating point value to its closest integer. Do so in
+** a way that avoids 'outside the range of representable values' warnings
+** from UBSAN.
+*/
+SQLITE_PRIVATE i64 sqlite3RealToI64(double r){
+ if( r<=(double)SMALLEST_INT64 ) return SMALLEST_INT64;
+ if( r>=(double)LARGEST_INT64) return LARGEST_INT64;
+ return (i64)r;
+}
+
/*
** Convert pMem so that it has type MEM_Real or MEM_Int.
** Invalidate any prior representations.
@@ -79444,7 +80723,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){
assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc);
if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1)
- || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r))
+ || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r)))
){
pMem->u.i = ix;
MemSetTypeFlag(pMem, MEM_Int);
@@ -79496,6 +80775,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){
sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding);
assert( pMem->flags & MEM_Str || pMem->db->mallocFailed );
pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero);
+ if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1;
return sqlite3VdbeChangeEncoding(pMem, encoding);
}
}
@@ -80631,6 +81911,9 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){
if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){
return p->n;
}
+ if( (p->flags & MEM_Str)!=0 && enc!=SQLITE_UTF8 && pVal->enc!=SQLITE_UTF8 ){
+ return p->n;
+ }
if( (p->flags & MEM_Blob)!=0 ){
if( p->flags & MEM_Zero ){
return p->n + p->u.nZero;
@@ -80676,10 +81959,10 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){
memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp));
p->db = db;
if( db->pVdbe ){
- db->pVdbe->pPrev = p;
+ db->pVdbe->ppVPrev = &p->pVNext;
}
- p->pNext = db->pVdbe;
- p->pPrev = 0;
+ p->pVNext = db->pVdbe;
+ p->ppVPrev = &db->pVdbe;
db->pVdbe = p;
assert( p->eVdbeState==VDBE_INIT_STATE );
p->pParse = pParse;
@@ -80761,21 +82044,28 @@ SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString(
#endif
/*
-** Swap all content between two VDBE structures.
+** Swap byte-code between two VDBE structures.
+**
+** This happens after pB was previously run and returned
+** SQLITE_SCHEMA. The statement was then reprepared in pA.
+** This routine transfers the new bytecode in pA over to pB
+** so that pB can be run again. The old pB byte code is
+** moved back to pA so that it will be cleaned up when pA is
+** finalized.
*/
SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
- Vdbe tmp, *pTmp;
+ Vdbe tmp, *pTmp, **ppTmp;
char *zTmp;
assert( pA->db==pB->db );
tmp = *pA;
*pA = *pB;
*pB = tmp;
- pTmp = pA->pNext;
- pA->pNext = pB->pNext;
- pB->pNext = pTmp;
- pTmp = pA->pPrev;
- pA->pPrev = pB->pPrev;
- pB->pPrev = pTmp;
+ pTmp = pA->pVNext;
+ pA->pVNext = pB->pVNext;
+ pB->pVNext = pTmp;
+ ppTmp = pA->ppVPrev;
+ pA->ppVPrev = pB->ppVPrev;
+ pB->ppVPrev = ppTmp;
zTmp = pA->zSql;
pA->zSql = pB->zSql;
pB->zSql = zTmp;
@@ -81027,6 +82317,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall(
addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function,
p1, p2, p3, (char*)pCtx, P4_FUNCCTX);
sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef);
+ sqlite3MayAbort(pParse);
return addr;
}
@@ -81095,7 +82386,7 @@ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt
iThis = v->nOp;
sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0,
zMsg, P4_DYNAMIC);
- sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z);
+ sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z);
if( bPush){
pParse->addrExplain = iThis;
}
@@ -81362,6 +82653,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
|| opcode==OP_VDestroy
|| opcode==OP_VCreate
|| opcode==OP_ParseSchema
+ || opcode==OP_Function || opcode==OP_PureFunc
|| ((opcode==OP_Halt || opcode==OP_HaltIfNull)
&& ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort))
){
@@ -81452,8 +82744,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
p->readOnly = 1;
p->bIsReader = 0;
pOp = &p->aOp[p->nOp-1];
- while(1){
-
+ assert( p->aOp[0].opcode==OP_Init );
+ while( 1 /* Loop termates when it reaches the OP_Init opcode */ ){
/* Only JUMP opcodes and the short list of special opcodes in the switch
** below need to be considered. The mkopcodeh.tcl generator script groups
** all these opcodes together near the front of the opcode list. Skip
@@ -81482,6 +82774,10 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
p->bIsReader = 1;
break;
}
+ case OP_Init: {
+ assert( pOp->p2>=0 );
+ goto resolve_p2_values_loop_exit;
+ }
#ifndef SQLITE_OMIT_VIRTUALTABLE
case OP_VUpdate: {
if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
@@ -81514,11 +82810,12 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
** have non-negative values for P2. */
assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0);
}
- if( pOp==p->aOp ) break;
+ assert( pOp>p->aOp );
pOp--;
}
+resolve_p2_values_loop_exit:
if( aLabel ){
- sqlite3DbFreeNN(p->db, pParse->aLabel);
+ sqlite3DbNNFreeNN(p->db, pParse->aLabel);
pParse->aLabel = 0;
}
pParse->nLabel = 0;
@@ -81767,15 +83064,19 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(
** for a specific instruction.
*/
SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){
+ assert( addr>=0 );
sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode;
}
SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){
+ assert( addr>=0 );
sqlite3VdbeGetOp(p,addr)->p1 = val;
}
SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){
+ assert( addr>=0 || p->db->mallocFailed );
sqlite3VdbeGetOp(p,addr)->p2 = val;
}
SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){
+ assert( addr>=0 );
sqlite3VdbeGetOp(p,addr)->p3 = val;
}
SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){
@@ -81783,6 +83084,18 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){
if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5;
}
+/*
+** If the previous opcode is an OP_Column that delivers results
+** into register iDest, then add the OPFLAG_TYPEOFARG flag to that
+** opcode.
+*/
+SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){
+ VdbeOp *pOp = sqlite3VdbeGetLastOp(p);
+ if( pOp->p3==iDest && pOp->opcode==OP_Column ){
+ pOp->p5 |= OPFLAG_TYPEOFARG;
+ }
+}
+
/*
** Change the P2 operand of instruction addr so that it points to
** the address of the next instruction to be coded.
@@ -81811,7 +83124,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){
|| p->aOp[addr].opcode==OP_FkIfZero );
assert( p->aOp[addr].p4type==0 );
#ifdef SQLITE_VDBE_COVERAGE
- sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */
+ sqlite3VdbeGetLastOp(p)->iSrcLine = 0; /* Erase VdbeCoverage() macros */
#endif
p->nOp--;
}else{
@@ -81825,8 +83138,9 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){
** the FuncDef is not ephermal, then do nothing.
*/
static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
+ assert( db!=0 );
if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
- sqlite3DbFreeNN(db, pDef);
+ sqlite3DbNNFreeNN(db, pDef);
}
}
@@ -81835,11 +83149,12 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
*/
static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){
if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
- sqlite3DbFreeNN(db, p);
+ sqlite3DbNNFreeNN(db, p);
}
static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){
+ assert( db!=0 );
freeEphemeralFunction(db, p->pFunc);
- sqlite3DbFreeNN(db, p);
+ sqlite3DbNNFreeNN(db, p);
}
static void freeP4(sqlite3 *db, int p4type, void *p4){
assert( db );
@@ -81852,7 +83167,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
case P4_INT64:
case P4_DYNAMIC:
case P4_INTARRAY: {
- sqlite3DbFree(db, p4);
+ if( p4 ) sqlite3DbNNFreeNN(db, p4);
break;
}
case P4_KEYINFO: {
@@ -81891,6 +83206,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
*/
static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){
assert( nOp>=0 );
+ assert( db!=0 );
if( aOp ){
Op *pOp = &aOp[nOp-1];
while(1){ /* Exit via break */
@@ -81901,7 +83217,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){
if( pOp==aOp ) break;
pOp--;
}
- sqlite3DbFreeNN(db, aOp);
+ sqlite3DbNNFreeNN(db, aOp);
}
}
@@ -82132,13 +83448,13 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){
** Set the value if the iSrcLine field for the previously coded instruction.
*/
SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){
- sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine;
+ sqlite3VdbeGetLastOp(v)->iSrcLine = iLine;
}
#endif /* SQLITE_VDBE_COVERAGE */
/*
-** Return the opcode for a given address. If the address is -1, then
-** return the most recently inserted opcode.
+** Return the opcode for a given address. The address must be non-negative.
+** See sqlite3VdbeGetLastOp() to get the most recently added opcode.
**
** If a memory allocation error has occurred prior to the calling of this
** routine, then a pointer to a dummy VdbeOp will be returned. That opcode
@@ -82154,9 +83470,6 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){
** zeros, which is correct. MSVC generates a warning, nevertheless. */
static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */
assert( p->eVdbeState==VDBE_INIT_STATE );
- if( addr<0 ){
- addr = p->nOp - 1;
- }
assert( (addr>=0 && addrnOp) || p->db->mallocFailed );
if( p->db->mallocFailed ){
return (VdbeOp*)&dummy;
@@ -82165,6 +83478,12 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){
}
}
+/* Return the most recently added opcode
+*/
+VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){
+ return sqlite3VdbeGetOp(p, p->nOp - 1);
+}
+
#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS)
/*
** Return an integer value for one of the parameters to the opcode pOp
@@ -82652,7 +83971,7 @@ static void releaseMemArray(Mem *p, int N){
sqlite3VdbeMemRelease(p);
p->flags = MEM_Undefined;
}else if( p->szMalloc ){
- sqlite3DbFreeNN(db, p->zMalloc);
+ sqlite3DbNNFreeNN(db, p->zMalloc);
p->szMalloc = 0;
p->flags = MEM_Undefined;
}
@@ -83644,7 +84963,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){
if( p->readOnly==0 ) nWrite++;
if( p->bIsReader ) nRead++;
}
- p = p->pNext;
+ p = p->pVNext;
}
assert( cnt==db->nVdbeActive );
assert( nWrite==db->nVdbeWrite );
@@ -84173,10 +85492,11 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp,
*/
static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
SubProgram *pSub, *pNext;
+ assert( db!=0 );
assert( p->db==0 || p->db==db );
if( p->aColName ){
releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
- sqlite3DbFreeNN(db, p->aColName);
+ sqlite3DbNNFreeNN(db, p->aColName);
}
for(pSub=p->pProgram; pSub; pSub=pNext){
pNext = pSub->pNext;
@@ -84185,11 +85505,11 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
}
if( p->eVdbeState!=VDBE_INIT_STATE ){
releaseMemArray(p->aVar, p->nVar);
- if( p->pVList ) sqlite3DbFreeNN(db, p->pVList);
- if( p->pFree ) sqlite3DbFreeNN(db, p->pFree);
+ if( p->pVList ) sqlite3DbNNFreeNN(db, p->pVList);
+ if( p->pFree ) sqlite3DbNNFreeNN(db, p->pFree);
}
vdbeFreeOpArray(db, p->aOp, p->nOp);
- sqlite3DbFree(db, p->zSql);
+ if( p->zSql ) sqlite3DbNNFreeNN(db, p->zSql);
#ifdef SQLITE_ENABLE_NORMALIZE
sqlite3DbFree(db, p->zNormSql);
{
@@ -84219,20 +85539,17 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){
assert( p!=0 );
db = p->db;
+ assert( db!=0 );
assert( sqlite3_mutex_held(db->mutex) );
sqlite3VdbeClearObject(db, p);
if( db->pnBytesFreed==0 ){
- if( p->pPrev ){
- p->pPrev->pNext = p->pNext;
- }else{
- assert( db->pVdbe==p );
- db->pVdbe = p->pNext;
- }
- if( p->pNext ){
- p->pNext->pPrev = p->pPrev;
+ assert( p->ppVPrev!=0 );
+ *p->ppVPrev = p->pVNext;
+ if( p->pVNext ){
+ p->pVNext->ppVPrev = p->ppVPrev;
}
}
- sqlite3DbFreeNN(db, p);
+ sqlite3DbNNFreeNN(db, p);
}
/*
@@ -85187,7 +86504,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
assert( pPKey2->pKeyInfo->aSortFlags!=0 );
assert( pPKey2->pKeyInfo->nKeyField>0 );
assert( idx1<=szHdr1 || CORRUPT_DB );
- do{
+ while( 1 /*exit-by-break*/ ){
u32 serial_type;
/* RHS is an integer */
@@ -85197,7 +86514,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
serial_type = aKey1[idx1];
testcase( serial_type==12 );
if( serial_type>=10 ){
- rc = +1;
+ rc = serial_type==10 ? -1 : +1;
}else if( serial_type==0 ){
rc = -1;
}else if( serial_type==7 ){
@@ -85222,7 +86539,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
** numbers). Types 10 and 11 are currently "reserved for future
** use", so it doesn't really matter what the results of comparing
** them to numberic values are. */
- rc = +1;
+ rc = serial_type==10 ? -1 : +1;
}else if( serial_type==0 ){
rc = -1;
}else{
@@ -85303,7 +86620,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
/* RHS is null */
else{
serial_type = aKey1[idx1];
- rc = (serial_type!=0);
+ rc = (serial_type!=0 && serial_type!=10);
}
if( rc!=0 ){
@@ -85325,8 +86642,13 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
if( i==pPKey2->nField ) break;
pRhs++;
d1 += sqlite3VdbeSerialTypeLen(serial_type);
+ if( d1>(unsigned)nKey1 ) break;
idx1 += sqlite3VarintLen(serial_type);
- }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 );
+ if( idx1>=(unsigned)szHdr1 ){
+ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT;
+ return 0; /* Corrupt index */
+ }
+ }
/* No memory allocation is ever used on mem1. Prove this using
** the following assert(). If the assert() fails, it indicates a
@@ -85727,7 +87049,7 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){
*/
SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){
Vdbe *p;
- for(p = db->pVdbe; p; p=p->pNext){
+ for(p = db->pVdbe; p; p=p->pVNext){
p->expired = iCode+1;
}
}
@@ -85848,13 +87170,14 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){
** the vdbeUnpackRecord() function found in vdbeapi.c.
*/
static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){
+ assert( db!=0 );
if( p ){
int i;
for(i=0; iaMem[i];
if( pMem->zMalloc ) sqlite3VdbeMemReleaseMalloc(pMem);
}
- sqlite3DbFreeNN(db, p);
+ sqlite3DbNNFreeNN(db, p);
}
}
#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
@@ -85925,7 +87248,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(
for(i=0; inField; i++){
sqlite3VdbeMemRelease(&preupdate.aNew[i]);
}
- sqlite3DbFreeNN(db, preupdate.aNew);
+ sqlite3DbNNFreeNN(db, preupdate.aNew);
}
}
#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
@@ -86042,7 +87365,9 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){
if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT;
sqlite3_mutex_enter(db->mutex);
checkProfileCallback(db, v);
- rc = sqlite3VdbeFinalize(v);
+ assert( v->eVdbeState>=VDBE_READY_STATE );
+ rc = sqlite3VdbeReset(v);
+ sqlite3VdbeDelete(v);
rc = sqlite3ApiExit(db, rc);
sqlite3LeaveMutexAndCloseZombie(db);
}
@@ -86250,6 +87575,9 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){
#endif
return aType[pVal->flags&MEM_AffMask];
}
+SQLITE_API int sqlite3_value_encoding(sqlite3_value *pVal){
+ return pVal->enc;
+}
/* Return true if a parameter to xUpdate represents an unchanged column */
SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){
@@ -87364,7 +88692,7 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){
** The error code stored in database p->db is overwritten with the return
** value in any case.
*/
-static int vdbeUnbind(Vdbe *p, int i){
+static int vdbeUnbind(Vdbe *p, unsigned int i){
Mem *pVar;
if( vdbeSafetyNotNull(p) ){
return SQLITE_MISUSE_BKPT;
@@ -87377,12 +88705,11 @@ static int vdbeUnbind(Vdbe *p, int i){
"bind on a busy prepared statement: [%s]", p->zSql);
return SQLITE_MISUSE_BKPT;
}
- if( i<1 || i>p->nVar ){
+ if( i>=(unsigned int)p->nVar ){
sqlite3Error(p->db, SQLITE_RANGE);
sqlite3_mutex_leave(p->db->mutex);
return SQLITE_RANGE;
}
- i--;
pVar = &p->aVar[i];
sqlite3VdbeMemRelease(pVar);
pVar->flags = MEM_Null;
@@ -87419,7 +88746,7 @@ static int bindText(
Mem *pVar;
int rc;
- rc = vdbeUnbind(p, i);
+ rc = vdbeUnbind(p, (u32)(i-1));
if( rc==SQLITE_OK ){
if( zData!=0 ){
pVar = &p->aVar[i-1];
@@ -87468,7 +88795,7 @@ SQLITE_API int sqlite3_bind_blob64(
SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){
int rc;
Vdbe *p = (Vdbe *)pStmt;
- rc = vdbeUnbind(p, i);
+ rc = vdbeUnbind(p, (u32)(i-1));
if( rc==SQLITE_OK ){
sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue);
sqlite3_mutex_leave(p->db->mutex);
@@ -87481,7 +88808,7 @@ SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){
SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){
int rc;
Vdbe *p = (Vdbe *)pStmt;
- rc = vdbeUnbind(p, i);
+ rc = vdbeUnbind(p, (u32)(i-1));
if( rc==SQLITE_OK ){
sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue);
sqlite3_mutex_leave(p->db->mutex);
@@ -87491,7 +88818,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu
SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){
int rc;
Vdbe *p = (Vdbe*)pStmt;
- rc = vdbeUnbind(p, i);
+ rc = vdbeUnbind(p, (u32)(i-1));
if( rc==SQLITE_OK ){
sqlite3_mutex_leave(p->db->mutex);
}
@@ -87506,7 +88833,7 @@ SQLITE_API int sqlite3_bind_pointer(
){
int rc;
Vdbe *p = (Vdbe*)pStmt;
- rc = vdbeUnbind(p, i);
+ rc = vdbeUnbind(p, (u32)(i-1));
if( rc==SQLITE_OK ){
sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor);
sqlite3_mutex_leave(p->db->mutex);
@@ -87584,7 +88911,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu
SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){
int rc;
Vdbe *p = (Vdbe *)pStmt;
- rc = vdbeUnbind(p, i);
+ rc = vdbeUnbind(p, (u32)(i-1));
if( rc==SQLITE_OK ){
#ifndef SQLITE_OMIT_INCRBLOB
sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n);
@@ -87744,7 +89071,7 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){
if( pStmt==0 ){
pNext = (sqlite3_stmt*)pDb->pVdbe;
}else{
- pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext;
+ pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pVNext;
}
sqlite3_mutex_leave(pDb->mutex);
return pNext;
@@ -87769,8 +89096,11 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){
sqlite3_mutex_enter(db->mutex);
v = 0;
db->pnBytesFreed = (int*)&v;
+ assert( db->lookaside.pEnd==db->lookaside.pTrueEnd );
+ db->lookaside.pEnd = db->lookaside.pStart;
sqlite3VdbeDelete(pVdbe);
db->pnBytesFreed = 0;
+ db->lookaside.pEnd = db->lookaside.pTrueEnd;
sqlite3_mutex_leave(db->mutex);
}else{
v = pVdbe->aCounter[op];
@@ -88610,7 +89940,8 @@ static VdbeCursor *allocateCursor(
** return false.
*/
static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){
- i64 iValue = (double)rValue;
+ i64 iValue;
+ iValue = sqlite3RealToI64(rValue);
if( sqlite3RealSameAsInt(rValue,iValue) ){
*piValue = iValue;
return 1;
@@ -88772,17 +90103,18 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){
** But it does set pMem->u.r and pMem->u.i appropriately.
*/
static u16 numericType(Mem *pMem){
- if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){
+ assert( (pMem->flags & MEM_Null)==0
+ || pMem->db==0 || pMem->db->mallocFailed );
+ if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null) ){
testcase( pMem->flags & MEM_Int );
testcase( pMem->flags & MEM_Real );
testcase( pMem->flags & MEM_IntReal );
- return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal);
- }
- if( pMem->flags & (MEM_Str|MEM_Blob) ){
- testcase( pMem->flags & MEM_Str );
- testcase( pMem->flags & MEM_Blob );
- return computeNumericType(pMem);
+ return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null);
}
+ assert( pMem->flags & (MEM_Str|MEM_Blob) );
+ testcase( pMem->flags & MEM_Str );
+ testcase( pMem->flags & MEM_Blob );
+ return computeNumericType(pMem);
return 0;
}
@@ -90027,7 +91359,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */
case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */
case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */
case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */
- u16 flags; /* Combined MEM_* flags from both inputs */
u16 type1; /* Numeric type of left operand */
u16 type2; /* Numeric type of right operand */
i64 iA; /* Integer value of left operand */
@@ -90036,12 +91367,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */
double rB; /* Real value of right operand */
pIn1 = &aMem[pOp->p1];
- type1 = numericType(pIn1);
+ type1 = pIn1->flags;
pIn2 = &aMem[pOp->p2];
- type2 = numericType(pIn2);
+ type2 = pIn2->flags;
pOut = &aMem[pOp->p3];
- flags = pIn1->flags | pIn2->flags;
if( (type1 & type2 & MEM_Int)!=0 ){
+int_math:
iA = pIn1->u.i;
iB = pIn2->u.i;
switch( pOp->opcode ){
@@ -90063,9 +91394,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */
}
pOut->u.i = iB;
MemSetTypeFlag(pOut, MEM_Int);
- }else if( (flags & MEM_Null)!=0 ){
+ }else if( ((type1 | type2) & MEM_Null)!=0 ){
goto arithmetic_result_is_null;
}else{
+ type1 = numericType(pIn1);
+ type2 = numericType(pIn2);
+ if( (type1 & type2 & MEM_Int)!=0 ) goto int_math;
fp_math:
rA = sqlite3VdbeRealValue(pIn1);
rB = sqlite3VdbeRealValue(pIn2);
@@ -90880,19 +92214,90 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */
break;
}
-/* Opcode: IsNullOrType P1 P2 P3 * *
-** Synopsis: if typeof(r[P1]) IN (P3,5) goto P2
+/* Opcode: IsType P1 P2 P3 P4 P5
+** Synopsis: if typeof(P1.P3) in P5 goto P2
+**
+** Jump to P2 if the type of a column in a btree is one of the types specified
+** by the P5 bitmask.
+**
+** P1 is normally a cursor on a btree for which the row decode cache is
+** valid through at least column P3. In other words, there should have been
+** a prior OP_Column for column P3 or greater. If the cursor is not valid,
+** then this opcode might give spurious results.
+** The the btree row has fewer than P3 columns, then use P4 as the
+** datatype.
+**
+** If P1 is -1, then P3 is a register number and the datatype is taken
+** from the value in that register.
+**
+** P5 is a bitmask of data types. SQLITE_INTEGER is the least significant
+** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04.
+** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10.
+**
+** Take the jump to address P2 if and only if the datatype of the
+** value determined by P1 and P3 corresponds to one of the bits in the
+** P5 bitmask.
**
-** Jump to P2 if the value in register P1 is NULL or has a datatype P3.
-** P3 is an integer which should be one of SQLITE_INTEGER, SQLITE_FLOAT,
-** SQLITE_BLOB, SQLITE_NULL, or SQLITE_TEXT.
*/
-case OP_IsNullOrType: { /* jump, in1 */
- int doTheJump;
- pIn1 = &aMem[pOp->p1];
- doTheJump = (pIn1->flags & MEM_Null)!=0 || sqlite3_value_type(pIn1)==pOp->p3;
- VdbeBranchTaken( doTheJump, 2);
- if( doTheJump ) goto jump_to_p2;
+case OP_IsType: { /* jump */
+ VdbeCursor *pC;
+ u16 typeMask;
+ u32 serialType;
+
+ assert( pOp->p1>=(-1) && pOp->p1nCursor );
+ assert( pOp->p1>=0 || (pOp->p3>=0 && pOp->p3<=(p->nMem+1 - p->nCursor)) );
+ if( pOp->p1>=0 ){
+ pC = p->apCsr[pOp->p1];
+ assert( pC!=0 );
+ assert( pOp->p3>=0 );
+ if( pOp->p3nHdrParsed ){
+ serialType = pC->aType[pOp->p3];
+ if( serialType>=12 ){
+ if( serialType&1 ){
+ typeMask = 0x04; /* SQLITE_TEXT */
+ }else{
+ typeMask = 0x08; /* SQLITE_BLOB */
+ }
+ }else{
+ static const unsigned char aMask[] = {
+ 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x2,
+ 0x01, 0x01, 0x10, 0x10
+ };
+ testcase( serialType==0 );
+ testcase( serialType==1 );
+ testcase( serialType==2 );
+ testcase( serialType==3 );
+ testcase( serialType==4 );
+ testcase( serialType==5 );
+ testcase( serialType==6 );
+ testcase( serialType==7 );
+ testcase( serialType==8 );
+ testcase( serialType==9 );
+ testcase( serialType==10 );
+ testcase( serialType==11 );
+ typeMask = aMask[serialType];
+ }
+ }else{
+ typeMask = 1 << (pOp->p4.i - 1);
+ testcase( typeMask==0x01 );
+ testcase( typeMask==0x02 );
+ testcase( typeMask==0x04 );
+ testcase( typeMask==0x08 );
+ testcase( typeMask==0x10 );
+ }
+ }else{
+ assert( memIsValid(&aMem[pOp->p3]) );
+ typeMask = 1 << (sqlite3_value_type((sqlite3_value*)&aMem[pOp->p3])-1);
+ testcase( typeMask==0x01 );
+ testcase( typeMask==0x02 );
+ testcase( typeMask==0x04 );
+ testcase( typeMask==0x08 );
+ testcase( typeMask==0x10 );
+ }
+ VdbeBranchTaken( (typeMask & pOp->p5)!=0, 2);
+ if( typeMask & pOp->p5 ){
+ goto jump_to_p2;
+ }
break;
}
@@ -90935,11 +92340,14 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */
** If it is, then set register P3 to NULL and jump immediately to P2.
** If P1 is not on a NULL row, then fall through without making any
** changes.
+**
+** If P1 is not an open cursor, then this opcode is a no-op.
*/
case OP_IfNullRow: { /* jump */
+ VdbeCursor *pC;
assert( pOp->p1>=0 && pOp->p1nCursor );
- assert( p->apCsr[pOp->p1]!=0 );
- if( p->apCsr[pOp->p1]->nullRow ){
+ pC = p->apCsr[pOp->p1];
+ if( ALWAYS(pC) && pC->nullRow ){
sqlite3VdbeMemSetNull(aMem + pOp->p3);
goto jump_to_p2;
}
@@ -90990,7 +92398,7 @@ case OP_Offset: { /* out3 */
** Interpret the data that cursor P1 points to as a structure built using
** the MakeRecord instruction. (See the MakeRecord opcode for additional
** information about the format of the data.) Extract the P2-th column
-** from this record. If there are less that (P2+1)
+** from this record. If there are less than (P2+1)
** values in the record, extract a NULL.
**
** The value extracted is stored in register P3.
@@ -90999,10 +92407,12 @@ case OP_Offset: { /* out3 */
** if the P4 argument is a P4_MEM use the value of the P4 argument as
** the result.
**
-** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then
-** the result is guaranteed to only be used as the argument of a length()
-** or typeof() function, respectively. The loading of large blobs can be
-** skipped for length() and all content loading can be skipped for typeof().
+** If the OPFLAG_LENGTHARG bit is set in P5 then the result is guaranteed
+** to only be used by the length() function or the equivalent. The content
+** of large blobs is not loaded, thus saving CPU cycles. If the
+** OPFLAG_TYPEOFARG bit is set then the result will only be used by the
+** typeof() function or the IS NULL or IS NOT NULL operators or the
+** equivalent. In this case, all content loading can be omitted.
*/
case OP_Column: {
u32 p2; /* column number to retrieve */
@@ -92941,7 +94351,13 @@ case OP_SeekGT: { /* jump, in3, group */
r.aMem = &aMem[pOp->p3];
#ifdef SQLITE_DEBUG
- { int i; for(i=0; i0 ) REGISTER_TRACE(pOp->p3+i, &r.aMem[i]);
+ }
+ }
#endif
r.eqSeen = 0;
rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res);
@@ -93004,7 +94420,7 @@ seek_not_found:
}
-/* Opcode: SeekScan P1 P2 * * *
+/* Opcode: SeekScan P1 P2 * * P5
** Synopsis: Scan-ahead up to P1 rows
**
** This opcode is a prefix opcode to OP_SeekGE. In other words, this
@@ -93014,8 +94430,8 @@ seek_not_found:
** This opcode uses the P1 through P4 operands of the subsequent
** OP_SeekGE. In the text that follows, the operands of the subsequent
** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only
-** the P1 and P2 operands of this opcode are also used, and are called
-** This.P1 and This.P2.
+** the P1, P2 and P5 operands of this opcode are also used, and are called
+** This.P1, This.P2 and This.P5.
**
** This opcode helps to optimize IN operators on a multi-column index
** where the IN operator is on the later terms of the index by avoiding
@@ -93025,29 +94441,51 @@ seek_not_found:
**
** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which
** is the desired entry that we want the cursor SeekGE.P1 to be pointing
-** to. Call this SeekGE.P4/P5 row the "target".
+** to. Call this SeekGE.P3/P4 row the "target".
**
** If the SeekGE.P1 cursor is not currently pointing to a valid row,
** then this opcode is a no-op and control passes through into the OP_SeekGE.
**
** If the SeekGE.P1 cursor is pointing to a valid row, then that row
** might be the target row, or it might be near and slightly before the
-** target row. This opcode attempts to position the cursor on the target
-** row by, perhaps by invoking sqlite3BtreeStep() on the cursor
-** between 0 and This.P1 times.
+** target row, or it might be after the target row. If the cursor is
+** currently before the target row, then this opcode attempts to position
+** the cursor on or after the target row by invoking sqlite3BtreeStep()
+** on the cursor between 1 and This.P1 times.
**
-** There are three possible outcomes from this opcode:
+** The This.P5 parameter is a flag that indicates what to do if the
+** cursor ends up pointing at a valid row that is past the target
+** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If
+** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0
+** case occurs when there are no inequality constraints to the right of
+** the IN constraing. The jump to SeekGE.P2 ends the loop. The P5!=0 case
+** occurs when there are inequality constraints to the right of the IN
+** operator. In that case, the This.P2 will point either directly to or
+** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for
+** loop terminate.
**
-**
If after This.P1 steps, the cursor is still pointing to a place that
-** is earlier in the btree than the target row, then fall through
-** into the subsquence OP_SeekGE opcode.
+** Possible outcomes from this opcode:
**
-**
If the cursor is successfully moved to the target row by 0 or more
-** sqlite3BtreeNext() calls, then jump to This.P2, which will land just
-** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE.
+**
If the cursor is initally not pointed to any valid row, then
+** fall through into the subsequent OP_SeekGE opcode.
**
-**
If the cursor ends up past the target row (indicating the the target
-** row does not exist in the btree) then jump to SeekOP.P2.
+**
If the cursor is left pointing to a row that is before the target
+** row, even after making as many as This.P1 calls to
+** sqlite3BtreeNext(), then also fall through into OP_SeekGE.
+**
+**
If the cursor is left pointing at the target row, either because it
+** was at the target row to begin with or because one or more
+** sqlite3BtreeNext() calls moved the cursor to the target row,
+** then jump to This.P2..,
+**
+**
If the cursor started out before the target row and a call to
+** to sqlite3BtreeNext() moved the cursor off the end of the index
+** (indicating that the target row definitely does not exist in the
+** btree) then jump to SeekGE.P2, ending the loop.
+**
+**
If the cursor ends up on a valid row that is past the target row
+** (indicating that the target row does not exist in the btree) then
+** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0.
**
*/
case OP_SeekScan: {
@@ -93058,14 +94496,25 @@ case OP_SeekScan: {
assert( pOp[1].opcode==OP_SeekGE );
- /* pOp->p2 points to the first instruction past the OP_IdxGT that
- ** follows the OP_SeekGE. */
+ /* If pOp->p5 is clear, then pOp->p2 points to the first instruction past the
+ ** OP_IdxGT that follows the OP_SeekGE. Otherwise, it points to the first
+ ** opcode past the OP_SeekGE itself. */
assert( pOp->p2>=(int)(pOp-aOp)+2 );
- assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE );
- testcase( aOp[pOp->p2-1].opcode==OP_IdxGE );
- assert( pOp[1].p1==aOp[pOp->p2-1].p1 );
- assert( pOp[1].p2==aOp[pOp->p2-1].p2 );
- assert( pOp[1].p3==aOp[pOp->p2-1].p3 );
+#ifdef SQLITE_DEBUG
+ if( pOp->p5==0 ){
+ /* There are no inequality constraints following the IN constraint. */
+ assert( pOp[1].p1==aOp[pOp->p2-1].p1 );
+ assert( pOp[1].p2==aOp[pOp->p2-1].p2 );
+ assert( pOp[1].p3==aOp[pOp->p2-1].p3 );
+ assert( aOp[pOp->p2-1].opcode==OP_IdxGT
+ || aOp[pOp->p2-1].opcode==OP_IdxGE );
+ testcase( aOp[pOp->p2-1].opcode==OP_IdxGE );
+ }else{
+ /* There are inequality constraints. */
+ assert( pOp->p2==(int)(pOp-aOp)+2 );
+ assert( aOp[pOp->p2-1].opcode==OP_SeekGE );
+ }
+#endif
assert( pOp->p1>0 );
pC = p->apCsr[pOp[1].p1];
@@ -93099,8 +94548,9 @@ case OP_SeekScan: {
while(1){
rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res);
if( rc ) goto abort_due_to_error;
- if( res>0 ){
+ if( res>0 && pOp->p5==0 ){
seekscan_search_fail:
+ /* Jump to SeekGE.P2, ending the loop */
#ifdef SQLITE_DEBUG
if( db->flags&SQLITE_VdbeTrace ){
printf("... %d steps and then skip\n", pOp->p1 - nStep);
@@ -93110,7 +94560,8 @@ case OP_SeekScan: {
pOp++;
goto jump_to_p2;
}
- if( res==0 ){
+ if( res>=0 ){
+ /* Jump to This.P2, bypassing the OP_SeekGE opcode */
#ifdef SQLITE_DEBUG
if( db->flags&SQLITE_VdbeTrace ){
printf("... %d steps and then success\n", pOp->p1 - nStep);
@@ -93186,12 +94637,16 @@ case OP_SeekHit: {
/* Opcode: IfNotOpen P1 P2 * * *
** Synopsis: if( !csr[P1] ) goto P2
**
-** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through.
+** If cursor P1 is not open or if P1 is set to a NULL row using the
+** OP_NullRow opcode, then jump to instruction P2. Otherwise, fall through.
*/
case OP_IfNotOpen: { /* jump */
+ VdbeCursor *pCur;
+
assert( pOp->p1>=0 && pOp->p1nCursor );
- VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2);
- if( !p->apCsr[pOp->p1] ){
+ pCur = p->apCsr[pOp->p1];
+ VdbeBranchTaken(pCur==0 || pCur->nullRow, 2);
+ if( pCur==0 || pCur->nullRow ){
goto jump_to_p2_and_check_for_interrupt;
}
break;
@@ -94382,7 +95837,9 @@ case OP_SorterNext: { /* jump */
case OP_Prev: /* jump */
assert( pOp->p1>=0 && pOp->p1nCursor );
- assert( pOp->p5aCounter) );
+ assert( pOp->p5==0
+ || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP
+ || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX);
pC = p->apCsr[pOp->p1];
assert( pC!=0 );
assert( pC->deferredMoveto==0 );
@@ -94395,7 +95852,9 @@ case OP_Prev: /* jump */
case OP_Next: /* jump */
assert( pOp->p1>=0 && pOp->p1nCursor );
- assert( pOp->p5aCounter) );
+ assert( pOp->p5==0
+ || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP
+ || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX);
pC = p->apCsr[pOp->p1];
assert( pC!=0 );
assert( pC->deferredMoveto==0 );
@@ -94602,10 +96061,10 @@ case OP_IdxRowid: { /* out2 */
** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */
rc = sqlite3VdbeCursorRestore(pC);
- /* sqlite3VbeCursorRestore() can only fail if the record has been deleted
- ** out from under the cursor. That will never happens for an IdxRowid
- ** or Seek opcode */
- if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;
+ /* sqlite3VdbeCursorRestore() may fail if the cursor has been disturbed
+ ** since it was last positioned and an error (e.g. OOM or an IO error)
+ ** occurs while trying to reposition it. */
+ if( rc!=SQLITE_OK ) goto abort_due_to_error;
if( !pC->nullRow ){
rowid = 0; /* Not needed. Only used to silence a warning. */
@@ -95507,7 +96966,7 @@ case OP_IfPos: { /* jump, in1 */
** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)
**
** This opcode performs a commonly used computation associated with
-** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3]
+** LIMIT and OFFSET processing. r[P1] holds the limit counter. r[P3]
** holds the offset counter. The opcode computes the combined value
** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2]
** value computed is the total number of rows that will need to be
@@ -101105,6 +102564,8 @@ SQLITE_PRIVATE int sqlite3JournalOpen(
){
MemJournal *p = (MemJournal*)pJfd;
+ assert( zName || nSpill<0 || (flags & SQLITE_OPEN_EXCLUSIVE) );
+
/* Zero the file-handle object. If nSpill was passed zero, initialize
** it using the sqlite3OsOpen() function of the underlying VFS. In this
** case none of the code in this module is executed as a result of calls
@@ -101532,33 +102993,21 @@ static void resolveAlias(
sqlite3ExprDelete(db, pDup);
pDup = 0;
}else{
+ Expr temp;
incrAggFunctionDepth(pDup, nSubquery);
if( pExpr->op==TK_COLLATE ){
assert( !ExprHasProperty(pExpr, EP_IntValue) );
pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
}
-
- /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This
- ** prevents ExprDelete() from deleting the Expr structure itself,
- ** allowing it to be repopulated by the memcpy() on the following line.
- ** The pExpr->u.zToken might point into memory that will be freed by the
- ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
- ** make a copy of the token before doing the sqlite3DbFree().
- */
- ExprSetProperty(pExpr, EP_Static);
- sqlite3ExprDelete(db, pExpr);
- memcpy(pExpr, pDup, sizeof(*pExpr));
- if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){
- assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 );
- pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken);
- pExpr->flags |= EP_MemToken;
- }
+ memcpy(&temp, pDup, sizeof(Expr));
+ memcpy(pDup, pExpr, sizeof(Expr));
+ memcpy(pExpr, &temp, sizeof(Expr));
if( ExprHasProperty(pExpr, EP_WinFunc) ){
if( ALWAYS(pExpr->y.pWin!=0) ){
pExpr->y.pWin->pOwner = pExpr;
}
}
- sqlite3DbFree(db, pDup);
+ sqlite3ExprDeferredDelete(pParse, pDup);
}
}
@@ -101761,7 +103210,7 @@ static int lookupName(
pTab = pItem->pTab;
assert( pTab!=0 && pTab->zName!=0 );
assert( pTab->nCol>0 || pParse->nErr );
- assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
+ assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
if( pItem->fg.isNestedFrom ){
/* In this case, pItem is a subquery that has been formed from a
** parenthesized subset of the FROM clause terms. Example:
@@ -103652,9 +105101,8 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){
if( op==TK_REGISTER ) op = pExpr->op2;
if( op==TK_COLUMN || op==TK_AGG_COLUMN ){
assert( ExprUseYTab(pExpr) );
- if( pExpr->y.pTab ){
- return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn);
- }
+ assert( pExpr->y.pTab!=0 );
+ return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn);
}
if( op==TK_SELECT ){
assert( ExprUseXSelect(pExpr) );
@@ -103772,17 +105220,14 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){
int op = p->op;
if( op==TK_REGISTER ) op = p->op2;
if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){
+ int j;
assert( ExprUseYTab(p) );
- if( p->y.pTab!=0 ){
- /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally
- ** a TK_COLUMN but was previously evaluated and cached in a register */
- int j = p->iColumn;
- if( j>=0 ){
- const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]);
- pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0);
- }
- break;
+ assert( p->y.pTab!=0 );
+ if( (j = p->iColumn)>=0 ){
+ const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]);
+ pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0);
}
+ break;
}
if( op==TK_CAST || op==TK_UPLUS ){
p = p->pLeft;
@@ -104367,7 +105812,9 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){
*/
static void exprSetHeight(Expr *p){
int nHeight = p->pLeft ? p->pLeft->nHeight : 0;
- if( p->pRight && p->pRight->nHeight>nHeight ) nHeight = p->pRight->nHeight;
+ if( NEVER(p->pRight) && p->pRight->nHeight>nHeight ){
+ nHeight = p->pRight->nHeight;
+ }
if( ExprUseXSelect(p) ){
heightOfSelect(p->x.pSelect, &nHeight);
}else if( p->x.pList ){
@@ -104510,15 +105957,26 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(
sqlite3ExprDelete(db, pLeft);
sqlite3ExprDelete(db, pRight);
}else{
+ assert( ExprUseXList(pRoot) );
+ assert( pRoot->x.pSelect==0 );
if( pRight ){
pRoot->pRight = pRight;
pRoot->flags |= EP_Propagate & pRight->flags;
+#if SQLITE_MAX_EXPR_DEPTH>0
+ pRoot->nHeight = pRight->nHeight+1;
+ }else{
+ pRoot->nHeight = 1;
+#endif
}
if( pLeft ){
pRoot->pLeft = pLeft;
pRoot->flags |= EP_Propagate & pLeft->flags;
+#if SQLITE_MAX_EXPR_DEPTH>0
+ if( pLeft->nHeight>=pRoot->nHeight ){
+ pRoot->nHeight = pLeft->nHeight+1;
+ }
+#endif
}
- exprSetHeight(pRoot);
}
}
@@ -104804,6 +106262,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n
*/
static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
assert( p!=0 );
+ assert( db!=0 );
assert( !ExprUseUValue(p) || p->u.iValue>=0 );
assert( !ExprUseYWin(p) || !ExprUseYSub(p) );
assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed );
@@ -104835,12 +106294,8 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
#endif
}
}
- if( ExprHasProperty(p, EP_MemToken) ){
- assert( !ExprHasProperty(p, EP_IntValue) );
- sqlite3DbFree(db, p->u.zToken);
- }
if( !ExprHasProperty(p, EP_Static) ){
- sqlite3DbFreeNN(db, p);
+ sqlite3DbNNFreeNN(db, p);
}
}
SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){
@@ -104871,8 +106326,9 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){
** pExpr to the pParse->pConstExpr list with a register number of 0.
*/
SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){
- pParse->pConstExpr =
- sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr);
+ sqlite3ParserAddCleanup(pParse,
+ (void(*)(sqlite3*,void*))sqlite3ExprDelete,
+ pExpr);
}
/* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the
@@ -104946,7 +106402,6 @@ static int dupedExprStructSize(const Expr *p, int flags){
}else{
assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) );
assert( !ExprHasProperty(p, EP_OuterON) );
- assert( !ExprHasProperty(p, EP_MemToken) );
assert( !ExprHasVVAProperty(p, EP_NoReduce) );
if( p->pLeft || p->x.pList ){
nSize = EXPR_REDUCEDSIZE | EP_Reduced;
@@ -105050,7 +106505,7 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){
}
/* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */
- pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken);
+ pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static);
pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly);
pNew->flags |= staticFlag;
ExprClearVVAProperties(pNew);
@@ -105626,12 +107081,13 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){
int i = pList->nExpr;
struct ExprList_item *pItem = pList->a;
assert( pList->nExpr>0 );
+ assert( db!=0 );
do{
sqlite3ExprDelete(db, pItem->pExpr);
- sqlite3DbFree(db, pItem->zEName);
+ if( pItem->zEName ) sqlite3DbNNFreeNN(db, pItem->zEName);
pItem++;
}while( --i>0 );
- sqlite3DbFreeNN(db, pList);
+ sqlite3DbNNFreeNN(db, pList);
}
SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
if( pList ) exprListDeleteNN(db, pList);
@@ -106809,6 +108265,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN(
sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO);
}
if( addrOnce ){
+ sqlite3VdbeAddOp1(v, OP_NullRow, iTab);
sqlite3VdbeJumpHere(v, addrOnce);
/* Subroutine return */
assert( ExprUseYSub(pExpr) );
@@ -106922,7 +108379,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
pLimit = sqlite3PExpr(pParse, TK_NE,
sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit);
}
- sqlite3ExprDelete(db, pSel->pLimit->pLeft);
+ sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft);
pSel->pLimit->pLeft = pLimit;
}else{
/* If there is no pre-existing limit add a limit of 1 */
@@ -107375,10 +108832,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(
){
Column *pCol;
assert( v!=0 );
- if( pTab==0 ){
- sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut);
- return;
- }
+ assert( pTab!=0 );
if( iCol<0 || iCol==pTab->iPKey ){
sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut);
VdbeComment((v, "%s.rowid", pTab->zName));
@@ -107436,7 +108890,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(
assert( pParse->pVdbe!=0 );
sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg);
if( p5 ){
- VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1);
+ VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe);
if( pOp->opcode==OP_Column ) pOp->p5 = p5;
}
return iReg;
@@ -107505,7 +108959,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){
** so that a subsequent copy will not be merged into this one.
*/
static void setDoNotMergeFlagOnCopy(Vdbe *v){
- if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){
+ if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){
sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */
}
}
@@ -107628,6 +109082,53 @@ static int exprCodeInlineFunction(
return target;
}
+/*
+** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr.
+** If it is, then resolve the expression by reading from the index and
+** return the register into which the value has been read. If pExpr is
+** not an indexed expression, then return negative.
+*/
+static SQLITE_NOINLINE int sqlite3IndexedExprLookup(
+ Parse *pParse, /* The parsing context */
+ Expr *pExpr, /* The expression to potentially bypass */
+ int target /* Where to store the result of the expression */
+){
+ IndexedExpr *p;
+ Vdbe *v;
+ for(p=pParse->pIdxExpr; p; p=p->pIENext){
+ int iDataCur = p->iDataCur;
+ if( iDataCur<0 ) continue;
+ if( pParse->iSelfTab ){
+ if( p->iDataCur!=pParse->iSelfTab-1 ) continue;
+ iDataCur = -1;
+ }
+ if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue;
+ v = pParse->pVdbe;
+ assert( v!=0 );
+ if( p->bMaybeNullRow ){
+ /* If the index is on a NULL row due to an outer join, then we
+ ** cannot extract the value from the index. The value must be
+ ** computed using the original expression. */
+ int addr = sqlite3VdbeCurrentAddr(v);
+ sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target);
+ VdbeCoverage(v);
+ sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target);
+ VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol));
+ sqlite3VdbeGoto(v, 0);
+ p = pParse->pIdxExpr;
+ pParse->pIdxExpr = 0;
+ sqlite3ExprCode(pParse, pExpr, target);
+ pParse->pIdxExpr = p;
+ sqlite3VdbeJumpHere(v, addr+2);
+ }else{
+ sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target);
+ VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol));
+ }
+ return target;
+ }
+ return -1; /* Not found */
+}
+
/*
** Generate code into the current Vdbe to evaluate the given
@@ -107656,6 +109157,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target)
expr_code_doover:
if( pExpr==0 ){
op = TK_NULL;
+ }else if( pParse->pIdxExpr!=0
+ && !ExprHasProperty(pExpr, EP_Leaf)
+ && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0
+ ){
+ return r1;
}else{
assert( !ExprHasVVAProperty(pExpr,EP_Immutable) );
op = pExpr->op;
@@ -107676,7 +109182,7 @@ expr_code_doover:
pCol->iSorterColumn, target);
if( pCol->iColumn<0 ){
VdbeComment((v,"%s.rowid",pTab->zName));
- }else{
+ }else if( ALWAYS(pTab!=0) ){
VdbeComment((v,"%s.%s",
pTab->zName, pTab->aCol[pCol->iColumn].zCnName));
if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){
@@ -107701,11 +109207,8 @@ expr_code_doover:
int aff;
iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target);
assert( ExprUseYTab(pExpr) );
- if( pExpr->y.pTab ){
- aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn);
- }else{
- aff = pExpr->affExpr;
- }
+ assert( pExpr->y.pTab!=0 );
+ aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn);
if( aff>SQLITE_AFF_BLOB ){
static const char zAff[] = "B\000C\000D\000E";
assert( SQLITE_AFF_BLOB=='A' );
@@ -107767,12 +109270,10 @@ expr_code_doover:
}
}
assert( ExprUseYTab(pExpr) );
+ assert( pExpr->y.pTab!=0 );
iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab,
pExpr->iColumn, iTab, target,
pExpr->op2);
- if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){
- sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg);
- }
return iReg;
}
case TK_INTEGER: {
@@ -108271,6 +109772,21 @@ expr_code_doover:
case TK_IF_NULL_ROW: {
int addrINR;
u8 okConstFactor = pParse->okConstFactor;
+ AggInfo *pAggInfo = pExpr->pAggInfo;
+ if( pAggInfo ){
+ assert( pExpr->iAgg>=0 && pExpr->iAggnColumn );
+ if( !pAggInfo->directMode ){
+ inReg = pAggInfo->aCol[pExpr->iAgg].iMem;
+ break;
+ }
+ if( pExpr->pAggInfo->useSortingIdx ){
+ sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab,
+ pAggInfo->aCol[pExpr->iAgg].iSorterColumn,
+ target);
+ inReg = target;
+ break;
+ }
+ }
addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable);
/* Temporarily disable factoring of constant expressions, since
** even though expressions may appear to be constant, they are not
@@ -108612,7 +110128,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList(
if( inReg!=target+i ){
VdbeOp *pOp;
if( copyOp==OP_Copy
- && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy
+ && (pOp=sqlite3VdbeGetLastOp(v))->opcode==OP_Copy
&& pOp->p1+pOp->p3+1==inReg
&& pOp->p2+pOp->p3+1==target+i
&& pOp->p5==0 /* The do-not-merge flag must be clear */
@@ -108811,6 +110327,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int
assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL );
assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL );
r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1);
+ sqlite3VdbeTypeofColumn(v, r1);
sqlite3VdbeAddOp2(v, op, r1, dest);
VdbeCoverageIf(v, op==TK_ISNULL);
VdbeCoverageIf(v, op==TK_NOTNULL);
@@ -108985,6 +110502,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int
case TK_ISNULL:
case TK_NOTNULL: {
r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1);
+ sqlite3VdbeTypeofColumn(v, r1);
sqlite3VdbeAddOp2(v, op, r1, dest);
testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL);
testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL);
@@ -109138,7 +110656,13 @@ SQLITE_PRIVATE int sqlite3ExprCompare(
if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){
return 1;
}
- return 2;
+ if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN
+ && pB->iTable<0 && pA->iTable==iTab
+ ){
+ /* fall through */
+ }else{
+ return 2;
+ }
}
assert( !ExprHasProperty(pA, EP_IntValue) );
assert( !ExprHasProperty(pB, EP_IntValue) );
@@ -109440,10 +110964,10 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){
assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) );
assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) );
if( (pLeft->op==TK_COLUMN
- && pLeft->y.pTab!=0
+ && ALWAYS(pLeft->y.pTab!=0)
&& IsVirtual(pLeft->y.pTab))
|| (pRight->op==TK_COLUMN
- && pRight->y.pTab!=0
+ && ALWAYS(pRight->y.pTab!=0)
&& IsVirtual(pRight->y.pTab))
){
return WRC_Prune;
@@ -109648,6 +111172,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){
SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){
Walker w;
struct RefSrcList x;
+ assert( pParse->db!=0 );
memset(&w, 0, sizeof(w));
memset(&x, 0, sizeof(x));
w.xExprCallback = exprRefToSrcList;
@@ -109664,7 +111189,7 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList
sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter);
}
#endif
- sqlite3DbFree(pParse->db, x.aiExclude);
+ if( x.aiExclude ) sqlite3DbNNFreeNN(pParse->db, x.aiExclude);
if( w.eCode & 0x01 ){
return 1;
}else if( w.eCode ){
@@ -109695,8 +111220,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){
int iAgg = pExpr->iAgg;
Parse *pParse = pWalker->pParse;
sqlite3 *db = pParse->db;
- assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION );
- if( pExpr->op==TK_AGG_COLUMN ){
+ if( pExpr->op!=TK_AGG_FUNCTION ){
+ assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_IF_NULL_ROW );
assert( iAgg>=0 && iAggnColumn );
if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){
pExpr = sqlite3ExprDup(db, pExpr, 0);
@@ -109706,6 +111231,7 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){
}
}
}else{
+ assert( pExpr->op==TK_AGG_FUNCTION );
assert( iAgg>=0 && iAggnFunc );
if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){
pExpr = sqlite3ExprDup(db, pExpr, 0);
@@ -109776,10 +111302,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
assert( pNC->ncFlags & NC_UAggInfo );
switch( pExpr->op ){
+ case TK_IF_NULL_ROW:
case TK_AGG_COLUMN:
case TK_COLUMN: {
testcase( pExpr->op==TK_AGG_COLUMN );
testcase( pExpr->op==TK_COLUMN );
+ testcase( pExpr->op==TK_IF_NULL_ROW );
/* Check to see if the column is in one of the tables in the FROM
** clause of the aggregate query */
if( ALWAYS(pSrcList!=0) ){
@@ -109797,8 +111325,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
int k;
pCol = pAggInfo->aCol;
for(k=0; knColumn; k++, pCol++){
- if( pCol->iTable==pExpr->iTable &&
- pCol->iColumn==pExpr->iColumn ){
+ if( pCol->iTable==pExpr->iTable
+ && pCol->iColumn==pExpr->iColumn
+ && pExpr->op!=TK_IF_NULL_ROW
+ ){
break;
}
}
@@ -109813,15 +111343,17 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
pCol->iMem = ++pParse->nMem;
pCol->iSorterColumn = -1;
pCol->pCExpr = pExpr;
- if( pAggInfo->pGroupBy ){
+ if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){
int j, n;
ExprList *pGB = pAggInfo->pGroupBy;
struct ExprList_item *pTerm = pGB->a;
n = pGB->nExpr;
for(j=0; jpExpr;
- if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable &&
- pE->iColumn==pExpr->iColumn ){
+ if( pE->op==TK_COLUMN
+ && pE->iTable==pExpr->iTable
+ && pE->iColumn==pExpr->iColumn
+ ){
pCol->iSorterColumn = j;
break;
}
@@ -109838,7 +111370,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
*/
ExprSetVVAProperty(pExpr, EP_NoReduce);
pExpr->pAggInfo = pAggInfo;
- pExpr->op = TK_AGG_COLUMN;
+ if( pExpr->op==TK_COLUMN ){
+ pExpr->op = TK_AGG_COLUMN;
+ }
pExpr->iAgg = (i16)k;
break;
} /* endif pExpr->iTable==pItem->iCursor */
@@ -113258,6 +114792,7 @@ static void analyzeVdbeCommentIndexWithColumnName(
if( NEVER(i==XN_ROWID) ){
VdbeComment((v,"%s.rowid",pIdx->zName));
}else if( i==XN_EXPR ){
+ assert( pIdx->bHasExpr );
VdbeComment((v,"%s.expr(%d)",pIdx->zName, k));
}else{
VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName));
@@ -115260,6 +116795,7 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){
SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){
sqlite3 *db;
Vdbe *v;
+ int iDb, i;
assert( pParse->pToplevel==0 );
db = pParse->db;
@@ -115289,7 +116825,6 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){
if( pParse->bReturning ){
Returning *pReturning = pParse->u1.pReturning;
int addrRewind;
- int i;
int reg;
if( pReturning->nRetCol ){
@@ -115326,76 +116861,69 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){
** transaction on each used database and to verify the schema cookie
** on each used database.
*/
- if( db->mallocFailed==0
- && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr)
- ){
- int iDb, i;
- assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init );
- sqlite3VdbeJumpHere(v, 0);
- assert( db->nDb>0 );
- iDb = 0;
- do{
- Schema *pSchema;
- if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue;
- sqlite3VdbeUsesBtree(v, iDb);
- pSchema = db->aDb[iDb].pSchema;
- sqlite3VdbeAddOp4Int(v,
- OP_Transaction, /* Opcode */
- iDb, /* P1 */
- DbMaskTest(pParse->writeMask,iDb), /* P2 */
- pSchema->schema_cookie, /* P3 */
- pSchema->iGeneration /* P4 */
- );
- if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1);
- VdbeComment((v,
- "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite));
- }while( ++iDbnDb );
+ assert( pParse->nErr>0 || sqlite3VdbeGetOp(v, 0)->opcode==OP_Init );
+ sqlite3VdbeJumpHere(v, 0);
+ assert( db->nDb>0 );
+ iDb = 0;
+ do{
+ Schema *pSchema;
+ if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue;
+ sqlite3VdbeUsesBtree(v, iDb);
+ pSchema = db->aDb[iDb].pSchema;
+ sqlite3VdbeAddOp4Int(v,
+ OP_Transaction, /* Opcode */
+ iDb, /* P1 */
+ DbMaskTest(pParse->writeMask,iDb), /* P2 */
+ pSchema->schema_cookie, /* P3 */
+ pSchema->iGeneration /* P4 */
+ );
+ if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1);
+ VdbeComment((v,
+ "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite));
+ }while( ++iDbnDb );
#ifndef SQLITE_OMIT_VIRTUALTABLE
- for(i=0; inVtabLock; i++){
- char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]);
- sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB);
- }
- pParse->nVtabLock = 0;
+ for(i=0; inVtabLock; i++){
+ char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]);
+ sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB);
+ }
+ pParse->nVtabLock = 0;
#endif
- /* Once all the cookies have been verified and transactions opened,
- ** obtain the required table-locks. This is a no-op unless the
- ** shared-cache feature is enabled.
- */
- codeTableLocks(pParse);
+ /* Once all the cookies have been verified and transactions opened,
+ ** obtain the required table-locks. This is a no-op unless the
+ ** shared-cache feature is enabled.
+ */
+ codeTableLocks(pParse);
- /* Initialize any AUTOINCREMENT data structures required.
- */
- sqlite3AutoincrementBegin(pParse);
+ /* Initialize any AUTOINCREMENT data structures required.
+ */
+ sqlite3AutoincrementBegin(pParse);
- /* Code constant expressions that where factored out of inner loops.
- **
- ** The pConstExpr list might also contain expressions that we simply
- ** want to keep around until the Parse object is deleted. Such
- ** expressions have iConstExprReg==0. Do not generate code for
- ** those expressions, of course.
- */
- if( pParse->pConstExpr ){
- ExprList *pEL = pParse->pConstExpr;
- pParse->okConstFactor = 0;
- for(i=0; inExpr; i++){
- int iReg = pEL->a[i].u.iConstExprReg;
- if( iReg>0 ){
- sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg);
- }
- }
+ /* Code constant expressions that where factored out of inner loops.
+ **
+ ** The pConstExpr list might also contain expressions that we simply
+ ** want to keep around until the Parse object is deleted. Such
+ ** expressions have iConstExprReg==0. Do not generate code for
+ ** those expressions, of course.
+ */
+ if( pParse->pConstExpr ){
+ ExprList *pEL = pParse->pConstExpr;
+ pParse->okConstFactor = 0;
+ for(i=0; inExpr; i++){
+ int iReg = pEL->a[i].u.iConstExprReg;
+ sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg);
}
-
- if( pParse->bReturning ){
- Returning *pRet = pParse->u1.pReturning;
- if( pRet->nRetCol ){
- sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol);
- }
- }
-
- /* Finally, jump back to the beginning of the executable code. */
- sqlite3VdbeGoto(v, 1);
}
+
+ if( pParse->bReturning ){
+ Returning *pRet = pParse->u1.pReturning;
+ if( pRet->nRetCol ){
+ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol);
+ }
+ }
+
+ /* Finally, jump back to the beginning of the executable code. */
+ sqlite3VdbeGoto(v, 1);
}
/* Get the VDBE program ready for execution
@@ -115451,8 +116979,6 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){
memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ);
db->mDbFlags |= DBFLAG_PreferBuiltin;
sqlite3RunParser(pParse, zSql);
- sqlite3DbFree(db, pParse->zErrMsg);
- pParse->zErrMsg = 0;
db->mDbFlags = savedDbFlags;
sqlite3DbFree(db, zSql);
memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ);
@@ -115582,7 +117108,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable(
/* If zName is the not the name of a table in the schema created using
** CREATE, then check to see if it is the name of an virtual table that
** can be an eponymous virtual table. */
- if( pParse->disableVtab==0 && db->init.busy==0 ){
+ if( (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)==0 && db->init.busy==0 ){
Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName);
if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){
pMod = sqlite3PragmaVtabRegister(db, zName);
@@ -115595,7 +117121,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable(
#endif
if( flags & LOCATE_NOERR ) return 0;
pParse->checkSchema = 1;
- }else if( IsVirtual(p) && pParse->disableVtab ){
+ }else if( IsVirtual(p) && (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)!=0 ){
p = 0;
}
@@ -115904,16 +117430,17 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){
int i;
Column *pCol;
assert( pTable!=0 );
+ assert( db!=0 );
if( (pCol = pTable->aCol)!=0 ){
for(i=0; inCol; i++, pCol++){
assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) );
sqlite3DbFree(db, pCol->zCnName);
}
- sqlite3DbFree(db, pTable->aCol);
+ sqlite3DbNNFreeNN(db, pTable->aCol);
if( IsOrdinaryTable(pTable) ){
sqlite3ExprListDelete(db, pTable->u.tab.pDfltList);
}
- if( db==0 || db->pnBytesFreed==0 ){
+ if( db->pnBytesFreed==0 ){
pTable->aCol = 0;
pTable->nCol = 0;
if( IsOrdinaryTable(pTable) ){
@@ -115950,7 +117477,8 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
** a Table object that was going to be marked ephemeral. So do not check
** that no lookaside memory is used in this case either. */
int nLookaside = 0;
- if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){
+ assert( db!=0 );
+ if( !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){
nLookaside = sqlite3LookasideUsed(db, 0);
}
#endif
@@ -115960,7 +117488,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
pNext = pIndex->pNext;
assert( pIndex->pSchema==pTable->pSchema
|| (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) );
- if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){
+ if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){
char *zName = pIndex->zName;
TESTONLY ( Index *pOld = ) sqlite3HashInsert(
&pIndex->pSchema->idxHash, zName, 0
@@ -115997,8 +117525,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
}
SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){
/* Do not delete the table until the reference count reaches zero. */
+ assert( db!=0 );
if( !pTable ) return;
- if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return;
+ if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return;
deleteTable(db, pTable);
}
@@ -117402,7 +118931,8 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){
/* Recompute the colNotIdxed field of the Index.
**
** colNotIdxed is a bitmask that has a 0 bit representing each indexed
-** columns that are within the first 63 columns of the table. The
+** columns that are within the first 63 columns of the table and a 1 for
+** all other bits (all columns that are not in the index). The
** high-order bit of colNotIdxed is always 1. All unindexed columns
** of the table have a 1.
**
@@ -117430,7 +118960,7 @@ static void recomputeColumnsNotIndexed(Index *pIdx){
}
}
pIdx->colNotIdxed = ~m;
- assert( (pIdx->colNotIdxed>>63)==1 );
+ assert( (pIdx->colNotIdxed>>63)==1 ); /* See note-20221022-a */
}
/*
@@ -118171,7 +119701,7 @@ create_view_fail:
** the columns of the view in the pTable structure. Return the number
** of errors. If an error is seen leave an error message in pParse->zErrMsg.
*/
-SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
+static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){
Table *pSelTab; /* A fake table from which we get the result set */
Select *pSel; /* Copy of the SELECT that implements the view */
int nErr = 0; /* Number of errors encountered */
@@ -118196,9 +119726,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
#ifndef SQLITE_OMIT_VIEW
/* A positive nCol means the columns names for this view are
- ** already known.
+ ** already known. This routine is not called unless either the
+ ** table is virtual or nCol is zero.
*/
- if( pTable->nCol>0 ) return 0;
+ assert( pTable->nCol<=0 );
/* A negative nCol is a special marker meaning that we are currently
** trying to compute the column names. If we enter this routine with
@@ -118294,6 +119825,11 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
#endif /* SQLITE_OMIT_VIEW */
return nErr;
}
+SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
+ assert( pTable!=0 );
+ if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0;
+ return viewGetColumnNames(pParse, pTable);
+}
#endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */
#ifndef SQLITE_OMIT_VIEW
@@ -119159,7 +120695,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
}
if( !IN_RENAME_OBJECT ){
if( !db->init.busy ){
- if( sqlite3FindTable(db, zName, 0)!=0 ){
+ if( sqlite3FindTable(db, zName, pDb->zDbSName)!=0 ){
sqlite3ErrorMsg(pParse, "there is already a table named %s", zName);
goto exit_create_index;
}
@@ -119312,6 +120848,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
j = XN_EXPR;
pIndex->aiColumn[i] = XN_EXPR;
pIndex->uniqNotNull = 0;
+ pIndex->bHasExpr = 1;
}else{
j = pCExpr->iColumn;
assert( j<=0x7fff );
@@ -119323,6 +120860,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex(
}
if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){
pIndex->bHasVCol = 1;
+ pIndex->bHasExpr = 1;
}
}
pIndex->aiColumn[i] = (i16)j;
@@ -119812,12 +121350,13 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *
*/
SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){
int i;
+ assert( db!=0 );
if( pList==0 ) return;
assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */
for(i=0; inId; i++){
sqlite3DbFree(db, pList->a[i].zName);
}
- sqlite3DbFreeNN(db, pList);
+ sqlite3DbNNFreeNN(db, pList);
}
/*
@@ -120020,11 +121559,12 @@ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){
SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){
int i;
SrcItem *pItem;
+ assert( db!=0 );
if( pList==0 ) return;
for(pItem=pList->a, i=0; inSrc; i++, pItem++){
- if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase);
- sqlite3DbFree(db, pItem->zName);
- if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias);
+ if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase);
+ if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName);
+ if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias);
if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy);
if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg);
sqlite3DeleteTable(db, pItem->pTab);
@@ -120035,7 +121575,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){
sqlite3ExprDelete(db, pItem->u3.pOn);
}
}
- sqlite3DbFreeNN(db, pList);
+ sqlite3DbNNFreeNN(db, pList);
}
/*
@@ -121287,19 +122827,21 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){
Hash temp2;
HashElem *pElem;
Schema *pSchema = (Schema *)p;
+ sqlite3 xdb;
+ memset(&xdb, 0, sizeof(xdb));
temp1 = pSchema->tblHash;
temp2 = pSchema->trigHash;
sqlite3HashInit(&pSchema->trigHash);
sqlite3HashClear(&pSchema->idxHash);
for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){
- sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem));
+ sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem));
}
sqlite3HashClear(&temp2);
sqlite3HashInit(&pSchema->tblHash);
for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){
Table *pTab = sqliteHashData(pElem);
- sqlite3DeleteTable(0, pTab);
+ sqlite3DeleteTable(&xdb, pTab);
}
sqlite3HashClear(&temp1);
sqlite3HashClear(&pSchema->fkeyHash);
@@ -121398,18 +122940,42 @@ SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *
** 1) It is a virtual table and no implementation of the xUpdate method
** has been provided
**
-** 2) It is a system table (i.e. sqlite_schema), this call is not
+** 2) A trigger is currently being coded and the table is a virtual table
+** that is SQLITE_VTAB_DIRECTONLY or if PRAGMA trusted_schema=OFF and
+** the table is not SQLITE_VTAB_INNOCUOUS.
+**
+** 3) It is a system table (i.e. sqlite_schema), this call is not
** part of a nested parse and writable_schema pragma has not
** been specified
**
-** 3) The table is a shadow table, the database connection is in
+** 4) The table is a shadow table, the database connection is in
** defensive mode, and the current sqlite3_prepare()
** is for a top-level SQL statement.
*/
+static int vtabIsReadOnly(Parse *pParse, Table *pTab){
+ if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){
+ return 1;
+ }
+
+ /* Within triggers:
+ ** * Do not allow DELETE, INSERT, or UPDATE of SQLITE_VTAB_DIRECTONLY
+ ** virtual tables
+ ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS
+ ** virtual tables if PRAGMA trusted_schema=ON.
+ */
+ if( pParse->pToplevel!=0
+ && pTab->u.vtab.p->eVtabRisk >
+ ((pParse->db->flags & SQLITE_TrustedSchema)!=0)
+ ){
+ sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"",
+ pTab->zName);
+ }
+ return 0;
+}
static int tabIsReadOnly(Parse *pParse, Table *pTab){
sqlite3 *db;
if( IsVirtual(pTab) ){
- return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0;
+ return vtabIsReadOnly(pParse, pTab);
}
if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0;
db = pParse->db;
@@ -121421,9 +122987,11 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){
}
/*
-** Check to make sure the given table is writable. If it is not
-** writable, generate an error message and return 1. If it is
-** writable return 0;
+** Check to make sure the given table is writable.
+**
+** If pTab is not writable -> generate an error message and return 1.
+** If pTab is writable but other errors have occurred -> return 1.
+** If pTab is writable and no prior errors -> return 0;
*/
SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){
if( tabIsReadOnly(pParse, pTab) ){
@@ -121784,9 +123352,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom(
}
for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
assert( pIdx->pSchema==pTab->pSchema );
- sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb);
if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){
- sqlite3VdbeChangeP3(v, -1, memCnt ? memCnt : -1);
+ sqlite3VdbeAddOp3(v, OP_Clear, pIdx->tnum, iDb, memCnt ? memCnt : -1);
+ }else{
+ sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb);
}
}
}else
@@ -121986,7 +123555,7 @@ delete_from_cleanup:
sqlite3ExprListDelete(db, pOrderBy);
sqlite3ExprDelete(db, pLimit);
#endif
- sqlite3DbFree(db, aToOpen);
+ if( aToOpen ) sqlite3DbNNFreeNN(db, aToOpen);
return;
}
/* Make sure "isView" and other macros defined above are undefined. Otherwise
@@ -123069,7 +124638,7 @@ static int patternCompare(
** c but in the other case and search the input string for either
** c or cx.
*/
- if( c<=0x80 ){
+ if( c<0x80 ){
char zStop[3];
int bMatch;
if( noCase ){
@@ -123152,7 +124721,13 @@ static int patternCompare(
** non-zero if there is no match.
*/
SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){
- return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[');
+ if( zString==0 ){
+ return zGlobPattern!=0;
+ }else if( zGlobPattern==0 ){
+ return 1;
+ }else {
+ return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[');
+ }
}
/*
@@ -123160,7 +124735,13 @@ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){
** a miss - like strcmp().
*/
SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){
- return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc);
+ if( zStr==0 ){
+ return zPattern!=0;
+ }else if( zPattern==0 ){
+ return 1;
+ }else{
+ return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc);
+ }
}
/*
@@ -126154,11 +127735,12 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){
FKey *pNext; /* Copy of pFKey->pNextFrom */
assert( IsOrdinaryTable(pTab) );
+ assert( db!=0 );
for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){
assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) );
/* Remove the FK from the fkeyHash hash table. */
- if( !db || db->pnBytesFreed==0 ){
+ if( db->pnBytesFreed==0 ){
if( pFKey->pPrevTo ){
pFKey->pPrevTo->pNextTo = pFKey->pNextTo;
}else{
@@ -126288,6 +127870,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){
aff = SQLITE_AFF_INTEGER;
}else{
assert( x==XN_EXPR );
+ assert( pIdx->bHasExpr );
assert( pIdx->aColExpr!=0 );
aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr);
}
@@ -126301,6 +127884,28 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){
return pIdx->zColAff;
}
+/*
+** Compute an affinity string for a table. Space is obtained
+** from sqlite3DbMalloc(). The caller is responsible for freeing
+** the space when done.
+*/
+SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){
+ char *zColAff;
+ zColAff = (char *)sqlite3DbMallocRaw(db, pTab->nCol+1);
+ if( zColAff ){
+ int i, j;
+ for(i=j=0; inCol; i++){
+ if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){
+ zColAff[j++] = pTab->aCol[i].affinity;
+ }
+ }
+ do{
+ zColAff[j--] = 0;
+ }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB );
+ }
+ return zColAff;
+}
+
/*
** Make changes to the evolving bytecode to do affinity transformations
** of values that are about to be gathered into a row for table pTab.
@@ -126342,7 +127947,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){
** Apply the type checking to that array of registers.
*/
SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){
- int i, j;
+ int i;
char *zColAff;
if( pTab->tabFlags & TF_Strict ){
if( iReg==0 ){
@@ -126351,7 +127956,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){
** OP_MakeRecord is found */
VdbeOp *pPrev;
sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
- pPrev = sqlite3VdbeGetOp(v, -1);
+ pPrev = sqlite3VdbeGetLastOp(v);
assert( pPrev!=0 );
assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed );
pPrev->opcode = OP_TypeCheck;
@@ -126365,22 +127970,11 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){
}
zColAff = pTab->zColAff;
if( zColAff==0 ){
- sqlite3 *db = sqlite3VdbeDb(v);
- zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1);
+ zColAff = sqlite3TableAffinityStr(0, pTab);
if( !zColAff ){
- sqlite3OomFault(db);
+ sqlite3OomFault(sqlite3VdbeDb(v));
return;
}
-
- for(i=j=0; inCol; i++){
- assert( pTab->aCol[i].affinity!=0 || sqlite3VdbeParser(v)->nErr>0 );
- if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){
- zColAff[j++] = pTab->aCol[i].affinity;
- }
- }
- do{
- zColAff[j--] = 0;
- }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB );
pTab->zColAff = zColAff;
}
assert( zColAff!=0 );
@@ -126389,7 +127983,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){
if( iReg ){
sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i);
}else{
- assert( sqlite3VdbeGetOp(v, -1)->opcode==OP_MakeRecord
+ assert( sqlite3VdbeGetLastOp(v)->opcode==OP_MakeRecord
|| sqlite3VdbeDb(v)->mallocFailed );
sqlite3VdbeChangeP4(v, -1, zColAff, i);
}
@@ -126475,7 +128069,7 @@ SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns(
*/
sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore);
if( (pTab->tabFlags & TF_HasStored)!=0 ){
- pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1);
+ pOp = sqlite3VdbeGetLastOp(pParse->pVdbe);
if( pOp->opcode==OP_Affinity ){
/* Change the OP_Affinity argument to '@' (NONE) for all stored
** columns. '@' is the no-op affinity and those columns have not
@@ -127381,7 +128975,12 @@ SQLITE_PRIVATE void sqlite3Insert(
sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore);
}
}else{
- sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore);
+ Expr *pX = pList->a[k].pExpr;
+ int y = sqlite3ExprCodeTarget(pParse, pX, iRegStore);
+ if( y!=iRegStore ){
+ sqlite3VdbeAddOp2(v,
+ ExprHasProperty(pX, EP_Subquery) ? OP_Copy : OP_SCopy, y, iRegStore);
+ }
}
}
@@ -127518,7 +129117,9 @@ SQLITE_PRIVATE void sqlite3Insert(
sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert
);
- sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0);
+ if( db->flags & SQLITE_ForeignKeys ){
+ sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0);
+ }
/* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE
** constraints or (b) there are no triggers and this table is not a
@@ -127602,7 +129203,7 @@ insert_cleanup:
sqlite3UpsertDelete(db, pUpsert);
sqlite3SelectDelete(db, pSelect);
sqlite3IdListDelete(db, pColumn);
- sqlite3DbFree(db, aRegIdx);
+ if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx);
}
/* Make sure "isView" and other macros defined above are undefined. Otherwise
@@ -129829,9 +131430,9 @@ struct sqlite3_api_routines {
const char *(*filename_journal)(const char*);
const char *(*filename_wal)(const char*);
/* Version 3.32.0 and later */
- char *(*create_filename)(const char*,const char*,const char*,
+ const char *(*create_filename)(const char*,const char*,const char*,
int,const char**);
- void (*free_filename)(char*);
+ void (*free_filename)(const char*);
sqlite3_file *(*database_file_object)(const char*);
/* Version 3.34.0 and later */
int (*txn_state)(sqlite3*,const char*);
@@ -129855,6 +131456,8 @@ struct sqlite3_api_routines {
unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*,
unsigned int);
const char *(*db_name)(sqlite3*,int);
+ /* Version 3.40.0 and later */
+ int (*value_encoding)(sqlite3_value*);
};
/*
@@ -130179,6 +131782,8 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_serialize sqlite3_api->serialize
#endif
#define sqlite3_db_name sqlite3_api->db_name
+/* Version 3.40.0 and later */
+#define sqlite3_value_encoding sqlite3_api->value_encoding
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
@@ -130691,7 +132296,9 @@ static const sqlite3_api_routines sqlite3Apis = {
0,
0,
#endif
- sqlite3_db_name
+ sqlite3_db_name,
+ /* Version 3.40.0 and later */
+ sqlite3_value_type
};
/* True if x is the directory separator character
@@ -132708,6 +134315,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
**
*/
case PragTyp_TEMP_STORE_DIRECTORY: {
+ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
if( !zRight ){
returnSingleText(v, sqlite3_temp_directory);
}else{
@@ -132717,6 +134325,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res);
if( rc!=SQLITE_OK || res==0 ){
sqlite3ErrorMsg(pParse, "not a writable directory");
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
goto pragma_out;
}
}
@@ -132734,6 +134343,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
}
#endif /* SQLITE_OMIT_WSD */
}
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
break;
}
@@ -132752,6 +134362,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
**
*/
case PragTyp_DATA_STORE_DIRECTORY: {
+ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
if( !zRight ){
returnSingleText(v, sqlite3_data_directory);
}else{
@@ -132761,6 +134372,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res);
if( rc!=SQLITE_OK || res==0 ){
sqlite3ErrorMsg(pParse, "not a writable directory");
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
goto pragma_out;
}
}
@@ -132772,6 +134384,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
}
#endif /* SQLITE_OMIT_WSD */
}
+ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR));
break;
}
#endif
@@ -133485,15 +135098,24 @@ SQLITE_PRIVATE void sqlite3Pragma(
for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){
Table *pTab = sqliteHashData(x);
Index *pIdx, *pPk;
- Index *pPrior = 0;
+ Index *pPrior = 0; /* Previous index */
int loopTop;
int iDataCur, iIdxCur;
int r1 = -1;
- int bStrict;
+ int bStrict; /* True for a STRICT table */
+ int r2; /* Previous key for WITHOUT ROWID tables */
+ int mxCol; /* Maximum non-virtual column number */
if( !IsOrdinaryTable(pTab) ) continue;
if( pObjTab && pObjTab!=pTab ) continue;
- pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab);
+ if( isQuick || HasRowid(pTab) ){
+ pPk = 0;
+ r2 = 0;
+ }else{
+ pPk = sqlite3PrimaryKeyIndex(pTab);
+ r2 = sqlite3GetTempRange(pParse, pPk->nKeyCol);
+ sqlite3VdbeAddOp3(v, OP_Null, 1, r2, r2+pPk->nKeyCol-1);
+ }
sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0,
1, 0, &iDataCur, &iIdxCur);
/* reg[7] counts the number of entries in the table.
@@ -133507,52 +135129,157 @@ SQLITE_PRIVATE void sqlite3Pragma(
assert( sqlite3NoTempsInRange(pParse,1,7+j) );
sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v);
loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1);
- if( !isQuick ){
- /* Sanity check on record header decoding */
- sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3);
- sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG);
- VdbeComment((v, "(right-most column)"));
+
+ /* Fetch the right-most column from the table. This will cause
+ ** the entire record header to be parsed and sanity checked. It
+ ** will also prepopulate the cursor column cache that is used
+ ** by the OP_IsType code, so it is a required step.
+ */
+ mxCol = pTab->nCol-1;
+ while( mxCol>=0
+ && ((pTab->aCol[mxCol].colFlags & COLFLAG_VIRTUAL)!=0
+ || pTab->iPKey==mxCol) ) mxCol--;
+ if( mxCol>=0 ){
+ sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, mxCol, 3);
+ sqlite3VdbeTypeofColumn(v, 3);
}
- /* Verify that all NOT NULL columns really are NOT NULL. At the
- ** same time verify the type of the content of STRICT tables */
+
+ if( !isQuick ){
+ if( pPk ){
+ /* Verify WITHOUT ROWID keys are in ascending order */
+ int a1;
+ char *zErr;
+ a1 = sqlite3VdbeAddOp4Int(v, OP_IdxGT, iDataCur, 0,r2,pPk->nKeyCol);
+ VdbeCoverage(v);
+ sqlite3VdbeAddOp1(v, OP_IsNull, r2); VdbeCoverage(v);
+ zErr = sqlite3MPrintf(db,
+ "row not in PRIMARY KEY order for %s",
+ pTab->zName);
+ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
+ integrityCheckResultRow(v);
+ sqlite3VdbeJumpHere(v, a1);
+ sqlite3VdbeJumpHere(v, a1+1);
+ for(j=0; jnKeyCol; j++){
+ sqlite3ExprCodeLoadIndexColumn(pParse, pPk, iDataCur, j, r2+j);
+ }
+ }
+ }
+ /* Verify datatypes for all columns:
+ **
+ ** (1) NOT NULL columns may not contain a NULL
+ ** (2) Datatype must be exact for non-ANY columns in STRICT tables
+ ** (3) Datatype for TEXT columns in non-STRICT tables must be
+ ** NULL, TEXT, or BLOB.
+ ** (4) Datatype for numeric columns in non-STRICT tables must not
+ ** be a TEXT value that can be losslessly converted to numeric.
+ */
bStrict = (pTab->tabFlags & TF_Strict)!=0;
for(j=0; jnCol; j++){
char *zErr;
- Column *pCol = pTab->aCol + j;
- int doError, jmp2;
+ Column *pCol = pTab->aCol + j; /* The column to be checked */
+ int labelError; /* Jump here to report an error */
+ int labelOk; /* Jump here if all looks ok */
+ int p1, p3, p4; /* Operands to the OP_IsType opcode */
+ int doTypeCheck; /* Check datatypes (besides NOT NULL) */
+
if( j==pTab->iPKey ) continue;
- if( pCol->notNull==0 && !bStrict ) continue;
- doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0;
- sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3);
- if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){
- sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG);
+ if( bStrict ){
+ doTypeCheck = pCol->eCType>COLTYPE_ANY;
+ }else{
+ doTypeCheck = pCol->affinity>SQLITE_AFF_BLOB;
}
+ if( pCol->notNull==0 && !doTypeCheck ) continue;
+
+ /* Compute the operands that will be needed for OP_IsType */
+ p4 = SQLITE_NULL;
+ if( pCol->colFlags & COLFLAG_VIRTUAL ){
+ sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3);
+ p1 = -1;
+ p3 = 3;
+ }else{
+ if( pCol->iDflt ){
+ sqlite3_value *pDfltValue = 0;
+ sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db),
+ pCol->affinity, &pDfltValue);
+ if( pDfltValue ){
+ p4 = sqlite3_value_type(pDfltValue);
+ sqlite3ValueFree(pDfltValue);
+ }
+ }
+ p1 = iDataCur;
+ if( !HasRowid(pTab) ){
+ testcase( j!=sqlite3TableColumnToStorage(pTab, j) );
+ p3 = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), j);
+ }else{
+ p3 = sqlite3TableColumnToStorage(pTab,j);
+ testcase( p3!=j);
+ }
+ }
+
+ labelError = sqlite3VdbeMakeLabel(pParse);
+ labelOk = sqlite3VdbeMakeLabel(pParse);
if( pCol->notNull ){
- jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v);
+ /* (1) NOT NULL columns may not contain a NULL */
+ int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4);
+ sqlite3VdbeChangeP5(v, 0x0f);
+ VdbeCoverage(v);
zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName,
pCol->zCnName);
sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
- if( bStrict && pCol->eCType!=COLTYPE_ANY ){
- sqlite3VdbeGoto(v, doError);
+ if( doTypeCheck ){
+ sqlite3VdbeGoto(v, labelError);
+ sqlite3VdbeJumpHere(v, jmp2);
}else{
- integrityCheckResultRow(v);
+ /* VDBE byte code will fall thru */
}
- sqlite3VdbeJumpHere(v, jmp2);
}
- if( (pTab->tabFlags & TF_Strict)!=0
- && pCol->eCType!=COLTYPE_ANY
- ){
- jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0,
- sqlite3StdTypeMap[pCol->eCType-1]);
+ if( bStrict && doTypeCheck ){
+ /* (2) Datatype must be exact for non-ANY columns in STRICT tables*/
+ static unsigned char aStdTypeMask[] = {
+ 0x1f, /* ANY */
+ 0x18, /* BLOB */
+ 0x11, /* INT */
+ 0x11, /* INTEGER */
+ 0x13, /* REAL */
+ 0x14 /* TEXT */
+ };
+ sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4);
+ assert( pCol->eCType>=1 && pCol->eCType<=sizeof(aStdTypeMask) );
+ sqlite3VdbeChangeP5(v, aStdTypeMask[pCol->eCType-1]);
VdbeCoverage(v);
zErr = sqlite3MPrintf(db, "non-%s value in %s.%s",
sqlite3StdType[pCol->eCType-1],
pTab->zName, pTab->aCol[j].zCnName);
sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
- sqlite3VdbeResolveLabel(v, doError);
- integrityCheckResultRow(v);
- sqlite3VdbeJumpHere(v, jmp2);
+ }else if( !bStrict && pCol->affinity==SQLITE_AFF_TEXT ){
+ /* (3) Datatype for TEXT columns in non-STRICT tables must be
+ ** NULL, TEXT, or BLOB. */
+ sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4);
+ sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */
+ VdbeCoverage(v);
+ zErr = sqlite3MPrintf(db, "NUMERIC value in %s.%s",
+ pTab->zName, pTab->aCol[j].zCnName);
+ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
+ }else if( !bStrict && pCol->affinity>=SQLITE_AFF_NUMERIC ){
+ /* (4) Datatype for numeric columns in non-STRICT tables must not
+ ** be a TEXT value that can be converted to numeric. */
+ sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4);
+ sqlite3VdbeChangeP5(v, 0x1b); /* NULL, INT, FLOAT, or BLOB */
+ VdbeCoverage(v);
+ if( p1>=0 ){
+ sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3);
+ }
+ sqlite3VdbeAddOp4(v, OP_Affinity, 3, 1, 0, "C", P4_STATIC);
+ sqlite3VdbeAddOp4Int(v, OP_IsType, -1, labelOk, 3, p4);
+ sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */
+ VdbeCoverage(v);
+ zErr = sqlite3MPrintf(db, "TEXT value in %s.%s",
+ pTab->zName, pTab->aCol[j].zCnName);
+ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
}
+ sqlite3VdbeResolveLabel(v, labelError);
+ integrityCheckResultRow(v);
+ sqlite3VdbeResolveLabel(v, labelOk);
}
/* Verify CHECK constraints */
if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){
@@ -133640,6 +135367,9 @@ SQLITE_PRIVATE void sqlite3Pragma(
integrityCheckResultRow(v);
sqlite3VdbeJumpHere(v, addr);
}
+ if( pPk ){
+ sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol);
+ }
}
}
}
@@ -133790,6 +135520,11 @@ SQLITE_PRIVATE void sqlite3Pragma(
aOp[1].p2 = iCookie;
aOp[1].p3 = sqlite3Atoi(zRight);
aOp[1].p5 = 1;
+ if( iCookie==BTREE_SCHEMA_VERSION && (db->flags & SQLITE_Defensive)!=0 ){
+ /* Do not allow the use of PRAGMA schema_version=VALUE in defensive
+ ** mode. Change the OP_SetCookie opcode into a no-op. */
+ aOp[1].opcode = OP_Noop;
+ }
}else{
/* Read the specified cookie value */
static const VdbeOpList readCookie[] = {
@@ -135038,15 +136773,15 @@ SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){
assert( db->pParse==pParse );
assert( pParse->nested==0 );
#ifndef SQLITE_OMIT_SHARED_CACHE
- sqlite3DbFree(db, pParse->aTableLock);
+ if( pParse->aTableLock ) sqlite3DbNNFreeNN(db, pParse->aTableLock);
#endif
while( pParse->pCleanup ){
ParseCleanup *pCleanup = pParse->pCleanup;
pParse->pCleanup = pCleanup->pNext;
pCleanup->xCleanup(db, pCleanup->pPtr);
- sqlite3DbFreeNN(db, pCleanup);
+ sqlite3DbNNFreeNN(db, pCleanup);
}
- sqlite3DbFree(db, pParse->aLabel);
+ if( pParse->aLabel ) sqlite3DbNNFreeNN(db, pParse->aLabel);
if( pParse->pConstExpr ){
sqlite3ExprListDelete(db, pParse->pConstExpr);
}
@@ -135169,7 +136904,7 @@ static int sqlite3Prepare(
sParse.disableLookaside++;
DisableLookaside;
}
- sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0;
+ sParse.prepFlags = prepFlags & 0xff;
/* Check to verify that it is possible to get a read lock on all
** database schemas. The inability to get a read lock indicates that
@@ -135210,7 +136945,9 @@ static int sqlite3Prepare(
}
}
- sqlite3VtabUnlockList(db);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ if( db->pDisconnect ) sqlite3VtabUnlockList(db);
+#endif
if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){
char *zSqlCopy;
@@ -135605,6 +137342,7 @@ struct SortCtx {
** If bFree==0, Leave the first Select object unfreed
*/
static void clearSelect(sqlite3 *db, Select *p, int bFree){
+ assert( db!=0 );
while( p ){
Select *pPrior = p->pPrior;
sqlite3ExprListDelete(db, p->pEList);
@@ -135624,7 +137362,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){
sqlite3WindowUnlinkFromSelect(p->pWin);
}
#endif
- if( bFree ) sqlite3DbFreeNN(db, p);
+ if( bFree ) sqlite3DbNNFreeNN(db, p);
p = pPrior;
bFree = 1;
}
@@ -135853,7 +137591,7 @@ SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){
*/
SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){
assert( pItem!=0 );
- assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
+ assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
if( pItem->fg.isNestedFrom ){
ExprList *pResults;
assert( pItem->pSelect!=0 );
@@ -136815,6 +138553,9 @@ static void selectInnerLoop(
testcase( eDest==SRT_Fifo );
testcase( eDest==SRT_DistFifo );
sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg);
+ if( pDest->zAffSdst ){
+ sqlite3VdbeChangeP4(v, -1, pDest->zAffSdst, nResultCol);
+ }
#ifndef SQLITE_OMIT_CTE
if( eDest==SRT_DistFifo ){
/* If the destination is DistFifo, then cursor (iParm+1) is open
@@ -137030,9 +138771,10 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
*/
SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){
if( p ){
+ assert( p->db!=0 );
assert( p->nRef>0 );
p->nRef--;
- if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p);
+ if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p);
}
}
@@ -137217,7 +138959,7 @@ static void generateSortTail(
if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
VdbeCoverage(v);
- codeOffset(v, p->iOffset, addrContinue);
+ assert( p->iLimit==0 && p->iOffset==0 );
sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
bSeq = 0;
}else{
@@ -137225,6 +138967,9 @@ static void generateSortTail(
codeOffset(v, p->iOffset, addrContinue);
iSortTab = iTab;
bSeq = 1;
+ if( p->iOffset>0 ){
+ sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
+ }
}
for(i=0, iCol=nKey+bSeq-1; ipPrior ){
- sqlite3SelectDelete(db, pSplit->pPrior);
+ sqlite3ParserAddCleanup(pParse,
+ (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
}
pSplit->pPrior = pPrior;
pPrior->pNext = pSplit;
@@ -139250,7 +140993,7 @@ static int multiSelectOrderBy(
** the left operands of a RIGHT JOIN. In either case, we need to potentially
** bypass the substituted expression with OP_IfNullRow.
**
-** Suppose the original expression integer constant. Even though the table
+** Suppose the original expression is an integer constant. Even though the table
** has the nullRow flag set, because the expression is an integer constant,
** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode
** that checks to see if the nullRow flag is set on the table. If the nullRow
@@ -139276,6 +141019,7 @@ typedef struct SubstContext {
int iNewTable; /* New table number */
int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */
ExprList *pEList; /* Replacement expressions */
+ ExprList *pCList; /* Collation sequences for replacement expr */
} SubstContext;
/* Forward Declarations */
@@ -139317,9 +141061,10 @@ static Expr *substExpr(
#endif
{
Expr *pNew;
- Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;
+ int iColumn = pExpr->iColumn;
+ Expr *pCopy = pSubst->pEList->a[iColumn].pExpr;
Expr ifNullRow;
- assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr );
+ assert( pSubst->pEList!=0 && iColumnpEList->nExpr );
assert( pExpr->pRight==0 );
if( sqlite3ExprIsVector(pCopy) ){
sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
@@ -139330,6 +141075,7 @@ static Expr *substExpr(
ifNullRow.op = TK_IF_NULL_ROW;
ifNullRow.pLeft = pCopy;
ifNullRow.iTable = pSubst->iNewTable;
+ ifNullRow.iColumn = -99;
ifNullRow.flags = EP_IfNullRow;
pCopy = &ifNullRow;
}
@@ -139356,11 +141102,16 @@ static Expr *substExpr(
/* Ensure that the expression now has an implicit collation sequence,
** just as it did when it was a column of a view or sub-query. */
- if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){
- CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
- pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
- (pColl ? pColl->zName : "BINARY")
+ {
+ CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
+ CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
+ pSubst->pCList->a[iColumn].pExpr
);
+ if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){
+ pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
+ (pColl ? pColl->zName : "BINARY")
+ );
+ }
}
ExprClearProperty(pExpr, EP_Collate);
}
@@ -139553,6 +141304,18 @@ static void renumberCursors(
}
#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
+/*
+** If pSel is not part of a compound SELECT, return a pointer to its
+** expression list. Otherwise, return a pointer to the expression list
+** of the leftmost SELECT in the compound.
+*/
+static ExprList *findLeftmostExprlist(Select *pSel){
+ while( pSel->pPrior ){
+ pSel = pSel->pPrior;
+ }
+ return pSel->pEList;
+}
+
#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
/*
** This routine attempts to flatten subqueries as a performance optimization.
@@ -139597,7 +141360,8 @@ static void renumberCursors(
** (3a) the subquery may not be a join and
** (3b) the FROM clause of the subquery may not contain a virtual
** table and
-** (3c) the outer query may not be an aggregate.
+** (**) Was: "The outer query may not have a GROUP BY." This case
+** is now managed correctly
** (3d) the outer query may not be DISTINCT.
** See also (26) for restrictions on RIGHT JOIN.
**
@@ -139651,6 +141415,11 @@ static void renumberCursors(
** (17d2) DISTINCT
** (17e) the subquery may not contain window functions, and
** (17f) the subquery must not be the RHS of a LEFT JOIN.
+** (17g) either the subquery is the first element of the outer
+** query or there are no RIGHT or FULL JOINs in any arm
+** of the subquery. (This is a duplicate of condition (27b).)
+** (17h) The corresponding result set expressions in all arms of the
+** compound must have the same affinity.
**
** The parent and sub-query may contain WHERE clauses. Subject to
** rules (11), (13) and (14), they may also contain ORDER BY,
@@ -139702,15 +141471,13 @@ static void renumberCursors(
** See also (3) for restrictions on LEFT JOIN.
**
** (27) The subquery may not contain a FULL or RIGHT JOIN unless it
-** is the first element of the parent query.
+** is the first element of the parent query. Two subcases:
+** (27a) the subquery is not a compound query.
+** (27b) the subquery is a compound query and the RIGHT JOIN occurs
+** in any arm of the compound query. (See also (17g).)
**
** (28) The subquery is not a MATERIALIZED CTE.
**
-** (29) Either the subquery is not the right-hand operand of a join with an
-** ON or USING clause nor the right-hand operand of a NATURAL JOIN, or
-** the right-most table within the FROM clause of the subquery
-** is not part of an outer join.
-**
**
** In this routine, the "p" parameter is a pointer to the outer query.
** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query
@@ -139802,16 +141569,10 @@ static int flattenSubquery(
**
** which is not at all the same thing.
**
- ** If the subquery is the right operand of a LEFT JOIN, then the outer
- ** query cannot be an aggregate. (3c) This is an artifact of the way
- ** aggregates are processed - there is no mechanism to determine if
- ** the LEFT JOIN table should be all-NULL.
- **
** See also tickets #306, #350, and #3300.
*/
if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){
if( pSubSrc->nSrc>1 /* (3a) */
- || isAgg /* (3c) */
|| IsVirtual(pSubSrc->a[0].pTab) /* (3b) */
|| (p->selFlags & SF_Distinct)!=0 /* (3d) */
|| (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */
@@ -139820,59 +141581,22 @@ static int flattenSubquery(
}
isOuterJoin = 1;
}
-#ifdef SQLITE_EXTRA_IFNULLROW
- else if( iFrom>0 && !isAgg ){
- /* Setting isOuterJoin to -1 causes OP_IfNullRow opcodes to be generated for
- ** every reference to any result column from subquery in a join, even
- ** though they are not necessary. This will stress-test the OP_IfNullRow
- ** opcode. */
- isOuterJoin = -1;
- }
-#endif
assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */
if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
- return 0; /* Restriction (27) */
+ return 0; /* Restriction (27a) */
}
if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){
return 0; /* (28) */
}
- /* Restriction (29):
- **
- ** We do not want two constraints on the same term of the flattened
- ** query where one constraint has EP_InnerON and the other is EP_OuterON.
- ** To prevent this, one or the other of the following conditions must be
- ** false:
- **
- ** (29a) The right-most entry in the FROM clause of the subquery
- ** must not be part of an outer join.
- **
- ** (29b) The subquery itself must not be the right operand of a
- ** NATURAL join or a join that as an ON or USING clause.
- **
- ** These conditions are sufficient to keep an EP_OuterON from being
- ** flattened into an EP_InnerON. Restrictions (3a) and (27) prevent
- ** an EP_InnerON from being flattened into an EP_OuterON.
- */
- if( pSubSrc->nSrc>=2
- && (pSubSrc->a[pSubSrc->nSrc-1].fg.jointype & JT_OUTER)!=0
- ){
- if( (pSubitem->fg.jointype & JT_NATURAL)!=0
- || pSubitem->fg.isUsing
- || NEVER(pSubitem->u3.pOn!=0) /* ON clause already shifted into WHERE */
- || pSubitem->fg.isOn
- ){
- return 0;
- }
- }
-
/* Restriction (17): If the sub-query is a compound SELECT, then it must
** use only the UNION ALL operator. And none of the simple select queries
** that make up the compound SELECT are allowed to be aggregate or distinct
** queries.
*/
if( pSub->pPrior ){
+ int ii;
if( pSub->pOrderBy ){
return 0; /* Restriction (20) */
}
@@ -139894,12 +141618,17 @@ static int flattenSubquery(
){
return 0;
}
+ if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
+ /* Without this restriction, the JT_LTORJ flag would end up being
+ ** omitted on left-hand tables of the right join that is being
+ ** flattened. */
+ return 0; /* Restrictions (17g), (27b) */
+ }
testcase( pSub1->pSrc->nSrc>1 );
}
/* Restriction (18). */
if( p->pOrderBy ){
- int ii;
for(ii=0; iipOrderBy->nExpr; ii++){
if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0;
}
@@ -139908,6 +141637,21 @@ static int flattenSubquery(
/* Restriction (23) */
if( (p->selFlags & SF_Recursive) ) return 0;
+ /* Restriction (17h) */
+ for(ii=0; iipEList->nExpr; ii++){
+ char aff;
+ assert( pSub->pEList->a[ii].pExpr!=0 );
+ aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr);
+ for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){
+ assert( pSub1->pEList!=0 );
+ assert( pSub1->pEList->nExpr>ii );
+ assert( pSub1->pEList->a[ii].pExpr!=0 );
+ if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){
+ return 0;
+ }
+ }
+ }
+
if( pSrc->nSrc>1 ){
if( pParse->nSelect>500 ) return 0;
if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0;
@@ -140141,6 +141885,7 @@ static int flattenSubquery(
x.iNewTable = iNewParent;
x.isOuterJoin = isOuterJoin;
x.pEList = pSub->pEList;
+ x.pCList = findLeftmostExprlist(pSub);
substSelect(&x, pParent, 0);
}
@@ -140160,7 +141905,7 @@ static int flattenSubquery(
pSub->pLimit = 0;
}
- /* Recompute the SrcList_item.colUsed masks for the flattened
+ /* Recompute the SrcItem.colUsed masks for the flattened
** tables. */
for(i=0; ia[i+iFrom]);
@@ -140550,6 +142295,13 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){
** be materialized. (This restriction is implemented in the calling
** routine.)
**
+** (8) The subquery may not be a compound that uses UNION, INTERSECT,
+** or EXCEPT. (We could, perhaps, relax this restriction to allow
+** this case if none of the comparisons operators between left and
+** right arms of the compound use a collation other than BINARY.
+** But it is a lot of work to check that case for an obscure and
+** minor optimization, so we omit it for now.)
+**
** Return 0 if no changes are made and non-zero if one or more WHERE clause
** terms are duplicated into the subquery.
*/
@@ -140569,6 +142321,10 @@ static int pushDownWhereTerms(
if( pSubq->pPrior ){
Select *pSel;
for(pSel=pSubq; pSel; pSel=pSel->pPrior){
+ u8 op = pSel->op;
+ assert( op==TK_ALL || op==TK_SELECT
+ || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT );
+ if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */
if( pSel->pWin ) return 0; /* restriction (6b) */
}
}else{
@@ -140623,6 +142379,7 @@ static int pushDownWhereTerms(
x.iNewTable = pSrc->iCursor;
x.isOuterJoin = 0;
x.pEList = pSubq->pEList;
+ x.pCList = findLeftmostExprlist(pSubq);
pNew = substExpr(&x, pNew);
#ifndef SQLITE_OMIT_WINDOWFUNC
if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){
@@ -140726,6 +142483,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){
|| p->pSrc->nSrc!=1
|| p->pSrc->a[0].pSelect
|| pAggInfo->nFunc!=1
+ || p->pHaving
){
return 0;
}
@@ -141146,9 +142904,9 @@ SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){
#endif
/*
-** The SrcList_item structure passed as the second argument represents a
+** The SrcItem structure passed as the second argument represents a
** sub-query in the FROM clause of a SELECT statement. This function
-** allocates and populates the SrcList_item.pTab object. If successful,
+** allocates and populates the SrcItem.pTab object. If successful,
** SQLITE_OK is returned. Otherwise, if an OOM error is encountered,
** SQLITE_NOMEM.
*/
@@ -141427,7 +143185,7 @@ static int selectExpander(Walker *pWalker, Select *p){
zTabName = pTab->zName;
}
if( db->mallocFailed ) break;
- assert( pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) );
+ assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) );
if( pFrom->fg.isNestedFrom ){
assert( pFrom->pSelect!=0 );
pNestedFrom = pFrom->pSelect->pEList;
@@ -141981,7 +143739,7 @@ static void havingToWhere(Parse *pParse, Select *p){
/*
** Check to see if the pThis entry of pTabList is a self-join of a prior view.
-** If it is, then return the SrcList_item for the prior view. If it is not,
+** If it is, then return the SrcItem for the prior view. If it is not,
** then return 0.
*/
static SrcItem *isSelfJoinView(
@@ -142356,7 +144114,9 @@ SQLITE_PRIVATE int sqlite3Select(
){
SELECTTRACE(0x100,pParse,p,
("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
- sqlite3ExprListDelete(db, pSub->pOrderBy);
+ sqlite3ParserAddCleanup(pParse,
+ (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
+ pSub->pOrderBy);
pSub->pOrderBy = 0;
}
@@ -142597,7 +144357,10 @@ SQLITE_PRIVATE int sqlite3Select(
}
sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem));
+ dest.zAffSdst = sqlite3TableAffinityStr(db, pItem->pTab);
sqlite3Select(pParse, pSub, &dest);
+ sqlite3DbFree(db, dest.zAffSdst);
+ dest.zAffSdst = 0;
pItem->pTab->nRowLogEst = pSub->nSelectRow;
if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1);
@@ -142717,7 +144480,7 @@ SQLITE_PRIVATE int sqlite3Select(
if( (p->selFlags & SF_FixedLimit)==0 ){
p->nSelectRow = 320; /* 4 billion rows */
}
- computeLimitRegisters(pParse, p, iEnd);
+ if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd);
if( p->iLimit==0 && sSort.addrSortIndex>=0 ){
sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen);
sSort.sortFlags |= SORTFLAG_UseSorter;
@@ -142939,8 +144702,13 @@ SQLITE_PRIVATE int sqlite3Select(
sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY");
}
for(ii=0; iinColumn; ii++){
- sqlite3DebugPrintf("agg-column[%d] iMem=%d\n",
- ii, pAggInfo->aCol[ii].iMem);
+ struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
+ sqlite3DebugPrintf(
+ "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
+ " iSorterColumn=%d\n",
+ ii, pCol->pTab ? pCol->pTab->zName : "NULL",
+ pCol->iTable, pCol->iColumn, pCol->iMem,
+ pCol->iSorterColumn);
sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
}
for(ii=0; iinFunc; ii++){
@@ -143018,7 +144786,7 @@ SQLITE_PRIVATE int sqlite3Select(
sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
SELECTTRACE(1,pParse,p,("WhereBegin\n"));
pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct,
- 0, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY)
+ p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY)
| (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0
);
if( pWInfo==0 ){
@@ -143061,15 +144829,15 @@ SQLITE_PRIVATE int sqlite3Select(
regBase = sqlite3GetTempRange(pParse, nCol);
sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0);
j = nGroupBy;
+ pAggInfo->directMode = 1;
for(i=0; inColumn; i++){
struct AggInfo_col *pCol = &pAggInfo->aCol[i];
if( pCol->iSorterColumn>=j ){
- int r1 = j + regBase;
- sqlite3ExprCodeGetColumnOfTable(v,
- pCol->pTab, pCol->iTable, pCol->iColumn, r1);
+ sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase);
j++;
}
}
+ pAggInfo->directMode = 0;
regRecord = sqlite3GetTempReg(pParse);
sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord);
sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord);
@@ -143317,7 +145085,7 @@ SQLITE_PRIVATE int sqlite3Select(
SELECTTRACE(1,pParse,p,("WhereBegin\n"));
pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy,
- pDistinct, 0, minMaxFlag|distFlag, 0);
+ pDistinct, p, minMaxFlag|distFlag, 0);
if( pWInfo==0 ){
goto select_end;
}
@@ -143961,6 +145729,23 @@ SQLITE_PRIVATE void sqlite3FinishTrigger(
Vdbe *v;
char *z;
+ /* If this is a new CREATE TABLE statement, and if shadow tables
+ ** are read-only, and the trigger makes a change to a shadow table,
+ ** then raise an error - do not allow the trigger to be created. */
+ if( sqlite3ReadOnlyShadowTables(db) ){
+ TriggerStep *pStep;
+ for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){
+ if( pStep->zTarget!=0
+ && sqlite3ShadowTableName(db, pStep->zTarget)
+ ){
+ sqlite3ErrorMsg(pParse,
+ "trigger \"%s\" may not write to shadow table \"%s\"",
+ pTrig->zName, pStep->zTarget);
+ goto triggerfinish_cleanup;
+ }
+ }
+ }
+
/* Make an entry in the sqlite_schema table */
v = sqlite3GetVdbe(pParse);
if( v==0 ) goto triggerfinish_cleanup;
@@ -144784,7 +146569,7 @@ static TriggerPrg *codeRowTrigger(
sSubParse.zAuthContext = pTrigger->zName;
sSubParse.eTriggerOp = pTrigger->op;
sSubParse.nQueryLoop = pParse->nQueryLoop;
- sSubParse.disableVtab = pParse->disableVtab;
+ sSubParse.prepFlags = pParse->prepFlags;
v = sqlite3GetVdbe(&sSubParse);
if( v ){
@@ -145130,11 +146915,14 @@ static void updateVirtualTable(
** it has been converted into REAL.
*/
SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){
+ Column *pCol;
assert( pTab!=0 );
- if( !IsView(pTab) ){
+ assert( pTab->nCol>i );
+ pCol = &pTab->aCol[i];
+ if( pCol->iDflt ){
sqlite3_value *pValue = 0;
u8 enc = ENC(sqlite3VdbeDb(v));
- Column *pCol = &pTab->aCol[i];
+ assert( !IsView(pTab) );
VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName));
assert( inCol );
sqlite3ValueFromExpr(sqlite3VdbeDb(v),
@@ -145145,7 +146933,7 @@ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){
}
}
#ifndef SQLITE_OMIT_FLOATING_POINT
- if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){
+ if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){
sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg);
}
#endif
@@ -146585,6 +148373,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(
if( pIdx->aiColumn[ii]==XN_EXPR ){
assert( pIdx->aColExpr!=0 );
assert( pIdx->aColExpr->nExpr>ii );
+ assert( pIdx->bHasExpr );
pExpr = pIdx->aColExpr->a[ii].pExpr;
if( pExpr->op!=TK_COLLATE ){
sCol[0].pLeft = pExpr;
@@ -146898,6 +148687,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
int nDb; /* Number of attached databases */
const char *zDbMain; /* Schema name of database to vacuum */
const char *zOut; /* Name of output file */
+ u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */
if( !db->autoCommit ){
sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
@@ -146969,12 +148759,17 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum(
goto end_of_vacuum;
}
db->mDbFlags |= DBFLAG_VacuumInto;
+
+ /* For a VACUUM INTO, the pager-flags are set to the same values as
+ ** they are for the database being vacuumed, except that PAGER_CACHESPILL
+ ** is always set. */
+ pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK);
}
nRes = sqlite3BtreeGetRequestedReserve(pMain);
sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size);
sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0));
- sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL);
+ sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL);
/* Begin a transaction and take an exclusive lock on the main database
** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
@@ -147487,7 +149282,8 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){
*/
SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){
assert( IsVirtual(p) );
- if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p);
+ assert( db!=0 );
+ if( db->pnBytesFreed==0 ) vtabDisconnectAll(0, p);
if( p->u.vtab.azArg ){
int i;
for(i=0; iu.vtab.nArg; i++){
@@ -148287,7 +150083,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(
if( pExpr->op!=TK_COLUMN ) return pDef;
assert( ExprUseYTab(pExpr) );
pTab = pExpr->y.pTab;
- if( pTab==0 ) return pDef;
+ if( NEVER(pTab==0) ) return pDef;
if( !IsVirtual(pTab) ) return pDef;
pVtab = sqlite3GetVTable(db, pTab)->pVtab;
assert( pVtab!=0 );
@@ -148894,7 +150690,7 @@ struct WhereAndInfo {
** between VDBE cursor numbers and bits of the bitmasks in WhereTerm.
**
** The VDBE cursor numbers are small integers contained in
-** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE
+** SrcItem.iCursor and Expr.iTable fields. For any given WHERE
** clause, the cursor numbers might not begin with 0 and they might
** contain gaps in the numbering sequence. But we want to make maximum
** use of the bits in our bitmasks. This structure provides a mapping
@@ -148965,20 +150761,6 @@ struct WhereLoopBuilder {
# define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000
#endif
-/*
-** Each instance of this object records a change to a single node
-** in an expression tree to cause that node to point to a column
-** of an index rather than an expression or a virtual column. All
-** such transformations need to be undone at the end of WHERE clause
-** processing.
-*/
-typedef struct WhereExprMod WhereExprMod;
-struct WhereExprMod {
- WhereExprMod *pNext; /* Next translation on a list of them all */
- Expr *pExpr; /* The Expr node that was transformed */
- Expr orig; /* Original value of the Expr node */
-};
-
/*
** The WHERE clause processing routine has two halves. The
** first part does the start of the WHERE loop and the second
@@ -148994,10 +150776,10 @@ struct WhereInfo {
SrcList *pTabList; /* List of tables in the join */
ExprList *pOrderBy; /* The ORDER BY clause or NULL */
ExprList *pResultSet; /* Result set of the query */
+#if WHERETRACE_ENABLED
Expr *pWhere; /* The complete WHERE clause */
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- Select *pLimit; /* Used to access LIMIT expr/registers for vtabs */
#endif
+ Select *pSelect; /* The entire SELECT statement containing WHERE */
int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */
int iContinue; /* Jump here to continue with next record */
int iBreak; /* Jump here to break out of the loop */
@@ -149016,7 +150798,6 @@ struct WhereInfo {
int iTop; /* The very beginning of the WHERE loop */
int iEndWhere; /* End of the WHERE clause itself */
WhereLoop *pLoops; /* List of all WhereLoop objects */
- WhereExprMod *pExprMods; /* Expression modifications */
WhereMemBlock *pMemToFree;/* Memory to free when this object destroyed */
Bitmask revMask; /* Mask of ORDER BY terms that need reversing */
WhereClause sWC; /* Decomposition of the WHERE clause */
@@ -149164,6 +150945,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */
#define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */
#define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */
+#define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */
#endif /* !defined(SQLITE_WHEREINT_H) */
@@ -149772,7 +151554,8 @@ static int codeEqualityTerm(
}
sqlite3ExprDelete(db, pX);
}else{
- aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq);
+ int n = sqlite3ExprVectorSize(pX->pLeft);
+ aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n));
eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab);
}
pX = pExpr;
@@ -150042,7 +151825,7 @@ static void whereLikeOptimizationStringFixup(
if( pTerm->wtFlags & TERM_LIKEOPT ){
VdbeOp *pOp;
assert( pLevel->iLikeRepCntr>0 );
- pOp = sqlite3VdbeGetOp(v, -1);
+ pOp = sqlite3VdbeGetLastOp(v);
assert( pOp!=0 );
assert( pOp->opcode==OP_String8
|| pTerm->pWC->pWInfo->pParse->db->mallocFailed );
@@ -150366,143 +152149,6 @@ static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){
}
}
-/* An instance of the IdxExprTrans object carries information about a
-** mapping from an expression on table columns into a column in an index
-** down through the Walker.
-*/
-typedef struct IdxExprTrans {
- Expr *pIdxExpr; /* The index expression */
- int iTabCur; /* The cursor of the corresponding table */
- int iIdxCur; /* The cursor for the index */
- int iIdxCol; /* The column for the index */
- int iTabCol; /* The column for the table */
- WhereInfo *pWInfo; /* Complete WHERE clause information */
- sqlite3 *db; /* Database connection (for malloc()) */
-} IdxExprTrans;
-
-/*
-** Preserve pExpr on the WhereETrans list of the WhereInfo.
-*/
-static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){
- WhereExprMod *pNew;
- pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew));
- if( pNew==0 ) return;
- pNew->pNext = pTrans->pWInfo->pExprMods;
- pTrans->pWInfo->pExprMods = pNew;
- pNew->pExpr = pExpr;
- memcpy(&pNew->orig, pExpr, sizeof(*pExpr));
-}
-
-/* The walker node callback used to transform matching expressions into
-** a reference to an index column for an index on an expression.
-**
-** If pExpr matches, then transform it into a reference to the index column
-** that contains the value of pExpr.
-*/
-static int whereIndexExprTransNode(Walker *p, Expr *pExpr){
- IdxExprTrans *pX = p->u.pIdxTrans;
- if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){
- pExpr = sqlite3ExprSkipCollate(pExpr);
- preserveExpr(pX, pExpr);
- pExpr->affExpr = sqlite3ExprAffinity(pExpr);
- pExpr->op = TK_COLUMN;
- pExpr->iTable = pX->iIdxCur;
- pExpr->iColumn = pX->iIdxCol;
- testcase( ExprHasProperty(pExpr, EP_Unlikely) );
- ExprClearProperty(pExpr, EP_Skip|EP_Unlikely|EP_WinFunc|EP_Subrtn);
- pExpr->y.pTab = 0;
- return WRC_Prune;
- }else{
- return WRC_Continue;
- }
-}
-
-#ifndef SQLITE_OMIT_GENERATED_COLUMNS
-/* A walker node callback that translates a column reference to a table
-** into a corresponding column reference of an index.
-*/
-static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){
- if( pExpr->op==TK_COLUMN ){
- IdxExprTrans *pX = p->u.pIdxTrans;
- if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){
- assert( ExprUseYTab(pExpr) && pExpr->y.pTab!=0 );
- preserveExpr(pX, pExpr);
- pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn);
- pExpr->iTable = pX->iIdxCur;
- pExpr->iColumn = pX->iIdxCol;
- pExpr->y.pTab = 0;
- }
- }
- return WRC_Continue;
-}
-#endif /* SQLITE_OMIT_GENERATED_COLUMNS */
-
-/*
-** For an indexes on expression X, locate every instance of expression X
-** in pExpr and change that subexpression into a reference to the appropriate
-** column of the index.
-**
-** 2019-10-24: Updated to also translate references to a VIRTUAL column in
-** the table into references to the corresponding (stored) column of the
-** index.
-*/
-static void whereIndexExprTrans(
- Index *pIdx, /* The Index */
- int iTabCur, /* Cursor of the table that is being indexed */
- int iIdxCur, /* Cursor of the index itself */
- WhereInfo *pWInfo /* Transform expressions in this WHERE clause */
-){
- int iIdxCol; /* Column number of the index */
- ExprList *aColExpr; /* Expressions that are indexed */
- Table *pTab;
- Walker w;
- IdxExprTrans x;
- aColExpr = pIdx->aColExpr;
- if( aColExpr==0 && !pIdx->bHasVCol ){
- /* The index does not reference any expressions or virtual columns
- ** so no translations are needed. */
- return;
- }
- pTab = pIdx->pTable;
- memset(&w, 0, sizeof(w));
- w.u.pIdxTrans = &x;
- x.iTabCur = iTabCur;
- x.iIdxCur = iIdxCur;
- x.pWInfo = pWInfo;
- x.db = pWInfo->pParse->db;
- for(iIdxCol=0; iIdxColnColumn; iIdxCol++){
- i16 iRef = pIdx->aiColumn[iIdxCol];
- if( iRef==XN_EXPR ){
- assert( aColExpr!=0 && aColExpr->a[iIdxCol].pExpr!=0 );
- x.pIdxExpr = aColExpr->a[iIdxCol].pExpr;
- if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue;
- w.xExprCallback = whereIndexExprTransNode;
-#ifndef SQLITE_OMIT_GENERATED_COLUMNS
- }else if( iRef>=0
- && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0
- && ((pTab->aCol[iRef].colFlags & COLFLAG_HASCOLL)==0
- || sqlite3StrICmp(sqlite3ColumnColl(&pTab->aCol[iRef]),
- sqlite3StrBINARY)==0)
- ){
- /* Check to see if there are direct references to generated columns
- ** that are contained in the index. Pulling the generated column
- ** out of the index is an optimization only - the main table is always
- ** available if the index cannot be used. To avoid unnecessary
- ** complication, omit this optimization if the collating sequence for
- ** the column is non-standard */
- x.iTabCol = iRef;
- w.xExprCallback = whereIndexExprTransColumn;
-#endif /* SQLITE_OMIT_GENERATED_COLUMNS */
- }else{
- continue;
- }
- x.iIdxCol = iIdxCol;
- sqlite3WalkExpr(&w, pWInfo->pWhere);
- sqlite3WalkExprList(&w, pWInfo->pOrderBy);
- sqlite3WalkExprList(&w, pWInfo->pResultSet);
- }
-}
-
/*
** The pTruth expression is always true because it is the WHERE clause
** a partial index that is driving a query loop. Look through all of the
@@ -150571,6 +152217,8 @@ static SQLITE_NOINLINE void filterPullDown(
testcase( pTerm->wtFlags & TERM_VIRTUAL );
regRowid = sqlite3GetTempReg(pParse);
regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid);
+ sqlite3VdbeAddOp2(pParse->pVdbe, OP_MustBeInt, regRowid, addrNxt);
+ VdbeCoverage(pParse->pVdbe);
sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter,
addrNxt, regRowid, 1);
VdbeCoverage(pParse->pVdbe);
@@ -150722,9 +152370,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
&& pLoop->u.vtab.bOmitOffset
){
assert( pTerm->eOperator==WO_AUX );
- assert( pWInfo->pLimit!=0 );
- assert( pWInfo->pLimit->iOffset>0 );
- sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pLimit->iOffset);
+ assert( pWInfo->pSelect!=0 );
+ assert( pWInfo->pSelect->iOffset>0 );
+ sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pSelect->iOffset);
VdbeComment((v,"Zero OFFSET counter"));
}
}
@@ -150832,6 +152480,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg);
addrNxt = pLevel->addrNxt;
if( pLevel->regFilter ){
+ sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt);
+ VdbeCoverage(v);
sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt,
iRowidReg, 1);
VdbeCoverage(v);
@@ -151183,6 +152833,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
** guess. */
addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan,
(pIdx->aiRowLogEst[0]+9)/10);
+ if( pRangeStart ){
+ sqlite3VdbeChangeP5(v, 1);
+ sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1);
+ addrSeekScan = 0;
+ }
VdbeCoverage(v);
}
sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
@@ -151258,8 +152913,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
}
nConstraint++;
}
- sqlite3DbFree(db, zStartAff);
- sqlite3DbFree(db, zEndAff);
+ if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff);
+ if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff);
/* Top of the loop body */
if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v);
@@ -151321,27 +152976,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
}
if( pLevel->iLeftJoin==0 ){
- /* If pIdx is an index on one or more expressions, then look through
- ** all the expressions in pWInfo and try to transform matching expressions
- ** into reference to index columns. Also attempt to translate references
- ** to virtual columns in the table into references to (stored) columns
- ** of the index.
- **
- ** Do not do this for the RHS of a LEFT JOIN. This is because the
- ** expression may be evaluated after OP_NullRow has been executed on
- ** the cursor. In this case it is important to do the full evaluation,
- ** as the result of the expression may not be NULL, even if all table
- ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a
- **
- ** Also, do not do this when processing one index an a multi-index
- ** OR clause, since the transformation will become invalid once we
- ** move forward to the next index.
- ** https://sqlite.org/src/info/4e8e4857d32d401f
- */
- if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ){
- whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
- }
-
/* If a partial index is driving the loop, try to eliminate WHERE clause
** terms from the query that must be true due to the WHERE clause of
** the partial index.
@@ -151454,7 +153088,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
int nNotReady; /* The number of notReady tables */
SrcItem *origSrc; /* Original list of tables */
nNotReady = pWInfo->nLevel - iLevel - 1;
- pOrTab = sqlite3StackAllocRaw(db,
+ pOrTab = sqlite3DbMallocRawNN(db,
sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0]));
if( pOrTab==0 ) return notReady;
pOrTab->nAlloc = (u8)(nNotReady + 1);
@@ -151707,7 +153341,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
assert( pLevel->op==OP_Return );
pLevel->p2 = sqlite3VdbeCurrentAddr(v);
- if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); }
+ if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); }
if( !untestedTerms ) disableTerm(pLevel, pTerm);
}else
#endif /* SQLITE_OMIT_OR_OPTIMIZATION */
@@ -152335,7 +153969,7 @@ static int isLikeOrGlob(
if( pLeft->op!=TK_COLUMN
|| sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT
|| (ALWAYS( ExprUseYTab(pLeft) )
- && pLeft->y.pTab
+ && ALWAYS(pLeft->y.pTab)
&& IsVirtual(pLeft->y.pTab)) /* Might be numeric */
){
int isNum;
@@ -152452,8 +154086,7 @@ static int isAuxiliaryVtabOperator(
** MATCH(expression,vtab_column)
*/
pCol = pList->a[1].pExpr;
- assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) );
- testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 );
+ assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) );
if( ExprIsVtab(pCol) ){
for(i=0; ia[0].pExpr;
assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) );
- testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 );
+ assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) );
if( ExprIsVtab(pCol) ){
sqlite3_vtab *pVtab;
sqlite3_module *pMod;
@@ -152503,13 +154136,12 @@ static int isAuxiliaryVtabOperator(
int res = 0;
Expr *pLeft = pExpr->pLeft;
Expr *pRight = pExpr->pRight;
- assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) );
- testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 );
+ assert( pLeft->op!=TK_COLUMN || (ExprUseYTab(pLeft) && pLeft->y.pTab!=0) );
if( ExprIsVtab(pLeft) ){
res++;
}
- assert( pRight==0 || pRight->op!=TK_COLUMN || ExprUseYTab(pRight) );
- testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 );
+ assert( pRight==0 || pRight->op!=TK_COLUMN
+ || (ExprUseYTab(pRight) && pRight->y.pTab!=0) );
if( pRight && ExprIsVtab(pRight) ){
res++;
SWAP(Expr*, pLeft, pRight);
@@ -153058,6 +154690,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2(
if( pIdx->aColExpr==0 ) continue;
for(i=0; inKeyCol; i++){
if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
+ assert( pIdx->bHasExpr );
if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
aiCurCol[0] = iCur;
aiCurCol[1] = XN_EXPR;
@@ -153671,9 +155304,9 @@ static void whereAddLimitExpr(
** exist only so that they may be passed to the xBestIndex method of the
** single virtual table in the FROM clause of the SELECT.
*/
-SQLITE_PRIVATE void sqlite3WhereAddLimit(WhereClause *pWC, Select *p){
- assert( p==0 || (p->pGroupBy==0 && (p->selFlags & SF_Aggregate)==0) );
- if( (p && p->pLimit) /* 1 */
+SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){
+ assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */
+ if( p->pGroupBy==0
&& (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */
&& (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */
){
@@ -154666,6 +156299,43 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
#define whereTraceIndexInfoOutputs(A)
#endif
+/*
+** We know that pSrc is an operand of an outer join. Return true if
+** pTerm is a constraint that is compatible with that join.
+**
+** pTerm must be EP_OuterON if pSrc is the right operand of an
+** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
+** is the left operand of a RIGHT join.
+**
+** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
+** for an example of a WHERE clause constraints that may not be used on
+** the right table of a RIGHT JOIN because the constraint implies a
+** not-NULL condition on the left table of the RIGHT JOIN.
+*/
+static int constraintCompatibleWithOuterJoin(
+ const WhereTerm *pTerm, /* WHERE clause term to check */
+ const SrcItem *pSrc /* Table we are trying to access */
+){
+ assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
+ testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
+ testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
+ testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
+ testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
+ if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
+ || pTerm->pExpr->w.iJoin != pSrc->iCursor
+ ){
+ return 0;
+ }
+ if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
+ && ExprHasProperty(pTerm->pExpr, EP_InnerON)
+ ){
+ return 0;
+ }
+ return 1;
+}
+
+
+
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
/*
** Return TRUE if the WHERE clause term pTerm is of a form where it
@@ -154681,16 +156351,10 @@ static int termCanDriveIndex(
if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
- if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
- testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
- testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
- if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
- || pTerm->pExpr->w.iJoin != pSrc->iCursor
- ){
- return 0; /* See tag-20191211-001 */
- }
+ if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
+ && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
+ ){
+ return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
}
if( (pTerm->prereqRight & notReady)!=0 ) return 0;
assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
@@ -155102,22 +156766,10 @@ static sqlite3_index_info *allocateIndexInfo(
assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
assert( pTerm->u.x.leftColumn>=XN_ROWID );
assert( pTerm->u.x.leftColumnnCol );
-
- /* tag-20191211-002: WHERE-clause constraints are not useful to the
- ** right-hand table of a LEFT JOIN nor to the either table of a
- ** RIGHT JOIN. See tag-20191211-001 for the
- ** equivalent restriction for ordinary tables. */
- if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT );
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
- testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) );
- testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
- if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
- || pTerm->pExpr->w.iJoin != pSrc->iCursor
- ){
- continue;
- }
+ if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
+ && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
+ ){
+ continue;
}
nTerm++;
pTerm->wtFlags |= TERM_OK;
@@ -155356,7 +157008,7 @@ static int whereKeyStats(
#endif
assert( pRec!=0 );
assert( pIdx->nSample>0 );
- assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol );
+ assert( pRec->nField>0 );
/* Do a binary search to find the first sample greater than or equal
** to pRec. If pRec contains a single field, the set of samples to search
@@ -155402,7 +157054,7 @@ static int whereKeyStats(
** it is extended to two fields. The duplicates that this creates do not
** cause any problems.
*/
- nField = pRec->nField;
+ nField = MIN(pRec->nField, pIdx->nSample);
iCol = 0;
iSample = pIdx->nSample * nField;
do{
@@ -155490,7 +157142,7 @@ static int whereKeyStats(
** is larger than all samples in the array. */
tRowcnt iUpper, iGap;
if( i>=pIdx->nSample ){
- iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]);
+ iUpper = pIdx->nRowEst0;
}else{
iUpper = aSample[i].anLt[iCol];
}
@@ -156119,12 +157771,18 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
}
/*
-** Deallocate internal memory used by a WhereLoop object
+** Deallocate internal memory used by a WhereLoop object. Leave the
+** object in an initialized state, as if it had been newly allocated.
*/
static void whereLoopClear(sqlite3 *db, WhereLoop *p){
- if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
+ if( p->aLTerm!=p->aLTermSpace ){
+ sqlite3DbFreeNN(db, p->aLTerm);
+ p->aLTerm = p->aLTermSpace;
+ p->nLSlot = ArraySize(p->aLTermSpace);
+ }
whereLoopClearUnion(db, p);
- whereLoopInit(p);
+ p->nLTerm = 0;
+ p->wsFlags = 0;
}
/*
@@ -156148,7 +157806,9 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
*/
static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
whereLoopClearUnion(db, pTo);
- if( whereLoopResize(db, pTo, pFrom->nLTerm) ){
+ if( pFrom->nLTerm > pTo->nLSlot
+ && whereLoopResize(db, pTo, pFrom->nLTerm)
+ ){
memset(pTo, 0, WHERE_LOOP_XFER_SZ);
return SQLITE_NOMEM_BKPT;
}
@@ -156166,8 +157826,9 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
** Delete a WhereLoop object
*/
static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
+ assert( db!=0 );
whereLoopClear(db, p);
- sqlite3DbFreeNN(db, p);
+ sqlite3DbNNFreeNN(db, p);
}
/*
@@ -156175,30 +157836,19 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
*/
static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
assert( pWInfo!=0 );
+ assert( db!=0 );
sqlite3WhereClauseClear(&pWInfo->sWC);
while( pWInfo->pLoops ){
WhereLoop *p = pWInfo->pLoops;
pWInfo->pLoops = p->pNextLoop;
whereLoopDelete(db, p);
}
- assert( pWInfo->pExprMods==0 );
while( pWInfo->pMemToFree ){
WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
- sqlite3DbFreeNN(db, pWInfo->pMemToFree);
+ sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
pWInfo->pMemToFree = pNext;
}
- sqlite3DbFreeNN(db, pWInfo);
-}
-
-/* Undo all Expr node modifications
-*/
-static void whereUndoExprMods(WhereInfo *pWInfo){
- while( pWInfo->pExprMods ){
- WhereExprMod *p = pWInfo->pExprMods;
- pWInfo->pExprMods = p->pNext;
- memcpy(p->pExpr, &p->orig, sizeof(p->orig));
- sqlite3DbFree(pWInfo->pParse->db, p);
- }
+ sqlite3DbNNFreeNN(db, pWInfo);
}
/*
@@ -156765,32 +158415,11 @@ static int whereLoopAddBtreeIndex(
** to mix with a lower range bound from some other source */
if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
- /* tag-20191211-001: Do not allow constraints from the WHERE clause to
- ** be used by the right table of a LEFT JOIN nor by the left table of a
- ** RIGHT JOIN. Only constraints in the ON clause are allowed.
- ** See tag-20191211-002 for the vtab equivalent.
- **
- ** 2022-06-06: See https://sqlite.org/forum/forumpost/206d99a16dd9212f
- ** for an example of a WHERE clause constraints that may not be used on
- ** the right table of a RIGHT JOIN because the constraint implies a
- ** not-NULL condition on the left table of the RIGHT JOIN.
- **
- ** 2022-06-10: The same condition applies to termCanDriveIndex() above.
- ** https://sqlite.org/forum/forumpost/51e6959f61
- */
- if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT );
- testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
- testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
- testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
- if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
- || pTerm->pExpr->w.iJoin != pSrc->iCursor
- ){
- continue;
- }
+ if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
+ && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
+ ){
+ continue;
}
-
if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
}else{
@@ -156801,7 +158430,11 @@ static int whereLoopAddBtreeIndex(
pNew->u.btree.nBtm = saved_nBtm;
pNew->u.btree.nTop = saved_nTop;
pNew->nLTerm = saved_nLTerm;
- if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
+ if( pNew->nLTerm>=pNew->nLSlot
+ && whereLoopResize(db, pNew, pNew->nLTerm+1)
+ ){
+ break; /* OOM while trying to enlarge the pNew->aLTerm array */
+ }
pNew->aLTerm[pNew->nLTerm++] = pTerm;
pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
@@ -156894,38 +158527,39 @@ static int whereLoopAddBtreeIndex(
if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
}else if( eOp & WO_ISNULL ){
pNew->wsFlags |= WHERE_COLUMN_NULL;
- }else if( eOp & (WO_GT|WO_GE) ){
- testcase( eOp & WO_GT );
- testcase( eOp & WO_GE );
- pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
- pNew->u.btree.nBtm = whereRangeVectorLen(
- pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
- );
- pBtm = pTerm;
- pTop = 0;
- if( pTerm->wtFlags & TERM_LIKEOPT ){
- /* Range constraints that come from the LIKE optimization are
- ** always used in pairs. */
- pTop = &pTerm[1];
- assert( (pTop-(pTerm->pWC->a))pWC->nTerm );
- assert( pTop->wtFlags & TERM_LIKEOPT );
- assert( pTop->eOperator==WO_LT );
- if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
- pNew->aLTerm[pNew->nLTerm++] = pTop;
- pNew->wsFlags |= WHERE_TOP_LIMIT;
- pNew->u.btree.nTop = 1;
- }
}else{
- assert( eOp & (WO_LT|WO_LE) );
- testcase( eOp & WO_LT );
- testcase( eOp & WO_LE );
- pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
- pNew->u.btree.nTop = whereRangeVectorLen(
+ int nVecLen = whereRangeVectorLen(
pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
);
- pTop = pTerm;
- pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
- pNew->aLTerm[pNew->nLTerm-2] : 0;
+ if( eOp & (WO_GT|WO_GE) ){
+ testcase( eOp & WO_GT );
+ testcase( eOp & WO_GE );
+ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
+ pNew->u.btree.nBtm = nVecLen;
+ pBtm = pTerm;
+ pTop = 0;
+ if( pTerm->wtFlags & TERM_LIKEOPT ){
+ /* Range constraints that come from the LIKE optimization are
+ ** always used in pairs. */
+ pTop = &pTerm[1];
+ assert( (pTop-(pTerm->pWC->a))pWC->nTerm );
+ assert( pTop->wtFlags & TERM_LIKEOPT );
+ assert( pTop->eOperator==WO_LT );
+ if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
+ pNew->aLTerm[pNew->nLTerm++] = pTop;
+ pNew->wsFlags |= WHERE_TOP_LIMIT;
+ pNew->u.btree.nTop = 1;
+ }
+ }else{
+ assert( eOp & (WO_LT|WO_LE) );
+ testcase( eOp & WO_LT );
+ testcase( eOp & WO_LE );
+ pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
+ pNew->u.btree.nTop = nVecLen;
+ pTop = pTerm;
+ pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
+ pNew->aLTerm[pNew->nLTerm-2] : 0;
+ }
}
/* At this point pNew->nOut is set to the number of rows expected to
@@ -157169,6 +158803,94 @@ static int whereUsablePartialIndex(
return 0;
}
+/*
+** Structure passed to the whereIsCoveringIndex Walker callback.
+*/
+struct CoveringIndexCheck {
+ Index *pIdx; /* The index */
+ int iTabCur; /* Cursor number for the corresponding table */
+};
+
+/*
+** Information passed in is pWalk->u.pCovIdxCk. Call is pCk.
+**
+** If the Expr node references the table with cursor pCk->iTabCur, then
+** make sure that column is covered by the index pCk->pIdx. We know that
+** all columns less than 63 (really BMS-1) are covered, so we don't need
+** to check them. But we do need to check any column at 63 or greater.
+**
+** If the index does not cover the column, then set pWalk->eCode to
+** non-zero and return WRC_Abort to stop the search.
+**
+** If this node does not disprove that the index can be a covering index,
+** then just return WRC_Continue, to continue the search.
+*/
+static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
+ int i; /* Loop counter */
+ const Index *pIdx; /* The index of interest */
+ const i16 *aiColumn; /* Columns contained in the index */
+ u16 nColumn; /* Number of columns in the index */
+ if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue;
+ if( pExpr->iColumn<(BMS-1) ) return WRC_Continue;
+ if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue;
+ pIdx = pWalk->u.pCovIdxCk->pIdx;
+ aiColumn = pIdx->aiColumn;
+ nColumn = pIdx->nColumn;
+ for(i=0; iiColumn ) return WRC_Continue;
+ }
+ pWalk->eCode = 1;
+ return WRC_Abort;
+}
+
+
+/*
+** pIdx is an index that covers all of the low-number columns used by
+** pWInfo->pSelect (columns from 0 through 62). But there are columns
+** in pWInfo->pSelect beyond 62. This routine tries to answer the question
+** of whether pIdx covers *all* columns in the query.
+**
+** Return 0 if pIdx is a covering index. Return non-zero if pIdx is
+** not a covering index or if we are unable to determine if pIdx is a
+** covering index.
+**
+** This routine is an optimization. It is always safe to return non-zero.
+** But returning zero when non-zero should have been returned can lead to
+** incorrect bytecode and assertion faults.
+*/
+static SQLITE_NOINLINE u32 whereIsCoveringIndex(
+ WhereInfo *pWInfo, /* The WHERE clause context */
+ Index *pIdx, /* Index that is being tested */
+ int iTabCur /* Cursor for the table being indexed */
+){
+ int i;
+ struct CoveringIndexCheck ck;
+ Walker w;
+ if( pWInfo->pSelect==0 ){
+ /* We don't have access to the full query, so we cannot check to see
+ ** if pIdx is covering. Assume it is not. */
+ return 1;
+ }
+ for(i=0; inColumn; i++){
+ if( pIdx->aiColumn[i]>=BMS-1 ) break;
+ }
+ if( i>=pIdx->nColumn ){
+ /* pIdx does not index any columns greater than 62, but we know from
+ ** colMask that columns greater than 62 are used, so this is not a
+ ** covering index */
+ return 1;
+ }
+ ck.pIdx = pIdx;
+ ck.iTabCur = iTabCur;
+ memset(&w, 0, sizeof(w));
+ w.xExprCallback = whereIsCoveringIndexWalkCallback;
+ w.xSelectCallback = sqlite3SelectWalkNoop;
+ w.u.pCovIdxCk = &ck;
+ w.eCode = 0;
+ sqlite3WalkSelect(&w, pWInfo->pSelect);
+ return w.eCode;
+}
+
/*
** Add all WhereLoop objects for a single table of the join where the table
** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
@@ -157371,6 +159093,9 @@ static int whereLoopAddBtree(
#else
pNew->rRun = rSize + 16;
#endif
+ if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){
+ pNew->wsFlags |= WHERE_VIEWSCAN;
+ }
ApplyCostMultiplier(pNew->rRun, pTab->costMult);
whereLoopOutputAdjust(pWC, pNew, rSize);
rc = whereLoopInsert(pBuilder, pNew);
@@ -157383,6 +159108,9 @@ static int whereLoopAddBtree(
m = 0;
}else{
m = pSrc->colUsed & pProbe->colNotIdxed;
+ if( m==TOPBIT ){
+ m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
+ }
pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED;
}
@@ -158091,12 +159819,19 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
sqlite3 *db = pWInfo->pParse->db;
int rc = SQLITE_OK;
int bFirstPastRJ = 0;
+ int hasRightJoin = 0;
WhereLoop *pNew;
/* Loop over the tables in the join, from left to right */
pNew = pBuilder->pNew;
- whereLoopInit(pNew);
+
+ /* Verify that pNew has already been initialized */
+ assert( pNew->nLTerm==0 );
+ assert( pNew->wsFlags==0 );
+ assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
+ assert( pNew->aLTerm!=0 );
+
pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
for(iTab=0, pItem=pTabList->a; pItemfg.jointype & JT_LTORJ ) hasRightJoin = 1;
mPrereq |= mPrior;
bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
+ }else if( !hasRightJoin ){
+ mPrereq = 0;
}
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pItem->pTab) ){
@@ -158600,7 +160336,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
int mxChoice; /* Maximum number of simultaneous paths tracked */
int nLoop; /* Number of terms in the join */
Parse *pParse; /* Parsing context */
- sqlite3 *db; /* The database connection */
int iLoop; /* Loop counter over the terms of the join */
int ii, jj; /* Loop counters */
int mxI = 0; /* Index of next entry to replace */
@@ -158619,7 +160354,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
int nSpace; /* Bytes of space allocated at pSpace */
pParse = pWInfo->pParse;
- db = pParse->db;
nLoop = pWInfo->nLevel;
/* TUNING: For simple queries, only the best path is tracked.
** For 2-way joins, the 5 best paths are followed.
@@ -158642,7 +160376,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
/* Allocate and initialize space for aTo, aFrom and aSortCost[] */
nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
nSpace += sizeof(LogEst) * nOrderBy;
- pSpace = sqlite3DbMallocRawNN(db, nSpace);
+ pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
aTo = (WherePath*)pSpace;
aFrom = aTo+mxChoice;
@@ -158692,9 +160426,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
LogEst rCost; /* Cost of path (pFrom+pWLoop) */
LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
- i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */
+ i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
Bitmask maskNew; /* Mask of src visited by (..) */
- Bitmask revMask = 0; /* Mask of rev-order loops for (..) */
+ Bitmask revMask; /* Mask of rev-order loops for (..) */
if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
@@ -158713,7 +160447,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
nOut = pFrom->nRow + pWLoop->nOut;
maskNew = pFrom->maskLoop | pWLoop->maskSelf;
+ isOrdered = pFrom->isOrdered;
if( isOrdered<0 ){
+ revMask = 0;
isOrdered = wherePathSatisfiesOrderBy(pWInfo,
pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
iLoop, pWLoop, &revMask);
@@ -158741,6 +160477,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */
}
+ /* TUNING: A full-scan of a VIEW or subquery in the outer loop
+ ** is not so bad. */
+ if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){
+ rCost += -10;
+ nOut += -30;
+ }
+
/* Check to see if pWLoop should be added to the set of
** mxChoice best-so-far paths.
**
@@ -158891,7 +160634,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
if( nFrom==0 ){
sqlite3ErrorMsg(pParse, "no query solution");
- sqlite3DbFreeNN(db, pSpace);
+ sqlite3StackFreeNN(pParse->db, pSpace);
return SQLITE_ERROR;
}
@@ -158973,7 +160716,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
pWInfo->nRowOut = pFrom->nRow;
/* Free temporary memory and return success */
- sqlite3DbFreeNN(db, pSpace);
+ sqlite3StackFreeNN(pParse->db, pSpace);
return SQLITE_OK;
}
@@ -159272,6 +161015,77 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
}
}
+/*
+** This is an sqlite3ParserAddCleanup() callback that is invoked to
+** free the Parse->pIdxExpr list when the Parse object is destroyed.
+*/
+static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
+ Parse *pParse = (Parse*)pObject;
+ while( pParse->pIdxExpr!=0 ){
+ IndexedExpr *p = pParse->pIdxExpr;
+ pParse->pIdxExpr = p->pIENext;
+ sqlite3ExprDelete(db, p->pExpr);
+ sqlite3DbFreeNN(db, p);
+ }
+}
+
+/*
+** The index pIdx is used by a query and contains one or more expressions.
+** In other words pIdx is an index on an expression. iIdxCur is the cursor
+** number for the index and iDataCur is the cursor number for the corresponding
+** table.
+**
+** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for
+** each of the expressions in the index so that the expression code generator
+** will know to replace occurrences of the indexed expression with
+** references to the corresponding column of the index.
+*/
+static SQLITE_NOINLINE void whereAddIndexedExpr(
+ Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */
+ Index *pIdx, /* The index-on-expression that contains the expressions */
+ int iIdxCur, /* Cursor number for pIdx */
+ SrcItem *pTabItem /* The FROM clause entry for the table */
+){
+ int i;
+ IndexedExpr *p;
+ Table *pTab;
+ assert( pIdx->bHasExpr );
+ pTab = pIdx->pTable;
+ for(i=0; inColumn; i++){
+ Expr *pExpr;
+ int j = pIdx->aiColumn[i];
+ int bMaybeNullRow;
+ if( j==XN_EXPR ){
+ pExpr = pIdx->aColExpr->a[i].pExpr;
+ testcase( pTabItem->fg.jointype & JT_LEFT );
+ testcase( pTabItem->fg.jointype & JT_RIGHT );
+ testcase( pTabItem->fg.jointype & JT_LTORJ );
+ bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
+ }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
+ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
+ bMaybeNullRow = 0;
+ }else{
+ continue;
+ }
+ if( sqlite3ExprIsConstant(pExpr) ) continue;
+ p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
+ if( p==0 ) break;
+ p->pIENext = pParse->pIdxExpr;
+ p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
+ p->iDataCur = pTabItem->iCursor;
+ p->iIdxCur = iIdxCur;
+ p->iIdxCol = i;
+ p->bMaybeNullRow = bMaybeNullRow;
+#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
+ p->zIdxName = pIdx->zName;
+#endif
+ pParse->pIdxExpr = p;
+ if( p->pIENext==0 ){
+ sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse);
+ }
+ }
+}
+
/*
** Generate the beginning of the loop used for WHERE clause processing.
** The return value is a pointer to an opaque structure that contains
@@ -159366,7 +161180,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
Expr *pWhere, /* The WHERE clause */
ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
- Select *pLimit, /* Use this LIMIT/OFFSET clause, if any */
+ Select *pSelect, /* The entire SELECT statement */
u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
** If WHERE_USE_LIMIT, then the limit amount */
@@ -159435,7 +161249,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
pWInfo->pParse = pParse;
pWInfo->pTabList = pTabList;
pWInfo->pOrderBy = pOrderBy;
+#if WHERETRACE_ENABLED
pWInfo->pWhere = pWhere;
+#endif
pWInfo->pResultSet = pResultSet;
pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
pWInfo->nLevel = nTabList;
@@ -159443,9 +161259,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
pWInfo->wctrlFlags = wctrlFlags;
pWInfo->iLimit = iAuxArg;
pWInfo->savedNQueryLoop = pParse->nQueryLoop;
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- pWInfo->pLimit = pLimit;
-#endif
+ pWInfo->pSelect = pSelect;
memset(&pWInfo->nOBSat, 0,
offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
@@ -159514,7 +161328,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
/* Analyze all of the subexpressions. */
sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
- sqlite3WhereAddLimit(&pWInfo->sWC, pLimit);
+ if( pSelect && pSelect->pLimit ){
+ sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
+ }
if( pParse->nErr ) goto whereBeginError;
/* Special case: WHERE terms that do not refer to any tables in the join
@@ -159817,6 +161633,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
op = OP_ReopenIdx;
}else{
iIndexCur = pParse->nTab++;
+ if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
+ whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
+ }
}
pLevel->iIdxCur = iIndexCur;
assert( pIx!=0 );
@@ -159939,8 +161758,6 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
/* Jump here if malloc fails */
whereBeginError:
if( pWInfo ){
- testcase( pWInfo->pExprMods!=0 );
- whereUndoExprMods(pWInfo);
pParse->nQueryLoop = pWInfo->savedNQueryLoop;
whereInfoFree(db, pWInfo);
}
@@ -160159,7 +161976,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){
}
assert( pWInfo->nLevel<=pTabList->nSrc );
- if( pWInfo->pExprMods ) whereUndoExprMods(pWInfo);
for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){
int k, last;
VdbeOp *pOp, *pLastOp;
@@ -160213,6 +162029,16 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){
}else{
last = pWInfo->iEndWhere;
}
+ if( pIdx->bHasExpr ){
+ IndexedExpr *p = pParse->pIdxExpr;
+ while( p ){
+ if( p->iIdxCur==pLevel->iIdxCur ){
+ p->iDataCur = -1;
+ p->iIdxCur = -1;
+ }
+ p = p->pIENext;
+ }
+ }
k = pLevel->addrBody + 1;
#ifdef SQLITE_DEBUG
if( db->flags & SQLITE_VdbeAddopTrace ){
@@ -161206,7 +163032,6 @@ static ExprList *exprListAppendList(
for(i=0; inExpr; i++){
sqlite3 *db = pParse->db;
Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0);
- assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) );
if( db->mallocFailed ){
sqlite3ExprDelete(db, pDup);
break;
@@ -162477,10 +164302,9 @@ static void windowCodeRangeTest(
/* This block runs if reg1 is not NULL, but reg2 is. */
sqlite3VdbeJumpHere(v, addr);
- sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v);
- if( op==OP_Gt || op==OP_Ge ){
- sqlite3VdbeChangeP2(v, -1, addrDone);
- }
+ sqlite3VdbeAddOp2(v, OP_IsNull, reg2,
+ (op==OP_Gt || op==OP_Ge) ? addrDone : lbl);
+ VdbeCoverage(v);
}
/* Register reg1 currently contains csr1.peerVal (the peer-value from csr1).
@@ -169960,6 +171784,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){
mxSqlLen -= n;
if( mxSqlLen<0 ){
pParse->rc = SQLITE_TOOBIG;
+ pParse->nErr++;
break;
}
#ifndef SQLITE_OMIT_WINDOWFUNC
@@ -170056,7 +171881,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){
if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){
sqlite3DeleteTrigger(db, pParse->pNewTrigger);
}
- if( pParse->pVList ) sqlite3DbFreeNN(db, pParse->pVList);
+ if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList);
db->pParse = pParentParse;
assert( nErr==0 || pParse->rc!=SQLITE_OK );
return nErr;
@@ -171412,18 +173237,19 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
db->lookaside.bMalloced = pBuf==0 ?1:0;
db->lookaside.nSlot = nBig+nSm;
}else{
- db->lookaside.pStart = db;
+ db->lookaside.pStart = 0;
#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
db->lookaside.pSmallInit = 0;
db->lookaside.pSmallFree = 0;
- db->lookaside.pMiddle = db;
+ db->lookaside.pMiddle = 0;
#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */
- db->lookaside.pEnd = db;
+ db->lookaside.pEnd = 0;
db->lookaside.bDisable = 1;
db->lookaside.sz = 0;
db->lookaside.bMalloced = 0;
db->lookaside.nSlot = 0;
}
+ db->lookaside.pTrueEnd = db->lookaside.pEnd;
assert( sqlite3LookasideUsed(db,0)==0 );
#endif /* SQLITE_OMIT_LOOKASIDE */
return SQLITE_OK;
@@ -171502,6 +173328,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){
SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){
va_list ap;
int rc;
+ sqlite3_mutex_enter(db->mutex);
va_start(ap, op);
switch( op ){
case SQLITE_DBCONFIG_MAINDBNAME: {
@@ -171567,6 +173394,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){
}
}
va_end(ap);
+ sqlite3_mutex_leave(db->mutex);
return rc;
}
@@ -172057,8 +173885,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){
** Return a static string containing the name corresponding to the error code
** specified in the argument.
*/
-#if defined(SQLITE_NEED_ERR_NAME)
-SQLITE_PRIVATE const char *sqlite3ErrName(int rc){
+SQLITE_API const char *sqlite3ErrName(int rc){
const char *zName = 0;
int i, origRc = rc;
for(i=0; i<2 && zName==0; i++, rc &= 0xff){
@@ -172163,7 +173990,6 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){
}
return zName;
}
-#endif
/*
** Return a static string that describes the kind of error specified in the
@@ -173937,6 +175763,19 @@ static int openDatabase(
goto opendb_out;
}
+#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)
+ /* Process magic filenames ":localStorage:" and ":sessionStorage:" */
+ if( zFilename && zFilename[0]==':' ){
+ if( strcmp(zFilename, ":localStorage:")==0 ){
+ zFilename = "file:local?vfs=kvvfs";
+ flags |= SQLITE_OPEN_URI;
+ }else if( strcmp(zFilename, ":sessionStorage:")==0 ){
+ zFilename = "file:session?vfs=kvvfs";
+ flags |= SQLITE_OPEN_URI;
+ }
+ }
+#endif /* SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) */
+
/* Parse the filename/URI argument
**
** Only allow sensible combinations of bits in the flags argument.
@@ -173967,6 +175806,12 @@ static int openDatabase(
sqlite3_free(zErrMsg);
goto opendb_out;
}
+ assert( db->pVfs!=0 );
+#if SQLITE_OS_KV || defined(SQLITE_OS_KV_OPTIONAL)
+ if( sqlite3_stricmp(db->pVfs->zName, "kvvfs")==0 ){
+ db->temp_store = 2;
+ }
+#endif
/* Open the backend database driver */
rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0,
@@ -174700,8 +176545,11 @@ SQLITE_API int sqlite3_test_control(int op, ...){
sqlite3ShowTriggerStepList(0);
sqlite3ShowTrigger(0);
sqlite3ShowTriggerList(0);
+#ifndef SQLITE_OMIT_WINDOWFUNC
sqlite3ShowWindow(0);
sqlite3ShowWinFunc(0);
+#endif
+ sqlite3ShowSelect(0);
}
#endif
break;
@@ -175073,7 +176921,7 @@ static char *appendText(char *p, const char *z){
** Memory layout must be compatible with that generated by the pager
** and expected by sqlite3_uri_parameter() and databaseName().
*/
-SQLITE_API char *sqlite3_create_filename(
+SQLITE_API const char *sqlite3_create_filename(
const char *zDatabase,
const char *zJournal,
const char *zWal,
@@ -175109,10 +176957,10 @@ SQLITE_API char *sqlite3_create_filename(
** error to call this routine with any parameter other than a pointer
** previously obtained from sqlite3_create_filename() or a NULL pointer.
*/
-SQLITE_API void sqlite3_free_filename(char *p){
+SQLITE_API void sqlite3_free_filename(const char *p){
if( p==0 ) return;
- p = (char*)databaseName(p);
- sqlite3_free(p - 4);
+ p = databaseName(p);
+ sqlite3_free((char*)p - 4);
}
@@ -175363,8 +177211,8 @@ SQLITE_API int sqlite3_snapshot_open(
*/
SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){
int rc = SQLITE_ERROR;
- int iDb;
#ifndef SQLITE_OMIT_WAL
+ int iDb;
#ifdef SQLITE_ENABLE_API_ARMOR
if( !sqlite3SafetyCheckOk(db) ){
@@ -176919,7 +178767,7 @@ struct Fts3MultiSegReader {
int nAdvance; /* How many seg-readers to advance */
Fts3SegFilter *pFilter; /* Pointer to filter object */
char *aBuffer; /* Buffer to merge doclists in */
- int nBuffer; /* Allocated size of aBuffer[] in bytes */
+ i64 nBuffer; /* Allocated size of aBuffer[] in bytes */
int iColFilter; /* If >=0, filter for this column */
int bRestart;
@@ -179615,7 +181463,7 @@ static int fts3TermSelectMerge(
**
** Similar padding is added in the fts3DoclistOrMerge() function.
*/
- pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1);
+ pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1);
pTS->anOutput[0] = nDoclist;
if( pTS->aaOutput[0] ){
memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
@@ -181035,8 +182883,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
char *aPoslist = 0; /* Position list for deferred tokens */
int nPoslist = 0; /* Number of bytes in aPoslist */
int iPrev = -1; /* Token number of previous deferred token */
-
- assert( pPhrase->doclist.bFreeList==0 );
+ char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0);
for(iToken=0; iTokennToken; iToken++){
Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
@@ -181050,6 +182897,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
if( pList==0 ){
sqlite3_free(aPoslist);
+ sqlite3_free(aFree);
pPhrase->doclist.pList = 0;
pPhrase->doclist.nList = 0;
return SQLITE_OK;
@@ -181070,6 +182918,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
nPoslist = (int)(aOut - aPoslist);
if( nPoslist==0 ){
sqlite3_free(aPoslist);
+ sqlite3_free(aFree);
pPhrase->doclist.pList = 0;
pPhrase->doclist.nList = 0;
return SQLITE_OK;
@@ -181102,13 +182951,14 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
nDistance = iPrev - nMaxUndeferred;
}
- aOut = (char *)sqlite3_malloc(nPoslist+8);
+ aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING);
if( !aOut ){
sqlite3_free(aPoslist);
return SQLITE_NOMEM;
}
pPhrase->doclist.pList = aOut;
+ assert( p1 && p2 );
if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
pPhrase->doclist.bFreeList = 1;
pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList);
@@ -181121,6 +182971,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
}
}
+ if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree);
return SQLITE_OK;
}
#endif /* SQLITE_DISABLE_FTS4_DEFERRED */
@@ -181469,7 +183320,7 @@ static int fts3EvalIncrPhraseNext(
if( bEof==0 ){
int nList = 0;
int nByte = a[p->nToken-1].nList;
- char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING);
+ char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING);
if( !aDoclist ) return SQLITE_NOMEM;
memcpy(aDoclist, a[p->nToken-1].pList, nByte+1);
memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING);
@@ -182295,11 +184146,10 @@ static int fts3EvalTestExpr(
default: {
#ifndef SQLITE_DISABLE_FTS4_DEFERRED
- if( pCsr->pDeferred
- && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
- ){
+ if( pCsr->pDeferred && (pExpr->bDeferred || (
+ pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList
+ ))){
Fts3Phrase *pPhrase = pExpr->pPhrase;
- assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
if( pExpr->bDeferred ){
fts3EvalInvalidatePoslist(pPhrase);
}
@@ -185706,7 +187556,7 @@ static int porterNext(
if( n>c->nAllocated ){
char *pNew;
c->nAllocated = n+20;
- pNew = sqlite3_realloc(c->zToken, c->nAllocated);
+ pNew = sqlite3_realloc64(c->zToken, c->nAllocated);
if( !pNew ) return SQLITE_NOMEM;
c->zToken = pNew;
}
@@ -186458,7 +188308,7 @@ static int simpleNext(
if( n>c->nTokenAllocated ){
char *pNew;
c->nTokenAllocated = n+20;
- pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+ pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated);
if( !pNew ) return SQLITE_NOMEM;
c->pToken = pNew;
}
@@ -187620,7 +189470,7 @@ static int fts3PendingListAppendVarint(
/* Allocate or grow the PendingList as required. */
if( !p ){
- p = sqlite3_malloc(sizeof(*p) + 100);
+ p = sqlite3_malloc64(sizeof(*p) + 100);
if( !p ){
return SQLITE_NOMEM;
}
@@ -187629,14 +189479,14 @@ static int fts3PendingListAppendVarint(
p->nData = 0;
}
else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
- int nNew = p->nSpace * 2;
- p = sqlite3_realloc(p, sizeof(*p) + nNew);
+ i64 nNew = p->nSpace * 2;
+ p = sqlite3_realloc64(p, sizeof(*p) + nNew);
if( !p ){
sqlite3_free(*pp);
*pp = 0;
return SQLITE_NOMEM;
}
- p->nSpace = nNew;
+ p->nSpace = (int)nNew;
p->aData = (char *)&p[1];
}
@@ -188193,7 +190043,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
int nByte = sqlite3_blob_bytes(p->pSegments);
*pnBlob = nByte;
if( paBlob ){
- char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
+ char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING);
if( !aByte ){
rc = SQLITE_NOMEM;
}else{
@@ -188310,7 +190160,7 @@ static int fts3SegReaderNext(
int nTerm = fts3HashKeysize(pElem);
if( (nTerm+1)>pReader->nTermAlloc ){
sqlite3_free(pReader->zTerm);
- pReader->zTerm = (char*)sqlite3_malloc((nTerm+1)*2);
+ pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2);
if( !pReader->zTerm ) return SQLITE_NOMEM;
pReader->nTermAlloc = (nTerm+1)*2;
}
@@ -188318,7 +190168,7 @@ static int fts3SegReaderNext(
pReader->zTerm[nTerm] = '\0';
pReader->nTerm = nTerm;
- aCopy = (char*)sqlite3_malloc(nCopy);
+ aCopy = (char*)sqlite3_malloc64(nCopy);
if( !aCopy ) return SQLITE_NOMEM;
memcpy(aCopy, pList->aData, nCopy);
pReader->nNode = pReader->nDoclist = nCopy;
@@ -188605,7 +190455,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
nExtra = nRoot + FTS3_NODE_PADDING;
}
- pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
+ pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra);
if( !pReader ){
return SQLITE_NOMEM;
}
@@ -188697,7 +190547,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
if( nElem==nAlloc ){
Fts3HashElem **aElem2;
nAlloc += 16;
- aElem2 = (Fts3HashElem **)sqlite3_realloc(
+ aElem2 = (Fts3HashElem **)sqlite3_realloc64(
aElem, nAlloc*sizeof(Fts3HashElem *)
);
if( !aElem2 ){
@@ -189031,7 +190881,7 @@ static int fts3NodeAddTerm(
** this is not expected to be a serious problem.
*/
assert( pTree->aData==(char *)&pTree[1] );
- pTree->aData = (char *)sqlite3_malloc(nReq);
+ pTree->aData = (char *)sqlite3_malloc64(nReq);
if( !pTree->aData ){
return SQLITE_NOMEM;
}
@@ -189049,7 +190899,7 @@ static int fts3NodeAddTerm(
if( isCopyTerm ){
if( pTree->nMalloczMalloc, nTerm*2);
+ char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2);
if( !zNew ){
return SQLITE_NOMEM;
}
@@ -189075,7 +190925,7 @@ static int fts3NodeAddTerm(
** now. Instead, the term is inserted into the parent of pTree. If pTree
** has no parent, one is created here.
*/
- pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize);
+ pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize);
if( !pNew ){
return SQLITE_NOMEM;
}
@@ -189213,7 +191063,7 @@ static int fts3SegWriterAdd(
){
int nPrefix; /* Size of term prefix in bytes */
int nSuffix; /* Size of term suffix in bytes */
- int nReq; /* Number of bytes required on leaf page */
+ i64 nReq; /* Number of bytes required on leaf page */
int nData;
SegmentWriter *pWriter = *ppWriter;
@@ -189222,13 +191072,13 @@ static int fts3SegWriterAdd(
sqlite3_stmt *pStmt;
/* Allocate the SegmentWriter structure */
- pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
+ pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter));
if( !pWriter ) return SQLITE_NOMEM;
memset(pWriter, 0, sizeof(SegmentWriter));
*ppWriter = pWriter;
/* Allocate a buffer in which to accumulate data */
- pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize);
+ pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize);
if( !pWriter->aData ) return SQLITE_NOMEM;
pWriter->nSize = p->nNodeSize;
@@ -189303,7 +191153,7 @@ static int fts3SegWriterAdd(
** the buffer to make it large enough.
*/
if( nReq>pWriter->nSize ){
- char *aNew = sqlite3_realloc(pWriter->aData, nReq);
+ char *aNew = sqlite3_realloc64(pWriter->aData, nReq);
if( !aNew ) return SQLITE_NOMEM;
pWriter->aData = aNew;
pWriter->nSize = nReq;
@@ -189328,7 +191178,7 @@ static int fts3SegWriterAdd(
*/
if( isCopyTerm ){
if( nTerm>pWriter->nMalloc ){
- char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2);
+ char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2);
if( !zNew ){
return SQLITE_NOMEM;
}
@@ -189636,12 +191486,12 @@ static void fts3ColumnFilter(
static int fts3MsrBufferData(
Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
char *pList,
- int nList
+ i64 nList
){
if( nList>pMsr->nBuffer ){
char *pNew;
pMsr->nBuffer = nList*2;
- pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer);
+ pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer);
if( !pNew ) return SQLITE_NOMEM;
pMsr->aBuffer = pNew;
}
@@ -189697,7 +191547,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){
- rc = fts3MsrBufferData(pMsr, pList, nList+1);
+ rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1);
if( rc!=SQLITE_OK ) return rc;
assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 );
pList = pMsr->aBuffer;
@@ -189834,11 +191684,11 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){
return SQLITE_OK;
}
-static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){
+static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){
if( nReq>pCsr->nBuffer ){
char *aNew;
pCsr->nBuffer = nReq*2;
- aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
+ aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer);
if( !aNew ){
return SQLITE_NOMEM;
}
@@ -189929,7 +191779,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
){
pCsr->nDoclist = apSegment[0]->nDoclist;
if( fts3SegReaderIsPending(apSegment[0]) ){
- rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist);
+ rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist,
+ (i64)pCsr->nDoclist);
pCsr->aDoclist = pCsr->aBuffer;
}else{
pCsr->aDoclist = apSegment[0]->aDoclist;
@@ -189982,7 +191833,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
- rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING);
+ rc = fts3GrowSegReaderBuffer(pCsr,
+ (i64)nByte+nDoclist+FTS3_NODE_PADDING);
if( rc ) return rc;
if( isFirst ){
@@ -190008,7 +191860,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
fts3SegReaderSort(apSegment, nMerge, j, xCmp);
}
if( nDoclist>0 ){
- rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING);
+ rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING);
if( rc ) return rc;
memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING);
pCsr->aDoclist = pCsr->aBuffer;
@@ -190721,7 +192573,7 @@ struct NodeReader {
static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){
if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){
int nAlloc = nMin;
- char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc);
+ char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc);
if( a ){
pBlob->nAlloc = nAlloc;
pBlob->a = a;
@@ -191518,7 +193370,7 @@ static int fts3RepackSegdirLevel(
if( nIdx>=nAlloc ){
int *aNew;
nAlloc += 16;
- aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int));
+ aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int));
if( !aNew ){
rc = SQLITE_NOMEM;
break;
@@ -191892,7 +193744,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
/* Allocate space for the cursor, filter and writer objects */
const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);
- pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
+ pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc);
if( !pWriter ) return SQLITE_NOMEM;
pFilter = (Fts3SegFilter *)&pWriter[1];
pCsr = (Fts3MultiSegReader *)&pFilter[1];
@@ -192528,7 +194380,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(
return SQLITE_OK;
}
- pRet = (char *)sqlite3_malloc(p->pList->nData);
+ pRet = (char *)sqlite3_malloc64(p->pList->nData);
if( !pRet ) return SQLITE_NOMEM;
nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
@@ -192548,7 +194400,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken(
int iCol /* Column that token must appear in (or -1) */
){
Fts3DeferredToken *pDeferred;
- pDeferred = sqlite3_malloc(sizeof(*pDeferred));
+ pDeferred = sqlite3_malloc64(sizeof(*pDeferred));
if( !pDeferred ){
return SQLITE_NOMEM;
}
@@ -201263,7 +203115,7 @@ static int rtreeUpdate(
rtreeReference(pRtree);
assert(nData>=1);
- cell.iRowid = 0; /* Used only to suppress a compiler warning */
+ memset(&cell, 0, sizeof(cell));
/* Constraint handling. A write operation on an r-tree table may return
** SQLITE_CONSTRAINT for two reasons:
@@ -204127,7 +205979,7 @@ static int geopolyUpdate(
sqlite3_free(p);
nChange = 1;
}
- for(jj=1; jjnAux; jj++){
+ for(jj=1; jjdbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
if( p->zState==0 ){
const char *zFile = sqlite3_db_filename(p->dbRbu, "main");
- p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile);
+ p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile);
}
}
@@ -209103,32 +210986,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){
}
if( p->rc==SQLITE_OK ){
-#if defined(_WIN32_WCE)
- {
- LPWSTR zWideOal;
- LPWSTR zWideWal;
-
- zWideOal = rbuWinUtf8ToUnicode(zOal);
- if( zWideOal ){
- zWideWal = rbuWinUtf8ToUnicode(zWal);
- if( zWideWal ){
- if( MoveFileW(zWideOal, zWideWal) ){
- p->rc = SQLITE_OK;
- }else{
- p->rc = SQLITE_IOERR;
- }
- sqlite3_free(zWideWal);
- }else{
- p->rc = SQLITE_IOERR_NOMEM;
- }
- sqlite3_free(zWideOal);
- }else{
- p->rc = SQLITE_IOERR_NOMEM;
- }
- }
-#else
- p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK;
-#endif
+ p->rc = p->xRename(p->pRenameArg, zOal, zWal);
}
if( p->rc!=SQLITE_OK
@@ -209867,6 +211725,7 @@ static sqlite3rbu *openRbuHandle(
/* Create the custom VFS. */
memset(p, 0, sizeof(sqlite3rbu));
+ sqlite3rbu_rename_handler(p, 0, 0);
rbuCreateVfs(p);
/* Open the target, RBU and state databases */
@@ -210258,6 +212117,54 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){
return rc;
}
+/*
+** Default xRename callback for RBU.
+*/
+static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){
+ int rc = SQLITE_OK;
+#if defined(_WIN32_WCE)
+ {
+ LPWSTR zWideOld;
+ LPWSTR zWideNew;
+
+ zWideOld = rbuWinUtf8ToUnicode(zOld);
+ if( zWideOld ){
+ zWideNew = rbuWinUtf8ToUnicode(zNew);
+ if( zWideNew ){
+ if( MoveFileW(zWideOld, zWideNew) ){
+ rc = SQLITE_OK;
+ }else{
+ rc = SQLITE_IOERR;
+ }
+ sqlite3_free(zWideNew);
+ }else{
+ rc = SQLITE_IOERR_NOMEM;
+ }
+ sqlite3_free(zWideOld);
+ }else{
+ rc = SQLITE_IOERR_NOMEM;
+ }
+ }
+#else
+ rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK;
+#endif
+ return rc;
+}
+
+SQLITE_API void sqlite3rbu_rename_handler(
+ sqlite3rbu *pRbu,
+ void *pArg,
+ int (*xRename)(void *pArg, const char *zOld, const char *zNew)
+){
+ if( xRename ){
+ pRbu->xRename = xRename;
+ pRbu->pRenameArg = pArg;
+ }else{
+ pRbu->xRename = xDefaultRename;
+ pRbu->pRenameArg = 0;
+ }
+}
+
/**************************************************************************
** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour
** of a standard VFS in the following ways:
@@ -212388,12 +214295,18 @@ static int dbpageColumn(
}
case 1: { /* data */
DbPage *pDbPage = 0;
- rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0);
- if( rc==SQLITE_OK ){
- sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage,
- SQLITE_TRANSIENT);
+ if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){
+ /* The pending byte page. Assume it is zeroed out. Attempting to
+ ** request this page from the page is an SQLITE_CORRUPT error. */
+ sqlite3_result_zeroblob(ctx, pCsr->szPage);
+ }else{
+ rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage,
+ SQLITE_TRANSIENT);
+ }
+ sqlite3PagerUnref(pDbPage);
}
- sqlite3PagerUnref(pDbPage);
break;
}
default: { /* schema */
@@ -212402,7 +214315,7 @@ static int dbpageColumn(
break;
}
}
- return SQLITE_OK;
+ return rc;
}
static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
@@ -212462,11 +214375,12 @@ static int dbpageUpdate(
pPager = sqlite3BtreePager(pBt);
rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0);
if( rc==SQLITE_OK ){
- rc = sqlite3PagerWrite(pDbPage);
- if( rc==SQLITE_OK ){
- memcpy(sqlite3PagerGetData(pDbPage),
- sqlite3_value_blob(argv[3]),
- szPage);
+ const void *pData = sqlite3_value_blob(argv[3]);
+ assert( pData!=0 || pTab->db->mallocFailed );
+ if( pData
+ && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK
+ ){
+ memcpy(sqlite3PagerGetData(pDbPage), pData, szPage);
}
}
sqlite3PagerUnref(pDbPage);
@@ -212486,11 +214400,12 @@ static int dbpageBegin(sqlite3_vtab *pVtab){
DbpageTable *pTab = (DbpageTable *)pVtab;
sqlite3 *db = pTab->db;
int i;
- for(i=0; inDb; i++){
+ int rc = SQLITE_OK;
+ for(i=0; rc==SQLITE_OK && inDb; i++){
Btree *pBt = db->aDb[i].pBt;
- if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0);
+ if( pBt ) rc = sqlite3BtreeBeginTrans(pBt, 1, 0);
}
- return SQLITE_OK;
+ return rc;
}
@@ -219214,7 +221129,7 @@ static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
-#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
+#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
@@ -231091,7 +233006,9 @@ static void fts5WriteAppendRowid(
fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid);
}else{
assert_nc( p->rc || iRowid>pWriter->iPrevRowid );
- fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid);
+ fts5BufferAppendVarint(&p->rc, &pPage->buf,
+ (u64)iRowid - (u64)pWriter->iPrevRowid
+ );
}
pWriter->iPrevRowid = iRowid;
pWriter->bFirstRowidInDoclist = 0;
@@ -231855,7 +233772,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
static void fts5AppendRowid(
Fts5Index *p,
- i64 iDelta,
+ u64 iDelta,
Fts5Iter *pUnused,
Fts5Buffer *pBuf
){
@@ -231865,7 +233782,7 @@ static void fts5AppendRowid(
static void fts5AppendPoslist(
Fts5Index *p,
- i64 iDelta,
+ u64 iDelta,
Fts5Iter *pMulti,
Fts5Buffer *pBuf
){
@@ -231940,10 +233857,10 @@ static void fts5MergeAppendDocid(
}
#endif
-#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \
- assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \
- fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
- (iLastRowid) = (iRowid); \
+#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \
+ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \
+ fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \
+ (iLastRowid) = (iRowid); \
}
/*
@@ -232214,7 +234131,7 @@ static void fts5SetupPrefixIter(
int nMerge = 1;
void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
- void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*);
+ void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
xMerge = fts5MergeRowidLists;
xAppend = fts5AppendRowid;
@@ -232253,7 +234170,7 @@ static void fts5SetupPrefixIter(
Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
p1->xSetOutputs(p1, pSeg);
if( p1->base.nData ){
- xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist);
+ xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist);
iLastRowid = p1->base.iRowid;
}
}
@@ -232301,7 +234218,7 @@ static void fts5SetupPrefixIter(
iLastRowid = 0;
}
- xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist);
+ xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist);
iLastRowid = p1->base.iRowid;
}
@@ -233280,6 +235197,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum
/* If this is a new term, query for it. Update cksum3 with the results. */
fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
+ if( p->rc ) break;
if( eDetail==FTS5_DETAIL_NONE ){
if( 0==fts5MultiIterIsEmpty(p, pIter) ){
@@ -236617,7 +238535,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
- sqlite3_result_text(pCtx, "fts5: 2022-06-25 14:57:57 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918", -1, SQLITE_TRANSIENT);
+ sqlite3_result_text(pCtx, "fts5: 2022-11-16 12:10:08 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318", -1, SQLITE_TRANSIENT);
}
/*
@@ -241288,6 +243206,16 @@ SQLITE_EXTENSION_INIT1
#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+#define STMT_NUM_INTEGER_COLUMN 10
+typedef struct StmtRow StmtRow;
+struct StmtRow {
+ sqlite3_int64 iRowid; /* Rowid value */
+ char *zSql; /* column "sql" */
+ int aCol[STMT_NUM_INTEGER_COLUMN+1]; /* all other column values */
+ StmtRow *pNext; /* Next row to return */
+};
+
/* stmt_vtab is a subclass of sqlite3_vtab which will
** serve as the underlying representation of a stmt virtual table
*/
@@ -241305,8 +243233,7 @@ typedef struct stmt_cursor stmt_cursor;
struct stmt_cursor {
sqlite3_vtab_cursor base; /* Base class - must be first */
sqlite3 *db; /* Database connection for this cursor */
- sqlite3_stmt *pStmt; /* Statement cursor is currently pointing at */
- sqlite3_int64 iRowid; /* The rowid */
+ StmtRow *pRow; /* Current row */
};
/*
@@ -241350,7 +243277,7 @@ static int stmtConnect(
"CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep,"
"reprep,run,mem)");
if( rc==SQLITE_OK ){
- pNew = sqlite3_malloc( sizeof(*pNew) );
+ pNew = sqlite3_malloc64( sizeof(*pNew) );
*ppVtab = (sqlite3_vtab*)pNew;
if( pNew==0 ) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(*pNew));
@@ -241372,7 +243299,7 @@ static int stmtDisconnect(sqlite3_vtab *pVtab){
*/
static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
stmt_cursor *pCur;
- pCur = sqlite3_malloc( sizeof(*pCur) );
+ pCur = sqlite3_malloc64( sizeof(*pCur) );
if( pCur==0 ) return SQLITE_NOMEM;
memset(pCur, 0, sizeof(*pCur));
pCur->db = ((stmt_vtab*)p)->db;
@@ -241380,10 +243307,21 @@ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
return SQLITE_OK;
}
+static void stmtCsrReset(stmt_cursor *pCur){
+ StmtRow *pRow = 0;
+ StmtRow *pNext = 0;
+ for(pRow=pCur->pRow; pRow; pRow=pNext){
+ pNext = pRow->pNext;
+ sqlite3_free(pRow);
+ }
+ pCur->pRow = 0;
+}
+
/*
** Destructor for a stmt_cursor.
*/
static int stmtClose(sqlite3_vtab_cursor *cur){
+ stmtCsrReset((stmt_cursor*)cur);
sqlite3_free(cur);
return SQLITE_OK;
}
@@ -241394,8 +243332,9 @@ static int stmtClose(sqlite3_vtab_cursor *cur){
*/
static int stmtNext(sqlite3_vtab_cursor *cur){
stmt_cursor *pCur = (stmt_cursor*)cur;
- pCur->iRowid++;
- pCur->pStmt = sqlite3_next_stmt(pCur->db, pCur->pStmt);
+ StmtRow *pNext = pCur->pRow->pNext;
+ sqlite3_free(pCur->pRow);
+ pCur->pRow = pNext;
return SQLITE_OK;
}
@@ -241409,39 +243348,11 @@ static int stmtColumn(
int i /* Which column to return */
){
stmt_cursor *pCur = (stmt_cursor*)cur;
- switch( i ){
- case STMT_COLUMN_SQL: {
- sqlite3_result_text(ctx, sqlite3_sql(pCur->pStmt), -1, SQLITE_TRANSIENT);
- break;
- }
- case STMT_COLUMN_NCOL: {
- sqlite3_result_int(ctx, sqlite3_column_count(pCur->pStmt));
- break;
- }
- case STMT_COLUMN_RO: {
- sqlite3_result_int(ctx, sqlite3_stmt_readonly(pCur->pStmt));
- break;
- }
- case STMT_COLUMN_BUSY: {
- sqlite3_result_int(ctx, sqlite3_stmt_busy(pCur->pStmt));
- break;
- }
- default: {
- assert( i==STMT_COLUMN_MEM );
- i = SQLITE_STMTSTATUS_MEMUSED +
- STMT_COLUMN_NSCAN - SQLITE_STMTSTATUS_FULLSCAN_STEP;
- /* Fall thru */
- }
- case STMT_COLUMN_NSCAN:
- case STMT_COLUMN_NSORT:
- case STMT_COLUMN_NAIDX:
- case STMT_COLUMN_NSTEP:
- case STMT_COLUMN_REPREP:
- case STMT_COLUMN_RUN: {
- sqlite3_result_int(ctx, sqlite3_stmt_status(pCur->pStmt,
- i-STMT_COLUMN_NSCAN+SQLITE_STMTSTATUS_FULLSCAN_STEP, 0));
- break;
- }
+ StmtRow *pRow = pCur->pRow;
+ if( i==STMT_COLUMN_SQL ){
+ sqlite3_result_text(ctx, pRow->zSql, -1, SQLITE_TRANSIENT);
+ }else{
+ sqlite3_result_int(ctx, pRow->aCol[i]);
}
return SQLITE_OK;
}
@@ -241452,7 +243363,7 @@ static int stmtColumn(
*/
static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
stmt_cursor *pCur = (stmt_cursor*)cur;
- *pRowid = pCur->iRowid;
+ *pRowid = pCur->pRow->iRowid;
return SQLITE_OK;
}
@@ -241462,7 +243373,7 @@ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
*/
static int stmtEof(sqlite3_vtab_cursor *cur){
stmt_cursor *pCur = (stmt_cursor*)cur;
- return pCur->pStmt==0;
+ return pCur->pRow==0;
}
/*
@@ -241477,9 +243388,53 @@ static int stmtFilter(
int argc, sqlite3_value **argv
){
stmt_cursor *pCur = (stmt_cursor *)pVtabCursor;
- pCur->pStmt = 0;
- pCur->iRowid = 0;
- return stmtNext(pVtabCursor);
+ sqlite3_stmt *p = 0;
+ sqlite3_int64 iRowid = 1;
+ StmtRow **ppRow = 0;
+
+ stmtCsrReset(pCur);
+ ppRow = &pCur->pRow;
+ for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){
+ const char *zSql = sqlite3_sql(p);
+ sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0;
+ StmtRow *pNew = (StmtRow*)sqlite3_malloc64(sizeof(StmtRow) + nSql);
+
+ if( pNew==0 ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(StmtRow));
+ if( zSql ){
+ pNew->zSql = (char*)&pNew[1];
+ memcpy(pNew->zSql, zSql, nSql);
+ }
+ pNew->aCol[STMT_COLUMN_NCOL] = sqlite3_column_count(p);
+ pNew->aCol[STMT_COLUMN_RO] = sqlite3_stmt_readonly(p);
+ pNew->aCol[STMT_COLUMN_BUSY] = sqlite3_stmt_busy(p);
+ pNew->aCol[STMT_COLUMN_NSCAN] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0
+ );
+ pNew->aCol[STMT_COLUMN_NSORT] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_SORT, 0
+ );
+ pNew->aCol[STMT_COLUMN_NAIDX] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_AUTOINDEX, 0
+ );
+ pNew->aCol[STMT_COLUMN_NSTEP] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_VM_STEP, 0
+ );
+ pNew->aCol[STMT_COLUMN_REPREP] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_REPREPARE, 0
+ );
+ pNew->aCol[STMT_COLUMN_RUN] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_RUN, 0
+ );
+ pNew->aCol[STMT_COLUMN_MEM] = sqlite3_stmt_status(
+ p, SQLITE_STMTSTATUS_MEMUSED, 0
+ );
+ pNew->iRowid = iRowid++;
+ *ppRow = pNew;
+ ppRow = &pNew->pNext;
+ }
+
+ return SQLITE_OK;
}
/*
diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h
index 26b4a31f..2859cc09 100644
--- a/src/database/sqlite3.h
+++ b/src/database/sqlite3.h
@@ -146,9 +146,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.39.0"
-#define SQLITE_VERSION_NUMBER 3039000
-#define SQLITE_SOURCE_ID "2022-06-25 14:57:57 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918"
+#define SQLITE_VERSION "3.40.0"
+#define SQLITE_VERSION_NUMBER 3040000
+#define SQLITE_SOURCE_ID "2022-11-16 12:10:08 89c459e766ea7e9165d0beeb124708b955a4950d0f4792f457465d71b158d318"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -670,13 +670,17 @@ SQLITE_API int sqlite3_exec(
**
** SQLite uses one of these integer values as the second
** argument to calls it makes to the xLock() and xUnlock() methods
-** of an [sqlite3_io_methods] object.
+** of an [sqlite3_io_methods] object. These values are ordered from
+** lest restrictive to most restrictive.
+**
+** The argument to xLock() is always SHARED or higher. The argument to
+** xUnlock is either SHARED or NONE.
*/
-#define SQLITE_LOCK_NONE 0
-#define SQLITE_LOCK_SHARED 1
-#define SQLITE_LOCK_RESERVED 2
-#define SQLITE_LOCK_PENDING 3
-#define SQLITE_LOCK_EXCLUSIVE 4
+#define SQLITE_LOCK_NONE 0 /* xUnlock() only */
+#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */
+#define SQLITE_LOCK_RESERVED 2 /* xLock() only */
+#define SQLITE_LOCK_PENDING 3 /* xLock() only */
+#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */
/*
** CAPI3REF: Synchronization Type Flags
@@ -754,7 +758,14 @@ struct sqlite3_file {
**
[SQLITE_LOCK_PENDING], or
**
[SQLITE_LOCK_EXCLUSIVE].
**
-** xLock() increases the lock. xUnlock() decreases the lock.
+** xLock() upgrades the database file lock. In other words, xLock() moves the
+** database file lock in the direction NONE toward EXCLUSIVE. The argument to
+** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never
+** SQLITE_LOCK_NONE. If the database file lock is already at or above the
+** requested lock, then the call to xLock() is a no-op.
+** xUnlock() downgrades the database file lock to either SHARED or NONE.
+* If the lock is already at or below the requested lock state, then the call
+** to xUnlock() is a no-op.
** The xCheckReservedLock() method checks whether any database connection,
** either in this process or in some other process, is holding a RESERVED,
** PENDING, or EXCLUSIVE lock on the file. It returns true
@@ -859,9 +870,8 @@ struct sqlite3_io_methods {
** opcode causes the xFileControl method to write the current state of
** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED],
** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE])
-** into an integer that the pArg argument points to. This capability
-** is used during testing and is only available when the SQLITE_TEST
-** compile-time option is used.
+** into an integer that the pArg argument points to.
+** This capability is only available if SQLite is compiled with [SQLITE_DEBUG].
**
**
[[SQLITE_FCNTL_SIZE_HINT]]
** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS
@@ -1253,6 +1263,26 @@ typedef struct sqlite3_mutex sqlite3_mutex;
*/
typedef struct sqlite3_api_routines sqlite3_api_routines;
+/*
+** CAPI3REF: File Name
+**
+** Type [sqlite3_filename] is used by SQLite to pass filenames to the
+** xOpen method of a [VFS]. It may be cast to (const char*) and treated
+** as a normal, nul-terminated, UTF-8 buffer containing the filename, but
+** may also be passed to special APIs such as:
+**
+**
+**
sqlite3_filename_database()
+**
sqlite3_filename_journal()
+**
sqlite3_filename_wal()
+**
sqlite3_uri_parameter()
+**
sqlite3_uri_boolean()
+**
sqlite3_uri_int64()
+**
sqlite3_uri_key()
+**
+*/
+typedef const char *sqlite3_filename;
+
/*
** CAPI3REF: OS Interface Object
**
@@ -1431,7 +1461,7 @@ struct sqlite3_vfs {
sqlite3_vfs *pNext; /* Next registered VFS */
const char *zName; /* Name of this virtual file system */
void *pAppData; /* Pointer to application-specific data */
- int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*,
+ int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*,
int flags, int *pOutFlags);
int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir);
int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut);
@@ -2309,6 +2339,7 @@ struct sqlite3_mem_methods {
**
The database is opened [shared cache] enabled, overriding
** the default shared cache setting provided by
** [sqlite3_enable_shared_cache()].)^
+** The [use of shared cache mode is discouraged] and hence shared cache
+** capabilities may be omitted from many builds of SQLite. In such cases,
+** this option is a no-op.
**
** ^(
[SQLITE_OPEN_PRIVATECACHE]
**
The database is opened [shared cache] disabled, overriding
@@ -3439,7 +3473,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
** to return an extended result code.
**
** [[OPEN_NOFOLLOW]] ^(
[SQLITE_OPEN_NOFOLLOW]
-**
The database filename is not allowed to be a symbolic link
+**
The database filename is not allowed to contain a symbolic link
** )^
**
** If the 3rd parameter to sqlite3_open_v2() is not one of the
@@ -3698,10 +3732,10 @@ SQLITE_API int sqlite3_open_v2(
**
** See the [URI filename] documentation for additional information.
*/
-SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam);
-SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault);
-SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64);
-SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N);
+SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam);
+SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault);
+SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64);
+SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N);
/*
** CAPI3REF: Translate filenames
@@ -3730,9 +3764,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N);
** return value from [sqlite3_db_filename()], then the result is
** undefined and is likely a memory access violation.
*/
-SQLITE_API const char *sqlite3_filename_database(const char*);
-SQLITE_API const char *sqlite3_filename_journal(const char*);
-SQLITE_API const char *sqlite3_filename_wal(const char*);
+SQLITE_API const char *sqlite3_filename_database(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename);
/*
** CAPI3REF: Database File Corresponding To A Journal
@@ -3798,14 +3832,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*);
** then the corresponding [sqlite3_module.xClose() method should also be
** invoked prior to calling sqlite3_free_filename(Y).
*/
-SQLITE_API char *sqlite3_create_filename(
+SQLITE_API sqlite3_filename sqlite3_create_filename(
const char *zDatabase,
const char *zJournal,
const char *zWal,
int nParam,
const char **azParam
);
-SQLITE_API void sqlite3_free_filename(char*);
+SQLITE_API void sqlite3_free_filename(sqlite3_filename);
/*
** CAPI3REF: Error Codes And Messages
@@ -5508,6 +5542,16 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6
** then the conversion is performed. Otherwise no conversion occurs.
** The [SQLITE_INTEGER | datatype] after conversion is returned.)^
**
+** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8],
+** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding
+** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X)
+** returns something other than SQLITE_TEXT, then the return value from
+** sqlite3_value_encoding(X) is meaningless. ^Calls to
+** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X),
+** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or
+** sqlite3_value_bytes16(X) might change the encoding of the value X and
+** thus change the return from subsequent calls to sqlite3_value_encoding(X).
+**
** ^Within the [xUpdate] method of a [virtual table], the
** sqlite3_value_nochange(X) interface returns true if and only if
** the column corresponding to X is unchanged by the UPDATE operation
@@ -5572,6 +5616,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value*);
SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*);
SQLITE_API int sqlite3_value_nochange(sqlite3_value*);
SQLITE_API int sqlite3_value_frombind(sqlite3_value*);
+SQLITE_API int sqlite3_value_encoding(sqlite3_value*);
/*
** CAPI3REF: Finding The Subtype Of SQL Values
@@ -5625,7 +5670,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*);
**
** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer
** when first called if N is less than or equal to zero or if a memory
-** allocate error occurs.
+** allocation error occurs.
**
** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is
** determined by the N parameter on first successful call. Changing the
@@ -5830,9 +5875,10 @@ typedef void (*sqlite3_destructor_type)(void*);
** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE].
** ^SQLite takes the text result from the application from
** the 2nd parameter of the sqlite3_result_text* interfaces.
-** ^If the 3rd parameter to the sqlite3_result_text* interfaces
-** is negative, then SQLite takes result text from the 2nd parameter
-** through the first zero character.
+** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces
+** other than sqlite3_result_text64() is negative, then SQLite computes
+** the string length itself by searching the 2nd parameter for the first
+** zero character.
** ^If the 3rd parameter to the sqlite3_result_text* interfaces
** is non-negative, then as many bytes (not characters) of the text
** pointed to by the 2nd parameter are taken as the application-defined
@@ -6282,7 +6328,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*);
**
** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name
** for the N-th database on database connection D, or a NULL pointer of N is
-** out of range. An N alue of 0 means the main database file. An N of 1 is
+** out of range. An N value of 0 means the main database file. An N of 1 is
** the "temp" schema. Larger values of N correspond to various ATTACH-ed
** databases.
**
@@ -6328,7 +6374,7 @@ SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N);
**
[sqlite3_filename_wal()]
**
*/
-SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName);
+SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName);
/*
** CAPI3REF: Determine if a database is read-only
@@ -6465,7 +6511,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
** function C that is invoked prior to each autovacuum of the database
** file. ^The callback is passed a copy of the generic data pointer (P),
** the schema-name of the attached database that is being autovacuumed,
-** the the size of the database file in pages, the number of free pages,
+** the size of the database file in pages, the number of free pages,
** and the number of bytes per page, respectively. The callback should
** return the number of free pages that should be removed by the
** autovacuum. ^If the callback returns zero, then no autovacuum happens.
@@ -6586,6 +6632,11 @@ SQLITE_API void *sqlite3_update_hook(
** to the same database. Sharing is enabled if the argument is true
** and disabled if the argument is false.)^
**
+** This interface is omitted if SQLite is compiled with
+** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE]
+** compile-time option is recommended because the
+** [use of shared cache mode is discouraged].
+**
** ^Cache sharing is enabled and disabled for an entire process.
** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]).
** In prior versions of SQLite,
@@ -6684,7 +6735,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*);
** ^The soft heap limit may not be greater than the hard heap limit.
** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N)
** is invoked with a value of N that is greater than the hard heap limit,
-** the the soft heap limit is set to the value of the hard heap limit.
+** the soft heap limit is set to the value of the hard heap limit.
** ^The soft heap limit is automatically enabled whenever the hard heap
** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and
** the soft heap limit is outside the range of 1..N, then the soft heap
@@ -8979,7 +9030,7 @@ typedef struct sqlite3_backup sqlite3_backup;
** if the application incorrectly accesses the destination [database connection]
** and so no error code is reported, but the operations may malfunction
** nevertheless. Use of the destination database connection while a
-** backup is in progress might also also cause a mutex deadlock.
+** backup is in progress might also cause a mutex deadlock.
**
** If running in [shared cache mode], the application must
** guarantee that the shared cache used by the destination database
@@ -9407,7 +9458,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2(
*/
#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */
#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */
-#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */
+#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */
#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */
/*
diff --git a/src/datastructure.c b/src/datastructure.c
index 30c40575..958fe565 100644
--- a/src/datastructure.c
+++ b/src/datastructure.c
@@ -37,7 +37,7 @@ void strtolower(char *str)
}
// creates a simple hash of a string that fits into a uint32_t
-uint32_t hashStr(const char *s)
+uint32_t __attribute__ ((pure)) hashStr(const char *s)
{
uint32_t hash = 0;
// Jenkins' One-at-a-Time hash (http://www.burtleburtle.net/bob/hash/doobs.html)
@@ -319,13 +319,14 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim
}
}
-int findCacheID(int domainID, int clientID, enum query_type query_type)
+int _findCacheID(const int domainID, const int clientID, const enum query_type query_type,
+ const bool create_new, const char *func, int line, const char *file)
{
// Compare content of client against known client IP addresses
for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++)
{
// Get cache pointer
- DNSCacheData* dns_cache = getDNSCache(cacheID, true);
+ DNSCacheData* dns_cache = _getDNSCache(cacheID, true, line, func, file);
// Check if the returned pointer is valid before trying to access it
if(dns_cache == NULL)
@@ -339,11 +340,14 @@ int findCacheID(int domainID, int clientID, enum query_type query_type)
}
}
+ if(!create_new)
+ return -1;
+
// Get ID of new cache entry
const int cacheID = counters->dns_cache_size;
// Get client pointer
- DNSCacheData* dns_cache = getDNSCache(cacheID, false);
+ DNSCacheData* dns_cache = _getDNSCache(cacheID, false, line, func, file);
if(dns_cache == NULL)
{
@@ -358,6 +362,7 @@ int findCacheID(int domainID, int clientID, enum query_type query_type)
dns_cache->clientID = clientID;
dns_cache->query_type = query_type;
dns_cache->force_reply = 0u;
+ dns_cache->domainlist_id = -1; // -1 = not set
// Increase counter by one
counters->dns_cache_size++;
@@ -510,6 +515,9 @@ void FTL_reload_all_domainlists(void)
// only after having called gravityDB_open()
read_regex_from_database();
+ // Check for inaccessible adlist URLs
+ check_inaccessible_adlists();
+
// Reset FTL's internal DNS cache storing whether a specific domain
// has already been validated for a specific user
FTL_reset_per_client_domain_data();
@@ -607,6 +615,8 @@ const char * __attribute__ ((const)) get_query_status_str(const enum query_statu
return "DBBUSY";
case QUERY_SPECIAL_DOMAIN:
return "SPECIAL_DOMAIN";
+ case QUERY_CACHE_STALE:
+ return "CACHE_STALE";
case QUERY_STATUS_MAX:
default:
return "INVALID";
@@ -718,6 +728,7 @@ bool __attribute__ ((const)) is_blocked(const enum query_status status)
case QUERY_RETRIED:
case QUERY_RETRIED_DNSSEC:
case QUERY_IN_PROGRESS:
+ case QUERY_CACHE_STALE:
case QUERY_STATUS_MAX:
default:
return false;
@@ -759,22 +770,68 @@ int __attribute__ ((pure)) get_cached_count(void)
return counters->status[QUERY_CACHE];
}
-void _query_set_status(queriesData *query, const enum query_status new_status, const char *file, const int line)
+static const char* __attribute__ ((const)) query_status_str(const enum query_status status)
+{
+ switch (status)
+ {
+ case QUERY_UNKNOWN:
+ return "UNKNOWN";
+ case QUERY_GRAVITY:
+ return "GRAVITY";
+ case QUERY_FORWARDED:
+ return "FORWARDED";
+ case QUERY_CACHE:
+ return "CACHE";
+ case QUERY_REGEX:
+ return "REGEX";
+ case QUERY_DENYLIST:
+ return "DENYLIST";
+ case QUERY_EXTERNAL_BLOCKED_IP:
+ return "EXTERNAL_BLOCKED_IP";
+ case QUERY_EXTERNAL_BLOCKED_NULL:
+ return "EXTERNAL_BLOCKED_NULL";
+ case QUERY_EXTERNAL_BLOCKED_NXRA:
+ return "EXTERNAL_BLOCKED_NXRA";
+ case QUERY_GRAVITY_CNAME:
+ return "GRAVITY_CNAME";
+ case QUERY_REGEX_CNAME:
+ return "REGEX_CNAME";
+ case QUERY_DENYLIST_CNAME:
+ return "DENYLIST_CNAME";
+ case QUERY_RETRIED:
+ return "RETRIED";
+ case QUERY_RETRIED_DNSSEC:
+ return "RETRIED_DNSSEC";
+ case QUERY_IN_PROGRESS:
+ return "IN_PROGRESS";
+ case QUERY_DBBUSY:
+ return "DBBUSY";
+ case QUERY_SPECIAL_DOMAIN:
+ return "SPECIAL_DOMAIN";
+ case QUERY_CACHE_STALE:
+ return "CACHE_STALE";
+ case QUERY_STATUS_MAX:
+ return NULL;
+ }
+ return NULL;
+}
+
+void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file)
{
// Debug logging
if(config.debug & DEBUG_STATUS)
{
- const char *oldstr = get_query_status_str(query->status);
+ const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str(query->status) : "INVALID";
if(query->status == new_status)
{
- log_debug(DEBUG_STATUS, "Query status unchanged: %s (%d) in %s:%i",
- oldstr, query->status, short_path(file), line);
+ log_debug(DEBUG_STATUS, "Query %i: status unchanged: %s (%d) in %s() (%s:%i)",
+ query->id, oldstr, query->status, func, short_path(file), line);
}
else
{
- const char *newstr = get_query_status_str(new_status);
- log_debug(DEBUG_STATUS, "Query status changed: %s (%d) -> %s (%d) in %s:%i",
- oldstr, query->status, newstr, new_status, short_path(file), line);
+ const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str(new_status) : "INVALID";
+ log_debug(DEBUG_STATUS, "Query %i: status changed: %s (%d) -> %s (%d) in %s() (%s:%i)",
+ query->id, oldstr, query->status, newstr, new_status, func, short_path(file), line);
}
}
diff --git a/src/datastructure.h b/src/datastructure.h
index 97a3c84d..aa0a926d 100644
--- a/src/datastructure.h
+++ b/src/datastructure.h
@@ -113,16 +113,17 @@ typedef struct {
enum query_type query_type;
int domainID;
int clientID;
- int deny_regex_id;
+ int domainlist_id;
} DNSCacheData;
void strtolower(char *str);
-uint32_t hashStr(const char *s) __attribute__((const));
+uint32_t hashStr(const char *s) __attribute__((pure));
int findQueryID(const int id);
int findUpstreamID(const char * upstream, const in_port_t port);
int findDomainID(const char *domain, const bool count);
int findClientID(const char *client, const bool count, const bool aliasclient);
-int findCacheID(int domainID, int clientID, enum query_type query_type);
+#define findCacheID(domainID, clientID, query_type, create_new) _findCacheID(domainID, clientID, query_type, create_new, __FUNCTION__, __LINE__, __FILE__)
+int _findCacheID(const int domainID, const int clientID, const enum query_type query_type, const bool create_new, const char *func, const int line, const char *file);
bool isValidIPv4(const char *addr);
bool isValidIPv6(const char *addr);
@@ -130,9 +131,8 @@ bool is_blocked(const enum query_status status) __attribute__ ((const));
int get_blocked_count(void) __attribute__ ((pure));
int get_forwarded_count(void) __attribute__ ((pure));
int get_cached_count(void) __attribute__ ((pure));
-void query_set_status(queriesData *query, const enum query_status new_status);
-#define query_set_status(query, new_status) _query_set_status(query, new_status, __FILE__, __LINE__)
-void _query_set_status(queriesData *query, const enum query_status new_status, const char *file, const int line);
+#define query_set_status(query, new_status) _query_set_status(query, new_status, __FUNCTION__, __LINE__, __FILE__)
+void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file);
void FTL_reload_all_domainlists(void);
void FTL_reset_per_client_domain_data(void);
@@ -152,14 +152,14 @@ const char *get_blocking_mode_str(const enum blocking_mode mode) __attribute__ (
// Pointer getter functions
#define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__)
-queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file);
+queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file);
#define getClient(clientID, checkMagic) _getClient(clientID, checkMagic, __LINE__, __FUNCTION__, __FILE__)
-clientsData* _getClient(int clientID, bool checkMagic, int line, const char * function, const char * file);
+clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file);
#define getDomain(domainID, checkMagic) _getDomain(domainID, checkMagic, __LINE__, __FUNCTION__, __FILE__)
-domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * function, const char * file);
+domainsData* _getDomain(int domainID, bool checkMagic, int line, const char *func, const char *file);
#define getUpstream(upstreamID, checkMagic) _getUpstream(upstreamID, checkMagic, __LINE__, __FUNCTION__, __FILE__)
-upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char * function, const char * file);
+upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char *func, const char *file);
#define getDNSCache(cacheID, checkMagic) _getDNSCache(cacheID, checkMagic, __LINE__, __FUNCTION__, __FILE__)
-DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * function, const char * file);
+DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *func, const char *file);
#endif //DATASTRUCTURE_H
diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c
index 0712db82..0472c491 100644
--- a/src/dnsmasq/cache.c
+++ b/src/dnsmasq/cache.c
@@ -190,7 +190,7 @@ void rehash(int size)
else if (new_size <= hash_size || !(new = whine_malloc(new_size * sizeof(struct crec *))))
return;
- for(i = 0; i < new_size; i++)
+ for (i = 0; i < new_size; i++)
new[i] = NULL;
old = hash_table;
@@ -234,7 +234,8 @@ static void cache_hash(struct crec *crecp)
immortal entries are at the end of the hash-chain.
This allows reverse searches and garbage collection to be optimised */
- struct crec **up = hash_bucket(cache_get_name(crecp));
+ char *name = cache_get_name(crecp);
+ struct crec **up = hash_bucket(name);
if (!(crecp->flags & F_REVERSE))
{
@@ -245,6 +246,11 @@ static void cache_hash(struct crec *crecp)
while (*up && !((*up)->flags & F_IMMORTAL))
up = &((*up)->hash_next);
}
+
+ /* Preserve order when inserting the same name multiple times. */
+ while (*up && hostname_isequal(cache_get_name(*up), name))
+ up = &((*up)->hash_next);
+
crecp->hash_next = *up;
*up = crecp;
}
@@ -375,6 +381,19 @@ static int is_outdated_cname_pointer(struct crec *crecp)
static int is_expired(time_t now, struct crec *crecp)
{
+ /* Don't dump expired entries if they are within the accepted timeout range.
+ The cache becomes approx. LRU. Never use expired DS or DNSKEY entries.
+ Possible values for daemon->cache_max_expiry:
+ -1 == serve cached content regardless how long ago it expired
+ 0 == the option is disabled, expired content isn't served
+ == serve cached content only if it expire less than seconds
+ ago (where n is a positive integer) */
+ if (daemon->cache_max_expiry != 0 &&
+ (daemon->cache_max_expiry == -1 ||
+ difftime(now, crecp->ttd) < daemon->cache_max_expiry) &&
+ !(crecp->flags & (F_DS | F_DNSKEY)))
+ return 0;
+
if (crecp->flags & F_IMMORTAL)
return 0;
@@ -384,6 +403,27 @@ static int is_expired(time_t now, struct crec *crecp)
return 1;
}
+/* Remove entries with a given UID from the cache */
+unsigned int cache_remove_uid(const unsigned int uid)
+{
+ int i;
+ unsigned int removed = 0;
+ struct crec *crecp, **up;
+
+ for (i = 0; i < hash_size; i++)
+ for (crecp = hash_table[i], up = &hash_table[i]; crecp; crecp = crecp->hash_next)
+ if ((crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)) && crecp->uid == uid)
+ {
+ *up = crecp->hash_next;
+ free(crecp);
+ removed++;
+ }
+ else
+ up = &crecp->hash_next;
+
+ return removed;
+}
+
static struct crec *cache_scan_free(char *name, union all_addr *addr, unsigned short class, time_t now,
unsigned int flags, struct crec **target_crec, unsigned int *target_uid)
{
@@ -554,7 +594,7 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho
{
struct crec *new, *target_crec = NULL;
union bigname *big_name = NULL;
- int freed_all = flags & F_REVERSE;
+ int freed_all = (flags & F_REVERSE);
int free_avail = 0;
unsigned int target_uid;
@@ -629,8 +669,12 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho
{
/* For DNSSEC records, uid holds class. */
free_avail = 1; /* Must be free space now. */
+
+ /* condition valid when stale-caching */
+ if (difftime(now, new->ttd) < 0)
+ daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED]++;
+
cache_scan_free(cache_get_name(new), &new->addr, new->uid, now, new->flags, NULL, NULL);
- daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED]++;
}
else
{
@@ -693,7 +737,7 @@ static struct crec *really_insert(char *name, union all_addr *addr, unsigned sho
new->ttd = now + (time_t)ttl;
new->next = new_chain;
new_chain = new;
-
+
return new;
}
@@ -871,7 +915,7 @@ int cache_find_non_terminal(char *name, time_t now)
struct crec *cache_find_by_name(struct crec *crecp, char *name, time_t now, unsigned int prot)
{
struct crec *ans;
- int no_rr = prot & F_NO_RR;
+ int no_rr = (prot & F_NO_RR) || option_bool(OPT_NORR);
prot &= ~F_NO_RR;
@@ -1019,16 +1063,17 @@ struct crec *cache_find_by_addr(struct crec *crecp, union all_addr *addr,
void add_hosts_entry(struct crec *cache, union all_addr *addr, int addrlen,
unsigned int index, struct crec **rhash, int hashsz)
{
- struct crec *lookup = cache_find_by_name(NULL, cache_get_name(cache), 0, cache->flags & (F_IPV4 | F_IPV6));
int i;
unsigned int j;
+ struct crec *lookup = NULL;
/* Remove duplicates in hosts files. */
- if (lookup && (lookup->flags & F_HOSTS) && memcmp(&lookup->addr, addr, addrlen) == 0)
- {
- free(cache);
- return;
- }
+ while ((lookup = cache_find_by_name(lookup, cache_get_name(cache), 0, cache->flags & (F_IPV4 | F_IPV6))))
+ if ((lookup->flags & F_HOSTS) && memcmp(&lookup->addr, addr, addrlen) == 0)
+ {
+ free(cache);
+ return;
+ }
/* Ensure there is only one address -> name mapping (first one trumps)
We do this by steam here, The entries are kept in hash chains, linked
@@ -1133,7 +1178,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr
{
FILE *f = fopen(filename, "r");
char *token = daemon->namebuff, *domain_suffix = NULL;
- int addr_count = 0, name_count = cache_size, lineno = 1;
+ int names_done = 0, name_count = cache_size, lineno = 1;
unsigned int flags = 0;
union all_addr addr;
int atnl, addrlen = 0;
@@ -1169,8 +1214,6 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr
continue;
}
- addr_count++;
-
/* rehash every 1000 names. */
if (rhash && ((name_count - cache_size) > 1000))
{
@@ -1202,6 +1245,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr
cache->ttd = daemon->local_ttl;
add_hosts_entry(cache, &addr, addrlen, index, rhash, hashsz);
name_count++;
+ names_done++;
}
if ((cache = whine_malloc(SIZEOF_BARE_CREC + strlen(canon) + 1)))
{
@@ -1210,6 +1254,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr
cache->ttd = daemon->local_ttl;
add_hosts_entry(cache, &addr, addrlen, index, rhash, hashsz);
name_count++;
+ names_done++;
}
free(canon);
@@ -1226,7 +1271,7 @@ int read_hostsfile(char *filename, unsigned int index, int cache_size, struct cr
if (rhash)
rehash(name_count);
- my_syslog(LOG_INFO, _("read %s - %d addresses"), filename, addr_count);
+ my_syslog(LOG_INFO, _("read %s - %d names"), filename, names_done);
return name_count;
}
@@ -1683,10 +1728,8 @@ int cache_make_stat(struct txt_record *t)
{
/* expand buffer if necessary */
newlen = bytes_needed + 1 + bufflen - bytes_avail;
- if (!(new = whine_malloc(newlen)))
+ if (!(new = whine_realloc(buff, newlen)))
return 0;
- memcpy(new, buff, bufflen);
- free(buff);
p = new + (p - buff);
lenp = p - 1;
buff = new;
@@ -1770,6 +1813,8 @@ void dump_cache(time_t now)
daemon->cachesize, daemon->metrics[METRIC_DNS_CACHE_LIVE_FREED], daemon->metrics[METRIC_DNS_CACHE_INSERTED]);
my_syslog(LOG_INFO, _("queries forwarded %u, queries answered locally %u"),
daemon->metrics[METRIC_DNS_QUERIES_FORWARDED], daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]);
+ if (daemon->cache_max_expiry != 0)
+ my_syslog(LOG_INFO, _("queries answered from stale cache %u"), daemon->metrics[METRIC_DNS_STALE_ANSWERED]);
#ifdef HAVE_AUTH
my_syslog(LOG_INFO, _("queries for authoritative zones %u"), daemon->metrics[METRIC_DNS_AUTH_ANSWERED]);
#endif
@@ -1784,16 +1829,23 @@ void dump_cache(time_t now)
if (!(serv->flags & SERV_MARK))
{
int port;
- unsigned int queries = 0, failed_queries = 0;
+ unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0, retrys = 0;
+ unsigned int sigma_latency = 0, count_latency = 0;
+
for (serv1 = serv; serv1; serv1 = serv1->next)
if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr))
{
serv1->flags |= SERV_MARK;
queries += serv1->queries;
failed_queries += serv1->failed_queries;
+ nxdomain_replies += serv1->nxdomain_replies;
+ retrys += serv1->retrys;
+ sigma_latency += serv1->query_latency;
+ count_latency++;
}
port = prettyprint_addr(&serv->addr, daemon->addrbuff);
- my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried or failed %u"), daemon->addrbuff, port, queries, failed_queries);
+ my_syslog(LOG_INFO, _("server %s#%d: queries sent %u, retried %u, failed %u, nxdomain replies %u, avg. latency %ums"),
+ daemon->addrbuff, port, queries, retrys, failed_queries, nxdomain_replies, sigma_latency/count_latency);
}
if (option_bool(OPT_DEBUG) || option_bool(OPT_LOG))
@@ -1888,7 +1940,10 @@ void dump_cache(time_t now)
char *record_source(unsigned int index)
{
struct hostsfile *ah;
-
+#ifdef HAVE_INOTIFY
+ struct dyndir *dd;
+#endif
+
if (index == SRC_CONFIG)
return "config";
else if (index == SRC_HOSTS)
@@ -1899,9 +1954,11 @@ char *record_source(unsigned int index)
return ah->fname;
#ifdef HAVE_INOTIFY
- for (ah = daemon->dynamic_dirs; ah; ah = ah->next)
- if (ah->index == index)
- return ah->fname;
+ /* Dynamic directories contain multiple files */
+ for (dd = daemon->dynamic_dirs; dd; dd = dd->next)
+ for (ah = dd->files; ah; ah = ah->next)
+ if (ah->index == index)
+ return ah->fname;
#endif
return "";
@@ -2129,6 +2186,8 @@ void _log_query(unsigned int flags, char *name, union all_addr *addr, char *arg,
name = arg;
verb = daemon->addrbuff;
}
+ else if (flags & F_STALE)
+ source = "cached-stale";
else
source = "cached";
diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h
index 18429bf3..3ee9be1f 100644
--- a/src/dnsmasq/config.h
+++ b/src/dnsmasq/config.h
@@ -24,6 +24,7 @@
#define KEYBLOCK_LEN 40 /* choose to minimise fragmentation when storing DNSSEC keys */
#define DNSSEC_WORK 50 /* Max number of queries to validate one question */
#define TIMEOUT 10 /* drop UDP queries after TIMEOUT seconds */
+#define SMALL_PORT_RANGE 30 /* If DNS port range is smaller than this, use different allocation. */
#define FORWARD_TEST 1000 /* try all servers every 1000 queries */
#define FORWARD_TIME 600 /* or 10 minutes */
#define UDP_TEST_TIME 60 /* How often to reset our idea of max packet size. */
@@ -60,6 +61,8 @@
#define SOA_EXPIRY 1209600 /* SOA expiry default */
#define LOOP_TEST_DOMAIN "test" /* domain for loop testing, "test" is reserved by RFC 2606 and won't therefore clash */
#define LOOP_TEST_TYPE T_TXT
+#define DEFAULT_FAST_RETRY 1000 /* ms, default delay before fast retry */
+#define STALE_CACHE_EXPIRY 86400 /* 1 day in secs, default maximum expiry time for stale cache data */
/* compile-time options: uncomment below to enable or do eg.
make COPTS=-DHAVE_BROKEN_RTC
@@ -200,6 +203,17 @@ RESOLVFILE
/* #define HAVE_DNSSEC */
/* #define HAVE_NFTSET */
+/* Pi-hole definitions */
+#define HAVE_LUASCRIPT
+#define HAVE_IDN
+#define HAVE_DNSSEC
+#ifdef DNSMASQ_ALL_OPTS
+ #define HAVE_DBUS
+ #define HAVE_CONNTRACK
+ #define HAVE_NFTSET
+#endif
+/***********************/
+
/* Default locations for important system files. */
#ifndef LEASEFILE
diff --git a/src/dnsmasq/crypto.c b/src/dnsmasq/crypto.c
index 060e27ff..2678683e 100644
--- a/src/dnsmasq/crypto.c
+++ b/src/dnsmasq/crypto.c
@@ -309,14 +309,14 @@ static int dnsmasq_gostdsa_verify(struct blockdata *key_data, unsigned int key_l
mpz_init(y);
}
- mpz_import(x, 32 , 1, 1, 0, 0, p);
- mpz_import(y, 32 , 1, 1, 0, 0, p + 32);
+ mpz_import(x, 32, -1, 1, 0, 0, p);
+ mpz_import(y, 32, -1, 1, 0, 0, p + 32);
if (!ecc_point_set(gost_key, x, y))
- return 0;
+ return 0;
- mpz_import(sig_struct->r, 32, 1, 1, 0, 0, sig);
- mpz_import(sig_struct->s, 32, 1, 1, 0, 0, sig + 32);
+ mpz_import(sig_struct->s, 32, 1, 1, 0, 0, sig);
+ mpz_import(sig_struct->r, 32, 1, 1, 0, 0, sig + 32);
return nettle_gostdsa_verify(gost_key, digest_len, digest, sig_struct);
}
@@ -390,7 +390,12 @@ static int (*verify_func(int algo))(struct blockdata *key_data, unsigned int key
return dnsmasq_ecdsa_verify;
#if MIN_VERSION(3, 1)
- case 15: case 16:
+ case 15:
+ return dnsmasq_eddsa_verify;
+#endif
+
+#if MIN_VERSION(3, 6)
+ case 16:
return dnsmasq_eddsa_verify;
#endif
}
@@ -425,7 +430,9 @@ char *ds_digest_name(int digest)
{
case 1: return "sha1";
case 2: return "sha256";
- case 3: return "gosthash94";
+#if MIN_VERSION(3, 6)
+ case 3: return "gosthash94cp";
+#endif
case 4: return "sha384";
default: return NULL;
}
@@ -444,11 +451,17 @@ char *algo_digest_name(int algo)
case 7: return "sha1"; /* RSASHA1-NSEC3-SHA1 */
case 8: return "sha256"; /* RSA/SHA-256 */
case 10: return "sha512"; /* RSA/SHA-512 */
- case 12: return "gosthash94"; /* ECC-GOST */
+#if MIN_VERSION(3, 6)
+ case 12: return "gosthash94cp"; /* ECC-GOST */
+#endif
case 13: return "sha256"; /* ECDSAP256SHA256 */
case 14: return "sha384"; /* ECDSAP384SHA384 */
+#if MIN_VERSION(3, 1)
case 15: return "null_hash"; /* ED25519 */
+# if MIN_VERSION(3, 6)
case 16: return "null_hash"; /* ED448 */
+# endif
+#endif
default: return NULL;
}
}
diff --git a/src/dnsmasq/dbus.c b/src/dnsmasq/dbus.c
index 0c55ea5e..fd5d1ca6 100644
--- a/src/dnsmasq/dbus.c
+++ b/src/dnsmasq/dbus.c
@@ -91,6 +91,11 @@ const char* introspection_xml_template =
" \n"
" \n"
" \n"
+" \n"
+" \n"
+" \n"
+" \n"
+" \n"
" \n"
"\n";
@@ -287,6 +292,11 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings)
u16 flags = 0;
char interface[IF_NAMESIZE];
char *str_addr, *str_domain = NULL;
+ struct server_details sdetails = { 0 };
+ sdetails.addr = &addr;
+ sdetails.source_addr = &source_addr;
+ sdetails.interface = interface;
+ sdetails.flags = &flags;
if (strings)
{
@@ -369,20 +379,6 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings)
strcpy(str_addr, str);
}
- /* parse the IP address */
- if ((addr_err = parse_server(str_addr, &addr, &source_addr, (char *) &interface, &flags)))
- {
- error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS,
- "Invalid IP address '%s': %s",
- str, addr_err);
- break;
- }
-
- /* 0.0.0.0 for server address == NULL, for Dbus */
- if (addr.in.sin_family == AF_INET &&
- addr.in.sin_addr.s_addr == 0)
- flags |= SERV_LITERAL_ADDRESS;
-
if (strings)
{
char *p;
@@ -396,7 +392,31 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings)
else
p = NULL;
- add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str_domain, NULL);
+ if (strings && strlen(str_addr) == 0)
+ add_update_server(SERV_LITERAL_ADDRESS | SERV_FROM_DBUS, &addr, &source_addr, interface, str_domain, NULL);
+ else
+ {
+ if ((addr_err = parse_server(str_addr, &sdetails)))
+ {
+ error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS,
+ "Invalid IP address '%s': %s",
+ str, addr_err);
+ break;
+ }
+
+ while (parse_server_next(&sdetails))
+ {
+ if ((addr_err = parse_server_addr(&sdetails)))
+ {
+ error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS,
+ "Invalid IP address '%s': %s",
+ str, addr_err);
+ break;
+ }
+
+ add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str_domain, NULL);
+ }
+ }
} while ((str_domain = p));
}
else
@@ -410,11 +430,40 @@ static DBusMessage* dbus_read_servers_ex(DBusMessage *message, int strings)
if (dbus_message_iter_get_arg_type(&string_iter) == DBUS_TYPE_STRING)
dbus_message_iter_get_basic(&string_iter, &str);
dbus_message_iter_next (&string_iter);
+
+ if ((addr_err = parse_server(str_addr, &sdetails)))
+ {
+ error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS,
+ "Invalid IP address '%s': %s",
+ str, addr_err);
+ break;
+ }
- add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str, NULL);
+ while (parse_server_next(&sdetails))
+ {
+ if ((addr_err = parse_server_addr(&sdetails)))
+ {
+ error = dbus_message_new_error_printf(message, DBUS_ERROR_INVALID_ARGS,
+ "Invalid IP address '%s': %s",
+ str, addr_err);
+ break;
+ }
+
+ /* 0.0.0.0 for server address == NULL, for Dbus */
+ if (addr.in.sin_family == AF_INET &&
+ addr.in.sin_addr.s_addr == 0)
+ flags |= SERV_LITERAL_ADDRESS;
+ else
+ flags &= ~SERV_LITERAL_ADDRESS;
+
+ add_update_server(flags | SERV_FROM_DBUS, &addr, &source_addr, interface, str, NULL);
+ }
} while (dbus_message_iter_get_arg_type(&string_iter) == DBUS_TYPE_STRING);
}
-
+
+ if (sdetails.orig_hostinfo)
+ freeaddrinfo(sdetails.orig_hostinfo);
+
/* jump to next element in outer array */
dbus_message_iter_next(&array_iter);
}
@@ -644,6 +693,77 @@ static DBusMessage *dbus_get_metrics(DBusMessage* message)
return reply;
}
+static void add_dict_entry(DBusMessageIter *container, const char *key, const char *val)
+{
+ DBusMessageIter dict;
+
+ dbus_message_iter_open_container(container, DBUS_TYPE_DICT_ENTRY, NULL, &dict);
+ dbus_message_iter_append_basic(&dict, DBUS_TYPE_STRING, &key);
+ dbus_message_iter_append_basic(&dict, DBUS_TYPE_STRING, &val);
+ dbus_message_iter_close_container(container, &dict);
+}
+
+static void add_dict_int(DBusMessageIter *container, const char *key, const unsigned int val)
+{
+ snprintf(daemon->namebuff, MAXDNAME, "%u", val);
+
+ add_dict_entry(container, key, daemon->namebuff);
+}
+
+static DBusMessage *dbus_get_server_metrics(DBusMessage* message)
+{
+ DBusMessage *reply = dbus_message_new_method_return(message);
+ DBusMessageIter server_array, dict_array, server_iter;
+ struct server *serv;
+
+ dbus_message_iter_init_append(reply, &server_iter);
+ dbus_message_iter_open_container(&server_iter, DBUS_TYPE_ARRAY, "a{ss}", &server_array);
+
+ /* sum counts from different records for same server */
+ for (serv = daemon->servers; serv; serv = serv->next)
+ serv->flags &= ~SERV_MARK;
+
+ for (serv = daemon->servers; serv; serv = serv->next)
+ if (!(serv->flags & SERV_MARK))
+ {
+ unsigned int port;
+ unsigned int queries = 0, failed_queries = 0, nxdomain_replies = 0, retrys = 0;
+ unsigned int sigma_latency = 0, count_latency = 0;
+
+ struct server *serv1;
+
+ for (serv1 = serv; serv1; serv1 = serv1->next)
+ if (!(serv1->flags & SERV_MARK) && sockaddr_isequal(&serv->addr, &serv1->addr))
+ {
+ serv1->flags |= SERV_MARK;
+ queries += serv1->queries;
+ failed_queries += serv1->failed_queries;
+ nxdomain_replies += serv1->nxdomain_replies;
+ retrys += serv1->retrys;
+ sigma_latency += serv1->query_latency;
+ count_latency++;
+ }
+
+ dbus_message_iter_open_container(&server_array, DBUS_TYPE_ARRAY, "{ss}", &dict_array);
+
+ port = prettyprint_addr(&serv->addr, daemon->namebuff);
+ add_dict_entry(&dict_array, "address", daemon->namebuff);
+
+ add_dict_int(&dict_array, "port", port);
+ add_dict_int(&dict_array, "queries", serv->queries);
+ add_dict_int(&dict_array, "failed_queries", serv->failed_queries);
+ add_dict_int(&dict_array, "nxdomain", serv->nxdomain_replies);
+ add_dict_int(&dict_array, "retries", serv->retrys);
+ add_dict_int(&dict_array, "latency", sigma_latency/count_latency);
+
+ dbus_message_iter_close_container(&server_array, &dict_array);
+ }
+
+ dbus_message_iter_close_container(&server_iter, &server_array);
+
+ return reply;
+}
+
DBusHandlerResult message_handler(DBusConnection *connection,
DBusMessage *message,
void *user_data)
@@ -719,6 +839,14 @@ DBusHandlerResult message_handler(DBusConnection *connection,
{
reply = dbus_get_metrics(message);
}
+ else if (strcmp(method, "GetServerMetrics") == 0)
+ {
+ reply = dbus_get_server_metrics(message);
+ }
+ else if (strcmp(method, "ClearMetrics") == 0)
+ {
+ clear_metrics();
+ }
else if (strcmp(method, "ClearCache") == 0)
clear_cache = 1;
else
@@ -761,8 +889,11 @@ char *dbus_init(void)
dbus_error_init (&dbus_error);
if (!(connection = dbus_bus_get (DBUS_BUS_SYSTEM, &dbus_error)))
- return NULL;
-
+ {
+ dbus_error_free(&dbus_error);
+ return NULL;
+ }
+
dbus_connection_set_exit_on_disconnect(connection, FALSE);
dbus_connection_set_watch_functions(connection, add_watch, remove_watch,
NULL, NULL, NULL);
diff --git a/src/dnsmasq/dhcp-common.c b/src/dnsmasq/dhcp-common.c
index b33c5dfc..1b6cc849 100644
--- a/src/dnsmasq/dhcp-common.c
+++ b/src/dnsmasq/dhcp-common.c
@@ -685,6 +685,7 @@ const struct opttab_t {
{ "client-machine-id", 97, 0 },
{ "posix-timezone", 100, OT_NAME }, /* RFC 4833, Sec. 2 */
{ "tzdb-timezone", 101, OT_NAME }, /* RFC 4833, Sec. 2 */
+ { "ipv6-only", 108, 4 | OT_DEC }, /* RFC 8925 */
{ "subnet-select", 118, OT_INTERNAL },
{ "domain-search", 119, OT_RFC1035_NAME },
{ "sip-server", 120, 0 },
@@ -721,6 +722,8 @@ static const struct opttab_t opttab6[] = {
{ "sntp-server", 31, OT_ADDR_LIST },
{ "information-refresh-time", 32, OT_TIME },
{ "FQDN", 39, OT_INTERNAL | OT_RFC1035_NAME },
+ { "posix-timezone", 41, OT_NAME }, /* RFC 4833, Sec. 3 */
+ { "tzdb-timezone", 42, OT_NAME }, /* RFC 4833, Sec. 3 */
{ "ntp-server", 56, 0 /* OT_ADDR_LIST | OT_RFC1035_NAME */ },
{ "bootfile-url", 59, OT_NAME },
{ "bootfile-param", 60, OT_CSTRING },
@@ -1017,7 +1020,10 @@ void log_relay(int family, struct dhcp_relay *relay)
{
int broadcast = relay->server.addr4.s_addr == 0;
inet_ntop(family, &relay->local, daemon->addrbuff, ADDRSTRLEN);
- inet_ntop(family, &relay->server, daemon->namebuff, ADDRSTRLEN);
+ inet_ntop(family, &relay->server, daemon->namebuff, ADDRSTRLEN);
+
+ if (family == AF_INET && relay->port != DHCP_SERVER_PORT)
+ sprintf(daemon->namebuff + strlen(daemon->namebuff), "#%u", relay->port);
#ifdef HAVE_DHCP6
struct in6_addr multicast;
@@ -1025,7 +1031,11 @@ void log_relay(int family, struct dhcp_relay *relay)
inet_pton(AF_INET6, ALL_SERVERS, &multicast);
if (family == AF_INET6)
- broadcast = IN6_ARE_ADDR_EQUAL(&relay->server.addr6, &multicast);
+ {
+ broadcast = IN6_ARE_ADDR_EQUAL(&relay->server.addr6, &multicast);
+ if (relay->port != DHCPV6_SERVER_PORT)
+ sprintf(daemon->namebuff + strlen(daemon->namebuff), "#%u", relay->port);
+ }
#endif
diff --git a/src/dnsmasq/dhcp-protocol.h b/src/dnsmasq/dhcp-protocol.h
index 75c9cd3c..e281143a 100644
--- a/src/dnsmasq/dhcp-protocol.h
+++ b/src/dnsmasq/dhcp-protocol.h
@@ -64,6 +64,7 @@
#define OPTION_SIP_SERVER 120
#define OPTION_VENDOR_IDENT 124
#define OPTION_VENDOR_IDENT_OPT 125
+#define OPTION_MUD_URL_V4 161
#define OPTION_END 255
#define SUBOPT_CIRCUIT_ID 1
diff --git a/src/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c
index 6104c87d..42d819f0 100644
--- a/src/dnsmasq/dhcp.c
+++ b/src/dnsmasq/dhcp.c
@@ -177,8 +177,7 @@ void dhcp_packet(time_t now, int pxe_fd)
return;
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_DHCP, (void *)daemon->dhcp_packet.iov_base, sz, (union mysockaddr *)&dest, NULL,
- pxe_fd ? PXE_PORT : daemon->dhcp_server_port);
+ dump_packet_udp(DUMP_DHCP, (void *)daemon->dhcp_packet.iov_base, sz, (union mysockaddr *)&dest, NULL, fd);
#endif
#if defined (HAVE_LINUX_NETWORK)
@@ -464,8 +463,8 @@ void dhcp_packet(time_t now, int pxe_fd)
dest.sin_addr = mess->yiaddr;
dest.sin_port = htons(daemon->dhcp_client_port);
- dump_packet(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL,
- (union mysockaddr *)&dest, daemon->dhcp_server_port);
+ dump_packet_udp(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL,
+ (union mysockaddr *)&dest, fd);
#endif
send_via_bpf(mess, iov.iov_len, iface_addr, &ifr);
@@ -478,8 +477,8 @@ void dhcp_packet(time_t now, int pxe_fd)
#endif
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL,
- (union mysockaddr *)&dest, daemon->dhcp_server_port);
+ dump_packet_udp(DUMP_DHCP, (void *)iov.iov_base, iov.iov_len, NULL,
+ (union mysockaddr *)&dest, fd);
#endif
while(retry_send(sendmsg(fd, &msg, 0)));
@@ -1121,7 +1120,7 @@ static int relay_upstream4(int iface_index, struct dhcp_packet *mess, size_t sz)
to.sa.sa_family = AF_INET;
to.in.sin_addr = relay->server.addr4;
- to.in.sin_port = htons(daemon->dhcp_server_port);
+ to.in.sin_port = htons(relay->port);
/* Broadcasting to server. */
if (relay->server.addr4.s_addr == 0)
@@ -1147,8 +1146,8 @@ static int relay_upstream4(int iface_index, struct dhcp_packet *mess, size_t sz)
fromsock.in.sin_port = htons(daemon->dhcp_server_port);
fromsock.in.sin_addr = from.addr4;
fromsock.sa.sa_family = AF_INET;
-
- dump_packet(DUMP_DHCP, (void *)mess, sz, &fromsock, &to, 0);
+
+ dump_packet_udp(DUMP_DHCP, (void *)mess, sz, &fromsock, &to, -1);
}
#endif
diff --git a/src/dnsmasq/dhcp6-protocol.h b/src/dnsmasq/dhcp6-protocol.h
index 332d5364..ce166037 100644
--- a/src/dnsmasq/dhcp6-protocol.h
+++ b/src/dnsmasq/dhcp6-protocol.h
@@ -63,6 +63,7 @@
#define OPTION6_FQDN 39
#define OPTION6_NTP_SERVER 56
#define OPTION6_CLIENT_MAC 79
+#define OPTION6_MUD_URL 112
#define NTP_SUBOPTION_SRV_ADDR 1
#define NTP_SUBOPTION_MC_ADDR 2
diff --git a/src/dnsmasq/dhcp6.c b/src/dnsmasq/dhcp6.c
index edb87a4a..1c8c6794 100644
--- a/src/dnsmasq/dhcp6.c
+++ b/src/dnsmasq/dhcp6.c
@@ -119,8 +119,8 @@ void dhcp6_packet(time_t now)
return;
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_DHCPV6, (void *)daemon->dhcp_packet.iov_base, sz,
- (union mysockaddr *)&from, NULL, DHCPV6_SERVER_PORT);
+ dump_packet_udp(DUMP_DHCPV6, (void *)daemon->dhcp_packet.iov_base, sz,
+ (union mysockaddr *)&from, NULL, daemon->dhcp6fd);
#endif
for (cmptr = CMSG_FIRSTHDR(&msg); cmptr; cmptr = CMSG_NXTHDR(&msg, cmptr))
@@ -142,8 +142,8 @@ void dhcp6_packet(time_t now)
if (relay_reply6(&from, sz, ifr.ifr_name))
{
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL,
- (union mysockaddr *)&from, DHCPV6_SERVER_PORT);
+ dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL,
+ (union mysockaddr *)&from, daemon->dhcp6fd);
#endif
while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base,
@@ -254,8 +254,8 @@ void dhcp6_packet(time_t now)
from.sin6_port = htons(port);
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1),
- NULL, (union mysockaddr *)&from, DHCPV6_SERVER_PORT);
+ dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1),
+ NULL, (union mysockaddr *)&from, daemon->dhcp6fd);
#endif
while (retry_send(sendto(daemon->dhcp6fd, daemon->outpacket.iov_base,
diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c
index add4dd97..59bd4c15 100644
--- a/src/dnsmasq/dnsmasq.c
+++ b/src/dnsmasq/dnsmasq.c
@@ -272,6 +272,10 @@ int main_dnsmasq (int argc, char **argv)
if (daemon->max_port < daemon->min_port)
die(_("max_port cannot be smaller than min_port"), NULL, EC_BADCONF);
+
+ if (daemon->max_port != 0 &&
+ daemon->max_port - daemon->min_port + 1 < daemon->randport_limit)
+ die(_("port_limit must not be larger than available port range"), NULL, EC_BADCONF);
now = dnsmasq_time();
@@ -1068,19 +1072,20 @@ int main_dnsmasq (int argc, char **argv)
while (!terminate)
{
- int timeout = -1;
+ int timeout = fast_retry(now);
poll_reset();
/* Whilst polling for the dbus, or doing a tftp transfer, wake every quarter second */
- if (daemon->tftp_trans ||
- (option_bool(OPT_DBUS) && !daemon->dbus))
+ if ((daemon->tftp_trans || (option_bool(OPT_DBUS) && !daemon->dbus)) &&
+ (timeout == -1 || timeout > 250))
timeout = 250;
-
+
/* Wake every second whilst waiting for DAD to complete */
- else if (is_dad_listeners())
+ else if (is_dad_listeners() &&
+ (timeout == -1 || timeout > 1000))
timeout = 1000;
-
+
set_dns_listeners();
#ifdef HAVE_DBUS
@@ -1689,9 +1694,10 @@ static void poll_resolv(int force, int do_reload, time_t now)
else
{
res->logged = 0;
- if (force || (statbuf.st_mtime != res->mtime))
+ if (force || (statbuf.st_mtime != res->mtime || statbuf.st_ino != res->ino))
{
res->mtime = statbuf.st_mtime;
+ res->ino = statbuf.st_ino;
if (difftime(statbuf.st_mtime, last_change) > 0.0)
{
last_change = statbuf.st_mtime;
@@ -2050,9 +2056,6 @@ static void check_dns_listeners(time_t now)
FTL_TCP_worker_terminating(true);
/**********************************************/
- shutdown(confd, SHUT_RDWR);
- close(confd);
-
if (buff)
free(buff);
@@ -2228,10 +2231,11 @@ int delay_dhcp(time_t start, int sec, int fd, uint32_t addr, unsigned short id)
#endif /* HAVE_DHCP */
/******************************** Pi-hole modification ********************************/
-void print_dnsmasq_version(void)
+void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal)
{
- printf("****************************** dnsmasq ******************************\n");
- printf(_("Version: %s\n"), VERSION);
- printf(_("Compile options: %s\n\n"), compile_opts);
+ printf("****************************** %s%sdnsmasq%s ******************************\n",
+ bold, yellow, normal);
+ printf(_("Version: %s%s%s%s\n"), bold, green, VERSION, normal);
+ printf(_("Features: %s\n\n"), compile_opts);
}
/**************************************************************************************/
diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h
index dca47b74..ff02fd5c 100644
--- a/src/dnsmasq/dnsmasq.h
+++ b/src/dnsmasq/dnsmasq.h
@@ -14,13 +14,6 @@
along with this program. If not, see .
*/
-/* Pi-hole definitions */
-#define HAVE_DNSSEC
-#define HAVE_DNSSEC_STATIC
-#define HAVE_IDN
-#define HAVE_LUASCRIPT
-/***********************/
-
#define COPYRIGHT "Copyright (c) 2000-2022 Simon Kelley"
/* We do defines that influence behavior of stdio.h, so complain
@@ -140,6 +133,7 @@ typedef unsigned long long u64;
#include
#include
#include
+#include
#ifndef HAVE_LINUX_NETWORK
# include
#endif
@@ -286,7 +280,8 @@ struct event_desc {
#define OPT_FILTER_AAAA 68
#define OPT_STRIP_ECS 69
#define OPT_STRIP_MAC 70
-#define OPT_LAST 71
+#define OPT_NORR 71
+#define OPT_LAST 72
#define OPTION_BITS (sizeof(unsigned int)*8)
#define OPTION_SIZE ( (OPT_LAST/OPTION_BITS)+((OPT_LAST%OPTION_BITS)!=0) )
@@ -520,6 +515,7 @@ struct crec {
#define F_DOMAINSRV (1u<<28)
#define F_RCODE (1u<<29)
#define F_SRV (1u<<30)
+#define F_STALE (1u<<31)
#define UID_NONE 0
/* Values of uid in crecs with F_CONFIG bit set. */
@@ -593,7 +589,8 @@ struct server {
struct serverfd *sfd;
int tcpfd, edns_pktsz;
time_t pktsz_reduced;
- unsigned int queries, failed_queries;
+ unsigned int queries, failed_queries, nxdomain_replies, retrys;
+ unsigned int query_latency, mma_latency;
time_t forwardtime;
int forwardcount;
#ifdef HAVE_LOOP
@@ -677,6 +674,7 @@ struct resolvc {
struct resolvc *next;
int is_default, logged;
time_t mtime;
+ ino_t ino;
char *name;
#ifdef HAVE_INOTIFY
int wd; /* inotify watch descriptor */
@@ -695,10 +693,17 @@ struct hostsfile {
struct hostsfile *next;
int flags;
char *fname;
+ unsigned int index; /* matches to cache entries for logging */
+};
+
+struct dyndir {
+ struct dyndir *next;
+ struct hostsfile *files;
+ int flags;
+ char *dname;
#ifdef HAVE_INOTIFY
int wd; /* inotify watch descriptor */
#endif
- unsigned int index; /* matches to cache entries for logging */
};
/* packet-dump flags */
@@ -766,11 +771,13 @@ struct frec {
unsigned short new_id;
int forwardall, flags;
time_t time;
+ u32 forward_timestamp;
+ int forward_delay;
unsigned char *hash[HASH_SIZE];
-#ifdef HAVE_DNSSEC
- int class, work_counter;
struct blockdata *stash; /* Saved reply, whilst we validate */
size_t stash_len;
+#ifdef HAVE_DNSSEC
+ int class, work_counter;
struct frec *dependent; /* Query awaiting internally-generated DNSKEY or DS query */
struct frec *next_dependent; /* list of above. */
struct frec *blocking_query; /* Query which is blocking us. */
@@ -987,6 +994,8 @@ struct dhcp_bridge {
struct cond_domain {
char *domain, *prefix; /* prefix is text-prefix on domain name */
+ char *interface; /* These two set when domain comes from interface. */
+ struct addrlist *al;
struct in_addr start, end;
struct in6_addr start6, end6;
int is6, indexed, prefixlen;
@@ -1094,6 +1103,7 @@ struct dhcp_relay {
union all_addr local, server;
char *interface; /* Allowable interface for replies from server, and dest for IPv6 multicast */
int iface_index; /* working - interface in which requests arrived, for return */
+ int port; /* Port of relay we forward to. */
#ifdef HAVE_SCRIPT
struct snoop_record {
struct in6_addr client, prefix;
@@ -1147,6 +1157,7 @@ extern struct daemon {
int log_fac; /* log facility */
char *log_file; /* optional log file */
int max_logs; /* queue limit */
+ int randport_limit; /* Maximum number of source ports for query. */
int cachesize, ftabsize;
int port, query_port, min_port, max_port;
unsigned long local_ttl, neg_ttl, max_ttl, min_cache_ttl, max_cache_ttl, auth_ttl, dhcp_ttl, use_dhcp_ttl;
@@ -1154,6 +1165,7 @@ extern struct daemon {
u32 umbrella_org;
u32 umbrella_asset;
u8 umbrella_device[8];
+ int host_index;
struct hostsfile *addn_hosts;
struct dhcp_context *dhcp, *dhcp6;
struct ra_interface *ra_interfaces;
@@ -1174,7 +1186,8 @@ extern struct daemon {
int doing_ra, doing_dhcp6;
struct dhcp_netid_list *dhcp_ignore, *dhcp_ignore_names, *dhcp_gen_names;
struct dhcp_netid_list *force_broadcast, *bootp_dynamic;
- struct hostsfile *dhcp_hosts_file, *dhcp_opts_file, *dynamic_dirs;
+ struct hostsfile *dhcp_hosts_file, *dhcp_opts_file;
+ struct dyndir *dynamic_dirs;
int dhcp_max, tftp_max, tftp_mtu;
int dhcp_server_port, dhcp_client_port;
int start_tftp_port, end_tftp_port;
@@ -1191,6 +1204,8 @@ extern struct daemon {
int dump_mask;
unsigned long soa_sn, soa_refresh, soa_retry, soa_expiry;
u32 metrics[__METRIC_MAX];
+ int fast_retry_time, fast_retry_timeout;
+ int cache_max_expiry;
#ifdef HAVE_DNSSEC
struct ds_config *ds;
char *timestamp_file;
@@ -1283,6 +1298,14 @@ extern struct daemon {
#endif
} *daemon;
+struct server_details {
+ union mysockaddr *addr, *source_addr;
+ struct addrinfo *hostinfo, *orig_hostinfo;
+ char *interface, *source, *scope_id, *interface_opt;
+ int serv_port, source_port, addr_type, scope_index, valid;
+ u16 *flags;
+};
+
/* cache.c */
void cache_init(void);
void next_uid(struct crec *crecp);
@@ -1300,6 +1323,7 @@ struct crec *cache_find_by_name(struct crec *crecp,
char *name, time_t now, unsigned int prot);
void cache_end_insert(void);
void cache_start_insert(void);
+unsigned int cache_remove_uid(const unsigned int uid);
int cache_recv_insert(time_t now, int fd);
struct crec *cache_insert(char *name, union all_addr *addr, unsigned short class,
time_t now, unsigned long ttl, unsigned int flags);
@@ -1349,7 +1373,8 @@ void report_addresses(struct dns_header *header, size_t len, u32 mark);
#endif
size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
struct in_addr local_addr, struct in_addr local_netmask,
- time_t now, int ad_reqd, int do_bit, int have_pseudoheader);
+ time_t now, int ad_reqd, int do_bit, int have_pseudoheader,
+ int *stale);
int check_for_bogus_wildcard(struct dns_header *header, size_t qlen, char *name,
time_t now);
int check_for_ignored_address(struct dns_header *header, size_t qlen);
@@ -1408,12 +1433,15 @@ void *safe_malloc(size_t size);
void safe_strncpy(char *dest, const char *src, size_t size);
void safe_pipe(int *fd, int read_noblock);
void *whine_malloc(size_t size);
+void *whine_realloc(void *ptr, size_t size);
int sa_len(union mysockaddr *addr);
int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2);
+int sockaddr_isnull(const union mysockaddr *s);
int hostname_order(const char *a, const char *b);
int hostname_isequal(const char *a, const char *b);
int hostname_issubdomain(char *a, char *b);
time_t dnsmasq_time(void);
+u32 dnsmasq_milliseconds(void);
int netmask_length(struct in_addr mask);
int is_same_net(struct in_addr a, struct in_addr b, struct in_addr mask);
int is_same_net_prefix(struct in_addr a, struct in_addr b, int prefix);
@@ -1457,8 +1485,9 @@ void read_servers_file(void);
void set_option_bool(unsigned int opt);
void reset_option_bool(unsigned int opt);
struct hostsfile *expand_filelist(struct hostsfile *list);
-char *parse_server(char *arg, union mysockaddr *addr,
- union mysockaddr *source_addr, char *interface, u16 *flags);
+char *parse_server(char *arg, struct server_details *sdetails);
+char *parse_server_addr(struct server_details *sdetails);
+int parse_server_next(struct server_details *sdetails);
int option_read_dynfile(char *file, int flags);
/* forward.c */
@@ -1473,6 +1502,7 @@ int send_from(int fd, int nowild, char *packet, size_t len,
void resend_query(void);
int allocate_rfd(struct randfd_list **fdlp, struct server *serv);
void free_rfds(struct randfd_list **fdlp);
+int fast_retry(time_t now);
/* network.c */
int indextoname(int fd, int index, char *name);
@@ -1825,8 +1855,10 @@ int do_arp_script_run(void);
/* dump.c */
#ifdef HAVE_DUMPFILE
void dump_init(void);
-void dump_packet(int mask, void *packet, size_t len, union mysockaddr *src,
- union mysockaddr *dst, int port);
+void dump_packet_udp(int mask, void *packet, size_t len, union mysockaddr *src,
+ union mysockaddr *dst, int fd);
+void dump_packet_icmp(int mask, void *packet, size_t len, union mysockaddr *src,
+ union mysockaddr *dst);
#endif
/* domain-match.c */
diff --git a/src/dnsmasq/dnssec.c b/src/dnsmasq/dnssec.c
index 9965eea3..219ba9af 100644
--- a/src/dnsmasq/dnssec.c
+++ b/src/dnsmasq/dnssec.c
@@ -979,10 +979,13 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch
}
/* The DNS packet is expected to contain the answer to a DS query
- Put all DSs in the answer which are valid into the cache.
+ Put all DSs in the answer which are valid and have hash and signature algos
+ we support into the cache.
Also handles replies which prove that there's no DS at this location,
either because the zone is unsigned or this isn't a zone cut. These are
cached too.
+ If none of the DS's are for supported algos, treat the answer as if
+ it's a proof of no DS at this location. RFC4035 para 5.2.
return codes:
STAT_OK At least one valid DS found and in cache.
STAT_BOGUS no DS in reply or not signed, fails validation, bad packet.
@@ -993,8 +996,8 @@ int dnssec_validate_by_ds(time_t now, struct dns_header *header, size_t plen, ch
int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char *name, char *keyname, int class)
{
unsigned char *p = (unsigned char *)(header+1);
- int qtype, qclass, rc, i, neganswer, nons, neg_ttl = 0;
- int aclass, atype, rdlen;
+ int qtype, qclass, rc, i, neganswer, nons, neg_ttl = 0, found_supported = 0;
+ int aclass, atype, rdlen, flags;
unsigned long ttl;
union all_addr a;
@@ -1065,14 +1068,22 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char
algo = *p++;
digest = *p++;
- if ((key = blockdata_alloc((char*)p, rdlen - 4)))
+ if (!ds_digest_name(digest) || !algo_digest_name(algo))
+ {
+ a.log.keytag = keytag;
+ a.log.algo = algo;
+ a.log.digest = digest;
+ log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu (not supported)", 0);
+ neg_ttl = ttl;
+ }
+ else if ((key = blockdata_alloc((char*)p, rdlen - 4)))
{
a.ds.digest = digest;
a.ds.keydata = key;
a.ds.algo = algo;
a.ds.keytag = keytag;
a.ds.keylen = rdlen - 4;
-
+
if (!cache_insert(name, &a, class, now, ttl, F_FORWARD | F_DS | F_DNSSECOK))
{
blockdata_free(key);
@@ -1083,26 +1094,29 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char
a.log.keytag = keytag;
a.log.algo = algo;
a.log.digest = digest;
- if (ds_digest_name(digest) && algo_digest_name(algo))
- log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu", 0);
- else
- log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu (not supported)", 0);
+ log_query(F_NOEXTRA | F_KEYTAG | F_UPSTREAM, name, &a, "DS keytag %hu, algo %hu, digest %hu", 0);
+ found_supported = 1;
}
}
p = psave;
}
+
if (!ADD_RDLEN(header, p, plen, rdlen))
return STAT_BOGUS; /* bad packet */
}
cache_end_insert();
+ /* Fall through if no supported algo DS found. */
+ if (found_supported)
+ return STAT_OK;
}
- else
+
+ flags = F_FORWARD | F_DS | F_NEG | F_DNSSECOK;
+
+ if (neganswer)
{
- int flags = F_FORWARD | F_DS | F_NEG | F_DNSSECOK;
-
if (RCODE(header) == NXDOMAIN)
flags |= F_NXDOMAIN;
@@ -1110,17 +1124,18 @@ int dnssec_validate_ds(time_t now, struct dns_header *header, size_t plen, char
to store presence/absence of NS. */
if (nons)
flags &= ~F_DNSSECOK;
-
- cache_start_insert();
-
- /* Use TTL from NSEC for negative cache entries */
- if (!cache_insert(name, NULL, class, now, neg_ttl, flags))
- return STAT_BOGUS;
-
- cache_end_insert();
-
- log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, nons ? "no DS/cut" : "no DS", 0);
}
+
+ cache_start_insert();
+
+ /* Use TTL from NSEC for negative cache entries */
+ if (!cache_insert(name, NULL, class, now, neg_ttl, flags))
+ return STAT_BOGUS;
+
+ cache_end_insert();
+
+ if (neganswer)
+ log_query(F_NOEXTRA | F_UPSTREAM, name, NULL, nons ? "no DS/cut" : "no DS", 0);
return STAT_OK;
}
@@ -1851,7 +1866,7 @@ static int zone_status(char *name, int class, char *keyname, time_t now)
STAT_NEED_DS need DS to complete validation (name is returned in keyname)
daemon->rr_status points to a char array which corressponds to the RRs in the
- answer and auth sections. This is set to 1 for each RR which is validated, and 0 for any which aren't.
+ answer and auth sections. This is set to >1 for each RR which is validated, and 0 for any which aren't.
When validating replies to DS records, we're only interested in the NSEC{3} RRs in the auth section.
Other RRs in that section missing sigs will not cause am INSECURE reply. We determine this mode
diff --git a/src/dnsmasq/domain-match.c b/src/dnsmasq/domain-match.c
index 3ec49b80..fe8e25a3 100644
--- a/src/dnsmasq/domain-match.c
+++ b/src/dnsmasq/domain-match.c
@@ -213,9 +213,13 @@ int lookup_domain(char *domain, int flags, int *lowout, int *highout)
to continue generalising */
{
/* We've matched a setting which says to use servers without a domain.
- Continue the search with empty query */
+ Continue the search with empty query. We set the F_SERVER flag
+ so that --address=/#/... doesn't match. */
if (daemon->serverarray[nlow]->flags & SERV_USE_RESOLV)
- crop_query = qlen;
+ {
+ crop_query = qlen;
+ flags |= F_SERVER;
+ }
else
break;
}
@@ -299,7 +303,7 @@ int filter_servers(int seed, int flags, int *lowout, int *highout)
for (i = nlow; i < nhigh && (daemon->serverarray[i]->flags & SERV_6ADDR); i++);
- if (i != nlow && (flags & F_IPV6))
+ if (!(flags & F_SERVER) && i != nlow && (flags & F_IPV6))
nhigh = i;
else
{
@@ -307,7 +311,7 @@ int filter_servers(int seed, int flags, int *lowout, int *highout)
for (i = nlow; i < nhigh && (daemon->serverarray[i]->flags & SERV_4ADDR); i++);
- if (i != nlow && (flags & F_IPV4))
+ if (!(flags & F_SERVER) && i != nlow && (flags & F_IPV4))
nhigh = i;
else
{
@@ -315,7 +319,7 @@ int filter_servers(int seed, int flags, int *lowout, int *highout)
for (i = nlow; i < nhigh && (daemon->serverarray[i]->flags & SERV_ALL_ZEROS); i++);
- if (i != nlow && (flags & (F_IPV4 | F_IPV6)))
+ if (!(flags & F_SERVER) && i != nlow && (flags & (F_IPV4 | F_IPV6)))
nhigh = i;
else
{
@@ -542,11 +546,23 @@ static int order_qsort(const void *a, const void *b)
return rc;
}
+
+/* When loading large numbers of server=.... lines during startup,
+ there's no possibility that there will be server records that can be reused, but
+ searching a long list for each server added grows as O(n^2) and slows things down.
+ This flag is set only if is known there may be free server records that can be reused.
+ There's a call to mark_servers(0) in read_opts() to reset the flag before
+ main config read. */
+
+static int maybe_free_servers = 0;
+
/* Must be called before add_update_server() to set daemon->servers_tail */
void mark_servers(int flag)
{
- struct server *serv, **up;
+ struct server *serv, *next, **up;
+ maybe_free_servers = !!flag;
+
daemon->servers_tail = NULL;
/* mark everything with argument flag */
@@ -564,11 +580,13 @@ void mark_servers(int flag)
1) numerous and 2) not reloaded often. We just delete
and recreate. */
if (flag)
- for (serv = daemon->local_domains, up = &daemon->local_domains; serv; serv = serv->next)
+ for (serv = daemon->local_domains, up = &daemon->local_domains; serv; serv = next)
{
+ next = serv->next;
+
if (serv->flags & flag)
{
- *up = serv->next;
+ *up = next;
free(serv->domain);
free(serv);
}
@@ -663,25 +681,30 @@ int add_update_server(int flags,
and move to the end of the list, for order. The entry found may already
be at the end. */
struct server **up, *tmp;
-
- for (serv = daemon->servers, up = &daemon->servers; serv; serv = tmp)
- {
- tmp = serv->next;
- if ((serv->flags & SERV_MARK) &&
- hostname_isequal(alloc_domain, serv->domain))
- {
- /* Need to move down? */
- if (serv->next)
- {
- *up = serv->next;
- daemon->servers_tail->next = serv;
- daemon->servers_tail = serv;
- serv->next = NULL;
- }
- break;
- }
- }
+ serv = NULL;
+
+ if (maybe_free_servers)
+ for (serv = daemon->servers, up = &daemon->servers; serv; serv = tmp)
+ {
+ tmp = serv->next;
+ if ((serv->flags & SERV_MARK) &&
+ hostname_isequal(alloc_domain, serv->domain))
+ {
+ /* Need to move down? */
+ if (serv->next)
+ {
+ *up = serv->next;
+ daemon->servers_tail->next = serv;
+ daemon->servers_tail = serv;
+ serv->next = NULL;
+ }
+ break;
+ }
+ else
+ up = &serv->next;
+ }
+
if (serv)
{
free(alloc_domain);
diff --git a/src/dnsmasq/domain.c b/src/dnsmasq/domain.c
index 71664334..a893ce5a 100644
--- a/src/dnsmasq/domain.c
+++ b/src/dnsmasq/domain.c
@@ -230,9 +230,17 @@ int is_rev_synth(int flag, union all_addr *addr, char *name)
static int match_domain(struct in_addr addr, struct cond_domain *c)
{
- if (!c->is6 &&
- ntohl(addr.s_addr) >= ntohl(c->start.s_addr) &&
- ntohl(addr.s_addr) <= ntohl(c->end.s_addr))
+ if (c->interface)
+ {
+ struct addrlist *al;
+ for (al = c->al; al; al = al->next)
+ if (!(al->flags & ADDRLIST_IPV6) &&
+ is_same_net_prefix(addr, al->addr.addr4, al->prefixlen))
+ return 1;
+ }
+ else if (!c->is6 &&
+ ntohl(addr.s_addr) >= ntohl(c->start.s_addr) &&
+ ntohl(addr.s_addr) <= ntohl(c->end.s_addr))
return 1;
return 0;
@@ -259,12 +267,21 @@ char *get_domain(struct in_addr addr)
static int match_domain6(struct in6_addr *addr, struct cond_domain *c)
{
- u64 addrpart = addr6part(addr);
-
- if (c->is6)
+
+ /* subnet from interface address. */
+ if (c->interface)
+ {
+ struct addrlist *al;
+ for (al = c->al; al; al = al->next)
+ if (al->flags & ADDRLIST_IPV6 &&
+ is_same_net6(addr, &al->addr.addr6, al->prefixlen))
+ return 1;
+ }
+ else if (c->is6)
{
if (c->prefixlen >= 64)
{
+ u64 addrpart = addr6part(addr);
if (is_same_net6(addr, &c->start6, 64) &&
addrpart >= addr6part(&c->start6) &&
addrpart <= addr6part(&c->end6))
@@ -273,7 +290,7 @@ static int match_domain6(struct in6_addr *addr, struct cond_domain *c)
else if (is_same_net6(addr, &c->start6, c->prefixlen))
return 1;
}
-
+
return 0;
}
diff --git a/src/dnsmasq/dump.c b/src/dnsmasq/dump.c
index 7b15945c..57352a9d 100644
--- a/src/dnsmasq/dump.c
+++ b/src/dnsmasq/dump.c
@@ -21,6 +21,8 @@
#include
static u32 packet_count;
+static void do_dump_packet(int mask, void *packet, size_t len,
+ union mysockaddr *src, union mysockaddr *dst, int port, int proto);
/* https://wiki.wireshark.org/Development/LibpcapFileFormat */
struct pcap_hdr_s {
@@ -81,9 +83,45 @@ void dump_init(void)
}
}
-/* port == -1 ->ICMPv6 */
-void dump_packet(int mask, void *packet, size_t len,
- union mysockaddr *src, union mysockaddr *dst, int port)
+void dump_packet_udp(int mask, void *packet, size_t len,
+ union mysockaddr *src, union mysockaddr *dst, int fd)
+{
+ union mysockaddr fd_addr;
+ socklen_t addr_len = sizeof(fd_addr);
+
+ if (daemon->dumpfd != -1 && (mask & daemon->dump_mask))
+ {
+ /* if fd is negative it carries a port number (negated)
+ which we use as a source or destination when not otherwise
+ specified so wireshark can ID the packet.
+ If both src and dst are specified, set this to -1 to avoid
+ a spurious getsockname() call. */
+ int port = (fd < 0) ? -fd : -1;
+
+ /* fd >= 0 is a file descriptor and the address of that file descriptor is used
+ in place of a NULL src or dst. */
+ if (fd >= 0 && getsockname(fd, (struct sockaddr *)&fd_addr, &addr_len) != -1)
+ {
+ if (!src)
+ src = &fd_addr;
+
+ if (!dst)
+ dst = &fd_addr;
+ }
+
+ do_dump_packet(mask, packet, len, src, dst, port, IPPROTO_UDP);
+ }
+}
+
+void dump_packet_icmp(int mask, void *packet, size_t len,
+ union mysockaddr *src, union mysockaddr *dst)
+{
+ if (daemon->dumpfd != -1 && (mask & daemon->dump_mask))
+ do_dump_packet(mask, packet, len, src, dst, -1, IPPROTO_ICMP);
+}
+
+static void do_dump_packet(int mask, void *packet, size_t len,
+ union mysockaddr *src, union mysockaddr *dst, int port, int proto)
{
struct ip ip;
struct ip6_hdr ip6;
@@ -100,13 +138,14 @@ void dump_packet(int mask, void *packet, size_t len,
void *iphdr;
size_t ipsz;
int rc;
+
+ /* if port != -1 it carries a port number
+ which we use as a source or destination when not otherwise
+ specified so wireshark can ID the packet.
+ If both src and dst are specified, set this to -1 to avoid
+ a spurious getsockname() call. */
+ udp.uh_sport = udp.uh_dport = htons(port < 0 ? 0 : port);
- if (daemon->dumpfd == -1 || !(mask & daemon->dump_mask))
- return;
-
- /* So wireshark can Id the packet. */
- udp.uh_sport = udp.uh_dport = htons(port);
-
if (src)
family = src->sa.sa_family;
else
@@ -121,15 +160,12 @@ void dump_packet(int mask, void *packet, size_t len,
ip6.ip6_vfc = 6 << 4;
ip6.ip6_hops = 64;
- if (port == -1)
- {
- ip6.ip6_plen = htons(len);
- ip6.ip6_nxt = IPPROTO_ICMPV6;
- }
+ if ((ip6.ip6_nxt = proto) == IPPROTO_UDP)
+ ip6.ip6_plen = htons(sizeof(struct udphdr) + len);
else
{
- ip6.ip6_plen = htons(sizeof(struct udphdr) + len);
- ip6.ip6_nxt = IPPROTO_UDP;
+ proto = ip6.ip6_nxt = IPPROTO_ICMPV6;
+ ip6.ip6_plen = htons(len);
}
if (src)
@@ -161,15 +197,12 @@ void dump_packet(int mask, void *packet, size_t len,
ip.ip_hl = sizeof(struct ip) / 4;
ip.ip_ttl = IPDEFTTL;
- if (port == -1)
- {
- ip.ip_len = htons(sizeof(struct ip) + len);
- ip.ip_p = IPPROTO_ICMP;
- }
+ if ((ip.ip_p = proto) == IPPROTO_UDP)
+ ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len);
else
{
- ip.ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + len);
- ip.ip_p = IPPROTO_UDP;
+ ip.ip_len = htons(sizeof(struct ip) + len);
+ proto = ip.ip_p = IPPROTO_ICMP;
}
if (src)
@@ -191,7 +224,7 @@ void dump_packet(int mask, void *packet, size_t len,
sum = (sum & 0xffff) + (sum >> 16);
ip.ip_sum = (sum == 0xffff) ? sum : ~sum;
- /* start UDP checksum */
+ /* start UDP/ICMP checksum */
sum = ip.ip_src.s_addr & 0xffff;
sum += (ip.ip_src.s_addr >> 16) & 0xffff;
sum += ip.ip_dst.s_addr & 0xffff;
@@ -201,25 +234,7 @@ void dump_packet(int mask, void *packet, size_t len,
if (len & 1)
((unsigned char *)packet)[len] = 0; /* for checksum, in case length is odd. */
- if (port == -1)
- {
- /* ICMP - ICMPv6 packet is a superset of ICMP */
- struct icmp6_hdr *icmp = packet;
-
- /* See comment in UDP code below. */
- sum += htons((family == AF_INET6) ? IPPROTO_ICMPV6 : IPPROTO_ICMP);
- sum += htons(len);
-
- icmp->icmp6_cksum = 0;
- for (i = 0; i < (len + 1) / 2; i++)
- sum += ((u16 *)packet)[i];
- while (sum >> 16)
- sum = (sum & 0xffff) + (sum >> 16);
- icmp->icmp6_cksum = (sum == 0xffff) ? sum : ~sum;
-
- pcap_header.incl_len = pcap_header.orig_len = ipsz + len;
- }
- else
+ if (proto == IPPROTO_UDP)
{
/* Add Remaining part of the pseudoheader. Note that though the
IPv6 pseudoheader is very different to the IPv4 one, the
@@ -241,7 +256,25 @@ void dump_packet(int mask, void *packet, size_t len,
pcap_header.incl_len = pcap_header.orig_len = ipsz + sizeof(udp) + len;
}
-
+ else
+ {
+ /* ICMP - ICMPv6 packet is a superset of ICMP */
+ struct icmp6_hdr *icmp = packet;
+
+ /* See comment in UDP code above. */
+ sum += htons(proto);
+ sum += htons(len);
+
+ icmp->icmp6_cksum = 0;
+ for (i = 0; i < (len + 1) / 2; i++)
+ sum += ((u16 *)packet)[i];
+ while (sum >> 16)
+ sum = (sum & 0xffff) + (sum >> 16);
+ icmp->icmp6_cksum = (sum == 0xffff) ? sum : ~sum;
+
+ pcap_header.incl_len = pcap_header.orig_len = ipsz + len;
+ }
+
rc = gettimeofday(&time, NULL);
pcap_header.ts_sec = time.tv_sec;
pcap_header.ts_usec = time.tv_usec;
@@ -249,9 +282,11 @@ void dump_packet(int mask, void *packet, size_t len,
if (rc == -1 ||
!read_write(daemon->dumpfd, (void *)&pcap_header, sizeof(pcap_header), 0) ||
!read_write(daemon->dumpfd, iphdr, ipsz, 0) ||
- (port != -1 && !read_write(daemon->dumpfd, (void *)&udp, sizeof(udp), 0)) ||
+ (proto == IPPROTO_UDP && !read_write(daemon->dumpfd, (void *)&udp, sizeof(udp), 0)) ||
!read_write(daemon->dumpfd, (void *)packet, len, 0))
my_syslog(LOG_ERR, _("failed to write packet dump"));
+ else if (option_bool(OPT_EXTRALOG))
+ my_syslog(LOG_INFO, _("%u dumping packet %u mask 0x%04x"), daemon->log_display_id, ++packet_count, mask);
else
my_syslog(LOG_INFO, _("dumping packet %u mask 0x%04x"), ++packet_count, mask);
diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c
index 4c061227..c1070067 100644
--- a/src/dnsmasq/forward.c
+++ b/src/dnsmasq/forward.c
@@ -173,7 +173,7 @@ static int domain_no_rebind(char *domain)
static int forward_query(int udpfd, union mysockaddr *udpaddr,
union all_addr *dst_addr, unsigned int dst_iface,
struct dns_header *header, size_t plen, char *limit, time_t now,
- struct frec *forward, int ad_reqd, int do_bit)
+ struct frec *forward, int ad_reqd, int do_bit, int fast_retry)
{
unsigned int flags = 0;
unsigned int fwd_flags = 0;
@@ -247,6 +247,14 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
if (!daemon->free_frec_src)
{
query_full(now, NULL);
+ /* This is tricky; if we're blasted with the same query
+ over and over, we'll end up taking this path each time
+ and never resetting until the frec gets deleted by
+ aging followed by the receipt of a different query. This
+ is a bit of a DoS vuln. Avoid by explicitly deleting the
+ frec once it expires. */
+ if (difftime(now, forward->time) >= TIMEOUT)
+ free_frec(forward);
goto reply;
}
@@ -311,6 +319,13 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
goto reply;
/* table full - flags == 0, return REFUSED */
+ /* Keep copy of query if we're doing fast retry. */
+ if (daemon->fast_retry_time != 0)
+ {
+ forward->stash = blockdata_alloc((char *)header, plen);
+ forward->stash_len = plen;
+ }
+
forward->frec_src.log_id = daemon->log_id;
forward->frec_src.source = *udpaddr;
forward->frec_src.orig_id = ntohs(header->id);
@@ -358,14 +373,14 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
#ifdef HAVE_DNSSEC
/* If we've already got an answer to this query, but we're awaiting keys for validation,
there's no point retrying the query, retry the key query instead...... */
- if (forward->blocking_query)
+ while (forward->blocking_query)
+ forward = forward->blocking_query;
+
+ if (forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY))
{
int is_sign;
unsigned char *pheader;
- while (forward->blocking_query)
- forward = forward->blocking_query;
-
/* log_id should match previous DNSSEC query. */
daemon->log_display_id = forward->frec_src.log_id;
@@ -390,7 +405,10 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
#endif
{
/* retry on existing query, from original source. Send to all available servers */
- forward->sentto->failed_queries++;
+ if (udpfd == -1 && !fast_retry)
+ forward->sentto->failed_queries++;
+ else
+ forward->sentto->retrys++;
FTL_forwarding_retried(forward->sentto, forward->frec_src.log_id, daemon->log_display_id, false);
@@ -400,13 +418,13 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
master = daemon->serverarray[first];
/* Forward to all available servers on retry of query from same host. */
- if (!option_bool(OPT_ORDER) && old_src)
+ if (!option_bool(OPT_ORDER) && old_src && !fast_retry)
forward->forwardall = 1;
else
{
start = forward->sentto->arrayposn;
- if (option_bool(OPT_ORDER))
+ if (option_bool(OPT_ORDER) && !fast_retry)
{
/* In strict order mode, there must be a server later in the list
left to send to, otherwise without the forwardall mechanism,
@@ -520,7 +538,7 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
if (errno == 0)
{
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_UP_QUERY, (void *)header, plen, NULL, &srv->addr, daemon->port);
+ dump_packet_udp(DUMP_UP_QUERY, (void *)header, plen, NULL, &srv->addr, fd);
#endif
/* Keep info in case we want to re-send this packet */
@@ -537,8 +555,8 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
}
#ifdef HAVE_DNSSEC
else
- log_query_mysockaddr(F_NOEXTRA | F_DNSSEC, daemon->namebuff, &srv->addr,
- "dnssec-retry", (forward->flags & FREC_DNSKEY_QUERY) ? T_DNSKEY : T_DS);
+ log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, daemon->namebuff, &srv->addr,
+ (forward->flags & FREC_DNSKEY_QUERY) ? "dnssec-retry[DNSKEY]" : "dnssec-retry[DS]", 0);
#endif
srv->queries++;
@@ -555,7 +573,10 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
}
if (forwarded || is_dnssec)
- return 1;
+ {
+ forward->forward_timestamp = dnsmasq_milliseconds();
+ return 1;
+ }
/* could not send on, prepare to return */
header->id = htons(forward->frec_src.orig_id);
@@ -594,6 +615,61 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr,
return 0;
}
+/* Check if any frecs need to do a retry, and action that if so.
+ Return time in milliseconds until he next retry will be required,
+ or -1 if none. */
+int fast_retry(time_t now)
+{
+ struct frec *f;
+ int ret = -1;
+
+ if (daemon->fast_retry_time != 0)
+ {
+ u32 millis = dnsmasq_milliseconds();
+
+ for (f = daemon->frec_list; f; f = f->next)
+ if (f->sentto && f->stash && difftime(now, f->time) < daemon->fast_retry_timeout)
+ {
+#ifdef HAVE_DNSSEC
+ if (f->blocking_query)
+ continue;
+#endif
+ /* t is milliseconds since last query sent. */
+ int to_run, t = (int)(millis - f->forward_timestamp);
+
+ if (t < f->forward_delay)
+ to_run = f->forward_delay - t;
+ else
+ {
+ unsigned char *udpsz;
+ unsigned short udp_size = PACKETSZ; /* default if no EDNS0 */
+ struct dns_header *header = (struct dns_header *)daemon->packet;
+
+ /* packet buffer overwritten */
+ daemon->srv_save = NULL;
+
+ blockdata_retrieve(f->stash, f->stash_len, (void *)header);
+
+ /* UDP size already set in saved query. */
+ if (find_pseudoheader(header, f->stash_len, NULL, &udpsz, NULL, NULL))
+ GETSHORT(udp_size, udpsz);
+
+ daemon->log_display_id = f->frec_src.log_id;
+
+ forward_query(-1, NULL, NULL, 0, header, f->stash_len, ((char *) header) + udp_size, now, f,
+ f->flags & FREC_AD_QUESTION, f->flags & FREC_DO_QUESTION, 1);
+
+ to_run = f->forward_delay = 2 * f->forward_delay;
+ }
+
+ if (ret == -1 || ret > to_run)
+ ret = to_run;
+ }
+
+ }
+ return ret;
+}
+
static struct ipsets *domain_find_sets(struct ipsets *setlist, const char *domain) {
/* Similar algorithm to search_servers. */
struct ipsets *ipset_pos, *ret = NULL;
@@ -624,6 +700,9 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server
int munged = 0, is_sign;
unsigned int rcode = RCODE(header);
size_t plen;
+ /******** Pi-hole modification ********/
+ unsigned char *pheader_copy = NULL;
+ /**************************************/
(void)ad_reqd;
(void)do_bit;
@@ -757,22 +836,40 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server
n = rrfilter(header, n, RRFILTER_AAAA);
}
- /******************************** Pi-hole modification ********************************/
- int ret = extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored);
- if (ret == 2)
- {
- cache_secure = 0;
- // Generate DNS packet for reply, a possibly existing pseudo header
- // will be restored later inside resize_packet()
- n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede);
- }
- else if(ret)
- /**************************************************************************************/
+ switch (extract_addresses(header, n, daemon->namebuff, now, ipsets, nftsets, is_sign, check_rebind, no_cache, cache_secure, &doctored))
{
+ case 1:
my_syslog(LOG_WARNING, _("possible DNS-rebind attack detected: %s"), daemon->namebuff);
munged = 1;
cache_secure = 0;
ede = EDE_BLOCKED;
+ break;
+
+ /* extract_addresses() found a malformed answer. */
+ case 2:
+ munged = 1;
+ SET_RCODE(header, SERVFAIL);
+ cache_secure = 0;
+ ede = EDE_OTHER;
+ break;
+
+ /* Pi-hole modification */
+ case 99:
+ cache_secure = 0;
+ // Make a private copy of the pheader to ensure
+ // we are not accidentially rewriting what is in
+ // the pheader when we're creating a crafted reply
+ // further below (when a query is to be blocked)
+ if (pheader)
+ {
+ pheader_copy = calloc(1, plen);
+ memcpy(pheader_copy, pheader, plen);
+ }
+
+ // Generate DNS packet for reply, a possibly existing pseudo header
+ // will be restored later inside resize_packet()
+ n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede);
+ break;
}
if (doctored)
@@ -813,7 +910,13 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server
/* the bogus-nxdomain stuff, doctor and NXDOMAIN->NODATA munging can all elide
sections of the packet. Find the new length here and put back pseudoheader
if it was removed. */
- n = resize_packet(header, n, pheader, plen);
+ n = resize_packet(header, n, pheader_copy ? pheader_copy : pheader, plen);
+ /******** Pi-hole modification ********/
+ // The line above was modified to use
+ // pheader_copy instead of pheader
+ if(pheader_copy)
+ free(pheader_copy);
+ /**************************************/
if (pheader && ede != EDE_UNSET)
{
@@ -821,6 +924,9 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server
n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 1);
}
+ if (RCODE(header) == NXDOMAIN)
+ server->nxdomain_replies++;
+
return n;
}
@@ -861,8 +967,8 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header,
NULL, NULL, NULL);
#ifdef HAVE_DUMPFILE
if (STAT_ISEQUAL(status, STAT_BOGUS))
- dump_packet((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS,
- header, (size_t)plen, &forward->sentto->addr, NULL, daemon->port);
+ dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_BOGUS : DUMP_BOGUS,
+ header, (size_t)plen, &forward->sentto->addr, NULL, -daemon->port);
#endif
}
@@ -963,6 +1069,8 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header,
/* Save query for retransmission and de-dup */
new->stash = blockdata_alloc((char *)header, nn);
new->stash_len = nn;
+ if (daemon->fast_retry_time != 0)
+ new->forward_timestamp = dnsmasq_milliseconds();
/* Don't resend this. */
daemon->srv_save = NULL;
@@ -972,13 +1080,13 @@ static void dnssec_validate(struct frec *forward, struct dns_header *header,
set_outgoing_mark(orig, fd);
#endif
+ server_send(server, fd, header, nn, 0);
+ server->queries++;
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_SEC_QUERY, (void *)header, (size_t)nn, NULL, &server->addr, daemon->port);
+ dump_packet_udp(DUMP_SEC_QUERY, (void *)header, (size_t)nn, NULL, &server->addr, fd);
#endif
log_query_mysockaddr(F_NOEXTRA | F_DNSSEC | F_SERVER, daemon->keyname, &server->addr,
STAT_ISEQUAL(status, STAT_NEED_KEY) ? "dnssec-query[DNSKEY]" : "dnssec-query[DS]", 0);
- server_send(server, fd, header, nn, 0);
- server->queries++;
return;
}
@@ -1067,20 +1175,20 @@ void reply_query(int fd, time_t now)
if (difftime(now, server->pktsz_reduced) > UDP_TEST_TIME)
server->edns_pktsz = daemon->edns_pktsz;
-#ifdef HAVE_DUMPFILE
- dump_packet((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_REPLY : DUMP_UP_REPLY,
- (void *)header, n, &serveraddr, NULL, daemon->port);
-#endif
-
/* log_query gets called indirectly all over the place, so
pass these in global variables - sorry. */
daemon->log_display_id = forward->frec_src.log_id;
daemon->log_source_addr = &forward->frec_src.source;
+#ifdef HAVE_DUMPFILE
+ dump_packet_udp((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) ? DUMP_SEC_REPLY : DUMP_UP_REPLY,
+ (void *)header, n, &serveraddr, NULL, fd);
+#endif
+
if (daemon->ignore_addr && RCODE(header) == NOERROR &&
check_for_ignored_address(header, n))
return;
-
+
/* Note: if we send extra options in the EDNS0 header, we can't recreate
the query from the reply. */
if ((RCODE(header) == REFUSED || RCODE(header) == SERVFAIL) &&
@@ -1095,12 +1203,15 @@ void reply_query(int fd, time_t now)
size_t nn = 0;
#ifdef HAVE_DNSSEC
- /* DNSSEC queries have a copy of the original query stashed.
- The query MAY have got a good answer, and be awaiting
+ /* The query MAY have got a good answer, and be awaiting
the results of further queries, in which case
The Stash contains something else and we don't need to retry anyway. */
- if ((forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY)) && !forward->blocking_query)
+ if (forward->blocking_query)
+ return;
+
+ if (forward->flags & (FREC_DNSKEY_QUERY | FREC_DS_QUERY))
{
+ /* DNSSEC queries have a copy of the original query stashed. */
blockdata_retrieve(forward->stash, forward->stash_len, (void *)header);
nn = forward->stash_len;
udp_size = daemon->edns_pktsz;
@@ -1108,38 +1219,50 @@ void reply_query(int fd, time_t now)
else
#endif
{
- /* recreate query from reply */
- if ((pheader = find_pseudoheader(header, (size_t)n, &plen, &udpsz, &is_sign, NULL)))
- GETSHORT(udp_size, udpsz);
-
- /* If the client provides an EDNS0 UDP size, use that to limit our reply.
- (bounded by the maximum configured). If no EDNS0, then it
- defaults to 512 */
- if (udp_size > daemon->edns_pktsz)
- udp_size = daemon->edns_pktsz;
- else if (udp_size < PACKETSZ)
- udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */
-
- if (!is_sign &&
- (nn = resize_packet(header, (size_t)n, pheader, plen)) &&
- (forward->flags & FREC_DO_QUESTION))
- add_do_bit(header, nn, (unsigned char *)pheader + plen);
+ /* in fast retry mode, we have a copy of the query. */
+ if (daemon->fast_retry_time != 0 && forward->stash)
+ {
+ blockdata_retrieve(forward->stash, forward->stash_len, (void *)header);
+ nn = forward->stash_len;
+ /* UDP size already set in saved query. */
+ if (find_pseudoheader(header, (size_t)n, NULL, &udpsz, NULL, NULL))
+ GETSHORT(udp_size, udpsz);
+ }
+ else
+ {
+ /* recreate query from reply */
+ if ((pheader = find_pseudoheader(header, (size_t)n, &plen, &udpsz, &is_sign, NULL)))
+ GETSHORT(udp_size, udpsz);
+
+ /* If the client provides an EDNS0 UDP size, use that to limit our reply.
+ (bounded by the maximum configured). If no EDNS0, then it
+ defaults to 512 */
+ if (udp_size > daemon->edns_pktsz)
+ udp_size = daemon->edns_pktsz;
+ else if (udp_size < PACKETSZ)
+ udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */
+
+ header->ancount = htons(0);
+ header->nscount = htons(0);
+ header->arcount = htons(0);
+ header->hb3 &= ~(HB3_QR | HB3_AA | HB3_TC);
+ header->hb4 &= ~(HB4_RA | HB4_RCODE | HB4_CD | HB4_AD);
+ if (forward->flags & FREC_CHECKING_DISABLED)
+ header->hb4 |= HB4_CD;
+ if (forward->flags & FREC_AD_QUESTION)
+ header->hb4 |= HB4_AD;
- header->ancount = htons(0);
- header->nscount = htons(0);
- header->arcount = htons(0);
- header->hb3 &= ~(HB3_QR | HB3_AA | HB3_TC);
- header->hb4 &= ~(HB4_RA | HB4_RCODE | HB4_CD | HB4_AD);
- if (forward->flags & FREC_CHECKING_DISABLED)
- header->hb4 |= HB4_CD;
- if (forward->flags & FREC_AD_QUESTION)
- header->hb4 |= HB4_AD;
+ if (!is_sign &&
+ (nn = resize_packet(header, (size_t)n, pheader, plen)) &&
+ (forward->flags & FREC_DO_QUESTION))
+ add_do_bit(header, nn, (unsigned char *)pheader + plen);
+ }
}
-
+
if (nn)
{
forward_query(-1, NULL, NULL, 0, header, nn, ((char *) header) + udp_size, now, forward,
- forward->flags & FREC_AD_QUESTION, forward->flags & FREC_DO_QUESTION);
+ forward->flags & FREC_AD_QUESTION, forward->flags & FREC_DO_QUESTION, 0);
return;
}
}
@@ -1166,6 +1289,22 @@ void reply_query(int fd, time_t now)
}
forward->sentto = server;
+
+ /* We have a good answer, and will now validate it or return it.
+ It may be some time before this the validation completes, but we don't need
+ any more answers, so close the socket(s) on which we were expecting
+ answers, to conserve file descriptors, and to save work reading and
+ discarding answers for other upstreams. */
+ free_rfds(&forward->rfds);
+
+ /* calculate modified moving average of server latency */
+ if (server->query_latency == 0)
+ server->mma_latency = (dnsmasq_milliseconds() - forward->forward_timestamp) * 128; /* init */
+ else
+ server->mma_latency += dnsmasq_milliseconds() - forward->forward_timestamp - server->query_latency;
+ /* denominator controls how many queries we average over. */
+ server->query_latency = server->mma_latency/128;
+
#ifdef HAVE_DNSSEC
if ((forward->sentto->flags & SERV_DO_DNSSEC) &&
@@ -1272,10 +1411,6 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he
{
header->id = htons(src->orig_id);
-#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &src->source, daemon->port);
-#endif
-
#if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS)
if (option_bool(OPT_CMARK_ALST_EN))
{
@@ -1289,14 +1424,20 @@ static void return_reply(time_t now, struct frec *forward, struct dns_header *he
/* Pi-hole modification */
int first_ID = -1;
- send_from(src->fd, option_bool(OPT_NOWILD) || option_bool (OPT_CLEVERBIND), daemon->packet, nn,
- &src->source, &src->dest, src->iface);
-
- if (option_bool(OPT_EXTRALOG) && src != &forward->frec_src)
+ if (src->fd != -1)
{
- daemon->log_display_id = src->log_id;
- daemon->log_source_addr = &src->source;
- log_query(F_UPSTREAM, "query", NULL, "duplicate", 0);
+#ifdef HAVE_DUMPFILE
+ dump_packet_udp(DUMP_REPLY, daemon->packet, (size_t)nn, NULL, &src->source, src->fd);
+#endif
+ send_from(src->fd, option_bool(OPT_NOWILD) || option_bool (OPT_CLEVERBIND), daemon->packet, nn,
+ &src->source, &src->dest, src->iface);
+
+ if (option_bool(OPT_EXTRALOG) && src != &forward->frec_src)
+ {
+ daemon->log_display_id = src->log_id;
+ daemon->log_source_addr = &src->source;
+ log_query(F_UPSTREAM, "query", NULL, "duplicate", 0);
+ }
}
/* Pi-hole modification */
FTL_multiple_replies(src->log_id, &first_ID);
@@ -1392,7 +1533,7 @@ void receive_query(struct listener *listen, time_t now)
/************ Pi-hole modification ************/
bool piholeblocked = false;
/**********************************************/
-
+
/* packet buffer overwritten */
daemon->srv_save = NULL;
@@ -1601,7 +1742,7 @@ void receive_query(struct listener *listen, time_t now)
daemon->log_source_addr = &source_addr;
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_QUERY, daemon->packet, (size_t)n, &source_addr, NULL, daemon->port);
+ dump_packet_udp(DUMP_QUERY, daemon->packet, (size_t)n, &source_addr, NULL, listen->fd);
#endif
#ifdef HAVE_CONNTRACK
@@ -1661,13 +1802,17 @@ void receive_query(struct listener *listen, time_t now)
/* If the client provides an EDNS0 UDP size, use that to limit our reply.
(bounded by the maximum configured). If no EDNS0, then it
- defaults to 512 */
+ defaults to 512. We write this value into the query packet too, so that
+ if it's forwarded, we don't specify a maximum size greater than we can handle. */
if (udp_size > daemon->edns_pktsz)
udp_size = daemon->edns_pktsz;
else if (udp_size < PACKETSZ)
udp_size = PACKETSZ; /* Sanity check - can't reduce below default. RFC 6891 6.2.3 */
- }
+ pheader -= 6; /* ext_class */
+ PUTSHORT(udp_size, pheader); /* Bounding forwarded queries to maximum configured */
+ }
+
#ifdef HAVE_CONNTRACK
#ifdef HAVE_AUTH
if (!auth_dns || local_auth)
@@ -1691,7 +1836,7 @@ void receive_query(struct listener *listen, time_t now)
if (m >= 1)
{
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, daemon->port);
+ dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd);
#endif
send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND),
(char *)header, m, &source_addr, &dst_addr, if_index);
@@ -1707,7 +1852,7 @@ void receive_query(struct listener *listen, time_t now)
if (m >= 1)
{
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, daemon->port);
+ dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd);
#endif
#if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS)
if (local_auth)
@@ -1722,7 +1867,11 @@ void receive_query(struct listener *listen, time_t now)
#endif
else
{
+ int stale;
int ad_reqd = do_bit;
+ u16 hb3 = header->hb3, hb4 = header->hb4;
+ int fd = listen->fd;
+
/* RFC 6840 5.7 */
if (header->hb4 & HB4_AD)
ad_reqd = 1;
@@ -1758,12 +1907,19 @@ void receive_query(struct listener *listen, time_t now)
/**********************************************/
m = answer_request(header, ((char *) header) + udp_size, (size_t)n,
- dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader);
+ dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale);
if (m >= 1)
{
+ if (stale && have_pseudoheader)
+ {
+ u16 swap = htons(EDE_STALE);
+
+ m = add_pseudoheader(header, m, ((unsigned char *) header) + udp_size, daemon->edns_pktsz,
+ EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0);
+ }
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, daemon->port);
+ dump_packet_udp(DUMP_REPLY, daemon->packet, m, NULL, &source_addr, listen->fd);
#endif
#if defined(HAVE_CONNTRACK) && defined(HAVE_UBUS)
if (option_bool(OPT_CMARK_ALST_EN) && have_mark && ((u32)mark & daemon->allowlist_mask))
@@ -1772,12 +1928,39 @@ void receive_query(struct listener *listen, time_t now)
send_from(listen->fd, option_bool(OPT_NOWILD) || option_bool(OPT_CLEVERBIND),
(char *)header, m, &source_addr, &dst_addr, if_index);
daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++;
+ if (stale)
+ daemon->metrics[METRIC_DNS_STALE_ANSWERED]++;
+ }
+
+ if (m == 0 || stale)
+ {
+ if (m != 0)
+ {
+ size_t plen;
+
+ /* We answered with stale cache data, so forward the query anyway to
+ refresh that. Restore the query from the answer packet. */
+ pheader = find_pseudoheader(header, (size_t)m, &plen, NULL, NULL, NULL);
+
+ header->hb3 = hb3;
+ header->hb4 = hb4;
+ header->ancount = htons(0);
+ header->nscount = htons(0);
+ header->arcount = htons(0);
+
+ m = resize_packet(header, m, pheader, plen);
+
+ /* We've already answered the client, so don't send it the answer
+ when it comes back. */
+ fd = -1;
+ }
+
+ if (forward_query(fd, &source_addr, &dst_addr, if_index,
+ header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit, 0))
+ daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++;
+ else
+ daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++;
}
- else if (forward_query(listen->fd, &source_addr, &dst_addr, if_index,
- header, (size_t)n, ((char *) header) + udp_size, now, NULL, ad_reqd, do_bit))
- daemon->metrics[METRIC_DNS_QUERIES_FORWARDED]++;
- else
- daemon->metrics[METRIC_DNS_LOCAL_ANSWERED]++;
}
}
@@ -2004,8 +2187,9 @@ unsigned char *tcp_request(int confd, time_t now,
unsigned char *pheader;
unsigned int mark = 0;
int have_mark = 0;
- int first, last;
+ int first, last, stale, do_stale = 0;
unsigned int flags = 0;
+ u16 hb3, hb4;
/************ Pi-hole modification ************/
bool piholeblocked = false;
@@ -2064,13 +2248,37 @@ unsigned char *tcp_request(int confd, time_t now,
{
int ede = EDE_UNSET;
- if (query_count == TCP_MAX_QUERIES ||
- !packet ||
- !read_write(confd, &c1, 1, 1) || !read_write(confd, &c2, 1, 1) ||
- !(size = c1 << 8 | c2) ||
- !read_write(confd, payload, size, 1))
- return packet;
-
+ if (query_count == TCP_MAX_QUERIES)
+ return packet;
+
+ if (do_stale)
+ {
+ size_t plen;
+
+ /* We answered the last query with stale data. Now try and get fresh data.
+ Restore query from answer. */
+ pheader = find_pseudoheader(header, m, &plen, NULL, NULL, NULL);
+
+ header->hb3 = hb3;
+ header->hb4 = hb4;
+ header->ancount = htons(0);
+ header->nscount = htons(0);
+ header->arcount = htons(0);
+
+ size = resize_packet(header, m, pheader, plen);
+ }
+ else
+ {
+ if (!read_write(confd, &c1, 1, 1) || !read_write(confd, &c2, 1, 1) ||
+ !(size = c1 << 8 | c2) ||
+ !read_write(confd, payload, size, 1))
+ return packet;
+
+ /* for stale-answer processing. */
+ hb3 = header->hb3;
+ hb4 = header->hb4;
+ }
+
if (size < (int)sizeof(struct dns_header))
continue;
@@ -2101,28 +2309,31 @@ unsigned char *tcp_request(int confd, time_t now,
struct auth_zone *zone;
#endif
- log_query_mysockaddr(F_QUERY | F_FORWARD, daemon->namebuff,
- &peer_addr, auth_dns ? "auth" : "query", qtype);
-
- piholeblocked = FTL_new_query(F_QUERY | F_FORWARD, daemon->namebuff,
- &peer_addr, auth_dns ? "auth" : "query", qtype, daemon->log_display_id, &edns, TCP);
-
#ifdef HAVE_CONNTRACK
is_single_query = 1;
#endif
-
+
+ if (!do_stale)
+ {
+ log_query_mysockaddr(F_QUERY | F_FORWARD, daemon->namebuff,
+ &peer_addr, auth_dns ? "auth" : "query", qtype);
+
+ piholeblocked = FTL_new_query(F_QUERY | F_FORWARD, daemon->namebuff,
+ &peer_addr, auth_dns ? "auth" : "query", qtype, daemon->log_display_id, &edns, TCP);
+
#ifdef HAVE_AUTH
- /* find queries for zones we're authoritative for, and answer them directly */
- if (!auth_dns && !option_bool(OPT_LOCALISE))
- for (zone = daemon->auth_zones; zone; zone = zone->next)
- if (in_zone(zone, daemon->namebuff, NULL))
- {
- auth_dns = 1;
- local_auth = 1;
- break;
- }
+ /* find queries for zones we're authoritative for, and answer them directly */
+ if (!auth_dns && !option_bool(OPT_LOCALISE))
+ for (zone = daemon->auth_zones; zone; zone = zone->next)
+ if (in_zone(zone, daemon->namebuff, NULL))
+ {
+ auth_dns = 1;
+ local_auth = 1;
+ break;
+ }
#endif
+ }
}
norebind = domain_no_rebind(daemon->namebuff);
@@ -2184,6 +2395,7 @@ unsigned char *tcp_request(int confd, time_t now,
if(piholeblocked)
{
int ede = EDE_UNSET;
+ stale = 0;
// Generate DNS packet for reply
m = FTL_make_answer(header, ((char *) header) + 65536, size, &ede);
// The pseudoheader may contain important information such as EDNS0 version important for
@@ -2202,11 +2414,14 @@ unsigned char *tcp_request(int confd, time_t now,
else
{
/**********************************************/
+
+ if (do_stale)
+ m = 0;
+ else
+ /* m > 0 if answered from cache */
+ m = answer_request(header, ((char *) header) + 65536, (size_t)size,
+ dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader, &stale);
- /* m > 0 if answered from cache */
- m = answer_request(header, ((char *) header) + 65536, (size_t)size,
- dst_addr_4, netmask, now, ad_reqd, do_bit, have_pseudoheader);
-
/* Do this by steam now we're not in the select() loop */
check_log_writer(1);
@@ -2327,6 +2542,9 @@ unsigned char *tcp_request(int confd, time_t now,
/**********************************************/
}
+ if (do_stale)
+ break;
+
/* In case of local answer or no connections made. */
if (m == 0 && !piholeblocked) // Pi-hole modified to ensure we don't provide local answers when dropping the reply
{
@@ -2337,13 +2555,19 @@ unsigned char *tcp_request(int confd, time_t now,
if (have_pseudoheader)
{
u16 swap = htons((u16)ede);
-
- if (ede != EDE_UNSET)
- m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0);
- else
- m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0);
+
+ if (ede != EDE_UNSET)
+ m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0);
+ else
+ m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0);
}
}
+ else if (stale)
+ {
+ u16 swap = htons((u16)EDE_STALE);
+
+ m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0);
+ }
check_log_writer(1);
@@ -2358,8 +2582,26 @@ unsigned char *tcp_request(int confd, time_t now,
#endif
if (!read_write(confd, packet, m + sizeof(u16), 0))
break;
+
+ /* If we answered with stale data, this process will now try and get fresh data into
+ the cache then and cannot therefore accept new queries. Close the incoming
+ connection to signal that to the client. Then set do_stale and loop round
+ once more to try and get fresh data, after which we exit. */
+ if (stale)
+ {
+ shutdown(confd, SHUT_RDWR);
+ close(confd);
+ do_stale = 1;
+ }
}
-
+
+ /* If we ran once to get fresh data, confd is already closed. */
+ if (!do_stale)
+ {
+ shutdown(confd, SHUT_RDWR);
+ close(confd);
+ }
+
return packet;
}
@@ -2371,16 +2613,36 @@ static int random_sock(struct server *s)
if ((fd = socket(s->source_addr.sa.sa_family, SOCK_DGRAM, 0)) != -1)
{
+ /* We need to set IPV6ONLY so we can use the same ports
+ for IPv4 and IPV6, otherwise, in restriced port situations,
+ we can end up with all our available ports in use for
+ one address family, and the other address family cannot be used. */
+ if (s->source_addr.sa.sa_family == AF_INET6)
+ {
+ int opt = 1;
+
+ if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) == -1)
+ {
+ close(fd);
+ return -1;
+ }
+ }
+
if (local_bind(fd, &s->source_addr, s->interface, s->ifindex, 0))
return fd;
- if (s->interface[0] == 0)
- (void)prettyprint_addr(&s->source_addr, daemon->namebuff);
- else
- strcpy(daemon->namebuff, s->interface);
-
- my_syslog(LOG_ERR, _("failed to bind server socket to %s: %s"),
- daemon->namebuff, strerror(errno));
+ /* don't log errors due to running out of available ports, we handle those. */
+ if (!sockaddr_isnull(&s->source_addr) || errno != EADDRINUSE)
+ {
+ if (s->interface[0] == 0)
+ (void)prettyprint_addr(&s->source_addr, daemon->addrbuff);
+ else
+ safe_strncpy(daemon->addrbuff, s->interface, ADDRSTRLEN);
+
+ my_syslog(LOG_ERR, _("failed to bind server socket to %s: %s"),
+ daemon->addrbuff, strerror(errno));
+ }
+
close(fd);
}
@@ -2410,39 +2672,93 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv)
{
static int finger = 0;
int i, j = 0;
- struct randfd_list *rfl;
+ int ports_full = 0;
+ struct randfd_list **up, *rfl, *found, **found_link;
struct randfd *rfd = NULL;
int fd = 0;
+ int ports_avail = 0;
+
+ /* We can't have more randomsocks for this AF available than ports in our port range,
+ so check that here, to avoid trying and failing to bind every port
+ in local_bind(), called from random_sock(). The actual check is below when
+ ports_avail != 0 */
+ if (daemon->max_port != 0)
+ {
+ ports_avail = daemon->max_port - daemon->min_port + 1;
+ if (ports_avail >= SMALL_PORT_RANGE)
+ ports_avail = 0;
+ }
/* If server has a pre-allocated fd, use that. */
if (serv->sfd)
return serv->sfd->fd;
- /* existing suitable random port socket linked to this transaction? */
- for (rfl = *fdlp; rfl; rfl = rfl->next)
+ /* existing suitable random port socket linked to this transaction?
+ Find the last one in the list and count how many there are. */
+ for (found = NULL, found_link = NULL, i = 0, up = fdlp, rfl = *fdlp; rfl; up = &rfl->next, rfl = rfl->next)
if (server_isequal(serv, rfl->rfd->serv))
- return rfl->rfd->fd;
+ {
+ i++;
+ found = rfl;
+ found_link = up;
+ }
- /* No. need new link. */
+ /* We have the maximum number for this query already. Promote
+ the last one on the list to the head, to circulate them,
+ and return it. */
+ if (found && i >= daemon->randport_limit)
+ {
+ *found_link = found->next;
+ found->next = *fdlp;
+ *fdlp = found;
+ return found->rfd->fd;
+ }
+
+ /* check for all available ports in use. */
+ if (ports_avail != 0)
+ {
+ int ports_inuse;
+
+ for (ports_inuse = 0, i = 0; i < daemon->numrrand; i++)
+ if (daemon->randomsocks[i].refcount != 0 &&
+ daemon->randomsocks[i].serv->source_addr.sa.sa_family == serv->source_addr.sa.sa_family &&
+ ++ports_inuse >= ports_avail)
+ {
+ ports_full = 1;
+ break;
+ }
+ }
+
+ /* limit the number of sockets we have open to avoid starvation of
+ (eg) TFTP. Once we have a reasonable number, randomness should be OK */
+ if (!ports_full)
+ for (i = 0; i < daemon->numrrand; i++)
+ if (daemon->randomsocks[i].refcount == 0)
+ {
+ if ((fd = random_sock(serv)) != -1)
+ {
+ rfd = &daemon->randomsocks[i];
+ rfd->serv = serv;
+ rfd->fd = fd;
+ rfd->refcount = 1;
+ }
+ break;
+ }
+
+ /* No good existing. Need new link. */
if ((rfl = daemon->rfl_spare))
daemon->rfl_spare = rfl->next;
else if (!(rfl = whine_malloc(sizeof(struct randfd_list))))
- return -1;
-
- /* limit the number of sockets we have open to avoid starvation of
- (eg) TFTP. Once we have a reasonable number, randomness should be OK */
- for (i = 0; i < daemon->numrrand; i++)
- if (daemon->randomsocks[i].refcount == 0)
- {
- if ((fd = random_sock(serv)) != -1)
- {
- rfd = &daemon->randomsocks[i];
- rfd->serv = serv;
- rfd->fd = fd;
- rfd->refcount = 1;
- }
- break;
- }
+ {
+ /* malloc failed, don't leak allocated sock */
+ if (rfd)
+ {
+ close(rfd->fd);
+ rfd->refcount = 0;
+ }
+
+ return -1;
+ }
/* No free ones or cannot get new socket, grab an existing one */
if (!rfd)
@@ -2453,10 +2769,19 @@ int allocate_rfd(struct randfd_list **fdlp, struct server *serv)
server_isequal(serv, daemon->randomsocks[i].serv) &&
daemon->randomsocks[i].refcount != 0xfffe)
{
- finger = i + 1;
- rfd = &daemon->randomsocks[i];
- rfd->refcount++;
- break;
+ struct randfd_list *rl;
+ /* Don't pick one we already have. */
+ for (rl = *fdlp; rl; rl = rl->next)
+ if (rl->rfd == &daemon->randomsocks[i])
+ break;
+
+ if (!rl)
+ {
+ finger = i + 1;
+ rfd = &daemon->randomsocks[i];
+ rfd->refcount++;
+ break;
+ }
}
}
@@ -2568,13 +2893,13 @@ static void free_frec(struct frec *f)
f->sentto = NULL;
f->flags = 0;
-#ifdef HAVE_DNSSEC
if (f->stash)
{
blockdata_free(f->stash);
f->stash = NULL;
}
-
+
+#ifdef HAVE_DNSSEC
/* Anything we're waiting on is pointless now, too */
if (f->blocking_query)
{
@@ -2630,6 +2955,7 @@ static struct frec *get_new_frec(time_t now, struct server *master, int force)
{
if (difftime(now, f->time) >= 4*TIMEOUT)
{
+ daemon->metrics[METRIC_DNS_UNANSWERED_QUERY]++;
free_frec(f);
target = f;
}
@@ -2651,6 +2977,7 @@ static struct frec *get_new_frec(time_t now, struct server *master, int force)
if (!target && oldest && ((int)difftime(now, oldest->time)) >= TIMEOUT)
{
/* can't find empty one, use oldest if there is one and it's older than timeout */
+ daemon->metrics[METRIC_DNS_UNANSWERED_QUERY]++;
free_frec(oldest);
target = oldest;
}
@@ -2662,8 +2989,11 @@ static struct frec *get_new_frec(time_t now, struct server *master, int force)
}
if (target)
- target->time = now;
-
+ {
+ target->time = now;
+ target->forward_delay = daemon->fast_retry_time;
+ }
+
return target;
}
diff --git a/src/dnsmasq/hash-questions.c b/src/dnsmasq/hash-questions.c
index 4b1dd919..adcf62c8 100644
--- a/src/dnsmasq/hash-questions.c
+++ b/src/dnsmasq/hash-questions.c
@@ -23,7 +23,7 @@
The hash used is SHA-256. If we're building with DNSSEC support,
we use the Nettle cypto library. If not, we prefer not to
- add a dependency on Nettle, and use a stand-alone implementaion.
+ add a dependency on Nettle, and use a stand-alone implementation.
*/
#include "dnsmasq.h"
diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c
index 0bb2b5c4..ca5f8c9b 100644
--- a/src/dnsmasq/helper.c
+++ b/src/dnsmasq/helper.c
@@ -430,6 +430,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd)
end = extradata + data.ed_len;
buf = extradata;
+
+ lua_pushnumber(lua, data.ed_len == 0 ? 1 : 0);
+ lua_setfield(lua, -2, "data_missing");
if (!is6)
buf = grab_extradata_lua(buf, end, "vendor_class");
@@ -457,7 +460,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd)
buf = grab_extradata_lua(buf, end, "subscriber_id");
buf = grab_extradata_lua(buf, end, "remote_id");
}
-
+
+ buf = grab_extradata_lua(buf, end, "requested_options");
+ buf = grab_extradata_lua(buf, end, "mud_url");
buf = grab_extradata_lua(buf, end, "tags");
if (is6)
@@ -608,6 +613,9 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd)
end = extradata + data.ed_len;
buf = extradata;
+
+ if (data.ed_len == 0)
+ my_setenv("DNSMASQ_DATA_MISSING", "1", &err);
if (!is6)
buf = grab_extradata(buf, end, "DNSMASQ_VENDOR_CLASS", &err);
@@ -636,11 +644,12 @@ int create_helper(int event_fd, int err_fd, uid_t uid, gid_t gid, long max_fd)
buf = grab_extradata(buf, end, "DNSMASQ_CIRCUIT_ID", &err);
buf = grab_extradata(buf, end, "DNSMASQ_SUBSCRIBER_ID", &err);
buf = grab_extradata(buf, end, "DNSMASQ_REMOTE_ID", &err);
- buf = grab_extradata(buf, end, "DNSMASQ_REQUESTED_OPTIONS", &err);
}
+ buf = grab_extradata(buf, end, "DNSMASQ_REQUESTED_OPTIONS", &err);
+ buf = grab_extradata(buf, end, "DNSMASQ_MUD_URL", &err);
buf = grab_extradata(buf, end, "DNSMASQ_TAGS", &err);
-
+
if (is6)
buf = grab_extradata(buf, end, "DNSMASQ_RELAY_ADDRESS", &err);
else
diff --git a/src/dnsmasq/inotify.c b/src/dnsmasq/inotify.c
index 5687e37c..d3c8277b 100644
--- a/src/dnsmasq/inotify.c
+++ b/src/dnsmasq/inotify.c
@@ -133,81 +133,112 @@ void inotify_dnsmasq_init()
}
}
+static struct hostsfile *dyndir_addhosts(struct dyndir *dd, char *path)
+{
+ /* Check if this file is already known in dd->files */
+ struct hostsfile *ah = NULL;
+ for(ah = dd->files; ah; ah = ah->next)
+ if(ah && ah->fname && strcmp(path, ah->fname) == 0)
+ return ah;
+
+ /* Not known, create new hostsfile record for this dyndir */
+ struct hostsfile *newah = NULL;
+ if(!(newah = whine_malloc(sizeof(struct hostsfile))))
+ return NULL;
+
+ /* Add this file to the tip of the linked list */
+ newah->next = dd->files;
+ dd->files = newah;
+
+ /* Copy flags, set index and the full file path */
+ newah->flags = dd->flags;
+ newah->index = daemon->host_index++;
+ newah->fname = path;
+
+ return newah;
+}
+
/* initialisation for dynamic-dir. Set inotify watch for each directory, and read pre-existing files */
void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revhashsz)
{
- struct hostsfile *ah;
-
- for (ah = daemon->dynamic_dirs; ah; ah = ah->next)
+ struct dyndir *dd;
+
+ for (dd = daemon->dynamic_dirs; dd; dd = dd->next)
{
DIR *dir_stream = NULL;
struct dirent *ent;
struct stat buf;
-
- if (!(ah->flags & flag))
+
+ if (!(dd->flags & flag))
continue;
-
- if (stat(ah->fname, &buf) == -1)
+
+ if (stat(dd->dname, &buf) == -1)
{
my_syslog(LOG_ERR, _("bad dynamic directory %s: %s"),
- ah->fname, strerror(errno));
+ dd->dname, strerror(errno));
continue;
}
if (!(S_ISDIR(buf.st_mode)))
{
my_syslog(LOG_ERR, _("bad dynamic directory %s: %s"),
- ah->fname, _("not a directory"));
+ dd->dname, _("not a directory"));
continue;
}
-
- if (!(ah->flags & AH_WD_DONE))
+
+ if (!(dd->flags & AH_WD_DONE))
{
- ah->wd = inotify_add_watch(daemon->inotifyfd, ah->fname, IN_CLOSE_WRITE | IN_MOVED_TO);
- ah->flags |= AH_WD_DONE;
+ dd->wd = inotify_add_watch(daemon->inotifyfd, dd->dname, IN_CLOSE_WRITE | IN_MOVED_TO | IN_DELETE);
+ dd->flags |= AH_WD_DONE;
}
/* Read contents of dir _after_ calling add_watch, in the hope of avoiding
a race which misses files being added as we start */
- if (ah->wd == -1 || !(dir_stream = opendir(ah->fname)))
+ if (dd->wd == -1 || !(dir_stream = opendir(dd->dname)))
{
my_syslog(LOG_ERR, _("failed to create inotify for %s: %s"),
- ah->fname, strerror(errno));
+ dd->dname, strerror(errno));
continue;
}
while ((ent = readdir(dir_stream)))
{
- size_t lendir = strlen(ah->fname);
+ size_t lendir = strlen(dd->dname);
size_t lenfile = strlen(ent->d_name);
char *path;
-
+
/* ignore emacs backups and dotfiles */
if (lenfile == 0 ||
ent->d_name[lenfile - 1] == '~' ||
(ent->d_name[0] == '#' && ent->d_name[lenfile - 1] == '#') ||
ent->d_name[0] == '.')
continue;
-
+
if ((path = whine_malloc(lendir + lenfile + 2)))
{
- strcpy(path, ah->fname);
+ struct hostsfile *ah;
+
+ strcpy(path, dd->dname);
strcat(path, "/");
strcat(path, ent->d_name);
+
+ if (!(ah = dyndir_addhosts(dd, path)))
+ {
+ free(path);
+ continue;
+ }
/* ignore non-regular files */
if (stat(path, &buf) != -1 && S_ISREG(buf.st_mode))
{
- if (ah->flags & AH_HOSTS)
+ if (dd->flags & AH_HOSTS)
total_size = read_hostsfile(path, ah->index, total_size, rhash, revhashsz);
#ifdef HAVE_DHCP
- else if (ah->flags & (AH_DHCP_HST | AH_DHCP_OPT))
- option_read_dynfile(path, ah->flags);
+ else if (dd->flags & (AH_DHCP_HST | AH_DHCP_OPT))
+ option_read_dynfile(path, dd->flags);
#endif
}
-
- free(path);
}
}
@@ -218,7 +249,7 @@ void set_dynamic_inotify(int flag, int total_size, struct crec **rhash, int revh
int inotify_check(time_t now)
{
int hit = 0;
- struct hostsfile *ah;
+ struct dyndir *dd;
while (1)
{
@@ -249,36 +280,51 @@ int inotify_check(time_t now)
if (res->wd == in->wd && strcmp(res->file, in->name) == 0)
hit = 1;
- for (ah = daemon->dynamic_dirs; ah; ah = ah->next)
- if (ah->wd == in->wd)
+ for (dd = daemon->dynamic_dirs; dd; dd = dd->next)
+ if (dd->wd == in->wd)
{
- size_t lendir = strlen(ah->fname);
+ size_t lendir = strlen(dd->dname);
char *path;
-
+
if ((path = whine_malloc(lendir + in->len + 2)))
{
- strcpy(path, ah->fname);
+ struct hostsfile *ah = NULL;
+
+ strcpy(path, dd->dname);
strcat(path, "/");
strcat(path, in->name);
-
- my_syslog(LOG_INFO, _("inotify, new or changed file %s"), path);
- if (ah->flags & AH_HOSTS)
+ /* Is this is a deletion event? */
+ if (in->mask & IN_DELETE)
+ my_syslog(LOG_INFO, _("inotify: %s removed"), path);
+ else
+ my_syslog(LOG_INFO, _("inotify: %s new or modified"), path);
+
+ if (dd->flags & AH_HOSTS)
{
- read_hostsfile(path, ah->index, 0, NULL, 0);
-#ifdef HAVE_DHCP
- if (daemon->dhcp || daemon->doing_dhcp6)
+ if ((ah = dyndir_addhosts(dd, path)))
{
- /* Propagate the consequences of loading a new dhcp-host */
- dhcp_update_configs(daemon->dhcp_conf);
- lease_update_from_configs();
- lease_update_file(now);
- lease_update_dns(1);
- }
+ const unsigned int removed = cache_remove_uid(ah->index);
+ if (removed > 0)
+ my_syslog(LOG_INFO, _("inotify: flushed %u names read from %s"), removed, path);
+
+ /* (Re-)load hostsfile only if this event isn't triggered by deletion */
+ if (!(in->mask & IN_DELETE))
+ read_hostsfile(path, ah->index, 0, NULL, 0);
+#ifdef HAVE_DHCP
+ if (daemon->dhcp || daemon->doing_dhcp6)
+ {
+ /* Propagate the consequences of loading a new dhcp-host */
+ dhcp_update_configs(daemon->dhcp_conf);
+ lease_update_from_configs();
+ lease_update_file(now);
+ lease_update_dns(1);
+ }
#endif
+ }
}
#ifdef HAVE_DHCP
- else if (ah->flags & AH_DHCP_HST)
+ else if (dd->flags & AH_DHCP_HST)
{
if (option_read_dynfile(path, AH_DHCP_HST))
{
@@ -289,11 +335,12 @@ int inotify_check(time_t now)
lease_update_dns(1);
}
}
- else if (ah->flags & AH_DHCP_OPT)
+ else if (dd->flags & AH_DHCP_OPT)
option_read_dynfile(path, AH_DHCP_OPT);
#endif
- free(path);
+ if (!ah)
+ free(path);
}
}
}
diff --git a/src/dnsmasq/lease.c b/src/dnsmasq/lease.c
index 81477d54..8a7b9756 100644
--- a/src/dnsmasq/lease.c
+++ b/src/dnsmasq/lease.c
@@ -1180,17 +1180,11 @@ void lease_add_extradata(struct dhcp_lease *lease, unsigned char *data, unsigned
if ((lease->extradata_size - lease->extradata_len) < (len + 1))
{
size_t newsz = lease->extradata_len + len + 100;
- unsigned char *new = whine_malloc(newsz);
+ unsigned char *new = whine_realloc(lease->extradata, newsz);
if (!new)
return;
- if (lease->extradata)
- {
- memcpy(new, lease->extradata, lease->extradata_len);
- free(lease->extradata);
- }
-
lease->extradata = new;
lease->extradata_size = newsz;
}
diff --git a/src/dnsmasq/metrics.c b/src/dnsmasq/metrics.c
index 68735293..f3e6728a 100644
--- a/src/dnsmasq/metrics.c
+++ b/src/dnsmasq/metrics.c
@@ -22,6 +22,8 @@ const char * metric_names[] = {
"dns_queries_forwarded",
"dns_auth_answered",
"dns_local_answered",
+ "dns_stale_answered",
+ "dns_unanswered",
"bootp",
"pxe",
"dhcp_ack",
@@ -42,3 +44,23 @@ const char * metric_names[] = {
const char* get_metric_name(int i) {
return metric_names[i];
}
+
+void clear_metrics(void)
+{
+ int i;
+ struct server *serv;
+
+ for (i = 0; i < __METRIC_MAX; i++)
+ daemon->metrics[i] = 0;
+
+ for (serv = daemon->servers; serv; serv = serv->next)
+ {
+ serv->queries = 0;
+ serv->failed_queries = 0;
+ serv->failed_queries = 0;
+ serv->retrys = 0;
+ serv->nxdomain_replies = 0;
+ serv->query_latency = 0;
+ }
+}
+
diff --git a/src/dnsmasq/metrics.h b/src/dnsmasq/metrics.h
index df72ec68..6f62a406 100644
--- a/src/dnsmasq/metrics.h
+++ b/src/dnsmasq/metrics.h
@@ -21,6 +21,8 @@ enum {
METRIC_DNS_QUERIES_FORWARDED,
METRIC_DNS_AUTH_ANSWERED,
METRIC_DNS_LOCAL_ANSWERED,
+ METRIC_DNS_STALE_ANSWERED,
+ METRIC_DNS_UNANSWERED_QUERY,
METRIC_BOOTP,
METRIC_PXE,
METRIC_DHCPACK,
@@ -41,3 +43,4 @@ enum {
};
const char* get_metric_name(int);
+void clear_metrics(void);
diff --git a/src/dnsmasq/netlink.c b/src/dnsmasq/netlink.c
index da829430..c156cde3 100644
--- a/src/dnsmasq/netlink.c
+++ b/src/dnsmasq/netlink.c
@@ -258,7 +258,16 @@ int iface_enumerate(int family, void *parm, int (*callback)())
while (RTA_OK(rta, len1))
{
- if (rta->rta_type == IFA_ADDRESS)
+ /*
+ * Important comment: (from if_addr.h)
+ * IFA_ADDRESS is prefix address, rather than local interface address.
+ * It makes no difference for normally configured broadcast interfaces,
+ * but for point-to-point IFA_ADDRESS is DESTINATION address,
+ * local address is supplied in IFA_LOCAL attribute.
+ */
+ if (rta->rta_type == IFA_LOCAL)
+ addrp = ((struct in6_addr *)(rta+1));
+ else if (rta->rta_type == IFA_ADDRESS && !addrp)
addrp = ((struct in6_addr *)(rta+1));
else if (rta->rta_type == IFA_CACHEINFO)
{
diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c
index 67a2089e..c71dea5e 100644
--- a/src/dnsmasq/network.c
+++ b/src/dnsmasq/network.c
@@ -233,6 +233,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label,
union mysockaddr *addr, struct in_addr netmask, int prefixlen, int iface_flags)
{
struct irec *iface;
+ struct cond_domain *cond;
int loopback;
struct ifreq ifr;
int tftp_ok = !!option_bool(OPT_TFTP);
@@ -361,7 +362,7 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label,
if (int_name->flags & INP4)
{
- if (netmask.s_addr == 0xffff)
+ if (netmask.s_addr == 0xffffffff)
continue;
newaddr.s_addr = (addr->in.sin_addr.s_addr & netmask.s_addr) |
@@ -455,7 +456,37 @@ static int iface_allowed(struct iface_param *param, int if_index, char *label,
}
}
}
-
+
+ /* Update addresses for domain=, */
+ for (cond = daemon->cond_domain; cond; cond = cond->next)
+ if (cond->interface && strncmp(label, cond->interface, IF_NAMESIZE) == 0)
+ {
+ struct addrlist *al;
+
+ if (param->spare)
+ {
+ al = param->spare;
+ param->spare = al->next;
+ }
+ else
+ al = whine_malloc(sizeof(struct addrlist));
+
+ if (addr->sa.sa_family == AF_INET)
+ {
+ al->addr.addr4 = addr->in.sin_addr;
+ al->flags = 0;
+ }
+ else
+ {
+ al->addr.addr6 = addr->in6.sin6_addr;
+ al->flags = ADDRLIST_IPV6;
+ }
+
+ al->prefixlen = prefixlen;
+ al->next = cond->al;
+ cond->al = al;
+ }
+
/* check whether the interface IP has been added already
we call this routine multiple times. */
for (iface = daemon->interfaces; iface; iface = iface->next)
@@ -700,6 +731,7 @@ int enumerate_interfaces(int reset)
int errsave, ret = 1;
struct addrlist *addr, *tmp;
struct interface_name *intname;
+ struct cond_domain *cond;
struct irec *iface;
#ifdef HAVE_AUTH
struct auth_zone *zone;
@@ -759,6 +791,19 @@ again:
intname->addr = NULL;
}
+ /* remove addresses stored against cond-domains. */
+ for (cond = daemon->cond_domain; cond; cond = cond->next)
+ {
+ for (addr = cond->al; addr; addr = tmp)
+ {
+ tmp = addr->next;
+ addr->next = spare;
+ spare = addr;
+ }
+
+ cond->al = NULL;
+ }
+
/* Remove list of addresses of local interfaces */
for (addr = daemon->interface_addrs; addr; addr = tmp)
{
@@ -1346,7 +1391,7 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind
or both are set. Otherwise use the OS's random ephemeral port allocation by
leaving port == 0 and tries == 1 */
ports_avail = daemon->max_port - daemon->min_port + 1;
- tries = ports_avail < 30 ? 3 * ports_avail : 100;
+ tries = (ports_avail < SMALL_PORT_RANGE) ? ports_avail : 100;
port = htons(daemon->min_port + (rand16() % ports_avail));
}
@@ -1375,7 +1420,16 @@ int local_bind(int fd, union mysockaddr *addr, char *intname, unsigned int ifind
if (--tries == 0)
return 0;
- port = htons(daemon->min_port + (rand16() % ports_avail));
+ /* For small ranges, do a systematic search, not a random one. */
+ if (ports_avail < SMALL_PORT_RANGE)
+ {
+ unsigned short hport = ntohs(port);
+ if (hport++ == daemon->max_port)
+ hport = daemon->min_port;
+ port = htons(hport);
+ }
+ else
+ port = htons(daemon->min_port + (rand16() % ports_avail));
}
if (!is_tcp && ifindex > 0)
diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c
index eb181ff3..0c61a22f 100644
--- a/src/dnsmasq/option.c
+++ b/src/dnsmasq/option.c
@@ -185,6 +185,10 @@ struct myoption {
#define LOPT_STRIP_MAC 372
#define LOPT_CONF_OPT 373
#define LOPT_CONF_SCRIPT 374
+#define LOPT_RANDPORT_LIM 375
+#define LOPT_FAST_RETRY 376
+#define LOPT_STALE_CACHE 377
+#define LOPT_NORR 378
#ifdef HAVE_GETOPT_LONG
static const struct option opts[] =
@@ -237,6 +241,7 @@ static const struct myoption opts[] =
{ "localmx", 0, 0, 'L' },
{ "local-ttl", 1, 0, 'T' },
{ "no-negcache", 0, 0, 'N' },
+ { "no-round-robin", 0, 0, LOPT_NORR },
{ "addn-hosts", 1, 0, 'H' },
{ "hostsdir", 1, 0, LOPT_HOST_INOTIFY },
{ "query-port", 1, 0, 'Q' },
@@ -370,6 +375,9 @@ static const struct myoption opts[] =
{ "log-debug", 0, 0, LOPT_LOG_DEBUG },
{ "umbrella", 2, 0, LOPT_UMBRELLA },
{ "quiet-tftp", 0, 0, LOPT_QUIET_TFTP },
+ { "port-limit", 1, 0, LOPT_RANDPORT_LIM },
+ { "fast-dns-retry", 2, 0, LOPT_FAST_RETRY },
+ { "use-stale-cache", 2, 0 , LOPT_STALE_CACHE },
{ NULL, 0, 0, 0 }
};
@@ -427,6 +435,7 @@ static struct {
{ 'M', ARG_DUP, "", gettext_noop("Specify BOOTP options to DHCP server."), NULL },
{ 'n', OPT_NO_POLL, NULL, gettext_noop("Do NOT poll %s file, reload only on SIGHUP."), RESOLVFILE },
{ 'N', OPT_NO_NEG, NULL, gettext_noop("Do NOT cache failed search results."), NULL },
+ { LOPT_STALE_CACHE, ARG_ONE, "[=]", gettext_noop("Use expired cache data for faster reply."), NULL },
{ 'o', OPT_ORDER, NULL, gettext_noop("Use nameservers strictly in the order given in %s."), RESOLVFILE },
{ 'O', ARG_DUP, "", gettext_noop("Specify options to be sent to DHCP clients."), NULL },
{ LOPT_FORCE, ARG_DUP, "", gettext_noop("DHCP option sent even if the client does not request it."), NULL},
@@ -434,6 +443,7 @@ static struct {
{ 'P', ARG_ONE, "", gettext_noop("Maximum supported UDP packet size for EDNS.0 (defaults to %s)."), "*" },
{ 'q', ARG_DUP, NULL, gettext_noop("Log DNS queries."), NULL },
{ 'Q', ARG_ONE, "", gettext_noop("Force the originating port for upstream DNS queries."), NULL },
+ { LOPT_RANDPORT_LIM, ARG_ONE, "#ports", gettext_noop("Set maximum number of random originating ports for a query."), NULL },
{ 'R', OPT_NO_RESOLV, NULL, gettext_noop("Do NOT read resolv.conf."), NULL },
{ 'r', ARG_DUP, "", gettext_noop("Specify path to resolv.conf (defaults to %s)."), RESOLVFILE },
{ LOPT_SERVERS_FILE, ARG_ONE, "", gettext_noop("Specify path to file with server= options"), NULL },
@@ -447,6 +457,7 @@ static struct {
{ LOPT_MAXTTL, ARG_ONE, "", gettext_noop("Specify time-to-live in seconds for maximum TTL to send to clients."), NULL },
{ LOPT_MAXCTTL, ARG_ONE, "", gettext_noop("Specify time-to-live ceiling for cache."), NULL },
{ LOPT_MINCTTL, ARG_ONE, "", gettext_noop("Specify time-to-live floor for cache."), NULL },
+ { LOPT_FAST_RETRY, ARG_ONE, "", gettext_noop("Retry DNS queries after this many milliseconds."), NULL},
{ 'u', ARG_ONE, "", gettext_noop("Change to this user after startup. (defaults to %s)."), CHUSER },
{ 'U', ARG_DUP, "set:,", gettext_noop("Map DHCP vendor class to tag."), NULL },
{ 'v', 0, NULL, gettext_noop("Display dnsmasq version and copyright information."), NULL },
@@ -562,6 +573,7 @@ static struct {
{ LOPT_SCRIPT_TIME, OPT_LEASE_RENEW, NULL, gettext_noop("Call dhcp-script when lease expiry changes."), NULL },
{ LOPT_UMBRELLA, ARG_ONE, "[=]", gettext_noop("Send Cisco Umbrella identifiers including remote IP."), NULL },
{ LOPT_QUIET_TFTP, OPT_QUIET_TFTP, NULL, gettext_noop("Do not log routine TFTP."), NULL },
+ { LOPT_NORR, OPT_NORR, NULL, gettext_noop("Suppress round-robin ordering of DNS records."), NULL },
{ 0, 0, NULL, NULL, NULL }
};
@@ -847,117 +859,241 @@ static char *parse_mysockaddr(char *arg, union mysockaddr *addr)
return NULL;
}
-char *parse_server(char *arg, union mysockaddr *addr, union mysockaddr *source_addr, char *interface, u16 *flags)
+char *parse_server(char *arg, struct server_details *sdetails)
{
- int source_port = 0, serv_port = NAMESERVER_PORT;
- char *portno, *source;
- char *interface_opt = NULL;
- int scope_index = 0;
- char *scope_id;
-
- *interface = 0;
+ sdetails->serv_port = NAMESERVER_PORT;
+ char *portno;
+ int ecode = 0;
+ struct addrinfo hints;
+ memset(&hints, 0, sizeof(struct addrinfo));
+
+ *sdetails->interface = 0;
+ sdetails->addr_type = AF_UNSPEC;
+
if (strcmp(arg, "#") == 0)
{
- if (flags)
- *flags |= SERV_USE_RESOLV;
+ if (sdetails->flags)
+ *sdetails->flags |= SERV_USE_RESOLV;
+ sdetails->addr_type = AF_LOCAL;
+ sdetails->valid = 1;
return NULL;
}
- if ((source = split_chr(arg, '@')) && /* is there a source. */
- (portno = split_chr(source, '#')) &&
- !atoi_check16(portno, &source_port))
+ if ((sdetails->source = split_chr(arg, '@')) && /* is there a source. */
+ (portno = split_chr(sdetails->source, '#')) &&
+ !atoi_check16(portno, &sdetails->source_port))
return _("bad port");
if ((portno = split_chr(arg, '#')) && /* is there a port no. */
- !atoi_check16(portno, &serv_port))
+ !atoi_check16(portno, &sdetails->serv_port))
return _("bad port");
- scope_id = split_chr(arg, '%');
+ sdetails->scope_id = split_chr(arg, '%');
- if (source) {
- interface_opt = split_chr(source, '@');
+ if (sdetails->source) {
+ sdetails->interface_opt = split_chr(sdetails->source, '@');
- if (interface_opt)
+ if (sdetails->interface_opt)
{
#if defined(SO_BINDTODEVICE)
- safe_strncpy(interface, source, IF_NAMESIZE);
- source = interface_opt;
+ safe_strncpy(sdetails->interface, sdetails->source, IF_NAMESIZE);
+ sdetails->source = sdetails->interface_opt;
#else
return _("interface binding not supported");
#endif
}
}
- if (inet_pton(AF_INET, arg, &addr->in.sin_addr) > 0)
+ if (inet_pton(AF_INET, arg, &sdetails->addr->in.sin_addr) > 0)
+ sdetails->addr_type = AF_INET;
+ else if (inet_pton(AF_INET6, arg, &sdetails->addr->in6.sin6_addr) > 0)
+ sdetails->addr_type = AF_INET6;
+ else
{
- addr->in.sin_port = htons(serv_port);
- addr->sa.sa_family = source_addr->sa.sa_family = AF_INET;
-#ifdef HAVE_SOCKADDR_SA_LEN
- source_addr->in.sin_len = addr->in.sin_len = sizeof(struct sockaddr_in);
+ /* if the argument is neither an IPv4 not an IPv6 address, it might be a
+ hostname and we should try to resolve it to a suitable address. */
+ memset(&hints, 0, sizeof(hints));
+ /* The AI_ADDRCONFIG flag ensures that then IPv4 addresses are returned in
+ the result only if the local system has at least one IPv4 address
+ configured, and IPv6 addresses are returned only if the local system
+ has at least one IPv6 address configured. The loopback address is not
+ considered for this case as valid as a configured address. This flag is
+ useful on, for example, IPv4-only systems, to ensure that getaddrinfo()
+ does not return IPv6 socket addresses that would always fail in
+ subsequent connect() or bind() attempts. */
+ hints.ai_flags = AI_ADDRCONFIG;
+#if defined(HAVE_IDN) && defined(AI_IDN)
+ /* If the AI_IDN flag is specified and we have glibc 2.3.4 or newer, then
+ the node name given in node is converted to IDN format if necessary.
+ The source encoding is that of the current locale. */
+ hints.ai_flags |= AI_IDN;
#endif
- source_addr->in.sin_addr.s_addr = INADDR_ANY;
- source_addr->in.sin_port = htons(daemon->query_port);
-
- if (source)
- {
- if (flags)
- *flags |= SERV_HAS_SOURCE;
- source_addr->in.sin_port = htons(source_port);
- if (!(inet_pton(AF_INET, source, &source_addr->in.sin_addr) > 0))
- {
-#if defined(SO_BINDTODEVICE)
- if (interface_opt)
- return _("interface can only be specified once");
-
- source_addr->in.sin_addr.s_addr = INADDR_ANY;
- safe_strncpy(interface, source, IF_NAMESIZE);
-#else
- return _("interface binding not supported");
-#endif
- }
- }
- }
- else if (inet_pton(AF_INET6, arg, &addr->in6.sin6_addr) > 0)
- {
- if (scope_id && (scope_index = if_nametoindex(scope_id)) == 0)
- return _("bad interface name");
-
- addr->in6.sin6_port = htons(serv_port);
- addr->in6.sin6_scope_id = scope_index;
- source_addr->in6.sin6_addr = in6addr_any;
- source_addr->in6.sin6_port = htons(daemon->query_port);
- source_addr->in6.sin6_scope_id = 0;
- addr->sa.sa_family = source_addr->sa.sa_family = AF_INET6;
- addr->in6.sin6_flowinfo = source_addr->in6.sin6_flowinfo = 0;
-#ifdef HAVE_SOCKADDR_SA_LEN
- addr->in6.sin6_len = source_addr->in6.sin6_len = sizeof(addr->in6);
-#endif
- if (source)
- {
- if (flags)
- *flags |= SERV_HAS_SOURCE;
- source_addr->in6.sin6_port = htons(source_port);
- if (inet_pton(AF_INET6, source, &source_addr->in6.sin6_addr) == 0)
- {
-#if defined(SO_BINDTODEVICE)
- if (interface_opt)
- return _("interface can only be specified once");
-
- source_addr->in6.sin6_addr = in6addr_any;
- safe_strncpy(interface, source, IF_NAMESIZE);
-#else
- return _("interface binding not supported");
-#endif
- }
- }
- }
- else
- return _("bad address");
+ /* The value AF_UNSPEC indicates that getaddrinfo() should return socket
+ addresses for any address family (either IPv4 or IPv6, for example)
+ that can be used with node and service "domain". */
+ hints.ai_family = AF_UNSPEC;
+ /* Get addresses suitable for sending datagrams. We assume that we can use the
+ same addresses for TCP connections. Settting this to zero gets each address
+ threes times, for SOCK_STREAM, SOCK_RAW and SOCK_DGRAM, which is not useful. */
+ hints.ai_socktype = SOCK_DGRAM;
+
+ /* Get address associated with this hostname */
+ ecode = getaddrinfo(arg, NULL, &hints, &sdetails->hostinfo);
+ if (ecode == 0)
+ {
+ /* The getaddrinfo() function allocated and initialized a linked list of
+ addrinfo structures, one for each network address that matches node
+ and service, subject to the restrictions imposed by our
+ above, and returns a pointer to the start of the list in .
+ The items in the linked list are linked by the field. */
+ sdetails->valid = 1;
+ sdetails->orig_hostinfo = sdetails->hostinfo;
+ return NULL;
+ }
+ else
+ {
+ /* Lookup failed, return human readable error string */
+ if (ecode == EAI_AGAIN)
+ return _("Cannot resolve server name");
+ else
+ return _((char*)gai_strerror(ecode));
+ }
+ }
+
+ sdetails->valid = 1;
return NULL;
}
+char *parse_server_addr(struct server_details *sdetails)
+{
+ if (sdetails->addr_type == AF_INET)
+ {
+ sdetails->addr->in.sin_port = htons(sdetails->serv_port);
+ sdetails->addr->sa.sa_family = sdetails->source_addr->sa.sa_family = AF_INET;
+#ifdef HAVE_SOCKADDR_SA_LEN
+ sdetails->source_addr->in.sin_len = sdetails->addr->in.sin_len = sizeof(struct sockaddr_in);
+#endif
+ sdetails->source_addr->in.sin_addr.s_addr = INADDR_ANY;
+ sdetails->source_addr->in.sin_port = htons(daemon->query_port);
+
+ if (sdetails->source)
+ {
+ if (sdetails->flags)
+ *sdetails->flags |= SERV_HAS_SOURCE;
+ sdetails->source_addr->in.sin_port = htons(sdetails->source_port);
+ if (inet_pton(AF_INET, sdetails->source, &sdetails->source_addr->in.sin_addr) == 0)
+ {
+ if (inet_pton(AF_INET6, sdetails->source, &sdetails->source_addr->in6.sin6_addr) == 1)
+ {
+ sdetails->source_addr->sa.sa_family = AF_INET6;
+ /* When resolving a server IP by hostname, we can simply skip mismatching
+ server / source IP pairs. Otherwise, when an IP address is given directly,
+ this is a fatal error. */
+ if (!sdetails->orig_hostinfo)
+ return _("cannot use IPv4 server address with IPv6 source address");
+ }
+ else
+ {
+#if defined(SO_BINDTODEVICE)
+ if (sdetails->interface_opt)
+ return _("interface can only be specified once");
+
+ sdetails->source_addr->in.sin_addr.s_addr = INADDR_ANY;
+ safe_strncpy(sdetails->interface, sdetails->source, IF_NAMESIZE);
+#else
+ return _("interface binding not supported");
+#endif
+ }
+ }
+ }
+ }
+ else if (sdetails->addr_type == AF_INET6)
+ {
+ if (sdetails->scope_id && (sdetails->scope_index = if_nametoindex(sdetails->scope_id)) == 0)
+ return _("bad interface name");
+
+ sdetails->addr->in6.sin6_port = htons(sdetails->serv_port);
+ sdetails->addr->in6.sin6_scope_id = sdetails->scope_index;
+ sdetails->source_addr->in6.sin6_addr = in6addr_any;
+ sdetails->source_addr->in6.sin6_port = htons(daemon->query_port);
+ sdetails->source_addr->in6.sin6_scope_id = 0;
+ sdetails->addr->sa.sa_family = sdetails->source_addr->sa.sa_family = AF_INET6;
+ sdetails->addr->in6.sin6_flowinfo = sdetails->source_addr->in6.sin6_flowinfo = 0;
+#ifdef HAVE_SOCKADDR_SA_LEN
+ sdetails->addr->in6.sin6_len = sdetails->source_addr->in6.sin6_len = sizeof(sdetails->addr->in6);
+#endif
+ if (sdetails->source)
+ {
+ if (sdetails->flags)
+ *sdetails->flags |= SERV_HAS_SOURCE;
+ sdetails->source_addr->in6.sin6_port = htons(sdetails->source_port);
+ if (inet_pton(AF_INET6, sdetails->source, &sdetails->source_addr->in6.sin6_addr) == 0)
+ {
+ if (inet_pton(AF_INET, sdetails->source, &sdetails->source_addr->in.sin_addr) == 1)
+ {
+ sdetails->source_addr->sa.sa_family = AF_INET;
+ /* When resolving a server IP by hostname, we can simply skip mismatching
+ server / source IP pairs. Otherwise, when an IP address is given directly,
+ this is a fatal error. */
+ if(!sdetails->orig_hostinfo)
+ return _("cannot use IPv6 server address with IPv4 source address");
+ }
+ else
+ {
+#if defined(SO_BINDTODEVICE)
+ if (sdetails->interface_opt)
+ return _("interface can only be specified once");
+
+ sdetails->source_addr->in6.sin6_addr = in6addr_any;
+ safe_strncpy(sdetails->interface, sdetails->source, IF_NAMESIZE);
+#else
+ return _("interface binding not supported");
+#endif
+ }
+ }
+ }
+ }
+ else if (sdetails->addr_type != AF_LOCAL)
+ return _("bad address");
+
+ return NULL;
+}
+
+int parse_server_next(struct server_details *sdetails)
+{
+ /* Looping over resolved addresses? */
+ if (sdetails->hostinfo)
+ {
+ /* Get address type */
+ sdetails->addr_type = sdetails->hostinfo->ai_family;
+
+ /* Get address */
+ if (sdetails->addr_type == AF_INET)
+ memcpy(&sdetails->addr->in.sin_addr,
+ &((struct sockaddr_in *) sdetails->hostinfo->ai_addr)->sin_addr,
+ sizeof(sdetails->addr->in.sin_addr));
+ else if (sdetails->addr_type == AF_INET6)
+ memcpy(&sdetails->addr->in6.sin6_addr,
+ &((struct sockaddr_in6 *) sdetails->hostinfo->ai_addr)->sin6_addr,
+ sizeof(sdetails->addr->in6.sin6_addr));
+
+ /* Iterate to the next available address */
+ sdetails->valid = sdetails->hostinfo->ai_next != NULL;
+ sdetails->hostinfo = sdetails->hostinfo->ai_next;
+ return 1;
+ }
+ else if (sdetails->valid)
+ {
+ /* When using an IP address, we return the address only once */
+ sdetails->valid = 0;
+ return 1;
+ }
+ /* Stop iterating here, we used all available addresses */
+ return 0;
+}
+
static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int size)
{
int i, j;
@@ -968,22 +1104,29 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int
union mysockaddr serv_addr, source_addr;
char interface[IF_NAMESIZE+1];
int count = 1, rem, addrbytes, addrbits;
-
+ struct server_details sdetails;
+
+ memset(&sdetails, 0, sizeof(struct server_details));
+ sdetails.addr = &serv_addr;
+ sdetails.source_addr = &source_addr;
+ sdetails.interface = interface;
+ sdetails.flags = &flags;
+
if (!server)
flags = SERV_LITERAL_ADDRESS;
- else if ((string = parse_server(server, &serv_addr, &source_addr, interface, &flags)))
+ else if ((string = parse_server(server, &sdetails)))
return string;
-
+
if (from_file)
flags |= SERV_FROM_FILE;
-
+
rem = size & 0x7;
addrbytes = (32 - size) >> 3;
addrbits = (32 - size) & 7;
if (size > 32 || size < 1)
return _("bad IPv4 prefix length");
-
+
/* Zero out last address bits according to CIDR mask */
((u8 *)addr4)[3-addrbytes] &= ~((1 << addrbits)-1);
@@ -997,23 +1140,40 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int
*domain = 0;
string = domain;
msize = size/8;
-
+
for (j = (rem == 0) ? msize-1 : msize; j >= 0; j--)
{
int dig = ((unsigned char *)addr4)[j];
-
+
if (j == msize)
dig += i;
-
+
string += sprintf(string, "%d.", dig);
}
-
+
sprintf(string, "in-addr.arpa");
- if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL))
- return _("error");
- }
+ if (flags & SERV_LITERAL_ADDRESS)
+ {
+ if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL))
+ return _("error");
+ }
+ else
+ {
+ while (parse_server_next(&sdetails))
+ {
+ if ((string = parse_server_addr(&sdetails)))
+ return string;
+
+ if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL))
+ return _("error");
+ }
+ if (sdetails.orig_hostinfo)
+ freeaddrinfo(sdetails.orig_hostinfo);
+ }
+ }
+
return NULL;
}
@@ -1027,10 +1187,17 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in
union mysockaddr serv_addr, source_addr;
char interface[IF_NAMESIZE+1];
int count = 1, rem, addrbytes, addrbits;
-
+ struct server_details sdetails;
+
+ memset(&sdetails, 0, sizeof(struct server_details));
+ sdetails.addr = &serv_addr;
+ sdetails.source_addr = &source_addr;
+ sdetails.interface = interface;
+ sdetails.flags = &flags;
+
if (!server)
flags = SERV_LITERAL_ADDRESS;
- else if ((string = parse_server(server, &serv_addr, &source_addr, interface, &flags)))
+ else if ((string = parse_server(server, &sdetails)))
return string;
if (from_file)
@@ -1071,10 +1238,27 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in
sprintf(string, "ip6.arpa");
- if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL))
- return _("error");
- }
+ if (flags & SERV_LITERAL_ADDRESS)
+ {
+ if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL))
+ return _("error");
+ }
+ else
+ {
+ while (parse_server_next(&sdetails))
+ {
+ if ((string = parse_server_addr(&sdetails)))
+ return string;
+
+ if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL))
+ return _("error");
+ }
+ if (sdetails.orig_hostinfo)
+ freeaddrinfo(sdetails.orig_hostinfo);
+ }
+ }
+
return NULL;
}
@@ -2157,15 +2341,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
case LOPT_DHCP_HOST: /* --dhcp-hostsfile */
case LOPT_DHCP_OPTS: /* --dhcp-optsfile */
- case LOPT_DHCP_INOTIFY: /* --dhcp-hostsdir */
- case LOPT_DHOPT_INOTIFY: /* --dhcp-optsdir */
- case LOPT_HOST_INOTIFY: /* --hostsdir */
case 'H': /* --addn-hosts */
{
struct hostsfile *new = opt_malloc(sizeof(struct hostsfile));
- static unsigned int hosts_index = SRC_AH;
new->fname = opt_string_alloc(arg);
- new->index = hosts_index++;
+ new->index = daemon->host_index++;
new->flags = 0;
if (option == 'H')
{
@@ -2181,21 +2361,29 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
{
new->next = daemon->dhcp_opts_file;
daemon->dhcp_opts_file = new;
- }
- else
- {
- new->next = daemon->dynamic_dirs;
- daemon->dynamic_dirs = new;
- if (option == LOPT_DHCP_INOTIFY)
- new->flags |= AH_DHCP_HST;
- else if (option == LOPT_DHOPT_INOTIFY)
- new->flags |= AH_DHCP_OPT;
- else if (option == LOPT_HOST_INOTIFY)
- new->flags |= AH_HOSTS;
}
break;
}
+
+ case LOPT_DHCP_INOTIFY: /* --dhcp-hostsdir */
+ case LOPT_DHOPT_INOTIFY: /* --dhcp-optsdir */
+ case LOPT_HOST_INOTIFY: /* --hostsdir */
+ {
+ struct dyndir *new = opt_malloc(sizeof(struct dyndir));
+ new->dname = opt_string_alloc(arg);
+ new->flags = 0;
+ new->next = daemon->dynamic_dirs;
+ daemon->dynamic_dirs = new;
+ if (option == LOPT_DHCP_INOTIFY)
+ new->flags |= AH_DHCP_HST;
+ else if (option == LOPT_DHOPT_INOTIFY)
+ new->flags |= AH_DHCP_OPT;
+ else if (option == LOPT_HOST_INOTIFY)
+ new->flags |= AH_HOSTS;
+
+ break;
+ }
case LOPT_AUTHSERV: /* --auth-server */
comma = split(arg);
@@ -2496,9 +2684,15 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
else if (!inet_pton(AF_INET6, arg, &new->end6))
ret_err_free(gen_err, new);
}
- else
+ else if (option == 's')
+ {
+ /* subnet from interface. */
+ new->interface = opt_string_alloc(comma);
+ new->al = NULL;
+ }
+ else
ret_err_free(gen_err, new);
-
+
if (option != 's' && prefstr)
{
if (!(new->prefix = canonicalise_opt(prefstr)) ||
@@ -2775,13 +2969,20 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
case LOPT_LOCAL: /* --local */
case 'A': /* --address */
{
- char *lastdomain = NULL, *domain = "";
+ char *lastdomain = NULL, *domain = "", *cur_domain;
u16 flags = 0;
char *err;
union all_addr addr;
union mysockaddr serv_addr, source_addr;
char interface[IF_NAMESIZE+1];
+ struct server_details sdetails;
+ memset(&sdetails, 0, sizeof(struct server_details));
+ sdetails.addr = &serv_addr;
+ sdetails.source_addr = &source_addr;
+ sdetails.interface = interface;
+ sdetails.flags = &flags;
+
unhide_metas(arg);
/* split the domain args, if any and skip to the end of them. */
@@ -2814,36 +3015,55 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
}
else
{
- if ((err = parse_server(arg, &serv_addr, &source_addr, interface, &flags)))
+ if ((err = parse_server(arg, &sdetails)))
ret_err(err);
}
if (servers_only && option == 'S')
flags |= SERV_FROM_FILE;
-
- while (1)
- {
- /* server=//1.2.3.4 is special. */
- if (lastdomain)
- {
- if (strlen(domain) == 0)
- flags |= SERV_FOR_NODOTS;
- else
- flags &= ~SERV_FOR_NODOTS;
- /* address=/#/ matches the same as without domain */
- if (option == 'A' && domain[0] == '#' && domain[1] == 0)
- domain[0] = 0;
+ cur_domain = domain;
+ while ((flags & SERV_LITERAL_ADDRESS) || parse_server_next(&sdetails))
+ {
+ cur_domain = domain;
+
+ if (!(flags & SERV_LITERAL_ADDRESS) && (err = parse_server_addr(&sdetails)))
+ ret_err(err);
+
+ /* When source is set only use DNS records of the same type and skip all others */
+ if (flags & SERV_HAS_SOURCE && sdetails.addr_type != sdetails.source_addr->sa.sa_family)
+ continue;
+
+ while (1)
+ {
+ /* server=//1.2.3.4 is special. */
+ if (lastdomain)
+ {
+ if (strlen(cur_domain) == 0)
+ flags |= SERV_FOR_NODOTS;
+ else
+ flags &= ~SERV_FOR_NODOTS;
+
+ /* address=/#/ matches the same as without domain */
+ if (option == 'A' && cur_domain[0] == '#' && cur_domain[1] == 0)
+ cur_domain[0] = 0;
+ }
+
+ if (!add_update_server(flags, sdetails.addr, sdetails.source_addr, sdetails.interface, cur_domain, &addr))
+ ret_err(gen_err);
+
+ if (!lastdomain || cur_domain == lastdomain)
+ break;
+
+ cur_domain += strlen(cur_domain) + 1;
}
-
- if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, &addr))
- ret_err(gen_err);
-
- if (!lastdomain || domain == lastdomain)
+
+ if (flags & SERV_LITERAL_ADDRESS)
break;
-
- domain += strlen(domain) + 1;
}
+
+ if (sdetails.orig_hostinfo)
+ freeaddrinfo(sdetails.orig_hostinfo);
break;
}
@@ -3177,6 +3397,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
if (daemon->query_port == 0)
daemon->osport = 1;
break;
+
+ case LOPT_RANDPORT_LIM: /* --port-limit */
+ if (!atoi_check(arg, &daemon->randport_limit) || (daemon->randport_limit < 1))
+ ret_err(gen_err);
+ break;
case 'T': /* --local-ttl */
case LOPT_NEGTTL: /* --neg-ttl */
@@ -3212,7 +3437,30 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
daemon->local_ttl = (unsigned long)ttl;
break;
}
+
+ case LOPT_FAST_RETRY:
+ daemon->fast_retry_timeout = TIMEOUT;
+ if (!arg)
+ daemon->fast_retry_time = DEFAULT_FAST_RETRY;
+ else
+ {
+ int retry;
+
+ comma = split(arg);
+ if (!atoi_check(arg, &retry) || retry < 50)
+ ret_err(gen_err);
+ daemon->fast_retry_time = retry;
+
+ if (comma)
+ {
+ if (!atoi_check(comma, &retry))
+ ret_err(gen_err);
+ daemon->fast_retry_timeout = retry/1000;
+ }
+ }
+ break;
+
#ifdef HAVE_DHCP
case 'X': /* --dhcp-lease-max */
if (!atoi_check(arg, &daemon->dhcp_max))
@@ -4331,6 +4579,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
{
if (inet_pton(AF_INET, arg, &new->local))
{
+ char *hash = split_chr(two, '#');
+
+ if (!hash || !atoi_check16(hash, &new->port))
+ new->port = DHCP_SERVER_PORT;
+
if (!inet_pton(AF_INET, two, &new->server))
{
new->server.addr4.s_addr = 0;
@@ -4349,6 +4602,11 @@ static int one_opt(int option, char *arg, char *errstr, char *gen_err, int comma
#ifdef HAVE_DHCP6
else if (inet_pton(AF_INET6, arg, &new->local))
{
+ char *hash = split_chr(two, '#');
+
+ if (!hash || !atoi_check16(hash, &new->port))
+ new->port = DHCPV6_SERVER_PORT;
+
if (!inet_pton(AF_INET6, two, &new->server))
{
inet_pton(AF_INET6, ALL_SERVERS, &new->server.addr6);
@@ -4897,6 +5155,24 @@ err:
break;
}
+ case LOPT_STALE_CACHE:
+ {
+ int max_expiry = STALE_CACHE_EXPIRY;
+ if (arg)
+ {
+ /* Don't accept negative TTLs here, they'd have the counter-intuitive
+ side-effect of evicting cache records before they expire */
+ if (!atoi_check(arg, &max_expiry) || max_expiry < 0)
+ ret_err(gen_err);
+ /* Store "serve expired forever" as -1 internally, the option isn't
+ active for daemon->cache_max_expiry == 0 */
+ if (max_expiry == 0)
+ max_expiry = -1;
+ }
+ daemon->cache_max_expiry = max_expiry;
+ break;
+ }
+
#ifdef HAVE_DNSSEC
case LOPT_DNSSEC_STAMP: /* --dnssec-timestamp */
daemon->timestamp_file = opt_string_alloc(arg);
@@ -4978,26 +5254,20 @@ err:
return 1;
}
-static void read_file(char *file, FILE *f, int hard_opt)
+static void read_file(char *file, FILE *f, int hard_opt, int from_script)
{
volatile int lineno = 0;
char *buff = daemon->namebuff;
while (fgets(buff, MAXDNAME, f))
{
- int white, i, script = 0;
+ int white, i;
volatile int option;
char *errmess, *p, *arg, *start;
size_t len;
- if (hard_opt == LOPT_CONF_SCRIPT)
- {
- hard_opt = 0;
- script = 1;
- }
-
option = (hard_opt == LOPT_REV_SERV) ? 0 : hard_opt;
-
+
/* Memory allocation failure longjmps here if mem_recover == 1 */
if (option != 0 || hard_opt == LOPT_REV_SERV)
{
@@ -5005,7 +5275,7 @@ static void read_file(char *file, FILE *f, int hard_opt)
continue;
mem_recover = 1;
}
-
+
arg = NULL;
lineno++;
errmess = NULL;
@@ -5111,7 +5381,7 @@ static void read_file(char *file, FILE *f, int hard_opt)
if (errmess || !one_opt(option, arg, daemon->namebuff, _("error"), 0, hard_opt == LOPT_REV_SERV))
{
- if (script)
+ if (from_script)
sprintf(daemon->namebuff + strlen(daemon->namebuff), _(" in output from %s"), file);
else
sprintf(daemon->namebuff + strlen(daemon->namebuff), _(" at line %d of %s"), lineno, file);
@@ -5157,8 +5427,14 @@ static int one_file(char *file, int hard_opt)
hard_opt = 0;
nofile_ok = 1;
}
-
- if (hard_opt == 0 && strcmp(file, "-") == 0)
+
+ if (hard_opt == LOPT_CONF_SCRIPT)
+ {
+ hard_opt = 0;
+ do_popen = 1;
+ }
+
+ if (hard_opt == 0 && !do_popen && strcmp(file, "-") == 0)
{
if (read_stdin == 1)
return 1;
@@ -5171,12 +5447,6 @@ static int one_file(char *file, int hard_opt)
/* ignore repeated files. */
struct stat statbuf;
- if (hard_opt == LOPT_CONF_SCRIPT)
- {
- hard_opt = 0;
- do_popen = 1;
- }
-
if (hard_opt == 0 && stat(file, &statbuf) == 0)
{
struct fileread *r;
@@ -5215,7 +5485,7 @@ static int one_file(char *file, int hard_opt)
}
}
- read_file(file, f, do_popen ? LOPT_CONF_SCRIPT : hard_opt);
+ read_file(file, f, hard_opt, do_popen);
if (do_popen)
{
@@ -5369,7 +5639,7 @@ void read_servers_file(void)
}
mark_servers(SERV_FROM_FILE);
- read_file(daemon->servers_file, f, LOPT_REV_SERV);
+ read_file(daemon->servers_file, f, LOPT_REV_SERV, 0);
fclose(f);
cleanup_servers();
check_servers(0);
@@ -5488,6 +5758,8 @@ void read_opts(int argc, char **argv, char *compile_opts)
daemon->soa_refresh = SOA_REFRESH;
daemon->soa_retry = SOA_RETRY;
daemon->soa_expiry = SOA_EXPIRY;
+ daemon->randport_limit = 1;
+ daemon->host_index = SRC_AH;
#ifndef NO_ID
add_txt("version.bind", "dnsmasq-" VERSION, 0 );
@@ -5509,7 +5781,10 @@ void read_opts(int argc, char **argv, char *compile_opts)
/******** Pi-hole modification ********/
add_txt("version.FTL", (char*)get_FTL_version(), 0 );
/**************************************/
-
+
+ /* See comment above make_servers(). Optimises server-read code. */
+ mark_servers(0);
+
while (1)
{
#ifdef HAVE_GETOPT_LONG
diff --git a/src/dnsmasq/poll.c b/src/dnsmasq/poll.c
index 29b33a0c..bbb9009b 100644
--- a/src/dnsmasq/poll.c
+++ b/src/dnsmasq/poll.c
@@ -96,28 +96,21 @@ void poll_listen(int fd, short event)
pollfds[i].events |= event;
else
{
- if (arrsize != nfds)
- memmove(&pollfds[i+1], &pollfds[i], (nfds - i) * sizeof(struct pollfd));
- else
+ if (arrsize == nfds)
{
/* Array too small, extend. */
struct pollfd *new;
arrsize = (arrsize == 0) ? 64 : arrsize * 2;
- if (!(new = whine_malloc(arrsize * sizeof(struct pollfd))))
+ if (!(new = whine_realloc(pollfds, arrsize * sizeof(struct pollfd))))
return;
- if (pollfds)
- {
- memcpy(new, pollfds, i * sizeof(struct pollfd));
- memcpy(&new[i+1], &pollfds[i], (nfds - i) * sizeof(struct pollfd));
- free(pollfds);
- }
-
pollfds = new;
}
-
+
+ memmove(&pollfds[i+1], &pollfds[i], (nfds - i) * sizeof(struct pollfd));
+
pollfds[i].fd = fd;
pollfds[i].events = event;
nfds++;
diff --git a/src/dnsmasq/radv.c b/src/dnsmasq/radv.c
index 2b4326c0..5820f4a6 100644
--- a/src/dnsmasq/radv.c
+++ b/src/dnsmasq/radv.c
@@ -203,7 +203,7 @@ void icmp6_packet(time_t now)
int opt_sz;
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_RA, (void *)packet, sz, (union mysockaddr *)&from, NULL, -1);
+ dump_packet_icmp(DUMP_RA, (void *)packet, sz, (union mysockaddr *)&from, NULL);
#endif
/* look for link-layer address option for logging */
@@ -560,7 +560,13 @@ static void send_ra_alias(time_t now, int iface, char *iface_name, struct in6_ad
}
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_RA, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, (union mysockaddr *)&addr, -1);
+ {
+ struct sockaddr_in6 src;
+ src.sin6_family = AF_INET6;
+ src.sin6_addr = parm.link_local;
+
+ dump_packet_icmp(DUMP_RA, (void *)daemon->outpacket.iov_base, save_counter(-1), (union mysockaddr *)&src, (union mysockaddr *)&addr);
+ }
#endif
while (retry_send(sendto(daemon->icmp6fd, daemon->outpacket.iov_base,
diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c
index 6d7490d3..9b5f2650 100644
--- a/src/dnsmasq/rfc1035.c
+++ b/src/dnsmasq/rfc1035.c
@@ -539,8 +539,10 @@ static int print_txt(struct dns_header *header, const size_t qlen, char *name,
/* Note that the following code can create CNAME chains that don't point to a real record,
either because of lack of memory, or lack of SOA records. These are treated by the cache code as
expired and cleaned out that way.
- Return 1 if we reject an address because it look like part of dns-rebinding attack. */
-// Pi-hole: Return 2 if we reject a part of a CNAME chain
+ Return 1 if we reject an address because it look like part of dns-rebinding attack.
+ Return 2 if the packet is malformed.
+ Return 99 if we reject parts of a CNAME chain (*** Pi-hole modification ***)
+*/
int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t now,
struct ipsets *ipsets, struct ipsets *nftsets, int is_sign, int check_rebind,
int no_cache_dnssec, int secure, int *doctored)
@@ -591,7 +593,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
namep = p = (unsigned char *)(header+1);
if (ntohs(header->qdcount) != 1 || !extract_name(header, qlen, &p, name, 1, 4))
- return 0; /* bad packet */
+ return 2; /* bad packet */
GETSHORT(qtype, p);
GETSHORT(qclass, p);
@@ -609,13 +611,13 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
{
cname_loop:
if (!(p1 = skip_questions(header, qlen)))
- return 0;
+ return 2;
for (j = 0; j < ntohs(header->ancount); j++)
{
int secflag = 0;
if (!(res = extract_name(header, qlen, &p1, name, 0, 10)))
- return 0; /* bad packet */
+ return 2; /* bad packet */
GETSHORT(aqtype, p1);
GETSHORT(aqclass, p1);
@@ -652,7 +654,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
log_query(secflag | F_CNAME | F_FORWARD | F_UPSTREAM, name, NULL, NULL, 0);
if (!extract_name(header, qlen, &p1, name, 1, 0))
- return 0;
+ return 2;
if (aqtype == T_CNAME)
{
@@ -678,7 +680,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
p1 = endrr;
if (!CHECK_LEN(header, p1, qlen, 0))
- return 0; /* bad packet */
+ return 2; /* bad packet */
}
}
@@ -723,14 +725,14 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
cname_loop1:
if (!(p1 = skip_questions(header, qlen)))
- return 0;
+ return 2;
for (j = 0; j < ntohs(header->ancount); j++)
{
int secflag = 0;
if (!(res = extract_name(header, qlen, &p1, name, 0, 10)))
- return 0; /* bad packet */
+ return 2; /* bad packet */
GETSHORT(aqtype, p1);
GETSHORT(aqclass, p1);
@@ -748,7 +750,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
{
p1 = endrr;
if (!CHECK_LEN(header, p1, qlen, 0))
- return 0; /* bad packet */
+ return 2; /* bad packet */
continue;
}
@@ -763,16 +765,6 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
}
#endif
- // ****************************** Pi-hole modification ******************************
- if(FTL_CNAME(name, cpp, daemon->log_display_id))
- {
- // Found while processing a reply from upstream. We prevent cache insertion here
- // This query is to be blocked as we found a blocked
- // domain while walking the CNAME path. Log to pihole.log here
- log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0);
- return 2;
- }
- // **********************************************************************************
if (aqtype == T_CNAME)
{
@@ -802,8 +794,19 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
namep = p1;
if (!extract_name(header, qlen, &p1, name, 1, 0))
- return 0;
+ return 2;
+ // ****************************** Pi-hole modification ******************************
+ const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL;
+ if(FTL_CNAME(name, src, daemon->log_display_id))
+ {
+ // Found while processing a reply from upstream. We prevent cache insertion here
+ // This query is to be blocked as we found a blocked
+ // domain while walking the CNAME path. Log to pihole.log here
+ log_query(F_UPSTREAM, name, NULL, "blocked during CNAME inspection", 0);
+ return 99;
+ }
+ // **********************************************************************************
if (qtype != T_CNAME)
goto cname_loop1;
@@ -825,25 +828,25 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
unsigned char *tmp = namep;
if (!CHECK_LEN(header, p1, qlen, 6))
- return 0; /* bad packet */
+ return 2; /* bad packet */
GETSHORT(addr.srv.priority, p1);
GETSHORT(addr.srv.weight, p1);
GETSHORT(addr.srv.srvport, p1);
if (!extract_name(header, qlen, &p1, name, 1, 0))
- return 0;
+ return 2;
addr.srv.targetlen = strlen(name) + 1; /* include terminating zero */
if (!(addr.srv.target = blockdata_alloc(name, addr.srv.targetlen)))
return 0;
/* we overwrote the original name, so get it back here. */
if (!extract_name(header, qlen, &tmp, name, 1, 0))
- return 0;
+ return 2;
}
else if (flags & (F_IPV4 | F_IPV6))
{
/* copy address into aligned storage */
if (!CHECK_LEN(header, p1, qlen, addrlen))
- return 0; /* bad packet */
+ return 2; /* bad packet */
memcpy(&addr, p1, addrlen);
/* check for returned address in private space */
@@ -887,7 +890,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
if (aqtype == T_TXT)
{
if (!print_txt(header, qlen, name, p1, ardlen, secflag))
- return 0;
+ return 2;
}
else
log_query(flags | F_FORWARD | secflag | F_UPSTREAM, name, &addr, NULL, aqtype);
@@ -895,7 +898,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t
p1 = endrr;
if (!CHECK_LEN(header, p1, qlen, 0))
- return 0; /* bad packet */
+ return 2; /* bad packet */
}
if (!found && (qtype != T_ANY || (flags & F_NXDOMAIN)))
@@ -1372,8 +1375,15 @@ int add_resource_record(struct dns_header *header, char *limit, int *truncp, int
#undef CHECK_LIMIT
}
+static int crec_isstale(struct crec *crecp, time_t now)
+{
+ return (!(crecp->flags & F_IMMORTAL)) && difftime(crecp->ttd, now) < 0;
+}
+
static unsigned long crec_ttl(struct crec *crecp, time_t now)
{
+ signed long ttl = difftime(crecp->ttd, now);
+
/* Return 0 ttl for DHCP entries, which might change
before the lease expires, unless configured otherwise. */
@@ -1382,8 +1392,8 @@ static unsigned long crec_ttl(struct crec *crecp, time_t now)
int conf_ttl = daemon->use_dhcp_ttl ? daemon->dhcp_ttl : daemon->local_ttl;
/* Apply ceiling of actual lease length to configured TTL. */
- if (!(crecp->flags & F_IMMORTAL) && (crecp->ttd - now) < conf_ttl)
- return crecp->ttd - now;
+ if (!(crecp->flags & F_IMMORTAL) && ttl < conf_ttl)
+ return ttl;
return conf_ttl;
}
@@ -1392,9 +1402,13 @@ static unsigned long crec_ttl(struct crec *crecp, time_t now)
if (crecp->flags & F_IMMORTAL)
return crecp->ttd;
+ /* Stale cache entries. */
+ if (ttl < 0)
+ return 0;
+
/* Return the Max TTL value if it is lower than the actual TTL */
- if (daemon->max_ttl == 0 || ((unsigned)(crecp->ttd - now) < daemon->max_ttl))
- return crecp->ttd - now;
+ if (daemon->max_ttl == 0 || ((unsigned)ttl < daemon->max_ttl))
+ return ttl;
else
return daemon->max_ttl;
}
@@ -1407,7 +1421,8 @@ static int cache_validated(const struct crec *crecp)
/* return zero if we can't answer from cache, or packet size if we can */
size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
struct in_addr local_addr, struct in_addr local_netmask,
- time_t now, int ad_reqd, int do_bit, int have_pseudoheader)
+ time_t now, int ad_reqd, int do_bit, int have_pseudoheader,
+ int *stale)
{
char *name = daemon->namebuff;
unsigned char *p, *ansp;
@@ -1423,6 +1438,9 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
size_t len;
int rd_bit = (header->hb3 & HB3_RD);
+ if (stale)
+ *stale = 0;
+
/* never answer queries with RD unset, to avoid cache snooping. */
if (ntohs(header->ancount) != 0 ||
ntohs(header->nscount) != 0 ||
@@ -1471,13 +1489,22 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
while (--count != 0 && (crecp = cache_find_by_name(NULL, name, now, F_CNAME | F_NXDOMAIN)))
{
char *cname_target;
-
+ int stale_flag = 0;
+
+ if (crec_isstale(crecp, now))
+ {
+ if (stale)
+ *stale = 1;
+
+ stale_flag = F_STALE;
+ }
+
if (crecp->flags & F_NXDOMAIN)
{
if (qtype == T_CNAME)
{
if (!dryrun)
- log_query(crecp->flags, name, NULL, record_source(crecp->uid), 0);
+ log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0);
auth = 0;
nxdomain = 1;
ans = 1;
@@ -1499,7 +1526,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
if (!dryrun)
{
- log_query(crecp->flags, name, NULL, record_source(crecp->uid), 0);
+ log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0);
if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
crec_ttl(crecp, now), &nameoffset,
T_CNAME, C_IN, "d", cname_target))
@@ -1609,7 +1636,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
if (addrlist)
break;
- else
+ else if (!(intr->flags & INP4))
while (intr->next && strcmp(intr->intr, intr->next->intr) == 0)
intr = intr->next;
}
@@ -1624,7 +1651,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
if (addrlist)
break;
- else
+ else if (!(intr->flags & INP6))
while (intr->next && strcmp(intr->intr, intr->next->intr) == 0)
intr = intr->next;
}
@@ -1668,22 +1695,33 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
{
do
{
+ int stale_flag = 0;
+
+ if (crec_isstale(crecp, now))
+ {
+ if (stale)
+ *stale = 1;
+
+ stale_flag = F_STALE;
+ }
+
/* don't answer wildcard queries with data not from /etc/hosts or dhcp leases */
if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP)))
continue;
+
if (!(crecp->flags & F_DNSSECOK))
sec_data = 0;
-
+
ans = 1;
-
+
if (crecp->flags & F_NEG)
{
auth = 0;
if (crecp->flags & F_NXDOMAIN)
nxdomain = 1;
if (!dryrun)
- log_query(crecp->flags & ~F_FORWARD, name, &addr, NULL, 0);
+ log_query(stale_flag | (crecp->flags & ~F_FORWARD), name, &addr, NULL, 0);
}
else
{
@@ -1691,9 +1729,9 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
auth = 0;
if (!dryrun)
{
- log_query(crecp->flags & ~F_FORWARD, cache_get_name(crecp), &addr,
+ log_query(stale_flag | (crecp->flags & ~F_FORWARD), cache_get_name(crecp), &addr,
record_source(crecp->uid), 0);
-
+
if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
crec_ttl(crecp, now), NULL,
T_PTR, C_IN, "d", cache_get_name(crecp)))
@@ -1800,7 +1838,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
if ((crecp = cache_find_by_name(NULL, name, now, flag | F_NXDOMAIN | (dryrun ? F_NO_RR : 0))))
{
int localise = 0;
-
+
/* See if a putative address is on the network from which we received
the query, is so we'll filter other answers. */
if (local_addr.s_addr != 0 && option_bool(OPT_LOCALISE) && flag == F_IPV4)
@@ -1822,6 +1860,16 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
(rd_bit && (!do_bit || cache_validated(crecp)) ))
do
{
+ int stale_flag = 0;
+
+ if (crec_isstale(crecp, now))
+ {
+ if (stale)
+ *stale = 1;
+
+ stale_flag = F_STALE;
+ }
+
/* don't answer wildcard queries with data not from /etc/hosts
or DHCP leases */
if (qtype == T_ANY && !(crecp->flags & (F_HOSTS | F_DHCP | F_CONFIG)))
@@ -1839,7 +1887,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
if (!dryrun)
// Pi-hole modification: Added record_source(crecp->uid) such that the subroutines know
// where the reply came from (e.g. gravity.list)
- log_query(crecp->flags, name, NULL, record_source(crecp->uid), 0);
+ log_query(stale_flag | crecp->flags, name, NULL, record_source(crecp->uid), 0);
}
else
{
@@ -1856,10 +1904,11 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
ans = 1;
if (!dryrun)
{
- log_query(crecp->flags & ~F_REVERSE, name, &crecp->addr,
+ log_query(stale_flag | (crecp->flags & ~F_REVERSE), name, &crecp->addr,
record_source(crecp->uid), 0);
// ****************************** Pi-hole modification ******************************
- if(FTL_CNAME(name, crecp, daemon->log_display_id))
+ const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL;
+ if(FTL_CNAME(name, src, daemon->log_display_id))
{
// Served from cache. This can happen if a domain hidden in the CNAME path
// is only blocked for some but not all clients. In this case, the entire
@@ -1979,6 +2028,15 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
rd_bit && (!do_bit || (option_bool(OPT_DNSSEC_VALID) && !(crecp->flags & F_DNSSECOK))))
do
{
+ int stale_flag = 0;
+
+ if (crec_isstale(crecp, now))
+ {
+ if (stale)
+ *stale = 1;
+
+ stale_flag = F_STALE;
+ }
/* don't answer wildcard queries with data not from /etc/hosts or dhcp leases, except for NXDOMAIN */
if (qtype == T_ANY && !(crecp->flags & (F_NXDOMAIN)))
break;
@@ -1994,12 +2052,12 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen,
if (crecp->flags & F_NXDOMAIN)
nxdomain = 1;
if (!dryrun)
- log_query(crecp->flags, name, NULL, NULL, 0);
+ log_query(stale_flag | crecp->flags, name, NULL, NULL, 0);
}
else if (!dryrun)
{
char *target = blockdata_retrieve(crecp->addr.srv.target, crecp->addr.srv.targetlen, NULL);
- log_query(crecp->flags, name, NULL, NULL, 0);
+ log_query(stale_flag | crecp->flags, name, NULL, NULL, 0);
if (add_resource_record(header, limit, &trunc, nameoffset, &ansp,
crec_ttl(crecp, now), NULL, T_SRV, C_IN, "sssd",
diff --git a/src/dnsmasq/rfc2131.c b/src/dnsmasq/rfc2131.c
index ecda2d3e..17e97b52 100644
--- a/src/dnsmasq/rfc2131.c
+++ b/src/dnsmasq/rfc2131.c
@@ -1153,15 +1153,22 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index,
tagif_netid = run_tag_if(&context->netid);
}
- log_tags(tagif_netid, ntohl(mess->xid));
apply_delay(mess->xid, recvtime, tagif_netid);
if (option_bool(OPT_RAPID_COMMIT) && option_find(mess, sz, OPTION_RAPID_COMMIT, 0))
{
rapid_commit = 1;
+ /* If a lease exists for this host and another address, squash it. */
+ if (lease && lease->addr.s_addr != mess->yiaddr.s_addr)
+ {
+ lease_prune(lease, now);
+ lease = NULL;
+ }
goto rapid_commit;
}
+ log_tags(tagif_netid, ntohl(mess->xid));
+
daemon->metrics[METRIC_DHCPOFFER]++;
log_packet("DHCPOFFER" , &mess->yiaddr, emac, emac_len, iface_name, NULL, NULL, mess->xid);
@@ -1420,21 +1427,18 @@ size_t dhcp_reply(struct dhcp_context *context, char *iface_name, int int_index,
/* DNSMASQ_REQUESTED_OPTIONS */
if ((opt = option_find(mess, sz, OPTION_REQUESTED_OPTIONS, 1)))
{
- int len = option_len(opt);
+ int i, len = option_len(opt);
unsigned char *rop = option_ptr(opt, 0);
- char *q = daemon->namebuff;
- int i;
+
for (i = 0; i < len; i++)
- {
- q += snprintf(q, MAXDNAME - (q - daemon->namebuff), "%d%s", rop[i], i + 1 == len ? "" : ",");
- }
- lease_add_extradata(lease, (unsigned char *)daemon->namebuff, (q - daemon->namebuff), 0);
+ lease_add_extradata(lease, (unsigned char *)daemon->namebuff,
+ sprintf(daemon->namebuff, "%u", rop[i]), (i + 1) == len ? 0 : ',');
}
else
- {
- add_extradata_opt(lease, NULL);
- }
-
+ lease_add_extradata(lease, NULL, 0, 0);
+
+ add_extradata_opt(lease, option_find(mess, sz, OPTION_MUD_URL_V4, 1));
+
/* space-concat tag set */
if (!tagif_netid)
add_extradata_opt(lease, NULL);
diff --git a/src/dnsmasq/rfc3315.c b/src/dnsmasq/rfc3315.c
index cee8382c..87544816 100644
--- a/src/dnsmasq/rfc3315.c
+++ b/src/dnsmasq/rfc3315.c
@@ -33,9 +33,9 @@ struct state {
unsigned int mac_len, mac_type;
};
-static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz,
+static int dhcp6_maybe_relay(struct state *state, unsigned char *inbuff, size_t sz,
struct in6_addr *client_addr, int is_unicast, time_t now);
-static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_t sz, int is_unicast, time_t now);
+static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbuff, size_t sz, int is_unicast, time_t now);
static void log6_opts(int nest, unsigned int xid, void *start_opts, void *end_opts);
static void log6_packet(struct state *state, char *type, struct in6_addr *addr, char *string);
static void log6_quiet(struct state *state, char *type, struct in6_addr *addr, char *string);
@@ -104,12 +104,12 @@ unsigned short dhcp6_reply(struct dhcp_context *context, int interface, char *if
}
/* This cost me blood to write, it will probably cost you blood to understand - srk. */
-static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz,
+static int dhcp6_maybe_relay(struct state *state, unsigned char *inbuff, size_t sz,
struct in6_addr *client_addr, int is_unicast, time_t now)
{
void *end = inbuff + sz;
void *opts = inbuff + 34;
- int msg_type = *((unsigned char *)inbuff);
+ int msg_type = *inbuff;
unsigned char *outmsgtypep;
void *opt;
struct dhcp_vendor *vendor;
@@ -259,15 +259,15 @@ static int dhcp6_maybe_relay(struct state *state, void *inbuff, size_t sz,
return 1;
}
-static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_t sz, int is_unicast, time_t now)
+static int dhcp6_no_relay(struct state *state, int msg_type, unsigned char *inbuff, size_t sz, int is_unicast, time_t now)
{
void *opt;
- int i, o, o1, start_opts;
+ int i, o, o1, start_opts, start_msg;
struct dhcp_opt *opt_cfg;
struct dhcp_netid *tagif;
struct dhcp_config *config = NULL;
struct dhcp_netid known_id, iface_id, v6_id;
- unsigned char *outmsgtypep;
+ unsigned char outmsgtype;
struct dhcp_vendor *vendor;
struct dhcp_context *context_tmp;
struct dhcp_mac *mac_opt;
@@ -296,12 +296,13 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
v6_id.next = state->tags;
state->tags = &v6_id;
- /* copy over transaction-id, and save pointer to message type */
- if (!(outmsgtypep = put_opt6(inbuff, 4)))
+ start_msg = save_counter(-1);
+ /* copy over transaction-id */
+ if (!put_opt6(inbuff, 4))
return 0;
start_opts = save_counter(-1);
- state->xid = outmsgtypep[3] | outmsgtypep[2] << 8 | outmsgtypep[1] << 16;
-
+ state->xid = inbuff[3] | inbuff[2] << 8 | inbuff[1] << 16;
+
/* We're going to be linking tags from all context we use.
mark them as unused so we don't link one twice and break the list */
for (context_tmp = state->context; context_tmp; context_tmp = context_tmp->current)
@@ -347,7 +348,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
(msg_type == DHCP6REQUEST || msg_type == DHCP6RENEW || msg_type == DHCP6RELEASE || msg_type == DHCP6DECLINE))
{
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
o1 = new_opt6(OPTION6_STATUS_CODE);
put_opt6_short(DHCP6USEMULTI);
put_opt6_string("Use multicast");
@@ -619,11 +620,11 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
struct dhcp_netid *solicit_tags;
struct dhcp_context *c;
- *outmsgtypep = DHCP6ADVERTISE;
+ outmsgtype = DHCP6ADVERTISE;
if (opt6_find(state->packet_options, state->end, OPTION6_RAPID_COMMIT, 0))
{
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
state->lease_allocate = 1;
o = new_opt6(OPTION6_RAPID_COMMIT);
end_opt6(o);
@@ -809,7 +810,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
int start = save_counter(-1);
/* set reply message type */
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
state->lease_allocate = 1;
log6_quiet(state, "DHCPREQUEST", NULL, ignore ? _("ignored") : NULL);
@@ -924,7 +925,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
int address_assigned = 0;
/* set reply message type */
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
log6_quiet(state, msg_type == DHCP6RENEW ? "DHCPRENEW" : "DHCPREBIND", NULL, NULL);
@@ -1057,7 +1058,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
int good_addr = 0;
/* set reply message type */
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
log6_quiet(state, "DHCPCONFIRM", NULL, NULL);
@@ -1121,7 +1122,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
log6_quiet(state, "DHCPINFORMATION-REQUEST", NULL, ignore ? _("ignored") : state->hostname);
if (ignore)
return 0;
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
tagif = add_options(state, 1);
break;
}
@@ -1130,7 +1131,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
case DHCP6RELEASE:
{
/* set reply message type */
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
log6_quiet(state, "DHCPRELEASE", NULL, NULL);
@@ -1195,7 +1196,7 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
case DHCP6DECLINE:
{
/* set reply message type */
- *outmsgtypep = DHCP6REPLY;
+ outmsgtype = DHCP6REPLY;
log6_quiet(state, "DHCPDECLINE", NULL, NULL);
@@ -1275,7 +1276,12 @@ static int dhcp6_no_relay(struct state *state, int msg_type, void *inbuff, size_
}
}
-
+
+ /* Fill in the message type. Note that we store the offset,
+ not a direct pointer, since the packet memory may have been
+ reallocated. */
+ ((unsigned char *)(daemon->outpacket.iov_base))[start_msg] = outmsgtype;
+
log_tags(tagif, state->xid);
log6_opts(0, state->xid, daemon->outpacket.iov_base + start_opts, daemon->outpacket.iov_base + save_counter(-1));
@@ -1873,23 +1879,24 @@ static void update_leases(struct state *state, struct dhcp_context *context, str
#ifdef HAVE_SCRIPT
if (daemon->lease_change_command)
{
- void *class_opt;
+ void *opt;
+
lease->flags |= LEASE_CHANGED;
free(lease->extradata);
lease->extradata = NULL;
lease->extradata_size = lease->extradata_len = 0;
lease->vendorclass_count = 0;
- if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_CLASS, 4)))
+ if ((opt = opt6_find(state->packet_options, state->end, OPTION6_VENDOR_CLASS, 4)))
{
- void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt));
+ void *enc_opt, *enc_end = opt6_ptr(opt, opt6_len(opt));
lease->vendorclass_count++;
/* send enterprise number first */
- sprintf(daemon->dhcp_buff2, "%u", opt6_uint(class_opt, 0, 4));
+ sprintf(daemon->dhcp_buff2, "%u", opt6_uint(opt, 0, 4));
lease_add_extradata(lease, (unsigned char *)daemon->dhcp_buff2, strlen(daemon->dhcp_buff2), 0);
- if (opt6_len(class_opt) >= 6)
- for (enc_opt = opt6_ptr(class_opt, 4); enc_opt; enc_opt = opt6_next(enc_opt, enc_end))
+ if (opt6_len(opt) >= 6)
+ for (enc_opt = opt6_ptr(opt, 4); enc_opt; enc_opt = opt6_next(enc_opt, enc_end))
{
lease->vendorclass_count++;
lease_add_extradata(lease, opt6_ptr(enc_opt, 0), opt6_len(enc_opt), 0);
@@ -1899,6 +1906,24 @@ static void update_leases(struct state *state, struct dhcp_context *context, str
lease_add_extradata(lease, (unsigned char *)state->client_hostname,
state->client_hostname ? strlen(state->client_hostname) : 0, 0);
+ /* DNSMASQ_REQUESTED_OPTIONS */
+ if ((opt = opt6_find(state->packet_options, state->end, OPTION6_ORO, 2)))
+ {
+ int i, len = opt6_len(opt)/2;
+ u16 *rop = opt6_ptr(opt, 0);
+
+ for (i = 0; i < len; i++)
+ lease_add_extradata(lease, (unsigned char *)daemon->namebuff,
+ sprintf(daemon->namebuff, "%u", ntohs(rop[i])), (i + 1) == len ? 0 : ',');
+ }
+ else
+ lease_add_extradata(lease, NULL, 0, 0);
+
+ if ((opt = opt6_find(state->packet_options, state->end, OPTION6_MUD_URL, 1)))
+ lease_add_extradata(lease, opt6_ptr(opt, 0), opt6_len(opt), 0);
+ else
+ lease_add_extradata(lease, NULL, 0, 0);
+
/* space-concat tag set */
if (!tagif && !context->netid.net)
lease_add_extradata(lease, NULL, 0, 0);
@@ -1928,10 +1953,10 @@ static void update_leases(struct state *state, struct dhcp_context *context, str
lease_add_extradata(lease, (unsigned char *)daemon->addrbuff, state->link_address ? strlen(daemon->addrbuff) : 0, 0);
- if ((class_opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2)))
+ if ((opt = opt6_find(state->packet_options, state->end, OPTION6_USER_CLASS, 2)))
{
- void *enc_opt, *enc_end = opt6_ptr(class_opt, opt6_len(class_opt));
- for (enc_opt = opt6_ptr(class_opt, 0); enc_opt; enc_opt = opt6_next(enc_opt, enc_end))
+ void *enc_opt, *enc_end = opt6_ptr(opt, opt6_len(opt));
+ for (enc_opt = opt6_ptr(opt, 0); enc_opt; enc_opt = opt6_next(enc_opt, enc_end))
lease_add_extradata(lease, opt6_ptr(enc_opt, 0), opt6_len(enc_opt), 0);
}
}
@@ -2156,15 +2181,12 @@ int relay_upstream6(int iface_index, ssize_t sz,
if (relay->iface_index != 0 && relay->iface_index == iface_index)
{
union mysockaddr to;
- union all_addr from;
-
- /* source address == relay address */
- from.addr6 = relay->local.addr6;
+
memcpy(&header[2], &relay->local.addr6, IN6ADDRSZ);
to.sa.sa_family = AF_INET6;
to.in6.sin6_addr = relay->server.addr6;
- to.in6.sin6_port = htons(DHCPV6_SERVER_PORT);
+ to.in6.sin6_port = htons(relay->port);
to.in6.sin6_flowinfo = 0;
to.in6.sin6_scope_id = 0;
@@ -2181,18 +2203,11 @@ int relay_upstream6(int iface_index, ssize_t sz,
}
#ifdef HAVE_DUMPFILE
- {
- union mysockaddr fromsock;
- fromsock.in6.sin6_port = htons(DHCPV6_SERVER_PORT);
- fromsock.in6.sin6_addr = from.addr6;
- fromsock.sa.sa_family = AF_INET6;
- fromsock.in6.sin6_flowinfo = 0;
- fromsock.in6.sin6_scope_id = 0;
-
- dump_packet(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), &fromsock, &to, 0);
- }
+ dump_packet_udp(DUMP_DHCPV6, (void *)daemon->outpacket.iov_base, save_counter(-1), NULL, &to, daemon->dhcp6fd);
#endif
- send_from(daemon->dhcp6fd, 0, daemon->outpacket.iov_base, save_counter(-1), &to, &from, 0);
+
+ while (retry_send(sendto(daemon->dhcp6fd, (void *)daemon->outpacket.iov_base, save_counter(-1),
+ 0, (struct sockaddr *)&to, sa_len(&to))));
if (option_bool(OPT_LOG_OPTS))
{
diff --git a/src/dnsmasq/rrfilter.c b/src/dnsmasq/rrfilter.c
index f02f5a5e..42d9c210 100644
--- a/src/dnsmasq/rrfilter.c
+++ b/src/dnsmasq/rrfilter.c
@@ -159,7 +159,7 @@ static int check_rrs(unsigned char *p, struct dns_header *header, size_t plen, i
/* mode may be remove EDNS0 or DNSSEC RRs or remove A or AAAA from answer section. */
size_t rrfilter(struct dns_header *header, size_t plen, int mode)
{
- static unsigned char **rrs;
+ static unsigned char **rrs = NULL;
static int rr_sz = 0;
unsigned char *p = (unsigned char *)(header+1);
@@ -339,15 +339,11 @@ int expand_workspace(unsigned char ***wkspc, int *szp, int new)
return 0;
new += 5;
-
- if (!(p = whine_malloc(new * sizeof(unsigned char *))))
- return 0;
-
- if (old != 0 && *wkspc)
- {
- memcpy(p, *wkspc, old * sizeof(unsigned char *));
- free(*wkspc);
- }
+
+ if (!(p = whine_realloc(*wkspc, new * sizeof(unsigned char *))))
+ return 0;
+
+ memset(p+old, 0, new-old);
*wkspc = p;
*szp = new;
diff --git a/src/dnsmasq/tftp.c b/src/dnsmasq/tftp.c
index bdf37a38..0861f37c 100644
--- a/src/dnsmasq/tftp.c
+++ b/src/dnsmasq/tftp.c
@@ -97,7 +97,7 @@ void tftp_request(struct listener *listen, time_t now)
return;
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_TFTP, (void *)packet, len, (union mysockaddr *)&peer, NULL, TFTP_PORT);
+ dump_packet_udp(DUMP_TFTP, (void *)packet, len, (union mysockaddr *)&peer, NULL, listen->tftpfd);
#endif
/* Can always get recvd interface for IPv6 */
@@ -488,7 +488,7 @@ void tftp_request(struct listener *listen, time_t now)
send_from(transfer->sockfd, !option_bool(OPT_SINGLE_PORT), packet, len, &peer, &addra, if_index);
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_TFTP, (void *)packet, len, NULL, (union mysockaddr *)&peer, TFTP_PORT);
+ dump_packet_udp(DUMP_TFTP, (void *)packet, len, NULL, (union mysockaddr *)&peer, transfer->sockfd);
#endif
if (is_err)
@@ -610,7 +610,7 @@ void check_tftp_listeners(time_t now)
while(retry_send(sendto(transfer->sockfd, daemon->packet, len, 0, &peer.sa, sa_len(&peer))));
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&peer, TFTP_PORT);
+ dump_packet_udp(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&peer, transfer->sockfd);
#endif
}
}
@@ -650,7 +650,7 @@ void check_tftp_listeners(time_t now)
send_from(transfer->sockfd, !option_bool(OPT_SINGLE_PORT), daemon->packet, len,
&transfer->peer, &transfer->source, transfer->if_index);
#ifdef HAVE_DUMPFILE
- dump_packet(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&transfer->peer, TFTP_PORT);
+ dump_packet_udp(DUMP_TFTP, (void *)daemon->packet, len, NULL, (union mysockaddr *)&transfer->peer, transfer->sockfd);
#endif
}
diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c
index ae514d29..e0ce67d3 100644
--- a/src/dnsmasq/util.c
+++ b/src/dnsmasq/util.c
@@ -336,6 +336,16 @@ void *whine_malloc(size_t size)
return ret;
}
+void *whine_realloc(void *ptr, size_t size)
+{
+ void *ret = realloc(ptr, size);
+
+ if (!ret)
+ my_syslog(LOG_ERR, _("failed to reallocate %d bytes"), (int) size);
+
+ return ret;
+}
+
int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2)
{
if (s1->sa.sa_family == s2->sa.sa_family)
@@ -354,6 +364,19 @@ int sockaddr_isequal(const union mysockaddr *s1, const union mysockaddr *s2)
return 0;
}
+int sockaddr_isnull(const union mysockaddr *s)
+{
+ if (s->sa.sa_family == AF_INET &&
+ s->in.sin_addr.s_addr == 0)
+ return 1;
+
+ if (s->sa.sa_family == AF_INET6 &&
+ IN6_IS_ADDR_UNSPECIFIED(&s->in6.sin6_addr))
+ return 1;
+
+ return 0;
+}
+
int sa_len(union mysockaddr *addr)
{
#ifdef HAVE_SOCKADDR_SA_LEN
@@ -447,6 +470,15 @@ time_t dnsmasq_time(void)
#endif
}
+u32 dnsmasq_milliseconds(void)
+{
+ struct timeval tv;
+
+ gettimeofday(&tv, NULL);
+
+ return (tv.tv_sec) * 1000 + (tv.tv_usec / 1000);
+}
+
int netmask_length(struct in_addr mask)
{
int zero_count = 0;
diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c
index 28da14c3..289bb130 100644
--- a/src/dnsmasq_interface.c
+++ b/src/dnsmasq_interface.c
@@ -77,6 +77,7 @@ static void check_pihole_PTR(char *domain);
#define query_set_dnssec(query, dnssec) _query_set_dnssec(query, dnssec, __FILE__, __LINE__)
static void _query_set_dnssec(queriesData *query, const enum dnssec_status dnssec, const char *file, const int line);
static char *get_ptrname(struct in_addr *addr);
+static const char *check_dnsmasq_name(const char *name);
// Static blocking metadata
static const char *blockingreason = "";
@@ -98,22 +99,17 @@ static struct {
static union mysockaddr last_server = {{ 0 }};
unsigned char* pihole_privacylevel = &config.privacylevel;
-const char *flagnames[] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA ", "F_SERVFAIL", "F_RCODE"};
+const char *flagnames[] = {"F_IMMORTAL ", "F_NAMEP ", "F_REVERSE ", "F_FORWARD ", "F_DHCP ", "F_NEG ", "F_HOSTS ", "F_IPV4 ", "F_IPV6 ", "F_BIGNAME ", "F_NXDOMAIN ", "F_CNAME ", "F_DNSKEY ", "F_CONFIG ", "F_DS ", "F_DNSSECOK ", "F_UPSTREAM ", "F_RRNAME ", "F_SERVER ", "F_QUERY ", "F_NOERR ", "F_AUTH ", "F_DNSSEC ", "F_KEYTAG ", "F_SECSTAT ", "F_NO_RR ", "F_IPSET ", "F_NOEXTRA ", "F_SERVFAIL", "F_RCODE", "F_SRV", "F_STALE" };
-void FTL_hook(unsigned int flags, char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line)
+void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line)
{
// Extract filename from path
const char *path = short_path(file);
- log_debug(DEBUG_FLAGS, "Processing FTL hook from %s:%d...", path, line);
+ log_debug(DEBUG_FLAGS, "Processing FTL hook from %s:%d (name: \"%s\")...", path, line, name);
print_flags(flags);
- // Special domain name handling
- // 1. Substitute "(NULL)" if no name is available (should not happen)
- // 2. Substitute "." if we are querying the root domain (e.g. DNSKEY)
- if(!name)
- name = (char*)"(NULL)";
- else if(!name[0])
- name = (char*)".";
+ // Check domain name received from dnsmasq
+ name = check_dnsmasq_name(name);
// Note: The order matters here!
if((flags & F_QUERY) && (flags & F_FORWARD))
@@ -139,10 +135,29 @@ void FTL_hook(unsigned int flags, char *name, union all_addr *addr, char *arg, i
// derive the real query type from the arg string
unsigned short qtype = type;
if(strcmp(arg, "dnssec-query[DNSKEY]") == 0)
+ {
qtype = T_DNSKEY;
+ arg = (char*)"dnssec-query";
+ }
else if(strcmp(arg, "dnssec-query[DS]") == 0)
+ {
qtype = T_DS;
- arg = (char*)"dnssec-query";
+ arg = (char*)"dnssec-query";
+ }
+ else if(strcmp(arg, "dnssec-retry[DNSKEY]") == 0)
+ {
+ qtype = T_DNSKEY;
+ arg = (char*)"dnssec-retry";
+ }
+ else if(strcmp(arg, "dnssec-retry[DS]") == 0)
+ {
+ qtype = T_DS;
+ arg = (char*)"dnssec-retry";
+ }
+ else
+ {
+ arg = (char*)"dnssec-unknown";
+ }
_FTL_new_query(flags, name, NULL, arg, qtype, id, &edns, INTERNAL, file, line);
// forwarded upstream (type is used to store the upstream port)
@@ -159,6 +174,7 @@ void FTL_hook(unsigned int flags, char *name, union all_addr *addr, char *arg, i
// This is inspired by make_local_answer()
size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char *file, const int line)
{
+ log_debug(DEBUG_FLAGS, "FTL_make_answer() called from %s:%d", short_path(file), line);
// Exit early if there are no questions in this query
if(ntohs(header->qdcount) == 0)
return 0;
@@ -501,6 +517,9 @@ bool _FTL_new_query(const unsigned int flags, const char *name,
break;
}
+ // Check domain name received from dnsmasq
+ name = check_dnsmasq_name(name);
+
// If domain is "pi.hole" or the local hostname we skip analyzing this query
// and, instead, immediately reply with the IP address - these queries are not further analyzed
if(is_pihole_domain(name))
@@ -1074,7 +1093,7 @@ static bool check_domain_blocked(const char *domain, const int clientID,
return false;
// Check domains against exact blacklist
- enum db_result blacklist = in_denylist(domain, client);
+ enum db_result blacklist = in_denylist(domain, dns_cache, client);
if(blacklist == FOUND)
{
// Set new status
@@ -1140,10 +1159,9 @@ static bool check_domain_blocked(const char *domain, const int clientID,
return true;
}
- // Check domain against deny regex filters
- // Skipped when the domain is whitelisted or blocked by exact denylist or gravity
- int regex_idx = 0;
- if((regex_idx = match_regex(domain, dns_cache, client->id, REGEX_DENY, false)) > -1)
+ // Check domain against blacklist regex filters
+ // Skipped when the domain is whitelisted or blocked by exact blacklist or gravity
+ if(in_regex(domain, dns_cache, client-> id, REGEX_DENY))
{
// Set new status
*new_status = QUERY_REGEX;
@@ -1151,14 +1169,13 @@ static bool check_domain_blocked(const char *domain, const int clientID,
// Mark domain as regex matched for this client
set_dnscache_blockingstatus(dns_cache, client, REGEX_BLOCKED, domain);
- dns_cache->deny_regex_id = regex_idx;
// Regex may be overwriting reply type for this domain
if(dns_cache->force_reply != REPLY_UNKNOWN)
force_next_DNS_reply = dns_cache->force_reply;
// Store ID of this regex (fork-private)
- last_regex_idx = regex_idx;
+ last_regex_idx = dns_cache->domainlist_id;
// We block this domain
return true;
@@ -1235,7 +1252,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
}
// Get cache pointer
- unsigned int cacheID = findCacheID(domainID, clientID, query->type);
+ unsigned int cacheID = findCacheID(domainID, clientID, query->type, true);
DNSCacheData *dns_cache = getDNSCache(cacheID, true);
if(dns_cache == NULL)
{
@@ -1302,7 +1319,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
if(!query->flags.allowed)
{
force_next_DNS_reply = dns_cache->force_reply;
- last_regex_idx = dns_cache->deny_regex_id;
+ last_regex_idx = dns_cache->domainlist_id;
query_blocked(query, domain, client, QUERY_REGEX);
return true;
}
@@ -1327,7 +1344,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
log_debug(DEBUG_QUERIES, "%s is known as special domain", domainstr);
force_next_DNS_reply = dns_cache->force_reply;
- query_blocked(query, domain, client, QUERY_CACHE);
+ query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN);
return true;
break;
@@ -1370,9 +1387,13 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
domainstr = strdup(domainstr);
const char *blockedDomain = domainstr;
- // Check whitelist (exact + regex) for match
+ // Check exact whitelist for match
query->flags.allowed = in_allowlist(domainstr, dns_cache, client) == FOUND;
+ // If not found: Check regex whitelist for match
+ if(!query->flags.allowed)
+ query->flags.allowed = in_regex(domainstr, dns_cache, client->id, REGEX_ALLOW);
+
// Check blacklist (exact + regex) and gravity for queried domain
unsigned char new_status = QUERY_UNKNOWN;
bool db_okay = true;
@@ -1433,12 +1454,9 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
}
-bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line)
+bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file, const int line)
{
- // Get CNAME destination and source (if applicable)
- const char *dst = domain;
- const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL;
- log_debug(DEBUG_QUERIES, "FTL_CNAME called with: src = %s, dst = %s, id = %d", src, domain, id);
+ log_debug(DEBUG_QUERIES, "FTL_CNAME called with: src = %s, dst = %s, id = %d", src, dst, id);
// Does the user want to skip deep CNAME inspection?
if(!config.cname_deep_inspection)
@@ -1485,7 +1503,7 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
// child_domain = Intermediate domain in CNAME path
// This is the domain which was queried later in this chain
- char *child_domain = strdup(domain);
+ char *child_domain = strdup(dst);
// Convert to lowercase for matching
strtolower(child_domain);
const int child_domainID = findDomainID(child_domain, false);
@@ -1529,8 +1547,8 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
else if(query->status == QUERY_REGEX)
{
// Get parent and child DNS cache entries
- const int parent_cacheID = findCacheID(parent_domainID, clientID, query->type);
- const int child_cacheID = findCacheID(child_domainID, clientID, query->type);
+ const int parent_cacheID = findCacheID(parent_domainID, clientID, query->type, false);
+ const int child_cacheID = findCacheID(child_domainID, clientID, query->type, false);
// Get cache pointers
DNSCacheData *parent_cache = getDNSCache(parent_cacheID, true);
@@ -1538,9 +1556,7 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
// Propagate ID of responsible regex up from the child to the parent domain
if(parent_cache != NULL && child_cache != NULL)
- {
- child_cache->deny_regex_id = parent_cache->deny_regex_id;
- }
+ child_cache->domainlist_id = parent_cache->domainlist_id;
// Set status
query_set_status(query, QUERY_REGEX_CNAME);
@@ -1553,10 +1569,7 @@ bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const
}
// Debug logging for deep CNAME inspection (if enabled)
- if(src == NULL)
- log_debug(DEBUG_QUERIES, "Query %d: CNAME %s", id, dst);
- else
- log_debug(DEBUG_QUERIES, "Query %d: CNAME %s ---> %s", id, src, dst);
+ log_debug(DEBUG_QUERIES, "Query %d: CNAME %s ---> %s", id, src, dst);
// Mark query for updating in the database
query->flags.database.changed = true;
@@ -1728,6 +1741,7 @@ void FTL_dnsmasq_reload(void)
// Gravity database updates
// - (Re-)open gravity database connection
// - Get number of blocked domains
+ // - check adlist table for inaccessible adlists
// - Read and compile regex filters (incl. per-client)
// - Flush FTL's DNS cache
set_event(RELOAD_GRAVITY);
@@ -1858,6 +1872,9 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al
log_debug(DEBUG_FLAGS, "***** Unknown cache query");
}
+ // Is this a stale reply?
+ const bool stale = flags & F_STALE;
+
// Possible debugging output
if(config.debug & DEBUG_QUERIES)
{
@@ -1909,16 +1926,18 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al
if(cached || last_server.sa.sa_family == 0)
// Log cache or upstream reply from unknown source
- log_debug(DEBUG_QUERIES, "**** got %s reply: %s is %s (ID %i, %s:%i)",
- cached ? "cache" : "upstream", dispname, answer, id, file, line);
+ log_debug(DEBUG_QUERIES, "**** got %s%s reply: %s is %s (ID %i, %s:%i)",
+ stale ? "stale ": "", cached ? "cache" : "upstream",
+ dispname, answer, id, file, line);
else
{
char ip[ADDRSTRLEN+1] = { 0 };
in_port_t port = 0;
mysockaddr_extract_ip_port(&last_server, ip, &port);
// Log server which replied to our request
- log_debug(DEBUG_QUERIES, "**** got %s reply from %s#%d: %s is %s (ID %i, %s:%i)",
- cached ? "cache" : "upstream", ip, port, dispname, answer, id, file, line);
+ log_debug(DEBUG_QUERIES, "**** got %s%s reply from %s#%d: %s is %s (ID %i, %s:%i)",
+ stale ? "stale ": "", cached ? "cache" : "upstream",
+ ip, port, dispname, answer, id, file, line);
}
}
@@ -1967,13 +1986,16 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al
return;
}
+ // Determine query status (live or stale data?)
+ const enum query_status qs = stale ? QUERY_CACHE_STALE : QUERY_CACHE;
+
// This is either a reply served from cache or a blocked query (which appear
// to be from cache because of flags containing F_HOSTS)
if(cached)
{
// Set status of this query only if this is not a blocked query
if(!is_blocked(query->status))
- query_set_status(query, QUERY_CACHE);
+ query_set_status(query, qs);
// Detect if returned IP indicates that this query was blocked
const enum query_status new_status = detect_blocked_IP(flags, addr, query, domain);
@@ -2017,7 +2039,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al
// Answered from a custom (user provided) cache file or because
// we're the authoritative DNS server (e.g. DHCP server and this
// is our own domain)
- query_set_status(query, QUERY_CACHE);
+ query_set_status(query, qs);
// Save reply type and update individual reply counters
query_set_reply(flags, 0, addr, query, response);
@@ -2597,8 +2619,7 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep
if(config.debug & DEBUG_QUERIES)
{
const char *path = short_path(file);
- log_debug(DEBUG_QUERIES, "Set reply to %s (%d) in %s:%d", get_query_reply_str(query->reply), query->reply,
- path, line);
+ log_debug(DEBUG_QUERIES, "Set reply to %s (%d) in %s:%d", get_query_reply_str(new_reply), new_reply, path, line);
if(query->reply != REPLY_UNKNOWN && query->reply != new_reply)
log_debug(DEBUG_QUERIES, "Reply of query %i was %s now changing to %s", query->id,
get_query_reply_str(query->reply), get_query_reply_str(new_reply));
@@ -3278,8 +3299,8 @@ int check_struct_sizes(void)
result += check_one_struct("overTimeData", sizeof(overTimeData), 32, 24);
result += check_one_struct("regexData", sizeof(regexData), 56, 44);
result += check_one_struct("SharedMemory", sizeof(SharedMemory), 24, 12);
- result += check_one_struct("ShmSettings", sizeof(ShmSettings), 12, 12);
- result += check_one_struct("countersStruct", sizeof(countersStruct), 292, 292);
+ result += check_one_struct("ShmSettings", sizeof(ShmSettings), 16, 16);
+ result += check_one_struct("countersStruct", sizeof(countersStruct), 248, 248);
result += check_one_struct("sqlite3_stmt_vec", sizeof(sqlite3_stmt_vec), 32, 16);
if(result == 0)
@@ -3287,3 +3308,16 @@ int check_struct_sizes(void)
return result;
}
+
+static const char *check_dnsmasq_name(const char *name)
+{
+ // Special domain name handling
+ if(!name)
+ // 1. Substitute "(NULL)" if no name is available (should not happen)
+ return "(NULL)";
+ else if(!name[0])
+ // 2. Substitute "." if we are querying the root domain (e.g. DNSKEY)
+ return ".";
+ // else
+ return name;
+}
diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h
index f4767007..268996a0 100644
--- a/src/dnsmasq_interface.h
+++ b/src/dnsmasq_interface.h
@@ -16,12 +16,10 @@
#include "edns0.h"
#include "cache_info.h"
-extern int socketfd, telnetfd4, telnetfd6;
-extern unsigned char *pihole_privacylevel;
+extern unsigned char* pihole_privacylevel;
+enum protocol { TCP, UDP, INTERNAL };
-enum protocol { TCP, UDP, INTERNAL } __attribute__ ((packed));
-
-void FTL_hook(unsigned int flags, char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line);
+void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char *arg, int id, unsigned short type, const char* file, const int line);
#define FTL_iface(iface, addr, addrfamily) _FTL_iface(iface, addr, addrfamily, __FILE__, __LINE__)
void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_family_t addrfamily, const char* file, const int line);
@@ -37,8 +35,8 @@ void FTL_forwarding_retried(const struct server *server, const int oldID, const
#define FTL_make_answer(header, limit, len, ede) _FTL_make_answer(header, limit, len, ede, __FILE__, __LINE__)
size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char* file, const int line);
-#define FTL_CNAME(domain, cpp, id) _FTL_CNAME(domain, cpp, id, __FILE__, __LINE__)
-bool _FTL_CNAME(const char *domain, const struct crec *cpp, const int id, const char* file, const int line);
+#define FTL_CNAME(dst, src, id) _FTL_CNAME(dst, src, id, __FILE__, __LINE__)
+bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file, const int line);
unsigned int FTL_extract_question_flags(struct dns_header *header, const size_t qlen);
void FTL_query_in_progress(const int id);
diff --git a/src/enums.h b/src/enums.h
index 92bc2632..27908830 100644
--- a/src/enums.h
+++ b/src/enums.h
@@ -47,6 +47,7 @@ enum query_status {
QUERY_IN_PROGRESS,
QUERY_DBBUSY,
QUERY_SPECIAL_DOMAIN,
+ QUERY_CACHE_STALE,
QUERY_STATUS_MAX
} __attribute__ ((packed));
@@ -249,6 +250,13 @@ enum thread_types {
THREADS_MAX
} __attribute__ ((packed));
+enum telnet_type {
+ TELNETv4,
+ TELNETv6,
+ TELNET_SOCK,
+ TELNET_MAX
+} __attribute__ ((packed));
+
enum message_type {
REGEX_MESSAGE,
SUBNET_MESSAGE,
@@ -259,7 +267,8 @@ enum message_type {
LOAD_MESSAGE,
SHMEM_MESSAGE,
DISK_MESSAGE,
- MAX_MESSAGE
+ INACCESSIBLE_ADLIST_MESSAGE,
+ MAX_MESSAGE,
} __attribute__ ((packed));
enum ptr_type {
diff --git a/src/gc.c b/src/gc.c
index 9f5584a4..f51a29d6 100644
--- a/src/gc.c
+++ b/src/gc.c
@@ -77,17 +77,21 @@ time_t get_rate_limit_turnaround(const unsigned int rate_limit_count)
return (time_t)config.rate_limit.interval*how_often - (time(NULL) - lastRateLimitCleaner);
}
-static void check_space(const char *file)
+static int check_space(const char *file, int LastUsage)
{
if(config.check.disk == 0)
- return;
+ return 0;
int perc = 0;
char buffer[64] = { 0 };
// Warn if space usage at the device holding the corresponding file
- // exceeds the configured threshold
- if((perc = get_filepath_usage(file, buffer)) > config.check.disk)
+ // exceeds the configured threshold and current usage is higher than
+ // usage in the last run (to prevent log spam)
+ perc = get_filepath_usage(file, buffer);
+ if(perc > config.check.disk && perc > LastUsage )
log_resource_shortage(-1.0, 0, -1, perc, file, buffer);
+
+ return perc;
}
static void check_load(void)
@@ -120,6 +124,10 @@ void *GC_thread(void *val)
lastRateLimitCleaner = time(NULL);
time_t lastResourceCheck = 0;
+ // Remember disk usage
+ int LastLogStorageUsage = 0;
+ int LastDBStorageUsage = 0;
+
// Run as long as this thread is not canceled
while(!killed)
{
@@ -141,8 +149,8 @@ void *GC_thread(void *val)
if(now - lastResourceCheck >= RCinterval)
{
check_load();
- check_space(config.files.database);
- check_space(config.files.log);
+ LastDBStorageUsage = check_space(config.files.database, LastDBStorageUsage);
+ LastLogStorageUsage = check_space(config.files.log, LastLogStorageUsage);
lastResourceCheck = now;
}
@@ -210,6 +218,7 @@ void *GC_thread(void *val)
// Adjusting counters is done below in moveOverTimeMemory()
break;
case QUERY_CACHE:
+ case QUERY_CACHE_STALE:
// Answered from local cache _or_ local config
break;
case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through)
diff --git a/src/gen_version.cmake b/src/gen_version.cmake
index 3d70e524..a4130332 100644
--- a/src/gen_version.cmake
+++ b/src/gen_version.cmake
@@ -24,7 +24,7 @@ if(DEFINED ENV{GIT_HASH})
set(GIT_HASH "$ENV{GIT_HASH}")
else()
execute_process(
- COMMAND git --no-pager describe --always --dirty
+ COMMAND git --no-pager describe --always --abbrev=8 --dirty
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE GIT_HASH
ERROR_QUIET
@@ -36,7 +36,7 @@ if(DEFINED ENV{GIT_VERSION})
set(GIT_VERSION "$ENV{GIT_VERSION}")
else()
execute_process(
- COMMAND git --no-pager describe --tags --always --dirty
+ COMMAND git --no-pager describe --tags --always --abbrev=8 --dirty
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE GIT_VERSION
ERROR_QUIET
diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c
index 2da6cb22..2577b90b 100644
--- a/src/lua/ftl_lua.c
+++ b/src/lua/ftl_lua.c
@@ -16,6 +16,7 @@
#include "../log.h"
#include
#include
+#include "scripts/scripts.h"
int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug)
{
@@ -98,9 +99,53 @@ LUAMOD_API int luaopen_pihole(lua_State *L) {
return LUA_YIELD;
}
-// Load bundled libraries and make the available globally
+static bool ftl_lua_load_embedded_script(lua_State *L, const char *name, const char *script, const size_t script_len, const bool make_global)
+{
+ // Explanation:
+ // luaL_dostring(L, script) expands to (luaL_loadstring(L, script) || lua_pcall(L, 0, LUA_MULTRET, 0))
+ // luaL_loadstring(L, script) calls luaL_loadbuffer(L, s, strlen(s), s)
+ if (luaL_loadbufferx(L, script, script_len, name, NULL) || lua_pcall(L, 0, LUA_MULTRET, 0) != 0)
+ {
+ const char *lua_err = lua_tostring(L, -1);
+ printf("LUA error while trying to import %s.lua: %s\n", name, lua_err);
+ return false;
+ }
+
+ if(make_global)
+ {
+ /* Set global[name] = luaL_dostring return */
+ lua_setglobal(L, name);
+ }
+
+ return true;
+}
+
+struct {
+ const char *name;
+ const char *content;
+ const size_t contentlen;
+ const bool global;
+} scripts[] =
+{
+ {"inspect", inspect_lua, sizeof(inspect_lua), true},
+};
+
+// Loop over bundled LUA libraries and print their names on the console
+void print_embedded_scripts(void)
+{
+ for(unsigned int i = 0; i < sizeof(scripts)/sizeof(scripts[0]); i++)
+ {
+ char prefix[2] = { 0 };
+ double formatted = 0.0;
+ format_memory_size(prefix, scripts[i].contentlen, &formatted);
+
+ printf("%s.lua (%.2f %sB) ", scripts[i].name, formatted, prefix);
+ }
+}
+
+// Loop over bundled LUA libraries and load them
void ftl_lua_init(lua_State *L)
{
- if(dolibrary(L, "inspect") != LUA_OK)
- printf("Failed to load library inspect.lua\n");
+ for(unsigned int i = 0; i < sizeof(scripts)/sizeof(scripts[0]); i++)
+ ftl_lua_load_embedded_script(L, scripts[i].name, scripts[i].content, scripts[i].contentlen, scripts[i].global);
}
diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h
index c2ef12ab..29c4cb5f 100644
--- a/src/lua/ftl_lua.h
+++ b/src/lua/ftl_lua.h
@@ -21,8 +21,9 @@ int run_luac(const int argc, char **argv);
int lua_main (int argc, char **argv);
int luac_main (int argc, char **argv);
-extern int dolibrary (lua_State *L, const char *name);
+extern int dolibrary (lua_State *L, char *name);
+void print_embedded_scripts(void);
void ftl_lua_init(lua_State *L);
#endif //FTL_LUA_H
\ No newline at end of file
diff --git a/src/lua/lapi.c b/src/lua/lapi.c
index 9048245f..5ee65792 100644
--- a/src/lua/lapi.c
+++ b/src/lua/lapi.c
@@ -39,7 +39,7 @@ const char lua_ident[] =
/*
-** Test for a valid index.
+** Test for a valid index (one that is not the 'nilvalue').
** '!ttisnil(o)' implies 'o != &G(L)->nilvalue', so it is not needed.
** However, it covers the most common cases in a faster way.
*/
@@ -53,6 +53,10 @@ const char lua_ident[] =
#define isupvalue(i) ((i) < LUA_REGISTRYINDEX)
+/*
+** Convert an acceptable index to a pointer to its respective value.
+** Non-valid indices return the special nil value 'G(L)->nilvalue'.
+*/
static TValue *index2value (lua_State *L, int idx) {
CallInfo *ci = L->ci;
if (idx > 0) {
@@ -70,21 +74,28 @@ static TValue *index2value (lua_State *L, int idx) {
else { /* upvalues */
idx = LUA_REGISTRYINDEX - idx;
api_check(L, idx <= MAXUPVAL + 1, "upvalue index too large");
- if (ttislcf(s2v(ci->func))) /* light C function? */
- return &G(L)->nilvalue; /* it has no upvalues */
- else {
+ if (ttisCclosure(s2v(ci->func))) { /* C closure? */
CClosure *func = clCvalue(s2v(ci->func));
- return (idx <= func->nupvalues) ? &func->upvalue[idx-1] : &G(L)->nilvalue;
+ return (idx <= func->nupvalues) ? &func->upvalue[idx-1]
+ : &G(L)->nilvalue;
+ }
+ else { /* light C function or Lua function (through a hook)?) */
+ api_check(L, ttislcf(s2v(ci->func)), "caller not a C function");
+ return &G(L)->nilvalue; /* no upvalues */
}
}
}
-static StkId index2stack (lua_State *L, int idx) {
+
+/*
+** Convert a valid actual index (not a pseudo-index) to its address.
+*/
+l_sinline StkId index2stack (lua_State *L, int idx) {
CallInfo *ci = L->ci;
if (idx > 0) {
StkId o = ci->func + idx;
- api_check(L, o < L->top, "unacceptable index");
+ api_check(L, o < L->top, "invalid index");
return o;
}
else { /* non-positive index */
@@ -172,7 +183,7 @@ LUA_API int lua_gettop (lua_State *L) {
LUA_API void lua_settop (lua_State *L, int idx) {
CallInfo *ci;
- StkId func;
+ StkId func, newtop;
ptrdiff_t diff; /* difference for new top */
lua_lock(L);
ci = L->ci;
@@ -187,9 +198,26 @@ LUA_API void lua_settop (lua_State *L, int idx) {
api_check(L, -(idx+1) <= (L->top - (func + 1)), "invalid new top");
diff = idx + 1; /* will "subtract" index (as it is negative) */
}
- if (diff < 0 && hastocloseCfunc(ci->nresults))
- luaF_close(L, L->top + diff, LUA_OK);
- L->top += diff; /* correct top only after closing any upvalue */
+ api_check(L, L->tbclist < L->top, "previous pop of an unclosed slot");
+ newtop = L->top + diff;
+ if (diff < 0 && L->tbclist >= newtop) {
+ lua_assert(hastocloseCfunc(ci->nresults));
+ luaF_close(L, newtop, CLOSEKTOP, 0);
+ }
+ L->top = newtop; /* correct top only after closing any upvalue */
+ lua_unlock(L);
+}
+
+
+LUA_API void lua_closeslot (lua_State *L, int idx) {
+ StkId level;
+ lua_lock(L);
+ level = index2stack(L, idx);
+ api_check(L, hastocloseCfunc(L->ci->nresults) && L->tbclist == level,
+ "no variable to close at given level");
+ luaF_close(L, level, CLOSEKTOP, 0);
+ level = index2stack(L, idx); /* stack may be moved */
+ setnilvalue(s2v(level));
lua_unlock(L);
}
@@ -200,7 +228,7 @@ LUA_API void lua_settop (lua_State *L, int idx) {
** Note that we move(copy) only the value inside the stack.
** (We do not move additional fields that may exist.)
*/
-static void reverse (lua_State *L, StkId from, StkId to) {
+l_sinline void reverse (lua_State *L, StkId from, StkId to) {
for (; from < to; from++, to--) {
TValue temp;
setobj(L, &temp, s2v(from));
@@ -420,7 +448,7 @@ LUA_API lua_CFunction lua_tocfunction (lua_State *L, int idx) {
}
-static void *touserdata (const TValue *o) {
+l_sinline void *touserdata (const TValue *o) {
switch (ttype(o)) {
case LUA_TUSERDATA: return getudatamem(uvalue(o));
case LUA_TLIGHTUSERDATA: return pvalue(o);
@@ -612,7 +640,7 @@ LUA_API int lua_pushthread (lua_State *L) {
*/
-static int auxgetstr (lua_State *L, const TValue *t, const char *k) {
+l_sinline int auxgetstr (lua_State *L, const TValue *t, const char *k) {
const TValue *slot;
TString *str = luaS_new(L, k);
if (luaV_fastget(L, t, str, slot, luaH_getstr)) {
@@ -629,11 +657,21 @@ static int auxgetstr (lua_State *L, const TValue *t, const char *k) {
}
+/*
+** Get the global table in the registry. Since all predefined
+** indices in the registry were inserted right when the registry
+** was created and never removed, they must always be in the array
+** part of the registry.
+*/
+#define getGtable(L) \
+ (&hvalue(&G(L)->l_registry)->array[LUA_RIDX_GLOBALS - 1])
+
+
LUA_API int lua_getglobal (lua_State *L, const char *name) {
- Table *reg;
+ const TValue *G;
lua_lock(L);
- reg = hvalue(&G(L)->l_registry);
- return auxgetstr(L, luaH_getint(reg, LUA_RIDX_GLOBALS), name);
+ G = getGtable(L);
+ return auxgetstr(L, G, name);
}
@@ -677,7 +715,7 @@ LUA_API int lua_geti (lua_State *L, int idx, lua_Integer n) {
}
-static int finishrawget (lua_State *L, const TValue *val) {
+l_sinline int finishrawget (lua_State *L, const TValue *val) {
if (isempty(val)) /* avoid copying empty items to the stack */
setnilvalue(s2v(L->top));
else
@@ -811,10 +849,10 @@ static void auxsetstr (lua_State *L, const TValue *t, const char *k) {
LUA_API void lua_setglobal (lua_State *L, const char *name) {
- Table *reg;
+ const TValue *G;
lua_lock(L); /* unlock done in 'auxsetstr' */
- reg = hvalue(&G(L)->l_registry);
- auxsetstr(L, luaH_getint(reg, LUA_RIDX_GLOBALS), name);
+ G = getGtable(L);
+ auxsetstr(L, G, name);
}
@@ -861,12 +899,10 @@ LUA_API void lua_seti (lua_State *L, int idx, lua_Integer n) {
static void aux_rawset (lua_State *L, int idx, TValue *key, int n) {
Table *t;
- TValue *slot;
lua_lock(L);
api_checknelems(L, n);
t = gettable(L, idx);
- slot = luaH_set(L, t, key);
- setobj2t(L, slot, s2v(L->top - 1));
+ luaH_set(L, t, key, s2v(L->top - 1));
invalidateTMcache(t);
luaC_barrierback(L, obj2gco(t), s2v(L->top - 1));
L->top -= n;
@@ -1063,8 +1099,7 @@ LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data,
LClosure *f = clLvalue(s2v(L->top - 1)); /* get newly created function */
if (f->nupvalues >= 1) { /* does it have an upvalue? */
/* get global table from registry */
- Table *reg = hvalue(&G(L)->l_registry);
- const TValue *gt = luaH_getint(reg, LUA_RIDX_GLOBALS);
+ const TValue *gt = getGtable(L);
/* set global table as 1st upvalue of 'f' (may be LUA_ENV) */
setobj(L, f->upvals[0]->v, gt);
luaC_barrier(L, f->upvals[0], gt);
@@ -1101,18 +1136,19 @@ LUA_API int lua_status (lua_State *L) {
LUA_API int lua_gc (lua_State *L, int what, ...) {
va_list argp;
int res = 0;
- global_State *g;
+ global_State *g = G(L);
+ if (g->gcstp & GCSTPGC) /* internal stop? */
+ return -1; /* all options are invalid when stopped */
lua_lock(L);
- g = G(L);
va_start(argp, what);
switch (what) {
case LUA_GCSTOP: {
- g->gcrunning = 0;
+ g->gcstp = GCSTPUSR; /* stopped by the user */
break;
}
case LUA_GCRESTART: {
luaE_setdebt(g, 0);
- g->gcrunning = 1;
+ g->gcstp = 0; /* (GCSTPGC must be already zero here) */
break;
}
case LUA_GCCOLLECT: {
@@ -1131,8 +1167,8 @@ LUA_API int lua_gc (lua_State *L, int what, ...) {
case LUA_GCSTEP: {
int data = va_arg(argp, int);
l_mem debt = 1; /* =1 to signal that it did an actual step */
- lu_byte oldrunning = g->gcrunning;
- g->gcrunning = 1; /* allow GC to run */
+ lu_byte oldstp = g->gcstp;
+ g->gcstp = 0; /* allow GC to run (GCSTPGC must be zero here) */
if (data == 0) {
luaE_setdebt(g, 0); /* do a basic step */
luaC_step(L);
@@ -1142,7 +1178,7 @@ LUA_API int lua_gc (lua_State *L, int what, ...) {
luaE_setdebt(g, debt);
luaC_checkGC(L);
}
- g->gcrunning = oldrunning; /* restore previous state */
+ g->gcstp = oldstp; /* restore previous state */
if (debt > 0 && g->gcstate == GCSpause) /* end of cycle? */
res = 1; /* signal it */
break;
@@ -1160,7 +1196,7 @@ LUA_API int lua_gc (lua_State *L, int what, ...) {
break;
}
case LUA_GCISRUNNING: {
- res = g->gcrunning;
+ res = gcrunning(g);
break;
}
case LUA_GCGEN: {
@@ -1240,8 +1276,7 @@ LUA_API void lua_toclose (lua_State *L, int idx) {
lua_lock(L);
o = index2stack(L, idx);
nresults = L->ci->nresults;
- api_check(L, L->openupval == NULL || uplevel(L->openupval) <= o,
- "marked index below or equal new one");
+ api_check(L, L->tbclist < o, "given index below or equal a marked one");
luaF_newtbcupval(L, o); /* create new to-be-closed upvalue */
if (!hastocloseCfunc(nresults)) /* function not marked yet? */
L->ci->nresults = codeNresults(nresults); /* mark it */
@@ -1383,13 +1418,16 @@ LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n) {
static UpVal **getupvalref (lua_State *L, int fidx, int n, LClosure **pf) {
+ static const UpVal *const nullup = NULL;
LClosure *f;
TValue *fi = index2value(L, fidx);
api_check(L, ttisLclosure(fi), "Lua function expected");
f = clLvalue(fi);
- api_check(L, (1 <= n && n <= f->p->sizeupvalues), "invalid upvalue index");
if (pf) *pf = f;
- return &f->upvals[n - 1]; /* get its upvalue pointer */
+ if (1 <= n && n <= f->p->sizeupvalues)
+ return &f->upvals[n - 1]; /* get its upvalue pointer */
+ else
+ return (UpVal**)&nullup;
}
@@ -1401,11 +1439,14 @@ LUA_API void *lua_upvalueid (lua_State *L, int fidx, int n) {
}
case LUA_VCCL: { /* C closure */
CClosure *f = clCvalue(fi);
- api_check(L, 1 <= n && n <= f->nupvalues, "invalid upvalue index");
- return &f->upvalue[n - 1];
- }
+ if (1 <= n && n <= f->nupvalues)
+ return &f->upvalue[n - 1];
+ /* else */
+ } /* FALLTHROUGH */
+ case LUA_VLCF:
+ return NULL; /* light C functions have no upvalues */
default: {
- api_check(L, 0, "closure expected");
+ api_check(L, 0, "function expected");
return NULL;
}
}
@@ -1417,6 +1458,7 @@ LUA_API void lua_upvaluejoin (lua_State *L, int fidx1, int n1,
LClosure *f1;
UpVal **up1 = getupvalref(L, fidx1, n1, &f1);
UpVal **up2 = getupvalref(L, fidx2, n2, NULL);
+ api_check(L, *up1 != NULL && *up2 != NULL, "invalid upvalue index");
*up1 = *up2;
luaC_objbarrier(L, f1, *up1);
}
diff --git a/src/lua/lapi.h b/src/lua/lapi.h
index 41216b27..9e99cc44 100644
--- a/src/lua/lapi.h
+++ b/src/lua/lapi.h
@@ -42,6 +42,8 @@
#define hastocloseCfunc(n) ((n) < LUA_MULTRET)
+/* Map [-1, inf) (range of 'nresults') into (-inf, -2] */
#define codeNresults(n) (-(n) - 3)
+#define decodeNresults(n) (-(n) - 3)
#endif
diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c
index cbe9ed31..8ed1da11 100644
--- a/src/lua/lauxlib.c
+++ b/src/lua/lauxlib.c
@@ -190,7 +190,7 @@ LUALIB_API int luaL_argerror (lua_State *L, int arg, const char *extramsg) {
}
-int luaL_typeerror (lua_State *L, int arg, const char *tname) {
+LUALIB_API int luaL_typeerror (lua_State *L, int arg, const char *tname) {
const char *msg;
const char *typearg; /* name for the type of the actual argument */
if (luaL_getmetafield(L, arg, "__name") == LUA_TSTRING)
@@ -283,10 +283,10 @@ LUALIB_API int luaL_fileresult (lua_State *L, int stat, const char *fname) {
LUALIB_API int luaL_execresult (lua_State *L, int stat) {
- const char *what = "exit"; /* type of termination */
if (stat != 0 && errno != 0) /* error with an 'errno'? */
return luaL_fileresult(L, 0, NULL);
else {
+ const char *what = "exit"; /* type of termination */
l_inspectstat(stat, what); /* interpret result */
if (*what == 'e' && stat == 0) /* successful termination? */
lua_pushboolean(L, 1);
@@ -378,7 +378,7 @@ LUALIB_API int luaL_checkoption (lua_State *L, int arg, const char *def,
** but without 'msg'.)
*/
LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *msg) {
- if (!lua_checkstack(L, space)) {
+ if (l_unlikely(!lua_checkstack(L, space))) {
if (msg)
luaL_error(L, "stack overflow (%s)", msg);
else
@@ -388,20 +388,20 @@ LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *msg) {
LUALIB_API void luaL_checktype (lua_State *L, int arg, int t) {
- if (lua_type(L, arg) != t)
+ if (l_unlikely(lua_type(L, arg) != t))
tag_error(L, arg, t);
}
LUALIB_API void luaL_checkany (lua_State *L, int arg) {
- if (lua_type(L, arg) == LUA_TNONE)
+ if (l_unlikely(lua_type(L, arg) == LUA_TNONE))
luaL_argerror(L, arg, "value expected");
}
LUALIB_API const char *luaL_checklstring (lua_State *L, int arg, size_t *len) {
const char *s = lua_tolstring(L, arg, len);
- if (!s) tag_error(L, arg, LUA_TSTRING);
+ if (l_unlikely(!s)) tag_error(L, arg, LUA_TSTRING);
return s;
}
@@ -420,7 +420,7 @@ LUALIB_API const char *luaL_optlstring (lua_State *L, int arg,
LUALIB_API lua_Number luaL_checknumber (lua_State *L, int arg) {
int isnum;
lua_Number d = lua_tonumberx(L, arg, &isnum);
- if (!isnum)
+ if (l_unlikely(!isnum))
tag_error(L, arg, LUA_TNUMBER);
return d;
}
@@ -442,7 +442,7 @@ static void interror (lua_State *L, int arg) {
LUALIB_API lua_Integer luaL_checkinteger (lua_State *L, int arg) {
int isnum;
lua_Integer d = lua_tointegerx(L, arg, &isnum);
- if (!isnum) {
+ if (l_unlikely(!isnum)) {
interror(L, arg);
}
return d;
@@ -475,7 +475,7 @@ static void *resizebox (lua_State *L, int idx, size_t newsize) {
lua_Alloc allocf = lua_getallocf(L, &ud);
UBox *box = (UBox *)lua_touserdata(L, idx);
void *temp = allocf(ud, box->box, box->bsize, newsize);
- if (temp == NULL && newsize > 0) { /* allocation error? */
+ if (l_unlikely(temp == NULL && newsize > 0)) { /* allocation error? */
lua_pushliteral(L, "not enough memory");
lua_error(L); /* raise a memory error */
}
@@ -515,13 +515,22 @@ static void newbox (lua_State *L) {
#define buffonstack(B) ((B)->b != (B)->init.b)
+/*
+** Whenever buffer is accessed, slot 'idx' must either be a box (which
+** cannot be NULL) or it is a placeholder for the buffer.
+*/
+#define checkbufferlevel(B,idx) \
+ lua_assert(buffonstack(B) ? lua_touserdata(B->L, idx) != NULL \
+ : lua_touserdata(B->L, idx) == (void*)B)
+
+
/*
** Compute new size for buffer 'B', enough to accommodate extra 'sz'
** bytes.
*/
static size_t newbuffsize (luaL_Buffer *B, size_t sz) {
size_t newsize = B->size * 2; /* double buffer size */
- if (MAX_SIZET - sz < B->n) /* overflow in (B->n + sz)? */
+ if (l_unlikely(MAX_SIZET - sz < B->n)) /* overflow in (B->n + sz)? */
return luaL_error(B->L, "buffer too large");
if (newsize < B->n + sz) /* double is not big enough? */
newsize = B->n + sz;
@@ -531,10 +540,11 @@ static size_t newbuffsize (luaL_Buffer *B, size_t sz) {
/*
** Returns a pointer to a free area with at least 'sz' bytes in buffer
-** 'B'. 'boxidx' is the relative position in the stack where the
-** buffer's box is or should be.
+** 'B'. 'boxidx' is the relative position in the stack where is the
+** buffer's box or its placeholder.
*/
static char *prepbuffsize (luaL_Buffer *B, size_t sz, int boxidx) {
+ checkbufferlevel(B, boxidx);
if (B->size - B->n >= sz) /* enough space? */
return B->b + B->n;
else {
@@ -545,10 +555,9 @@ static char *prepbuffsize (luaL_Buffer *B, size_t sz, int boxidx) {
if (buffonstack(B)) /* buffer already has a box? */
newbuff = (char *)resizebox(L, boxidx, newsize); /* resize it */
else { /* no box yet */
- lua_pushnil(L); /* reserve slot for final result */
+ lua_remove(L, boxidx); /* remove placeholder */
newbox(L); /* create a new box */
- /* move box (and slot) to its intended position */
- lua_rotate(L, boxidx - 1, 2);
+ lua_insert(L, boxidx); /* move box to its intended position */
lua_toclose(L, boxidx);
newbuff = (char *)resizebox(L, boxidx, newsize);
memcpy(newbuff, B->b, B->n * sizeof(char)); /* copy original content */
@@ -583,11 +592,11 @@ LUALIB_API void luaL_addstring (luaL_Buffer *B, const char *s) {
LUALIB_API void luaL_pushresult (luaL_Buffer *B) {
lua_State *L = B->L;
+ checkbufferlevel(B, -1);
lua_pushlstring(L, B->b, B->n);
- if (buffonstack(B)) {
- lua_copy(L, -1, -3); /* move string to reserved slot */
- lua_pop(L, 2); /* pop string and box (closing the box) */
- }
+ if (buffonstack(B))
+ lua_closeslot(L, -2); /* close the box */
+ lua_remove(L, -2); /* remove box or placeholder from the stack */
}
@@ -622,6 +631,7 @@ LUALIB_API void luaL_buffinit (lua_State *L, luaL_Buffer *B) {
B->b = B->init.b;
B->n = 0;
B->size = LUAL_BUFFERSIZE;
+ lua_pushlightuserdata(L, (void*)B); /* push placeholder */
}
@@ -639,10 +649,14 @@ LUALIB_API char *luaL_buffinitsize (lua_State *L, luaL_Buffer *B, size_t sz) {
** =======================================================
*/
-/* index of free-list header */
-#define freelist 0
-
+/* index of free-list header (after the predefined values) */
+#define freelist (LUA_RIDX_LAST + 1)
+/*
+** The previously freed references form a linked list:
+** t[freelist] is the index of a first free index, or zero if list is
+** empty; t[t[freelist]] is the index of the second element; etc.
+*/
LUALIB_API int luaL_ref (lua_State *L, int t) {
int ref;
if (lua_isnil(L, -1)) {
@@ -650,9 +664,16 @@ LUALIB_API int luaL_ref (lua_State *L, int t) {
return LUA_REFNIL; /* 'nil' has a unique fixed reference */
}
t = lua_absindex(L, t);
- lua_rawgeti(L, t, freelist); /* get first free element */
- ref = (int)lua_tointeger(L, -1); /* ref = t[freelist] */
- lua_pop(L, 1); /* remove it from stack */
+ if (lua_rawgeti(L, t, freelist) == LUA_TNIL) { /* first access? */
+ ref = 0; /* list is empty */
+ lua_pushinteger(L, 0); /* initialize as an empty list */
+ lua_rawseti(L, t, freelist); /* ref = t[freelist] = 0 */
+ }
+ else { /* already initialized */
+ lua_assert(lua_isinteger(L, -1));
+ ref = (int)lua_tointeger(L, -1); /* ref = t[freelist] */
+ }
+ lua_pop(L, 1); /* remove element from stack */
if (ref != 0) { /* any free element? */
lua_rawgeti(L, t, ref); /* remove it from list */
lua_rawseti(L, t, freelist); /* (t[freelist] = t[ref]) */
@@ -668,6 +689,7 @@ LUALIB_API void luaL_unref (lua_State *L, int t, int ref) {
if (ref >= 0) {
t = lua_absindex(L, t);
lua_rawgeti(L, t, freelist);
+ lua_assert(lua_isinteger(L, -1));
lua_rawseti(L, t, ref); /* t[ref] = t[freelist] */
lua_pushinteger(L, ref);
lua_rawseti(L, t, freelist); /* t[freelist] = ref */
@@ -851,7 +873,7 @@ LUALIB_API lua_Integer luaL_len (lua_State *L, int idx) {
int isnum;
lua_len(L, idx);
l = lua_tointegerx(L, -1, &isnum);
- if (!isnum)
+ if (l_unlikely(!isnum))
luaL_error(L, "object length is not an integer");
lua_pop(L, 1); /* remove object */
return l;
@@ -859,6 +881,7 @@ LUALIB_API lua_Integer luaL_len (lua_State *L, int idx) {
LUALIB_API const char *luaL_tolstring (lua_State *L, int idx, size_t *len) {
+ idx = lua_absindex(L,idx);
if (luaL_callmeta(L, idx, "__tostring")) { /* metafield? */
if (!lua_isstring(L, -1))
luaL_error(L, "'__tostring' must return a string");
@@ -1006,43 +1029,67 @@ static int panic (lua_State *L) {
/*
-** Emit a warning. '*warnstate' means:
-** 0 - warning system is off;
-** 1 - ready to start a new message;
-** 2 - previous message is to be continued.
+** Warning functions:
+** warnfoff: warning system is off
+** warnfon: ready to start a new message
+** warnfcont: previous message is to be continued
*/
-static void warnf (void *ud, const char *message, int tocont) {
- int *warnstate = (int *)ud;
- if (*warnstate != 2 && !tocont && *message == '@') { /* control message? */
- if (strcmp(message, "@off") == 0)
- *warnstate = 0;
- else if (strcmp(message, "@on") == 0)
- *warnstate = 1;
- return;
+static void warnfoff (void *ud, const char *message, int tocont);
+static void warnfon (void *ud, const char *message, int tocont);
+static void warnfcont (void *ud, const char *message, int tocont);
+
+
+/*
+** Check whether message is a control message. If so, execute the
+** control or ignore it if unknown.
+*/
+static int checkcontrol (lua_State *L, const char *message, int tocont) {
+ if (tocont || *(message++) != '@') /* not a control message? */
+ return 0;
+ else {
+ if (strcmp(message, "off") == 0)
+ lua_setwarnf(L, warnfoff, L); /* turn warnings off */
+ else if (strcmp(message, "on") == 0)
+ lua_setwarnf(L, warnfon, L); /* turn warnings on */
+ return 1; /* it was a control message */
}
- else if (*warnstate == 0) /* warnings off? */
- return;
- if (*warnstate == 1) /* previous message was the last? */
- lua_writestringerror("%s", "Lua warning: "); /* start a new warning */
+}
+
+
+static void warnfoff (void *ud, const char *message, int tocont) {
+ checkcontrol((lua_State *)ud, message, tocont);
+}
+
+
+/*
+** Writes the message and handle 'tocont', finishing the message
+** if needed and setting the next warn function.
+*/
+static void warnfcont (void *ud, const char *message, int tocont) {
+ lua_State *L = (lua_State *)ud;
lua_writestringerror("%s", message); /* write message */
if (tocont) /* not the last part? */
- *warnstate = 2; /* to be continued */
+ lua_setwarnf(L, warnfcont, L); /* to be continued */
else { /* last part */
lua_writestringerror("%s", "\n"); /* finish message with end-of-line */
- *warnstate = 1; /* ready to start a new message */
+ lua_setwarnf(L, warnfon, L); /* next call is a new message */
}
}
+static void warnfon (void *ud, const char *message, int tocont) {
+ if (checkcontrol((lua_State *)ud, message, tocont)) /* control message? */
+ return; /* nothing else to be done */
+ lua_writestringerror("%s", "Lua warning: "); /* start a new warning */
+ warnfcont(ud, message, tocont); /* finish processing */
+}
+
+
LUALIB_API lua_State *luaL_newstate (void) {
lua_State *L = lua_newstate(l_alloc, NULL);
- if (L) {
- int *warnstate; /* space for warning state */
+ if (l_likely(L)) {
lua_atpanic(L, &panic);
- warnstate = (int *)lua_newuserdatauv(L, sizeof(int), 0);
- luaL_ref(L, LUA_REGISTRYINDEX); /* make sure it won't be collected */
- *warnstate = 0; /* default is warnings off */
- lua_setwarnf(L, warnf, warnstate);
+ lua_setwarnf(L, warnfoff, L); /* default is warnings off */
}
return L;
}
diff --git a/src/lua/lauxlib.h b/src/lua/lauxlib.h
index 59fef6af..5b977e2a 100644
--- a/src/lua/lauxlib.h
+++ b/src/lua/lauxlib.h
@@ -12,6 +12,7 @@
#include
#include
+#include "luaconf.h"
#include "lua.h"
@@ -101,7 +102,7 @@ LUALIB_API lua_State *(luaL_newstate) (void);
LUALIB_API lua_Integer (luaL_len) (lua_State *L, int idx);
-LUALIB_API void luaL_addgsub (luaL_Buffer *b, const char *s,
+LUALIB_API void (luaL_addgsub) (luaL_Buffer *b, const char *s,
const char *p, const char *r);
LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s,
const char *p, const char *r);
@@ -130,10 +131,10 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname,
(luaL_checkversion(L), luaL_newlibtable(L,l), luaL_setfuncs(L,l,0))
#define luaL_argcheck(L, cond,arg,extramsg) \
- ((void)((cond) || luaL_argerror(L, (arg), (extramsg))))
+ ((void)(luai_likely(cond) || luaL_argerror(L, (arg), (extramsg))))
#define luaL_argexpected(L,cond,arg,tname) \
- ((void)((cond) || luaL_typeerror(L, (arg), (tname))))
+ ((void)(luai_likely(cond) || luaL_typeerror(L, (arg), (tname))))
#define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL))
#define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL))
@@ -153,10 +154,34 @@ LUALIB_API void (luaL_requiref) (lua_State *L, const char *modname,
#define luaL_loadbuffer(L,s,sz,n) luaL_loadbufferx(L,s,sz,n,NULL)
+/*
+** Perform arithmetic operations on lua_Integer values with wrap-around
+** semantics, as the Lua core does.
+*/
+#define luaL_intop(op,v1,v2) \
+ ((lua_Integer)((lua_Unsigned)(v1) op (lua_Unsigned)(v2)))
+
+
/* push the value used to represent failure/error */
#define luaL_pushfail(L) lua_pushnil(L)
+/*
+** Internal assertions for in-house debugging
+*/
+#if !defined(lua_assert)
+
+#if defined LUAI_ASSERT
+ #include
+ #define lua_assert(c) assert(c)
+#else
+ #define lua_assert(c) ((void)0)
+#endif
+
+#endif
+
+
+
/*
** {======================================================
** Generic Buffer manipulation
diff --git a/src/lua/lbaselib.c b/src/lua/lbaselib.c
index 747fd45a..1d60c9de 100644
--- a/src/lua/lbaselib.c
+++ b/src/lua/lbaselib.c
@@ -138,7 +138,7 @@ static int luaB_setmetatable (lua_State *L) {
int t = lua_type(L, 2);
luaL_checktype(L, 1, LUA_TTABLE);
luaL_argexpected(L, t == LUA_TNIL || t == LUA_TTABLE, 2, "nil or table");
- if (luaL_getmetafield(L, 1, "__metatable") != LUA_TNIL)
+ if (l_unlikely(luaL_getmetafield(L, 1, "__metatable") != LUA_TNIL))
return luaL_error(L, "cannot change a protected metatable");
lua_settop(L, 2);
lua_setmetatable(L, 1);
@@ -182,11 +182,20 @@ static int luaB_rawset (lua_State *L) {
static int pushmode (lua_State *L, int oldmode) {
- lua_pushstring(L, (oldmode == LUA_GCINC) ? "incremental" : "generational");
+ if (oldmode == -1)
+ luaL_pushfail(L); /* invalid call to 'lua_gc' */
+ else
+ lua_pushstring(L, (oldmode == LUA_GCINC) ? "incremental"
+ : "generational");
return 1;
}
+/*
+** check whether call to 'lua_gc' was valid (not inside a finalizer)
+*/
+#define checkvalres(res) { if (res == -1) break; }
+
static int luaB_collectgarbage (lua_State *L) {
static const char *const opts[] = {"stop", "restart", "collect",
"count", "step", "setpause", "setstepmul",
@@ -199,12 +208,14 @@ static int luaB_collectgarbage (lua_State *L) {
case LUA_GCCOUNT: {
int k = lua_gc(L, o);
int b = lua_gc(L, LUA_GCCOUNTB);
+ checkvalres(k);
lua_pushnumber(L, (lua_Number)k + ((lua_Number)b/1024));
return 1;
}
case LUA_GCSTEP: {
int step = (int)luaL_optinteger(L, 2, 0);
int res = lua_gc(L, o, step);
+ checkvalres(res);
lua_pushboolean(L, res);
return 1;
}
@@ -212,11 +223,13 @@ static int luaB_collectgarbage (lua_State *L) {
case LUA_GCSETSTEPMUL: {
int p = (int)luaL_optinteger(L, 2, 0);
int previous = lua_gc(L, o, p);
+ checkvalres(previous);
lua_pushinteger(L, previous);
return 1;
}
case LUA_GCISRUNNING: {
int res = lua_gc(L, o);
+ checkvalres(res);
lua_pushboolean(L, res);
return 1;
}
@@ -233,10 +246,13 @@ static int luaB_collectgarbage (lua_State *L) {
}
default: {
int res = lua_gc(L, o);
+ checkvalres(res);
lua_pushinteger(L, res);
return 1;
}
}
+ luaL_pushfail(L); /* invalid call (inside a finalizer) */
+ return 1;
}
@@ -260,6 +276,11 @@ static int luaB_next (lua_State *L) {
}
+static int pairscont (lua_State *L, int status, lua_KContext k) {
+ (void)L; (void)status; (void)k; /* unused */
+ return 3;
+}
+
static int luaB_pairs (lua_State *L) {
luaL_checkany(L, 1);
if (luaL_getmetafield(L, 1, "__pairs") == LUA_TNIL) { /* no metamethod? */
@@ -269,7 +290,7 @@ static int luaB_pairs (lua_State *L) {
}
else {
lua_pushvalue(L, 1); /* argument 'self' to metamethod */
- lua_call(L, 1, 3); /* get 3 values from metamethod */
+ lua_callk(L, 1, 3, 0, pairscont); /* get 3 values from metamethod */
}
return 3;
}
@@ -279,7 +300,8 @@ static int luaB_pairs (lua_State *L) {
** Traversal function for 'ipairs'
*/
static int ipairsaux (lua_State *L) {
- lua_Integer i = luaL_checkinteger(L, 2) + 1;
+ lua_Integer i = luaL_checkinteger(L, 2);
+ i = luaL_intop(+, i, 1);
lua_pushinteger(L, i);
return (lua_geti(L, 1, i) == LUA_TNIL) ? 1 : 2;
}
@@ -299,7 +321,7 @@ static int luaB_ipairs (lua_State *L) {
static int load_aux (lua_State *L, int status, int envidx) {
- if (status == LUA_OK) {
+ if (l_likely(status == LUA_OK)) {
if (envidx != 0) { /* 'env' parameter? */
lua_pushvalue(L, envidx); /* environment for loaded function */
if (!lua_setupvalue(L, -2, 1)) /* set it as 1st upvalue */
@@ -355,7 +377,7 @@ static const char *generic_reader (lua_State *L, void *ud, size_t *size) {
*size = 0;
return NULL;
}
- else if (!lua_isstring(L, -1))
+ else if (l_unlikely(!lua_isstring(L, -1)))
luaL_error(L, "reader function must return a string");
lua_replace(L, RESERVEDSLOT); /* save string in reserved slot */
return lua_tolstring(L, RESERVEDSLOT, size);
@@ -393,7 +415,7 @@ static int dofilecont (lua_State *L, int d1, lua_KContext d2) {
static int luaB_dofile (lua_State *L) {
const char *fname = luaL_optstring(L, 1, NULL);
lua_settop(L, 1);
- if (luaL_loadfile(L, fname) != LUA_OK)
+ if (l_unlikely(luaL_loadfile(L, fname) != LUA_OK))
return lua_error(L);
lua_callk(L, 0, LUA_MULTRET, 0, dofilecont);
return dofilecont(L, 0, 0);
@@ -401,7 +423,7 @@ static int luaB_dofile (lua_State *L) {
static int luaB_assert (lua_State *L) {
- if (lua_toboolean(L, 1)) /* condition is true? */
+ if (l_likely(lua_toboolean(L, 1))) /* condition is true? */
return lua_gettop(L); /* return all arguments */
else { /* error */
luaL_checkany(L, 1); /* there must be a condition */
@@ -437,7 +459,7 @@ static int luaB_select (lua_State *L) {
** ignored).
*/
static int finishpcall (lua_State *L, int status, lua_KContext extra) {
- if (status != LUA_OK && status != LUA_YIELD) { /* error? */
+ if (l_unlikely(status != LUA_OK && status != LUA_YIELD)) { /* error? */
lua_pushboolean(L, 0); /* first result (false) */
lua_pushvalue(L, -2); /* error message */
return 2; /* return false, msg */
diff --git a/src/lua/lcode.c b/src/lua/lcode.c
index 6f241c94..06425a1d 100644
--- a/src/lua/lcode.c
+++ b/src/lua/lcode.c
@@ -10,6 +10,7 @@
#include "lprefix.h"
+#include
#include
#include
#include
@@ -314,15 +315,6 @@ void luaK_patchtohere (FuncState *fs, int list) {
}
-/*
-** MAXimum number of successive Instructions WiTHout ABSolute line
-** information.
-*/
-#if !defined(MAXIWTHABS)
-#define MAXIWTHABS 120
-#endif
-
-
/* limit for difference between lines in relative line info. */
#define LIMLINEDIFF 0x80
@@ -337,13 +329,13 @@ void luaK_patchtohere (FuncState *fs, int list) {
static void savelineinfo (FuncState *fs, Proto *f, int line) {
int linedif = line - fs->previousline;
int pc = fs->pc - 1; /* last instruction coded */
- if (abs(linedif) >= LIMLINEDIFF || fs->iwthabs++ > MAXIWTHABS) {
+ if (abs(linedif) >= LIMLINEDIFF || fs->iwthabs++ >= MAXIWTHABS) {
luaM_growvector(fs->ls->L, f->abslineinfo, fs->nabslineinfo,
f->sizeabslineinfo, AbsLineInfo, MAX_INT, "lines");
f->abslineinfo[fs->nabslineinfo].pc = pc;
f->abslineinfo[fs->nabslineinfo++].line = line;
linedif = ABSLINEINFO; /* signal that there is absolute information */
- fs->iwthabs = 0; /* restart counter */
+ fs->iwthabs = 1; /* restart counter */
}
luaM_growvector(fs->ls->L, f->lineinfo, pc, f->sizelineinfo, ls_byte,
MAX_INT, "opcodes");
@@ -545,11 +537,14 @@ static void freeexps (FuncState *fs, expdesc *e1, expdesc *e2) {
** and try to reuse constants. Because some values should not be used
** as keys (nil cannot be a key, integer keys can collapse with float
** keys), the caller must provide a useful 'key' for indexing the cache.
+** Note that all functions share the same table, so entering or exiting
+** a function can make some indices wrong.
*/
static int addk (FuncState *fs, TValue *key, TValue *v) {
+ TValue val;
lua_State *L = fs->ls->L;
Proto *f = fs->f;
- TValue *idx = luaH_set(L, fs->ls->h, key); /* index scanner table */
+ const TValue *idx = luaH_get(fs->ls->h, key); /* query scanner table */
int k, oldsize;
if (ttisinteger(idx)) { /* is there an index there? */
k = cast_int(ivalue(idx));
@@ -563,7 +558,8 @@ static int addk (FuncState *fs, TValue *key, TValue *v) {
k = fs->nk;
/* numerical value does not need GC barrier;
table has no metatable, so it does not need to invalidate cache */
- setivalue(idx, k);
+ setivalue(&val, k);
+ luaH_finishset(L, fs->ls->h, key, idx, &val);
luaM_growvector(L, f->k, k, f->sizek, TValue, MAXARG_Ax, "constants");
while (oldsize < f->sizek) setnilvalue(&f->k[oldsize++]);
setobj(L, &f->k[k], v);
@@ -585,24 +581,41 @@ static int stringK (FuncState *fs, TString *s) {
/*
** Add an integer to list of constants and return its index.
-** Integers use userdata as keys to avoid collision with floats with
-** same value; conversion to 'void*' is used only for hashing, so there
-** are no "precision" problems.
*/
static int luaK_intK (FuncState *fs, lua_Integer n) {
- TValue k, o;
- setpvalue(&k, cast_voidp(cast_sizet(n)));
+ TValue o;
setivalue(&o, n);
- return addk(fs, &k, &o);
+ return addk(fs, &o, &o); /* use integer itself as key */
}
/*
-** Add a float to list of constants and return its index.
+** Add a float to list of constants and return its index. Floats
+** with integral values need a different key, to avoid collision
+** with actual integers. To that, we add to the number its smaller
+** power-of-two fraction that is still significant in its scale.
+** For doubles, that would be 1/2^52.
+** (This method is not bulletproof: there may be another float
+** with that value, and for floats larger than 2^53 the result is
+** still an integer. At worst, this only wastes an entry with
+** a duplicate.)
*/
static int luaK_numberK (FuncState *fs, lua_Number r) {
TValue o;
+ lua_Integer ik;
setfltvalue(&o, r);
- return addk(fs, &o, &o); /* use number itself as key */
+ if (!luaV_flttointeger(r, &ik, F2Ieq)) /* not an integral value? */
+ return addk(fs, &o, &o); /* use number itself as key */
+ else { /* must build an alternative key */
+ const int nbm = l_floatatt(MANT_DIG);
+ const lua_Number q = l_mathop(ldexp)(l_mathop(1.0), -nbm + 1);
+ const lua_Number k = (ik == 0) ? q : r + r*q; /* new key */
+ TValue kv;
+ setfltvalue(&kv, k);
+ /* result is not an integral value, unless value is too large */
+ lua_assert(!luaV_flttointeger(k, &ik, F2Ieq) ||
+ l_mathop(fabs)(r) >= l_mathop(1e6));
+ return addk(fs, &kv, &o);
+ }
}
@@ -753,7 +766,7 @@ void luaK_setoneret (FuncState *fs, expdesc *e) {
/*
-** Ensure that expression 'e' is not a variable (nor a constant).
+** Ensure that expression 'e' is not a variable (nor a ).
** (Expression still may have jump lists.)
*/
void luaK_dischargevars (FuncState *fs, expdesc *e) {
@@ -763,7 +776,7 @@ void luaK_dischargevars (FuncState *fs, expdesc *e) {
break;
}
case VLOCAL: { /* already in a register */
- e->u.info = e->u.var.sidx;
+ e->u.info = e->u.var.ridx;
e->k = VNONRELOC; /* becomes a non-relocatable value */
break;
}
@@ -805,8 +818,8 @@ void luaK_dischargevars (FuncState *fs, expdesc *e) {
/*
-** Ensures expression value is in register 'reg' (and therefore
-** 'e' will become a non-relocatable expression).
+** Ensure expression value is in register 'reg', making 'e' a
+** non-relocatable expression.
** (Expression still may have jump lists.)
*/
static void discharge2reg (FuncState *fs, expdesc *e, int reg) {
@@ -860,7 +873,8 @@ static void discharge2reg (FuncState *fs, expdesc *e, int reg) {
/*
-** Ensures expression value is in any register.
+** Ensure expression value is in a register, making 'e' a
+** non-relocatable expression.
** (Expression still may have jump lists.)
*/
static void discharge2anyreg (FuncState *fs, expdesc *e) {
@@ -946,8 +960,11 @@ int luaK_exp2anyreg (FuncState *fs, expdesc *e) {
exp2reg(fs, e, e->u.info); /* put final result in it */
return e->u.info;
}
+ /* else expression has jumps and cannot change its register
+ to hold the jump values, because it is a local variable.
+ Go through to the default case. */
}
- luaK_exp2nextreg(fs, e); /* otherwise, use next available register */
+ luaK_exp2nextreg(fs, e); /* default: use next available register */
return e->u.info;
}
@@ -1032,7 +1049,7 @@ void luaK_storevar (FuncState *fs, expdesc *var, expdesc *ex) {
switch (var->k) {
case VLOCAL: {
freeexp(fs, ex);
- exp2reg(fs, ex, var->u.var.sidx); /* compute 'ex' into proper place */
+ exp2reg(fs, ex, var->u.var.ridx); /* compute 'ex' into proper place */
return;
}
case VUPVAL: {
@@ -1272,7 +1289,7 @@ void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k) {
}
else {
/* register index of the table */
- t->u.ind.t = (t->k == VLOCAL) ? t->u.var.sidx: t->u.info;
+ t->u.ind.t = (t->k == VLOCAL) ? t->u.var.ridx: t->u.info;
if (isKstr(fs, k)) {
t->u.ind.idx = k->u.info; /* literal string */
t->k = VINDEXSTR;
@@ -1299,7 +1316,8 @@ static int validop (int op, TValue *v1, TValue *v2) {
case LUA_OPBAND: case LUA_OPBOR: case LUA_OPBXOR:
case LUA_OPSHL: case LUA_OPSHR: case LUA_OPBNOT: { /* conversion errors */
lua_Integer i;
- return (tointegerns(v1, &i) && tointegerns(v2, &i));
+ return (luaV_tointegerns(v1, &i, LUA_FLOORN2I) &&
+ luaV_tointegerns(v2, &i, LUA_FLOORN2I));
}
case LUA_OPDIV: case LUA_OPIDIV: case LUA_OPMOD: /* division by 0 */
return (nvalue(v2) != 0);
diff --git a/src/lua/lcorolib.c b/src/lua/lcorolib.c
index c165031d..785a1e81 100644
--- a/src/lua/lcorolib.c
+++ b/src/lua/lcorolib.c
@@ -31,14 +31,14 @@ static lua_State *getco (lua_State *L) {
*/
static int auxresume (lua_State *L, lua_State *co, int narg) {
int status, nres;
- if (!lua_checkstack(co, narg)) {
+ if (l_unlikely(!lua_checkstack(co, narg))) {
lua_pushliteral(L, "too many arguments to resume");
return -1; /* error flag */
}
lua_xmove(L, co, narg);
status = lua_resume(co, L, narg, &nres);
- if (status == LUA_OK || status == LUA_YIELD) {
- if (!lua_checkstack(L, nres + 1)) {
+ if (l_likely(status == LUA_OK || status == LUA_YIELD)) {
+ if (l_unlikely(!lua_checkstack(L, nres + 1))) {
lua_pop(co, nres); /* remove results anyway */
lua_pushliteral(L, "too many results to resume");
return -1; /* error flag */
@@ -57,7 +57,7 @@ static int luaB_coresume (lua_State *L) {
lua_State *co = getco(L);
int r;
r = auxresume(L, co, lua_gettop(L) - 1);
- if (r < 0) {
+ if (l_unlikely(r < 0)) {
lua_pushboolean(L, 0);
lua_insert(L, -2);
return 2; /* return false + error message */
@@ -73,10 +73,13 @@ static int luaB_coresume (lua_State *L) {
static int luaB_auxwrap (lua_State *L) {
lua_State *co = lua_tothread(L, lua_upvalueindex(1));
int r = auxresume(L, co, lua_gettop(L));
- if (r < 0) { /* error? */
+ if (l_unlikely(r < 0)) { /* error? */
int stat = lua_status(co);
- if (stat != LUA_OK && stat != LUA_YIELD) /* error in the coroutine? */
- lua_resetthread(co); /* close its tbc variables */
+ if (stat != LUA_OK && stat != LUA_YIELD) { /* error in the coroutine? */
+ stat = lua_resetthread(co); /* close its tbc variables */
+ lua_assert(stat != LUA_OK);
+ lua_xmove(co, L, 1); /* move error message to the caller */
+ }
if (stat != LUA_ERRMEM && /* not a memory error and ... */
lua_type(L, -1) == LUA_TSTRING) { /* ... error object is a string? */
luaL_where(L, 1); /* add extra info, if available */
@@ -176,7 +179,7 @@ static int luaB_close (lua_State *L) {
}
else {
lua_pushboolean(L, 0);
- lua_xmove(co, L, 1); /* copy error message */
+ lua_xmove(co, L, 1); /* move error message */
return 2;
}
}
diff --git a/src/lua/ldblib.c b/src/lua/ldblib.c
index 59eb8f0e..6dcbaa98 100644
--- a/src/lua/ldblib.c
+++ b/src/lua/ldblib.c
@@ -33,7 +33,7 @@ static const char *const HOOKKEY = "_HOOKKEY";
** checked.
*/
static void checkstack (lua_State *L, lua_State *L1, int n) {
- if (L != L1 && !lua_checkstack(L1, n))
+ if (l_unlikely(L != L1 && !lua_checkstack(L1, n)))
luaL_error(L, "stack overflow");
}
@@ -152,6 +152,7 @@ static int db_getinfo (lua_State *L) {
lua_State *L1 = getthread(L, &arg);
const char *options = luaL_optstring(L, arg+2, "flnSrtu");
checkstack(L, L1, 3);
+ luaL_argcheck(L, options[0] != '>', arg + 2, "invalid option '>'");
if (lua_isfunction(L, arg + 1)) { /* info about a function? */
options = lua_pushfstring(L, ">%s", options); /* add '>' to 'options' */
lua_pushvalue(L, arg + 1); /* move function to 'L1' stack */
@@ -212,7 +213,7 @@ static int db_getlocal (lua_State *L) {
lua_Debug ar;
const char *name;
int level = (int)luaL_checkinteger(L, arg + 1);
- if (!lua_getstack(L1, level, &ar)) /* out of range? */
+ if (l_unlikely(!lua_getstack(L1, level, &ar))) /* out of range? */
return luaL_argerror(L, arg+1, "level out of range");
checkstack(L, L1, 1);
name = lua_getlocal(L1, &ar, nvar);
@@ -237,7 +238,7 @@ static int db_setlocal (lua_State *L) {
lua_Debug ar;
int level = (int)luaL_checkinteger(L, arg + 1);
int nvar = (int)luaL_checkinteger(L, arg + 2);
- if (!lua_getstack(L1, level, &ar)) /* out of range? */
+ if (l_unlikely(!lua_getstack(L1, level, &ar))) /* out of range? */
return luaL_argerror(L, arg+1, "level out of range");
luaL_checkany(L, arg+3);
lua_settop(L, arg+3);
@@ -281,25 +282,33 @@ static int db_setupvalue (lua_State *L) {
** Check whether a given upvalue from a given closure exists and
** returns its index
*/
-static int checkupval (lua_State *L, int argf, int argnup) {
+static void *checkupval (lua_State *L, int argf, int argnup, int *pnup) {
+ void *id;
int nup = (int)luaL_checkinteger(L, argnup); /* upvalue index */
luaL_checktype(L, argf, LUA_TFUNCTION); /* closure */
- luaL_argcheck(L, (lua_getupvalue(L, argf, nup) != NULL), argnup,
- "invalid upvalue index");
- return nup;
+ id = lua_upvalueid(L, argf, nup);
+ if (pnup) {
+ luaL_argcheck(L, id != NULL, argnup, "invalid upvalue index");
+ *pnup = nup;
+ }
+ return id;
}
static int db_upvalueid (lua_State *L) {
- int n = checkupval(L, 1, 2);
- lua_pushlightuserdata(L, lua_upvalueid(L, 1, n));
+ void *id = checkupval(L, 1, 2, NULL);
+ if (id != NULL)
+ lua_pushlightuserdata(L, id);
+ else
+ luaL_pushfail(L);
return 1;
}
static int db_upvaluejoin (lua_State *L) {
- int n1 = checkupval(L, 1, 2);
- int n2 = checkupval(L, 3, 4);
+ int n1, n2;
+ checkupval(L, 1, 2, &n1);
+ checkupval(L, 3, 4, &n2);
luaL_argcheck(L, !lua_iscfunction(L, 1), 1, "Lua function expected");
luaL_argcheck(L, !lua_iscfunction(L, 3), 3, "Lua function expected");
lua_upvaluejoin(L, 1, n1, 3, n2);
@@ -369,7 +378,7 @@ static int db_sethook (lua_State *L) {
}
if (!luaL_getsubtable(L, LUA_REGISTRYINDEX, HOOKKEY)) {
/* table just created; initialize it */
- lua_pushstring(L, "k");
+ lua_pushliteral(L, "k");
lua_setfield(L, -2, "__mode"); /** hooktable.__mode = "k" */
lua_pushvalue(L, -1);
lua_setmetatable(L, -2); /* metatable(hooktable) = hooktable */
@@ -412,7 +421,7 @@ static int db_debug (lua_State *L) {
for (;;) {
char buffer[250];
lua_writestringerror("%s", "lua_debug> ");
- if (fgets(buffer, sizeof(buffer), stdin) == 0 ||
+ if (fgets(buffer, sizeof(buffer), stdin) == NULL ||
strcmp(buffer, "cont\n") == 0)
return 0;
if (luaL_loadbuffer(L, buffer, strlen(buffer), "=(debug command)") ||
@@ -440,10 +449,7 @@ static int db_traceback (lua_State *L) {
static int db_setcstacklimit (lua_State *L) {
int limit = (int)luaL_checkinteger(L, 1);
int res = lua_setcstacklimit(L, limit);
- if (res == 0)
- lua_pushboolean(L, 0);
- else
- lua_pushinteger(L, res);
+ lua_pushinteger(L, res);
return 1;
}
diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c
index 8cb00e51..a716d95e 100644
--- a/src/lua/ldebug.c
+++ b/src/lua/ldebug.c
@@ -33,11 +33,9 @@
#define noLuaClosure(f) ((f) == NULL || (f)->c.tt == LUA_VCCL)
-/* inverse of 'pcRel' */
-#define invpcRel(pc, p) ((p)->code + (pc) + 1)
-static const char *funcnamefromcode (lua_State *L, CallInfo *ci,
- const char **name);
+static const char *funcnamefromcall (lua_State *L, CallInfo *ci,
+ const char **name);
static int currentpc (CallInfo *ci) {
@@ -48,10 +46,16 @@ static int currentpc (CallInfo *ci) {
/*
** Get a "base line" to find the line corresponding to an instruction.
-** For that, search the array of absolute line info for the largest saved
-** instruction smaller or equal to the wanted instruction. A special
-** case is when there is no absolute info or the instruction is before
-** the first absolute one.
+** Base lines are regularly placed at MAXIWTHABS intervals, so usually
+** an integer division gets the right place. When the source file has
+** large sequences of empty/comment lines, it may need extra entries,
+** so the original estimate needs a correction.
+** If the original estimate is -1, the initial 'if' ensures that the
+** 'while' will run at least once.
+** The assertion that the estimate is a lower bound for the correct base
+** is valid as long as the debug info has been generated with the same
+** value for MAXIWTHABS or smaller. (Previous releases use a little
+** smaller value.)
*/
static int getbaseline (const Proto *f, int pc, int *basepc) {
if (f->sizeabslineinfo == 0 || pc < f->abslineinfo[0].pc) {
@@ -59,20 +63,12 @@ static int getbaseline (const Proto *f, int pc, int *basepc) {
return f->linedefined;
}
else {
- unsigned int i;
- if (pc >= f->abslineinfo[f->sizeabslineinfo - 1].pc)
- i = f->sizeabslineinfo - 1; /* instruction is after last saved one */
- else { /* binary search */
- unsigned int j = f->sizeabslineinfo - 1; /* pc < anchorlines[j] */
- i = 0; /* abslineinfo[i] <= pc */
- while (i < j - 1) {
- unsigned int m = (j + i) / 2;
- if (pc >= f->abslineinfo[m].pc)
- i = m;
- else
- j = m;
- }
- }
+ int i = cast_uint(pc) / MAXIWTHABS - 1; /* get an estimate */
+ /* estimate must be a lower bound of the correct base */
+ lua_assert(i < 0 ||
+ (i < f->sizeabslineinfo && f->abslineinfo[i].pc <= pc));
+ while (i + 1 < f->sizeabslineinfo && pc >= f->abslineinfo[i + 1].pc)
+ i++; /* low estimate; adjust it */
*basepc = f->abslineinfo[i].pc;
return f->abslineinfo[i].line;
}
@@ -305,8 +301,15 @@ static void collectvalidlines (lua_State *L, Closure *f) {
sethvalue2s(L, L->top, t); /* push it on stack */
api_incr_top(L);
setbtvalue(&v); /* boolean 'true' to be the value of all indices */
- for (i = 0; i < p->sizelineinfo; i++) { /* for all lines with code */
- currentline = nextline(p, currentline, i);
+ if (!p->is_vararg) /* regular function? */
+ i = 0; /* consider all instructions */
+ else { /* vararg function */
+ lua_assert(GET_OPCODE(p->code[0]) == OP_VARARGPREP);
+ currentline = nextline(p, currentline, 0);
+ i = 1; /* skip first instruction (OP_VARARGPREP) */
+ }
+ for (; i < p->sizelineinfo; i++) { /* for each instruction */
+ currentline = nextline(p, currentline, i); /* get its line */
luaH_setint(L, t, currentline, &v); /* table[line] = true */
}
}
@@ -314,15 +317,9 @@ static void collectvalidlines (lua_State *L, Closure *f) {
static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name) {
- if (ci == NULL) /* no 'ci'? */
- return NULL; /* no info */
- else if (ci->callstatus & CIST_FIN) { /* is this a finalizer? */
- *name = "__gc";
- return "metamethod"; /* report it as such */
- }
- /* calling function is a known Lua function? */
- else if (!(ci->callstatus & CIST_TAIL) && isLua(ci->previous))
- return funcnamefromcode(L, ci->previous, name);
+ /* calling function is a known function? */
+ if (ci != NULL && !(ci->callstatus & CIST_TAIL))
+ return funcnamefromcall(L, ci->previous, name);
else return NULL; /* no way to find a name */
}
@@ -594,16 +591,10 @@ static const char *getobjname (const Proto *p, int lastpc, int reg,
** Returns what the name is (e.g., "for iterator", "method",
** "metamethod") and sets '*name' to point to the name.
*/
-static const char *funcnamefromcode (lua_State *L, CallInfo *ci,
- const char **name) {
+static const char *funcnamefromcode (lua_State *L, const Proto *p,
+ int pc, const char **name) {
TMS tm = (TMS)0; /* (initial value avoids warnings) */
- const Proto *p = ci_func(ci)->p; /* calling function */
- int pc = currentpc(ci); /* calling instruction index */
Instruction i = p->code[pc]; /* calling instruction */
- if (ci->callstatus & CIST_HOOKED) { /* was it called inside a hook? */
- *name = "?";
- return "hook";
- }
switch (GET_OPCODE(i)) {
case OP_CALL:
case OP_TAILCALL:
@@ -629,12 +620,10 @@ static const char *funcnamefromcode (lua_State *L, CallInfo *ci,
case OP_LEN: tm = TM_LEN; break;
case OP_CONCAT: tm = TM_CONCAT; break;
case OP_EQ: tm = TM_EQ; break;
- case OP_LT: case OP_LE: case OP_LTI: case OP_LEI:
- *name = "order"; /* '<=' can call '__lt', etc. */
- return "metamethod";
- case OP_CLOSE: case OP_RETURN:
- *name = "close";
- return "metamethod";
+ /* no cases for OP_EQI and OP_EQK, as they don't call metamethods */
+ case OP_LT: case OP_LTI: case OP_GTI: tm = TM_LT; break;
+ case OP_LE: case OP_LEI: case OP_GEI: tm = TM_LE; break;
+ case OP_CLOSE: case OP_RETURN: tm = TM_CLOSE; break;
default:
return NULL; /* cannot find a reasonable name */
}
@@ -642,19 +631,43 @@ static const char *funcnamefromcode (lua_State *L, CallInfo *ci,
return "metamethod";
}
+
+/*
+** Try to find a name for a function based on how it was called.
+*/
+static const char *funcnamefromcall (lua_State *L, CallInfo *ci,
+ const char **name) {
+ if (ci->callstatus & CIST_HOOKED) { /* was it called inside a hook? */
+ *name = "?";
+ return "hook";
+ }
+ else if (ci->callstatus & CIST_FIN) { /* was it called as a finalizer? */
+ *name = "__gc";
+ return "metamethod"; /* report it as such */
+ }
+ else if (isLua(ci))
+ return funcnamefromcode(L, ci_func(ci)->p, currentpc(ci), name);
+ else
+ return NULL;
+}
+
/* }====================================================== */
/*
-** The subtraction of two potentially unrelated pointers is
-** not ISO C, but it should not crash a program; the subsequent
-** checks are ISO C and ensure a correct result.
+** Check whether pointer 'o' points to some value in the stack
+** frame of the current function. Because 'o' may not point to a
+** value in this stack, we cannot compare it with the region
+** boundaries (undefined behaviour in ISO C).
*/
static int isinstack (CallInfo *ci, const TValue *o) {
- StkId base = ci->func + 1;
- ptrdiff_t i = cast(StkId, o) - base;
- return (0 <= i && i < (ci->top - base) && s2v(base + i) == o);
+ StkId pos;
+ for (pos = ci->func + 1; pos < ci->top; pos++) {
+ if (o == s2v(pos))
+ return 1;
+ }
+ return 0; /* not found */
}
@@ -677,9 +690,21 @@ static const char *getupvalname (CallInfo *ci, const TValue *o,
}
+static const char *formatvarinfo (lua_State *L, const char *kind,
+ const char *name) {
+ if (kind == NULL)
+ return ""; /* no information */
+ else
+ return luaO_pushfstring(L, " (%s '%s')", kind, name);
+}
+
+/*
+** Build a string with a "description" for the value 'o', such as
+** "variable 'x'" or "upvalue 'y'".
+*/
static const char *varinfo (lua_State *L, const TValue *o) {
- const char *name = NULL; /* to avoid warnings */
CallInfo *ci = L->ci;
+ const char *name = NULL; /* to avoid warnings */
const char *kind = NULL;
if (isLua(ci)) {
kind = getupvalname(ci, o, &name); /* check whether 'o' is an upvalue */
@@ -687,13 +712,40 @@ static const char *varinfo (lua_State *L, const TValue *o) {
kind = getobjname(ci_func(ci)->p, currentpc(ci),
cast_int(cast(StkId, o) - (ci->func + 1)), &name);
}
- return (kind) ? luaO_pushfstring(L, " (%s '%s')", kind, name) : "";
+ return formatvarinfo(L, kind, name);
}
-l_noret luaG_typeerror (lua_State *L, const TValue *o, const char *op) {
+/*
+** Raise a type error
+*/
+static l_noret typeerror (lua_State *L, const TValue *o, const char *op,
+ const char *extra) {
const char *t = luaT_objtypename(L, o);
- luaG_runerror(L, "attempt to %s a %s value%s", op, t, varinfo(L, o));
+ luaG_runerror(L, "attempt to %s a %s value%s", op, t, extra);
+}
+
+
+/*
+** Raise a type error with "standard" information about the faulty
+** object 'o' (using 'varinfo').
+*/
+l_noret luaG_typeerror (lua_State *L, const TValue *o, const char *op) {
+ typeerror(L, o, op, varinfo(L, o));
+}
+
+
+/*
+** Raise an error for calling a non-callable object. Try to find a name
+** for the object based on how it was called ('funcnamefromcall'); if it
+** cannot get a name there, try 'varinfo'.
+*/
+l_noret luaG_callerror (lua_State *L, const TValue *o) {
+ CallInfo *ci = L->ci;
+ const char *name = NULL; /* to avoid warnings */
+ const char *kind = funcnamefromcall(L, ci, &name);
+ const char *extra = kind ? formatvarinfo(L, kind, name) : varinfo(L, o);
+ typeerror(L, o, "call", extra);
}
@@ -722,7 +774,7 @@ l_noret luaG_opinterror (lua_State *L, const TValue *p1,
*/
l_noret luaG_tointerror (lua_State *L, const TValue *p1, const TValue *p2) {
lua_Integer temp;
- if (!tointegerns(p1, &temp))
+ if (!luaV_tointegerns(p1, &temp, LUA_FLOORN2I))
p2 = p1;
luaG_runerror(L, "number%s has no integer representation", varinfo(L, p2));
}
@@ -780,16 +832,30 @@ l_noret luaG_runerror (lua_State *L, const char *fmt, ...) {
/*
** Check whether new instruction 'newpc' is in a different line from
-** previous instruction 'oldpc'.
+** previous instruction 'oldpc'. More often than not, 'newpc' is only
+** one or a few instructions after 'oldpc' (it must be after, see
+** caller), so try to avoid calling 'luaG_getfuncline'. If they are
+** too far apart, there is a good chance of a ABSLINEINFO in the way,
+** so it goes directly to 'luaG_getfuncline'.
*/
static int changedline (const Proto *p, int oldpc, int newpc) {
if (p->lineinfo == NULL) /* no debug information? */
return 0;
- while (oldpc++ < newpc) {
- if (p->lineinfo[oldpc] != 0)
- return (luaG_getfuncline(p, oldpc - 1) != luaG_getfuncline(p, newpc));
+ if (newpc - oldpc < MAXIWTHABS / 2) { /* not too far apart? */
+ int delta = 0; /* line diference */
+ int pc = oldpc;
+ for (;;) {
+ int lineinfo = p->lineinfo[++pc];
+ if (lineinfo == ABSLINEINFO)
+ break; /* cannot compute delta; fall through */
+ delta += lineinfo;
+ if (pc == newpc)
+ return (delta != 0); /* delta computed successfully */
+ }
}
- return 0; /* no line changes between positions */
+ /* either instructions are too far apart or there is an absolute line
+ info in the way; compute line difference explicitly */
+ return (luaG_getfuncline(p, oldpc) != luaG_getfuncline(p, newpc));
}
@@ -797,20 +863,19 @@ static int changedline (const Proto *p, int oldpc, int newpc) {
** Traces the execution of a Lua function. Called before the execution
** of each opcode, when debug is on. 'L->oldpc' stores the last
** instruction traced, to detect line changes. When entering a new
-** function, 'npci' will be zero and will test as a new line without
-** the need for 'oldpc'; so, 'oldpc' does not need to be initialized
-** before. Some exceptional conditions may return to a function without
-** updating 'oldpc'. In that case, 'oldpc' may be invalid; if so, it is
-** reset to zero. (A wrong but valid 'oldpc' at most causes an extra
-** call to a line hook.)
+** function, 'npci' will be zero and will test as a new line whatever
+** the value of 'oldpc'. Some exceptional conditions may return to
+** a function without setting 'oldpc'. In that case, 'oldpc' may be
+** invalid; if so, use zero as a valid value. (A wrong but valid 'oldpc'
+** at most causes an extra call to a line hook.)
+** This function is not "Protected" when called, so it should correct
+** 'L->top' before calling anything that can run the GC.
*/
int luaG_traceexec (lua_State *L, const Instruction *pc) {
CallInfo *ci = L->ci;
lu_byte mask = L->hookmask;
const Proto *p = ci_func(ci)->p;
int counthook;
- /* 'L->oldpc' may be invalid; reset it in this case */
- int oldpc = (L->oldpc < p->sizecode) ? L->oldpc : 0;
if (!(mask & (LUA_MASKLINE | LUA_MASKCOUNT))) { /* no hooks? */
ci->u.l.trap = 0; /* don't need to stop again */
return 0; /* turn off 'trap' */
@@ -826,15 +891,16 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) {
ci->callstatus &= ~CIST_HOOKYIELD; /* erase mark */
return 1; /* do not call hook again (VM yielded, so it did not move) */
}
- if (!isIT(*(ci->u.l.savedpc - 1)))
- L->top = ci->top; /* prepare top */
+ if (!isIT(*(ci->u.l.savedpc - 1))) /* top not being used? */
+ L->top = ci->top; /* correct top */
if (counthook)
luaD_hook(L, LUA_HOOKCOUNT, -1, 0, 0); /* call count hook */
if (mask & LUA_MASKLINE) {
+ /* 'L->oldpc' may be invalid; use zero in this case */
+ int oldpc = (L->oldpc < p->sizecode) ? L->oldpc : 0;
int npci = pcRel(pc, p);
- if (npci == 0 || /* call linehook when enter a new function, */
- pc <= invpcRel(oldpc, p) || /* when jump back (loop), or when */
- changedline(p, oldpc, npci)) { /* enter new line */
+ if (npci <= oldpc || /* call hook when jump back (loop), */
+ changedline(p, oldpc, npci)) { /* or when enter new line */
int newline = luaG_getfuncline(p, npci);
luaD_hook(L, LUA_HOOKLINE, newline, 0, 0); /* call line hook */
}
diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h
index a0a58486..974960e9 100644
--- a/src/lua/ldebug.h
+++ b/src/lua/ldebug.h
@@ -26,11 +26,22 @@
*/
#define ABSLINEINFO (-0x80)
+
+/*
+** MAXimum number of successive Instructions WiTHout ABSolute line
+** information. (A power of two allows fast divisions.)
+*/
+#if !defined(MAXIWTHABS)
+#define MAXIWTHABS 128
+#endif
+
+
LUAI_FUNC int luaG_getfuncline (const Proto *f, int pc);
LUAI_FUNC const char *luaG_findlocal (lua_State *L, CallInfo *ci, int n,
StkId *pos);
LUAI_FUNC l_noret luaG_typeerror (lua_State *L, const TValue *o,
const char *opname);
+LUAI_FUNC l_noret luaG_callerror (lua_State *L, const TValue *o);
LUAI_FUNC l_noret luaG_forerror (lua_State *L, const TValue *o,
const char *what);
LUAI_FUNC l_noret luaG_concaterror (lua_State *L, const TValue *p1,
diff --git a/src/lua/ldo.c b/src/lua/ldo.c
index 5473815a..a48e35f9 100644
--- a/src/lua/ldo.c
+++ b/src/lua/ldo.c
@@ -98,11 +98,12 @@ void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop) {
setsvalue2s(L, oldtop, luaS_newliteral(L, "error in error handling"));
break;
}
- case CLOSEPROTECT: {
+ case LUA_OK: { /* special case only for closing upvalues */
setnilvalue(s2v(oldtop)); /* no error message */
break;
}
default: {
+ lua_assert(errorstatus(errcode)); /* real error */
setobjs2s(L, oldtop, L->top - 1); /* error message on current top */
break;
}
@@ -118,17 +119,13 @@ l_noret luaD_throw (lua_State *L, int errcode) {
}
else { /* thread has no error handler */
global_State *g = G(L);
- errcode = luaF_close(L, L->stack, errcode); /* close all upvalues */
- L->status = cast_byte(errcode); /* mark it as dead */
+ errcode = luaE_resetthread(L, errcode); /* close all upvalues */
if (g->mainthread->errorJmp) { /* main thread has a handler? */
setobjs2s(L, g->mainthread->top++, L->top - 1); /* copy error obj. */
luaD_throw(g->mainthread, errcode); /* re-throw in main thread */
}
else { /* no handler at all; abort */
if (g->panic) { /* panic function? */
- luaD_seterrorobj(L, errcode, L->top); /* assume EXTRA_STACK */
- if (L->ci->top < L->top)
- L->ci->top = L->top; /* pushing msg. can break this invariant */
lua_unlock(L);
g->panic(L); /* call panic function (last chance to jump out) */
}
@@ -139,8 +136,7 @@ l_noret luaD_throw (lua_State *L, int errcode) {
int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) {
- global_State *g = G(L);
- l_uint32 oldnCcalls = g->Cstacklimit - (L->nCcalls + L->nci);
+ l_uint32 oldnCcalls = L->nCcalls;
struct lua_longjmp lj;
lj.status = LUA_OK;
lj.previous = L->errorJmp; /* chain new error handler */
@@ -149,7 +145,7 @@ int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) {
(*f)(L, ud);
);
L->errorJmp = lj.previous; /* restore old error handler */
- L->nCcalls = g->Cstacklimit - oldnCcalls - L->nci;
+ L->nCcalls = oldnCcalls;
return lj.status;
}
@@ -164,9 +160,8 @@ int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) {
static void correctstack (lua_State *L, StkId oldstack, StkId newstack) {
CallInfo *ci;
UpVal *up;
- if (oldstack == newstack)
- return; /* stack address did not change */
L->top = (L->top - oldstack) + newstack;
+ L->tbclist = (L->tbclist - oldstack) + newstack;
for (up = L->openupval; up != NULL; up = up->u.open.next)
up->v = s2v((uplevel(up) - oldstack) + newstack);
for (ci = L->ci; ci != NULL; ci = ci->previous) {
@@ -182,22 +177,37 @@ static void correctstack (lua_State *L, StkId oldstack, StkId newstack) {
#define ERRORSTACKSIZE (LUAI_MAXSTACK + 200)
+/*
+** Reallocate the stack to a new size, correcting all pointers into
+** it. (There are pointers to a stack from its upvalues, from its list
+** of call infos, plus a few individual pointers.) The reallocation is
+** done in two steps (allocation + free) because the correction must be
+** done while both addresses (the old stack and the new one) are valid.
+** (In ISO C, any pointer use after the pointer has been deallocated is
+** undefined behavior.)
+** In case of allocation error, raise an error or return false according
+** to 'raiseerror'.
+*/
int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) {
- int lim = L->stacksize;
- StkId newstack = luaM_reallocvector(L, L->stack, lim, newsize, StackValue);
+ int oldsize = stacksize(L);
+ int i;
+ StkId newstack = luaM_reallocvector(L, NULL, 0,
+ newsize + EXTRA_STACK, StackValue);
lua_assert(newsize <= LUAI_MAXSTACK || newsize == ERRORSTACKSIZE);
- lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK);
- if (unlikely(newstack == NULL)) { /* reallocation failed? */
+ if (l_unlikely(newstack == NULL)) { /* reallocation failed? */
if (raiseerror)
luaM_error(L);
else return 0; /* do not raise an error */
}
- for (; lim < newsize; lim++)
- setnilvalue(s2v(newstack + lim)); /* erase new segment */
+ /* number of elements to be copied to the new stack */
+ i = ((oldsize <= newsize) ? oldsize : newsize) + EXTRA_STACK;
+ memcpy(newstack, L->stack, i * sizeof(StackValue));
+ for (; i < newsize + EXTRA_STACK; i++)
+ setnilvalue(s2v(newstack + i)); /* erase new segment */
correctstack(L, L->stack, newstack);
+ luaM_freearray(L, L->stack, oldsize + EXTRA_STACK);
L->stack = newstack;
- L->stacksize = newsize;
- L->stack_last = L->stack + newsize - EXTRA_STACK;
+ L->stack_last = L->stack + newsize;
return 1;
}
@@ -207,51 +217,73 @@ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) {
** is true, raises any error; otherwise, return 0 in case of errors.
*/
int luaD_growstack (lua_State *L, int n, int raiseerror) {
- int size = L->stacksize;
- int newsize = 2 * size; /* tentative new size */
- if (unlikely(size > LUAI_MAXSTACK)) { /* need more space after extra size? */
+ int size = stacksize(L);
+ if (l_unlikely(size > LUAI_MAXSTACK)) {
+ /* if stack is larger than maximum, thread is already using the
+ extra space reserved for errors, that is, thread is handling
+ a stack error; cannot grow further than that. */
+ lua_assert(stacksize(L) == ERRORSTACKSIZE);
if (raiseerror)
luaD_throw(L, LUA_ERRERR); /* error inside message handler */
- else return 0;
+ return 0; /* if not 'raiseerror', just signal it */
}
else {
- int needed = cast_int(L->top - L->stack) + n + EXTRA_STACK;
+ int newsize = 2 * size; /* tentative new size */
+ int needed = cast_int(L->top - L->stack) + n;
if (newsize > LUAI_MAXSTACK) /* cannot cross the limit */
newsize = LUAI_MAXSTACK;
if (newsize < needed) /* but must respect what was asked for */
newsize = needed;
- if (unlikely(newsize > LUAI_MAXSTACK)) { /* stack overflow? */
+ if (l_likely(newsize <= LUAI_MAXSTACK))
+ return luaD_reallocstack(L, newsize, raiseerror);
+ else { /* stack overflow */
/* add extra size to be able to handle the error message */
luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror);
if (raiseerror)
luaG_runerror(L, "stack overflow");
- else return 0;
+ return 0;
}
- } /* else no errors */
- return luaD_reallocstack(L, newsize, raiseerror);
+ }
}
static int stackinuse (lua_State *L) {
CallInfo *ci;
+ int res;
StkId lim = L->top;
for (ci = L->ci; ci != NULL; ci = ci->previous) {
if (lim < ci->top) lim = ci->top;
}
lua_assert(lim <= L->stack_last);
- return cast_int(lim - L->stack) + 1; /* part of stack in use */
+ res = cast_int(lim - L->stack) + 1; /* part of stack in use */
+ if (res < LUA_MINSTACK)
+ res = LUA_MINSTACK; /* ensure a minimum size */
+ return res;
}
+/*
+** If stack size is more than 3 times the current use, reduce that size
+** to twice the current use. (So, the final stack size is at most 2/3 the
+** previous size, and half of its entries are empty.)
+** As a particular case, if stack was handling a stack overflow and now
+** it is not, 'max' (limited by LUAI_MAXSTACK) will be smaller than
+** stacksize (equal to ERRORSTACKSIZE in this case), and so the stack
+** will be reduced to a "regular" size.
+*/
void luaD_shrinkstack (lua_State *L) {
int inuse = stackinuse(L);
- int goodsize = inuse + BASIC_STACK_SIZE;
- if (goodsize > LUAI_MAXSTACK)
- goodsize = LUAI_MAXSTACK; /* respect stack limit */
+ int nsize = inuse * 2; /* proposed new size */
+ int max = inuse * 3; /* maximum "reasonable" size */
+ if (max > LUAI_MAXSTACK) {
+ max = LUAI_MAXSTACK; /* respect stack limit */
+ if (nsize > LUAI_MAXSTACK)
+ nsize = LUAI_MAXSTACK;
+ }
/* if thread is currently not handling a stack overflow and its
- good size is smaller than current size, shrink its stack */
- if (inuse <= (LUAI_MAXSTACK - EXTRA_STACK) && goodsize < L->stacksize)
- luaD_reallocstack(L, goodsize, 0); /* ok if that fails */
+ size is larger than maximum "reasonable" size, shrink it */
+ if (inuse <= LUAI_MAXSTACK && stacksize(L) > max)
+ luaD_reallocstack(L, nsize, 0); /* ok if that fails */
else /* don't change stack */
condmovestack(L,{},{}); /* (change only for debugging) */
luaE_shrinkCI(L); /* shrink CI list */
@@ -277,8 +309,8 @@ void luaD_hook (lua_State *L, int event, int line,
if (hook && L->allowhook) { /* make sure there is a hook */
int mask = CIST_HOOKED;
CallInfo *ci = L->ci;
- ptrdiff_t top = savestack(L, L->top);
- ptrdiff_t ci_top = savestack(L, ci->top);
+ ptrdiff_t top = savestack(L, L->top); /* preserve original 'top' */
+ ptrdiff_t ci_top = savestack(L, ci->top); /* idem for 'ci->top' */
lua_Debug ar;
ar.event = event;
ar.currentline = line;
@@ -288,8 +320,10 @@ void luaD_hook (lua_State *L, int event, int line,
ci->u2.transferinfo.ftransfer = ftransfer;
ci->u2.transferinfo.ntransfer = ntransfer;
}
+ if (isLua(ci) && L->top < ci->top)
+ L->top = ci->top; /* protect entire activation register */
luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */
- if (L->top + LUA_MINSTACK > ci->top)
+ if (ci->top < L->top + LUA_MINSTACK)
ci->top = L->top + LUA_MINSTACK;
L->allowhook = 0; /* cannot call hooks inside a hook */
ci->callstatus |= mask;
@@ -311,55 +345,60 @@ void luaD_hook (lua_State *L, int event, int line,
** active.
*/
void luaD_hookcall (lua_State *L, CallInfo *ci) {
- int hook = (ci->callstatus & CIST_TAIL) ? LUA_HOOKTAILCALL : LUA_HOOKCALL;
- Proto *p;
- if (!(L->hookmask & LUA_MASKCALL)) /* some other hook? */
- return; /* don't call hook */
- p = clLvalue(s2v(ci->func))->p;
- L->top = ci->top; /* prepare top */
- ci->u.l.savedpc++; /* hooks assume 'pc' is already incremented */
- luaD_hook(L, hook, -1, 1, p->numparams);
- ci->u.l.savedpc--; /* correct 'pc' */
+ L->oldpc = 0; /* set 'oldpc' for new function */
+ if (L->hookmask & LUA_MASKCALL) { /* is call hook on? */
+ int event = (ci->callstatus & CIST_TAIL) ? LUA_HOOKTAILCALL
+ : LUA_HOOKCALL;
+ Proto *p = ci_func(ci)->p;
+ ci->u.l.savedpc++; /* hooks assume 'pc' is already incremented */
+ luaD_hook(L, event, -1, 1, p->numparams);
+ ci->u.l.savedpc--; /* correct 'pc' */
+ }
}
-static StkId rethook (lua_State *L, CallInfo *ci, StkId firstres, int nres) {
- ptrdiff_t oldtop = savestack(L, L->top); /* hook may change top */
- int delta = 0;
- if (isLuacode(ci)) {
- Proto *p = ci_func(ci)->p;
- if (p->is_vararg)
- delta = ci->u.l.nextraargs + p->numparams + 1;
- if (L->top < ci->top)
- L->top = ci->top; /* correct top to run hook */
- }
+/*
+** Executes a return hook for Lua and C functions and sets/corrects
+** 'oldpc'. (Note that this correction is needed by the line hook, so it
+** is done even when return hooks are off.)
+*/
+static void rethook (lua_State *L, CallInfo *ci, int nres) {
if (L->hookmask & LUA_MASKRET) { /* is return hook on? */
+ StkId firstres = L->top - nres; /* index of first result */
+ int delta = 0; /* correction for vararg functions */
int ftransfer;
+ if (isLua(ci)) {
+ Proto *p = ci_func(ci)->p;
+ if (p->is_vararg)
+ delta = ci->u.l.nextraargs + p->numparams + 1;
+ }
ci->func += delta; /* if vararg, back to virtual 'func' */
ftransfer = cast(unsigned short, firstres - ci->func);
luaD_hook(L, LUA_HOOKRET, -1, ftransfer, nres); /* call it */
ci->func -= delta;
}
if (isLua(ci = ci->previous))
- L->oldpc = pcRel(ci->u.l.savedpc, ci_func(ci)->p); /* update 'oldpc' */
- return restorestack(L, oldtop);
+ L->oldpc = pcRel(ci->u.l.savedpc, ci_func(ci)->p); /* set 'oldpc' */
}
/*
** Check whether 'func' has a '__call' metafield. If so, put it in the
-** stack, below original 'func', so that 'luaD_call' can call it. Raise
+** stack, below original 'func', so that 'luaD_precall' can call it. Raise
** an error if there is no '__call' metafield.
*/
-void luaD_tryfuncTM (lua_State *L, StkId func) {
- const TValue *tm = luaT_gettmbyobj(L, s2v(func), TM_CALL);
+StkId luaD_tryfuncTM (lua_State *L, StkId func) {
+ const TValue *tm;
StkId p;
- if (unlikely(ttisnil(tm)))
- luaG_typeerror(L, s2v(func), "call"); /* nothing to call */
+ checkstackGCp(L, 1, func); /* space for metamethod */
+ tm = luaT_gettmbyobj(L, s2v(func), TM_CALL); /* (after previous GC) */
+ if (l_unlikely(ttisnil(tm)))
+ luaG_callerror(L, s2v(func)); /* nothing to call */
for (p = L->top; p > func; p--) /* open space for metamethod */
setobjs2s(L, p, p-1);
L->top++; /* stack space pre-allocated by the caller */
setobj2s(L, func, tm); /* metamethod is the new function to be called */
+ return func;
}
@@ -369,7 +408,7 @@ void luaD_tryfuncTM (lua_State *L, StkId func) {
** expressions, multiple results for tail calls/single parameters)
** separated.
*/
-static void moveresults (lua_State *L, StkId res, int nres, int wanted) {
+l_sinline void moveresults (lua_State *L, StkId res, int nres, int wanted) {
StkId firstresult;
int i;
switch (wanted) { /* handle typical cases separately */
@@ -379,27 +418,34 @@ static void moveresults (lua_State *L, StkId res, int nres, int wanted) {
case 1: /* one value needed */
if (nres == 0) /* no results? */
setnilvalue(s2v(res)); /* adjust with nil */
- else
+ else /* at least one result */
setobjs2s(L, res, L->top - nres); /* move it to proper place */
L->top = res + 1;
return;
case LUA_MULTRET:
wanted = nres; /* we want all results */
break;
- default: /* multiple results (or to-be-closed variables) */
+ default: /* two/more results and/or to-be-closed variables */
if (hastocloseCfunc(wanted)) { /* to-be-closed variables? */
ptrdiff_t savedres = savestack(L, res);
- luaF_close(L, res, LUA_OK); /* may change the stack */
- res = restorestack(L, savedres);
- wanted = codeNresults(wanted); /* correct value */
+ L->ci->callstatus |= CIST_CLSRET; /* in case of yields */
+ L->ci->u2.nres = nres;
+ luaF_close(L, res, CLOSEKTOP, 1);
+ L->ci->callstatus &= ~CIST_CLSRET;
+ if (L->hookmask) /* if needed, call hook after '__close's */
+ rethook(L, L->ci, nres);
+ res = restorestack(L, savedres); /* close and hook can move stack */
+ wanted = decodeNresults(wanted);
if (wanted == LUA_MULTRET)
- wanted = nres;
+ wanted = nres; /* we want all results */
}
break;
}
+ /* generic case */
firstresult = L->top - nres; /* index of first result */
- /* move all results to correct place */
- for (i = 0; i < nres && i < wanted; i++)
+ if (nres > wanted) /* extra results? */
+ nres = wanted; /* don't need them */
+ for (i = 0; i < nres; i++) /* move all results to correct place */
setobjs2s(L, res + i, firstresult + i);
for (; i < wanted; i++) /* complete wanted number of results */
setnilvalue(s2v(res + i));
@@ -408,15 +454,21 @@ static void moveresults (lua_State *L, StkId res, int nres, int wanted) {
/*
-** Finishes a function call: calls hook if necessary, removes CallInfo,
-** moves current number of results to proper place.
+** Finishes a function call: calls hook if necessary, moves current
+** number of results to proper place, and returns to previous call
+** info. If function has to close variables, hook must be called after
+** that.
*/
void luaD_poscall (lua_State *L, CallInfo *ci, int nres) {
- if (L->hookmask)
- L->top = rethook(L, ci, L->top - nres, nres);
- L->ci = ci->previous; /* back to caller */
+ int wanted = ci->nresults;
+ if (l_unlikely(L->hookmask && !hastocloseCfunc(wanted)))
+ rethook(L, ci, nres);
/* move results to proper place */
- moveresults(L, ci->func, nres, ci->nresults);
+ moveresults(L, ci->func, nres, wanted);
+ /* function cannot be in any of these cases when returning */
+ lua_assert(!(ci->callstatus &
+ (CIST_HOOKED | CIST_YPCALL | CIST_FIN | CIST_TRAN | CIST_CLSRET)));
+ L->ci = ci->previous; /* back to caller (after closing variables) */
}
@@ -424,66 +476,101 @@ void luaD_poscall (lua_State *L, CallInfo *ci, int nres) {
#define next_ci(L) (L->ci->next ? L->ci->next : luaE_extendCI(L))
-/*
-** Prepare a function for a tail call, building its call info on top
-** of the current call info. 'narg1' is the number of arguments plus 1
-** (so that it includes the function itself).
-*/
-void luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int narg1) {
- Proto *p = clLvalue(s2v(func))->p;
- int fsize = p->maxstacksize; /* frame size */
- int nfixparams = p->numparams;
- int i;
- for (i = 0; i < narg1; i++) /* move down function and arguments */
- setobjs2s(L, ci->func + i, func + i);
- checkstackGC(L, fsize);
- func = ci->func; /* moved-down function */
- for (; narg1 <= nfixparams; narg1++)
- setnilvalue(s2v(func + narg1)); /* complete missing arguments */
- ci->top = func + 1 + fsize; /* top for new function */
- lua_assert(ci->top <= L->stack_last);
- ci->u.l.savedpc = p->code; /* starting point */
- ci->callstatus |= CIST_TAIL;
- L->top = func + narg1; /* set top */
+l_sinline CallInfo *prepCallInfo (lua_State *L, StkId func, int nret,
+ int mask, StkId top) {
+ CallInfo *ci = L->ci = next_ci(L); /* new frame */
+ ci->func = func;
+ ci->nresults = nret;
+ ci->callstatus = mask;
+ ci->top = top;
+ return ci;
}
/*
-** Call a function (C or Lua). The function to be called is at *func.
-** The arguments are on the stack, right after the function.
-** When returns, all the results are on the stack, starting at the original
-** function position.
+** precall for C functions
*/
-void luaD_call (lua_State *L, StkId func, int nresults) {
- lua_CFunction f;
+l_sinline int precallC (lua_State *L, StkId func, int nresults,
+ lua_CFunction f) {
+ int n; /* number of returns */
+ CallInfo *ci;
+ checkstackGCp(L, LUA_MINSTACK, func); /* ensure minimum stack size */
+ L->ci = ci = prepCallInfo(L, func, nresults, CIST_C,
+ L->top + LUA_MINSTACK);
+ lua_assert(ci->top <= L->stack_last);
+ if (l_unlikely(L->hookmask & LUA_MASKCALL)) {
+ int narg = cast_int(L->top - func) - 1;
+ luaD_hook(L, LUA_HOOKCALL, -1, 1, narg);
+ }
+ lua_unlock(L);
+ n = (*f)(L); /* do the actual call */
+ lua_lock(L);
+ api_checknelems(L, n);
+ luaD_poscall(L, ci, n);
+ return n;
+}
+
+
+/*
+** Prepare a function for a tail call, building its call info on top
+** of the current call info. 'narg1' is the number of arguments plus 1
+** (so that it includes the function itself). Return the number of
+** results, if it was a C function, or -1 for a Lua function.
+*/
+int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func,
+ int narg1, int delta) {
retry:
switch (ttypetag(s2v(func))) {
case LUA_VCCL: /* C closure */
- f = clCvalue(s2v(func))->f;
- goto Cfunc;
+ return precallC(L, func, LUA_MULTRET, clCvalue(s2v(func))->f);
case LUA_VLCF: /* light C function */
- f = fvalue(s2v(func));
- Cfunc: {
- int n; /* number of returns */
- CallInfo *ci;
- checkstackGCp(L, LUA_MINSTACK, func); /* ensure minimum stack size */
- L->ci = ci = next_ci(L);
- ci->nresults = nresults;
- ci->callstatus = CIST_C;
- ci->top = L->top + LUA_MINSTACK;
- ci->func = func;
+ return precallC(L, func, LUA_MULTRET, fvalue(s2v(func)));
+ case LUA_VLCL: { /* Lua function */
+ Proto *p = clLvalue(s2v(func))->p;
+ int fsize = p->maxstacksize; /* frame size */
+ int nfixparams = p->numparams;
+ int i;
+ checkstackGCp(L, fsize - delta, func);
+ ci->func -= delta; /* restore 'func' (if vararg) */
+ for (i = 0; i < narg1; i++) /* move down function and arguments */
+ setobjs2s(L, ci->func + i, func + i);
+ func = ci->func; /* moved-down function */
+ for (; narg1 <= nfixparams; narg1++)
+ setnilvalue(s2v(func + narg1)); /* complete missing arguments */
+ ci->top = func + 1 + fsize; /* top for new function */
lua_assert(ci->top <= L->stack_last);
- if (L->hookmask & LUA_MASKCALL) {
- int narg = cast_int(L->top - func) - 1;
- luaD_hook(L, LUA_HOOKCALL, -1, 1, narg);
- }
- lua_unlock(L);
- n = (*f)(L); /* do the actual call */
- lua_lock(L);
- api_checknelems(L, n);
- luaD_poscall(L, ci, n);
- break;
+ ci->u.l.savedpc = p->code; /* starting point */
+ ci->callstatus |= CIST_TAIL;
+ L->top = func + narg1; /* set top */
+ return -1;
}
+ default: { /* not a function */
+ func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */
+ /* return luaD_pretailcall(L, ci, func, narg1 + 1, delta); */
+ narg1++;
+ goto retry; /* try again */
+ }
+ }
+}
+
+
+/*
+** Prepares the call to a function (C or Lua). For C functions, also do
+** the call. The function to be called is at '*func'. The arguments
+** are on the stack, right after the function. Returns the CallInfo
+** to be executed, if it was a Lua function. Otherwise (a C function)
+** returns NULL, with all the results on the stack, starting at the
+** original function position.
+*/
+CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) {
+ retry:
+ switch (ttypetag(s2v(func))) {
+ case LUA_VCCL: /* C closure */
+ precallC(L, func, nresults, clCvalue(s2v(func))->f);
+ return NULL;
+ case LUA_VLCF: /* light C function */
+ precallC(L, func, nresults, fvalue(s2v(func)));
+ return NULL;
case LUA_VLCL: { /* Lua function */
CallInfo *ci;
Proto *p = clLvalue(s2v(func))->p;
@@ -491,22 +578,16 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
int nfixparams = p->numparams;
int fsize = p->maxstacksize; /* frame size */
checkstackGCp(L, fsize, func);
- L->ci = ci = next_ci(L);
- ci->nresults = nresults;
+ L->ci = ci = prepCallInfo(L, func, nresults, 0, func + 1 + fsize);
ci->u.l.savedpc = p->code; /* starting point */
- ci->callstatus = 0;
- ci->top = func + 1 + fsize;
- ci->func = func;
- L->ci = ci;
for (; narg < nfixparams; narg++)
setnilvalue(s2v(L->top++)); /* complete missing arguments */
lua_assert(ci->top <= L->stack_last);
- luaV_execute(L, ci); /* run the function */
- break;
+ return ci;
}
default: { /* not a function */
- checkstackGCp(L, 1, func); /* space for metamethod */
- luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */
+ func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */
+ /* return luaD_precall(L, func, nresults); */
goto retry; /* try again with metamethod */
}
}
@@ -514,41 +595,108 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
/*
-** Similar to 'luaD_call', but does not allow yields during the call.
+** Call a function (C or Lua) through C. 'inc' can be 1 (increment
+** number of recursive invocations in the C stack) or nyci (the same
+** plus increment number of non-yieldable calls).
*/
-void luaD_callnoyield (lua_State *L, StkId func, int nResults) {
- incXCcalls(L);
- if (getCcalls(L) <= CSTACKERR) { /* possible C stack overflow? */
- luaE_exitCcall(L); /* to compensate decrement in next call */
- luaE_enterCcall(L); /* check properly */
+l_sinline void ccall (lua_State *L, StkId func, int nResults, int inc) {
+ CallInfo *ci;
+ L->nCcalls += inc;
+ if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS))
+ luaE_checkcstack(L);
+ if ((ci = luaD_precall(L, func, nResults)) != NULL) { /* Lua function? */
+ ci->callstatus = CIST_FRESH; /* mark that it is a "fresh" execute */
+ luaV_execute(L, ci); /* call it */
}
- luaD_call(L, func, nResults);
- decXCcalls(L);
+ L->nCcalls -= inc;
}
/*
-** Completes the execution of an interrupted C function, calling its
-** continuation function.
+** External interface for 'ccall'
*/
-static void finishCcall (lua_State *L, int status) {
- CallInfo *ci = L->ci;
- int n;
- /* must have a continuation and must be able to call it */
- lua_assert(ci->u.c.k != NULL && yieldable(L));
- /* error status can only happen in a protected call */
- lua_assert((ci->callstatus & CIST_YPCALL) || status == LUA_YIELD);
- if (ci->callstatus & CIST_YPCALL) { /* was inside a pcall? */
- ci->callstatus &= ~CIST_YPCALL; /* continuation is also inside it */
- L->errfunc = ci->u.c.old_errfunc; /* with the same error function */
+void luaD_call (lua_State *L, StkId func, int nResults) {
+ ccall(L, func, nResults, 1);
+}
+
+
+/*
+** Similar to 'luaD_call', but does not allow yields during the call.
+*/
+void luaD_callnoyield (lua_State *L, StkId func, int nResults) {
+ ccall(L, func, nResults, nyci);
+}
+
+
+/*
+** Finish the job of 'lua_pcallk' after it was interrupted by an yield.
+** (The caller, 'finishCcall', does the final call to 'adjustresults'.)
+** The main job is to complete the 'luaD_pcall' called by 'lua_pcallk'.
+** If a '__close' method yields here, eventually control will be back
+** to 'finishCcall' (when that '__close' method finally returns) and
+** 'finishpcallk' will run again and close any still pending '__close'
+** methods. Similarly, if a '__close' method errs, 'precover' calls
+** 'unroll' which calls ''finishCcall' and we are back here again, to
+** close any pending '__close' methods.
+** Note that, up to the call to 'luaF_close', the corresponding
+** 'CallInfo' is not modified, so that this repeated run works like the
+** first one (except that it has at least one less '__close' to do). In
+** particular, field CIST_RECST preserves the error status across these
+** multiple runs, changing only if there is a new error.
+*/
+static int finishpcallk (lua_State *L, CallInfo *ci) {
+ int status = getcistrecst(ci); /* get original status */
+ if (l_likely(status == LUA_OK)) /* no error? */
+ status = LUA_YIELD; /* was interrupted by an yield */
+ else { /* error */
+ StkId func = restorestack(L, ci->u2.funcidx);
+ L->allowhook = getoah(ci->callstatus); /* restore 'allowhook' */
+ luaF_close(L, func, status, 1); /* can yield or raise an error */
+ func = restorestack(L, ci->u2.funcidx); /* stack may be moved */
+ luaD_seterrorobj(L, status, func);
+ luaD_shrinkstack(L); /* restore stack size in case of overflow */
+ setcistrecst(ci, LUA_OK); /* clear original status */
+ }
+ ci->callstatus &= ~CIST_YPCALL;
+ L->errfunc = ci->u.c.old_errfunc;
+ /* if it is here, there were errors or yields; unlike 'lua_pcallk',
+ do not change status */
+ return status;
+}
+
+
+/*
+** Completes the execution of a C function interrupted by an yield.
+** The interruption must have happened while the function was either
+** closing its tbc variables in 'moveresults' or executing
+** 'lua_callk'/'lua_pcallk'. In the first case, it just redoes
+** 'luaD_poscall'. In the second case, the call to 'finishpcallk'
+** finishes the interrupted execution of 'lua_pcallk'. After that, it
+** calls the continuation of the interrupted function and finally it
+** completes the job of the 'luaD_call' that called the function. In
+** the call to 'adjustresults', we do not know the number of results
+** of the function called by 'lua_callk'/'lua_pcallk', so we are
+** conservative and use LUA_MULTRET (always adjust).
+*/
+static void finishCcall (lua_State *L, CallInfo *ci) {
+ int n; /* actual number of results from C function */
+ if (ci->callstatus & CIST_CLSRET) { /* was returning? */
+ lua_assert(hastocloseCfunc(ci->nresults));
+ n = ci->u2.nres; /* just redo 'luaD_poscall' */
+ /* don't need to reset CIST_CLSRET, as it will be set again anyway */
+ }
+ else {
+ int status = LUA_YIELD; /* default if there were no errors */
+ /* must have a continuation and must be able to call it */
+ lua_assert(ci->u.c.k != NULL && yieldable(L));
+ if (ci->callstatus & CIST_YPCALL) /* was inside a 'lua_pcallk'? */
+ status = finishpcallk(L, ci); /* finish it */
+ adjustresults(L, LUA_MULTRET); /* finish 'lua_callk' */
+ lua_unlock(L);
+ n = (*ci->u.c.k)(L, status, ci->u.c.ctx); /* call continuation */
+ lua_lock(L);
+ api_checknelems(L, n);
}
- /* finish 'lua_callk'/'lua_pcall'; CIST_YPCALL and 'errfunc' already
- handled */
- adjustresults(L, ci->nresults);
- lua_unlock(L);
- n = (*ci->u.c.k)(L, status, ci->u.c.ctx); /* call continuation function */
- lua_lock(L);
- api_checknelems(L, n);
luaD_poscall(L, ci, n); /* finish 'luaD_call' */
}
@@ -556,18 +704,14 @@ static void finishCcall (lua_State *L, int status) {
/*
** Executes "full continuation" (everything in the stack) of a
** previously interrupted coroutine until the stack is empty (or another
-** interruption long-jumps out of the loop). If the coroutine is
-** recovering from an error, 'ud' points to the error status, which must
-** be passed to the first continuation function (otherwise the default
-** status is LUA_YIELD).
+** interruption long-jumps out of the loop).
*/
static void unroll (lua_State *L, void *ud) {
CallInfo *ci;
- if (ud != NULL) /* error status? */
- finishCcall(L, *(int *)ud); /* finish 'lua_pcallk' callee */
+ UNUSED(ud);
while ((ci = L->ci) != &L->base_ci) { /* something in the stack */
if (!isLua(ci)) /* C function? */
- finishCcall(L, LUA_YIELD); /* complete its execution */
+ finishCcall(L, ci); /* complete its execution */
else { /* Lua function */
luaV_finishOp(L); /* finish interrupted instruction */
luaV_execute(L, ci); /* execute down to higher C 'boundary' */
@@ -590,28 +734,6 @@ static CallInfo *findpcall (lua_State *L) {
}
-/*
-** Recovers from an error in a coroutine. Finds a recover point (if
-** there is one) and completes the execution of the interrupted
-** 'luaD_pcall'. If there is no recover point, returns zero.
-*/
-static int recover (lua_State *L, int status) {
- StkId oldtop;
- CallInfo *ci = findpcall(L);
- if (ci == NULL) return 0; /* no recovery point */
- /* "finish" luaD_pcall */
- oldtop = restorestack(L, ci->u2.funcidx);
- luaF_close(L, oldtop, status); /* may change the stack */
- oldtop = restorestack(L, ci->u2.funcidx);
- luaD_seterrorobj(L, status, oldtop);
- L->ci = ci;
- L->allowhook = getoah(ci->callstatus); /* restore original 'allowhook' */
- luaD_shrinkstack(L);
- L->errfunc = ci->u.c.old_errfunc;
- return 1; /* continue running the coroutine */
-}
-
-
/*
** Signal an error in the call to 'lua_resume', not in the execution
** of the coroutine itself. (Such errors should not be handled by any
@@ -637,14 +759,15 @@ static void resume (lua_State *L, void *ud) {
int n = *(cast(int*, ud)); /* number of arguments */
StkId firstArg = L->top - n; /* first argument */
CallInfo *ci = L->ci;
- if (L->status == LUA_OK) { /* starting a coroutine? */
- luaD_call(L, firstArg - 1, LUA_MULTRET);
- }
+ if (L->status == LUA_OK) /* starting a coroutine? */
+ ccall(L, firstArg - 1, LUA_MULTRET, 0); /* just call its body */
else { /* resuming from previous yield */
lua_assert(L->status == LUA_YIELD);
L->status = LUA_OK; /* mark that it is running (again) */
- if (isLua(ci)) /* yielded inside a hook? */
+ if (isLua(ci)) { /* yielded inside a hook? */
+ L->top = firstArg; /* discard arguments */
luaV_execute(L, ci); /* just continue running Lua code */
+ }
else { /* 'common' yield */
if (ci->u.c.k != NULL) { /* does it have a continuation function? */
lua_unlock(L);
@@ -658,6 +781,26 @@ static void resume (lua_State *L, void *ud) {
}
}
+
+/*
+** Unrolls a coroutine in protected mode while there are recoverable
+** errors, that is, errors inside a protected call. (Any error
+** interrupts 'unroll', and this loop protects it again so it can
+** continue.) Stops with a normal end (status == LUA_OK), an yield
+** (status == LUA_YIELD), or an unprotected error ('findpcall' doesn't
+** find a recover point).
+*/
+static int precover (lua_State *L, int status) {
+ CallInfo *ci;
+ while (errorstatus(status) && (ci = findpcall(L)) != NULL) {
+ L->ci = ci; /* go down to recovery functions */
+ setcistrecst(ci, status); /* status to finish 'pcall' */
+ status = luaD_rawrunprotected(L, unroll, NULL);
+ }
+ return status;
+}
+
+
LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
int *nresults) {
int status;
@@ -670,21 +813,16 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
}
else if (L->status != LUA_YIELD) /* ended with errors? */
return resume_error(L, "cannot resume dead coroutine", nargs);
- if (from == NULL)
- L->nCcalls = CSTACKTHREAD;
- else /* correct 'nCcalls' for this thread */
- L->nCcalls = getCcalls(from) - L->nci - CSTACKCF;
- if (L->nCcalls <= CSTACKERR)
+ L->nCcalls = (from) ? getCcalls(from) : 0;
+ if (getCcalls(L) >= LUAI_MAXCCALLS)
return resume_error(L, "C stack overflow", nargs);
+ L->nCcalls++;
luai_userstateresume(L, nargs);
api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
status = luaD_rawrunprotected(L, resume, &nargs);
/* continue running after recoverable errors */
- while (errorstatus(status) && recover(L, status)) {
- /* unroll continuation */
- status = luaD_rawrunprotected(L, unroll, &status);
- }
- if (likely(!errorstatus(status)))
+ status = precover(L, status);
+ if (l_likely(!errorstatus(status)))
lua_assert(status == L->status); /* normal end or yield */
else { /* unrecoverable error */
L->status = cast_byte(status); /* mark thread as 'dead' */
@@ -710,22 +848,22 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
lua_lock(L);
ci = L->ci;
api_checknelems(L, nresults);
- if (unlikely(!yieldable(L))) {
+ if (l_unlikely(!yieldable(L))) {
if (L != G(L)->mainthread)
luaG_runerror(L, "attempt to yield across a C-call boundary");
else
luaG_runerror(L, "attempt to yield from outside a coroutine");
}
L->status = LUA_YIELD;
+ ci->u2.nyield = nresults; /* save number of results */
if (isLua(ci)) { /* inside a hook? */
lua_assert(!isLuacode(ci));
+ api_check(L, nresults == 0, "hooks cannot yield values");
api_check(L, k == NULL, "hooks cannot continue after yielding");
- ci->u2.nyield = 0; /* no results */
}
else {
if ((ci->u.c.k = k) != NULL) /* is there a continuation? */
ci->u.c.ctx = ctx; /* save context */
- ci->u2.nyield = nresults; /* save number of results */
luaD_throw(L, LUA_YIELD);
}
lua_assert(ci->callstatus & CIST_HOOKED); /* must be inside a hook */
@@ -734,6 +872,45 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
}
+/*
+** Auxiliary structure to call 'luaF_close' in protected mode.
+*/
+struct CloseP {
+ StkId level;
+ int status;
+};
+
+
+/*
+** Auxiliary function to call 'luaF_close' in protected mode.
+*/
+static void closepaux (lua_State *L, void *ud) {
+ struct CloseP *pcl = cast(struct CloseP *, ud);
+ luaF_close(L, pcl->level, pcl->status, 0);
+}
+
+
+/*
+** Calls 'luaF_close' in protected mode. Return the original status
+** or, in case of errors, the new status.
+*/
+int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status) {
+ CallInfo *old_ci = L->ci;
+ lu_byte old_allowhooks = L->allowhook;
+ for (;;) { /* keep closing upvalues until no more errors */
+ struct CloseP pcl;
+ pcl.level = restorestack(L, level); pcl.status = status;
+ status = luaD_rawrunprotected(L, &closepaux, &pcl);
+ if (l_likely(status == LUA_OK)) /* no more errors? */
+ return pcl.status;
+ else { /* an error occurred; restore saved state and repeat */
+ L->ci = old_ci;
+ L->allowhook = old_allowhooks;
+ }
+ }
+}
+
+
/*
** Call the C function 'func' in protected mode, restoring basic
** thread information ('allowhook', etc.) and in particular
@@ -747,14 +924,12 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
ptrdiff_t old_errfunc = L->errfunc;
L->errfunc = ef;
status = luaD_rawrunprotected(L, func, u);
- if (unlikely(status != LUA_OK)) { /* an error occurred? */
- StkId oldtop = restorestack(L, old_top);
+ if (l_unlikely(status != LUA_OK)) { /* an error occurred? */
L->ci = old_ci;
L->allowhook = old_allowhooks;
- status = luaF_close(L, oldtop, status);
- oldtop = restorestack(L, old_top); /* previous call may change stack */
- luaD_seterrorobj(L, status, oldtop);
- luaD_shrinkstack(L);
+ status = luaD_closeprotected(L, old_top, status);
+ luaD_seterrorobj(L, status, restorestack(L, old_top));
+ luaD_shrinkstack(L); /* restore stack size in case of overflow */
}
L->errfunc = old_errfunc;
return status;
diff --git a/src/lua/ldo.h b/src/lua/ldo.h
index 6c6cb285..911e67f6 100644
--- a/src/lua/ldo.h
+++ b/src/lua/ldo.h
@@ -23,7 +23,7 @@
** at every check.
*/
#define luaD_checkstackaux(L,n,pre,pos) \
- if (L->stack_last - L->top <= (n)) \
+ if (l_unlikely(L->stack_last - L->top <= (n))) \
{ pre; luaD_growstack(L, n, 1); pos; } \
else { condmovestack(L,pre,pos); }
@@ -58,10 +58,12 @@ LUAI_FUNC int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
LUAI_FUNC void luaD_hook (lua_State *L, int event, int line,
int fTransfer, int nTransfer);
LUAI_FUNC void luaD_hookcall (lua_State *L, CallInfo *ci);
-LUAI_FUNC void luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int n);
+LUAI_FUNC int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int narg1, int delta);
+LUAI_FUNC CallInfo *luaD_precall (lua_State *L, StkId func, int nResults);
LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults);
LUAI_FUNC void luaD_callnoyield (lua_State *L, StkId func, int nResults);
-LUAI_FUNC void luaD_tryfuncTM (lua_State *L, StkId func);
+LUAI_FUNC StkId luaD_tryfuncTM (lua_State *L, StkId func);
+LUAI_FUNC int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status);
LUAI_FUNC int luaD_pcall (lua_State *L, Pfunc func, void *u,
ptrdiff_t oldtop, ptrdiff_t ef);
LUAI_FUNC void luaD_poscall (lua_State *L, CallInfo *ci, int nres);
diff --git a/src/lua/lfunc.c b/src/lua/lfunc.c
index 88d45328..f5889a21 100644
--- a/src/lua/lfunc.c
+++ b/src/lua/lfunc.c
@@ -53,7 +53,7 @@ void luaF_initupvals (lua_State *L, LClosure *cl) {
uv->v = &uv->u.value; /* make it closed */
setnilvalue(uv->v);
cl->upvals[i] = uv;
- luaC_objbarrier(L, cl, o);
+ luaC_objbarrier(L, cl, uv);
}
}
@@ -100,115 +100,83 @@ UpVal *luaF_findupval (lua_State *L, StkId level) {
}
-static void callclose (lua_State *L, void *ud) {
- UNUSED(ud);
- luaD_callnoyield(L, L->top - 3, 0);
-}
-
-
/*
-** Prepare closing method plus its arguments for object 'obj' with
-** error message 'err'. (This function assumes EXTRA_STACK.)
+** Call closing method for object 'obj' with error message 'err'. The
+** boolean 'yy' controls whether the call is yieldable.
+** (This function assumes EXTRA_STACK.)
*/
-static int prepclosingmethod (lua_State *L, TValue *obj, TValue *err) {
+static void callclosemethod (lua_State *L, TValue *obj, TValue *err, int yy) {
StkId top = L->top;
const TValue *tm = luaT_gettmbyobj(L, obj, TM_CLOSE);
- if (ttisnil(tm)) /* no metamethod? */
- return 0; /* nothing to call */
setobj2s(L, top, tm); /* will call metamethod... */
setobj2s(L, top + 1, obj); /* with 'self' as the 1st argument */
setobj2s(L, top + 2, err); /* and error msg. as 2nd argument */
L->top = top + 3; /* add function and arguments */
- return 1;
+ if (yy)
+ luaD_call(L, top, 0);
+ else
+ luaD_callnoyield(L, top, 0);
}
/*
-** Raise an error with message 'msg', inserting the name of the
-** local variable at position 'level' in the stack.
+** Check whether object at given level has a close metamethod and raise
+** an error if not.
*/
-static void varerror (lua_State *L, StkId level, const char *msg) {
- int idx = cast_int(level - L->ci->func);
- const char *vname = luaG_findlocal(L, L->ci, idx, NULL);
- if (vname == NULL) vname = "?";
- luaG_runerror(L, msg, vname);
+static void checkclosemth (lua_State *L, StkId level) {
+ const TValue *tm = luaT_gettmbyobj(L, s2v(level), TM_CLOSE);
+ if (ttisnil(tm)) { /* no metamethod? */
+ int idx = cast_int(level - L->ci->func); /* variable index */
+ const char *vname = luaG_findlocal(L, L->ci, idx, NULL);
+ if (vname == NULL) vname = "?";
+ luaG_runerror(L, "variable '%s' got a non-closable value", vname);
+ }
}
/*
-** Prepare and call a closing method. If status is OK, code is still
-** inside the original protected call, and so any error will be handled
-** there. Otherwise, a previous error already activated the original
-** protected call, and so the call to the closing method must be
-** protected here. (A status == CLOSEPROTECT behaves like a previous
-** error, to also run the closing method in protected mode).
-** If status is OK, the call to the closing method will be pushed
-** at the top of the stack. Otherwise, values are pushed after
-** the 'level' of the upvalue being closed, as everything after
-** that won't be used again.
+** Prepare and call a closing method.
+** If status is CLOSEKTOP, the call to the closing method will be pushed
+** at the top of the stack. Otherwise, values can be pushed right after
+** the 'level' of the upvalue being closed, as everything after that
+** won't be used again.
*/
-static int callclosemth (lua_State *L, StkId level, int status) {
+static void prepcallclosemth (lua_State *L, StkId level, int status, int yy) {
TValue *uv = s2v(level); /* value being closed */
- if (likely(status == LUA_OK)) {
- if (prepclosingmethod(L, uv, &G(L)->nilvalue)) /* something to call? */
- callclose(L, NULL); /* call closing method */
- else if (!l_isfalse(uv)) /* non-closable non-false value? */
- varerror(L, level, "attempt to close non-closable variable '%s'");
+ TValue *errobj;
+ if (status == CLOSEKTOP)
+ errobj = &G(L)->nilvalue; /* error object is nil */
+ else { /* 'luaD_seterrorobj' will set top to level + 2 */
+ errobj = s2v(level + 1); /* error object goes after 'uv' */
+ luaD_seterrorobj(L, status, level + 1); /* set error object */
}
- else { /* must close the object in protected mode */
- ptrdiff_t oldtop;
- level++; /* space for error message */
- oldtop = savestack(L, level + 1); /* top will be after that */
- luaD_seterrorobj(L, status, level); /* set error message */
- if (prepclosingmethod(L, uv, s2v(level))) { /* something to call? */
- int newstatus = luaD_pcall(L, callclose, NULL, oldtop, 0);
- if (newstatus != LUA_OK && status == CLOSEPROTECT) /* first error? */
- status = newstatus; /* this will be the new error */
- else {
- if (newstatus != LUA_OK) /* suppressed error? */
- luaE_warnerror(L, "__close metamethod");
- /* leave original error (or nil) on top */
- L->top = restorestack(L, oldtop);
- }
- }
- /* else no metamethod; ignore this case and keep original error */
- }
- return status;
+ callclosemethod(L, uv, errobj, yy);
}
/*
-** Try to create a to-be-closed upvalue
-** (can raise a memory-allocation error)
+** Maximum value for deltas in 'tbclist', dependent on the type
+** of delta. (This macro assumes that an 'L' is in scope where it
+** is used.)
*/
-static void trynewtbcupval (lua_State *L, void *ud) {
- newupval(L, 1, cast(StkId, ud), &L->openupval);
-}
+#define MAXDELTA \
+ ((256ul << ((sizeof(L->stack->tbclist.delta) - 1) * 8)) - 1)
/*
-** Create a to-be-closed upvalue. If there is a memory error
-** when creating the upvalue, the closing method must be called here,
-** as there is no upvalue to call it later.
+** Insert a variable in the list of to-be-closed variables.
*/
void luaF_newtbcupval (lua_State *L, StkId level) {
- TValue *obj = s2v(level);
- lua_assert(L->openupval == NULL || uplevel(L->openupval) < level);
- if (!l_isfalse(obj)) { /* false doesn't need to be closed */
- int status;
- const TValue *tm = luaT_gettmbyobj(L, obj, TM_CLOSE);
- if (ttisnil(tm)) /* no metamethod? */
- varerror(L, level, "variable '%s' got a non-closable value");
- status = luaD_rawrunprotected(L, trynewtbcupval, level);
- if (unlikely(status != LUA_OK)) { /* memory error creating upvalue? */
- lua_assert(status == LUA_ERRMEM);
- luaD_seterrorobj(L, LUA_ERRMEM, level + 1); /* save error message */
- /* next call must succeed, as object is closable */
- prepclosingmethod(L, s2v(level), s2v(level + 1));
- callclose(L, NULL); /* call closing method */
- luaD_throw(L, LUA_ERRMEM); /* throw memory error */
- }
+ lua_assert(level > L->tbclist);
+ if (l_isfalse(s2v(level)))
+ return; /* false doesn't need to be closed */
+ checkclosemth(L, level); /* value must have a close method */
+ while (cast_uint(level - L->tbclist) > MAXDELTA) {
+ L->tbclist += MAXDELTA; /* create a dummy node at maximum delta */
+ L->tbclist->tbclist.delta = 0;
}
+ level->tbclist.delta = cast(unsigned short, level - L->tbclist);
+ L->tbclist = level;
}
@@ -220,18 +188,16 @@ void luaF_unlinkupval (UpVal *uv) {
}
-int luaF_close (lua_State *L, StkId level, int status) {
+/*
+** Close all upvalues up to the given stack level.
+*/
+void luaF_closeupval (lua_State *L, StkId level) {
UpVal *uv;
- while ((uv = L->openupval) != NULL && uplevel(uv) >= level) {
+ StkId upl; /* stack index pointed by 'uv' */
+ while ((uv = L->openupval) != NULL && (upl = uplevel(uv)) >= level) {
TValue *slot = &uv->u.value; /* new position for value */
lua_assert(uplevel(uv) < L->top);
- if (uv->tbc && status != NOCLOSINGMETH) {
- /* must run closing method, which may change the stack */
- ptrdiff_t levelrel = savestack(L, level);
- status = callclosemth(L, uplevel(uv), status);
- level = restorestack(L, levelrel);
- }
- luaF_unlinkupval(uv);
+ luaF_unlinkupval(uv); /* remove upvalue from 'openupval' list */
setobj(L, slot, uv->v); /* move value to upvalue slot */
uv->v = slot; /* now current value lives here */
if (!iswhite(uv)) { /* neither white nor dead? */
@@ -239,7 +205,35 @@ int luaF_close (lua_State *L, StkId level, int status) {
luaC_barrier(L, uv, slot);
}
}
- return status;
+}
+
+
+/*
+** Remove firt element from the tbclist plus its dummy nodes.
+*/
+static void poptbclist (lua_State *L) {
+ StkId tbc = L->tbclist;
+ lua_assert(tbc->tbclist.delta > 0); /* first element cannot be dummy */
+ tbc -= tbc->tbclist.delta;
+ while (tbc > L->stack && tbc->tbclist.delta == 0)
+ tbc -= MAXDELTA; /* remove dummy nodes */
+ L->tbclist = tbc;
+}
+
+
+/*
+** Close all upvalues and to-be-closed variables up to the given stack
+** level.
+*/
+void luaF_close (lua_State *L, StkId level, int status, int yy) {
+ ptrdiff_t levelrel = savestack(L, level);
+ luaF_closeupval(L, level); /* first, close the upvalues */
+ while (L->tbclist >= level) { /* traverse tbc's down to that level */
+ StkId tbc = L->tbclist; /* get variable index */
+ poptbclist(L); /* remove it from list */
+ prepcallclosemth(L, tbc, status, yy); /* close variable */
+ level = restorestack(L, levelrel);
+ }
}
diff --git a/src/lua/lfunc.h b/src/lua/lfunc.h
index 8d6f965c..dc1cebcc 100644
--- a/src/lua/lfunc.h
+++ b/src/lua/lfunc.h
@@ -42,15 +42,9 @@
#define MAXMISS 10
-/*
-** Special "status" for 'luaF_close'
-*/
-/* close upvalues without running their closing methods */
-#define NOCLOSINGMETH (-1)
-
-/* close upvalues running all closing methods in protected mode */
-#define CLOSEPROTECT (-2)
+/* special status to close upvalues preserving the top of the stack */
+#define CLOSEKTOP (-1)
LUAI_FUNC Proto *luaF_newproto (lua_State *L);
@@ -59,7 +53,8 @@ LUAI_FUNC LClosure *luaF_newLclosure (lua_State *L, int nupvals);
LUAI_FUNC void luaF_initupvals (lua_State *L, LClosure *cl);
LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level);
LUAI_FUNC void luaF_newtbcupval (lua_State *L, StkId level);
-LUAI_FUNC int luaF_close (lua_State *L, StkId level, int status);
+LUAI_FUNC void luaF_closeupval (lua_State *L, StkId level);
+LUAI_FUNC void luaF_close (lua_State *L, StkId level, int status, int yy);
LUAI_FUNC void luaF_unlinkupval (UpVal *uv);
LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f);
LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number,
diff --git a/src/lua/lgc.c b/src/lua/lgc.c
index 4a7bcaed..42a73d81 100644
--- a/src/lua/lgc.c
+++ b/src/lua/lgc.c
@@ -161,18 +161,17 @@ static void linkgclist_ (GCObject *o, GCObject **pnext, GCObject **list) {
/*
-** Clear keys for empty entries in tables. If entry is empty
-** and its key is not marked, mark its entry as dead. This allows the
-** collection of the key, but keeps its entry in the table (its removal
-** could break a chain). The main feature of a dead key is that it must
-** be different from any other value, to do not disturb searches.
-** Other places never manipulate dead keys, because its associated empty
-** value is enough to signal that the entry is logically empty.
+** Clear keys for empty entries in tables. If entry is empty, mark its
+** entry as dead. This allows the collection of the key, but keeps its
+** entry in the table: its removal could break a chain and could break
+** a table traversal. Other places never manipulate dead keys, because
+** its associated empty value is enough to signal that the entry is
+** logically empty.
*/
static void clearkey (Node *n) {
lua_assert(isempty(gval(n)));
- if (keyiswhite(n))
- setdeadkey(n); /* unused and unmarked key; remove it */
+ if (keyiscollectable(n))
+ setdeadkey(n); /* unused key; remove it */
}
@@ -301,7 +300,7 @@ static void reallymarkobject (global_State *g, GCObject *o) {
if (upisopen(uv))
set2gray(uv); /* open upvalues are kept gray */
else
- set2black(o); /* closed upvalues are visited here */
+ set2black(uv); /* closed upvalues are visited here */
markvalue(g, uv->v); /* mark its content */
break;
}
@@ -309,7 +308,7 @@ static void reallymarkobject (global_State *g, GCObject *o) {
Udata *u = gco2u(o);
if (u->nuvalue == 0) { /* no user values? */
markobjectN(g, u->metatable); /* mark its metatable */
- set2black(o); /* nothing else to mark */
+ set2black(u); /* nothing else to mark */
break;
}
/* else... */
@@ -633,9 +632,8 @@ static int traversethread (global_State *g, lua_State *th) {
for (uv = th->openupval; uv != NULL; uv = uv->u.open.next)
markobject(g, uv); /* open upvalues cannot be collected */
if (g->gcstate == GCSatomic) { /* final traversal? */
- StkId lim = th->stack + th->stacksize; /* real end of stack */
- for (; o < lim; o++) /* clear not-marked stack slice */
- setnilvalue(s2v(o));
+ for (; o < th->stack_last + EXTRA_STACK; o++)
+ setnilvalue(s2v(o)); /* clear dead stack slice */
/* 'remarkupvals' may have removed thread from 'twups' list */
if (!isintwups(th) && th->openupval != NULL) {
th->twups = g->twups; /* link it back to the list */
@@ -644,7 +642,7 @@ static int traversethread (global_State *g, lua_State *th) {
}
else if (!g->gcemergency)
luaD_shrinkstack(th); /* do not change stack in emergency cycle */
- return 1 + th->stacksize;
+ return 1 + stacksize(th);
}
@@ -771,12 +769,16 @@ static void freeobj (lua_State *L, GCObject *o) {
case LUA_VUPVAL:
freeupval(L, gco2upv(o));
break;
- case LUA_VLCL:
- luaM_freemem(L, o, sizeLclosure(gco2lcl(o)->nupvalues));
+ case LUA_VLCL: {
+ LClosure *cl = gco2lcl(o);
+ luaM_freemem(L, cl, sizeLclosure(cl->nupvalues));
break;
- case LUA_VCCL:
- luaM_freemem(L, o, sizeCclosure(gco2ccl(o)->nupvalues));
+ }
+ case LUA_VCCL: {
+ CClosure *cl = gco2ccl(o);
+ luaM_freemem(L, cl, sizeCclosure(cl->nupvalues));
break;
+ }
case LUA_VTABLE:
luaH_free(L, gco2t(o));
break;
@@ -788,13 +790,17 @@ static void freeobj (lua_State *L, GCObject *o) {
luaM_freemem(L, o, sizeudata(u->nuvalue, u->len));
break;
}
- case LUA_VSHRSTR:
- luaS_remove(L, gco2ts(o)); /* remove it from hash table */
- luaM_freemem(L, o, sizelstring(gco2ts(o)->shrlen));
+ case LUA_VSHRSTR: {
+ TString *ts = gco2ts(o);
+ luaS_remove(L, ts); /* remove it from hash table */
+ luaM_freemem(L, ts, sizelstring(ts->shrlen));
break;
- case LUA_VLNGSTR:
- luaM_freemem(L, o, sizelstring(gco2ts(o)->u.lnglen));
+ }
+ case LUA_VLNGSTR: {
+ TString *ts = gco2ts(o);
+ luaM_freemem(L, ts, sizelstring(ts->u.lnglen));
break;
+ }
default: lua_assert(0);
}
}
@@ -900,18 +906,18 @@ static void GCTM (lua_State *L) {
if (!notm(tm)) { /* is there a finalizer? */
int status;
lu_byte oldah = L->allowhook;
- int running = g->gcrunning;
+ int oldgcstp = g->gcstp;
+ g->gcstp |= GCSTPGC; /* avoid GC steps */
L->allowhook = 0; /* stop debug hooks during GC metamethod */
- g->gcrunning = 0; /* avoid GC steps */
setobj2s(L, L->top++, tm); /* push finalizer... */
setobj2s(L, L->top++, &v); /* ... and its argument */
L->ci->callstatus |= CIST_FIN; /* will run a finalizer */
status = luaD_pcall(L, dothecall, NULL, savestack(L, L->top - 2), 0);
L->ci->callstatus &= ~CIST_FIN; /* not running a finalizer anymore */
L->allowhook = oldah; /* restore hooks */
- g->gcrunning = running; /* restore state */
- if (unlikely(status != LUA_OK)) { /* error while running __gc? */
- luaE_warnerror(L, "__gc metamethod");
+ g->gcstp = oldgcstp; /* restore state */
+ if (l_unlikely(status != LUA_OK)) { /* error while running __gc? */
+ luaE_warnerror(L, "__gc");
L->top--; /* pops error object */
}
}
@@ -1005,7 +1011,8 @@ static void correctpointers (global_State *g, GCObject *o) {
void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt) {
global_State *g = G(L);
if (tofinalize(o) || /* obj. is already marked... */
- gfasttm(g, mt, TM_GC) == NULL) /* or has no finalizer? */
+ gfasttm(g, mt, TM_GC) == NULL || /* or has no finalizer... */
+ (g->gcstp & GCSTPCLS)) /* or closing state? */
return; /* nothing to be done */
else { /* move 'o' to 'finobj' list */
GCObject **p;
@@ -1496,12 +1503,13 @@ static void deletelist (lua_State *L, GCObject *p, GCObject *limit) {
*/
void luaC_freeallobjects (lua_State *L) {
global_State *g = G(L);
+ g->gcstp = GCSTPCLS; /* no extra finalizers after here */
luaC_changemode(L, KGC_INC);
separatetobefnz(g, 1); /* separate all objects with finalizers */
lua_assert(g->finobj == NULL);
callallpendingfinalizers(L);
deletelist(L, g->allgc, obj2gco(g->mainthread));
- deletelist(L, g->finobj, NULL);
+ lua_assert(g->finobj == NULL); /* no new finalizers */
deletelist(L, g->fixedgc, NULL); /* collect fixed objects */
lua_assert(g->strt.nuse == 0);
}
@@ -1569,52 +1577,64 @@ static int sweepstep (lua_State *L, global_State *g,
static lu_mem singlestep (lua_State *L) {
global_State *g = G(L);
+ lu_mem work;
+ lua_assert(!g->gcstopem); /* collector is not reentrant */
+ g->gcstopem = 1; /* no emergency collections while collecting */
switch (g->gcstate) {
case GCSpause: {
restartcollection(g);
g->gcstate = GCSpropagate;
- return 1;
+ work = 1;
+ break;
}
case GCSpropagate: {
if (g->gray == NULL) { /* no more gray objects? */
g->gcstate = GCSenteratomic; /* finish propagate phase */
- return 0;
+ work = 0;
}
else
- return propagatemark(g); /* traverse one gray object */
+ work = propagatemark(g); /* traverse one gray object */
+ break;
}
case GCSenteratomic: {
- lu_mem work = atomic(L); /* work is what was traversed by 'atomic' */
+ work = atomic(L); /* work is what was traversed by 'atomic' */
entersweep(L);
g->GCestimate = gettotalbytes(g); /* first estimate */;
- return work;
+ break;
}
case GCSswpallgc: { /* sweep "regular" objects */
- return sweepstep(L, g, GCSswpfinobj, &g->finobj);
+ work = sweepstep(L, g, GCSswpfinobj, &g->finobj);
+ break;
}
case GCSswpfinobj: { /* sweep objects with finalizers */
- return sweepstep(L, g, GCSswptobefnz, &g->tobefnz);
+ work = sweepstep(L, g, GCSswptobefnz, &g->tobefnz);
+ break;
}
case GCSswptobefnz: { /* sweep objects to be finalized */
- return sweepstep(L, g, GCSswpend, NULL);
+ work = sweepstep(L, g, GCSswpend, NULL);
+ break;
}
case GCSswpend: { /* finish sweeps */
checkSizes(L, g);
g->gcstate = GCScallfin;
- return 0;
+ work = 0;
+ break;
}
case GCScallfin: { /* call remaining finalizers */
if (g->tobefnz && !g->gcemergency) {
- int n = runafewfinalizers(L, GCFINMAX);
- return n * GCFINALIZECOST;
+ g->gcstopem = 0; /* ok collections during finalizers */
+ work = runafewfinalizers(L, GCFINMAX) * GCFINALIZECOST;
}
else { /* emergency mode or no more finalizers */
g->gcstate = GCSpause; /* finish collection */
- return 0;
+ work = 0;
}
+ break;
}
default: lua_assert(0); return 0;
}
+ g->gcstopem = 0;
+ return work;
}
@@ -1629,6 +1649,7 @@ void luaC_runtilstate (lua_State *L, int statesmask) {
}
+
/*
** Performs a basic incremental step. The debt and step size are
** converted from bytes to "units of work"; then the function loops
@@ -1660,7 +1681,7 @@ static void incstep (lua_State *L, global_State *g) {
void luaC_step (lua_State *L) {
global_State *g = G(L);
lua_assert(!g->gcemergency);
- if (g->gcrunning) { /* running? */
+ if (gcrunning(g)) { /* running? */
if(isdecGCmodegen(g))
genstep(L, g);
else
diff --git a/src/lua/lgc.h b/src/lua/lgc.h
index 073e2a40..4a125634 100644
--- a/src/lua/lgc.h
+++ b/src/lua/lgc.h
@@ -148,6 +148,16 @@
*/
#define isdecGCmodegen(g) (g->gckind == KGC_GEN || g->lastatomic != 0)
+
+/*
+** Control when GC is running:
+*/
+#define GCSTPUSR 1 /* bit true when GC stopped by user */
+#define GCSTPGC 2 /* bit true when GC stopped by itself */
+#define GCSTPCLS 4 /* bit true when closing Lua state */
+#define gcrunning(g) ((g)->gcstp == 0)
+
+
/*
** Does one step of collection when debt becomes positive. 'pre'/'pos'
** allows some adjustments to be done only when needed. macro
diff --git a/src/lua/linit.c b/src/lua/linit.c
index 446982ef..9a5bcfdc 100644
--- a/src/lua/linit.c
+++ b/src/lua/linit.c
@@ -50,9 +50,9 @@ static const luaL_Reg loadedlibs[] = {
{LUA_MATHLIBNAME, luaopen_math},
{LUA_UTF8LIBNAME, luaopen_utf8},
{LUA_DBLIBNAME, luaopen_debug},
- /************** Pi-hole modification ***************/
+ /****** Pi-hole modification ******/
{LUA_PIHOLELIBNAME, luaopen_pihole},
- /***************************************************/
+ /**********************************/
{NULL, NULL}
};
diff --git a/src/lua/liolib.c b/src/lua/liolib.c
index 60ab1bfa..b08397da 100644
--- a/src/lua/liolib.c
+++ b/src/lua/liolib.c
@@ -52,12 +52,6 @@ static int l_checkmode (const char *mode) {
** =======================================================
*/
-#if !defined(l_checkmodep)
-/* By default, Lua accepts only "r" or "w" as mode */
-#define l_checkmodep(m) ((m[0] == 'r' || m[0] == 'w') && m[1] == '\0')
-#endif
-
-
#if !defined(l_popen) /* { */
#if defined(LUA_USE_POSIX) /* { */
@@ -70,6 +64,12 @@ static int l_checkmode (const char *mode) {
#define l_popen(L,c,m) (_popen(c,m))
#define l_pclose(L,file) (_pclose(file))
+#if !defined(l_checkmodep)
+/* Windows accepts "[rw][bt]?" as valid modes */
+#define l_checkmodep(m) ((m[0] == 'r' || m[0] == 'w') && \
+ (m[1] == '\0' || ((m[1] == 'b' || m[1] == 't') && m[2] == '\0')))
+#endif
+
#else /* }{ */
/* ISO C definitions */
@@ -83,6 +83,12 @@ static int l_checkmode (const char *mode) {
#endif /* } */
+
+#if !defined(l_checkmodep)
+/* By default, Lua accepts only "r" or "w" as valid modes */
+#define l_checkmodep(m) ((m[0] == 'r' || m[0] == 'w') && m[1] == '\0')
+#endif
+
/* }====================================================== */
@@ -180,7 +186,7 @@ static int f_tostring (lua_State *L) {
static FILE *tofile (lua_State *L) {
LStream *p = tolstream(L);
- if (isclosed(p))
+ if (l_unlikely(isclosed(p)))
luaL_error(L, "attempt to use a closed file");
lua_assert(p->f);
return p->f;
@@ -255,7 +261,7 @@ static LStream *newfile (lua_State *L) {
static void opencheck (lua_State *L, const char *fname, const char *mode) {
LStream *p = newfile(L);
p->f = fopen(fname, mode);
- if (p->f == NULL)
+ if (l_unlikely(p->f == NULL))
luaL_error(L, "cannot open file '%s' (%s)", fname, strerror(errno));
}
@@ -303,7 +309,7 @@ static FILE *getiofile (lua_State *L, const char *findex) {
LStream *p;
lua_getfield(L, LUA_REGISTRYINDEX, findex);
p = (LStream *)lua_touserdata(L, -1);
- if (isclosed(p))
+ if (l_unlikely(isclosed(p)))
luaL_error(L, "default %s file is closed", findex + IOPREF_LEN);
return p->f;
}
@@ -430,7 +436,7 @@ typedef struct {
** Add current char to buffer (if not out of space) and read next one
*/
static int nextc (RN *rn) {
- if (rn->n >= L_MAXLENNUM) { /* buffer overflow? */
+ if (l_unlikely(rn->n >= L_MAXLENNUM)) { /* buffer overflow? */
rn->buff[0] = '\0'; /* invalidate result */
return 0; /* fail */
}
@@ -493,8 +499,8 @@ static int read_number (lua_State *L, FILE *f) {
ungetc(rn.c, rn.f); /* unread look-ahead char */
l_unlockfile(rn.f);
rn.buff[rn.n] = '\0'; /* finish string */
- if (lua_stringtonumber(L, rn.buff)) /* is this a valid number? */
- return 1; /* ok */
+ if (l_likely(lua_stringtonumber(L, rn.buff)))
+ return 1; /* ok, it is a valid number */
else { /* invalid format */
lua_pushnil(L); /* "result" to be removed */
return 0; /* read fails */
@@ -670,7 +676,8 @@ static int g_write (lua_State *L, FILE *f, int arg) {
status = status && (fwrite(s, sizeof(char), l, f) == l);
}
}
- if (status) return 1; /* file handle already on stack top */
+ if (l_likely(status))
+ return 1; /* file handle already on stack top */
else return luaL_fileresult(L, status, NULL);
}
@@ -697,7 +704,7 @@ static int f_seek (lua_State *L) {
luaL_argcheck(L, (lua_Integer)offset == p3, 3,
"not an integer in proper range");
op = l_fseek(f, offset, mode[op]);
- if (op)
+ if (l_unlikely(op))
return luaL_fileresult(L, 0, NULL); /* error */
else {
lua_pushinteger(L, (lua_Integer)l_ftell(f));
diff --git a/src/lua/llex.c b/src/lua/llex.c
index 3d6b2b97..e9915178 100644
--- a/src/lua/llex.c
+++ b/src/lua/llex.c
@@ -122,26 +122,29 @@ l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
/*
-** creates a new string and anchors it in scanner's table so that
-** it will not be collected until the end of the compilation
-** (by that time it should be anchored somewhere)
+** Creates a new string and anchors it in scanner's table so that it
+** will not be collected until the end of the compilation; by that time
+** it should be anchored somewhere. It also internalizes long strings,
+** ensuring there is only one copy of each unique string. The table
+** here is used as a set: the string enters as the key, while its value
+** is irrelevant. We use the string itself as the value only because it
+** is a TValue readly available. Later, the code generation can change
+** this value.
*/
TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
lua_State *L = ls->L;
- TValue *o; /* entry for 'str' */
TString *ts = luaS_newlstr(L, str, l); /* create new string */
- setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */
- o = luaH_set(L, ls->h, s2v(L->top - 1));
- if (isempty(o)) { /* not in use yet? */
- /* boolean value does not need GC barrier;
- table is not a metatable, so it does not need to invalidate cache */
- setbtvalue(o); /* t[string] = true */
+ const TValue *o = luaH_getstr(ls->h, ts);
+ if (!ttisnil(o)) /* string already present? */
+ ts = keystrval(nodefromval(o)); /* get saved copy */
+ else { /* not in use yet */
+ TValue *stv = s2v(L->top++); /* reserve stack space for string */
+ setsvalue(L, stv, ts); /* temporarily anchor the string */
+ luaH_finishset(L, ls->h, stv, o, stv); /* t[string] = string */
+ /* table is not a metatable, so it does not need to invalidate cache */
luaC_checkGC(L);
+ L->top--; /* remove string from stack */
}
- else { /* string already present */
- ts = keystrval(nodefromval(o)); /* re-use value previously stored */
- }
- L->top--; /* remove string from stack */
return ts;
}
@@ -254,9 +257,10 @@ static int read_numeral (LexState *ls, SemInfo *seminfo) {
/*
-** reads a sequence '[=*[' or ']=*]', leaving the last bracket.
-** If sequence is well formed, return its number of '='s + 2; otherwise,
-** return 1 if there is no '='s or 0 otherwise (an unfinished '[==...').
+** read a sequence '[=*[' or ']=*]', leaving the last bracket. If
+** sequence is well formed, return its number of '='s + 2; otherwise,
+** return 1 if it is a single bracket (no '='s and no 2nd bracket);
+** otherwise (an unfinished '[==...') return 0.
*/
static size_t skip_sep (LexState *ls) {
size_t count = 0;
@@ -481,34 +485,34 @@ static int llex (LexState *ls, SemInfo *seminfo) {
}
case '=': {
next(ls);
- if (check_next1(ls, '=')) return TK_EQ;
+ if (check_next1(ls, '=')) return TK_EQ; /* '==' */
else return '=';
}
case '<': {
next(ls);
- if (check_next1(ls, '=')) return TK_LE;
- else if (check_next1(ls, '<')) return TK_SHL;
+ if (check_next1(ls, '=')) return TK_LE; /* '<=' */
+ else if (check_next1(ls, '<')) return TK_SHL; /* '<<' */
else return '<';
}
case '>': {
next(ls);
- if (check_next1(ls, '=')) return TK_GE;
- else if (check_next1(ls, '>')) return TK_SHR;
+ if (check_next1(ls, '=')) return TK_GE; /* '>=' */
+ else if (check_next1(ls, '>')) return TK_SHR; /* '>>' */
else return '>';
}
case '/': {
next(ls);
- if (check_next1(ls, '/')) return TK_IDIV;
+ if (check_next1(ls, '/')) return TK_IDIV; /* '//' */
else return '/';
}
case '~': {
next(ls);
- if (check_next1(ls, '=')) return TK_NE;
+ if (check_next1(ls, '=')) return TK_NE; /* '~=' */
else return '~';
}
case ':': {
next(ls);
- if (check_next1(ls, ':')) return TK_DBCOLON;
+ if (check_next1(ls, ':')) return TK_DBCOLON; /* '::' */
else return ':';
}
case '"': case '\'': { /* short literal strings */
@@ -547,7 +551,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
return TK_NAME;
}
}
- else { /* single-char tokens (+ - / ...) */
+ else { /* single-char tokens ('+', '*', '%', '{', '}', ...) */
int c = ls->current;
next(ls);
return c;
diff --git a/src/lua/llimits.h b/src/lua/llimits.h
index 48c97f95..52a32f92 100644
--- a/src/lua/llimits.h
+++ b/src/lua/llimits.h
@@ -149,22 +149,6 @@ typedef LUAI_UACINT l_uacInt;
#endif
-/*
-** macros to improve jump prediction (used mainly for error handling)
-*/
-#if !defined(likely)
-
-#if defined(__GNUC__)
-#define likely(x) (__builtin_expect(((x) != 0), 1))
-#define unlikely(x) (__builtin_expect(((x) != 0), 0))
-#else
-#define likely(x) (x)
-#define unlikely(x) (x)
-#endif
-
-#endif
-
-
/*
** non-return type
*/
@@ -181,6 +165,20 @@ typedef LUAI_UACINT l_uacInt;
#endif
+/*
+** Inline functions
+*/
+#if !defined(LUA_USE_C89)
+#define l_inline inline
+#elif defined(__GNUC__)
+#define l_inline __inline__
+#else
+#define l_inline /* empty */
+#endif
+
+#define l_sinline static l_inline
+
+
/*
** type for virtual-machine instructions;
** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h)
@@ -234,6 +232,17 @@ typedef l_uint32 Instruction;
#endif
+/*
+** Maximum depth for nested C calls, syntactical nested non-terminals,
+** and other features implemented through recursion in C. (Value must
+** fit in a 16-bit unsigned integer. It must also be compatible with
+** the size of the C stack.)
+*/
+#if !defined(LUAI_MAXCCALLS)
+#define LUAI_MAXCCALLS 200
+#endif
+
+
/*
** macros that are executed whenever program enters the Lua core
** ('lua_lock') and leaves the core ('lua_unlock')
@@ -315,7 +324,8 @@ typedef l_uint32 Instruction;
/* exponentiation */
#if !defined(luai_numpow)
-#define luai_numpow(L,a,b) ((void)L, l_mathop(pow)(a,b))
+#define luai_numpow(L,a,b) \
+ ((void)L, (b == 2) ? (a)*(a) : l_mathop(pow)(a,b))
#endif
/* the others are quite standard operations */
@@ -344,14 +354,14 @@ typedef l_uint32 Instruction;
#else
/* realloc stack keeping its size */
#define condmovestack(L,pre,pos) \
- { int sz_ = (L)->stacksize; pre; luaD_reallocstack((L), sz_, 0); pos; }
+ { int sz_ = stacksize(L); pre; luaD_reallocstack((L), sz_, 0); pos; }
#endif
#if !defined(HARDMEMTESTS)
#define condchangemem(L,pre,pos) ((void)0)
#else
#define condchangemem(L,pre,pos) \
- { if (G(L)->gcrunning) { pre; luaC_fullgc(L, 0); pos; } }
+ { if (gcrunning(G(L))) { pre; luaC_fullgc(L, 0); pos; } }
#endif
#endif
diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c
index 86def470..e0c61a16 100644
--- a/src/lua/lmathlib.c
+++ b/src/lua/lmathlib.c
@@ -73,7 +73,7 @@ static int math_atan (lua_State *L) {
static int math_toint (lua_State *L) {
int valid;
lua_Integer n = lua_tointegerx(L, 1, &valid);
- if (valid)
+ if (l_likely(valid))
lua_pushinteger(L, n);
else {
luaL_checkany(L, 1);
@@ -175,7 +175,8 @@ static int math_log (lua_State *L) {
lua_Number base = luaL_checknumber(L, 2);
#if !defined(LUA_USE_C89)
if (base == l_mathop(2.0))
- res = l_mathop(log2)(x); else
+ res = l_mathop(log2)(x);
+ else
#endif
if (base == l_mathop(10.0))
res = l_mathop(log10)(x);
@@ -474,7 +475,7 @@ static lua_Number I2d (Rand64 x) {
/* 2^(-FIGS) = 1.0 / 2^30 / 2^3 / 2^(FIGS-33) */
#define scaleFIG \
- ((lua_Number)1.0 / (UONE << 30) / 8.0 / (UONE << (FIGS - 33)))
+ (l_mathop(1.0) / (UONE << 30) / l_mathop(8.0) / (UONE << (FIGS - 33)))
/*
** use FIGS - 32 bits from lower half, throwing out the other
@@ -485,7 +486,7 @@ static lua_Number I2d (Rand64 x) {
/*
** higher 32 bits go after those (FIGS - 32) bits: shiftHI = 2^(FIGS - 32)
*/
-#define shiftHI ((lua_Number)(UONE << (FIGS - 33)) * 2.0)
+#define shiftHI ((lua_Number)(UONE << (FIGS - 33)) * l_mathop(2.0))
static lua_Number I2d (Rand64 x) {
diff --git a/src/lua/lmem.c b/src/lua/lmem.c
index 43739bff..9029d588 100644
--- a/src/lua/lmem.c
+++ b/src/lua/lmem.c
@@ -24,12 +24,12 @@
#if defined(EMERGENCYGCTESTS)
/*
-** First allocation will fail whenever not building initial state
-** and not shrinking a block. (This fail will trigger 'tryagain' and
-** a full GC cycle at every allocation.)
+** First allocation will fail whenever not building initial state.
+** (This fail will trigger 'tryagain' and a full GC cycle at every
+** allocation.)
*/
static void *firsttry (global_State *g, void *block, size_t os, size_t ns) {
- if (ttisnil(&g->nilvalue) && ns > os)
+ if (completestate(g) && ns > 0) /* frees never fail */
return NULL; /* fail */
else /* normal allocation */
return (*g->frealloc)(g->ud, block, os, ns);
@@ -83,7 +83,7 @@ void *luaM_growaux_ (lua_State *L, void *block, int nelems, int *psize,
if (nelems + 1 <= size) /* does one extra element still fit? */
return block; /* nothing to be done */
if (size >= limit / 2) { /* cannot double it? */
- if (unlikely(size >= limit)) /* cannot grow even a little? */
+ if (l_unlikely(size >= limit)) /* cannot grow even a little? */
luaG_runerror(L, "too many %s (limit is %d)", what, limit);
size = limit; /* still have at least one free place */
}
@@ -138,15 +138,17 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) {
/*
-** In case of allocation fail, this function will call the GC to try
-** to free some memory and then try the allocation again.
-** (It should not be called when shrinking a block, because then the
-** interpreter may be in the middle of a collection step.)
+** In case of allocation fail, this function will do an emergency
+** collection to free some memory and then try the allocation again.
+** The GC should not be called while state is not fully built, as the
+** collector is not yet fully initialized. Also, it should not be called
+** when 'gcstopem' is true, because then the interpreter is in the
+** middle of a collection step.
*/
static void *tryagain (lua_State *L, void *block,
size_t osize, size_t nsize) {
global_State *g = G(L);
- if (ttisnil(&g->nilvalue)) { /* is state fully build? */
+ if (completestate(g) && !g->gcstopem) {
luaC_fullgc(L, 1); /* try to free some memory... */
return (*g->frealloc)(g->ud, block, osize, nsize); /* try again */
}
@@ -156,17 +158,14 @@ static void *tryagain (lua_State *L, void *block,
/*
** Generic allocation routine.
-** If allocation fails while shrinking a block, do not try again; the
-** GC shrinks some blocks and it is not reentrant.
*/
void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
void *newblock;
global_State *g = G(L);
lua_assert((osize == 0) == (block == NULL));
newblock = firsttry(g, block, osize, nsize);
- if (unlikely(newblock == NULL && nsize > 0)) {
- if (nsize > osize) /* not shrinking a block? */
- newblock = tryagain(L, block, osize, nsize);
+ if (l_unlikely(newblock == NULL && nsize > 0)) {
+ newblock = tryagain(L, block, osize, nsize);
if (newblock == NULL) /* still no memory? */
return NULL; /* do not update 'GCdebt' */
}
@@ -179,7 +178,7 @@ void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
void *luaM_saferealloc_ (lua_State *L, void *block, size_t osize,
size_t nsize) {
void *newblock = luaM_realloc_(L, block, osize, nsize);
- if (unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */
+ if (l_unlikely(newblock == NULL && nsize > 0)) /* allocation failed? */
luaM_error(L);
return newblock;
}
@@ -191,7 +190,7 @@ void *luaM_malloc_ (lua_State *L, size_t size, int tag) {
else {
global_State *g = G(L);
void *newblock = firsttry(g, NULL, tag, size);
- if (unlikely(newblock == NULL)) {
+ if (l_unlikely(newblock == NULL)) {
newblock = tryagain(L, NULL, tag, size);
if (newblock == NULL)
luaM_error(L);
diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c
index c0ec9a13..6f9fa373 100644
--- a/src/lua/loadlib.c
+++ b/src/lua/loadlib.c
@@ -132,14 +132,16 @@ static void lsys_unloadlib (void *lib) {
static void *lsys_load (lua_State *L, const char *path, int seeglb) {
void *lib = dlopen(path, RTLD_NOW | (seeglb ? RTLD_GLOBAL : RTLD_LOCAL));
- if (lib == NULL) lua_pushstring(L, dlerror());
+ if (l_unlikely(lib == NULL))
+ lua_pushstring(L, dlerror());
return lib;
}
static lua_CFunction lsys_sym (lua_State *L, void *lib, const char *sym) {
lua_CFunction f = cast_func(dlsym(lib, sym));
- if (f == NULL) lua_pushstring(L, dlerror());
+ if (l_unlikely(f == NULL))
+ lua_pushstring(L, dlerror());
return f;
}
@@ -410,7 +412,7 @@ static int ll_loadlib (lua_State *L) {
const char *path = luaL_checkstring(L, 1);
const char *init = luaL_checkstring(L, 2);
int stat = lookforfunc(L, path, init);
- if (stat == 0) /* no errors? */
+ if (l_likely(stat == 0)) /* no errors? */
return 1; /* return the loaded function */
else { /* error; error message is on stack top */
luaL_pushfail(L);
@@ -523,14 +525,14 @@ static const char *findfile (lua_State *L, const char *name,
const char *path;
lua_getfield(L, lua_upvalueindex(1), pname);
path = lua_tostring(L, -1);
- if (path == NULL)
+ if (l_unlikely(path == NULL))
luaL_error(L, "'package.%s' must be a string", pname);
return searchpath(L, name, path, ".", dirsep);
}
static int checkload (lua_State *L, int stat, const char *filename) {
- if (stat) { /* module loaded successfully? */
+ if (l_likely(stat)) { /* module loaded successfully? */
lua_pushstring(L, filename); /* will be 2nd argument to module */
return 2; /* return open function and file name */
}
@@ -623,13 +625,14 @@ static void findloader (lua_State *L, const char *name) {
int i;
luaL_Buffer msg; /* to build error message */
/* push 'package.searchers' to index 3 in the stack */
- if (lua_getfield(L, lua_upvalueindex(1), "searchers") != LUA_TTABLE)
+ if (l_unlikely(lua_getfield(L, lua_upvalueindex(1), "searchers")
+ != LUA_TTABLE))
luaL_error(L, "'package.searchers' must be a table");
luaL_buffinit(L, &msg);
/* iterate over available searchers to find a loader */
for (i = 1; ; i++) {
luaL_addstring(&msg, "\n\t"); /* error-message prefix */
- if (lua_rawgeti(L, 3, i) == LUA_TNIL) { /* no more searchers? */
+ if (l_unlikely(lua_rawgeti(L, 3, i) == LUA_TNIL)) { /* no more searchers? */
lua_pop(L, 1); /* remove nil */
luaL_buffsub(&msg, 2); /* remove prefix */
luaL_pushresult(&msg); /* create error message */
diff --git a/src/lua/lobject.c b/src/lua/lobject.c
index f8ea917a..301aa900 100644
--- a/src/lua/lobject.c
+++ b/src/lua/lobject.c
@@ -164,7 +164,7 @@ static int isneg (const char **s) {
*/
static lua_Number lua_strx2number (const char *s, char **endptr) {
int dot = lua_getlocaledecpoint();
- lua_Number r = 0.0; /* result (accumulator) */
+ lua_Number r = l_mathop(0.0); /* result (accumulator) */
int sigdig = 0; /* number of significant digits */
int nosigdig = 0; /* number of non-significant digits */
int e = 0; /* exponent correction */
@@ -174,7 +174,7 @@ static lua_Number lua_strx2number (const char *s, char **endptr) {
while (lisspace(cast_uchar(*s))) s++; /* skip initial spaces */
neg = isneg(&s); /* check sign */
if (!(*s == '0' && (*(s + 1) == 'x' || *(s + 1) == 'X'))) /* check '0x' */
- return 0.0; /* invalid format (no '0x') */
+ return l_mathop(0.0); /* invalid format (no '0x') */
for (s += 2; ; s++) { /* skip '0x' and read numeral */
if (*s == dot) {
if (hasdot) break; /* second dot? stop loop */
@@ -184,14 +184,14 @@ static lua_Number lua_strx2number (const char *s, char **endptr) {
if (sigdig == 0 && *s == '0') /* non-significant digit (zero)? */
nosigdig++;
else if (++sigdig <= MAXSIGDIG) /* can read it without overflow? */
- r = (r * cast_num(16.0)) + luaO_hexavalue(*s);
+ r = (r * l_mathop(16.0)) + luaO_hexavalue(*s);
else e++; /* too many digits; ignore, but still count for exponent */
if (hasdot) e--; /* decimal digit? correct exponent */
}
else break; /* neither a dot nor a digit */
}
if (nosigdig + sigdig == 0) /* no digits? */
- return 0.0; /* invalid format */
+ return l_mathop(0.0); /* invalid format */
*endptr = cast_charp(s); /* valid up to here */
e *= 4; /* each digit multiplies/divides value by 2^4 */
if (*s == 'p' || *s == 'P') { /* exponent part? */
@@ -200,7 +200,7 @@ static lua_Number lua_strx2number (const char *s, char **endptr) {
s++; /* skip 'p' */
neg1 = isneg(&s); /* sign */
if (!lisdigit(cast_uchar(*s)))
- return 0.0; /* invalid; must have at least one digit */
+ return l_mathop(0.0); /* invalid; must have at least one digit */
while (lisdigit(cast_uchar(*s))) /* read exponent */
exp1 = exp1 * 10 + *(s++) - '0';
if (neg1) exp1 = -exp1;
@@ -258,7 +258,7 @@ static const char *l_str2d (const char *s, lua_Number *result) {
if (endptr == NULL) { /* failed? may be a different locale */
char buff[L_MAXLENNUM + 1];
const char *pdot = strchr(s, '.');
- if (strlen(s) > L_MAXLENNUM || pdot == NULL)
+ if (pdot == NULL || strlen(s) > L_MAXLENNUM)
return NULL; /* string too long or no dot; fail */
strcpy(buff, s); /* copy string to buffer */
buff[pdot - s] = lua_getlocaledecpoint(); /* correct decimal point */
diff --git a/src/lua/lobject.h b/src/lua/lobject.h
index a9d45785..0e05b3e4 100644
--- a/src/lua/lobject.h
+++ b/src/lua/lobject.h
@@ -21,10 +21,12 @@
*/
#define LUA_TUPVAL LUA_NUMTYPES /* upvalues */
#define LUA_TPROTO (LUA_NUMTYPES+1) /* function prototypes */
+#define LUA_TDEADKEY (LUA_NUMTYPES+2) /* removed keys in tables */
+
/*
-** number of all possible types (including LUA_TNONE)
+** number of all possible types (including LUA_TNONE but excluding DEADKEY)
*/
#define LUA_TOTALTYPES (LUA_TPROTO + 2)
@@ -66,7 +68,7 @@ typedef struct TValue {
#define val_(o) ((o)->value_)
-#define valraw(o) (&val_(o))
+#define valraw(o) (val_(o))
/* raw type tag of a TValue */
@@ -110,7 +112,7 @@ typedef struct TValue {
#define settt_(o,t) ((o)->tt_=(t))
-/* main macro to copy values (from 'obj1' to 'obj2') */
+/* main macro to copy values (from 'obj2' to 'obj1') */
#define setobj(L,obj1,obj2) \
{ TValue *io1=(obj1); const TValue *io2=(obj2); \
io1->value_ = io2->value_; settt_(io1, io2->tt_); \
@@ -134,10 +136,19 @@ typedef struct TValue {
/*
-** Entries in the Lua stack
+** Entries in a Lua stack. Field 'tbclist' forms a list of all
+** to-be-closed variables active in this stack. Dummy entries are
+** used when the distance between two tbc variables does not fit
+** in an unsigned short. They are represented by delta==0, and
+** their real delta is always the maximum value that fits in
+** that field.
*/
typedef union StackValue {
TValue val;
+ struct {
+ TValuefields;
+ unsigned short delta;
+ } tbclist;
} StackValue;
@@ -555,7 +566,7 @@ typedef struct Proto {
/*
** {==================================================================
-** Closures
+** Functions
** ===================================================================
*/
@@ -568,10 +579,11 @@ typedef struct Proto {
#define LUA_VCCL makevariant(LUA_TFUNCTION, 2) /* C closure */
#define ttisfunction(o) checktype(o, LUA_TFUNCTION)
-#define ttisclosure(o) ((rawtt(o) & 0x1F) == LUA_VLCL)
#define ttisLclosure(o) checktag((o), ctb(LUA_VLCL))
#define ttislcf(o) checktag((o), LUA_VLCF)
#define ttisCclosure(o) checktag((o), ctb(LUA_VCCL))
+#define ttisclosure(o) (ttisLclosure(o) || ttisCclosure(o))
+
#define isLfunction(o) ttisLclosure(o)
@@ -743,13 +755,13 @@ typedef struct Table {
/*
-** Use a "nil table" to mark dead keys in a table. Those keys serve
-** to keep space for removed entries, which may still be part of
-** chains. Note that the 'keytt' does not have the BIT_ISCOLLECTABLE
-** set, so these values are considered not collectable and are different
-** from any valid value.
+** Dead keys in tables have the tag DEADKEY but keep their original
+** gcvalue. This distinguishes them from regular keys but allows them to
+** be found when searched in a special way. ('next' needs that to find
+** keys removed from a table during a traversal.)
*/
-#define setdeadkey(n) (keytt(n) = LUA_TTABLE, gckey(n) = NULL)
+#define setdeadkey(node) (keytt(node) = LUA_TDEADKEY)
+#define keyisdead(node) (keytt(node) == LUA_TDEADKEY)
/* }================================================================== */
diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h
index 122e5d21..7c274515 100644
--- a/src/lua/lopcodes.h
+++ b/src/lua/lopcodes.h
@@ -190,7 +190,8 @@ enum OpMode {iABC, iABx, iAsBx, iAx, isJ}; /* basic instruction formats */
/*
-** grep "ORDER OP" if you change these enums
+** Grep "ORDER OP" if you change these enums. Opcodes marked with a (*)
+** has extra descriptions in the notes after the enumeration.
*/
typedef enum {
@@ -203,7 +204,7 @@ OP_LOADF,/* A sBx R[A] := (lua_Number)sBx */
OP_LOADK,/* A Bx R[A] := K[Bx] */
OP_LOADKX,/* A R[A] := K[extra arg] */
OP_LOADFALSE,/* A R[A] := false */
-OP_LFALSESKIP,/*A R[A] := false; pc++ */
+OP_LFALSESKIP,/*A R[A] := false; pc++ (*) */
OP_LOADTRUE,/* A R[A] := true */
OP_LOADNIL,/* A B R[A], R[A+1], ..., R[A+B] := nil */
OP_GETUPVAL,/* A B R[A] := UpValue[B] */
@@ -225,13 +226,13 @@ OP_SELF,/* A B C R[A+1] := R[B]; R[A] := R[B][RK(C):string] */
OP_ADDI,/* A B sC R[A] := R[B] + sC */
-OP_ADDK,/* A B C R[A] := R[B] + K[C] */
-OP_SUBK,/* A B C R[A] := R[B] - K[C] */
-OP_MULK,/* A B C R[A] := R[B] * K[C] */
-OP_MODK,/* A B C R[A] := R[B] % K[C] */
-OP_POWK,/* A B C R[A] := R[B] ^ K[C] */
-OP_DIVK,/* A B C R[A] := R[B] / K[C] */
-OP_IDIVK,/* A B C R[A] := R[B] // K[C] */
+OP_ADDK,/* A B C R[A] := R[B] + K[C]:number */
+OP_SUBK,/* A B C R[A] := R[B] - K[C]:number */
+OP_MULK,/* A B C R[A] := R[B] * K[C]:number */
+OP_MODK,/* A B C R[A] := R[B] % K[C]:number */
+OP_POWK,/* A B C R[A] := R[B] ^ K[C]:number */
+OP_DIVK,/* A B C R[A] := R[B] / K[C]:number */
+OP_IDIVK,/* A B C R[A] := R[B] // K[C]:number */
OP_BANDK,/* A B C R[A] := R[B] & K[C]:integer */
OP_BORK,/* A B C R[A] := R[B] | K[C]:integer */
@@ -254,14 +255,14 @@ OP_BXOR,/* A B C R[A] := R[B] ~ R[C] */
OP_SHL,/* A B C R[A] := R[B] << R[C] */
OP_SHR,/* A B C R[A] := R[B] >> R[C] */
-OP_MMBIN,/* A B C call C metamethod over R[A] and R[B] */
+OP_MMBIN,/* A B C call C metamethod over R[A] and R[B] (*) */
OP_MMBINI,/* A sB C k call C metamethod over R[A] and sB */
OP_MMBINK,/* A B C k call C metamethod over R[A] and K[B] */
OP_UNM,/* A B R[A] := -R[B] */
OP_BNOT,/* A B R[A] := ~R[B] */
OP_NOT,/* A B R[A] := not R[B] */
-OP_LEN,/* A B R[A] := length of R[B] */
+OP_LEN,/* A B R[A] := #R[B] (length operator) */
OP_CONCAT,/* A B R[A] := R[A].. ... ..R[A + B - 1] */
@@ -280,7 +281,7 @@ OP_GTI,/* A sB k if ((R[A] > sB) ~= k) then pc++ */
OP_GEI,/* A sB k if ((R[A] >= sB) ~= k) then pc++ */
OP_TEST,/* A k if (not R[A] == k) then pc++ */
-OP_TESTSET,/* A B k if (not R[B] == k) then pc++ else R[A] := R[B] */
+OP_TESTSET,/* A B k if (not R[B] == k) then pc++ else R[A] := R[B] (*) */
OP_CALL,/* A B C R[A], ... ,R[A+C-2] := R[A](R[A+1], ... ,R[A+B-1]) */
OP_TAILCALL,/* A B C k return R[A](R[A+1], ... ,R[A+B-1]) */
@@ -297,7 +298,7 @@ OP_TFORPREP,/* A Bx create upvalue for R[A + 3]; pc+=Bx */
OP_TFORCALL,/* A C R[A+4], ... ,R[A+3+C] := R[A](R[A+1], R[A+2]); */
OP_TFORLOOP,/* A Bx if R[A+2] ~= nil then { R[A]=R[A+2]; pc -= Bx } */
-OP_SETLIST,/* A B C k R[A][(C-1)*FPF+i] := R[A+i], 1 <= i <= B */
+OP_SETLIST,/* A B C k R[A][C+i] := R[A+i], 1 <= i <= B */
OP_CLOSURE,/* A Bx R[A] := closure(KPROTO[Bx]) */
@@ -315,6 +316,18 @@ OP_EXTRAARG/* Ax extra (larger) argument for previous opcode */
/*===========================================================================
Notes:
+
+ (*) Opcode OP_LFALSESKIP is used to convert a condition to a boolean
+ value, in a code equivalent to (not cond ? false : true). (It
+ produces false and skips the next instruction producing true.)
+
+ (*) Opcodes OP_MMBIN and variants follow each arithmetic and
+ bitwise opcode. If the operation succeeds, it skips this next
+ opcode. Otherwise, this opcode calls the corresponding metamethod.
+
+ (*) Opcode OP_TESTSET is used in short-circuit expressions that need
+ both to jump and to produce a value, such as (a = b or c).
+
(*) In OP_CALL, if (B == 0) then B = top - A. If (C == 0), then
'top' is set to last_result+1, so next open instruction (OP_CALL,
OP_RETURN*, OP_SETLIST) may use 'top'.
diff --git a/src/lua/loslib.c b/src/lua/loslib.c
index e65e188b..3e20d622 100644
--- a/src/lua/loslib.c
+++ b/src/lua/loslib.c
@@ -170,7 +170,7 @@ static int os_tmpname (lua_State *L) {
char buff[LUA_TMPNAMBUFSIZE];
int err;
lua_tmpnam(buff, err);
- if (err)
+ if (l_unlikely(err))
return luaL_error(L, "unable to generate a unique filename");
lua_pushstring(L, buff);
return 1;
@@ -208,7 +208,7 @@ static int os_clock (lua_State *L) {
*/
static void setfield (lua_State *L, const char *key, int value, int delta) {
#if (defined(LUA_NUMTIME) && LUA_MAXINTEGER <= INT_MAX)
- if (value > LUA_MAXINTEGER - delta)
+ if (l_unlikely(value > LUA_MAXINTEGER - delta))
luaL_error(L, "field '%s' is out-of-bound", key);
#endif
lua_pushinteger(L, (lua_Integer)value + delta);
@@ -253,9 +253,9 @@ static int getfield (lua_State *L, const char *key, int d, int delta) {
int t = lua_getfield(L, -1, key); /* get field and its type */
lua_Integer res = lua_tointegerx(L, -1, &isnum);
if (!isnum) { /* field is not an integer? */
- if (t != LUA_TNIL) /* some other value? */
+ if (l_unlikely(t != LUA_TNIL)) /* some other value? */
return luaL_error(L, "field '%s' is not an integer", key);
- else if (d < 0) /* absent field; no default? */
+ else if (l_unlikely(d < 0)) /* absent field; no default? */
return luaL_error(L, "field '%s' missing in date table", key);
res = d;
}
diff --git a/src/lua/lparser.c b/src/lua/lparser.c
index bc7d9a4f..3abe3d75 100644
--- a/src/lua/lparser.c
+++ b/src/lua/lparser.c
@@ -128,7 +128,7 @@ static void checknext (LexState *ls, int c) {
** in line 'where' (if that is not the current line).
*/
static void check_match (LexState *ls, int what, int who, int where) {
- if (unlikely(!testnext(ls, what))) {
+ if (l_unlikely(!testnext(ls, what))) {
if (where == ls->linenumber) /* all in the same line? */
error_expected(ls, what); /* do not need a complex message */
else {
@@ -222,26 +222,26 @@ static Vardesc *getlocalvardesc (FuncState *fs, int vidx) {
/*
-** Convert 'nvar', a compiler index level, to it corresponding
-** stack index level. For that, search for the highest variable
-** below that level that is in the stack and uses its stack
-** index ('sidx').
+** Convert 'nvar', a compiler index level, to its corresponding
+** register. For that, search for the highest variable below that level
+** that is in a register and uses its register index ('ridx') plus one.
*/
-static int stacklevel (FuncState *fs, int nvar) {
+static int reglevel (FuncState *fs, int nvar) {
while (nvar-- > 0) {
- Vardesc *vd = getlocalvardesc(fs, nvar); /* get variable */
- if (vd->vd.kind != RDKCTC) /* is in the stack? */
- return vd->vd.sidx + 1;
+ Vardesc *vd = getlocalvardesc(fs, nvar); /* get previous variable */
+ if (vd->vd.kind != RDKCTC) /* is in a register? */
+ return vd->vd.ridx + 1;
}
- return 0; /* no variables in the stack */
+ return 0; /* no variables in registers */
}
/*
-** Return the number of variables in the stack for function 'fs'
+** Return the number of variables in the register stack for the given
+** function.
*/
int luaY_nvarstack (FuncState *fs) {
- return stacklevel(fs, fs->nactvar);
+ return reglevel(fs, fs->nactvar);
}
@@ -267,7 +267,7 @@ static void init_var (FuncState *fs, expdesc *e, int vidx) {
e->f = e->t = NO_JUMP;
e->k = VLOCAL;
e->u.var.vidx = vidx;
- e->u.var.sidx = getlocalvardesc(fs, vidx)->vd.sidx;
+ e->u.var.ridx = getlocalvardesc(fs, vidx)->vd.ridx;
}
@@ -310,12 +310,12 @@ static void check_readonly (LexState *ls, expdesc *e) {
*/
static void adjustlocalvars (LexState *ls, int nvars) {
FuncState *fs = ls->fs;
- int stklevel = luaY_nvarstack(fs);
+ int reglevel = luaY_nvarstack(fs);
int i;
for (i = 0; i < nvars; i++) {
int vidx = fs->nactvar++;
Vardesc *var = getlocalvardesc(fs, vidx);
- var->vd.sidx = stklevel++;
+ var->vd.ridx = reglevel++;
var->vd.pidx = registerlocalvar(ls, fs, var->vd.name);
}
}
@@ -366,7 +366,7 @@ static int newupvalue (FuncState *fs, TString *name, expdesc *v) {
FuncState *prev = fs->prev;
if (v->k == VLOCAL) {
up->instack = 1;
- up->idx = v->u.var.sidx;
+ up->idx = v->u.var.ridx;
up->kind = getlocalvardesc(prev, v->u.var.vidx)->vd.kind;
lua_assert(eqstr(name, getlocalvardesc(prev, v->u.var.vidx)->vd.name));
}
@@ -416,6 +416,17 @@ static void markupval (FuncState *fs, int level) {
}
+/*
+** Mark that current block has a to-be-closed variable.
+*/
+static void marktobeclosed (FuncState *fs) {
+ BlockCnt *bl = fs->bl;
+ bl->upval = 1;
+ bl->insidetbc = 1;
+ fs->needclose = 1;
+}
+
+
/*
** Find a variable with the given name 'n'. If it is an upvalue, add
** this upvalue into all intermediate functions. If it is a global, set
@@ -489,12 +500,10 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) {
}
-/*
-** Macros to limit the maximum recursion depth while parsing
-*/
-#define enterlevel(ls) luaE_enterCcall((ls)->L)
+#define enterlevel(ls) luaE_incCstack(ls->L)
-#define leavelevel(ls) luaE_exitCcall((ls)->L)
+
+#define leavelevel(ls) ((ls)->L->nCcalls--)
/*
@@ -519,7 +528,7 @@ static void solvegoto (LexState *ls, int g, Labeldesc *label) {
Labellist *gl = &ls->dyd->gt; /* list of goto's */
Labeldesc *gt = &gl->arr[g]; /* goto to be resolved */
lua_assert(eqstr(gt->name, label->name));
- if (unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */
+ if (l_unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */
jumpscopeerror(ls, gt);
luaK_patchlist(ls->fs, gt->pc, label->pc);
for (i = g; i < gl->n - 1; i++) /* remove goto from pending list */
@@ -622,7 +631,7 @@ static void movegotosout (FuncState *fs, BlockCnt *bl) {
for (i = bl->firstgoto; i < gl->n; i++) { /* for each pending goto */
Labeldesc *gt = &gl->arr[i];
/* leaving a variable scope? */
- if (stacklevel(fs, gt->nactvar) > stacklevel(fs, bl->nactvar))
+ if (reglevel(fs, gt->nactvar) > reglevel(fs, bl->nactvar))
gt->close |= bl->upval; /* jump may need a close */
gt->nactvar = bl->nactvar; /* update goto level */
}
@@ -663,7 +672,7 @@ static void leaveblock (FuncState *fs) {
BlockCnt *bl = fs->bl;
LexState *ls = fs->ls;
int hasclose = 0;
- int stklevel = stacklevel(fs, bl->nactvar); /* level outside the block */
+ int stklevel = reglevel(fs, bl->nactvar); /* level outside the block */
if (bl->isloop) /* fix pending breaks? */
hasclose = createlabel(ls, luaS_newliteral(ls->L, "break"), 0, 0);
if (!hasclose && bl->previous && bl->upval)
@@ -947,7 +956,7 @@ static void setvararg (FuncState *fs, int nparams) {
static void parlist (LexState *ls) {
- /* parlist -> [ param { ',' param } ] */
+ /* parlist -> [ {NAME ','} (NAME | '...') ] */
FuncState *fs = ls->fs;
Proto *f = fs->f;
int nparams = 0;
@@ -955,12 +964,12 @@ static void parlist (LexState *ls) {
if (ls->t.token != ')') { /* is 'parlist' not empty? */
do {
switch (ls->t.token) {
- case TK_NAME: { /* param -> NAME */
+ case TK_NAME: {
new_localvar(ls, str_checkname(ls));
nparams++;
break;
}
- case TK_DOTS: { /* param -> '...' */
+ case TK_DOTS: {
luaX_next(ls);
isvararg = 1;
break;
@@ -1332,13 +1341,13 @@ static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) {
}
}
else { /* table is a register */
- if (v->k == VLOCAL && lh->v.u.ind.t == v->u.var.sidx) {
+ if (v->k == VLOCAL && lh->v.u.ind.t == v->u.var.ridx) {
conflict = 1; /* table is the local being assigned now */
lh->v.u.ind.t = extra; /* assignment will use safe copy */
}
/* is index the local being assigned? */
if (lh->v.k == VINDEXED && v->k == VLOCAL &&
- lh->v.u.ind.idx == v->u.var.sidx) {
+ lh->v.u.ind.idx == v->u.var.ridx) {
conflict = 1;
lh->v.u.ind.idx = extra; /* previous assignment will use safe copy */
}
@@ -1348,7 +1357,7 @@ static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) {
if (conflict) {
/* copy upvalue/local value to a temporary (in position 'extra') */
if (v->k == VLOCAL)
- luaK_codeABC(fs, OP_MOVE, extra, v->u.var.sidx, 0);
+ luaK_codeABC(fs, OP_MOVE, extra, v->u.var.ridx, 0);
else
luaK_codeABC(fs, OP_GETUPVAL, extra, v->u.info, 0);
luaK_reserveregs(fs, 1);
@@ -1413,7 +1422,7 @@ static void gotostat (LexState *ls) {
newgotoentry(ls, name, line, luaK_jump(fs));
else { /* found a label */
/* backward jump; will be resolved here */
- int lblevel = stacklevel(fs, lb->nactvar); /* label level */
+ int lblevel = reglevel(fs, lb->nactvar); /* label level */
if (luaY_nvarstack(fs) > lblevel) /* leaving the scope of a variable? */
luaK_codeABC(fs, OP_CLOSE, lblevel, 0, 0);
/* create jump and link it to the label */
@@ -1437,7 +1446,7 @@ static void breakstat (LexState *ls) {
*/
static void checkrepeated (LexState *ls, TString *name) {
Labeldesc *lb = findlabel(ls, name);
- if (unlikely(lb != NULL)) { /* already defined? */
+ if (l_unlikely(lb != NULL)) { /* already defined? */
const char *msg = "label '%s' already defined on line %d";
msg = luaO_pushfstring(ls->L, msg, getstr(name), lb->line);
luaK_semerror(ls, msg); /* error */
@@ -1490,7 +1499,7 @@ static void repeatstat (LexState *ls, int line) {
if (bl2.upval) { /* upvalues? */
int exit = luaK_jump(fs); /* normal exit must jump over fix */
luaK_patchtohere(fs, condexit); /* repetition must close upvalues */
- luaK_codeABC(fs, OP_CLOSE, stacklevel(fs, bl2.nactvar), 0, 0);
+ luaK_codeABC(fs, OP_CLOSE, reglevel(fs, bl2.nactvar), 0, 0);
condexit = luaK_jump(fs); /* repeat after closing upvalues */
luaK_patchtohere(fs, exit); /* normal exit comes to here */
}
@@ -1522,7 +1531,7 @@ static void fixforjump (FuncState *fs, int pc, int dest, int back) {
int offset = dest - (pc + 1);
if (back)
offset = -offset;
- if (unlikely(offset > MAXARG_Bx))
+ if (l_unlikely(offset > MAXARG_Bx))
luaX_syntaxerror(fs->ls, "control structure too long");
SETARG_Bx(*jmp, offset);
}
@@ -1601,7 +1610,7 @@ static void forlist (LexState *ls, TString *indexname) {
line = ls->linenumber;
adjust_assign(ls, 4, explist(ls, &e), &e);
adjustlocalvars(ls, 4); /* control variables */
- markupval(fs, fs->nactvar); /* last control var. must be closed */
+ marktobeclosed(fs); /* last control var. must be closed */
luaK_checkstack(fs, 3); /* extra space to call generator */
forbody(ls, base, line, nvars - 4, 1);
}
@@ -1625,59 +1634,21 @@ static void forstat (LexState *ls, int line) {
}
-/*
-** Check whether next instruction is a single jump (a 'break', a 'goto'
-** to a forward label, or a 'goto' to a backward label with no variable
-** to close). If so, set the name of the 'label' it is jumping to
-** ("break" for a 'break') or to where it is jumping to ('target') and
-** return true. If not a single jump, leave input unchanged, to be
-** handled as a regular statement.
-*/
-static int issinglejump (LexState *ls, TString **label, int *target) {
- if (testnext(ls, TK_BREAK)) { /* a break? */
- *label = luaS_newliteral(ls->L, "break");
- return 1;
- }
- else if (ls->t.token != TK_GOTO || luaX_lookahead(ls) != TK_NAME)
- return 0; /* not a valid goto */
- else {
- TString *lname = ls->lookahead.seminfo.ts; /* label's id */
- Labeldesc *lb = findlabel(ls, lname);
- if (lb) { /* a backward jump? */
- /* does it need to close variables? */
- if (luaY_nvarstack(ls->fs) > stacklevel(ls->fs, lb->nactvar))
- return 0; /* not a single jump; cannot optimize */
- *target = lb->pc;
- }
- else /* jump forward */
- *label = lname;
- luaX_next(ls); /* skip goto */
- luaX_next(ls); /* skip name */
- return 1;
- }
-}
-
-
static void test_then_block (LexState *ls, int *escapelist) {
/* test_then_block -> [IF | ELSEIF] cond THEN block */
BlockCnt bl;
- int line;
FuncState *fs = ls->fs;
- TString *jlb = NULL;
- int target = NO_JUMP;
expdesc v;
int jf; /* instruction to skip 'then' code (if condition is false) */
luaX_next(ls); /* skip IF or ELSEIF */
expr(ls, &v); /* read condition */
checknext(ls, TK_THEN);
- line = ls->linenumber;
- if (issinglejump(ls, &jlb, &target)) { /* 'if x then goto' ? */
- luaK_goiffalse(ls->fs, &v); /* will jump to label if condition is true */
+ if (ls->t.token == TK_BREAK) { /* 'if x then break' ? */
+ int line = ls->linenumber;
+ luaK_goiffalse(ls->fs, &v); /* will jump if condition is true */
+ luaX_next(ls); /* skip 'break' */
enterblock(fs, &bl, 0); /* must enter block before 'goto' */
- if (jlb != NULL) /* forward jump? */
- newgotoentry(ls, jlb, line, v.t); /* will be resolved later */
- else /* backward jump */
- luaK_patchlist(fs, v.t, target); /* jump directly to 'target' */
+ newgotoentry(ls, luaS_newliteral(ls->L, "break"), line, v.t);
while (testnext(ls, ';')) {} /* skip semicolons */
if (block_follow(ls, 0)) { /* jump is the entire block? */
leaveblock(fs);
@@ -1686,7 +1657,7 @@ static void test_then_block (LexState *ls, int *escapelist) {
else /* must skip over 'then' part if condition is false */
jf = luaK_jump(fs);
}
- else { /* regular case (not a jump) */
+ else { /* regular case (not a break) */
luaK_goiftrue(ls->fs, &v); /* skip over block if condition is false */
enterblock(fs, &bl, 0);
jf = v.f;
@@ -1743,18 +1714,16 @@ static int getlocalattribute (LexState *ls) {
}
-static void checktoclose (LexState *ls, int level) {
+static void checktoclose (FuncState *fs, int level) {
if (level != -1) { /* is there a to-be-closed variable? */
- FuncState *fs = ls->fs;
- markupval(fs, level + 1);
- fs->bl->insidetbc = 1; /* in the scope of a to-be-closed variable */
- luaK_codeABC(fs, OP_TBC, stacklevel(fs, level), 0, 0);
+ marktobeclosed(fs);
+ luaK_codeABC(fs, OP_TBC, reglevel(fs, level), 0, 0);
}
}
static void localstat (LexState *ls) {
- /* stat -> LOCAL ATTRIB NAME {',' ATTRIB NAME} ['=' explist] */
+ /* stat -> LOCAL NAME ATTRIB { ',' NAME ATTRIB } ['=' explist] */
FuncState *fs = ls->fs;
int toclose = -1; /* index of to-be-closed variable (if any) */
Vardesc *var; /* last variable */
@@ -1791,7 +1760,7 @@ static void localstat (LexState *ls) {
adjust_assign(ls, nvars, nexps, &e);
adjustlocalvars(ls, nvars);
}
- checktoclose(ls, toclose);
+ checktoclose(fs, toclose);
}
@@ -1816,6 +1785,7 @@ static void funcstat (LexState *ls, int line) {
luaX_next(ls); /* skip FUNCTION */
ismethod = funcname(ls, &v);
body(ls, &b, ismethod, line);
+ check_readonly(ls, &v);
luaK_storevar(ls->fs, &v, &b);
luaK_fixline(ls->fs, line); /* definition "happens" in the first line */
}
diff --git a/src/lua/lparser.h b/src/lua/lparser.h
index 618cb010..5e4500f1 100644
--- a/src/lua/lparser.h
+++ b/src/lua/lparser.h
@@ -23,7 +23,7 @@
/* kinds of variables/expressions */
typedef enum {
- VVOID, /* when 'expdesc' describes the last expression a list,
+ VVOID, /* when 'expdesc' describes the last expression of a list,
this kind means an empty list (so, no expression) */
VNIL, /* constant nil */
VTRUE, /* constant true */
@@ -35,10 +35,11 @@ typedef enum {
(string is fixed by the lexer) */
VNONRELOC, /* expression has its value in a fixed register;
info = result register */
- VLOCAL, /* local variable; var.sidx = stack index (local register);
+ VLOCAL, /* local variable; var.ridx = register index;
var.vidx = relative index in 'actvar.arr' */
VUPVAL, /* upvalue variable; info = index of upvalue in 'upvalues' */
- VCONST, /* compile-time constant; info = absolute index in 'actvar.arr' */
+ VCONST, /* compile-time variable;
+ info = absolute index in 'actvar.arr' */
VINDEXED, /* indexed variable;
ind.t = table register;
ind.idx = key's R index */
@@ -76,7 +77,7 @@ typedef struct expdesc {
lu_byte t; /* table (register or upvalue) */
} ind;
struct { /* for local variables */
- lu_byte sidx; /* index in the stack */
+ lu_byte ridx; /* register holding the variable */
unsigned short vidx; /* compiler index (in 'actvar.arr') */
} var;
} u;
@@ -96,7 +97,7 @@ typedef union Vardesc {
struct {
TValuefields; /* constant value (if it is a compile-time constant) */
lu_byte kind;
- lu_byte sidx; /* index of the variable in the stack */
+ lu_byte ridx; /* register holding the variable */
short pidx; /* index of the variable in the Proto's 'locvars' array */
TString *name; /* variable name */
} vd;
diff --git a/src/lua/lstate.c b/src/lua/lstate.c
index 86b3761f..1ffe1a0f 100644
--- a/src/lua/lstate.c
+++ b/src/lua/lstate.c
@@ -76,7 +76,7 @@ static unsigned int luai_makeseed (lua_State *L) {
addbuff(buff, p, &h); /* local variable */
addbuff(buff, p, &lua_newstate); /* public function */
lua_assert(p == sizeof(buff));
- return luaS_hash(buff, p, h, 1);
+ return luaS_hash(buff, p, h);
}
#endif
@@ -97,66 +97,14 @@ void luaE_setdebt (global_State *g, l_mem debt) {
LUA_API int lua_setcstacklimit (lua_State *L, unsigned int limit) {
- global_State *g = G(L);
- int ccalls;
- luaE_freeCI(L); /* release unused CIs */
- ccalls = getCcalls(L);
- if (limit >= 40000)
- return 0; /* out of bounds */
- limit += CSTACKERR;
- if (L != g-> mainthread)
- return 0; /* only main thread can change the C stack */
- else if (ccalls <= CSTACKERR)
- return 0; /* handling overflow */
- else {
- int diff = limit - g->Cstacklimit;
- if (ccalls + diff <= CSTACKERR)
- return 0; /* new limit would cause an overflow */
- g->Cstacklimit = limit; /* set new limit */
- L->nCcalls += diff; /* correct 'nCcalls' */
- return limit - diff - CSTACKERR; /* success; return previous limit */
- }
-}
-
-
-/*
-** Decrement count of "C calls" and check for overflows. In case of
-** a stack overflow, check appropriate error ("regular" overflow or
-** overflow while handling stack overflow). If 'nCcalls' is smaller
-** than CSTACKERR but larger than CSTACKMARK, it means it has just
-** entered the "overflow zone", so the function raises an overflow
-** error. If 'nCcalls' is smaller than CSTACKMARK (which means it is
-** already handling an overflow) but larger than CSTACKERRMARK, does
-** not report an error (to allow message handling to work). Otherwise,
-** report a stack overflow while handling a stack overflow (probably
-** caused by a repeating error in the message handling function).
-*/
-
-void luaE_enterCcall (lua_State *L) {
- int ncalls = getCcalls(L);
- L->nCcalls--;
- if (ncalls <= CSTACKERR) { /* possible overflow? */
- luaE_freeCI(L); /* release unused CIs */
- ncalls = getCcalls(L); /* update call count */
- if (ncalls <= CSTACKERR) { /* still overflow? */
- if (ncalls <= CSTACKERRMARK) /* below error-handling zone? */
- luaD_throw(L, LUA_ERRERR); /* error while handling stack error */
- else if (ncalls >= CSTACKMARK) {
- /* not in error-handling zone; raise the error now */
- L->nCcalls = (CSTACKMARK - 1); /* enter error-handling zone */
- luaG_runerror(L, "C stack overflow");
- }
- /* else stack is in the error-handling zone;
- allow message handler to work */
- }
- }
+ UNUSED(L); UNUSED(limit);
+ return LUAI_MAXCCALLS; /* warning?? */
}
CallInfo *luaE_extendCI (lua_State *L) {
CallInfo *ci;
lua_assert(L->ci->next == NULL);
- luaE_enterCcall(L);
ci = luaM_new(L, CallInfo);
lua_assert(L->ci->next == NULL);
L->ci->next = ci;
@@ -175,13 +123,11 @@ void luaE_freeCI (lua_State *L) {
CallInfo *ci = L->ci;
CallInfo *next = ci->next;
ci->next = NULL;
- L->nCcalls += L->nci; /* add removed elements back to 'nCcalls' */
while ((ci = next) != NULL) {
next = ci->next;
luaM_free(L, ci);
L->nci--;
}
- L->nCcalls -= L->nci; /* adjust result */
}
@@ -194,7 +140,6 @@ void luaE_shrinkCI (lua_State *L) {
CallInfo *next;
if (ci == NULL)
return; /* no extra elements */
- L->nCcalls += L->nci; /* add removed elements back to 'nCcalls' */
while ((next = ci->next) != NULL) { /* two extra elements? */
CallInfo *next2 = next->next; /* next's next */
ci->next = next2; /* remove next from the list */
@@ -207,19 +152,40 @@ void luaE_shrinkCI (lua_State *L) {
ci = next2; /* continue */
}
}
- L->nCcalls -= L->nci; /* adjust result */
+}
+
+
+/*
+** Called when 'getCcalls(L)' larger or equal to LUAI_MAXCCALLS.
+** If equal, raises an overflow error. If value is larger than
+** LUAI_MAXCCALLS (which means it is handling an overflow) but
+** not much larger, does not report an error (to allow overflow
+** handling to work).
+*/
+void luaE_checkcstack (lua_State *L) {
+ if (getCcalls(L) == LUAI_MAXCCALLS)
+ luaG_runerror(L, "C stack overflow");
+ else if (getCcalls(L) >= (LUAI_MAXCCALLS / 10 * 11))
+ luaD_throw(L, LUA_ERRERR); /* error while handling stack error */
+}
+
+
+LUAI_FUNC void luaE_incCstack (lua_State *L) {
+ L->nCcalls++;
+ if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS))
+ luaE_checkcstack(L);
}
static void stack_init (lua_State *L1, lua_State *L) {
int i; CallInfo *ci;
/* initialize stack array */
- L1->stack = luaM_newvector(L, BASIC_STACK_SIZE, StackValue);
- L1->stacksize = BASIC_STACK_SIZE;
- for (i = 0; i < BASIC_STACK_SIZE; i++)
+ L1->stack = luaM_newvector(L, BASIC_STACK_SIZE + EXTRA_STACK, StackValue);
+ L1->tbclist = L1->stack;
+ for (i = 0; i < BASIC_STACK_SIZE + EXTRA_STACK; i++)
setnilvalue(s2v(L1->stack + i)); /* erase new stack */
L1->top = L1->stack;
- L1->stack_last = L1->stack + L1->stacksize - EXTRA_STACK;
+ L1->stack_last = L1->stack + BASIC_STACK_SIZE;
/* initialize first ci */
ci = &L1->base_ci;
ci->next = ci->previous = NULL;
@@ -240,7 +206,7 @@ static void freestack (lua_State *L) {
L->ci = &L->base_ci; /* free the entire 'ci' list */
luaE_freeCI(L);
lua_assert(L->nci == 0);
- luaM_freearray(L, L->stack, L->stacksize); /* free stack array */
+ luaM_freearray(L, L->stack, stacksize(L) + EXTRA_STACK); /* free stack */
}
@@ -248,24 +214,19 @@ static void freestack (lua_State *L) {
** Create registry table and its predefined values
*/
static void init_registry (lua_State *L, global_State *g) {
- TValue temp;
/* create registry */
Table *registry = luaH_new(L);
sethvalue(L, &g->l_registry, registry);
luaH_resize(L, registry, LUA_RIDX_LAST, 0);
/* registry[LUA_RIDX_MAINTHREAD] = L */
- setthvalue(L, &temp, L); /* temp = L */
- luaH_setint(L, registry, LUA_RIDX_MAINTHREAD, &temp);
- /* registry[LUA_RIDX_GLOBALS] = table of globals */
- sethvalue(L, &temp, luaH_new(L)); /* temp = new table (global table) */
- luaH_setint(L, registry, LUA_RIDX_GLOBALS, &temp);
+ setthvalue(L, ®istry->array[LUA_RIDX_MAINTHREAD - 1], L);
+ /* registry[LUA_RIDX_GLOBALS] = new table (table of globals) */
+ sethvalue(L, ®istry->array[LUA_RIDX_GLOBALS - 1], luaH_new(L));
}
/*
** open parts of the state that may cause memory-allocation errors.
-** ('g->nilvalue' being a nil value flags that the state was completely
-** build.)
*/
static void f_luaopen (lua_State *L, void *ud) {
global_State *g = G(L);
@@ -275,8 +236,8 @@ static void f_luaopen (lua_State *L, void *ud) {
luaS_init(L);
luaT_init(L);
luaX_init(L);
- g->gcrunning = 1; /* allow gc */
- setnilvalue(&g->nilvalue);
+ g->gcstp = 0; /* allow gc */
+ setnilvalue(&g->nilvalue); /* now state is complete */
luai_userstateopen(L);
}
@@ -290,8 +251,8 @@ static void preinit_thread (lua_State *L, global_State *g) {
L->stack = NULL;
L->ci = NULL;
L->nci = 0;
- L->stacksize = 0;
L->twups = L; /* thread has no upvalues */
+ L->nCcalls = 0;
L->errorJmp = NULL;
L->hook = NULL;
L->hookmask = 0;
@@ -307,10 +268,14 @@ static void preinit_thread (lua_State *L, global_State *g) {
static void close_state (lua_State *L) {
global_State *g = G(L);
- luaF_close(L, L->stack, CLOSEPROTECT); /* close all upvalues */
- luaC_freeallobjects(L); /* collect all objects */
- if (ttisnil(&g->nilvalue)) /* closing a fully built state? */
+ if (!completestate(g)) /* closing a partially built state? */
+ luaC_freeallobjects(L); /* just collect its objects */
+ else { /* closing a fully built state */
+ L->ci = &L->base_ci; /* unwind CallInfo list */
+ luaD_closeprotected(L, 1, LUA_OK); /* close all upvalues */
+ luaC_freeallobjects(L); /* collect all objects */
luai_userstateclose(L);
+ }
luaM_freearray(L, G(L)->strt.hash, G(L)->strt.size);
freestack(L);
lua_assert(gettotalbytes(g) == sizeof(LG));
@@ -335,7 +300,6 @@ LUA_API lua_State *lua_newthread (lua_State *L) {
setthvalue2s(L, L->top, L1);
api_incr_top(L);
preinit_thread(L1, g);
- L1->nCcalls = getCcalls(L);
L1->hookmask = L->hookmask;
L1->basehookcount = L->basehookcount;
L1->hook = L->hook;
@@ -352,7 +316,7 @@ LUA_API lua_State *lua_newthread (lua_State *L) {
void luaE_freethread (lua_State *L, lua_State *L1) {
LX *l = fromstate(L1);
- luaF_close(L1, L1->stack, NOCLOSINGMETH); /* close all upvalues */
+ luaF_closeupval(L1, L1->stack); /* close all upvalues */
lua_assert(L1->openupval == NULL);
luai_userstatefree(L, L1);
freestack(L1);
@@ -360,23 +324,29 @@ void luaE_freethread (lua_State *L, lua_State *L1) {
}
-int lua_resetthread (lua_State *L) {
- CallInfo *ci;
- int status;
- lua_lock(L);
- L->ci = ci = &L->base_ci; /* unwind CallInfo list */
+int luaE_resetthread (lua_State *L, int status) {
+ CallInfo *ci = L->ci = &L->base_ci; /* unwind CallInfo list */
setnilvalue(s2v(L->stack)); /* 'function' entry for basic 'ci' */
ci->func = L->stack;
ci->callstatus = CIST_C;
- status = luaF_close(L, L->stack, CLOSEPROTECT);
- if (status != CLOSEPROTECT) /* real errors? */
- luaD_seterrorobj(L, status, L->stack + 1);
- else {
+ if (status == LUA_YIELD)
status = LUA_OK;
+ L->status = LUA_OK; /* so it can run __close metamethods */
+ status = luaD_closeprotected(L, 1, status);
+ if (status != LUA_OK) /* errors? */
+ luaD_seterrorobj(L, status, L->stack + 1);
+ else
L->top = L->stack + 1;
- }
ci->top = L->top + LUA_MINSTACK;
- L->status = status;
+ luaD_reallocstack(L, cast_int(ci->top - L->stack), 0);
+ return status;
+}
+
+
+LUA_API int lua_resetthread (lua_State *L) {
+ int status;
+ lua_lock(L);
+ status = luaE_resetthread(L, L->status);
lua_unlock(L);
return status;
}
@@ -396,7 +366,6 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
preinit_thread(L, g);
g->allgc = obj2gco(L); /* by now, only object is the main thread */
L->next = NULL;
- g->Cstacklimit = L->nCcalls = LUAI_MAXCSTACK + CSTACKERR;
incnny(L); /* main thread is always non yieldable */
g->frealloc = f;
g->ud = ud;
@@ -404,13 +373,14 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
g->ud_warn = NULL;
g->mainthread = L;
g->seed = luai_makeseed(L);
- g->gcrunning = 0; /* no GC while building state */
+ g->gcstp = GCSTPGC; /* no GC while building state */
g->strt.size = g->strt.nuse = 0;
g->strt.hash = NULL;
setnilvalue(&g->l_registry);
g->panic = NULL;
g->gcstate = GCSpause;
g->gckind = KGC_INC;
+ g->gcstopem = 0;
g->gcemergency = 0;
g->finobj = g->tobefnz = g->fixedgc = NULL;
g->firstold1 = g->survival = g->old1 = g->reallyold = NULL;
diff --git a/src/lua/lstate.h b/src/lua/lstate.h
index c1c38204..61e82cde 100644
--- a/src/lua/lstate.h
+++ b/src/lua/lstate.h
@@ -87,49 +87,13 @@
/*
-** About 'nCcalls': each thread in Lua (a lua_State) keeps a count of
-** how many "C calls" it still can do in the C stack, to avoid C-stack
-** overflow. This count is very rough approximation; it considers only
-** recursive functions inside the interpreter, as non-recursive calls
-** can be considered using a fixed (although unknown) amount of stack
-** space.
-**
-** The count has two parts: the lower part is the count itself; the
-** higher part counts the number of non-yieldable calls in the stack.
-** (They are together so that we can change both with one instruction.)
-**
-** Because calls to external C functions can use an unknown amount
-** of space (e.g., functions using an auxiliary buffer), calls
-** to these functions add more than one to the count (see CSTACKCF).
-**
-** The proper count excludes the number of CallInfo structures allocated
-** by Lua, as a kind of "potential" calls. So, when Lua calls a function
-** (and "consumes" one CallInfo), it needs neither to decrement nor to
-** check 'nCcalls', as its use of C stack is already accounted for.
+** About 'nCcalls': This count has two parts: the lower 16 bits counts
+** the number of recursive invocations in the C stack; the higher
+** 16 bits counts the number of non-yieldable calls in the stack.
+** (They are together so that we can change and save both with one
+** instruction.)
*/
-/* number of "C stack slots" used by an external C function */
-#define CSTACKCF 10
-
-
-/*
-** The C-stack size is sliced in the following zones:
-** - larger than CSTACKERR: normal stack;
-** - [CSTACKMARK, CSTACKERR]: buffer zone to signal a stack overflow;
-** - [CSTACKCF, CSTACKERRMARK]: error-handling zone;
-** - below CSTACKERRMARK: buffer zone to signal overflow during overflow;
-** (Because the counter can be decremented CSTACKCF at once, we need
-** the so called "buffer zones", with at least that size, to properly
-** detect a change from one zone to the next.)
-*/
-#define CSTACKERR (8 * CSTACKCF)
-#define CSTACKMARK (CSTACKERR - (CSTACKCF + 2))
-#define CSTACKERRMARK (CSTACKCF + 2)
-
-
-/* initial limit for the C-stack of threads */
-#define CSTACKTHREAD (2 * CSTACKERR)
-
/* true if this thread does not have non-yieldable calls in the stack */
#define yieldable(L) (((L)->nCcalls & 0xffff0000) == 0)
@@ -144,13 +108,8 @@
/* Decrement the number of non-yieldable calls */
#define decnny(L) ((L)->nCcalls -= 0x10000)
-/* Increment the number of non-yieldable calls and decrement nCcalls */
-#define incXCcalls(L) ((L)->nCcalls += 0x10000 - CSTACKCF)
-
-/* Decrement the number of non-yieldable calls and increment nCcalls */
-#define decXCcalls(L) ((L)->nCcalls -= 0x10000 - CSTACKCF)
-
-
+/* Non-yieldable call increment */
+#define nyci (0x10000 | 1)
@@ -168,12 +127,20 @@ struct lua_longjmp; /* defined in ldo.c */
#endif
-/* extra stack space to handle TM calls and some other extras */
+/*
+** Extra stack space to handle TM calls and some other extras. This
+** space is not included in 'stack_last'. It is used only to avoid stack
+** checks, either because the element will be promptly popped or because
+** there will be a stack check soon after the push. Function frames
+** never use this extra space, so it does not need to be kept clean.
+*/
#define EXTRA_STACK 5
#define BASIC_STACK_SIZE (2*LUA_MINSTACK)
+#define stacksize(th) cast_int((th)->stack_last - (th)->stack)
+
/* kinds of Garbage Collection */
#define KGC_INC 0 /* incremental gc */
@@ -189,6 +156,18 @@ typedef struct stringtable {
/*
** Information about a call.
+** About union 'u':
+** - field 'l' is used only for Lua functions;
+** - field 'c' is used only for C functions.
+** About union 'u2':
+** - field 'funcidx' is used only by C functions while doing a
+** protected call;
+** - field 'nyield' is used only while a function is "doing" an
+** yield (from the yield until the next resume);
+** - field 'nres' is used only while closing tbc variables when
+** returning from a function;
+** - field 'transferinfo' is used only during call/returnhooks,
+** before the function starts or after it ends.
*/
typedef struct CallInfo {
StkId func; /* function index in the stack */
@@ -209,6 +188,7 @@ typedef struct CallInfo {
union {
int funcidx; /* called-function index */
int nyield; /* number of values yielded */
+ int nres; /* number of values returned */
struct { /* info about transferred values (for call/return hooks) */
unsigned short ftransfer; /* offset of first value transferred */
unsigned short ntransfer; /* number of values transferred */
@@ -224,16 +204,34 @@ typedef struct CallInfo {
*/
#define CIST_OAH (1<<0) /* original value of 'allowhook' */
#define CIST_C (1<<1) /* call is running a C function */
-#define CIST_HOOKED (1<<2) /* call is running a debug hook */
-#define CIST_YPCALL (1<<3) /* call is a yieldable protected call */
-#define CIST_TAIL (1<<4) /* call was tail called */
-#define CIST_HOOKYIELD (1<<5) /* last hook called yielded */
-#define CIST_FIN (1<<6) /* call is running a finalizer */
-#define CIST_TRAN (1<<7) /* 'ci' has transfer information */
+#define CIST_FRESH (1<<2) /* call is on a fresh "luaV_execute" frame */
+#define CIST_HOOKED (1<<3) /* call is running a debug hook */
+#define CIST_YPCALL (1<<4) /* doing a yieldable protected call */
+#define CIST_TAIL (1<<5) /* call was tail called */
+#define CIST_HOOKYIELD (1<<6) /* last hook called yielded */
+#define CIST_FIN (1<<7) /* function "called" a finalizer */
+#define CIST_TRAN (1<<8) /* 'ci' has transfer information */
+#define CIST_CLSRET (1<<9) /* function is closing tbc variables */
+/* Bits 10-12 are used for CIST_RECST (see below) */
+#define CIST_RECST 10
#if defined(LUA_COMPAT_LT_LE)
-#define CIST_LEQ (1<<8) /* using __lt for __le */
+#define CIST_LEQ (1<<13) /* using __lt for __le */
#endif
+
+/*
+** Field CIST_RECST stores the "recover status", used to keep the error
+** status while closing to-be-closed variables in coroutines, so that
+** Lua can correctly resume after an yield from a __close method called
+** because of an error. (Three bits are enough for error status.)
+*/
+#define getcistrecst(ci) (((ci)->callstatus >> CIST_RECST) & 7)
+#define setcistrecst(ci,st) \
+ check_exp(((st) & 7) == (st), /* status must fit in three bits */ \
+ ((ci)->callstatus = ((ci)->callstatus & ~(7 << CIST_RECST)) \
+ | ((st) << CIST_RECST)))
+
+
/* active function is a Lua function */
#define isLua(ci) (!((ci)->callstatus & CIST_C))
@@ -262,9 +260,10 @@ typedef struct global_State {
lu_byte currentwhite;
lu_byte gcstate; /* state of garbage collector */
lu_byte gckind; /* kind of GC running */
+ lu_byte gcstopem; /* stops emergency collections */
lu_byte genminormul; /* control for minor generational collections */
lu_byte genmajormul; /* control for major generational collections */
- lu_byte gcrunning; /* true if GC is running */
+ lu_byte gcstp; /* control whether GC is running */
lu_byte gcemergency; /* true if this is an emergency collection */
lu_byte gcpause; /* size of pause between successive GCs */
lu_byte gcstepmul; /* GC "speed" */
@@ -296,7 +295,6 @@ typedef struct global_State {
TString *strcache[STRCACHE_N][STRCACHE_M]; /* cache for strings in API */
lua_WarnFunction warnf; /* warning function */
void *ud_warn; /* auxiliary data to 'warnf' */
- unsigned int Cstacklimit; /* current limit for the C stack */
} global_State;
@@ -311,18 +309,18 @@ struct lua_State {
StkId top; /* first free slot in the stack */
global_State *l_G;
CallInfo *ci; /* call info for current function */
- StkId stack_last; /* last free slot in the stack */
+ StkId stack_last; /* end of stack (last element + 1) */
StkId stack; /* stack base */
UpVal *openupval; /* list of open upvalues in this stack */
+ StkId tbclist; /* list of to-be-closed variables */
GCObject *gclist;
struct lua_State *twups; /* list of threads with open upvalues */
struct lua_longjmp *errorJmp; /* current error recover point */
CallInfo base_ci; /* CallInfo for first level (C calling Lua) */
volatile lua_Hook hook;
ptrdiff_t errfunc; /* current error handling function (stack index) */
- l_uint32 nCcalls; /* number of allowed nested C calls - 'nci' */
+ l_uint32 nCcalls; /* number of nested (non-yieldable | C) calls */
int oldpc; /* last pc traced */
- int stacksize;
int basehookcount;
int hookcount;
volatile l_signalT hookmask;
@@ -331,6 +329,12 @@ struct lua_State {
#define G(L) (L->l_G)
+/*
+** 'g->nilvalue' being a nil value flags that the state was completely
+** build.
+*/
+#define completestate(g) ttisnil(&g->nilvalue)
+
/*
** Union of all collectable objects (only for conversions)
@@ -389,12 +393,12 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1);
LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L);
LUAI_FUNC void luaE_freeCI (lua_State *L);
LUAI_FUNC void luaE_shrinkCI (lua_State *L);
-LUAI_FUNC void luaE_enterCcall (lua_State *L);
+LUAI_FUNC void luaE_checkcstack (lua_State *L);
+LUAI_FUNC void luaE_incCstack (lua_State *L);
LUAI_FUNC void luaE_warning (lua_State *L, const char *msg, int tocont);
LUAI_FUNC void luaE_warnerror (lua_State *L, const char *where);
+LUAI_FUNC int luaE_resetthread (lua_State *L, int status);
-#define luaE_exitCcall(L) ((L)->nCcalls++)
-
#endif
diff --git a/src/lua/lstring.c b/src/lua/lstring.c
index 6f157473..13dcaf42 100644
--- a/src/lua/lstring.c
+++ b/src/lua/lstring.c
@@ -22,16 +22,6 @@
#include "lstring.h"
-/*
-** Lua will use at most ~(2^LUAI_HASHLIMIT) bytes from a long string to
-** compute its hash
-*/
-#if !defined(LUAI_HASHLIMIT)
-#define LUAI_HASHLIMIT 5
-#endif
-
-
-
/*
** Maximum size for string table.
*/
@@ -50,10 +40,9 @@ int luaS_eqlngstr (TString *a, TString *b) {
}
-unsigned int luaS_hash (const char *str, size_t l, unsigned int seed,
- size_t step) {
+unsigned int luaS_hash (const char *str, size_t l, unsigned int seed) {
unsigned int h = seed ^ cast_uint(l);
- for (; l >= step; l -= step)
+ for (; l > 0; l--)
h ^= ((h<<5) + (h>>2) + cast_byte(str[l - 1]));
return h;
}
@@ -63,8 +52,7 @@ unsigned int luaS_hashlongstr (TString *ts) {
lua_assert(ts->tt == LUA_VLNGSTR);
if (ts->extra == 0) { /* no hash? */
size_t len = ts->u.lnglen;
- size_t step = (len >> LUAI_HASHLIMIT) + 1;
- ts->hash = luaS_hash(getstr(ts), len, ts->hash, step);
+ ts->hash = luaS_hash(getstr(ts), len, ts->hash);
ts->extra = 1; /* now it has its hash */
}
return ts->hash;
@@ -101,7 +89,7 @@ void luaS_resize (lua_State *L, int nsize) {
if (nsize < osize) /* shrinking table? */
tablerehash(tb->hash, osize, nsize); /* depopulate shrinking part */
newvect = luaM_reallocvector(L, tb->hash, osize, nsize, TString*);
- if (unlikely(newvect == NULL)) { /* reallocation failed? */
+ if (l_unlikely(newvect == NULL)) { /* reallocation failed? */
if (nsize < osize) /* was it shrinking table? */
tablerehash(tb->hash, nsize, osize); /* restore to original size */
/* leave table as it was */
@@ -184,7 +172,7 @@ void luaS_remove (lua_State *L, TString *ts) {
static void growstrtab (lua_State *L, stringtable *tb) {
- if (unlikely(tb->nuse == MAX_INT)) { /* too many strings? */
+ if (l_unlikely(tb->nuse == MAX_INT)) { /* too many strings? */
luaC_fullgc(L, 1); /* try to free some... */
if (tb->nuse == MAX_INT) /* still too many? */
luaM_error(L); /* cannot even create a message... */
@@ -201,7 +189,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
TString *ts;
global_State *g = G(L);
stringtable *tb = &g->strt;
- unsigned int h = luaS_hash(str, l, g->seed, 1);
+ unsigned int h = luaS_hash(str, l, g->seed);
TString **list = &tb->hash[lmod(h, tb->size)];
lua_assert(str != NULL); /* otherwise 'memcmp'/'memcpy' are undefined */
for (ts = *list; ts != NULL; ts = ts->u.hnext) {
@@ -235,7 +223,7 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) {
return internshrstr(L, str, l);
else {
TString *ts;
- if (unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char)))
+ if (l_unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char)))
luaM_toobig(L);
ts = luaS_createlngstrobj(L, l);
memcpy(getstr(ts), str, l * sizeof(char));
@@ -271,7 +259,7 @@ Udata *luaS_newudata (lua_State *L, size_t s, int nuvalue) {
Udata *u;
int i;
GCObject *o;
- if (unlikely(s > MAX_SIZE - udatamemoffset(nuvalue)))
+ if (l_unlikely(s > MAX_SIZE - udatamemoffset(nuvalue)))
luaM_toobig(L);
o = luaC_newobj(L, LUA_VUSERDATA, sizeudata(nuvalue, s));
u = gco2u(o);
diff --git a/src/lua/lstring.h b/src/lua/lstring.h
index a413a9d3..450c2390 100644
--- a/src/lua/lstring.h
+++ b/src/lua/lstring.h
@@ -41,8 +41,7 @@
#define eqshrstr(a,b) check_exp((a)->tt == LUA_VSHRSTR, (a) == (b))
-LUAI_FUNC unsigned int luaS_hash (const char *str, size_t l,
- unsigned int seed, size_t step);
+LUAI_FUNC unsigned int luaS_hash (const char *str, size_t l, unsigned int seed);
LUAI_FUNC unsigned int luaS_hashlongstr (TString *ts);
LUAI_FUNC int luaS_eqlngstr (TString *a, TString *b);
LUAI_FUNC void luaS_resize (lua_State *L, int newsize);
diff --git a/src/lua/lstrlib.c b/src/lua/lstrlib.c
index 2ba8bde4..0b4fdbb7 100644
--- a/src/lua/lstrlib.c
+++ b/src/lua/lstrlib.c
@@ -152,8 +152,9 @@ static int str_rep (lua_State *L) {
const char *s = luaL_checklstring(L, 1, &l);
lua_Integer n = luaL_checkinteger(L, 2);
const char *sep = luaL_optlstring(L, 3, "", &lsep);
- if (n <= 0) lua_pushliteral(L, "");
- else if (l + lsep < l || l + lsep > MAXSIZE / n) /* may overflow? */
+ if (n <= 0)
+ lua_pushliteral(L, "");
+ else if (l_unlikely(l + lsep < l || l + lsep > MAXSIZE / n))
return luaL_error(L, "resulting string too large");
else {
size_t totallen = (size_t)n * l + (size_t)(n - 1) * lsep;
@@ -181,7 +182,7 @@ static int str_byte (lua_State *L) {
size_t pose = getendpos(L, 3, pi, l);
int n, i;
if (posi > pose) return 0; /* empty interval; return no values */
- if (pose - posi >= (size_t)INT_MAX) /* arithmetic overflow? */
+ if (l_unlikely(pose - posi >= (size_t)INT_MAX)) /* arithmetic overflow? */
return luaL_error(L, "string slice too long");
n = (int)(pose - posi) + 1;
luaL_checkstack(L, n, "string slice too long");
@@ -235,7 +236,7 @@ static int str_dump (lua_State *L) {
luaL_checktype(L, 1, LUA_TFUNCTION);
lua_settop(L, 1); /* ensure function is on the top of the stack */
state.init = 0;
- if (lua_dump(L, writer, &state, strip) != 0)
+ if (l_unlikely(lua_dump(L, writer, &state, strip) != 0))
return luaL_error(L, "unable to dump given function");
luaL_pushresult(&state.B);
return 1;
@@ -275,7 +276,8 @@ static int tonum (lua_State *L, int arg) {
static void trymt (lua_State *L, const char *mtname) {
lua_settop(L, 2); /* back to the original arguments */
- if (lua_type(L, 2) == LUA_TSTRING || !luaL_getmetafield(L, 2, mtname))
+ if (l_unlikely(lua_type(L, 2) == LUA_TSTRING ||
+ !luaL_getmetafield(L, 2, mtname)))
luaL_error(L, "attempt to %s a '%s' with a '%s'", mtname + 2,
luaL_typename(L, -2), luaL_typename(L, -1));
lua_insert(L, -3); /* put metamethod before arguments */
@@ -383,7 +385,8 @@ static const char *match (MatchState *ms, const char *s, const char *p);
static int check_capture (MatchState *ms, int l) {
l -= '1';
- if (l < 0 || l >= ms->level || ms->capture[l].len == CAP_UNFINISHED)
+ if (l_unlikely(l < 0 || l >= ms->level ||
+ ms->capture[l].len == CAP_UNFINISHED))
return luaL_error(ms->L, "invalid capture index %%%d", l + 1);
return l;
}
@@ -400,14 +403,14 @@ static int capture_to_close (MatchState *ms) {
static const char *classend (MatchState *ms, const char *p) {
switch (*p++) {
case L_ESC: {
- if (p == ms->p_end)
+ if (l_unlikely(p == ms->p_end))
luaL_error(ms->L, "malformed pattern (ends with '%%')");
return p+1;
}
case '[': {
if (*p == '^') p++;
do { /* look for a ']' */
- if (p == ms->p_end)
+ if (l_unlikely(p == ms->p_end))
luaL_error(ms->L, "malformed pattern (missing ']')");
if (*(p++) == L_ESC && p < ms->p_end)
p++; /* skip escapes (e.g. '%]') */
@@ -482,7 +485,7 @@ static int singlematch (MatchState *ms, const char *s, const char *p,
static const char *matchbalance (MatchState *ms, const char *s,
const char *p) {
- if (p >= ms->p_end - 1)
+ if (l_unlikely(p >= ms->p_end - 1))
luaL_error(ms->L, "malformed pattern (missing arguments to '%%b')");
if (*s != *p) return NULL;
else {
@@ -565,7 +568,7 @@ static const char *match_capture (MatchState *ms, const char *s, int l) {
static const char *match (MatchState *ms, const char *s, const char *p) {
- if (ms->matchdepth-- == 0)
+ if (l_unlikely(ms->matchdepth-- == 0))
luaL_error(ms->L, "pattern too complex");
init: /* using goto's to optimize tail recursion */
if (p != ms->p_end) { /* end of pattern? */
@@ -599,7 +602,7 @@ static const char *match (MatchState *ms, const char *s, const char *p) {
case 'f': { /* frontier? */
const char *ep; char previous;
p += 2;
- if (*p != '[')
+ if (l_unlikely(*p != '['))
luaL_error(ms->L, "missing '[' after '%%f' in pattern");
ep = classend(ms, p); /* points to what is next */
previous = (s == ms->src_init) ? '\0' : *(s - 1);
@@ -699,7 +702,7 @@ static const char *lmemfind (const char *s1, size_t l1,
static size_t get_onecapture (MatchState *ms, int i, const char *s,
const char *e, const char **cap) {
if (i >= ms->level) {
- if (i != 0)
+ if (l_unlikely(i != 0))
luaL_error(ms->L, "invalid capture index %%%d", i + 1);
*cap = s;
return e - s;
@@ -707,7 +710,7 @@ static size_t get_onecapture (MatchState *ms, int i, const char *s,
else {
ptrdiff_t capl = ms->capture[i].len;
*cap = ms->capture[i].init;
- if (capl == CAP_UNFINISHED)
+ if (l_unlikely(capl == CAP_UNFINISHED))
luaL_error(ms->L, "unfinished capture");
else if (capl == CAP_POSITION)
lua_pushinteger(ms->L, (ms->capture[i].init - ms->src_init) + 1);
@@ -926,7 +929,7 @@ static int add_value (MatchState *ms, luaL_Buffer *b, const char *s,
luaL_addlstring(b, s, e - s); /* keep original text */
return 0; /* no changes */
}
- else if (!lua_isstring(L, -1))
+ else if (l_unlikely(!lua_isstring(L, -1)))
return luaL_error(L, "invalid replacement value (a %s)",
luaL_typename(L, -1));
else {
@@ -1058,7 +1061,7 @@ static int lua_number2strx (lua_State *L, char *buff, int sz,
for (i = 0; i < n; i++)
buff[i] = toupper(uchar(buff[i]));
}
- else if (fmt[SIZELENMOD] != 'a')
+ else if (l_unlikely(fmt[SIZELENMOD] != 'a'))
return luaL_error(L, "modifiers for format '%%a'/'%%A' not implemented");
return n;
}
@@ -1087,13 +1090,31 @@ static int lua_number2strx (lua_State *L, char *buff, int sz,
/* valid flags in a format specification */
-#if !defined(L_FMTFLAGS)
-#define L_FMTFLAGS "-+ #0"
+#if !defined(L_FMTFLAGSF)
+
+/* valid flags for a, A, e, E, f, F, g, and G conversions */
+#define L_FMTFLAGSF "-+#0 "
+
+/* valid flags for o, x, and X conversions */
+#define L_FMTFLAGSX "-#0"
+
+/* valid flags for d and i conversions */
+#define L_FMTFLAGSI "-+0 "
+
+/* valid flags for u conversions */
+#define L_FMTFLAGSU "-0"
+
+/* valid flags for c, p, and s conversions */
+#define L_FMTFLAGSC "-"
+
#endif
/*
-** maximum size of each format specification (such as "%-099.99d")
+** Maximum size of each format specification (such as "%-099.99d"):
+** Initial '%', flags (up to 5), width (2), period, precision (2),
+** length modifier (8), conversion specifier, and final '\0', plus some
+** extra.
*/
#define MAX_FORMAT 32
@@ -1186,25 +1207,53 @@ static void addliteral (lua_State *L, luaL_Buffer *b, int arg) {
}
-static const char *scanformat (lua_State *L, const char *strfrmt, char *form) {
- const char *p = strfrmt;
- while (*p != '\0' && strchr(L_FMTFLAGS, *p) != NULL) p++; /* skip flags */
- if ((size_t)(p - strfrmt) >= sizeof(L_FMTFLAGS)/sizeof(char))
- luaL_error(L, "invalid format (repeated flags)");
- if (isdigit(uchar(*p))) p++; /* skip width */
- if (isdigit(uchar(*p))) p++; /* (2 digits at most) */
- if (*p == '.') {
- p++;
- if (isdigit(uchar(*p))) p++; /* skip precision */
- if (isdigit(uchar(*p))) p++; /* (2 digits at most) */
+static const char *get2digits (const char *s) {
+ if (isdigit(uchar(*s))) {
+ s++;
+ if (isdigit(uchar(*s))) s++; /* (2 digits at most) */
}
- if (isdigit(uchar(*p)))
- luaL_error(L, "invalid format (width or precision too long)");
+ return s;
+}
+
+
+/*
+** Check whether a conversion specification is valid. When called,
+** first character in 'form' must be '%' and last character must
+** be a valid conversion specifier. 'flags' are the accepted flags;
+** 'precision' signals whether to accept a precision.
+*/
+static void checkformat (lua_State *L, const char *form, const char *flags,
+ int precision) {
+ const char *spec = form + 1; /* skip '%' */
+ spec += strspn(spec, flags); /* skip flags */
+ if (*spec != '0') { /* a width cannot start with '0' */
+ spec = get2digits(spec); /* skip width */
+ if (*spec == '.' && precision) {
+ spec++;
+ spec = get2digits(spec); /* skip precision */
+ }
+ }
+ if (!isalpha(uchar(*spec))) /* did not go to the end? */
+ luaL_error(L, "invalid conversion specification: '%s'", form);
+}
+
+
+/*
+** Get a conversion specification and copy it to 'form'.
+** Return the address of its last character.
+*/
+static const char *getformat (lua_State *L, const char *strfrmt,
+ char *form) {
+ /* spans flags, width, and precision ('0' is included as a flag) */
+ size_t len = strspn(strfrmt, L_FMTFLAGSF "123456789.");
+ len++; /* adds following character (should be the specifier) */
+ /* still needs space for '%', '\0', plus a length modifier */
+ if (len >= MAX_FORMAT - 10)
+ luaL_error(L, "invalid format (too long)");
*(form++) = '%';
- memcpy(form, strfrmt, ((p - strfrmt) + 1) * sizeof(char));
- form += (p - strfrmt) + 1;
- *form = '\0';
- return p;
+ memcpy(form, strfrmt, len * sizeof(char));
+ *(form + len) = '\0';
+ return strfrmt + len - 1;
}
@@ -1227,6 +1276,7 @@ static int str_format (lua_State *L) {
size_t sfl;
const char *strfrmt = luaL_checklstring(L, arg, &sfl);
const char *strfrmt_end = strfrmt+sfl;
+ const char *flags;
luaL_Buffer b;
luaL_buffinit(L, &b);
while (strfrmt < strfrmt_end) {
@@ -1236,25 +1286,35 @@ static int str_format (lua_State *L) {
luaL_addchar(&b, *strfrmt++); /* %% */
else { /* format item */
char form[MAX_FORMAT]; /* to store the format ('%...') */
- int maxitem = MAX_ITEM;
- char *buff = luaL_prepbuffsize(&b, maxitem); /* to put formatted item */
- int nb = 0; /* number of bytes in added item */
+ int maxitem = MAX_ITEM; /* maximum length for the result */
+ char *buff = luaL_prepbuffsize(&b, maxitem); /* to put result */
+ int nb = 0; /* number of bytes in result */
if (++arg > top)
return luaL_argerror(L, arg, "no value");
- strfrmt = scanformat(L, strfrmt, form);
+ strfrmt = getformat(L, strfrmt, form);
switch (*strfrmt++) {
case 'c': {
+ checkformat(L, form, L_FMTFLAGSC, 0);
nb = l_sprintf(buff, maxitem, form, (int)luaL_checkinteger(L, arg));
break;
}
case 'd': case 'i':
- case 'o': case 'u': case 'x': case 'X': {
+ flags = L_FMTFLAGSI;
+ goto intcase;
+ case 'u':
+ flags = L_FMTFLAGSU;
+ goto intcase;
+ case 'o': case 'x': case 'X':
+ flags = L_FMTFLAGSX;
+ intcase: {
lua_Integer n = luaL_checkinteger(L, arg);
+ checkformat(L, form, flags, 1);
addlenmod(form, LUA_INTEGER_FRMLEN);
nb = l_sprintf(buff, maxitem, form, (LUAI_UACINT)n);
break;
}
case 'a': case 'A':
+ checkformat(L, form, L_FMTFLAGSF, 1);
addlenmod(form, LUA_NUMBER_FRMLEN);
nb = lua_number2strx(L, buff, maxitem, form,
luaL_checknumber(L, arg));
@@ -1265,12 +1325,14 @@ static int str_format (lua_State *L) {
/* FALLTHROUGH */
case 'e': case 'E': case 'g': case 'G': {
lua_Number n = luaL_checknumber(L, arg);
+ checkformat(L, form, L_FMTFLAGSF, 1);
addlenmod(form, LUA_NUMBER_FRMLEN);
nb = l_sprintf(buff, maxitem, form, (LUAI_UACNUMBER)n);
break;
}
case 'p': {
const void *p = lua_topointer(L, arg);
+ checkformat(L, form, L_FMTFLAGSC, 0);
if (p == NULL) { /* avoid calling 'printf' with argument NULL */
p = "(null)"; /* result */
form[strlen(form) - 1] = 's'; /* format it as a string */
@@ -1291,7 +1353,8 @@ static int str_format (lua_State *L) {
luaL_addvalue(&b); /* keep entire string */
else {
luaL_argcheck(L, l == strlen(s), arg, "string contains zeros");
- if (!strchr(form, '.') && l >= 100) {
+ checkformat(L, form, L_FMTFLAGSC, 1);
+ if (strchr(form, '.') == NULL && l >= 100) {
/* no precision and string is too long to be formatted */
luaL_addvalue(&b); /* keep entire string */
}
@@ -1349,26 +1412,6 @@ static const union {
} nativeendian = {1};
-/* dummy structure to get native alignment requirements */
-struct cD {
- char c;
- union { double d; void *p; lua_Integer i; lua_Number n; } u;
-};
-
-#define MAXALIGN (offsetof(struct cD, u))
-
-
-/*
-** Union for serializing floats
-*/
-typedef union Ftypes {
- float f;
- double d;
- lua_Number n;
- char buff[5 * sizeof(lua_Number)]; /* enough for any float type */
-} Ftypes;
-
-
/*
** information to pack/unpack stuff
*/
@@ -1385,7 +1428,9 @@ typedef struct Header {
typedef enum KOption {
Kint, /* signed integers */
Kuint, /* unsigned integers */
- Kfloat, /* floating-point numbers */
+ Kfloat, /* single-precision floating-point numbers */
+ Knumber, /* Lua "native" floating-point numbers */
+ Kdouble, /* double-precision floating-point numbers */
Kchar, /* fixed-length strings */
Kstring, /* strings with prefixed length */
Kzstr, /* zero-terminated strings */
@@ -1420,7 +1465,7 @@ static int getnum (const char **fmt, int df) {
*/
static int getnumlimit (Header *h, const char **fmt, int df) {
int sz = getnum(fmt, df);
- if (sz > MAXINTSIZE || sz <= 0)
+ if (l_unlikely(sz > MAXINTSIZE || sz <= 0))
return luaL_error(h->L, "integral size (%d) out of limits [1,%d]",
sz, MAXINTSIZE);
return sz;
@@ -1441,6 +1486,8 @@ static void initheader (lua_State *L, Header *h) {
** Read and classify next option. 'size' is filled with option's size.
*/
static KOption getoption (Header *h, const char **fmt, int *size) {
+ /* dummy structure to get native alignment requirements */
+ struct cD { char c; union { LUAI_MAXALIGN; } u; };
int opt = *((*fmt)++);
*size = 0; /* default */
switch (opt) {
@@ -1454,14 +1501,14 @@ static KOption getoption (Header *h, const char **fmt, int *size) {
case 'J': *size = sizeof(lua_Integer); return Kuint;
case 'T': *size = sizeof(size_t); return Kuint;
case 'f': *size = sizeof(float); return Kfloat;
- case 'd': *size = sizeof(double); return Kfloat;
- case 'n': *size = sizeof(lua_Number); return Kfloat;
+ case 'n': *size = sizeof(lua_Number); return Knumber;
+ case 'd': *size = sizeof(double); return Kdouble;
case 'i': *size = getnumlimit(h, fmt, sizeof(int)); return Kint;
case 'I': *size = getnumlimit(h, fmt, sizeof(int)); return Kuint;
case 's': *size = getnumlimit(h, fmt, sizeof(size_t)); return Kstring;
case 'c':
*size = getnum(fmt, -1);
- if (*size == -1)
+ if (l_unlikely(*size == -1))
luaL_error(h->L, "missing size for format option 'c'");
return Kchar;
case 'z': return Kzstr;
@@ -1471,7 +1518,11 @@ static KOption getoption (Header *h, const char **fmt, int *size) {
case '<': h->islittle = 1; break;
case '>': h->islittle = 0; break;
case '=': h->islittle = nativeendian.little; break;
- case '!': h->maxalign = getnumlimit(h, fmt, MAXALIGN); break;
+ case '!': {
+ const int maxalign = offsetof(struct cD, u);
+ h->maxalign = getnumlimit(h, fmt, maxalign);
+ break;
+ }
default: luaL_error(h->L, "invalid format option '%c'", opt);
}
return Knop;
@@ -1500,7 +1551,7 @@ static KOption getdetails (Header *h, size_t totalsize,
else {
if (align > h->maxalign) /* enforce maximum alignment */
align = h->maxalign;
- if ((align & (align - 1)) != 0) /* is 'align' not a power of 2? */
+ if (l_unlikely((align & (align - 1)) != 0)) /* not a power of 2? */
luaL_argerror(h->L, 1, "format asks for alignment not power of 2");
*ntoalign = (align - (int)(totalsize & (align - 1))) & (align - 1);
}
@@ -1535,12 +1586,10 @@ static void packint (luaL_Buffer *b, lua_Unsigned n,
** Copy 'size' bytes from 'src' to 'dest', correcting endianness if
** given 'islittle' is different from native endianness.
*/
-static void copywithendian (volatile char *dest, volatile const char *src,
+static void copywithendian (char *dest, const char *src,
int size, int islittle) {
- if (islittle == nativeendian.little) {
- while (size-- != 0)
- *(dest++) = *(src++);
- }
+ if (islittle == nativeendian.little)
+ memcpy(dest, src, size);
else {
dest += size - 1;
while (size-- != 0)
@@ -1583,15 +1632,27 @@ static int str_pack (lua_State *L) {
packint(&b, (lua_Unsigned)n, h.islittle, size, 0);
break;
}
- case Kfloat: { /* floating-point options */
- volatile Ftypes u;
- char *buff = luaL_prepbuffsize(&b, size);
- lua_Number n = luaL_checknumber(L, arg); /* get argument */
- if (size == sizeof(u.f)) u.f = (float)n; /* copy it into 'u' */
- else if (size == sizeof(u.d)) u.d = (double)n;
- else u.n = n;
- /* move 'u' to final result, correcting endianness if needed */
- copywithendian(buff, u.buff, size, h.islittle);
+ case Kfloat: { /* C float */
+ float f = (float)luaL_checknumber(L, arg); /* get argument */
+ char *buff = luaL_prepbuffsize(&b, sizeof(f));
+ /* move 'f' to final result, correcting endianness if needed */
+ copywithendian(buff, (char *)&f, sizeof(f), h.islittle);
+ luaL_addsize(&b, size);
+ break;
+ }
+ case Knumber: { /* Lua float */
+ lua_Number f = luaL_checknumber(L, arg); /* get argument */
+ char *buff = luaL_prepbuffsize(&b, sizeof(f));
+ /* move 'f' to final result, correcting endianness if needed */
+ copywithendian(buff, (char *)&f, sizeof(f), h.islittle);
+ luaL_addsize(&b, size);
+ break;
+ }
+ case Kdouble: { /* C double */
+ double f = (double)luaL_checknumber(L, arg); /* get argument */
+ char *buff = luaL_prepbuffsize(&b, sizeof(f));
+ /* move 'f' to final result, correcting endianness if needed */
+ copywithendian(buff, (char *)&f, sizeof(f), h.islittle);
luaL_addsize(&b, size);
break;
}
@@ -1682,7 +1743,7 @@ static lua_Integer unpackint (lua_State *L, const char *str,
else if (size > SZINT) { /* must check unread bytes */
int mask = (!issigned || (lua_Integer)res >= 0) ? 0 : MC;
for (i = limit; i < size; i++) {
- if ((unsigned char)str[islittle ? i : size - 1 - i] != mask)
+ if (l_unlikely((unsigned char)str[islittle ? i : size - 1 - i] != mask))
luaL_error(L, "%d-byte integer does not fit into Lua Integer", size);
}
}
@@ -1717,13 +1778,21 @@ static int str_unpack (lua_State *L) {
break;
}
case Kfloat: {
- volatile Ftypes u;
- lua_Number num;
- copywithendian(u.buff, data + pos, size, h.islittle);
- if (size == sizeof(u.f)) num = (lua_Number)u.f;
- else if (size == sizeof(u.d)) num = (lua_Number)u.d;
- else num = u.n;
- lua_pushnumber(L, num);
+ float f;
+ copywithendian((char *)&f, data + pos, sizeof(f), h.islittle);
+ lua_pushnumber(L, (lua_Number)f);
+ break;
+ }
+ case Knumber: {
+ lua_Number f;
+ copywithendian((char *)&f, data + pos, sizeof(f), h.islittle);
+ lua_pushnumber(L, f);
+ break;
+ }
+ case Kdouble: {
+ double f;
+ copywithendian((char *)&f, data + pos, sizeof(f), h.islittle);
+ lua_pushnumber(L, (lua_Number)f);
break;
}
case Kchar: {
@@ -1738,7 +1807,7 @@ static int str_unpack (lua_State *L) {
break;
}
case Kzstr: {
- size_t len = (int)strlen(data + pos);
+ size_t len = strlen(data + pos);
luaL_argcheck(L, pos + len < ld, 2,
"unfinished string for format 'z'");
lua_pushlstring(L, data + pos, len);
diff --git a/src/lua/ltable.c b/src/lua/ltable.c
index 5a0d066f..1b1cd241 100644
--- a/src/lua/ltable.c
+++ b/src/lua/ltable.c
@@ -68,18 +68,21 @@
#define MAXHSIZE luaM_limitN(1u << MAXHBITS, Node)
+/*
+** When the original hash value is good, hashing by a power of 2
+** avoids the cost of '%'.
+*/
#define hashpow2(t,n) (gnode(t, lmod((n), sizenode(t))))
+/*
+** for other types, it is better to avoid modulo by power of 2, as
+** they can have many 2 factors.
+*/
+#define hashmod(t,n) (gnode(t, ((n) % ((sizenode(t)-1)|1))))
+
+
#define hashstr(t,str) hashpow2(t, (str)->hash)
#define hashboolean(t,p) hashpow2(t, p)
-#define hashint(t,i) hashpow2(t, i)
-
-
-/*
-** for some types, it is better to avoid modulus by power of 2, as
-** they tend to have many 2 factors.
-*/
-#define hashmod(t,n) (gnode(t, ((n) % ((sizenode(t)-1)|1))))
#define hashpointer(t,p) hashmod(t, point2uint(p))
@@ -96,6 +99,20 @@ static const Node dummynode_ = {
static const TValue absentkey = {ABSTKEYCONSTANT};
+/*
+** Hash for integers. To allow a good hash, use the remainder operator
+** ('%'). If integer fits as a non-negative int, compute an int
+** remainder, which is faster. Otherwise, use an unsigned-integer
+** remainder, which uses all bits and ensures a non-negative result.
+*/
+static Node *hashint (const Table *t, lua_Integer i) {
+ lua_Unsigned ui = l_castS2U(i);
+ if (ui <= (unsigned int)INT_MAX)
+ return hashmod(t, cast_int(ui));
+ else
+ return hashmod(t, ui);
+}
+
/*
** Hash for floating-point numbers.
@@ -129,54 +146,78 @@ static int l_hashfloat (lua_Number n) {
/*
** returns the 'main' position of an element in a table (that is,
-** the index of its hash value). The key comes broken (tag in 'ktt'
-** and value in 'vkl') so that we can call it on keys inserted into
-** nodes.
+** the index of its hash value).
*/
-static Node *mainposition (const Table *t, int ktt, const Value *kvl) {
- switch (withvariant(ktt)) {
- case LUA_VNUMINT:
- return hashint(t, ivalueraw(*kvl));
- case LUA_VNUMFLT:
- return hashmod(t, l_hashfloat(fltvalueraw(*kvl)));
- case LUA_VSHRSTR:
- return hashstr(t, tsvalueraw(*kvl));
- case LUA_VLNGSTR:
- return hashpow2(t, luaS_hashlongstr(tsvalueraw(*kvl)));
+static Node *mainpositionTV (const Table *t, const TValue *key) {
+ switch (ttypetag(key)) {
+ case LUA_VNUMINT: {
+ lua_Integer i = ivalue(key);
+ return hashint(t, i);
+ }
+ case LUA_VNUMFLT: {
+ lua_Number n = fltvalue(key);
+ return hashmod(t, l_hashfloat(n));
+ }
+ case LUA_VSHRSTR: {
+ TString *ts = tsvalue(key);
+ return hashstr(t, ts);
+ }
+ case LUA_VLNGSTR: {
+ TString *ts = tsvalue(key);
+ return hashpow2(t, luaS_hashlongstr(ts));
+ }
case LUA_VFALSE:
return hashboolean(t, 0);
case LUA_VTRUE:
return hashboolean(t, 1);
- case LUA_VLIGHTUSERDATA:
- return hashpointer(t, pvalueraw(*kvl));
- case LUA_VLCF:
- return hashpointer(t, fvalueraw(*kvl));
- default:
- return hashpointer(t, gcvalueraw(*kvl));
+ case LUA_VLIGHTUSERDATA: {
+ void *p = pvalue(key);
+ return hashpointer(t, p);
+ }
+ case LUA_VLCF: {
+ lua_CFunction f = fvalue(key);
+ return hashpointer(t, f);
+ }
+ default: {
+ GCObject *o = gcvalue(key);
+ return hashpointer(t, o);
+ }
}
}
-/*
-** Returns the main position of an element given as a 'TValue'
-*/
-static Node *mainpositionTV (const Table *t, const TValue *key) {
- return mainposition(t, rawtt(key), valraw(key));
+l_sinline Node *mainpositionfromnode (const Table *t, Node *nd) {
+ TValue key;
+ getnodekey(cast(lua_State *, NULL), &key, nd);
+ return mainpositionTV(t, &key);
}
/*
-** Check whether key 'k1' is equal to the key in node 'n2'.
-** This equality is raw, so there are no metamethods. Floats
-** with integer values have been normalized, so integers cannot
-** be equal to floats. It is assumed that 'eqshrstr' is simply
-** pointer equality, so that short strings are handled in the
-** default case.
+** Check whether key 'k1' is equal to the key in node 'n2'. This
+** equality is raw, so there are no metamethods. Floats with integer
+** values have been normalized, so integers cannot be equal to
+** floats. It is assumed that 'eqshrstr' is simply pointer equality, so
+** that short strings are handled in the default case.
+** A true 'deadok' means to accept dead keys as equal to their original
+** values. All dead keys are compared in the default case, by pointer
+** identity. (Only collectable objects can produce dead keys.) Note that
+** dead long strings are also compared by identity.
+** Once a key is dead, its corresponding value may be collected, and
+** then another value can be created with the same address. If this
+** other value is given to 'next', 'equalkey' will signal a false
+** positive. In a regular traversal, this situation should never happen,
+** as all keys given to 'next' came from the table itself, and therefore
+** could not have been collected. Outside a regular traversal, we
+** have garbage in, garbage out. What is relevant is that this false
+** positive does not break anything. (In particular, 'next' will return
+** some other valid item on the table or nil.)
*/
-static int equalkey (const TValue *k1, const Node *n2) {
- if (rawtt(k1) != keytt(n2)) /* not the same variants? */
+static int equalkey (const TValue *k1, const Node *n2, int deadok) {
+ if ((rawtt(k1) != keytt(n2)) && /* not the same variants? */
+ !(deadok && keyisdead(n2) && iscollectable(k1)))
return 0; /* cannot be same key */
- switch (ttypetag(k1)) {
+ switch (keytt(n2)) {
case LUA_VNIL: case LUA_VFALSE: case LUA_VTRUE:
return 1;
case LUA_VNUMINT:
@@ -187,7 +228,7 @@ static int equalkey (const TValue *k1, const Node *n2) {
return pvalue(k1) == pvalueraw(keyval(n2));
case LUA_VLCF:
return fvalue(k1) == fvalueraw(keyval(n2));
- case LUA_VLNGSTR:
+ case ctb(LUA_VLNGSTR):
return luaS_eqlngstr(tsvalue(k1), keystrval(n2));
default:
return gcvalue(k1) == gcvalueraw(keyval(n2));
@@ -251,11 +292,12 @@ static unsigned int setlimittosize (Table *t) {
/*
** "Generic" get version. (Not that generic: not valid for integers,
** which may be in array part, nor for floats with integral values.)
+** See explanation about 'deadok' in function 'equalkey'.
*/
-static const TValue *getgeneric (Table *t, const TValue *key) {
+static const TValue *getgeneric (Table *t, const TValue *key, int deadok) {
Node *n = mainpositionTV(t, key);
for (;;) { /* check whether 'key' is somewhere in the chain */
- if (equalkey(key, n))
+ if (equalkey(key, n, deadok))
return gval(n); /* that's it */
else {
int nx = gnext(n);
@@ -292,8 +334,8 @@ static unsigned int findindex (lua_State *L, Table *t, TValue *key,
if (i - 1u < asize) /* is 'key' inside array part? */
return i; /* yes; that's the index */
else {
- const TValue *n = getgeneric(t, key);
- if (unlikely(isabstkey(n)))
+ const TValue *n = getgeneric(t, key, 1);
+ if (l_unlikely(isabstkey(n)))
luaG_runerror(L, "invalid key to 'next'"); /* key not found */
i = cast_int(nodefromval(n) - gnode(t, 0)); /* key index in hash table */
/* hash elements are numbered after array ones */
@@ -471,7 +513,7 @@ static void reinsert (lua_State *L, Table *ot, Table *t) {
already present in the table */
TValue k;
getnodekey(L, &k, old);
- setobjt2t(L, luaH_set(L, t, &k), gval(old));
+ luaH_set(L, t, &k, gval(old));
}
}
}
@@ -527,7 +569,7 @@ void luaH_resize (lua_State *L, Table *t, unsigned int newasize,
}
/* allocate new array */
newarray = luaM_reallocvector(L, t->array, oldasize, newasize, TValue);
- if (unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */
+ if (l_unlikely(newarray == NULL && newasize > 0)) { /* allocation failed? */
freehash(L, &newt); /* release new hash part */
luaM_error(L); /* raise error (with array unchanged) */
}
@@ -618,10 +660,10 @@ static Node *getfreepos (Table *t) {
** put new key in its main position; otherwise (colliding node is in its main
** position), new key goes to an empty position.
*/
-TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
+void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) {
Node *mp;
TValue aux;
- if (unlikely(ttisnil(key)))
+ if (l_unlikely(ttisnil(key)))
luaG_runerror(L, "table index is nil");
else if (ttisfloat(key)) {
lua_Number f = fltvalue(key);
@@ -630,9 +672,11 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
setivalue(&aux, k);
key = &aux; /* insert it as an integer */
}
- else if (unlikely(luai_numisnan(f)))
+ else if (l_unlikely(luai_numisnan(f)))
luaG_runerror(L, "table index is NaN");
}
+ if (ttisnil(value))
+ return; /* do not insert nil values */
mp = mainpositionTV(t, key);
if (!isempty(gval(mp)) || isdummy(t)) { /* main position is taken? */
Node *othern;
@@ -640,10 +684,11 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
if (f == NULL) { /* cannot find a free place? */
rehash(L, t, key); /* grow table */
/* whatever called 'newkey' takes care of TM cache */
- return luaH_set(L, t, key); /* insert key into grown table */
+ luaH_set(L, t, key, value); /* insert key into grown table */
+ return;
}
lua_assert(!isdummy(t));
- othern = mainposition(t, keytt(mp), &keyval(mp));
+ othern = mainpositionfromnode(t, mp);
if (othern != mp) { /* is colliding node out of its main position? */
/* yes; move colliding node into free position */
while (othern + gnext(othern) != mp) /* find previous */
@@ -668,7 +713,7 @@ TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key) {
setnodekey(L, mp, key);
luaC_barrierback(L, obj2gco(t), key);
lua_assert(isempty(gval(mp)));
- return gval(mp);
+ setobj2t(L, gval(mp), value);
}
@@ -730,7 +775,7 @@ const TValue *luaH_getstr (Table *t, TString *key) {
else { /* for long strings, use generic case */
TValue ko;
setsvalue(cast(lua_State *, NULL), &ko, key);
- return getgeneric(t, &ko);
+ return getgeneric(t, &ko, 0);
}
}
@@ -750,34 +795,45 @@ const TValue *luaH_get (Table *t, const TValue *key) {
/* else... */
} /* FALLTHROUGH */
default:
- return getgeneric(t, key);
+ return getgeneric(t, key, 0);
}
}
+/*
+** Finish a raw "set table" operation, where 'slot' is where the value
+** should have been (the result of a previous "get table").
+** Beware: when using this function you probably need to check a GC
+** barrier and invalidate the TM cache.
+*/
+void luaH_finishset (lua_State *L, Table *t, const TValue *key,
+ const TValue *slot, TValue *value) {
+ if (isabstkey(slot))
+ luaH_newkey(L, t, key, value);
+ else
+ setobj2t(L, cast(TValue *, slot), value);
+}
+
+
/*
** beware: when using this function you probably need to check a GC
** barrier and invalidate the TM cache.
*/
-TValue *luaH_set (lua_State *L, Table *t, const TValue *key) {
- const TValue *p = luaH_get(t, key);
- if (!isabstkey(p))
- return cast(TValue *, p);
- else return luaH_newkey(L, t, key);
+void luaH_set (lua_State *L, Table *t, const TValue *key, TValue *value) {
+ const TValue *slot = luaH_get(t, key);
+ luaH_finishset(L, t, key, slot, value);
}
void luaH_setint (lua_State *L, Table *t, lua_Integer key, TValue *value) {
const TValue *p = luaH_getint(t, key);
- TValue *cell;
- if (!isabstkey(p))
- cell = cast(TValue *, p);
- else {
+ if (isabstkey(p)) {
TValue k;
setivalue(&k, key);
- cell = luaH_newkey(L, t, &k);
+ luaH_newkey(L, t, &k, value);
}
- setobj2t(L, cell, value);
+ else
+ setobj2t(L, cast(TValue *, p), value);
}
diff --git a/src/lua/ltable.h b/src/lua/ltable.h
index c0060f4b..7bbbcb21 100644
--- a/src/lua/ltable.h
+++ b/src/lua/ltable.h
@@ -41,8 +41,12 @@ LUAI_FUNC void luaH_setint (lua_State *L, Table *t, lua_Integer key,
LUAI_FUNC const TValue *luaH_getshortstr (Table *t, TString *key);
LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key);
LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key);
-LUAI_FUNC TValue *luaH_newkey (lua_State *L, Table *t, const TValue *key);
-LUAI_FUNC TValue *luaH_set (lua_State *L, Table *t, const TValue *key);
+LUAI_FUNC void luaH_newkey (lua_State *L, Table *t, const TValue *key,
+ TValue *value);
+LUAI_FUNC void luaH_set (lua_State *L, Table *t, const TValue *key,
+ TValue *value);
+LUAI_FUNC void luaH_finishset (lua_State *L, Table *t, const TValue *key,
+ const TValue *slot, TValue *value);
LUAI_FUNC Table *luaH_new (lua_State *L);
LUAI_FUNC void luaH_resize (lua_State *L, Table *t, unsigned int nasize,
unsigned int nhsize);
diff --git a/src/lua/ltablib.c b/src/lua/ltablib.c
index d344a47e..868d78fd 100644
--- a/src/lua/ltablib.c
+++ b/src/lua/ltablib.c
@@ -59,8 +59,9 @@ static void checktab (lua_State *L, int arg, int what) {
static int tinsert (lua_State *L) {
- lua_Integer e = aux_getn(L, 1, TAB_RW) + 1; /* first empty element */
lua_Integer pos; /* where to insert new element */
+ lua_Integer e = aux_getn(L, 1, TAB_RW);
+ e = luaL_intop(+, e, 1); /* first empty element */
switch (lua_gettop(L)) {
case 2: { /* called with only 2 arguments */
pos = e; /* insert new element at the end */
@@ -145,9 +146,9 @@ static int tmove (lua_State *L) {
static void addfield (lua_State *L, luaL_Buffer *b, lua_Integer i) {
lua_geti(L, 1, i);
- if (!lua_isstring(L, -1))
- luaL_error(L, "invalid value (%s) at index %d in table for 'concat'",
- luaL_typename(L, -1), i);
+ if (l_unlikely(!lua_isstring(L, -1)))
+ luaL_error(L, "invalid value (%s) at index %I in table for 'concat'",
+ luaL_typename(L, -1), (LUAI_UACINT)i);
luaL_addvalue(b);
}
@@ -196,7 +197,8 @@ static int tunpack (lua_State *L) {
lua_Integer e = luaL_opt(L, luaL_checkinteger, 3, luaL_len(L, 1));
if (i > e) return 0; /* empty range */
n = (lua_Unsigned)e - i; /* number of elements minus 1 (avoid overflows) */
- if (n >= (unsigned int)INT_MAX || !lua_checkstack(L, (int)(++n)))
+ if (l_unlikely(n >= (unsigned int)INT_MAX ||
+ !lua_checkstack(L, (int)(++n))))
return luaL_error(L, "too many results to unpack");
for (; i < e; i++) { /* push arg[i..e - 1] (to avoid overflows) */
lua_geti(L, 1, i);
@@ -300,14 +302,14 @@ static IdxT partition (lua_State *L, IdxT lo, IdxT up) {
for (;;) {
/* next loop: repeat ++i while a[i] < P */
while ((void)lua_geti(L, 1, ++i), sort_comp(L, -1, -2)) {
- if (i == up - 1) /* a[i] < P but a[up - 1] == P ?? */
+ if (l_unlikely(i == up - 1)) /* a[i] < P but a[up - 1] == P ?? */
luaL_error(L, "invalid order function for sorting");
lua_pop(L, 1); /* remove a[i] */
}
/* after the loop, a[i] >= P and a[lo .. i - 1] < P */
/* next loop: repeat --j while P < a[j] */
while ((void)lua_geti(L, 1, --j), sort_comp(L, -3, -1)) {
- if (j < i) /* j < i but a[j] > P ?? */
+ if (l_unlikely(j < i)) /* j < i but a[j] > P ?? */
luaL_error(L, "invalid order function for sorting");
lua_pop(L, 1); /* remove a[j] */
}
diff --git a/src/lua/ltm.c b/src/lua/ltm.c
index 4770f96b..b657b783 100644
--- a/src/lua/ltm.c
+++ b/src/lua/ltm.c
@@ -147,7 +147,7 @@ static int callbinTM (lua_State *L, const TValue *p1, const TValue *p2,
void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2,
StkId res, TMS event) {
- if (!callbinTM(L, p1, p2, res, event)) {
+ if (l_unlikely(!callbinTM(L, p1, p2, res, event))) {
switch (event) {
case TM_BAND: case TM_BOR: case TM_BXOR:
case TM_SHL: case TM_SHR: case TM_BNOT: {
@@ -166,7 +166,8 @@ void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2,
void luaT_tryconcatTM (lua_State *L) {
StkId top = L->top;
- if (!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2, TM_CONCAT))
+ if (l_unlikely(!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2,
+ TM_CONCAT)))
luaG_concaterror(L, s2v(top - 2), s2v(top - 1));
}
diff --git a/src/lua/lua.c b/src/lua/lua.c
index fa52d252..03815a05 100644
--- a/src/lua/lua.c
+++ b/src/lua/lua.c
@@ -20,9 +20,9 @@
#include "lauxlib.h"
#include "lualib.h"
-/************** Pi-hole modification ***************/
+/** Pi-hole modification **/
#include "ftl_lua.h"
-/***************************************************/
+/**************************/
#if !defined(LUA_PROGNAME)
@@ -41,6 +41,26 @@ static lua_State *globalL = NULL;
static const char *progname = LUA_PROGNAME;
+#if defined(LUA_USE_POSIX) /* { */
+
+/*
+** Use 'sigaction' when available.
+*/
+static void setsignal (int sig, void (*handler)(int)) {
+ struct sigaction sa;
+ sa.sa_handler = handler;
+ sa.sa_flags = 0;
+ sigemptyset(&sa.sa_mask); /* do not mask any signal */
+ sigaction(sig, &sa, NULL);
+}
+
+#else /* }{ */
+
+#define setsignal signal
+
+#endif /* } */
+
+
/*
** Hook set by signal function to stop the interpreter.
*/
@@ -59,7 +79,7 @@ static void lstop (lua_State *L, lua_Debug *ar) {
*/
static void laction (int i) {
int flag = LUA_MASKCALL | LUA_MASKRET | LUA_MASKLINE | LUA_MASKCOUNT;
- signal(i, SIG_DFL); /* if another SIGINT happens, terminate process */
+ setsignal(i, SIG_DFL); /* if another SIGINT happens, terminate process */
lua_sethook(globalL, lstop, flag, 1);
}
@@ -73,14 +93,15 @@ static void print_usage (const char *badoption) {
lua_writestringerror(
"usage: %s [options] [script [args]]\n"
"Available options are:\n"
- " -e stat execute string 'stat'\n"
- " -i enter interactive mode after executing 'script'\n"
- " -l name require library 'name' into global 'name'\n"
- " -v show version information\n"
- " -E ignore environment variables\n"
- " -W turn warnings on\n"
- " -- stop handling options\n"
- " - stop handling options and execute stdin\n"
+ " -e stat execute string 'stat'\n"
+ " -i enter interactive mode after executing 'script'\n"
+ " -l mod require library 'mod' into global 'mod'\n"
+ " -l g=mod require library 'mod' into global 'g'\n"
+ " -v show version information\n"
+ " -E ignore environment variables\n"
+ " -W turn warnings on\n"
+ " -- stop handling options\n"
+ " - stop handling options and execute stdin\n"
,
progname);
}
@@ -139,9 +160,9 @@ static int docall (lua_State *L, int narg, int nres) {
lua_pushcfunction(L, msghandler); /* push message handler */
lua_insert(L, base); /* put it under function and args */
globalL = L; /* to be available to 'laction' */
- signal(SIGINT, laction); /* set C-signal handler */
+ setsignal(SIGINT, laction); /* set C-signal handler */
status = lua_pcall(L, narg, nres, base);
- signal(SIGINT, SIG_DFL); /* reset C-signal handler */
+ setsignal(SIGINT, SIG_DFL); /* reset C-signal handler */
lua_remove(L, base); /* remove message handler from the stack */
return status;
}
@@ -191,18 +212,24 @@ static int dostring (lua_State *L, const char *s, const char *name) {
/*
-** Calls 'require(name)' and stores the result in a global variable
-** with the given name.
+** Receives 'globname[=modname]' and runs 'globname = require(modname)'.
*/
/************** Pi-hole modification ***************/
-int dolibrary (lua_State *L, const char *name) {
+int dolibrary (lua_State *L, char *globname) {
/***************************************************/
int status;
+ char *modname = strchr(globname, '=');
+ if (modname == NULL) /* no explicit name? */
+ modname = globname; /* module name is equal to global name */
+ else {
+ *modname = '\0'; /* global name ends here */
+ modname++; /* module name starts after the '=' */
+ }
lua_getglobal(L, "require");
- lua_pushstring(L, name);
- status = docall(L, 1, 1); /* call 'require(name)' */
+ lua_pushstring(L, modname);
+ status = docall(L, 1, 1); /* call 'require(modname)' */
if (status == LUA_OK)
- lua_setglobal(L, name); /* global[name] = require return */
+ lua_setglobal(L, globname); /* globname = require(modname) */
return report(L, status);
}
@@ -313,7 +340,7 @@ static int runargs (lua_State *L, char **argv, int n) {
switch (option) {
case 'e': case 'l': {
int status;
- const char *extra = argv[i] + 2; /* both options need an argument */
+ char *extra = argv[i] + 2; /* both options need an argument */
if (*extra == '\0') extra = argv[++i];
lua_assert(extra != NULL);
status = (option == 'e')
@@ -422,14 +449,18 @@ static int handle_luainit (lua_State *L) {
/*
-** Returns the string to be used as a prompt by the interpreter.
+** Return the string to be used as a prompt by the interpreter. Leave
+** the string (or nil, if using the default value) on the stack, to keep
+** it anchored.
*/
static const char *get_prompt (lua_State *L, int firstline) {
- const char *p;
- lua_getglobal(L, firstline ? "_PROMPT" : "_PROMPT2");
- p = lua_tostring(L, -1);
- if (p == NULL) p = (firstline ? LUA_PROMPT : LUA_PROMPT2);
- return p;
+ if (lua_getglobal(L, firstline ? "_PROMPT" : "_PROMPT2") == LUA_TNIL)
+ return (firstline ? LUA_PROMPT : LUA_PROMPT2); /* use the default */
+ else { /* apply 'tostring' over the value */
+ const char *p = luaL_tolstring(L, -1, NULL);
+ lua_remove(L, -2); /* remove original value */
+ return p;
+ }
}
/* mark in error messages for incomplete statements */
@@ -604,7 +635,6 @@ static int pmain (lua_State *L) {
return 0; /* error running LUA_INIT */
}
-
/************** Pi-hole modification ***************/
// Load and enable libraries bundled with Pi-hole
ftl_lua_init(L);
@@ -629,7 +659,9 @@ static int pmain (lua_State *L) {
}
+/******* Pi-hole modification ********/
int lua_main (int argc, char **argv) {
+/*************************************/
int status, result;
lua_State *L = luaL_newstate(); /* create state */
if (L == NULL) {
diff --git a/src/lua/lua.h b/src/lua/lua.h
index 08c6a64a..e6618392 100644
--- a/src/lua/lua.h
+++ b/src/lua/lua.h
@@ -18,14 +18,14 @@
#define LUA_VERSION_MAJOR "5"
#define LUA_VERSION_MINOR "4"
-#define LUA_VERSION_RELEASE "1"
+#define LUA_VERSION_RELEASE "4"
#define LUA_VERSION_NUM 504
-#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 0)
+#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 4)
#define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR
#define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE
-#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2020 Lua.org, PUC-Rio"
+#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2022 Lua.org, PUC-Rio"
#define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo, W. Celes"
@@ -347,7 +347,8 @@ LUA_API size_t (lua_stringtonumber) (lua_State *L, const char *s);
LUA_API lua_Alloc (lua_getallocf) (lua_State *L, void **ud);
LUA_API void (lua_setallocf) (lua_State *L, lua_Alloc f, void *ud);
-LUA_API void (lua_toclose) (lua_State *L, int idx);
+LUA_API void (lua_toclose) (lua_State *L, int idx);
+LUA_API void (lua_closeslot) (lua_State *L, int idx);
/*
@@ -491,7 +492,7 @@ struct lua_Debug {
/******************************************************************************
-* Copyright (C) 1994-2020 Lua.org, PUC-Rio.
+* Copyright (C) 1994-2022 Lua.org, PUC-Rio.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
diff --git a/src/lua/luac.c b/src/lua/luac.c
index d7d219e4..6b556b26 100644
--- a/src/lua/luac.c
+++ b/src/lua/luac.c
@@ -155,6 +155,7 @@ static const Proto* combine(lua_State* L, int n)
f->p[i]=toproto(L,i-n-1);
if (f->p[i]->sizeupvalues>0) f->p[i]->upvalues[0].instack=0;
}
+ luaM_freearray(L,f->lineinfo,f->sizelineinfo);
f->sizelineinfo=0;
return f;
}
@@ -194,7 +195,9 @@ static int pmain(lua_State* L)
return 0;
}
+/******* Pi-hole modification ********/
int luac_main(int argc, char* argv[])
+/*************************************/
{
lua_State* L;
int i=doargs(argc,argv);
@@ -600,11 +603,11 @@ static void PrintCode(const Proto* f)
if (c==0) printf("all out"); else printf("%d out",c-1);
break;
case OP_TAILCALL:
- printf("%d %d %d",a,b,c);
+ printf("%d %d %d%s",a,b,c,ISK);
printf(COMMENT "%d in",b-1);
break;
case OP_RETURN:
- printf("%d %d %d",a,b,c);
+ printf("%d %d %d%s",a,b,c,ISK);
printf(COMMENT);
if (b==0) printf("all out"); else printf("%d out",b-1);
break;
@@ -619,7 +622,7 @@ static void PrintCode(const Proto* f)
break;
case OP_FORPREP:
printf("%d %d",a,bx);
- printf(COMMENT "to %d",pc+bx+2);
+ printf(COMMENT "exit to %d",pc+bx+3);
break;
case OP_TFORPREP:
printf("%d %d",a,bx);
diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h
index bdf927e7..d42d14b7 100644
--- a/src/lua/luaconf.h
+++ b/src/lua/luaconf.h
@@ -16,13 +16,13 @@
** ===================================================================
** General Configuration File for Lua
**
-** Some definitions here can be changed externally, through the
-** compiler (e.g., with '-D' options). Those are protected by
-** '#if !defined' guards. However, several other definitions should
-** be changed directly here, either because they affect the Lua
-** ABI (by making the changes here, you ensure that all software
-** connected to Lua, such as C libraries, will be compiled with the
-** same configuration); or because they are seldom changed.
+** Some definitions here can be changed externally, through the compiler
+** (e.g., with '-D' options): They are commented out or protected
+** by '#if !defined' guards. However, several other definitions
+** should be changed directly here, either because they affect the
+** Lua ABI (by making the changes here, you ensure that all software
+** connected to Lua, such as C libraries, will be compiled with the same
+** configuration); or because they are seldom changed.
**
** Search for "@@" to find all configurable definitions.
** ===================================================================
@@ -36,21 +36,6 @@
** =====================================================================
*/
-/*
-@@ LUAI_MAXCSTACK defines the maximum depth for nested calls and
-** also limits the maximum depth of other recursive algorithms in
-** the implementation, such as syntactic analysis. A value too
-** large may allow the interpreter to crash (C-stack overflow).
-** The default value seems ok for regular machines, but may be
-** too high for restricted hardware.
-** The test file 'cstack.lua' may help finding a good limit.
-** (It will crash with a limit too high.)
-*/
-#if !defined(LUAI_MAXCSTACK)
-#define LUAI_MAXCSTACK 2000
-#endif
-
-
/*
@@ LUA_USE_C89 controls the use of non-ISO-C89 features.
** Define it if you want Lua to avoid the use of a few C99 features
@@ -96,26 +81,12 @@
/*
** {==================================================================
-** Configuration for Number types.
+** Configuration for Number types. These options should not be
+** set externally, because any other code connected to Lua must
+** use the same configuration.
** ===================================================================
*/
-/*
-@@ LUA_32BITS enables Lua with 32-bit integers and 32-bit floats.
-*/
-/* #define LUA_32BITS */
-
-
-/*
-@@ LUA_C89_NUMBERS ensures that Lua uses the largest types available for
-** C89 ('long' and 'double'); Windows always has '__int64', so it does
-** not need to use this case.
-*/
-#if defined(LUA_USE_C89) && !defined(LUA_USE_WINDOWS)
-#define LUA_C89_NUMBERS
-#endif
-
-
/*
@@ LUA_INT_TYPE defines the type for Lua integers.
@@ LUA_FLOAT_TYPE defines the type for Lua floats.
@@ -136,7 +107,31 @@
#define LUA_FLOAT_DOUBLE 2
#define LUA_FLOAT_LONGDOUBLE 3
-#if defined(LUA_32BITS) /* { */
+
+/* Default configuration ('long long' and 'double', for 64-bit Lua) */
+#define LUA_INT_DEFAULT LUA_INT_LONGLONG
+#define LUA_FLOAT_DEFAULT LUA_FLOAT_DOUBLE
+
+
+/*
+@@ LUA_32BITS enables Lua with 32-bit integers and 32-bit floats.
+*/
+#define LUA_32BITS 0
+
+
+/*
+@@ LUA_C89_NUMBERS ensures that Lua uses the largest types available for
+** C89 ('long' and 'double'); Windows always has '__int64', so it does
+** not need to use this case.
+*/
+#if defined(LUA_USE_C89) && !defined(LUA_USE_WINDOWS)
+#define LUA_C89_NUMBERS 1
+#else
+#define LUA_C89_NUMBERS 0
+#endif
+
+
+#if LUA_32BITS /* { */
/*
** 32-bit integers and 'float'
*/
@@ -147,27 +142,22 @@
#endif
#define LUA_FLOAT_TYPE LUA_FLOAT_FLOAT
-#elif defined(LUA_C89_NUMBERS) /* }{ */
+#elif LUA_C89_NUMBERS /* }{ */
/*
** largest types available for C89 ('long' and 'double')
*/
#define LUA_INT_TYPE LUA_INT_LONG
#define LUA_FLOAT_TYPE LUA_FLOAT_DOUBLE
+#else /* }{ */
+/* use defaults */
+
+#define LUA_INT_TYPE LUA_INT_DEFAULT
+#define LUA_FLOAT_TYPE LUA_FLOAT_DEFAULT
+
#endif /* } */
-/*
-** default configuration for 64-bit Lua ('long long' and 'double')
-*/
-#if !defined(LUA_INT_TYPE)
-#define LUA_INT_TYPE LUA_INT_LONGLONG
-#endif
-
-#if !defined(LUA_FLOAT_TYPE)
-#define LUA_FLOAT_TYPE LUA_FLOAT_DOUBLE
-#endif
-
/* }================================================================== */
@@ -388,14 +378,13 @@
/*
** {==================================================================
-** Configuration for Numbers.
+** Configuration for Numbers (low-level part).
** Change these definitions if no predefined LUA_FLOAT_* / LUA_INT_*
** satisfy your needs.
** ===================================================================
*/
/*
-@@ LUA_NUMBER is the floating-point type used by Lua.
@@ LUAI_UACNUMBER is the result of a 'default argument promotion'
@@ over a floating number.
@@ l_floatatt(x) corrects float attribute 'x' to the proper float type
@@ -488,10 +477,7 @@
/*
-@@ LUA_INTEGER is the integer type used by Lua.
-**
@@ LUA_UNSIGNED is the unsigned version of LUA_INTEGER.
-**
@@ LUAI_UACINT is the result of a 'default argument promotion'
@@ over a LUA_INTEGER.
@@ LUA_INTEGER_FRMLEN is the length modifier for reading/writing integers.
@@ -499,7 +485,6 @@
@@ LUA_MAXINTEGER is the maximum value for a LUA_INTEGER.
@@ LUA_MININTEGER is the minimum value for a LUA_INTEGER.
@@ LUA_MAXUNSIGNED is the maximum value for a LUA_UNSIGNED.
-@@ LUA_UNSIGNEDBITS is the number of bits in a LUA_UNSIGNED.
@@ lua_integer2str converts an integer to a string.
*/
@@ -520,9 +505,6 @@
#define LUA_UNSIGNED unsigned LUAI_UACINT
-#define LUA_UNSIGNEDBITS (sizeof(LUA_UNSIGNED) * CHAR_BIT)
-
-
/* now the variable definitions */
#if LUA_INT_TYPE == LUA_INT_INT /* { int */
@@ -674,6 +656,34 @@
#define lua_getlocaledecpoint() (localeconv()->decimal_point[0])
#endif
+
+/*
+** macros to improve jump prediction, used mostly for error handling
+** and debug facilities. (Some macros in the Lua API use these macros.
+** Define LUA_NOBUILTIN if you do not want '__builtin_expect' in your
+** code.)
+*/
+#if !defined(luai_likely)
+
+#if defined(__GNUC__) && !defined(LUA_NOBUILTIN)
+#define luai_likely(x) (__builtin_expect(((x) != 0), 1))
+#define luai_unlikely(x) (__builtin_expect(((x) != 0), 0))
+#else
+#define luai_likely(x) (x)
+#define luai_unlikely(x) (x)
+#endif
+
+#endif
+
+
+#if defined(LUA_CORE) || defined(LUA_LIB)
+/* shorter names for Lua's own use */
+#define l_likely(x) luai_likely(x)
+#define l_unlikely(x) luai_unlikely(x)
+#endif
+
+
+
/* }================================================================== */
diff --git a/src/lua/lualib.h b/src/lua/lualib.h
index a78ecdb2..a7cab4eb 100644
--- a/src/lua/lualib.h
+++ b/src/lua/lualib.h
@@ -44,19 +44,13 @@ LUAMOD_API int (luaopen_debug) (lua_State *L);
#define LUA_LOADLIBNAME "package"
LUAMOD_API int (luaopen_package) (lua_State *L);
-/************** Pi-hole modification ***************/
+/************ Pi-hole modification *************/
#define LUA_PIHOLELIBNAME "pihole"
LUAMOD_API int (luaopen_pihole) (lua_State *L);
-/***************************************************/
+/***********************************************/
/* open all previous libraries */
LUALIB_API void (luaL_openlibs) (lua_State *L);
-
-#if !defined(lua_assert)
-#define lua_assert(x) ((void)0)
-#endif
-
-
#endif
diff --git a/src/lua/lutf8lib.c b/src/lua/lutf8lib.c
index 901d985f..e7bf098f 100644
--- a/src/lua/lutf8lib.c
+++ b/src/lua/lutf8lib.c
@@ -224,14 +224,11 @@ static int byteoffset (lua_State *L) {
static int iter_aux (lua_State *L, int strict) {
size_t len;
const char *s = luaL_checklstring(L, 1, &len);
- lua_Integer n = lua_tointeger(L, 2) - 1;
- if (n < 0) /* first iteration? */
- n = 0; /* start from here */
- else if (n < (lua_Integer)len) {
- n++; /* skip current byte */
- while (iscont(s + n)) n++; /* and its continuations */
+ lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2);
+ if (n < len) {
+ while (iscont(s + n)) n++; /* skip continuation bytes */
}
- if (n >= (lua_Integer)len)
+ if (n >= len) /* (also handles original 'n' being negative) */
return 0; /* no more codepoints */
else {
utfint code;
diff --git a/src/lua/lvm.c b/src/lua/lvm.c
index 08681af1..2ec34400 100644
--- a/src/lua/lvm.c
+++ b/src/lua/lvm.c
@@ -229,17 +229,17 @@ static int forprep (lua_State *L, StkId ra) {
count /= l_castS2U(-(step + 1)) + 1u;
}
/* store the counter in place of the limit (which won't be
- needed anymore */
+ needed anymore) */
setivalue(plimit, l_castU2S(count));
}
}
else { /* try making all values floats */
lua_Number init; lua_Number limit; lua_Number step;
- if (unlikely(!tonumber(plimit, &limit)))
+ if (l_unlikely(!tonumber(plimit, &limit)))
luaG_forerror(L, plimit, "limit");
- if (unlikely(!tonumber(pstep, &step)))
+ if (l_unlikely(!tonumber(pstep, &step)))
luaG_forerror(L, pstep, "step");
- if (unlikely(!tonumber(pinit, &init)))
+ if (l_unlikely(!tonumber(pinit, &init)))
luaG_forerror(L, pinit, "initial value");
if (step == 0)
luaG_runerror(L, "'for' step is zero");
@@ -292,7 +292,7 @@ void luaV_finishget (lua_State *L, const TValue *t, TValue *key, StkId val,
if (slot == NULL) { /* 't' is not a table? */
lua_assert(!ttistable(t));
tm = luaT_gettmbyobj(L, t, TM_INDEX);
- if (unlikely(notm(tm)))
+ if (l_unlikely(notm(tm)))
luaG_typeerror(L, t, "index"); /* no metamethod */
/* else will try the metamethod */
}
@@ -337,10 +337,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key,
lua_assert(isempty(slot)); /* slot must be empty */
tm = fasttm(L, h->metatable, TM_NEWINDEX); /* get metamethod */
if (tm == NULL) { /* no metamethod? */
- if (isabstkey(slot)) /* no previous entry? */
- slot = luaH_newkey(L, h, key); /* create one */
- /* no metamethod and (now) there is an entry with given key */
- setobj2t(L, cast(TValue *, slot), val); /* set its new value */
+ luaH_finishset(L, h, key, slot, val); /* set new value */
invalidateTMcache(h);
luaC_barrierback(L, obj2gco(h), val);
return;
@@ -349,7 +346,7 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key,
}
else { /* not a table; check metamethod */
tm = luaT_gettmbyobj(L, t, TM_NEWINDEX);
- if (unlikely(notm(tm)))
+ if (l_unlikely(notm(tm)))
luaG_typeerror(L, t, "index");
}
/* try the metamethod */
@@ -409,7 +406,7 @@ static int l_strcmp (const TString *ls, const TString *rs) {
** from float to int.)
** When 'f' is NaN, comparisons must result in false.
*/
-static int LTintfloat (lua_Integer i, lua_Number f) {
+l_sinline int LTintfloat (lua_Integer i, lua_Number f) {
if (l_intfitsf(i))
return luai_numlt(cast_num(i), f); /* compare them as floats */
else { /* i < f <=> i < ceil(f) */
@@ -426,7 +423,7 @@ static int LTintfloat (lua_Integer i, lua_Number f) {
** Check whether integer 'i' is less than or equal to float 'f'.
** See comments on previous function.
*/
-static int LEintfloat (lua_Integer i, lua_Number f) {
+l_sinline int LEintfloat (lua_Integer i, lua_Number f) {
if (l_intfitsf(i))
return luai_numle(cast_num(i), f); /* compare them as floats */
else { /* i <= f <=> i <= floor(f) */
@@ -443,7 +440,7 @@ static int LEintfloat (lua_Integer i, lua_Number f) {
** Check whether float 'f' is less than integer 'i'.
** See comments on previous function.
*/
-static int LTfloatint (lua_Number f, lua_Integer i) {
+l_sinline int LTfloatint (lua_Number f, lua_Integer i) {
if (l_intfitsf(i))
return luai_numlt(f, cast_num(i)); /* compare them as floats */
else { /* f < i <=> floor(f) < i */
@@ -460,7 +457,7 @@ static int LTfloatint (lua_Number f, lua_Integer i) {
** Check whether float 'f' is less than or equal to integer 'i'.
** See comments on previous function.
*/
-static int LEfloatint (lua_Number f, lua_Integer i) {
+l_sinline int LEfloatint (lua_Number f, lua_Integer i) {
if (l_intfitsf(i))
return luai_numle(f, cast_num(i)); /* compare them as floats */
else { /* f <= i <=> ceil(f) <= i */
@@ -476,7 +473,7 @@ static int LEfloatint (lua_Number f, lua_Integer i) {
/*
** Return 'l < r', for numbers.
*/
-static int LTnum (const TValue *l, const TValue *r) {
+l_sinline int LTnum (const TValue *l, const TValue *r) {
lua_assert(ttisnumber(l) && ttisnumber(r));
if (ttisinteger(l)) {
lua_Integer li = ivalue(l);
@@ -498,7 +495,7 @@ static int LTnum (const TValue *l, const TValue *r) {
/*
** Return 'l <= r', for numbers.
*/
-static int LEnum (const TValue *l, const TValue *r) {
+l_sinline int LEnum (const TValue *l, const TValue *r) {
lua_assert(ttisnumber(l) && ttisnumber(r));
if (ttisinteger(l)) {
lua_Integer li = ivalue(l);
@@ -571,8 +568,13 @@ int luaV_equalobj (lua_State *L, const TValue *t1, const TValue *t2) {
if (ttype(t1) != ttype(t2) || ttype(t1) != LUA_TNUMBER)
return 0; /* only numbers can be equal with different variants */
else { /* two numbers with different variants */
- lua_Integer i1, i2; /* compare them as integers */
- return (tointegerns(t1, &i1) && tointegerns(t2, &i2) && i1 == i2);
+ /* One of them is an integer. If the other does not have an
+ integer value, they cannot be equal; otherwise, compare their
+ integer values. */
+ lua_Integer i1, i2;
+ return (luaV_tointegerns(t1, &i1, F2Ieq) &&
+ luaV_tointegerns(t2, &i2, F2Ieq) &&
+ i1 == i2);
}
}
/* values have same type and same variant */
@@ -654,7 +656,7 @@ void luaV_concat (lua_State *L, int total) {
/* collect total length and number of strings */
for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) {
size_t l = vslen(s2v(top - n - 1));
- if (unlikely(l >= (MAX_SIZE/sizeof(char)) - tl))
+ if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl))
luaG_runerror(L, "string length overflow");
tl += l;
}
@@ -698,7 +700,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) {
}
default: { /* try metamethod */
tm = luaT_gettmbyobj(L, rb, TM_LEN);
- if (unlikely(notm(tm))) /* no metamethod? */
+ if (l_unlikely(notm(tm))) /* no metamethod? */
luaG_typeerror(L, rb, "get length of");
break;
}
@@ -714,7 +716,7 @@ void luaV_objlen (lua_State *L, StkId ra, const TValue *rb) {
** otherwise 'floor(q) == trunc(q) - 1'.
*/
lua_Integer luaV_idiv (lua_State *L, lua_Integer m, lua_Integer n) {
- if (unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */
+ if (l_unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */
if (n == 0)
luaG_runerror(L, "attempt to divide by zero");
return intop(-, 0, m); /* n==-1; avoid overflow with 0x80000...//-1 */
@@ -734,7 +736,7 @@ lua_Integer luaV_idiv (lua_State *L, lua_Integer m, lua_Integer n) {
** about luaV_idiv.)
*/
lua_Integer luaV_mod (lua_State *L, lua_Integer m, lua_Integer n) {
- if (unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */
+ if (l_unlikely(l_castS2U(n) + 1u <= 1u)) { /* special cases: -1 or 0 */
if (n == 0)
luaG_runerror(L, "attempt to perform 'n%%0'");
return 0; /* m % -1 == 0; avoid overflow with 0x80000...%-1 */
@@ -764,7 +766,8 @@ lua_Number luaV_modf (lua_State *L, lua_Number m, lua_Number n) {
/*
** Shift left operation. (Shift right just negates 'y'.)
*/
-#define luaV_shiftr(x,y) luaV_shiftl(x,-(y))
+#define luaV_shiftr(x,y) luaV_shiftl(x,intop(-, 0, y))
+
lua_Integer luaV_shiftl (lua_Integer x, lua_Integer y) {
if (y < 0) { /* shift right? */
@@ -845,6 +848,19 @@ void luaV_finishOp (lua_State *L) {
luaV_concat(L, total); /* concat them (may yield again) */
break;
}
+ case OP_CLOSE: { /* yielded closing variables */
+ ci->u.l.savedpc--; /* repeat instruction to close other vars. */
+ break;
+ }
+ case OP_RETURN: { /* yielded closing variables */
+ StkId ra = base + GETARG_A(inst);
+ /* adjust top to signal correct number of returns, in case the
+ return is "up to top" ('isIT') */
+ L->top = ra + ci->u2.nres;
+ /* repeat instruction to close other vars. and complete the return */
+ ci->u.l.savedpc--;
+ break;
+ }
default: {
/* only these other opcodes can yield */
lua_assert(op == OP_TFORCALL || op == OP_CALL ||
@@ -920,7 +936,7 @@ void luaV_finishOp (lua_State *L) {
*/
#define op_arithfK(L,fop) { \
TValue *v1 = vRB(i); \
- TValue *v2 = KC(i); \
+ TValue *v2 = KC(i); lua_assert(ttisnumber(v2)); \
op_arithf_aux(L, v1, v2, fop); }
@@ -949,7 +965,7 @@ void luaV_finishOp (lua_State *L) {
*/
#define op_arithK(L,iop,fop) { \
TValue *v1 = vRB(i); \
- TValue *v2 = KC(i); \
+ TValue *v2 = KC(i); lua_assert(ttisnumber(v2)); \
op_arith_aux(L, v1, v2, iop, fop); }
@@ -1048,7 +1064,8 @@ void luaV_finishOp (lua_State *L) {
#define updatebase(ci) (base = ci->func + 1)
-#define updatestack(ci) { if (trap) { updatebase(ci); ra = RA(i); } }
+#define updatestack(ci) \
+ { if (l_unlikely(trap)) { updatebase(ci); ra = RA(i); } }
/*
@@ -1092,15 +1109,11 @@ void luaV_finishOp (lua_State *L) {
#define ProtectNT(exp) (savepc(L), (exp), updatetrap(ci))
/*
-** Protect code that will finish the loop (returns) or can only raise
-** errors. (That is, it will not return to the interpreter main loop
-** after changing the stack or hooks.)
+** Protect code that can only raise errors. (That is, it cannot change
+** the stack or hooks.)
*/
#define halfProtect(exp) (savestate(L,ci), (exp))
-/* idem, but without changing the stack */
-#define halfProtectNT(exp) (savepc(L), (exp))
-
/* 'c' is the limit of live values in the stack */
#define checkGC(L,c) \
{ luaC_condGC(L, (savepc(L), L->top = (c)), \
@@ -1110,7 +1123,7 @@ void luaV_finishOp (lua_State *L) {
/* fetch an instruction and prepare its execution */
#define vmfetch() { \
- if (trap) { /* stack reallocation or hooks? */ \
+ if (l_unlikely(trap)) { /* stack reallocation or hooks? */ \
trap = luaG_traceexec(L, pc); /* handle hooks */ \
updatebase(ci); /* correct stack */ \
} \
@@ -1132,17 +1145,20 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
#if LUA_USE_JUMPTABLE
#include "ljumptab.h"
#endif
- tailcall:
+ startfunc:
trap = L->hookmask;
+ returning: /* trap already set */
cl = clLvalue(s2v(ci->func));
k = cl->p->k;
pc = ci->u.l.savedpc;
- if (trap) {
- if (cl->p->is_vararg)
- trap = 0; /* hooks will start after VARARGPREP instruction */
- else if (pc == cl->p->code) /* first instruction (not resuming)? */
- luaD_hookcall(L, ci);
- ci->u.l.trap = 1; /* there may be other hooks */
+ if (l_unlikely(trap)) {
+ if (pc == cl->p->code) { /* first instruction (not resuming)? */
+ if (cl->p->is_vararg)
+ trap = 0; /* hooks will start after VARARGPREP instruction */
+ else /* check 'call' hook */
+ luaD_hookcall(L, ci);
+ }
+ ci->u.l.trap = 1; /* assume trap is on, for now */
}
base = ci->func + 1;
/* main loop of interpreter */
@@ -1150,8 +1166,12 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
Instruction i; /* instruction being executed */
StkId ra; /* instruction's A register */
vmfetch();
+ #if 0
+ /* low-level line tracing for debugging Lua */
+ printf("line: %d\n", luaG_getfuncline(cl->p, pcRel(pc, cl->p)));
+ #endif
lua_assert(base == ci->func + 1);
- lua_assert(base <= L->top && L->top < L->stack + L->stacksize);
+ lua_assert(base <= L->top && L->top < L->stack_last);
/* invalidate top for instructions not expecting it */
lua_assert(isIT(i) || (cast_void(L->top = base), 1));
vmdispatch (GET_OPCODE(i)) {
@@ -1528,7 +1548,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
vmbreak;
}
vmcase(OP_CLOSE) {
- Protect(luaF_close(L, ra, LUA_OK));
+ Protect(luaF_close(L, ra, LUA_OK, 1));
vmbreak;
}
vmcase(OP_TBC) {
@@ -1606,47 +1626,45 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
vmbreak;
}
vmcase(OP_CALL) {
+ CallInfo *newci;
int b = GETARG_B(i);
int nresults = GETARG_C(i) - 1;
if (b != 0) /* fixed number of arguments? */
L->top = ra + b; /* top signals number of arguments */
/* else previous instruction set top */
- ProtectNT(luaD_call(L, ra, nresults));
+ savepc(L); /* in case of errors */
+ if ((newci = luaD_precall(L, ra, nresults)) == NULL)
+ updatetrap(ci); /* C call; nothing else to be done */
+ else { /* Lua call: run function in this same C frame */
+ ci = newci;
+ goto startfunc;
+ }
vmbreak;
}
vmcase(OP_TAILCALL) {
int b = GETARG_B(i); /* number of arguments + 1 (function) */
+ int n; /* number of results when calling a C function */
int nparams1 = GETARG_C(i);
- /* delat is virtual 'func' - real 'func' (vararg functions) */
+ /* delta is virtual 'func' - real 'func' (vararg functions) */
int delta = (nparams1) ? ci->u.l.nextraargs + nparams1 : 0;
if (b != 0)
L->top = ra + b;
else /* previous instruction set top */
b = cast_int(L->top - ra);
- savepc(ci); /* some calls here can raise errors */
+ savepc(ci); /* several calls here can raise errors */
if (TESTARG_k(i)) {
- /* close upvalues from current call; the compiler ensures
- that there are no to-be-closed variables here, so this
- call cannot change the stack */
- luaF_close(L, base, NOCLOSINGMETH);
+ luaF_closeupval(L, base); /* close upvalues from current call */
+ lua_assert(L->tbclist < base); /* no pending tbc variables */
lua_assert(base == ci->func + 1);
}
- while (!ttisfunction(s2v(ra))) { /* not a function? */
- luaD_tryfuncTM(L, ra); /* try '__call' metamethod */
- b++; /* there is now one extra argument */
- checkstackGCp(L, 1, ra);
+ if ((n = luaD_pretailcall(L, ci, ra, b, delta)) < 0) /* Lua function? */
+ goto startfunc; /* execute the callee */
+ else { /* C function? */
+ ci->func -= delta; /* restore 'func' (if vararg) */
+ luaD_poscall(L, ci, n); /* finish caller */
+ updatetrap(ci); /* 'luaD_poscall' can change hooks */
+ goto ret; /* caller returns after the tail call */
}
- if (!ttisLclosure(s2v(ra))) { /* C function? */
- luaD_call(L, ra, LUA_MULTRET); /* call it */
- updatetrap(ci);
- updatestack(ci); /* stack may have been relocated */
- ci->func -= delta;
- luaD_poscall(L, ci, cast_int(L->top - ra));
- return;
- }
- ci->func -= delta;
- luaD_pretailcall(L, ci, ra, b); /* prepare call frame */
- goto tailcall;
}
vmcase(OP_RETURN) {
int n = GETARG_B(i) - 1; /* number of results */
@@ -1655,9 +1673,10 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
n = cast_int(L->top - ra); /* get what is available */
savepc(ci);
if (TESTARG_k(i)) { /* may there be open upvalues? */
+ ci->u2.nres = n; /* save number of returns */
if (L->top < ci->top)
L->top = ci->top;
- luaF_close(L, base, LUA_OK);
+ luaF_close(L, base, CLOSEKTOP, 1);
updatetrap(ci);
updatestack(ci);
}
@@ -1665,26 +1684,31 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
ci->func -= ci->u.l.nextraargs + nparams1;
L->top = ra + n; /* set call for 'luaD_poscall' */
luaD_poscall(L, ci, n);
- return;
+ updatetrap(ci); /* 'luaD_poscall' can change hooks */
+ goto ret;
}
vmcase(OP_RETURN0) {
- if (L->hookmask) {
+ if (l_unlikely(L->hookmask)) {
L->top = ra;
- halfProtectNT(luaD_poscall(L, ci, 0)); /* no hurry... */
+ savepc(ci);
+ luaD_poscall(L, ci, 0); /* no hurry... */
+ trap = 1;
}
else { /* do the 'poscall' here */
- int nres = ci->nresults;
+ int nres;
L->ci = ci->previous; /* back to caller */
L->top = base - 1;
- while (nres-- > 0)
+ for (nres = ci->nresults; l_unlikely(nres > 0); nres--)
setnilvalue(s2v(L->top++)); /* all results are nil */
}
- return;
+ goto ret;
}
vmcase(OP_RETURN1) {
- if (L->hookmask) {
+ if (l_unlikely(L->hookmask)) {
L->top = ra + 1;
- halfProtectNT(luaD_poscall(L, ci, 1)); /* no hurry... */
+ savepc(ci);
+ luaD_poscall(L, ci, 1); /* no hurry... */
+ trap = 1;
}
else { /* do the 'poscall' here */
int nres = ci->nresults;
@@ -1694,11 +1718,17 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
else {
setobjs2s(L, base - 1, ra); /* at least this result */
L->top = base;
- while (--nres > 0) /* complete missing results */
- setnilvalue(s2v(L->top++));
+ for (; l_unlikely(nres > 1); nres--)
+ setnilvalue(s2v(L->top++)); /* complete missing results */
}
}
- return;
+ ret: /* return from a Lua function */
+ if (ci->callstatus & CIST_FRESH)
+ return; /* end this frame */
+ else {
+ ci = ci->previous;
+ goto returning; /* continue running caller in this frame */
+ }
}
vmcase(OP_FORLOOP) {
if (ttisinteger(s2v(ra + 2))) { /* integer loop? */
@@ -1792,7 +1822,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
}
vmcase(OP_VARARGPREP) {
ProtectNT(luaT_adjustvarargs(L, GETARG_A(i), ci, cl->p));
- if (trap) {
+ if (l_unlikely(trap)) { /* previous "Protect" updated trap */
luaD_hookcall(L, ci);
L->oldpc = 1; /* next opcode will be seen as a "new" line */
}
diff --git a/src/lua/lvm.h b/src/lua/lvm.h
index 2d4ac160..1bc16f3a 100644
--- a/src/lua/lvm.h
+++ b/src/lua/lvm.h
@@ -60,12 +60,14 @@ typedef enum {
/* convert an object to an integer (including string coercion) */
#define tointeger(o,i) \
- (ttisinteger(o) ? (*(i) = ivalue(o), 1) : luaV_tointeger(o,i,LUA_FLOORN2I))
+ (l_likely(ttisinteger(o)) ? (*(i) = ivalue(o), 1) \
+ : luaV_tointeger(o,i,LUA_FLOORN2I))
/* convert an object to an integer (without string coercion) */
#define tointegerns(o,i) \
- (ttisinteger(o) ? (*(i) = ivalue(o), 1) : luaV_tointegerns(o,i,LUA_FLOORN2I))
+ (l_likely(ttisinteger(o)) ? (*(i) = ivalue(o), 1) \
+ : luaV_tointegerns(o,i,LUA_FLOORN2I))
#define intop(op,v1,v2) l_castU2S(l_castS2U(v1) op l_castS2U(v2))
diff --git a/src/lua/scripts/CMakeLists.txt b/src/lua/scripts/CMakeLists.txt
new file mode 100644
index 00000000..1278f8f9
--- /dev/null
+++ b/src/lua/scripts/CMakeLists.txt
@@ -0,0 +1,35 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2022 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine
+# /src/lua/scripts/CMakeList.txt
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+set(sources
+ inspect.lua.hex
+ scripts.h
+ )
+
+# Compile files from raw/ into hex/
+find_program(RESOURCE_COMPILER xxd)
+file(GLOB_RECURSE COMPILED_RESOURCES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/" "./*.lua")
+foreach(INPUT_FILE ${COMPILED_RESOURCES})
+ set(IN ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_FILE})
+ set(OUTPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_FILE}.hex)
+ add_custom_command(
+ OUTPUT ${INPUT_FILE}.hex
+ COMMAND ${RESOURCE_COMPILER} -i < ${IN} > ${OUTPUT_FILE}
+ COMMENT "Compiling ${INPUT_FILE} to binary"
+ VERBATIM)
+ list(APPEND COMPILED_RESOURCES ${OUTPUT_FILE})
+endforeach()
+
+# Ensure target lua_scripts is build before target lua
+add_dependencies(lua lua_scripts)
+
+add_library(lua_scripts OBJECT ${sources})
+target_compile_options(lua_scripts PRIVATE ${EXTRAWARN})
+target_include_directories(lua_scripts PRIVATE ${PROJECT_SOURCE_DIR}/src)
diff --git a/src/lua/scripts/inspect.lua b/src/lua/scripts/inspect.lua
new file mode 100644
index 00000000..ebe722d9
--- /dev/null
+++ b/src/lua/scripts/inspect.lua
@@ -0,0 +1,333 @@
+local _tl_compat; if (tonumber((_VERSION or ''):match('[%d.]*$')) or 0) < 5.3 then local p, m = pcall(require, 'compat53.module'); if p then _tl_compat = m end end; local math = _tl_compat and _tl_compat.math or math; local string = _tl_compat and _tl_compat.string or string; local table = _tl_compat and _tl_compat.table or table
+local inspect = {Options = {}, }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+inspect._VERSION = 'inspect.lua 3.1.0'
+inspect._URL = 'http://github.com/kikito/inspect.lua'
+inspect._DESCRIPTION = 'human-readable representations of tables'
+inspect._LICENSE = [[
+ MIT LICENSE
+ Copyright (c) 2022 Enrique García Cota
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+]]
+inspect.KEY = setmetatable({}, { __tostring = function() return 'inspect.KEY' end })
+inspect.METATABLE = setmetatable({}, { __tostring = function() return 'inspect.METATABLE' end })
+
+local tostring = tostring
+local rep = string.rep
+local match = string.match
+local char = string.char
+local gsub = string.gsub
+local fmt = string.format
+
+local function rawpairs(t)
+ return next, t, nil
+end
+
+
+
+local function smartQuote(str)
+ if match(str, '"') and not match(str, "'") then
+ return "'" .. str .. "'"
+ end
+ return '"' .. gsub(str, '"', '\\"') .. '"'
+end
+
+
+local shortControlCharEscapes = {
+ ["\a"] = "\\a", ["\b"] = "\\b", ["\f"] = "\\f", ["\n"] = "\\n",
+ ["\r"] = "\\r", ["\t"] = "\\t", ["\v"] = "\\v", ["\127"] = "\\127",
+}
+local longControlCharEscapes = { ["\127"] = "\127" }
+for i = 0, 31 do
+ local ch = char(i)
+ if not shortControlCharEscapes[ch] then
+ shortControlCharEscapes[ch] = "\\" .. i
+ longControlCharEscapes[ch] = fmt("\\%03d", i)
+ end
+end
+
+local function escape(str)
+ return (gsub(gsub(gsub(str, "\\", "\\\\"),
+ "(%c)%f[0-9]", longControlCharEscapes),
+ "%c", shortControlCharEscapes))
+end
+
+local function isIdentifier(str)
+ return type(str) == "string" and not not str:match("^[_%a][_%a%d]*$")
+end
+
+local flr = math.floor
+local function isSequenceKey(k, sequenceLength)
+ return type(k) == "number" and
+ flr(k) == k and
+ 1 <= (k) and
+ k <= sequenceLength
+end
+
+local defaultTypeOrders = {
+ ['number'] = 1, ['boolean'] = 2, ['string'] = 3, ['table'] = 4,
+ ['function'] = 5, ['userdata'] = 6, ['thread'] = 7,
+}
+
+local function sortKeys(a, b)
+ local ta, tb = type(a), type(b)
+
+
+ if ta == tb and (ta == 'string' or ta == 'number') then
+ return (a) < (b)
+ end
+
+ local dta = defaultTypeOrders[ta] or 100
+ local dtb = defaultTypeOrders[tb] or 100
+
+
+ return dta == dtb and ta < tb or dta < dtb
+end
+
+local function getKeys(t)
+
+ local seqLen = 1
+ while rawget(t, seqLen) ~= nil do
+ seqLen = seqLen + 1
+ end
+ seqLen = seqLen - 1
+
+ local keys, keysLen = {}, 0
+ for k in rawpairs(t) do
+ if not isSequenceKey(k, seqLen) then
+ keysLen = keysLen + 1
+ keys[keysLen] = k
+ end
+ end
+ table.sort(keys, sortKeys)
+ return keys, keysLen, seqLen
+end
+
+local function countCycles(x, cycles)
+ if type(x) == "table" then
+ if cycles[x] then
+ cycles[x] = cycles[x] + 1
+ else
+ cycles[x] = 1
+ for k, v in rawpairs(x) do
+ countCycles(k, cycles)
+ countCycles(v, cycles)
+ end
+ countCycles(getmetatable(x), cycles)
+ end
+ end
+end
+
+local function makePath(path, a, b)
+ local newPath = {}
+ local len = #path
+ for i = 1, len do newPath[i] = path[i] end
+
+ newPath[len + 1] = a
+ newPath[len + 2] = b
+
+ return newPath
+end
+
+
+local function processRecursive(process,
+ item,
+ path,
+ visited)
+ if item == nil then return nil end
+ if visited[item] then return visited[item] end
+
+ local processed = process(item, path)
+ if type(processed) == "table" then
+ local processedCopy = {}
+ visited[item] = processedCopy
+ local processedKey
+
+ for k, v in rawpairs(processed) do
+ processedKey = processRecursive(process, k, makePath(path, k, inspect.KEY), visited)
+ if processedKey ~= nil then
+ processedCopy[processedKey] = processRecursive(process, v, makePath(path, processedKey), visited)
+ end
+ end
+
+ local mt = processRecursive(process, getmetatable(processed), makePath(path, inspect.METATABLE), visited)
+ if type(mt) ~= 'table' then mt = nil end
+ setmetatable(processedCopy, mt)
+ processed = processedCopy
+ end
+ return processed
+end
+
+local function puts(buf, str)
+ buf.n = buf.n + 1
+ buf[buf.n] = str
+end
+
+
+
+local Inspector = {}
+
+
+
+
+
+
+
+
+
+
+local Inspector_mt = { __index = Inspector }
+
+local function tabify(inspector)
+ puts(inspector.buf, inspector.newline .. rep(inspector.indent, inspector.level))
+end
+
+function Inspector:getId(v)
+ local id = self.ids[v]
+ local ids = self.ids
+ if not id then
+ local tv = type(v)
+ id = (ids[tv] or 0) + 1
+ ids[v], ids[tv] = id, id
+ end
+ return tostring(id)
+end
+
+function Inspector:putValue(v)
+ local buf = self.buf
+ local tv = type(v)
+ if tv == 'string' then
+ puts(buf, smartQuote(escape(v)))
+ elseif tv == 'number' or tv == 'boolean' or tv == 'nil' or
+ tv == 'cdata' or tv == 'ctype' then
+ puts(buf, tostring(v))
+ elseif tv == 'table' and not self.ids[v] then
+ local t = v
+
+ if t == inspect.KEY or t == inspect.METATABLE then
+ puts(buf, tostring(t))
+ elseif self.level >= self.depth then
+ puts(buf, '{...}')
+ else
+ if self.cycles[t] > 1 then puts(buf, fmt('<%d>', self:getId(t))) end
+
+ local keys, keysLen, seqLen = getKeys(t)
+
+ puts(buf, '{')
+ self.level = self.level + 1
+
+ for i = 1, seqLen + keysLen do
+ if i > 1 then puts(buf, ',') end
+ if i <= seqLen then
+ puts(buf, ' ')
+ self:putValue(t[i])
+ else
+ local k = keys[i - seqLen]
+ tabify(self)
+ if isIdentifier(k) then
+ puts(buf, k)
+ else
+ puts(buf, "[")
+ self:putValue(k)
+ puts(buf, "]")
+ end
+ puts(buf, ' = ')
+ self:putValue(t[k])
+ end
+ end
+
+ local mt = getmetatable(t)
+ if type(mt) == 'table' then
+ if seqLen + keysLen > 0 then puts(buf, ',') end
+ tabify(self)
+ puts(buf, ' = ')
+ self:putValue(mt)
+ end
+
+ self.level = self.level - 1
+
+ if keysLen > 0 or type(mt) == 'table' then
+ tabify(self)
+ elseif seqLen > 0 then
+ puts(buf, ' ')
+ end
+
+ puts(buf, '}')
+ end
+
+ else
+ puts(buf, fmt('<%s %d>', tv, self:getId(v)))
+ end
+end
+
+
+
+
+function inspect.inspect(root, options)
+ options = options or {}
+
+ local depth = options.depth or (math.huge)
+ local newline = options.newline or '\n'
+ local indent = options.indent or ' '
+ local process = options.process
+
+ if process then
+ root = processRecursive(process, root, {}, {})
+ end
+
+ local cycles = {}
+ countCycles(root, cycles)
+
+ local inspector = setmetatable({
+ buf = { n = 0 },
+ ids = {},
+ cycles = cycles,
+ depth = depth,
+ level = 0,
+ newline = newline,
+ indent = indent,
+ }, Inspector_mt)
+
+ inspector:putValue(root)
+
+ return table.concat(inspector.buf)
+end
+
+setmetatable(inspect, {
+ __call = function(_, root, options)
+ return inspect.inspect(root, options)
+ end,
+})
+
+return inspect
\ No newline at end of file
diff --git a/src/lua/scripts/scripts.h b/src/lua/scripts/scripts.h
new file mode 100644
index 00000000..2a78d5a4
--- /dev/null
+++ b/src/lua/scripts/scripts.h
@@ -0,0 +1,17 @@
+/* Pi-hole: A black hole for Internet advertisements
+* (c) 2022 Pi-hole, LLC (https://pi-hole.net)
+* Network-wide ad blocking via your own hardware.
+*
+* FTL Engine
+* Embedded LUA scripts processor
+*
+* This file is copyright under the latest version of the EUPL.
+* Please see LICENSE file for your rights under this license. */
+#ifndef LUA_SCRIPTS_H
+#define LUA_SCRIPTS_H
+
+static const char inspect_lua[] = {
+#include "inspect.lua.hex"
+};
+
+#endif // LUA_SCRIPTS_H
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 71ada6e2..50f5340d 100644
--- a/src/main.c
+++ b/src/main.c
@@ -59,12 +59,15 @@ int main (int argc, char *argv[])
log_info("########## FTL started on %s! ##########", hostname());
log_FTL_version(false);
+ // Process pihole-FTL.toml configuration file
+ readFTLconf();
+
// Catch signals not handled by dnsmasq
// We configure real-time signals later (after dnsmasq has forked)
handle_signals();
// Initialize shared memory
- if(!init_shmem(true))
+ if(!init_shmem())
{
log_crit("Initialization of shared memory failed.");
// Check if there is already a running FTL process
@@ -72,14 +75,15 @@ int main (int argc, char *argv[])
return EXIT_FAILURE;
}
- // Process FTL config file
- readFTLconf();
-
// pihole-FTL should really be run as user "pihole" to not mess up with file permissions
// print warning otherwise
if(strcmp(username, "pihole") != 0)
log_warn("Starting pihole-FTL as user %s is not recommended", username);
+ // Write PID early on so systemd cannot be fooled during DELAY_STARTUP
+ // times. The PID in this file will later be overwritten after forking
+ savepid();
+
// Delay startup (if requested)
// Do this before reading the database to make this option not only
// useful for interfaces that aren't ready but also for fake-hwclocks
diff --git a/src/regex.c b/src/regex.c
index 9c1dbb69..0cdbff00 100644
--- a/src/regex.c
+++ b/src/regex.c
@@ -296,8 +296,8 @@ bool compile_regex(const char *regexin, regexData *regex, char **message)
return true;
}
-int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID,
- const enum regex_type regexid, const bool regextest)
+static int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID,
+ const enum regex_type regexid, const bool regextest)
{
int match_idx = -1;
#ifdef USE_TRE_REGEX
@@ -453,6 +453,24 @@ int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID,
return match_idx;
}
+bool in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid)
+{
+ // For performance reasons, the regex evaluations is executed only if the
+ // exact whitelist lookup does not deliver a positive match. This is an
+ // optimization as the database lookup will most likely hit (a) more domains
+ // and (b) will be faster (given a sufficiently large number of regex
+ // whitelisting filters).
+ const int regex_id = match_regex(domain, dns_cache, clientID, regexid, false);
+ if(regex_id != -1)
+ {
+ // We found a match
+ dns_cache->domainlist_id = regex_id;
+ return true;
+ }
+
+ return false;
+}
+
static void free_regex(void)
{
// Return early if we don't use any regex filters
diff --git a/src/regex_r.h b/src/regex_r.h
index 00cacab1..8e351f88 100644
--- a/src/regex_r.h
+++ b/src/regex_r.h
@@ -46,8 +46,7 @@ typedef struct {
#define REGEX_MSG_LEN 256
bool compile_regex(const char *regexin, regexData *regex, char **message);
unsigned int get_num_regex(const enum regex_type regexid) __attribute__((pure));
-int match_regex(const char *input, DNSCacheData* dns_cache, const int clientID,
- const enum regex_type regexid, const bool regextest);
+bool in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, const enum regex_type regexid);
void allocate_regex_client_enabled(clientsData *client, const int clientID);
void reload_per_client_regex(clientsData *client);
void read_regex_from_database(void);
diff --git a/src/shmem.c b/src/shmem.c
index fb1843ec..94750787 100644
--- a/src/shmem.c
+++ b/src/shmem.c
@@ -34,6 +34,8 @@
#include "files.h"
// log_resource_shortage()
#include "database/message-table.h"
+// check_running_FTL()
+#include "procps.h"
/// The version of shared memory used
#define SHARED_MEMORY_VERSION 14
@@ -113,6 +115,7 @@ static ShmSettings *shmSettings = NULL;
static int pagesize;
static unsigned int local_shm_counter = 0;
+static pid_t shmem_pid = 0;
static size_t used_shmem = 0u;
static size_t get_optimal_object_size(const size_t objsize, const size_t minsize);
@@ -137,6 +140,41 @@ static int get_dev_shm_usage(char buffer[64])
return percentage;
}
+// Verify the PID stored during shared memory initialization is the same as ours
+// (while we initialized the shared memory objects)
+static void verify_shmem_pid(void)
+{
+ // Open shared memory settings object
+ const int settingsfd = shm_open(SHARED_SETTINGS_NAME, O_RDONLY, S_IRUSR | S_IWUSR);
+ if(settingsfd == -1)
+ {
+ log_crit("verify_shmem_pid(): Failed to open shared memory object \"%s\": %s",
+ SHARED_SETTINGS_NAME, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ShmSettings shms = { 0 };
+ if(read(settingsfd, &shms, sizeof(shms)) != sizeof(shms))
+ {
+ log_crit("verify_shmem_pid(): Failed to read %zu bytes from shared memory object \"%s\": %s",
+ sizeof(shms), SHARED_SETTINGS_NAME, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ close(settingsfd);
+
+ // Compare the SHM's PID to the one we had when creating the SHM objects
+ if(shms.pid == shmem_pid)
+ return;
+
+ // If we reach here, we are in serious trouble. Terminating with error
+ // code is the most sensible thing we can do at this point
+ log_crit("Shared memory is owned by a different process (PID %d)", shms.pid);
+ check_running_FTL();
+ log_crit("Exiting now!");
+ exit(EXIT_FAILURE);
+}
+
// chown_shmem() changes the file ownership of a given shared memory object
static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw)
{
@@ -282,14 +320,14 @@ size_t addstr(const char *input)
return (shmSettings->next_str_pos - len);
}
-const char *getstr(const size_t pos)
+const char *_getstr(const size_t pos, const char *func, const int line, const char *file)
{
// Only access the string memory if this memory region has already been set
if(pos < shmSettings->next_str_pos)
return &((const char*)shm_strings.ptr)[pos];
else
{
- log_warn("Tried to access %zu but next_str_pos is %u", pos, shmSettings->next_str_pos);
+ log_warn("Tried to access %zu in %s() (%s:%i) but next_str_pos is %u", pos, func, file, line, shmSettings->next_str_pos);
return "";
}
}
@@ -361,7 +399,7 @@ static void remap_shm(void)
}
// Obtain SHMEM lock
-void _lock_shm(const char* func, const int line, const char * file)
+void _lock_shm(const char *func, const int line, const char *file)
{
log_debug(DEBUG_LOCKS, "Waiting for SHM lock in %s() (%s:%i)", func, file, line);
@@ -447,138 +485,123 @@ bool is_our_lock(void)
return false;
}
-bool init_shmem(bool create_new)
+bool init_shmem()
{
// Get kernel's page size
pagesize = getpagesize();
/****************************** shared memory lock ******************************/
// Try to create shared memory object
- shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(ShmLock), create_new);
+ shm_lock = create_shm(SHARED_LOCK_NAME, sizeof(ShmLock));
if(shm_lock.ptr == NULL)
return false;
- shmLock = (ShmLock*) shm_lock.ptr;
- if(create_new)
- {
- create_mutex(&shmLock->lock.outer);
- create_mutex(&shmLock->lock.inner);
- }
+
+ shmLock = (ShmLock*)shm_lock.ptr;
+ create_mutex(&shmLock->lock.outer);
+ create_mutex(&shmLock->lock.inner);
/****************************** shared counters struct ******************************/
// Try to create shared memory object
- shm_counters = create_shm(SHARED_COUNTERS_NAME, sizeof(countersStruct), create_new);
+ shm_counters = create_shm(SHARED_COUNTERS_NAME, sizeof(countersStruct));
if(shm_counters.ptr == NULL)
return false;
+
counters = (countersStruct*)shm_counters.ptr;
/****************************** shared settings struct ******************************/
// Try to create shared memory object
- shm_settings = create_shm(SHARED_SETTINGS_NAME, sizeof(ShmSettings), create_new);
+ shm_settings = create_shm(SHARED_SETTINGS_NAME, sizeof(ShmSettings));
if(shm_settings.ptr == NULL)
return false;
+
shmSettings = (ShmSettings*)shm_settings.ptr;
- if(create_new)
- {
- shmSettings->version = SHARED_MEMORY_VERSION;
- shmSettings->global_shm_counter = 0;
- }
- else
- {
- if(shmSettings->version != SHARED_MEMORY_VERSION)
- {
- log_err("Shared memory version mismatch, found %d, expected %d!",
- shmSettings->version, SHARED_MEMORY_VERSION);
- return false;
- }
- }
+ shmSettings->version = SHARED_MEMORY_VERSION;
+ shmSettings->global_shm_counter = 0;
+ shmSettings->pid = shmem_pid = getpid();
/****************************** shared strings buffer ******************************/
// Try to create shared memory object
- shm_strings = create_shm(SHARED_STRINGS_NAME, STRINGS_ALLOC_STEP, create_new);
+ shm_strings = create_shm(SHARED_STRINGS_NAME, STRINGS_ALLOC_STEP);
if(shm_strings.ptr == NULL)
return false;
- if(create_new)
- {
- counters->strings_MAX = shm_strings.size;
- // Initialize shared string object with an empty string at position zero
- ((char*)shm_strings.ptr)[0] = '\0';
- shmSettings->next_str_pos = 1;
- }
+ counters->strings_MAX = shm_strings.size;
+
+ // Initialize shared string object with an empty string at position zero
+ ((char*)shm_strings.ptr)[0] = '\0';
+ shmSettings->next_str_pos = 1;
/****************************** shared domains struct ******************************/
size_t size = get_optimal_object_size(sizeof(domainsData), 1);
// Try to create shared memory object
- shm_domains = create_shm(SHARED_DOMAINS_NAME, size*sizeof(domainsData), create_new);
+ shm_domains = create_shm(SHARED_DOMAINS_NAME, size*sizeof(domainsData));
if(shm_domains.ptr == NULL)
return false;
+
domains = (domainsData*)shm_domains.ptr;
- if(create_new)
- counters->domains_MAX = size;
+ counters->domains_MAX = size;
/****************************** shared clients struct ******************************/
size = get_optimal_object_size(sizeof(clientsData), 1);
// Try to create shared memory object
- shm_clients = create_shm(SHARED_CLIENTS_NAME, size*sizeof(clientsData), create_new);
+ shm_clients = create_shm(SHARED_CLIENTS_NAME, size*sizeof(clientsData));
if(shm_clients.ptr == NULL)
return false;
+
clients = (clientsData*)shm_clients.ptr;
- if(create_new)
- counters->clients_MAX = size;
+ counters->clients_MAX = size;
/****************************** shared upstreams struct ******************************/
size = get_optimal_object_size(sizeof(upstreamsData), 1);
// Try to create shared memory object
- shm_upstreams = create_shm(SHARED_UPSTREAMS_NAME, size*sizeof(upstreamsData), create_new);
+ shm_upstreams = create_shm(SHARED_UPSTREAMS_NAME, size*sizeof(upstreamsData));
if(shm_upstreams.ptr == NULL)
return false;
upstreams = (upstreamsData*)shm_upstreams.ptr;
- if(create_new)
- counters->upstreams_MAX = size;
+
+ counters->upstreams_MAX = size;
/****************************** shared queries struct ******************************/
// Try to create shared memory object
- shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesData), create_new);
+ shm_queries = create_shm(SHARED_QUERIES_NAME, pagesize*sizeof(queriesData));
if(shm_queries.ptr == NULL)
return false;
queries = (queriesData*)shm_queries.ptr;
- if(create_new)
- counters->queries_MAX = pagesize;
+
+ counters->queries_MAX = pagesize;
/****************************** shared overTime struct ******************************/
size = get_optimal_object_size(sizeof(overTimeData), OVERTIME_SLOTS);
// Try to create shared memory object
- shm_overTime = create_shm(SHARED_OVERTIME_NAME, size*sizeof(overTimeData), create_new);
+ shm_overTime = create_shm(SHARED_OVERTIME_NAME, size*sizeof(overTimeData));
if(shm_overTime.ptr == NULL)
return false;
- if(create_new)
- {
- // set global pointer in overTime.c
- overTime = (overTimeData*)shm_overTime.ptr;
- }
+
+ // set global pointer in overTime.c
+ overTime = (overTimeData*)shm_overTime.ptr;
/****************************** shared DNS cache struct ******************************/
size = get_optimal_object_size(sizeof(DNSCacheData), 1);
// Try to create shared memory object
- shm_dns_cache = create_shm(SHARED_DNS_CACHE, size*sizeof(DNSCacheData), create_new);
+ shm_dns_cache = create_shm(SHARED_DNS_CACHE, size*sizeof(DNSCacheData));
if(shm_dns_cache.ptr == NULL)
return false;
+
dns_cache = (DNSCacheData*)shm_dns_cache.ptr;
- if(create_new)
- counters->dns_cache_MAX = size;
+ counters->dns_cache_MAX = size;
/****************************** shared per-client regex buffer ******************************/
size = pagesize; // Allocate one pagesize initially. This may be expanded later on
// Try to create shared memory object
- shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size, create_new);
+ shm_per_client_regex = create_shm(SHARED_PER_CLIENT_REGEX, size);
if(shm_per_client_regex.ptr == NULL)
return false;
- if(create_new)
- counters->per_client_regex_MAX = size;
+
+ counters->per_client_regex_MAX = size;
/****************************** shared fifo_buffer struct ******************************/
// Try to create shared memory object
- shm_fifo_log = create_shm(SHARED_FIFO_LOG_NAME, sizeof(fifologData), create_new);
+ shm_fifo_log = create_shm(SHARED_FIFO_LOG_NAME, sizeof(fifologData));
if(shm_fifo_log.ptr == NULL)
return false;
fifo_log = (fifologData*)shm_fifo_log.ptr;
@@ -613,10 +636,9 @@ void destroy_shmem(void)
///
/// \param name the name of the shared memory
/// \param size the size to allocate
-/// \param create_new true = delete old file, create new, false = connect to existing object or fail
/// \return a structure with a pointer to the mounted shared memory. The pointer
/// will always be valid, because if it failed FTL will have exited.
-static SharedMemory create_shm(const char *name, const size_t size, bool create_new)
+static SharedMemory create_shm(const char *name, const size_t size)
{
char df[64] = { 0 };
const int percentage = get_dev_shm_usage(df);
@@ -633,21 +655,19 @@ static SharedMemory create_shm(const char *name, const size_t size, bool create_
.ptr = NULL
};
- // O_RDWR: Open the object for read-write access (we need to be able to modify the locks)
- // When creating a new shared memory object, we add to this
- // - O_CREAT: Create the shared memory object if it does not exist.
- // - O_EXCL: Return an error if a shared memory object with the given name already exists.
- const int shm_oflags = create_new ? O_RDWR | O_CREAT | O_EXCL : O_RDWR;
-
- // Create the shared memory file in read/write mode with 600 permissions
+ // Create the shared memory file in read/write mode with 600 (u+rw) permissions
+ // and the following open flags:
+ // - O_RDWR: Open the object for read-write access (we need to be able to modify the locks)
+ // - O_CREAT: Create the shared memory object if it does not exist.
+ // - O_EXCL: Return an error if a shared memory object with the given name already exists.
errno = 0;
- const int fd = shm_open(sharedMemory.name, shm_oflags, S_IRUSR | S_IWUSR);
+ const int fd = shm_open(sharedMemory.name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
// Check for `shm_open` error
if(fd == -1)
{
- log_err("create_shm(): Failed to %s shared memory object \"%s\": %s",
- create_new ? "create" : "open", name, strerror(errno));
+ log_err("create_shm(): Failed to create shared memory object \"%s\": %s",
+ name, strerror(errno));
return sharedMemory;
}
@@ -775,6 +795,9 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si
// TCP requests.
if(resize)
{
+ // Verify shared memory ownership
+ verify_shmem_pid();
+
// Open shared memory object
const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR);
if(fd == -1)
@@ -1018,15 +1041,15 @@ void set_per_client_regex(const int clientID, const int regexID, const bool valu
((bool*) shm_per_client_regex.ptr)[id] = value;
}
-static inline bool check_range(int ID, int MAXID, const char* type, int line, const char * function, const char * file)
+static inline bool check_range(int ID, int MAXID, const char* type, const char *func, int line, const char *file)
{
// Check bounds
if(ID < 0 || ID > MAXID)
{
if(config.debug)
{
- log_err("Trying to access %s ID %i, but maximum is %i in %s() (%s:%i)",
- type, ID, MAXID, function, file, line);
+ log_err("Trying to access %s ID %i, but maximum is %i", type, ID, MAXID);
+ log_err("found in %s() (%s:%i)", func, short_path(file), line);
}
return false;
}
@@ -1035,15 +1058,15 @@ static inline bool check_range(int ID, int MAXID, const char* type, int line, co
return true;
}
-static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, const char* type, int line, const char * function, const char * file)
+static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, const char *type, const char *func, int line, const char *file)
{
// Check magic only if requested (skipped for new entries which are uninitialized)
if(checkMagic && magic != MAGICBYTE)
{
if(config.debug)
{
- log_err("Trying to access %s ID %i, but magic byte is %x in %s() (%s:%i)",
- type, ID, magic, function, file, line);
+ log_err("Trying to access %s ID %i, but magic byte is %x", type, ID, magic);
+ log_err("found in %s() (%s:%i)", func, short_path(file), line);
}
return false;
}
@@ -1052,7 +1075,7 @@ static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, con
return true;
}
-queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * function, const char * file)
+queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file)
{
// This does not exist, return a NULL pointer
if(queryID == -1)
@@ -1061,20 +1084,23 @@ queriesData* _getQuery(int queryID, bool checkMagic, int line, const char * func
// We are not in a locked situation, return a NULL pointer
if(config.debug & DEBUG_LOCKS && !is_our_lock())
{
- log_err("Tried to obtain query pointer without lock in %s() (%s:%i)!",
- function, file, line);
- generate_backtrace();
+ if(config.debug)
+ {
+ log_err("Tried to obtain query pointer without lock in %s() (%s:%i)!",
+ func, short_path(file), line);
+ generate_backtrace();
+ }
return NULL;
}
- if(check_range(queryID, counters->queries_MAX, "query", line, function, file) &&
- check_magic(queryID, checkMagic, queries[queryID].magic, "query", line, function, file))
+ if(check_range(queryID, counters->queries_MAX, "query", func, line, file) &&
+ check_magic(queryID, checkMagic, queries[queryID].magic, "query", func, line, file))
return &queries[queryID];
else
return NULL;
}
-clientsData* _getClient(int clientID, bool checkMagic, int line, const char * function, const char * file)
+clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file)
{
// This does not exist, we return a NULL pointer
if(clientID == -1)
@@ -1083,20 +1109,23 @@ clientsData* _getClient(int clientID, bool checkMagic, int line, const char * fu
// We are not in a locked situation, return a NULL pointer
if(config.debug & DEBUG_LOCKS && !is_our_lock())
{
- log_err("Tried to obtain client pointer without lock in %s() (%s:%i)!",
- function, file, line);
- generate_backtrace();
+ if(config.debug)
+ {
+ log_err("Tried to obtain client pointer without lock in %s() (%s:%i)!",
+ func, short_path(file), line);
+ generate_backtrace();
+ }
return NULL;
}
- if(check_range(clientID, counters->clients_MAX, "client", line, function, file) &&
- check_magic(clientID, checkMagic, clients[clientID].magic, "client", line, function, file))
+ if(check_range(clientID, counters->clients_MAX, "client", func, line, file) &&
+ check_magic(clientID, checkMagic, clients[clientID].magic, "client", func, line, file))
return &clients[clientID];
else
return NULL;
}
-domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * function, const char * file)
+domainsData* _getDomain(int domainID, bool checkMagic, int line, const char *func, const char *file)
{
// This does not exist, we return a NULL pointer
if(domainID == -1)
@@ -1105,20 +1134,23 @@ domainsData* _getDomain(int domainID, bool checkMagic, int line, const char * fu
// We are not in a locked situation, return a NULL pointer
if(config.debug & DEBUG_LOCKS && !is_our_lock())
{
- log_err("Tried to obtain domain pointer without lock in %s() (%s:%i)!",
- function, file, line);
- generate_backtrace();
+ if(config.debug)
+ {
+ log_err("Tried to obtain domain pointer without lock in %s() (%s:%i)!",
+ func, short_path(file), line);
+ generate_backtrace();
+ }
return NULL;
}
- if(check_range(domainID, counters->domains_MAX, "domain", line, function, file) &&
- check_magic(domainID, checkMagic, domains[domainID].magic, "domain", line, function, file))
+ if(check_range(domainID, counters->domains_MAX, "domain", func, line, file) &&
+ check_magic(domainID, checkMagic, domains[domainID].magic, "domain", func, line, file))
return &domains[domainID];
else
return NULL;
}
-upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char * function, const char * file)
+upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char *func, const char *file)
{
// This does not exist, we return a NULL pointer
if(upstreamID == -1)
@@ -1127,20 +1159,23 @@ upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const cha
// We are not in a locked situation, return a NULL pointer
if(config.debug & DEBUG_LOCKS && !is_our_lock())
{
- log_err("Tried to obtain upstream pointer without lock in %s() (%s:%i)!",
- function, file, line);
- generate_backtrace();
+ if(config.debug)
+ {
+ log_err("Tried to obtain upstream pointer without lock in %s() (%s:%i)!",
+ func, short_path(file), line);
+ generate_backtrace();
+ }
return NULL;
}
- if(check_range(upstreamID, counters->upstreams_MAX, "upstream", line, function, file) &&
- check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", line, function, file))
+ if(check_range(upstreamID, counters->upstreams_MAX, "upstream", func, line, file) &&
+ check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", func, line, file))
return &upstreams[upstreamID];
else
return NULL;
}
-DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char * function, const char * file)
+DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *func, const char *file)
{
// This does not exist, we return a NULL pointer
if(cacheID == -1)
@@ -1149,14 +1184,17 @@ DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *
// We are not in a locked situation, return a NULL pointer
if(config.debug & DEBUG_LOCKS && !is_our_lock())
{
- log_err("Tried to obtain cache pointer without lock in %s() (%s:%i)!",
- function, file, line);
- generate_backtrace();
+ if(config.debug)
+ {
+ log_err("Tried to obtain cache pointer without lock in %s() (%s:%i)!",
+ func, short_path(file), line);
+ generate_backtrace();
+ }
return NULL;
}
- if(check_range(cacheID, counters->dns_cache_MAX, "dns_cache", line, function, file) &&
- check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", line, function, file))
+ if(check_range(cacheID, counters->dns_cache_MAX, "dns_cache", func, line, file) &&
+ check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", func, line, file))
return &dns_cache[cacheID];
else
return NULL;
diff --git a/src/shmem.h b/src/shmem.h
index 25ee5636..92b3e013 100644
--- a/src/shmem.h
+++ b/src/shmem.h
@@ -26,6 +26,7 @@ typedef struct {
typedef struct {
int version;
+ pid_t pid;
unsigned int global_shm_counter;
unsigned int next_str_pos;
} ShmSettings;
@@ -72,10 +73,9 @@ extern countersStruct *counters;
///
/// \param name the name of the shared memory
/// \param size the size to allocate
-/// \param create_new true = delete old file, create new, false = connect to existing object or fail
/// \return a structure with a pointer to the mounted shared memory. The pointer
/// will always be valid, because if it failed FTL will have exited.
-static SharedMemory create_shm(const char *name, const size_t size, bool create_new);
+static SharedMemory create_shm(const char *name, const size_t size);
/// Reallocate shared memory
///
@@ -114,10 +114,11 @@ void _unlock_log(const char* func, const int line, const char * file);
/// Block until a lock can be obtained
-bool init_shmem(bool create_new);
+bool init_shmem(void);
void destroy_shmem(void);
size_t addstr(const char *str);
-const char *getstr(const size_t pos);
+#define getstr(pos) _getstr(pos, __FUNCTION__, __LINE__, __FILE__)
+const char *_getstr(const size_t pos, const char *func, const int line, const char *file);
/**
* Escapes a string by replacing special characters, such as spaces
diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c
index 010184ed..710a7f3d 100644
--- a/src/syscalls/accept.c
+++ b/src/syscalls/accept.c
@@ -15,7 +15,7 @@
#undef accept
int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line)
{
- int ret = 0;
+ int ret = 0;
do
{
// Reset errno before trying to write
@@ -26,11 +26,17 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char
// incoming signal
while(ret < 0 && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(ret < 0)
log_warn("Could not accept() in %s() (%s:%i): %s",
func, file, line, strerror(errno));
- return ret;
+ // Restore errno value
+ errno = _errno;
+
+ return ret;
}
\ No newline at end of file
diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c
index bb3f19d6..60c2d2f0 100644
--- a/src/syscalls/calloc.c
+++ b/src/syscalls/calloc.c
@@ -30,10 +30,17 @@ void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const s
// an incoming signal
while(ptr == NULL && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Handle other errors than EINTR
if(ptr == NULL)
log_err("Memory allocation (%zu x %zu) failed in %s() (%s:%i)",
nmemb, size, func, file, line);
+ // Restore errno value
+ errno = _errno;
+
+ // Return memory pointer
return ptr;
}
diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c
index eaa54e6c..71912a69 100644
--- a/src/syscalls/fopen.c
+++ b/src/syscalls/fopen.c
@@ -15,7 +15,7 @@
static uint8_t already_writing = 0;
#undef fopen
-FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line)
+FILE * __attribute__ ((__malloc__)) FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line)
{
FILE *file_ptr = 0;
do
@@ -28,16 +28,22 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c
// incoming signal
while(file_ptr == NULL && errno == EINTR);
- // Final error checking (may have faild for some other reason then an
- // EINTR = interrupted system call). The already_writing coutner
- // prevents a possible infinite loop. We accept "No such file or
- // directory" as this is something we'll deal with elsewhere
- if(file_ptr == NULL && errno != ENOENT && (already_writing++) == 1)
+ // Backup errno value
+ const int _errno = errno;
+
+ // Final error checking (may have failed for some other reason then an
+ // EINTR = interrupted system call)
+ // The already_writing counter prevents a possible infinite loop
+ if(file_ptr == NULL && (already_writing++) == 1)
log_warn("Could not fopen(\"%s\", \"%s\") in %s() (%s:%i): %s",
pathname, mode, func, file, line, strerror(errno));
// Decrement warning counter
already_writing--;
+ // Restore errno value
+ errno = _errno;
+
+ // Return file pointer
return file_ptr;
}
diff --git a/src/syscalls/ftlallocate.c b/src/syscalls/ftlallocate.c
index 14dbd9a8..8140f1b8 100644
--- a/src/syscalls/ftlallocate.c
+++ b/src/syscalls/ftlallocate.c
@@ -35,5 +35,6 @@ int FTLfallocate(const int fd, const off_t offset, const off_t len, const char *
// Set errno ourselves as posix_fallocate doesn't do it
errno = ret;
+
return ret;
}
diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c
index 433eab07..ab4b0112 100644
--- a/src/syscalls/pthread_mutex_lock.c
+++ b/src/syscalls/pthread_mutex_lock.c
@@ -17,7 +17,7 @@
#undef pthread_mutex_lock
int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line)
{
- int ret = 0;
+ ssize_t ret = 0;
do
{
ret = pthread_mutex_lock(__mutex);
@@ -26,11 +26,17 @@ int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char
// incoming signal
while(ret == EINTR);
- // Final errer checking (may have faild for some other reason then an
+ // Backup errno value
+ const int _errno = errno;
+
+ // Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
- if(ret != 0)
+ if(ret < 0)
log_warn("Could not pthread_mutex_lock() in %s() (%s:%i): %s",
- func, file, line, strerror(ret));
+ func, file, line, strerror(errno));
+
+ // Restore errno value
+ errno = _errno;
return ret;
}
diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c
index ec320b7b..98e6d0a6 100644
--- a/src/syscalls/realloc.c
+++ b/src/syscalls/realloc.c
@@ -34,10 +34,16 @@ void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size,
// an incoming signal
while(ptr_out == NULL && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Handle other errors than EINTR
if(ptr_out == NULL)
log_err("Memory reallocation (%p -> %zu) failed in %s() (%s:%i)",
ptr_in, size, func, file, line);
+ // Restore errno value
+ errno = _errno;
+
return ptr_out;
}
\ No newline at end of file
diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c
index 65128d8b..c0e77ecf 100644
--- a/src/syscalls/recv.c
+++ b/src/syscalls/recv.c
@@ -17,7 +17,7 @@
#undef recv
ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, const char *func, const int line)
{
- ssize_t ret = 0;
+ ssize_t ret = 0;
do
{
// Reset errno before trying to write
@@ -28,11 +28,17 @@ ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file,
// incoming signal
while(ret < 0 && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(ret < 0)
log_warn("Could not recv() in %s() (%s:%i): %s",
func, file, line, strerror(errno));
- return ret;
+ // Restore errno value
+ errno = _errno;
+
+ return ret;
}
\ No newline at end of file
diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c
index 86b8bebc..f16dd6f1 100644
--- a/src/syscalls/recvfrom.c
+++ b/src/syscalls/recvfrom.c
@@ -18,7 +18,7 @@
#undef recvfrom
ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen, const char *file, const char *func, const int line)
{
- ssize_t ret = 0;
+ ssize_t ret = 0;
do
{
// Reset errno before trying to write
@@ -29,11 +29,17 @@ ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockadd
// incoming signal
while(ret < 0 && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(ret < 0)
log_warn("Could not recvfrom() in %s() (%s:%i): %s",
func, file, line, strerror(errno));
- return ret;
+ // Restore errno value
+ errno = _errno;
+
+ return ret;
}
\ No newline at end of file
diff --git a/src/syscalls/select.c b/src/syscalls/select.c
index 605a883a..5eb07c9a 100644
--- a/src/syscalls/select.c
+++ b/src/syscalls/select.c
@@ -17,7 +17,7 @@
#undef select
int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout, const char *file, const char *func, const int line)
{
- int ret = 0;
+ int ret = 0;
do
{
// Reset errno before trying to write
@@ -28,11 +28,17 @@ int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, st
// incoming signal
while(ret < 0 && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(ret < 0)
log_warn("Could not select() in %s() (%s:%i): %s",
func, file, line, strerror(errno));
- return ret;
+ // Restore errno value
+ errno = _errno;
+
+ return ret;
}
\ No newline at end of file
diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c
index 4ed35e9a..276a32dd 100644
--- a/src/syscalls/sendto.c
+++ b/src/syscalls/sendto.c
@@ -29,11 +29,17 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc
// incoming signal
while(ret < 0 && errno == EINTR);
+ // Backup errno value
+ const int _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(ret < 0)
log_warn("Could not sendto() in %s() (%s:%i): %s",
func, file, line, strerror(errno));
- return ret;
+ // Restore errno value
+ errno = _errno;
+
+ return ret;
}
\ No newline at end of file
diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h
index 1b2787cd..dfe1a178 100644
--- a/src/syscalls/syscalls.h
+++ b/src/syscalls/syscalls.h
@@ -45,7 +45,7 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc
int FTLpthread_mutex_lock(pthread_mutex_t *__mutex, const char *file, const char *func, const int line);
// Interrupt-safe file routines
-FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line);
+FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const char *func, const int line) __attribute__ ((__malloc__));
// Syscall helpers
void syscalls_report_error(const char *error, FILE *stream, const int _errno, const char *format, const char *func, const char *file, const int line);
diff --git a/src/syscalls/vasprintf.c b/src/syscalls/vasprintf.c
index 8300a8c5..3ac340e6 100644
--- a/src/syscalls/vasprintf.c
+++ b/src/syscalls/vasprintf.c
@@ -54,6 +54,9 @@ int FTLvasprintf(const char *file, const char *func, const int line, char **buff
stdout, _errno, format, func, file, line);
}
+ // Restore errno value
+ errno = _errno;
+
// Return number of written bytes
return length;
}
diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c
index b945ae4f..7a5ffae9 100644
--- a/src/syscalls/vfprintf.c
+++ b/src/syscalls/vfprintf.c
@@ -134,6 +134,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line
if(buffer != NULL)
free(buffer);
+ // Restore errno value
+ errno = _errno;
+
// Return early, there isn't anything we can do here
return length;
}
@@ -152,6 +155,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line
// to an interruption by an incoming signal
while(_buffer < buffer && errno == EINTR);
+ // Backup errno value
+ _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(_buffer < buffer)
@@ -163,6 +169,9 @@ int FTLvfprintf(FILE *stream, const char *file, const char *func, const int line
// Free allocated memory
free(buffer);
+ // Restore errno value
+ errno = _errno;
+
// Return number of written bytes
return length;
}
diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c
index 2e1b5f08..4f4badfc 100644
--- a/src/syscalls/vsnprintf.c
+++ b/src/syscalls/vsnprintf.c
@@ -47,6 +47,10 @@ int FTLvsnprintf(const char *file, const char *func, const int line, char *__res
stdout, _errno, format, func, file, line);
}
+
+ // Restore errno value
+ errno = _errno;
+
// Return number of written bytes
return length;
}
diff --git a/src/syscalls/vsprintf.c b/src/syscalls/vsprintf.c
index b31ba414..cce0b35f 100644
--- a/src/syscalls/vsprintf.c
+++ b/src/syscalls/vsprintf.c
@@ -54,6 +54,9 @@ int FTLvsprintf(const char *file, const char *func, const int line, char *__rest
stdout, _errno, format, func, file, line);
}
+ // Restore errno value
+ errno = _errno;
+
// Return number of written bytes
return length;
}
diff --git a/src/syscalls/write.c b/src/syscalls/write.c
index b7d8d984..df62adda 100644
--- a/src/syscalls/write.c
+++ b/src/syscalls/write.c
@@ -28,19 +28,26 @@ ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const
// Reset errno before trying to write
errno = 0;
ret = write(fd, buf, total);
- if(ret > 0)
- written += ret;
+ if(ret > 0)
+ written += ret;
}
// Try to write the remaining content into the stream if
// (a) we haven't written all the data, however, there was no other error
// (b) the last write() call failed due to an interruption by an incoming signal
while((written < total && errno == 0) || (ret < 0 && errno == EINTR));
+ // Backup errno value
+ const int _errno = errno;
+
// Final error checking (may have failed for some other reason then an
// EINTR = interrupted system call)
if(written < total)
log_warn("Could not write() everything in %s() [%s:%i]: %s",
func, file, line, strerror(errno));
- return written;
+ // Restore errno value
+ errno = _errno;
+
+ // Return number of written bytes
+ return written;
}
\ No newline at end of file
diff --git a/test/arch_test.sh b/test/arch_test.sh
index e97bb7d5..6a8ef0cd 100644
--- a/test/arch_test.sh
+++ b/test/arch_test.sh
@@ -73,11 +73,15 @@ check_static() {
echo "Static executable check: OK"
}
-if [[ "${CI_ARCH}" == "x86_64" ]]; then
+if [[ "${CI_ARCH}" == "x86_64" || "${CI_ARCH}" == "x86_64_full" ]]; then
check_machine "ELF64" "Advanced Micro Devices X86-64"
- check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]"
- check_file "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.32, not stripped"
+ if [[ "${CI_ARCH}" == "x86_64_full" ]]; then
+ check_libs "[libm.so.6] [librt.so.1] [libdbus-1.so.3] [libmnl.so.0] [libnftables.so.1] [libnftnl.so.11] [libnfnetlink.so.0] [libnetfilter_conntrack.so.3] [libpthread.so.0] [libc.so.6]"
+ else
+ check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]"
+ fi
+ check_file "ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 3.2.0, with debug_info, not stripped"
elif [[ "${CI_ARCH}" == "x86_64-musl" ]]; then
@@ -90,13 +94,13 @@ elif [[ "${CI_ARCH}" == "x86_32" ]]; then
check_machine "ELF32" "Intel 80386"
check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6]"
- check_file "ELF 32-bit LSB shared object, Intel 80386, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.2, for GNU/Linux 2.6.32, not stripped"
+ check_file "ELF 32-bit LSB shared object, Intel 80386, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.2, for GNU/Linux 3.2.0, with debug_info, not stripped"
elif [[ "${CI_ARCH}" == "aarch64" ]]; then
check_machine "ELF64" "AArch64"
check_libs "[libm.so.6] [librt.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-aarch64.so.1]"
- check_file "ELF 64-bit LSB shared object, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, not stripped"
+ check_file "ELF 64-bit LSB shared object, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, with debug_info, not stripped"
elif [[ "${CI_ARCH}" == "armv4t" ]]; then
@@ -111,16 +115,16 @@ elif [[ "${CI_ARCH}" == "armv5te" ]]; then
check_machine "ELF32" "ARM"
check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux.so.3]"
- check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, not stripped"
+ check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped"
- check_CPU_arch "v4T"
+ check_CPU_arch "v5TE"
check_FP_arch "" # No specified FP arch
elif [[ "${CI_ARCH}" == "armv6hf" ]]; then
check_machine "ELF32" "ARM"
check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]"
- check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 2.6.32, not stripped"
+ check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped"
check_CPU_arch "v6"
check_FP_arch "VFPv2"
@@ -129,7 +133,7 @@ elif [[ "${CI_ARCH}" == "armv7hf" ]]; then
check_machine "ELF32" "ARM"
check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]"
- check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, not stripped"
+ check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped"
check_CPU_arch "v7"
check_FP_arch "VFPv3-D16"
@@ -138,7 +142,7 @@ elif [[ "${CI_ARCH}" == "armv8a" ]]; then
check_machine "ELF32" "ARM"
check_libs "[libm.so.6] [librt.so.1] [libgcc_s.so.1] [libpthread.so.0] [libc.so.6] [ld-linux-armhf.so.3]"
- check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, not stripped"
+ check_file "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, with debug_info, not stripped"
check_CPU_arch "v8"
check_FP_arch "VFPv3-D16"
diff --git a/test/gravity.db.sql b/test/gravity.db.sql
index 45604aa4..1a43780e 100644
--- a/test/gravity.db.sql
+++ b/test/gravity.db.sql
@@ -25,12 +25,16 @@ CREATE TABLE domainlist
CREATE TABLE adlist
(
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- address TEXT UNIQUE NOT NULL,
- enabled BOOLEAN NOT NULL DEFAULT 1,
- date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
- date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
- comment TEXT
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ address TEXT UNIQUE NOT NULL,
+ enabled BOOLEAN NOT NULL DEFAULT 1,
+ date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+ date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+ comment TEXT,
+ date_updated INTEGER,
+ number INTEGER NOT NULL DEFAULT 0,
+ invalid_domains INTEGER NOT NULL DEFAULT 0,
+ status INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE adlist_by_group
@@ -208,12 +212,13 @@ INSERT INTO domainlist VALUES(14,3,'^regex-notA$;querytype=!A',1,1559928803,1559
/* Other special domains */
INSERT INTO domainlist VALUES(15,1,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group');
-INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list');
+INSERT INTO adlist VALUES(1,'https://hosts-file.net/ad_servers.txt',1,1559928803,1559928803,'Migrated from /etc/pihole/adlists.list',1559928803,2000,2,1);
INSERT INTO gravity VALUES('allowed.ftl',1);
INSERT INTO gravity VALUES('gravity.ftl',1);
+INSERT INTO gravity VALUES('gravity-aaaa.ftl',1);
INSERT INTO gravity VALUES('gravity-allowed.ftl',1);
-INSERT INTO info VALUES('gravity_count',3);
+INSERT INTO info VALUES('gravity_count',4);
INSERT INTO "group" VALUES(1,0,'Test group',1559928803,1559928803,'A disabled test group');
INSERT INTO domainlist_by_group VALUES(15,1);
diff --git a/test/pdns/pdns.conf b/test/pdns/pdns.conf
index ebe41ead..2449edf5 100644
--- a/test/pdns/pdns.conf
+++ b/test/pdns/pdns.conf
@@ -9,7 +9,6 @@
# Local DNS address and port
local-address=127.0.0.1:5554
-local-ipv6=
# Do not enforce TCP for ANY queries
any-to-tcp=false
@@ -19,3 +18,6 @@ launch=gsqlite3
# Database location
gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
+
+# Used when creating a new zone
+default-soa-content=ns1.@ hostmaster.@ 0 10800 3600 604800 3600
diff --git a/test/pdns/recursor.conf b/test/pdns/recursor.conf
index 6919a556..06f40946 100644
--- a/test/pdns/recursor.conf
+++ b/test/pdns/recursor.conf
@@ -12,3 +12,12 @@ local-address=127.0.0.1:5555
# Use authoritative server for ftl. and arpa. zones
forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554
+
+# In this mode the Recursor acts as a “security aware, non-validating”
+# nameserver, meaning it will set the DO-bit on outgoing queries and will
+# provide DNSSEC related RRsets (NSEC, RRSIG) to clients that ask for them (by
+# means of a DO-bit in the query), except for zones provided through the
+# auth-zones setting. It will not do any validation in this mode, not even when
+# requested by the client.
+# The default mode until PowerDNS Recursor 4.5.0.
+dnssec=process-no-validate
diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh
index d5697936..50ff6420 100644
--- a/test/pdns/setup.sh
+++ b/test/pdns/setup.sh
@@ -40,7 +40,7 @@ else
fi
# Create zone ftl
pdnsutil create-zone ftl ns1.ftl
-pdnsutil add-record ftl. . SOA "ns1.ftl. hostmaster.ftl. 1 10800 3600 604800 3600"
+pdnsutil disable-dnssec ftl
# Create A records
pdnsutil add-record ftl. a A 192.168.1.1
@@ -65,6 +65,7 @@ pdnsutil add-record ftl. regex-REPLYv4 AAAA fe80::2c01
pdnsutil add-record ftl. regex-REPLYv6 AAAA fe80::2c02
pdnsutil add-record ftl. regex-REPLYv46 AAAA fe80::2c03
pdnsutil add-record ftl. any AAAA fe80::3c01
+pdnsutil add-record ftl. gravity-aaaa AAAA fe80::4c01
# Create CNAME records
pdnsutil add-record ftl. cname-1 CNAME gravity.ftl
@@ -79,6 +80,10 @@ pdnsutil add-record ftl. cname-ok CNAME a.ftl
# Create CNAME for SOA test domain
pdnsutil add-record ftl. soa CNAME ftl
+# Create CNAME for NODATA tests
+pdnsutil add-record ftl. aaaa-cname CNAME gravity-aaaa.ftl
+pdnsutil add-record ftl. a-cname CNAME gravity.ftl
+
# Create PTR records
pdnsutil add-record ftl. ptr PTR ptr.ftl.
@@ -92,17 +97,10 @@ pdnsutil add-record ftl. naptr NAPTR '20 10 "s" "http+N2L+N2C+N2R" "" ftl.'
pdnsutil add-record ftl. mx MX "50 ns1.ftl."
# SVCB + HTTPS
-if ! pdnsutil add-record ftl. svcb SVCB '1 port="80"'; then
- # see RFC3597: Handling of Unknown DNS Resource Record (RR) Types
- # and https://ypcs.fi/howto/2020/09/30/announce-https-via-dns/
- pdnsutil add-record ftl. svcb TYPE64 "\# 13 000109706F72743D2238302200"
-fi
+pdnsutil add-record ftl. svcb SVCB '1 port="80"'
# HTTPS
-if ! pdnsutil add-record ftl. https HTTPS '1 . alpn="h3,h2"'; then
- # comment above applies
- pdnsutil add-record ftl. https TYPE65 "\# 15 000100000100080322683303683222"
-fi
+pdnsutil add-record ftl. https HTTPS '1 . alpn="h3,h2"'
# Create reverse lookup zone
pdnsutil create-zone arpa ns1.ftl
@@ -120,6 +118,8 @@ pdnsutil rectify-all-zones
pdnsutil check-zone ftl
pdnsutil check-zone arpa
+pdnsutil list-all-zones
+
echo "********* Done installing PowerDNS configuration **********"
# Start services
diff --git a/test/run.sh b/test/run.sh
index 22f6bf82..4df87190 100755
--- a/test/run.sh
+++ b/test/run.sh
@@ -1,7 +1,7 @@
#!/bin/bash
-# Only run tests on x86_64, x86_64-musl, and x86_32 targets
-if [[ ${CI} == "true" && "${CI_ARCH}" != "x86_64" && "${CI_ARCH}" != "x86_64-musl" && "${CI_ARCH}" != "x86_32" ]]; then
+# Only run tests on x86_* targets (where the CI can natively run the binaries)
+if [[ ${CI} == "true" && "${CI_ARCH}" != "x86_"* ]]; then
echo "Skipping tests (CI_ARCH: ${CI_ARCH})!"
exit 0
fi
@@ -15,7 +15,7 @@ fi
while pidof -s pihole-FTL > /dev/null; do
pid="$(pidof -s pihole-FTL)"
echo "Terminating running pihole-FTL process with PID ${pid}"
- kill $pid
+ kill "$pid"
sleep 1
done
@@ -62,10 +62,6 @@ bash test/pdns/setup.sh
OLDUMASK=$(umask)
umask 0022
-# Prepare LUA scripts
-mkdir -p /opt/pihole/libs
-wget -O /opt/pihole/libs/inspect.lua https://ftl.pi-hole.net/libraries/inspect.lua
-
# Start FTL
if ! su pihole -s /bin/sh -c /home/pihole/pihole-FTL; then
echo "pihole-FTL failed to start"
@@ -99,10 +95,14 @@ curl_to_tricorder() {
}
if [[ $RET != 0 ]]; then
- echo -n "pihole.log: "
+ echo -n "pihole/pihole.log: "
curl_to_tricorder /var/log/pihole/pihole.log
echo ""
+<<<<<<< HEAD
echo -n "FTL.log: "
+=======
+ echo -n "pihole/FTL.log: "
+>>>>>>> development
curl_to_tricorder /var/log/pihole/FTL.log
echo ""
echo -n "dig.log: "
@@ -110,6 +110,9 @@ if [[ $RET != 0 ]]; then
echo ""
echo -n "ptr.log: "
curl_to_tricorder ./ptr.log
+ echo ""getallqueries
+ echo -n "getallqueries.log: "
+ curl_to_tricorder ./getallqueries.log
echo ""
echo -n "HTTP_info.log: "
openssl s_client -quiet -connect tricorder.pi-hole.net:9998 2> /dev/null < /var/log/pihole/HTTP_info.log
@@ -120,10 +123,10 @@ if [[ $RET != 0 ]]; then
fi
# Kill pihole-FTL after having completed tests
-kill $(pidof pihole-FTL)
+kill "$(pidof pihole-FTL)"
# Restore umask
-umask $OLDUMASK
+umask "$OLDUMASK"
# Remove copied file
rm /home/pihole/pihole-FTL
diff --git a/test/test_suite.bats b/test/test_suite.bats
index d27c8cbf..e08f0da9 100644
--- a/test/test_suite.bats
+++ b/test/test_suite.bats
@@ -32,6 +32,17 @@
[[ ${lines[9]} == *"pihole-FTL is already running!" ]]
}
+@test "dnsmasq options as expected" {
+ run bash -c './pihole-FTL -vv | grep "cryptohash"'
+ printf "%s\n" "${lines[@]}"
+ if [[ "${CI_ARCH}" == "x86_64_full" ]]; then
+ [[ ${lines[0]} == "Features: IPv6 GNU-getopt DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]]
+ else
+ [[ ${lines[0]} == "Features: IPv6 GNU-getopt no-DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP no-conntrack ipset no-nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]]
+ fi
+ [[ ${lines[1]} == "" ]]
+}
+
@test "Starting tests without prior history" {
run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole/FTL.log'
printf "%s\n" "${lines[@]}"
@@ -317,7 +328,7 @@
@test "Local DNS test: SOA ftl" {
run bash -c "dig SOA ftl @127.0.0.1 +short"
printf "%s\n" "${lines[@]}"
- [[ ${lines[0]} == "ns1.ftl. hostmaster.ftl. 1 10800 3600 604800 3600" ]]
+ [[ ${lines[0]} == "ns1.ftl. hostmaster.ftl. 0 10800 3600 604800 3600" ]]
[[ ${lines[1]} == "" ]]
}
@@ -357,16 +368,16 @@
}
@test "Local DNS test: SVCB svcb.ftl" {
- run bash -c "dig +unknown TYPE64 svcb.ftl @127.0.0.1 +short"
+ run bash -c "dig SVCB svcb.ftl @127.0.0.1 +short"
printf "%s\n" "${lines[@]}"
- [[ ${lines[0]} == '\# 13 000109706F72743D2238302200' ]]
+ [[ ${lines[0]} == '1 port=\"80\".' ]]
[[ ${lines[1]} == "" ]]
}
@test "Local DNS test: HTTPS https.ftl" {
- run bash -c "dig +unknown TYPE65 https.ftl @127.0.0.1 +short"
+ run bash -c "dig HTTPS https.ftl @127.0.0.1 +short"
printf "%s\n" "${lines[@]}"
- [[ ${lines[0]} == '\# 15 000100000100080322683303683222' ]]
+ [[ ${lines[0]} == '1 . alpn="h3,h2"' ]]
[[ ${lines[1]} == "" ]]
}
@@ -384,14 +395,33 @@
[[ ${lines[1]} == "" ]]
}
+@test "CNAME inspection: NODATA CNAME targets are blocked" {
+ run bash -c "dig A a-cname.ftl @127.0.0.1 +short"
+ printf "%s\n" "${lines[@]}"
+ [[ ${lines[0]} == "0.0.0.0" ]]
+ [[ ${lines[1]} == "" ]]
+ run bash -c "dig AAAA a-cname.ftl @127.0.0.1 +short"
+ printf "%s\n" "${lines[@]}"
+ [[ ${lines[0]} == "::" ]]
+ [[ ${lines[1]} == "" ]]
+ run bash -c "dig A aaaa-cname.ftl @127.0.0.1 +short"
+ printf "%s\n" "${lines[@]}"
+ [[ ${lines[0]} == "0.0.0.0" ]]
+ [[ ${lines[1]} == "" ]]
+ run bash -c "dig AAAA aaaa-cname.ftl @127.0.0.1 +short"
+ printf "%s\n" "${lines[@]}"
+ [[ ${lines[0]} == "::" ]]
+ [[ ${lines[1]} == "" ]]
+}
+
@test "DNSSEC: SECURE domain is resolved" {
- run bash -c "dig A sigok.verteiltesysteme.net @127.0.0.1"
+ run bash -c "dig A dnssec.works @127.0.0.1"
printf "%s\n" "${lines[@]}"
[[ ${lines[@]} == *"status: NOERROR"* ]]
}
@test "DNSSEC: BOGUS domain is rejected" {
- run bash -c "dig A sigfail.verteiltesysteme.net @127.0.0.1"
+ run bash -c "dig A fail01.dnssec.works @127.0.0.1"
printf "%s\n" "${lines[@]}"
[[ ${lines[@]} == *"status: SERVFAIL"* ]]
}
@@ -441,8 +471,7 @@
@test "Help CLI argument return help text" {
run bash -c '/home/pihole/pihole-FTL help'
printf "%s\n" "${lines[@]}"
- [[ ${lines[0]} == "pihole-FTL - The Pi-hole FTL engine" ]]
- [[ ${lines[3]} == "Available arguments:" ]]
+ [[ ${lines[0]} == "The Pi-hole FTL engine - "* ]]
}
#@test "No WARNING messages in FTL.log (besides known capability issues)" {
-
+