diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 652e4ece..dbb9de7a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "FTL x86_64 Build Env", - "image": "ghcr.io/pi-hole/ftl-build:v2.5", + "image": "ghcr.io/pi-hole/ftl-build:nightly", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], "customizations": { "vscode": { @@ -8,13 +8,15 @@ "jetmartin.bats", "ms-vscode.cpptools", "ms-vscode.cmake-tools", - "eamodio.gitlens" + "eamodio.gitlens", + "github.copilot", + "ms-python.python" ] } }, "mounts": [ "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly", - "type=bind,source=/var/www/html,target=/var/www/html,readonly" + "type=bind,source=/var/www/html/admin,target=/var/www/html/admin,readonly" ] } diff --git a/.github/.codespellignore b/.github/.codespellignore index cdccd1cd..c1b04b2a 100644 --- a/.github/.codespellignore +++ b/.github/.codespellignore @@ -7,3 +7,9 @@ requestor requestors punycode bitap +mmapped +dnsmasq +iif +prefered +padd +rabit diff --git a/.github/.codespellignore_lines b/.github/.codespellignore_lines index 49f7eebf..a592a395 100644 --- a/.github/.codespellignore_lines +++ b/.github/.codespellignore_lines @@ -1 +1,3 @@ self.errors.append("Exception when GETing from FTL: " + str(e)) +// sitten -> sittin (substitution of "i" for "e"), +// sittin -> sitting (insertion of "g" at the end). diff --git a/.github/Dockerfile b/.github/Dockerfile index 2731b8a2..094c9ef2 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,25 +1,31 @@ -FROM ghcr.io/pi-hole/ftl-build:v2.5 AS builder +FROM ghcr.io/pi-hole/ftl-build:v2.8 AS builder WORKDIR /app COPY . /app ARG CI_ARCH="linux/amd64" -ENV CI_ARCH ${CI_ARCH} +ENV CI_ARCH=${CI_ARCH} ARG GIT_BRANCH="test" -ENV GIT_BRANCH ${GIT_BRANCH} +ENV GIT_BRANCH=${GIT_BRANCH} ARG GIT_TAG="test" -ENV GIT_TAG ${GIT_TAG} +ENV GIT_TAG=${GIT_TAG} +ARG BUILD_OPTS="" +ENV BUILD_OPTS=${BUILD_OPTS} + +# Setting TERM is needed for pretty output in BATS tests +ENV TERM=xterm + +# Monkeypatch BATS to remove duplicate output of starting and finished test +# BATS uses ANSI escape codes to overwrite the line after the test has finished +# This is not supported by Github Actions as it does not provide a TTY to the docker build container +RUN sed -i '/buffer_with_truncation /d' /bats-core/libexec/bats-core/bats-format-pretty # Build FTL # Remove possible old build files RUN rm -rf cmake && \ -# Build FTL - bash build.sh "-DSTATIC=${STATIC}" && \ -# Run binary architecture tests - bash test/arch_test.sh && \ -# Run full test suite - bash test/run.sh && \ +# Build and test FTL + bash build.sh "-DSTATIC=${STATIC}" test ${BUILD_OPTS} && \ # Move FTL binary to root directory cd / &&\ mv /app/pihole-FTL . && \ diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index 731f382e..ce073c59 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -5,6 +5,9 @@ inputs: platform: required: true description: The platform to build for + build_opts: + required: true + description: Any extra build opts to use git_branch: required: true description: The branch to build from @@ -76,6 +79,7 @@ runs: "CI_ARCH=${{ inputs.platform }}" "GIT_BRANCH=${{ inputs.git_branch }}" "GIT_TAG=${{ inputs.git_tag }}" + "BUILD_OPTS=${{ inputs.build_opts }}" - name: List files in current directory shell: bash @@ -98,22 +102,34 @@ runs: with: name: ${{ inputs.artifact_name }} path: '${{ inputs.bin_name }}*' + - + name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + # Skip attestation if ACTIONS_ID_TOKEN_REQUEST_URL env variable is not + # available (e.g., PR originating from a fork) + if: ${{ env.ACTIONS_ID_TOKEN_REQUEST_URL != '' }} + with: + subject-path: ${{ inputs.bin_name }} - name: Extract documentation files from container - if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' + if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' && inputs.build_opts == '' shell: bash run: | tar -xf build.tar api-docs.tar.gz - name: Upload documentation artifacts for deployoment - if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' + if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64' && inputs.build_opts == '' uses: actions/upload-artifact@v4.3.1 with: name: pihole-api-docs path: 'api-docs.tar.gz' - name: Deploy - if: inputs.event_name != 'pull_request' + # Skip deployment step if: + # - this is a triggered by a PR event (we only push on commit to branch + # events) + # - no SSH key is provided (this is a PR from a fork) + if: inputs.event_name != 'pull_request' && ${{ inputs.SSH_KEY != '' }} uses: ./.github/actions/deploy with: pattern: ${{ inputs.bin_name }}-binary diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2f025b2e..ff4ebf1b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,23 +17,3 @@ updates: github_action-dependencies: patterns: - "*" - -# As above, but for development-v6 -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - day: saturday - time: "10:00" - open-pull-requests-limit: 10 - target-branch: development-v6 - reviewers: - - "pi-hole/ftl-maintainers" - pull-request-branch-name: - # Separate sections of the branch name with a hyphen - separator: "-" - groups: - github_action-dependencies: - patterns: - - "*" - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07dd5280..e3d69c3b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,10 @@ name: Build, Test, Deploy +permissions: + id-token: write + contents: read + attestations: write + on: push: branches: @@ -26,7 +31,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: "Calculate required variables" id: variables @@ -59,10 +64,16 @@ jobs: include: - platform: linux/amd64 bin_name: pihole-FTL-amd64 + build_opts: "" + - platform: linux/amd64 + bin_name: pihole-FTL-amd64-clang + build_opts: clang - platform: linux/386 bin_name: pihole-FTL-386 + build_opts: "" - platform: linux/riscv64 bin_name: pihole-FTL-riscv64 + build_opts: "" env: CI_ARCH: ${{ matrix.platform }} GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }} @@ -70,13 +81,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test with: platform: ${{ matrix.platform }} bin_name: ${{ matrix.bin_name }} + build_opts: ${{ matrix.build_opts }} artifact_name: ${{ matrix.bin_name }}-binary target_dir: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }} git_branch: ${{ needs.smoke-tests.outputs.GIT_BRANCH }} @@ -108,7 +120,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: Build and test and deploy FTL uses: ./.github/actions/build-and-test diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..a3878d25 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,147 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" +env: + nettleversion: 3.9.1 + mbedtlsversion: 3.6.1 + +on: + push: + branches: [ "master", "development", "special/CI*", "update/dnsmasq" ] + pull_request: + branches: [ "master", "development", "special/CI*", "update/dnsmasq" ] + schedule: + - cron: '45 10 * * 6' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: c-cpp + build-mode: manual + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libidn2-0 libidn2-0-dev libunistring-dev + + - name: Install nettle + run: | + curl -sSL https://ftl.pi-hole.net/libraries/nettle-${nettleversion}.tar.gz | tar -xz + cd nettle-${nettleversion} + ./configure --enable-static --disable-shared --disable-openssl --disable-mini-gmp -disable-gcov --disable-documentation + sudo make -j $(nproc) install + + - name: Install mbedTLS + # Build static mbedTLS with pthread support + # Disable AESNI on linux/386 asit would possibly result in an incompatible + # binary in processors lacking the AESNI and SSE2 instruction sets + run: | + curl -sSL https://ftl.pi-hole.net/libraries/mbedtls-${mbedtlsversion}.tar.bz2 | tar -xj + cd mbedtls-${mbedtlsversion} + sed -i '/#define MBEDTLS_THREADING_C/s*^//**g' include/mbedtls/mbedtls_config.h + sed -i '/#define MBEDTLS_THREADING_PTHREAD/s*^//**g' include/mbedtls/mbedtls_config.h + sudo make -j $(nproc) install + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + queries: security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + ./build.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" + upload: failure-only # upload only in case of failure, otherwise upload later after filtering + output: codeql-results + + - name: Filter SARIF + uses: advanced-security/filter-sarif@v1 + with: + # filter out third-party dependencies + patterns: | + -src/dnsmasq/* + -src/webserver/civetweb/* + -src/webserver/cJSON/* + -src/tre-regex/* + -src/config/tomlc99/* + -src/database/shell.c + -src/database/sqlite3.c + -src/database/sqlite3.h + -src/zip/miniz/* + -src/lua/* + +src/lua/ftl_* + input: codeql-results/cpp.sarif + output: codeql-results/cpp.sarif + + - name: Upload SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: codeql-results/cpp.sarif + + - name: Upload CodeQL results as an artifact + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: codeql-results + path: codeql-results + retention-days: 5 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 2989b5b4..8ce5dfe2 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 43b59de4..24b299fc 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v2.1.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.2 with: dirtyLabel: "Merge conflicts" repoToken: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/openapi-validator.yml b/.github/workflows/openapi-validator.yml index ccafd283..8827d074 100644 --- a/.github/workflows/openapi-validator.yml +++ b/.github/workflows/openapi-validator.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: Set Node.js version uses: actions/setup-node@v4 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 87b75885..41e1793a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 36085247..e15d1049 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.7 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.gitignore b/.gitignore index 075dad8f..a3c7c317 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,8 @@ version~ # IDE files .idea/ *.sw* -/.vscode -.vscode/ -/.vscode/ +.vscode/* +!.vscode/c_cpp_properties.json /build/ # __pycache__ files (API tests) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..901ccbde --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,18 @@ +{ + "configurations": [ + { + "name": "Linux", + "includePath": [ + "${workspaceFolder}/src/**" + ], + "compileCommands": "${workspaceFolder}/build/compile_commands.json", + "defines": [], + "compilerPath": "/usr/bin/gcc", + "cStandard": "gnu17", + "cppStandard": "gnu++17", + "intelliSenseMode": "linux-gcc-x64", + "configurationProvider": "ms-vscode.cmake-tools" + } + ], + "version": 4 +} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index a5c421ad..30e2d624 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,10 +8,14 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -cmake_minimum_required(VERSION 2.8.12) +# C17 supports requires minimum CMake version 3.21 +# GCC 8.1.0 +# LLVM Clang 7.0.0 +cmake_minimum_required(VERSION 3.21) +set(CMAKE_C_STANDARD 17) project(PIHOLE_FTL C) -set(DNSMASQ_VERSION pi-hole-v2.90+1) +set(DNSMASQ_VERSION pi-hole-v2.90+2) add_subdirectory(src) diff --git a/build.sh b/build.sh index b83062a4..2df0f8f1 100755 --- a/build.sh +++ b/build.sh @@ -23,6 +23,7 @@ do "-C" | "CLEAN" ) clean=1 && nobuild=1;; "-i" | "install" ) install=1;; "-t" | "test" ) test=1;; + "clang" ) clang=1;; "ci" ) builddir="cmake_ci/";; esac done @@ -60,6 +61,13 @@ for scriptname in src/lua/scripts/*.lua; do fi done +# Set compiler to clang if requested +if [[ -n "${clang}" ]]; then + export CC=clang + export CXX=clang++ + export STATIC="false" +fi + # Configure build, pass CMake CACHE entries if present # Wrap multiple options in "" as first argument to ./build.sh: # ./build.sh "-DA=1 -DB=2" install @@ -88,5 +96,6 @@ fi # If we are asked to run tests, we do this here if [[ -n "${test}" ]]; then cd .. - ./test/run.sh + bash test/arch_test.sh + bash test/run.sh fi diff --git a/patch/civetweb.sh b/patch/civetweb.sh index f33dc8de..3fcc732a 100644 --- a/patch/civetweb.sh +++ b/patch/civetweb.sh @@ -1,13 +1,23 @@ #!/bin/sh set -e +echo "Applying patches for civetweb" +echo "Applying patch 0001-add-pihole-mods.patch" patch -p1 < patch/civetweb/0001-add-pihole-mods.patch -patch -p1 < patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch + +echo "Applying patch 0001-Always-Kepler-syntax-for-Lua-server-pages.patch" patch -p1 < patch/civetweb/0001-Always-Kepler-syntax-for-Lua-server-pages.patch + +echo "Applying patch 0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch" patch -p1 < patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch + +echo "Applying patch 0001-Add-mbedTLS-debug-logging-hook.patch" patch -p1 < patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch + +echo "Applying patch 0001-Add-Register-CSRF-token-in-conn-request_info.patch" patch -p1 < patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch + +echo "Applying patch 0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch" patch -p1 < patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch -patch -p1 < patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch echo "ALL PATCHES APPLIED OKAY" diff --git a/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch b/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch index f5439791..f0f78ea0 100644 --- a/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch +++ b/patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch @@ -14,14 +14,14 @@ index 0d293f1f..44f6cf3d 100644 --- a/src/webserver/civetweb/civetweb.c +++ b/src/webserver/civetweb/civetweb.c @@ -7754,6 +7754,8 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ - mg_snprintf( - conn, &truncated, filename, filename_buf_len - 1, "%s%s", root, uri); + roots[i], + uri); -+ FTL_rewrite_pattern(filename, filename_buf_len - 1, root, uri); ++ FTL_rewrite_pattern(filename, filename_buf_len - 1); + - if (truncated) { - goto interpret_cleanup; - } + if (truncated) { + goto interpret_cleanup; + } diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h index e71dfedc..2ad76693 100644 --- a/src/webserver/civetweb/civetweb.h @@ -30,8 +30,8 @@ index e71dfedc..2ad76693 100644 int status, const char* mime_type, long long content_length); -+void FTL_rewrite_pattern(char *filename, size_t filename_buf_len, -+ const char *root, const char *uri); ++void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len); ++ + // Buffer used for additional "Set-Cookie" headers #define PIHOLE_HEADERS_MAXLEN 1024 diff --git a/patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch b/patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch deleted file mode 100644 index ee48ec54..00000000 --- a/patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 1b81285fed48df6939d4b2569bba9e572f4c1137 Mon Sep 17 00:00:00 2001 -From: DL6ER -Date: Fri, 13 Jan 2023 21:37:31 +0100 -Subject: [PATCH] Add NO_DLOPEN option to civetweb's LUA routines - -Signed-off-by: DL6ER ---- - src/webserver/civetweb/mod_lua.inl | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl -index 5cc94318..59c4f2b3 100644 ---- a/src/webserver/civetweb/mod_lua.inl -+++ b/src/webserver/civetweb/mod_lua.inl -@@ -3634,7 +3634,7 @@ lua_init_optional_libraries(void) - lua_shared_init(); - - /* UUID library */ --#if !defined(_WIN32) -+#if !defined(_WIN32) && !defined(NO_DLOPEN) - lib_handle_uuid = dlopen("libuuid.so", RTLD_LAZY); - pf_uuid_generate.p = - (lib_handle_uuid ? dlsym(lib_handle_uuid, "uuid_generate") : 0); -@@ -3648,7 +3648,7 @@ static void - lua_exit_optional_libraries(void) - { - /* UUID library */ --#if !defined(_WIN32) -+#if !defined(_WIN32) && !defined(NO_DLOPEN) - if (lib_handle_uuid) { - dlclose(lib_handle_uuid); - } --- -2.34.1 - diff --git a/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch b/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch index ba7e8b2a..71f5c0ea 100644 --- a/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch +++ b/patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch @@ -14,8 +14,8 @@ index 2ad76693..52724199 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h @@ -938,6 +938,10 @@ int my_send_http_error_headers(struct mg_connection *conn, - void FTL_rewrite_pattern(char *filename, size_t filename_buf_len, - const char *root, const char *uri); + void FTL_rewrite_pattern(char *filename, size_t filename_buf_len); + +#define MG_CONFIG_MBEDTLS_DEBUG 3 +void FTL_mbed_debug(void *user_param, int level, const char *file, @@ -36,9 +36,9 @@ index e72685f4..00b9280a 100644 + mbedtls_ssl_conf_dbg(conf, FTL_mbed_debug, NULL); + /****************************************************/ + - #ifdef MBEDTLS_SSL_PROTO_TLS1_3 - psa_status_t status = psa_crypto_init(); - if (status != PSA_SUCCESS) { + /* Initialize TLS key and cert */ + mbedtls_pk_init(&ctx->pkey); + mbedtls_ctr_drbg_init(&ctx->ctr); -- 2.34.1 diff --git a/patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch b/patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch deleted file mode 100644 index d54ab29e..00000000 --- a/patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch +++ /dev/null @@ -1,35 +0,0 @@ -From ebb27741b10ed2eac51ac356708800ae96cdd17a Mon Sep 17 00:00:00 2001 -From: DL6ER -Date: Tue, 31 Oct 2023 08:35:31 +0100 -Subject: [PATCH] Allow extended ASCII characters in URIs - -Signed-off-by: DL6ER ---- - src/webserver/civetweb/civetweb.c | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c -index 9b0c6308..5320c4d4 100644 ---- a/src/webserver/civetweb/civetweb.c -+++ b/src/webserver/civetweb/civetweb.c -@@ -10734,7 +10734,7 @@ skip_to_end_of_word_and_terminate(char **ppw, int eol) - { - /* Forward until a space is found - use isgraph here */ - /* See http://www.cplusplus.com/reference/cctype/ */ -- while (isgraph((unsigned char)**ppw)) { -+ while ((unsigned char)**ppw > 127 || isgraph((unsigned char)**ppw)) { - (*ppw)++; - } - -@@ -18473,7 +18473,7 @@ get_uri_type(const char *uri) - * and % encoded symbols. - */ - for (i = 0; uri[i] != 0; i++) { -- if (uri[i] < 33) { -+ if ((unsigned char)uri[i] < 33) { - /* control characters and spaces are invalid */ - return 0; - } --- -2.34.1 - diff --git a/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch b/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch index c396cb0f..9dc655e8 100644 --- a/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch +++ b/patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch @@ -27,28 +27,6 @@ index 3df8eab9..9b0c6308 100644 #endif /* DEBUG */ #endif /* DEBUG_TRACE */ -diff --git a/src/webserver/civetweb/mod_mbedtls.inl b/src/webserver/civetweb/mod_mbedtls.inl -index 00b9280a..6a450ba3 100644 ---- a/src/webserver/civetweb/mod_mbedtls.inl -+++ b/src/webserver/civetweb/mod_mbedtls.inl -@@ -213,7 +213,7 @@ mbed_ssl_accept(mbedtls_ssl_context **ssl, - return -1; - } - -- DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->state); -+ DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->MBEDTLS_PRIVATE(state)); - return 0; - } - -@@ -239,7 +239,7 @@ mbed_ssl_handshake(mbedtls_ssl_context *ssl) - } - } - -- DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->state); -+ DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->MBEDTLS_PRIVATE(state)); - return rc; - } - -- 2.34.1 diff --git a/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch b/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch index 575c5649..50506f0f 100644 --- a/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch +++ b/patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch @@ -6,9 +6,9 @@ Subject: [PATCH] Register CSRF token and is_authenticated boolean in conn->reque Signed-off-by: DL6ER --- src/webserver/civetweb/civetweb.c | 3 +++ - src/webserver/civetweb/civetweb.h | 2 ++ - src/webserver/civetweb/mod_lua.inl | 3 +++ - 3 files changed, 8 insertions(+) + src/webserver/civetweb/civetweb.h | 3 +++ + src/webserver/civetweb/mod_lua.inl | 4 ++++ + 3 files changed, 10 insertions(+) diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c index 233b342a..f44b17ba 100644 @@ -20,7 +20,6 @@ index 233b342a..f44b17ba 100644 + /* Pi-hole addition */ + memset(conn->request_info.csrf_token, 0, sizeof(conn->request_info.csrf_token)); -+ reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0); + #if defined(USE_SERVER_STATS) conn->processing_time = 0; @@ -29,7 +28,7 @@ diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb index 5b3d596b..291ef683 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h -@@ -183,6 +183,8 @@ struct mg_request_info { +@@ -183,6 +183,9 @@ struct mg_request_info { const char *acceptedWebSocketSubprotocol; /* websocket subprotocol, * accepted during handshake */ @@ -43,13 +42,14 @@ diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua index e9a13835..92066b3f 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl -@@ -2603,6 +2603,9 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L) +@@ -2603,6 +2603,10 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L) reg_string(L, "finger", conn->request_info.client_cert->finger); lua_rawset(L, -3); } + + /* Pi-hole addition */ + reg_string(L, "csrf_token", conn->request_info.csrf_token); ++ reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0); } diff --git a/patch/lua.sh b/patch/lua.sh index 9987b222..80bc6f70 100644 --- a/patch/lua.sh +++ b/patch/lua.sh @@ -2,5 +2,7 @@ set -e patch -p1 < patch/lua/0001-add-pihole-library.patch +patch -p1 < patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch +patch -p1 < patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch echo "ALL PATCHES APPLIED OKAY" diff --git a/patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch b/patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch new file mode 100644 index 00000000..2270a56f --- /dev/null +++ b/patch/lua/0001-Add-bundled-script-loading-into-luaL_openlibs-to-mak.patch @@ -0,0 +1,90 @@ +From 0ff00e1c838ec91a31970c2b51a7651954cba3d6 Mon Sep 17 00:00:00 2001 +From: DL6ER +Date: Mon, 23 Sep 2024 21:42:21 +0200 +Subject: [PATCH] Add bundled script loading into luaL_openlibs to make them + available globally (also in the webserver) + +Signed-off-by: DL6ER +--- + src/lua/ftl_lua.h | 2 -- + src/lua/linit.c | 6 ++++++ + src/lua/lua.c | 13 +------------ + 3 files changed, 7 insertions(+), 14 deletions(-) + +diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h +index d986498a..30bad1f9 100644 +--- a/src/lua/ftl_lua.h ++++ b/src/lua/ftl_lua.h +@@ -21,8 +21,6 @@ int run_luac(const int argc, char **argv); + int lua_main (int argc, char **argv); + int luac_main (int argc, char **argv); + +-extern int dolibrary (lua_State *L, char *name); +- + void print_embedded_scripts(void); + void ftl_lua_init(lua_State *L); + +diff --git a/src/lua/linit.c b/src/lua/linit.c +index 9a5bcfdc..787865c0 100644 +--- a/src/lua/linit.c ++++ b/src/lua/linit.c +@@ -8,6 +8,10 @@ + #define linit_c + #define LUA_LIB + ++/** Pi-hole modification **/ ++#include "ftl_lua.h" ++/**************************/ ++ + /* + ** If you embed Lua in your program and need to open the standard + ** libraries, call luaL_openlibs in your program. If you need a +@@ -64,5 +68,7 @@ LUALIB_API void luaL_openlibs (lua_State *L) { + luaL_requiref(L, lib->name, lib->func, 1); + lua_pop(L, 1); /* remove lib */ + } ++ // Load and enable libraries bundled with Pi-hole ++ ftl_lua_init(L); + } + +diff --git a/src/lua/lua.c b/src/lua/lua.c +index 35fb281d..111a1b2b 100644 +--- a/src/lua/lua.c ++++ b/src/lua/lua.c +@@ -20,10 +20,6 @@ + #include "lauxlib.h" + #include "lualib.h" + +-/** Pi-hole modification **/ +-#include "ftl_lua.h" +-/**************************/ +- + + #if !defined(LUA_PROGNAME) + #define LUA_PROGNAME "lua" +@@ -218,9 +214,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { + ** If there is no explicit modname and globname contains a '-', cut + ** the suffix after '-' (the "version") to make the global name. + */ +-/************** Pi-hole modification ***************/ +-int dolibrary (lua_State *L, char *globname) { +-/***************************************************/ ++static int dolibrary (lua_State *L, char *globname) { + int status; + char *suffix = NULL; + char *modname = strchr(globname, '='); +@@ -655,11 +649,6 @@ static int pmain (lua_State *L) { + return 0; /* error running LUA_INIT */ + } + +- /************** Pi-hole modification ***************/ +- // Load and enable libraries bundled with Pi-hole +- ftl_lua_init(L); +- /***************************************************/ +- + if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ + return 0; /* something failed */ + if (script > 0) { /* execute main script (if there is one) */ +-- +2.34.1 + diff --git a/patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch b/patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch new file mode 100644 index 00000000..64e2658f --- /dev/null +++ b/patch/lua/0001-Increase-LUA_IDSIZE-so-that-long-script-filenames-as.patch @@ -0,0 +1,27 @@ +From 835933f8501e517a781b380b8cfafa656adc6fa7 Mon Sep 17 00:00:00 2001 +From: DL6ER +Date: Mon, 23 Sep 2024 13:25:34 +0200 +Subject: [PATCH] Increase LUA_IDSIZE so that long script filenames as well as + long script lines fit into the error logging buffer + +Signed-off-by: DL6ER +--- + src/lua/luaconf.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h +index 33bb580d..dacc5221 100644 +--- a/src/lua/luaconf.h ++++ b/src/lua/luaconf.h +@@ -765,7 +765,7 @@ + ** of a function in debug information. + ** CHANGE it if you want a different size. + */ +-#define LUA_IDSIZE 60 ++#define LUA_IDSIZE 256 + + + /* +-- +2.34.1 + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 185ba5dc..778764a6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,8 +8,6 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -set(CMAKE_C_STANDARD 11) - # Default to a release with debug info build if (NOT EXISTS ${CMAKE_BINARY_DIR}/CMakeCache.txt) if (NOT CMAKE_BUILD_TYPE) @@ -29,7 +27,6 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature. # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption. # SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime). -# HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB) # SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2) # SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator. @@ -37,7 +34,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk. # SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8. # SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL. -set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200") +set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200") # Code hardening and debugging improvements # -fstack-protector-strong: The program will be resistant to having its stack overflowed @@ -53,8 +50,10 @@ set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -D # -Wl,-z,now: Disable lazy binding # -Wl,-z,relro: Read-only segments after relocation # -fno-common: Emit globals without explicit initializer from `.bss` to `.data`. This causes GCC to reject multiple definitions of global variables. This is the new default from GCC-10 on. -set(HARDENING_FLAGS "-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -Wl,-z,relro,-z,now -fexceptions -funwind-tables -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro -fno-common") -set(DEBUG_FLAGS "-rdynamic -fno-omit-frame-pointer") +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(HARDENING_FLAGS "-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -Wl,-z,relro,-z,now -fexceptions -funwind-tables -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro -fno-common") + set(DEBUG_FLAGS "-rdynamic -fno-omit-frame-pointer") +endif() # -Wall: This enables all the warnings about constructions that some users consider questionable, and that are easy to avoid (or modify to prevent the warning), even in conjunction with macros. This also enables some language-specific warnings described in C++ Dialect Options and Objective-C and Objective-C++ Dialect Options. # -Wextra: This enables some extra warning flags that are not enabled by -Wall. @@ -155,11 +154,27 @@ else() set(EXTRAWARN_GCC13 "") endif() -set(EXTRAWARN "${EXTRAWARN_GCC6} \ - ${EXTRAWARN_GCC7} \ - ${EXTRAWARN_GCC8} \ - ${EXTRAWARN_GCC12} \ - ${EXTRAWARN_GCC13}") +# Set extrawarn flags if CC is GCC +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(EXTRAWARN "${EXTRAWARN_GCC6} \ + ${EXTRAWARN_GCC7} \ + ${EXTRAWARN_GCC8} \ + ${EXTRAWARN_GCC12} \ + ${EXTRAWARN_GCC13}") +elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(EXTRAWARN " + -Werror \ + -Wnewline-eof \ + -Wno-dangling-else \ + -Wno-gnu-zero-variadic-macro-arguments \ + -Wno-gnu-variable-sized-type-not-at-end \ + -Wno-declaration-after-statement \ + -Wno-reserved-identifier \ + -Wno-reserved-macro-identifier") +else() + message(WARNING "Unknown compiler, not setting warnings flags") + set(EXTRAWARN "") +endif() # Remove extra spaces from EXTRAWARN string(REGEX REPLACE " +" " " EXTRAWARN "${EXTRAWARN}") @@ -185,11 +200,14 @@ else() message(STATUS "Compiling dynamically linked executable") endif() # -pie -fPIE: (Dynamic) position independent executable -set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE") + +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE") +endif() # -FILE_OFFSET_BITS=64: used by stat(). Avoids problems with files > 2 GB on 32bit machines # We define HAVE_POLL_H as this is needed for the musl builds to succeed -set(CMAKE_C_FLAGS "-pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}") +set(CMAKE_C_FLAGS "-std=c99 -pipe ${WARN_FLAGS} -D_FILE_OFFSET_BITS=64 ${HARDENING_FLAGS} ${DEBUG_FLAGS} ${CMAKE_C_FLAGS} -DHAVE_POLL_H ${SQLITE_DEFINES}") set(CMAKE_C_FLAGS_DEBUG "-O0 -g3") set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") @@ -250,15 +268,14 @@ add_custom_target( COMMAND ${CMAKE_COMMAND} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_version.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -add_library(FTL OBJECT ${sources}) -target_compile_options(FTL PRIVATE ${EXTRAWARN}) -target_compile_definitions(FTL PRIVATE DNSMASQ_VERSION=\"${DNSMASQ_VERSION}\") -target_include_directories(FTL PRIVATE ${PROJECT_SOURCE_DIR}/src) -add_dependencies(FTL gen_version) - +add_library(core OBJECT ${sources}) +target_compile_options(core PRIVATE ${EXTRAWARN}) +target_compile_definitions(core PRIVATE DNSMASQ_VERSION=\"${DNSMASQ_VERSION}\") +target_include_directories(core PRIVATE ${PROJECT_SOURCE_DIR}/src) +add_dependencies(core gen_version) add_executable(pihole-FTL - $ + $ $ $ $ @@ -270,32 +287,36 @@ add_executable(pihole-FTL $ $ $ + $ $ $ $ $ $ + $ ) if(STATIC) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON) set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_END_STATIC ON) - target_link_libraries(pihole-FTL -static-libgcc -static -pie) + target_link_libraries(pihole-FTL -static-libgcc -static) + set(LIBRARY_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") else() find_library(LIBMATH m) target_link_libraries(pihole-FTL ${LIBMATH}) + set(LIBRARY_SUFFIX "") endif() set(CMAKE_THREAD_PREFER_PTHREAD TRUE) set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) # for DNSSEC we need the nettle (+ hogweed) crypto and the gmp math libraries -find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64) -find_library(LIBGMP NAMES libgmp${CMAKE_STATIC_LIBRARY_SUFFIX} gmp) -find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64) +find_library(LIBHOGWEED NAMES libhogweed${LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64) +find_library(LIBGMP NAMES libgmp${LIBRARY_SUFFIX} gmp) +find_library(LIBNETTLE NAMES libnettle${LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64) # for IDN2 we need the idn2 library which in turn depends on the unistring library -find_library(LIBIDN2 NAMES libidn2${CMAKE_STATIC_LIBRARY_SUFFIX} idn2) -find_library(LIBUNISTRING NAMES libunistring${CMAKE_STATIC_LIBRARY_SUFFIX} unistring) +find_library(LIBIDN2 NAMES libidn2${LIBRARY_SUFFIX} idn2) +find_library(LIBUNISTRING NAMES libunistring${LIBRARY_SUFFIX} unistring) target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN2} ${LIBUNISTRING}) @@ -304,55 +325,6 @@ if(LUA_DL STREQUAL "true") target_link_libraries(pihole-FTL ${LIBDL}) endif() -find_library(LIBREADLINE NAMES libreadline${CMAKE_STATIC_LIBRARY_SUFFIX} readline) -find_library(LIBHISTORY NAMES libhistory${CMAKE_STATIC_LIBRARY_SUFFIX} history) -find_library(LIBTERMCAP NAMES libtermcap${CMAKE_STATIC_LIBRARY_SUFFIX} termcap) -if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) - message(STATUS "Building FTL with readline support: YES") - target_compile_definitions(FTL PRIVATE LUA_USE_READLINE) - target_compile_definitions(pihole-FTL PRIVATE LUA_USE_READLINE) - target_link_libraries(pihole-FTL ${LIBREADLINE} ${LIBHISTORY} ${LIBTERMCAP}) -else() - message(STATUS "Building FTL with readline support: NO") -endif() - -# Do we want to compile an all-in FTL version? -if(DEFINED ENV{CI_ARCH}) - if($ENV{CI_ARCH} STREQUAL "x86_64_full") - add_definitions(-DDNSMASQ_ALL_OPTS) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) - find_package(DBus REQUIRED) - # Use results of find_package() call. - include_directories(${DBUS_INCLUDE_DIRS}) - target_link_libraries(pihole-FTL ${DBUS_LIBRARIES}) - find_library(LIBMNL mnl) - find_library(LIBNFTNL nftnl) - find_library(LIBNFTABLES nftables) - find_library(LIBNFNETLINK nfnetlink) - find_library(LIBNETFILTER_CONNTRACK netfilter_conntrack) - target_link_libraries(pihole-FTL ${LIBMNL} ${LIBNFTABLES} ${LIBNFTNL} ${LIBNFNETLINK} ${LIBNETFILTER_CONNTRACK}) - endif() -endif() - -if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE) -endif() - -find_library(LIBMBEDCRYPTO NAMES lmbedcrypto${CMAKE_STATIC_LIBRARY_SUFFIX} mbedcrypto) -find_library(LIBMBEDX509 NAMES lmbedx509${CMAKE_STATIC_LIBRARY_SUFFIX} mbedx509) -find_library(LIBMBEDTLS NAMES lmbedtls${CMAKE_STATIC_LIBRARY_SUFFIX} mbedtls) -if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS) - # Link against the mbedTLS libraries, the order is important (!) - target_compile_definitions(FTL PRIVATE HAVE_MBEDTLS) - target_link_libraries(pihole-FTL ${LIBMBEDTLS} ${LIBMBEDX509} ${LIBMBEDCRYPTO}) -endif() - -find_program(SETCAP setcap) -install(TARGETS pihole-FTL - RUNTIME DESTINATION bin - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) -install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_CHOWN+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)") - add_subdirectory(api) add_subdirectory(webserver) add_subdirectory(zip) @@ -364,3 +336,51 @@ add_subdirectory(tre-regex) add_subdirectory(syscalls) add_subdirectory(config) add_subdirectory(tools) +add_subdirectory(ntp) + +find_library(LIBREADLINE NAMES libreadline${LIBRARY_SUFFIX} readline) +find_library(LIBHISTORY NAMES libhistory${LIBRARY_SUFFIX} history) +find_library(LIBTERMCAP NAMES libtermcap${LIBRARY_SUFFIX} termcap) +if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) + message(STATUS "Building FTL with readline support: YES") + target_compile_definitions(lua PRIVATE LUA_USE_READLINE) + target_compile_definitions(sqlite3 PRIVATE HAVE_READLINE) + target_link_libraries(pihole-FTL ${LIBREADLINE} ${LIBHISTORY} ${LIBTERMCAP}) +else() + message(STATUS "Building FTL with readline support: NO") +endif() + +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE) +endif() + +find_library(LIBMBEDCRYPTO NAMES lmbedcrypto${LIBRARY_SUFFIX} mbedcrypto) +find_library(LIBMBEDX509 NAMES lmbedx509${LIBRARY_SUFFIX} mbedx509) +find_library(LIBMBEDTLS NAMES lmbedtls${LIBRARY_SUFFIX} mbedtls) +if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS) + # Enable TLS support in civetweb if mbedTLS is available + message(STATUS "Building FTL with TLS support: YES") + target_compile_definitions(core PRIVATE HAVE_MBEDTLS) + target_compile_definitions(civetweb PRIVATE USE_MBEDTLS) + target_compile_definitions(webserver PRIVATE HAVE_MBEDTLS) + # Link against the mbedTLS libraries, the order is important (!) + target_link_libraries(pihole-FTL ${LIBMBEDTLS} ${LIBMBEDX509} ${LIBMBEDCRYPTO}) +else() + # Disable TLS support in civetweb if mbedTLS is not available + message(STATUS "Building FTL with TLS support: NO") + target_compile_definitions(civetweb PRIVATE NO_SSL) +endif() + +# After finishing building the FTL binary, we append the sha256sum of the binary in raw form to itself +add_custom_command(TARGET pihole-FTL POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ $/pihole-FTL.tmp + COMMAND sha256sum $.tmp | cut -d ' ' -f 1 | xxd -r -p >> $.tmp + COMMAND mv $.tmp $ + ) + +find_program(SETCAP setcap) +install(TARGETS pihole-FTL + RUNTIME DESTINATION bin + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) +install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_CHOWN,CAP_SYS_TIME+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)") + diff --git a/src/FTL.h b/src/FTL.h index 9edd2034..5eb91005 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -33,7 +33,6 @@ #include #include #include -//#include #include // syslog #include @@ -48,7 +47,7 @@ // MIN(x,y) is already defined in dnsmasq.h // Number of elements in an array -#define ArraySize(X) (sizeof(X)/sizeof(X[0])) +#define ArraySize(X) (sizeof(X)/sizeof(*X)) // Constant socket buffer length #define SOCKETBUFFERLEN 1024 @@ -124,12 +123,13 @@ // Default: 180 [seconds] #define DELAY_UPTIME 180 -// DB_QUERY_MAX_ITER defines how many queries we check periodically for updates to be added -// to the in-memory database. This value may need to be increased on *very* busy systems. -// However, there is an algorithm in place that tries to ensure we are not missing queries -// on systems with > 100 queries per second -// Default: 100 (per second) -#define DB_QUERY_MAX_ITER 100 +// REPLY_TIMEOUT defines until how far back in the history of queries we are +// checking for changed/updated queries. This value should not be set too high +// to avoid unnecessary spinning in the updating loop of the queries running +// every second. The value should be set to a value that is high enough to +// catch all queries that are still in the process of being resolved. +// Default: 30 [seconds] +#define REPLY_TIMEOUT 30 // Special exit code used to signal that FTL wants to restart #define RESTART_FTL_CODE 22 @@ -142,12 +142,16 @@ // Default: 2592000 (once per month) #define DATABASE_MACVENDOR_INTERVAL 2592000 +// Over how many seconds should the query-per-second (QPS) value be averaged? +// Default: 30 (seconds) +#define QPS_AVGLEN 30 + // Use out own syscalls handling functions that will detect possible errors // and report accordingly in the log. This will make debugging FTL crash // caused by insufficient memory or by code bugs (not properly dealing // with NULL pointers) much easier. #undef strdup // strdup() is a macro in itself, it needs special handling -#define free(ptr) FTLfree((void**)&ptr, __FILE__, __FUNCTION__, __LINE__) +#define free(ptr) { FTLfree(ptr, __FILE__, __FUNCTION__, __LINE__); ptr = NULL; } #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/api/2fa.c b/src/api/2fa.c index 4e255df2..164f7000 100644 --- a/src/api/2fa.c +++ b/src/api/2fa.c @@ -15,7 +15,7 @@ #include "config/config.h" // getrandom() #include "daemon.h" -// generate_app_password() +// generate_password() #include "config/password.h" // TOTP+HMAC @@ -313,7 +313,7 @@ int generateAppPw(struct ftl_conn *api) { // Generate and set app password char *password = NULL, *pwhash = NULL; - if(!generate_app_password(&password, &pwhash)) + if(!generate_password(&password, &pwhash)) { return send_json_error(api, 500, diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index e42a2505..a96a0f30 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -20,6 +20,7 @@ set(sources dhcp.c dns.c network.c + padd.c history.c info.c list.c diff --git a/src/api/action.c b/src/api/action.c index 138c1c6c..4a55d6e5 100644 --- a/src/api/action.c +++ b/src/api/action.c @@ -128,10 +128,7 @@ int api_action_restartDNS(struct ftl_conn *api) "Restarting DNS is not allowed", "Check setting webserver.api.allow_destructive"); - log_info("Restarting FTL due to API action request"); - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); + restart_ftl("API action request"); return send_json_success(api); } diff --git a/src/api/api.c b/src/api/api.c index bb26c2a4..444dc45a 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -90,6 +90,7 @@ static struct { { "/api/config", "/{element}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/config", "/{element}/{value}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_DELETE | HTTP_PUT }, { "/api/network/gateway", "", api_network_gateway, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/routes", "", api_network_routes, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/network/interfaces", "", api_network_interfaces, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/network/devices", "", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/network/devices", "/{device_id}", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, @@ -101,6 +102,7 @@ static struct { { "/api/action/restartdns", "", api_action_restartDNS, { API_PARSE_JSON, 0 }, true, HTTP_POST }, { "/api/action/flush/logs", "", api_action_flush_logs, { API_PARSE_JSON, 0 }, true, HTTP_POST }, { "/api/action/flush/arp", "", api_action_flush_arp, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/padd", "", api_padd, { API_PARSE_JSON, 0 }, true, HTTP_GET }, { "/api/docs", "", api_docs, { API_PARSE_JSON, 0 }, false, HTTP_GET }, }; @@ -113,10 +115,12 @@ int api_handler(struct mg_connection *conn, void *ignored) http_method(conn), NULL, NULL, + NULL, API_AUTH_UNAUTHORIZED, double_time(), { false, NULL, NULL, NULL, 0u }, { false }, + NULL, { API_FLAG_NONE, 0 } }; @@ -170,22 +174,7 @@ int api_handler(struct mg_connection *conn, void *ignored) } // Verify requesting client is allowed to see this resource - if(api_request[i].func == api_search) - { - // Handle /api/search special as it may be allowed for local users due to webserver.api.searchAPIauth - if(!config.webserver.api.searchAPIauth.v.b && is_local_api_user(api.request->remote_addr)) - { - // Local users does not need to authenticate when searchAPIauth is false - ; - } - else if(api_request[i].require_auth && check_client_auth(&api, true) == API_AUTH_UNAUTHORIZED) - { - // Users need to authenticate but authentication failed - unauthorized = true; - break; - } - } - else if(api_request[i].require_auth && check_client_auth(&api, true) == API_AUTH_UNAUTHORIZED) + if(api_request[i].require_auth && check_client_auth(&api, true) == API_AUTH_UNAUTHORIZED) { unauthorized = true; break; @@ -270,12 +259,7 @@ int api_handler(struct mg_connection *conn, void *ignored) // Restart FTL if requested if(api.ftl.restart) - { - log_info("Restarting FTL due to API config change"); - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); - } + restart_ftl(api.ftl.restart_reason); return ret; } diff --git a/src/api/api.h b/src/api/api.h index e8e57964..b72a1e92 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -17,6 +17,8 @@ #include "webserver/http-common.h" // regex_t #include "regex_r.h" +// enum conf_type +#include "config/config.h" // Common definitions #define LOCALHOSTv4 "127.0.0.1" @@ -27,12 +29,19 @@ int api_handler(struct mg_connection *conn, void *ignored); // Statistic methods int __attribute__((pure)) cmpdesc(const void *a, const void *b); +unsigned int get_active_clients(void); int api_stats_summary(struct ftl_conn *api); int api_stats_query_types(struct ftl_conn *api); int api_stats_upstreams(struct ftl_conn *api); int api_stats_top_domains(struct ftl_conn *api); int api_stats_top_clients(struct ftl_conn *api); int api_stats_recentblocked(struct ftl_conn *api); +cJSON *get_top_domains(struct ftl_conn *api, const int count, + const bool blocked, const bool domains_only); +cJSON *get_top_clients(struct ftl_conn *api, const int count, + const bool blocked, const bool clients_only, + const bool names_only, const bool ip_if_no_name); +cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only); // History methods int api_history(struct ftl_conn *api); @@ -65,18 +74,26 @@ int api_info_messages_count(struct ftl_conn *api); int api_info_messages(struct ftl_conn *api); int api_info_metrics(struct ftl_conn *api); int api_info_login(struct ftl_conn *api); +cJSON *read_sys_property(const char *path); +int get_system_obj(struct ftl_conn *api, cJSON *system); +int get_sensors_obj(struct ftl_conn *api, cJSON *sensors, const bool add_list); +int get_version_obj(struct ftl_conn *api, cJSON *version); // Config methods int api_config(struct ftl_conn *api); +int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed); +cJSON *addJSONConfValue(const enum conf_type conf_type, union conf_value *val); // Log methods int api_logs(struct ftl_conn *api); // Network methods int api_network_gateway(struct ftl_conn *api); +int api_network_routes(struct ftl_conn *api); int api_network_interfaces(struct ftl_conn *api); int api_network_devices(struct ftl_conn *api); int api_client_suggestions(struct ftl_conn *api); +int get_gateway(struct ftl_conn *api, cJSON * json, const bool detailed); // DNS methods int api_dns_blocking(struct ftl_conn *api); @@ -125,4 +142,7 @@ int api_search(struct ftl_conn *api); int api_dhcp_leases_GET(struct ftl_conn *api); int api_dhcp_leases_DELETE(struct ftl_conn *api); +// PADD methods +int api_padd(struct ftl_conn *api); + #endif // ROUTES_H diff --git a/src/api/auth.c b/src/api/auth.c index 714ca9d5..2aea675e 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -78,17 +78,10 @@ bool __attribute__((pure)) is_local_api_user(const char *remote_addr) // Returns >= 0 for any valid authentication int check_client_auth(struct ftl_conn *api, const bool is_api) { - // Is the user requesting from localhost? - // This may be allowed without authentication depending on the configuration - if(!config.webserver.api.localAPIauth.v.b && is_local_api_user(api->request->remote_addr)) - { - add_request_info(api, NULL); - return API_AUTH_LOCALHOST; - } - // When the pwhash is unset, authentication is disabled if(config.webserver.api.pwhash.v.s[0] == '\0') { + api->message = "no password set"; add_request_info(api, NULL); return API_AUTH_EMPTYPASS; } @@ -186,7 +179,8 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) if(!sid_avail) { - log_debug(DEBUG_API, "API Authentication: FAIL (no SID provided)"); + api->message = "no SID provided"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message); return API_AUTH_UNAUTHORIZED; } @@ -212,21 +206,28 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } else { - log_debug(DEBUG_API, "API Authentication: FAIL (Cookie authentication without CSRF token)"); + api->message = "Cookie authentication without CSRF token"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message); return API_AUTH_UNAUTHORIZED; } } + bool expired = false; for(unsigned int i = 0; i < max_sessions; i++) { if(auth_data[i].used && - auth_data[i].valid_until >= now && strcmp(auth_data[i].sid, sid) == 0) { + // Check if session is known but expired + if(auth_data[i].valid_until < now) + expired = true; + + // Check CSRF if authentiating via cookie if(need_csrf && strcmp(auth_data[i].csrf, csrf) != 0) { - log_debug(DEBUG_API, "API Authentication: FAIL (CSRF token mismatch, received \"%s\", expected \"%s\")", - csrf, auth_data[i].csrf); + api->message = "CSRF token mismatch"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s, received \"%s\", expected \"%s\")", + api->message, csrf, auth_data[i].csrf); return API_AUTH_UNAUTHORIZED; } user_id = i; @@ -258,7 +259,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // Debug logging if(config.debug.api.v.b) { - char timestr[128]; + char timestr[TIMESTR_SIZE]; get_timestr(timestr, auth_data[user_id].valid_until, false, false); log_debug(DEBUG_API, "Recognized known user: user_id %i, valid_until: %s, remote_addr %s (%s at login)", user_id, timestr, api->request->remote_addr, auth_data[user_id].remote_addr); @@ -266,12 +267,15 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } else { - log_debug(DEBUG_API, "API Authentication: FAIL (SID invalid/expired)"); + api->message = expired ? "session expired" : "session unknown"; + log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message); return API_AUTH_UNAUTHORIZED; } api->user_id = user_id; + api->session = &auth_data[user_id]; + api->message = "correct password"; return user_id; } @@ -295,8 +299,16 @@ static int get_all_sessions(struct ftl_conn *api, cJSON *json) JSON_ADD_NUMBER_TO_OBJECT(session, "last_active", auth_data[i].valid_until - config.webserver.session.timeout.v.ui); JSON_ADD_NUMBER_TO_OBJECT(session, "valid_until", auth_data[i].valid_until); JSON_REF_STR_IN_OBJECT(session, "remote_addr", auth_data[i].remote_addr); - JSON_REF_STR_IN_OBJECT(session, "user_agent", auth_data[i].user_agent); + if(auth_data[i].user_agent[0] != '\0') + JSON_REF_STR_IN_OBJECT(session, "user_agent", auth_data[i].user_agent); + else + JSON_ADD_NULL_TO_OBJECT(session, "user_agent"); + if(auth_data[i].x_forwarded_for[0] != '\0') + JSON_REF_STR_IN_OBJECT(session, "x_forwarded_for", auth_data[i].x_forwarded_for); + else + JSON_ADD_NULL_TO_OBJECT(session, "x_forwarded_for"); JSON_ADD_BOOL_TO_OBJECT(session, "app", auth_data[i].app); + JSON_ADD_BOOL_TO_OBJECT(session, "cli", auth_data[i].cli); JSON_ADD_ITEM_TO_ARRAY(sessions, session); } JSON_ADD_ITEM_TO_OBJECT(json, "sessions", sessions); @@ -308,12 +320,13 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ cJSON *session = JSON_NEW_OBJECT(); // Authentication not needed - if(user_id == API_AUTH_LOCALHOST || user_id == API_AUTH_EMPTYPASS) + if(user_id == API_AUTH_EMPTYPASS) { JSON_ADD_BOOL_TO_OBJECT(session, "valid", true); JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0); JSON_ADD_NULL_TO_OBJECT(session, "sid"); JSON_ADD_NUMBER_TO_OBJECT(session, "validity", -1); + JSON_REF_STR_IN_OBJECT(session, "message", api->message); JSON_ADD_ITEM_TO_OBJECT(json, "session", session); return 0; } @@ -326,6 +339,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ JSON_REF_STR_IN_OBJECT(session, "sid", auth_data[user_id].sid); JSON_REF_STR_IN_OBJECT(session, "csrf", auth_data[user_id].csrf); JSON_ADD_NUMBER_TO_OBJECT(session, "validity", auth_data[user_id].valid_until - now); + JSON_REF_STR_IN_OBJECT(session, "message", api->message); JSON_ADD_ITEM_TO_OBJECT(json, "session", session); return 0; } @@ -335,6 +349,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0); JSON_ADD_NULL_TO_OBJECT(session, "sid"); JSON_ADD_NUMBER_TO_OBJECT(session, "validity", -1); + JSON_REF_STR_IN_OBJECT(session, "message", api->message); JSON_ADD_ITEM_TO_OBJECT(json, "session", session); return 0; } @@ -401,14 +416,6 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized } } - else if(user_id == API_AUTH_LOCALHOST) - { - log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)"); - - cJSON *json = JSON_NEW_OBJECT(); - get_session_object(api, json, user_id, now); - JSON_SEND_OBJECT(json); - } else if(user_id == API_AUTH_EMPTYPASS) { log_debug(DEBUG_API, "API Auth status: OK (empty password)"); @@ -461,19 +468,9 @@ int api_auth(struct ftl_conn *api) if(api->method == HTTP_POST) { // Try to extract response from payload - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + const int ret = check_json_payload(api); + if(ret != 0) + return ret; // Check if password is available cJSON *json_password; @@ -531,7 +528,9 @@ int api_auth(struct ftl_conn *api) else result = verify_login(password); - if(result == PASSWORD_CORRECT || result == APPPASSWORD_CORRECT) + if(result == PASSWORD_CORRECT || + result == APPPASSWORD_CORRECT || + result == CLIPASSWORD_CORRECT) { // Accepted @@ -542,7 +541,7 @@ int api_auth(struct ftl_conn *api) // Check possible 2FA token // Successful login with empty password does not require 2FA - if(strlen(config.webserver.api.totp_secret.v.s) > 0 && result != APPPASSWORD_CORRECT) + if(strlen(config.webserver.api.totp_secret.v.s) > 0 && result == PASSWORD_CORRECT) { // Get 2FA token from payload cJSON *json_totp; @@ -583,7 +582,7 @@ int api_auth(struct ftl_conn *api) auth_data[i].valid_until < now) { log_debug(DEBUG_API, "API: Session of client %u (%s) expired, freeing...", - i, auth_data[i].remote_addr); + i, auth_data[i].remote_addr); delete_session(i); } @@ -609,10 +608,22 @@ int api_auth(struct ftl_conn *api) { auth_data[i].user_agent[0] = '\0'; } + // Store X-Forwarded-For (if available) + const char *x_forwarded_for = mg_get_header(api->conn, "X-Forwarded-For"); + if(x_forwarded_for != NULL) + { + strncpy(auth_data[i].x_forwarded_for, x_forwarded_for, sizeof(auth_data[i].x_forwarded_for)); + auth_data[i].x_forwarded_for[sizeof(auth_data[i].x_forwarded_for)-1] = '\0'; + } + else + { + auth_data[i].x_forwarded_for[0] = '\0'; + } auth_data[i].tls.login = api->request->is_ssl; auth_data[i].tls.mixed = false; auth_data[i].app = result == APPPASSWORD_CORRECT; + auth_data[i].cli = result == CLIPASSWORD_CORRECT; // Generate new SID and CSRF token generateSID(auth_data[i].sid); @@ -626,7 +637,7 @@ int api_auth(struct ftl_conn *api) // Debug logging if(config.debug.api.v.b && user_id > API_AUTH_UNAUTHORIZED) { - char timestr[128]; + char timestr[TIMESTR_SIZE]; get_timestr(timestr, auth_data[user_id].valid_until, false, false); log_debug(DEBUG_API, "API: Registered new user: user_id %i valid_until: %s remote_addr %s (accepted due to %s)", user_id, timestr, auth_data[user_id].remote_addr, @@ -642,6 +653,8 @@ int api_auth(struct ftl_conn *api) "API seats exceeded", "increase webserver.api.max_sessions"); } + + api->message = result == APPPASSWORD_CORRECT ? "app-password correct" : "password correct"; } else if(result == PASSWORD_RATE_LIMITED) { @@ -654,10 +667,12 @@ int api_auth(struct ftl_conn *api) else if(result == NO_PASSWORD_SET) { // No password set + api->message = "password incorrect"; log_debug(DEBUG_API, "API: Trying to auth with password but none set: '%s'", password); } else { + api->message = "password incorrect"; log_debug(DEBUG_API, "API: Password incorrect: '%s'", password); } diff --git a/src/api/auth.h b/src/api/auth.h index 53663026..9f5cc9b5 100644 --- a/src/api/auth.h +++ b/src/api/auth.h @@ -48,6 +48,7 @@ struct session { bool used; bool app; + bool cli; struct { bool login; bool mixed; @@ -56,8 +57,9 @@ struct session { time_t valid_until; char remote_addr[48]; // Large enough for IPv4 and IPv6 addresses, hard-coded in civetweb.h as mg_request_info.remote_addr char user_agent[128]; + char x_forwarded_for[48]; // see remote_addr note char sid[SID_SIZE]; char csrf[SID_SIZE]; }; -#endif // AUTH_H \ No newline at end of file +#endif // AUTH_H diff --git a/src/api/config.c b/src/api/config.c index 3d3563e9..690f8103 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -37,6 +37,7 @@ static struct { { { "dns", "DNS", "DNS server settings" }, { "dhcp", "DHCP", "DHCP server settings" }, + { "ntp", "NTP", "Network Time Sync settings" }, { "resolver", "Resolver", "Resolver settings" }, { "database", "Database", "Database settings" }, { "webserver", "HTTP/API", "Webserver and API settings" }, @@ -91,7 +92,7 @@ static cJSON *get_or_create_object(cJSON *parent, const char *path_element) // This function is used to add a property to the JSON output using the // appropriate type of the config item to add. -static cJSON *addJSONvalue(const enum conf_type conf_type, union conf_value *val) +cJSON *addJSONConfValue(const enum conf_type conf_type, union conf_value *val) { switch(conf_type) { @@ -128,6 +129,8 @@ static cJSON *addJSONvalue(const enum conf_type conf_type, union conf_value *val return cJSON_CreateStringReference(get_web_theme_str(val->web_theme)); case CONF_ENUM_TEMP_UNIT: return cJSON_CreateStringReference(get_temp_unit_str(val->temp_unit)); + case CONF_ENUM_BLOCKING_EDNS_MODE: + return cJSON_CreateStringReference(get_edns_mode_str(val->edns_mode)); case CONF_STRUCT_IN_ADDR: { // Special case 0.0.0.0 -> return empty string @@ -238,7 +241,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct // 1. Check it is a number // 2. Check the number is within the allowed range for the given data type if(!cJSON_IsNumber(elem) || - elem->valuedouble < LONG_MIN || elem->valuedouble > LONG_MAX) + elem->valuedouble < (double)LONG_MIN || elem->valuedouble > (double)LONG_MAX) return "not of type long"; // Set item conf_item->v.l = elem->valuedouble; @@ -250,7 +253,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct // 1. Check it is a number // 2. Check the number is within the allowed range for the given data type if(!cJSON_IsNumber(elem) || - elem->valuedouble < 0 || elem->valuedouble > ULONG_MAX) + elem->valuedouble < 0 || elem->valuedouble > (double)ULONG_MAX) return "not of type unsigned long"; // Set item conf_item->v.ul = elem->valuedouble; @@ -278,6 +281,7 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct free(conf_item->v.s); // Set item conf_item->v.s = strdup(elem->valuestring); + conf_item->t = CONF_STRING_ALLOCATED; // allocated now log_debug(DEBUG_CONFIG, "%s = \"%s\"", conf_item->k, conf_item->v.s); break; } @@ -389,6 +393,19 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct log_debug(DEBUG_CONFIG, "%s = %d", conf_item->k, conf_item->v.temp_unit); break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + // Check type + if(!cJSON_IsString(elem)) + return "not of type string"; + const int edns_mode = get_edns_mode_val(elem->valuestring); + if(edns_mode == -1) + return "invalid option"; + // Set item + conf_item->v.edns_mode = edns_mode; + log_debug(DEBUG_CONFIG, "%s = %d", conf_item->k, conf_item->v.edns_mode); + break; + } case CONF_ENUM_PRIVACY_LEVEL: { // Check type @@ -463,17 +480,9 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct return NULL; } -static int api_config_get(struct ftl_conn *api) +int get_json_config(struct ftl_conn *api, cJSON *json, const bool detailed) { - // Parse query string parameters - bool detailed = false; - if(api->request->query_string != NULL) - { - // Check if we should return detailed config information - get_bool_var(api->request->query_string, "detailed", &detailed); - } - - // Create root JSON object + // Create root config object cJSON *config_j = JSON_NEW_OBJECT(); // Does the user request only a subset of /config? @@ -551,7 +560,7 @@ static int api_config_get(struct ftl_conn *api) else { // Add current value - cJSON *val = addJSONvalue(conf_item->t, &conf_item->v); + cJSON *val = addJSONConfValue(conf_item->t, &conf_item->v); if(val == NULL) { log_warn("Cannot format config item type %s of type %i", @@ -562,7 +571,7 @@ static int api_config_get(struct ftl_conn *api) } // Add default value - cJSON *dval = addJSONvalue(conf_item->t, &conf_item->d); + cJSON *dval = addJSONConfValue(conf_item->t, &conf_item->d); if(dval == NULL) { log_warn("Cannot format config item type %s of type %i", @@ -576,7 +585,6 @@ static int api_config_get(struct ftl_conn *api) // Add config item flags cJSON *flags = JSON_NEW_OBJECT(); JSON_ADD_BOOL_TO_OBJECT(flags, "restart_dnsmasq", conf_item->f & FLAG_RESTART_FTL); - JSON_ADD_BOOL_TO_OBJECT(flags, "advanced", conf_item->f & FLAG_ADVANCED_SETTING); JSON_ADD_BOOL_TO_OBJECT(flags, "session_reset", conf_item->f & FLAG_INVALIDATE_SESSIONS); JSON_ADD_BOOL_TO_OBJECT(flags, "env_var", conf_item->f & FLAG_ENV_VAR); JSON_ADD_ITEM_TO_OBJECT(leaf, "flags", flags); @@ -592,7 +600,7 @@ static int api_config_get(struct ftl_conn *api) else { // Create the config item leaf object - cJSON *leaf = addJSONvalue(conf_item->t, &conf_item->v); + cJSON *leaf = addJSONConfValue(conf_item->t, &conf_item->v); if(leaf == NULL) { log_warn("Cannot format config item type %s of type %i", @@ -607,8 +615,6 @@ static int api_config_get(struct ftl_conn *api) // Release allocated memory free_config_path(requested_path); - cJSON *json = JSON_NEW_OBJECT(); - // Add topics and DNS server suggestions if in detailed mode if(detailed) { @@ -650,25 +656,33 @@ static int api_config_get(struct ftl_conn *api) // Build and return JSON response JSON_ADD_ITEM_TO_OBJECT(json, "config", config_j); + + return 0; +} + +static int api_config_get(struct ftl_conn *api) +{ + // Parse query string parameters + bool detailed = false; + if(api->request->query_string != NULL) + { + // Check if we should return detailed config information + get_bool_var(api->request->query_string, "detailed", &detailed); + } + + cJSON *json = JSON_NEW_OBJECT(); + get_json_config(api, json, detailed); + + // Build and return JSON response JSON_SEND_OBJECT(json); } static int api_config_patch(struct ftl_conn *api) { // Is there a payload with valid JSON data? - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + const int ret = check_json_payload(api); + if(ret != 0) + return ret; // Is there a "config" object at the root of the received JSON payload? cJSON *conf = cJSON_GetObjectItem(api->payload.json, "config"); @@ -680,6 +694,16 @@ static int api_config_patch(struct ftl_conn *api) NULL); } + // Return early if the user tries to change some settings but the config + // is in read-only mode + if(config.misc.readOnly.v.b) + { + return send_json_error(api, 403, + "forbidden", + "The config is currently in read-only mode", + NULL); + } + // Read all known config items bool config_changed = false; bool dnsmasq_changed = false; @@ -706,6 +730,16 @@ static int api_config_patch(struct ftl_conn *api) continue; } + if(new_item->f & FLAG_READ_ONLY && cJSON_IsBool(elem) && elem->valueint == 1) + { + char *key = strdup(new_item->k); + free_config(&newconf); + return send_json_error_free(api, 400, + "bad_request", + "This config option can only be set in pihole.toml, not via the API", + key, true, true); + } + // Check if this is a write-only config item with the placeholder value if(new_item->f & FLAG_WRITE_ONLY && cJSON_IsString(elem) && strcmp(elem->valuestring, PASSWORD_VALUE) == 0) @@ -734,7 +768,7 @@ static int api_config_patch(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "Config item is invalid", - hint, true); + hint, true, true); } // Get pointer to memory location of this conf_item (global) @@ -749,7 +783,7 @@ static int api_config_patch(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "Config items set via environment variables cannot be changed via the API", - key, true); + key, true, true); } // Skip processing if value didn't change compared to current value @@ -788,7 +822,10 @@ static int api_config_patch(struct ftl_conn *api) // If the privacy level was decreased, we need to restart if(new_item == &newconf.misc.privacylevel && new_item->v.privacy_level < conf_item->v.privacy_level) + { + api->ftl.restart_reason = "Privacy level decreased"; api->ftl.restart = true; + } // Check if this item changed the password, if so, we need to // invalidate all currently active sessions @@ -804,7 +841,10 @@ static int api_config_patch(struct ftl_conn *api) { char errbuf[ERRBUF_SIZE] = { 0 }; if(write_dnsmasq_config(&newconf, true, errbuf)) + { + api->ftl.restart_reason = "dnsmasq config changed"; api->ftl.restart = true; + } else { free_config(&newconf); @@ -855,9 +895,9 @@ static int api_config_put_delete(struct ftl_conn *api) const char *hint = NULL, *message = NULL; if(api->method == HTTP_PUT) - hint = "Use, e.g., PUT /api/config/dnsmasq/upstreams/127.0.0.1 to add \"127.0.0.1\" to config.dns.upstreams"; + hint = "Use, e.g., PUT /api/config/dns/upstreams/127.0.0.1 to add \"127.0.0.1\" to config.dns.upstreams"; else - hint = "Use, e.g., DELETE /api/config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"; + hint = "Use, e.g., DELETE /api/config/dns/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"; if(min_level < 2) { @@ -912,7 +952,7 @@ static int api_config_put_delete(struct ftl_conn *api) return send_json_error_free(api, 400, "bad_request", "Config items set via environment variables cannot be changed via the API", - key, true); + key, true, true); } // Check if this entry exists in the array @@ -1012,7 +1052,10 @@ static int api_config_put_delete(struct ftl_conn *api) char errbuf[ERRBUF_SIZE] = { 0 }; // Request restart of FTL if(write_dnsmasq_config(&newconf, true, errbuf)) + { + api->ftl.restart_reason = "dnsmasq config changed"; api->ftl.restart = true; + } else { // The new config did not work @@ -1048,6 +1091,25 @@ int api_config(struct ftl_conn *api) if(api->method == HTTP_GET) return api_config_get(api); + // Check if this is an app session and reject the request if app sudo + // mode is disabled + if(api->session != NULL && api->session->app && !config.webserver.api.app_sudo.v.b) + { + return send_json_error(api, 403, + "forbidden", + "Unable to change configuration (read-only)", + "The current app session is not allowed to modify Pi-hole config settings (webserver.api.app_sudo is false)"); + } + + // Check if this is a CLI session and reject the request + if(api->session != NULL && api->session->cli) + { + return send_json_error(api, 403, + "forbidden", + "Unable to change configuration (read-only)", + "The current CLI session is not allowed to modify Pi-hole config settings"); + } + // POST: Create a new config (not supported) // PATCH: Replace parts of the the config with the provided one // PUT: Replaces the entire config with the provided one (not supported diff --git a/src/api/dhcp.c b/src/api/dhcp.c index 16898ca9..4e9e67af 100644 --- a/src/api/dhcp.c +++ b/src/api/dhcp.c @@ -110,4 +110,4 @@ int api_dhcp_leases_DELETE(struct ftl_conn *api) // - 404 Not Found (if no lease was found) cJSON *json = JSON_NEW_OBJECT(); JSON_SEND_OBJECT_CODE(json, found ? 204 : 404); -} \ No newline at end of file +} diff --git a/src/api/dns.c b/src/api/dns.c index 525c727e..a7747868 100644 --- a/src/api/dns.c +++ b/src/api/dns.c @@ -31,21 +31,8 @@ static int get_blocking(struct ftl_conn *api) // Return current status cJSON *json = JSON_NEW_OBJECT(); const enum blocking_status blocking = get_blockingstatus(); - switch(blocking) - { - case BLOCKING_ENABLED: - JSON_REF_STR_IN_OBJECT(json, "blocking", "enabled"); - break; - case BLOCKING_DISABLED: - JSON_REF_STR_IN_OBJECT(json, "blocking", "disabled"); - break; - case DNS_FAILED: - JSON_REF_STR_IN_OBJECT(json, "blocking", "failure"); - break; - case BLOCKING_UNKNOWN: - JSON_REF_STR_IN_OBJECT(json, "blocking", "unknown"); - break; - } + const char *status = get_blocking_status_str(blocking); + JSON_REF_STR_IN_OBJECT(json, "blocking", status); // Get timer information (if applicable) double delay; @@ -74,19 +61,10 @@ static int set_blocking(struct ftl_conn *api) NULL); } - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + // Check if the payload is valid JSON + const int ret = check_json_payload(api); + if(ret != 0) + return ret; cJSON *elem = cJSON_GetObjectItemCaseSensitive(api->payload.json, "blocking"); if (!cJSON_IsBool(elem)) @@ -109,7 +87,7 @@ static int set_blocking(struct ftl_conn *api) // The blocking status does not need to be changed // Delete a possibly running timer - set_blockingmode_timer(-1.0, true); + set_blockingmode_timer(timer, true); log_debug(DEBUG_API, "No change in blocking mode, resetting timer"); } diff --git a/src/api/docs/CMakeLists.txt b/src/api/docs/CMakeLists.txt index 8df3045a..8e22c45d 100644 --- a/src/api/docs/CMakeLists.txt +++ b/src/api/docs/CMakeLists.txt @@ -34,6 +34,7 @@ set(sources hex/specs/logs.yaml hex/specs/main.yaml hex/specs/network.yaml + hex/specs/padd.yaml hex/specs/queries.yaml hex/specs/search.yaml hex/specs/stats.yaml diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 8ea10a13..4d75549c 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -280,6 +280,7 @@ components: - sid - csrf - validity + - message - totp properties: valid: @@ -299,6 +300,10 @@ components: validity: type: integer description: Remaining lifetime of this session unless refreshed (seconds) + message: + type: string + description: Human-readable message describing the session status + nullable: true password: type: object @@ -340,6 +345,9 @@ components: app: type: boolean description: Indicator if this session was initiated using an application password + cli: + type: boolean + description: Indicator if this session was initiated using the command-line interface (CLI) login_at: type: integer description: Timestamp of login (seconds since epoch) @@ -354,7 +362,12 @@ components: description: IP address of the client user_agent: type: string - description: User agent of the client + nullable: true + description: User agent of the client (optional) + x_forwarded_for: + type: string + nullable: true + description: IP address of the client (if behind a proxy, optional) example: - id: 1 current_session: true @@ -363,11 +376,13 @@ components: login: true mixed: false app: false + cli: false login_at: 1580000000 last_active: 1580000000 valid_until: 1580000300 remote_addr: "192.168.0.34" user_agent: "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0" + x_forwarded_for: null totp: type: object description: TOTP secret suggestion @@ -431,7 +446,7 @@ components: examples: auth_okay: - summary: Authentication valid + summary: Session valid value: session: valid: true @@ -439,6 +454,7 @@ components: sid: null csrf: null validity: 300 + message: null login_okay: summary: Login successful value: @@ -448,6 +464,7 @@ components: sid: "vFA+EP4MQ5JJvJg+3Q2Jnw=" csrf: "Ux87YTIiMOf/GKCefVIOMw=" validity: 300 + message: correct password no_login_required: summary: No login required for this client value: @@ -457,6 +474,7 @@ components: sid: null csrf: null validity: -1 + message: no auth for local user login_required: summary: Login required, 2FA disabled value: @@ -466,6 +484,7 @@ components: sid: null csrf: null validity: -1 + message: password incorrect login_required_2fa: summary: Login required, 2FA enabled value: @@ -475,6 +494,7 @@ components: sid: null csrf: null validity: -1 + message: password incorrect login_failed: summary: Login failed value: @@ -484,6 +504,7 @@ components: sid: null csrf: null validity: -1 + message: no SID provided errors: no_payload: summary: Bad request (no valid JSON payload) diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 72f0a471..687cb2cb 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -11,7 +11,7 @@ components: - "Client management" operationId: "get_clients" description: | - `{client}` is optional. Specifying it will result in only the requested client being returned. + `{client}` is optional. If it is specified, it will result in only the requested client being returned. This parameter needs to be URI-encoded. Valid combinations are: - `/api/clients` (all clients) @@ -42,7 +42,7 @@ components: - "Client management" operationId: "replace_client" description: | - Items may be updated by replacing them. `{client}` is required. + Items may be updated by replacing them. `{client}` is required and needs to be URI-encoded. Ensure to send all the required parameters (such as `comment` or `groups`) to ensure these properties are retained. The read-only fields `id` and `date_added` are preserved, `date_modified` is automatically updated on success. @@ -91,7 +91,7 @@ components: - "Client management" operationId: "delete_client" description: | - *Note:* There will be no content on success. + *Note:* There will be no content on success. `{client}` is required and needs to be URI-encoded. responses: '204': description: Item deleted @@ -383,6 +383,12 @@ components: type: integer readOnly: true example: 1611239099 + name: + description: hostname (only if available) + type: string + readOnly: true + nullable: true + example: localhost lists_processed: type: object properties: diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 0b785754..d9d7c612 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -239,6 +239,8 @@ components: type: integer optimizer: type: integer + upstreamBlockedTTL: + type: integer revServers: type: array items: @@ -250,6 +252,8 @@ components: type: boolean mode: type: string + edns: + type: string specialDomains: type: object properties: @@ -320,10 +324,51 @@ components: type: boolean logging: type: boolean + ignoreUnknownClients: + type: boolean hosts: type: array items: type: string + ntp: + type: object + properties: + ipv4: + type: object + properties: + active: + type: boolean + address: + type: string + x-format: ipv4 + ipv6: + type: object + properties: + active: + type: boolean + address: + type: string + x-format: ipv6 + sync: + type: object + properties: + active: + type: boolean + server: + type: string + interval: + type: integer + count: + type: integer + rtc: + type: object + properties: + set: + type: boolean + device: + type: string + utc: + type: boolean resolver: type: object properties: @@ -372,8 +417,6 @@ components: tls: type: object properties: - rev_proxy: - type: boolean cert: type: string paths: @@ -393,10 +436,6 @@ components: api: type: object properties: - localAPIauth: - type: boolean - searchAPIauth: - type: boolean max_sessions: type: integer prettyJSON: @@ -411,6 +450,10 @@ components: type: string app_pwhash: type: string + app_sudo: + type: boolean + cli_pw: + type: boolean excludeClients: type: array items: @@ -479,6 +522,8 @@ components: type: string extraLogging: type: boolean + readOnly: + type: boolean check: type: object properties: @@ -545,6 +590,8 @@ components: type: boolean reserved: type: boolean + ntp: + type: boolean all: type: boolean topics: @@ -620,11 +667,13 @@ components: cache: size: 10000 optimizer: 3600 + upstreamBlockedTTL: 86400 revServers: - "true,192.168.0.0/24,192.168.0.1,lan" blocking: active: true mode: 'NULL' + edns: 'NONE' specialDomains: mozillaCanary: true iCloudPrivateRelay: true @@ -653,9 +702,26 @@ components: rapidCommit: false multiDNS: false logging: false + ignoreUnknownClients: false hosts: - "11:22:33:44:55:66,192.168.1.123" - "11:22:33:44:55:67,192.168.1.124,hostname" + ntp: + ipv4: + active: true + address: "" + ipv6: + active: true + address: "" + sync: + active: true + server: "pool.ntp.org" + interval: 3600 + count: 8 + rtc: + set: true + device: "" + utc: true resolver: resolveIPv4: true resolveIPv6: true @@ -677,7 +743,6 @@ components: timeout: 300 restore: true tls: - rev_proxy: false cert: "/etc/pihole/tls.pem" paths: webroot: "/var/www/html" @@ -686,14 +751,14 @@ components: boxed: true theme: "default-darker" api: - localAPIauth: false - searchAPIauth: false max_sessions: 16 prettyJSON: false password: "********" pwhash: '' totp_secret: '' app_pwhash: '' + app_sudo: false + cli_pw: true excludeClients: [ '1\.2\.3\.4', 'localhost', 'fe80::345' ] excludeDomains: [ 'google\\.de', 'pi-hole\.net' ] maxHistory: 86400 @@ -723,6 +788,7 @@ components: etc_dnsmasq_d: false dnsmasq_lines: [ ] extraLogging: false + readOnly: false check: load: true shmem: 90 @@ -755,6 +821,7 @@ components: webserver: false extra: false reserved: false + ntp: false all: false config_one: summary: One option diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index dadaee27..6be2b1ac 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -717,10 +717,25 @@ components: type: integer description: Number of denied domains example: 3 + regex: + type: object + properties: + allowed: + type: integer + description: Number of allowed regex filters + example: 4 + denied: + type: integer + description: Number of denied regex filters + example: 2 privacy_level: type: integer description: Currently used privacy level example: 0 + query_frequency: + type: number + description: Average number of queries per second + example: 1.1 clients: type: object properties: diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index 1762aed4..5f1e35ea 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -63,6 +63,8 @@ tags: description: Methods used to gather advanced information about your network - name: "Actions" description: Methods used to trigger certain actions on your Pi-hole + - name: "PADD" + description: Methods used to query Pi-hole from PADD @@ -241,6 +243,9 @@ paths: /network/gateway: $ref: 'network.yaml#/components/paths/gateway' + /network/routes: + $ref: 'network.yaml#/components/paths/routes' + /network/interfaces: $ref: 'network.yaml#/components/paths/interfaces' @@ -271,6 +276,9 @@ paths: /docs: $ref: 'docs.yaml#/components/paths/docs' + /padd: + $ref: 'padd.yaml#/components/paths/padd' + components: securitySchemes: query_sid: diff --git a/src/api/docs/content/specs/network.yaml b/src/api/docs/content/specs/network.yaml index 1c497a65..079e8238 100644 --- a/src/api/docs/content/specs/network.yaml +++ b/src/api/docs/content/specs/network.yaml @@ -10,6 +10,10 @@ components: operationId: "get_gateway" description: | This API hook returns infos about the gateway of your Pi-hole. + + If the optional parameter `detailed` is set to `true`, the response will include detailed information about the individual interfaces and routes. Note that the available information is dependent on the interface type and state. + parameters: + - $ref: 'network.yaml#/components/parameters/devices/detailed' responses: '200': description: OK @@ -27,14 +31,47 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + routes: + get: + summary: Get info about the routes of your Pi-hole + tags: + - "Network information" + operationId: "get_routes" + parameters: + - $ref: 'network.yaml#/components/parameters/devices/detailed' + description: | + This API hook returns infos about the networking routes of your Pi-hole. Note that not all described fields are applicable to any routing type. Users must not rely on the presence of any field without checking the route type first. + + If the optional parameter `detailed` is set to `true`, the response will include more detailed information about the individual routes where the available information is dependent on the route type and state. + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: 'network.yaml#/components/schemas/routes' + - $ref: 'common.yaml#/components/schemas/took' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' interfaces: get: summary: Get info about the interfaces of your Pi-hole tags: - "Network information" operationId: "get_interfaces" + parameters: + - $ref: 'network.yaml#/components/parameters/devices/detailed' description: | - This API hook returns infos about the networking interfaces of your Pi-hole. + This API hook returns infos about the networking interfaces of your Pi-hole. Note that not all described fields are applicable to any routing type. Users must not rely on the presence of any field without checking the route type first. + + If the optional parameter `detailed` is set to `true`, the response will include more detailed information about the individual interfaces where the available information is dependent on the interface type and state. responses: '200': description: OK @@ -119,14 +156,144 @@ components: gateway: type: object properties: - address: - type: string - description: Address of the gateway - example: "192.168.0.1" - interface: - type: string - description: Interface of your Pi-hole connected to the gateway - example: "eth0" + gateway: + type: array + items: + type: object + properties: + family: + type: string + description: Address family + interface: + type: string + description: Interface name + address: + type: string + description: Gateway address + local: + type: array + description: Local interface addresses + items: + type: string + example: + - family: "inet" + interface: "eth0" + address: "192.168.0.1" + local: + - "192.168.0.22" + - family: "inet6" + interface: "eth0" + address: "fe80::3587:2fff:f11a:1" + local: + - "fe80::3587:2fff:f11a:4321" + routes: + type: object + properties: + routes: + type: array + description: Array of routes + items: + type: object + properties: + gateway: + type: string + description: Gateway address + family: + type: string + enum: [ "inet", "inet6", "link", "mpls", "bridge", "???" ] + description: Address family + table: + type: integer + description: Routing table ID (0 = unspecified, 253 = default, 254 = local, 255 = local, other = user-defined) + protocol: + type: string + description: Routing protocol + scope: + type: string + description: Routing scope + type: + type: string + description: Routing type + flags: + type: array + description: Array of route flags + items: + type: string + oif: + type: string + description: Outgoing interface + iif: + type: string + description: Incoming interface + dst: + type: string + description: Destination address (or "default" for the default route) + src: + type: string + description: Source address + prefsrc: + type: string + description: Preferred source address + priority: + type: integer + description: Route priority + pref: + type: integer + description: Route preference + + example: + - family: "inet" + table: 254 + protocol: "static" + scope: "universe" + type: "unicast" + flags: [] + gateway: "192.168.0.1" + oif: "eth0" + - family: "inet" + table: 254 + protocol: "boot" + scope: "link" + type: "unicast" + flags: [] + dst: "10.1.0.0" + oif: "wg0" + - family: "inet" + table: 255 + protocol: "kernel" + scope: "host" + type: "local" + flags: [] + dst: "127.0.0.1" + prefsrc: "127.0.0.1" + oif: "lo" + - family: "inet6" + table: 255 + protocol: "kernel" + scope: "universe" + type: "local" + flags: [] + dst: "::1" + priority: 0 + oif: "eth0" + - family: "inet6" + table: 254 + protocol: "static" + scope: "universe" + type: "unicast" + flags: [] + gateway: "fe80::3587:2fff:f11a:4321" + oif: "eth0" + - family: "inet6" + table: 255 + protocol: "kernel" + scope: "universe" + type: "multicast" + flags: [] + dst: "fd00:4711::" + priority: 5 + oif: "wg0" + interfaces: type: object properties: @@ -138,84 +305,259 @@ components: properties: name: type: string - nullable: true description: Interface name - default: - type: boolean - description: If the interface is the default gateway - carrier: - type: boolean - description: If the interface is connected speed: type: integer - description: Speed of the interface in Mbit/s (-1 if not applicable) - tx: - type: object - properties: - num: - type: number - description: Number of transmitted data since boot - unit: - type: string - description: Unit of transmitted data since boot - rx: - type: object - properties: - num: - type: number - description: Number of received data since boot - unit: - type: string - description: Unit of received data since boot - ipv4: - type: array nullable: true - description: Array of associated IPv4 addresses + description: Speed of the interface in Mbit/s (`null` if not applicable) + carrier: + type: boolean + description: Whether the interface is connected + type: + type: string + description: Type of the interface + flags: + type: array + description: Array of address flags items: type: string - ipv6: + state: + type: string + description: State of the interface + proto_down: + type: boolean + description: Whether the interface is administratively down + address: + type: string + description: Interface hardware address + broadcast: + type: string + description: Interface broadcast address + perm_address: + type: string + description: Interface permanent hardware address + stats: + type: object + properties: + rx_bytes: + type: object + description: Interface received bytes + properties: + value: + type: number + description: Number of received bytes + unit: + type: string + description: Unit of the received bytes + tx_bytes: + type: object + description: Interface transmitted bytes + properties: + value: + type: number + description: Number of transmitted bytes + unit: + type: string + description: Unit of the transmitted bytes + addresses: type: array nullable: true - description: Array of associated IPv6 addresses + description: Array of associated IPv addresses items: - type: string + type: object + properties: + address: + type: string + description: Interface address + address_type: + type: string + description: Type of the interface address + broadcast: + type: string + description: Interface broadcast address + broadcast_type: + type: string + description: Type of the broadcast address + local: + type: string + description: Local address + local_type: + type: string + description: Type of the local address + label: + type: string + description: Interface label + family: + type: string + enum: [ "inet", "inet6", "link", "mpls", "bridge", "???" ] + description: Address family + flags: + type: array + description: Array of address flags + items: + type: string + prefixlen: + type: integer + description: Prefix length of the interface address + scope: + type: string + description: Address scope + prefered: + type: integer + description: Preferred lifetime of the address (`4294967295` = forever) + valid: + type: integer + description: Valid lifetime of the address (`4294967295` = forever) + cstamp: + type: number + description: Creation timestamp of the address + tstamp: + type: number + description: Updated timestamp of the address example: + - name: "lo" + speed: null + type: "loopback" + flags: [ "up", "loopback", "running", "lower_up" ] + state: "unknown" + carrier: true + address: "00:00:00:00:00:00" + broadcast: "00:00:00:00:00:00" + stats: + rx_bytes: + value: 81.6571641 + unit: "MB" + tx_bytes: + value: 648.818 + unit: "MB" + addresses: + - address: "127.0.0.1" + address_type: "loopback" + local: "127.0.0.1" + local_type: "loopback" + family: "inet" + scope: "host" + flags: [ "permanent" ] + prefixlen: 8 + label: "lo" + prefered: 4294967295 + valid: 4294967295 + cstamp: 1720989931 + tstamp: 1720989931 + - address: "::1" + address_type: "loopback" + local: "::1" + local_type: "loopback" + family: "inet6" + scope: "host" + flags: [ "permanent" ] + prefixlen: 128 + label: "lo" + prefered: 4294967295 + valid: 4294967295 + cstamp: 1720989931.1 + tstamp: 1720989931.1 - name: "eth0" - default: true - carrier: true speed: 1000 - tx: - num: 10.4 - unit: "MB" - rx: - num: 8.1 - unit: "MB" - ipv4: ["192.168.0.123"] - ipv6: ["fe80::1234:5678:9abc:def0", "2001:db8::1234:5678:9abc:def0"] - - name: "wlan0" - default: false - carrier: false - speed: -1 - tx: - num: 0 - unit: "B" - rx: - num: 0 - unit: "B" - ipv4: [] - ipv6: [] - - name: "wg0" - default: false + type: "ether" + flags: [ "up", "broadcast", "running", "multicast", "lower_up" ] + state: "up" carrier: true - speed: -1 - tx: - num: 170.3 - unit: "kB" - rx: - num: 222.3 - unit: "kB" - ipv4: ["10.1.0.1"] - ipv6: ["fd00:4711::1"] + address: "00:11:22:33:44:55" + broadcast: "ff:ff:ff:ff:ff:ff" + perm_address: "00:11:22:33:44:55" + stats: + rx_bytes: + value: 15.5585 + unit: "GB" + tx_bytes: + value: 1.55858 + unit: "GB" + addresses: + - address: "192.168.0.123" + address_type: "private" + local: "192.168.0.123" + local_type: "private" + family: "inet" + scope: "universe" + flags: [ "permanent" ] + prefixlen: 24 + label: "eth0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 1720989931.1 + tstamp: 1720989931.1 + - address: "2001:db8::1234:5678:9abc:def0" + address_type: "global (GUA)" + family: "inet6" + scope: "universe" + flags: [] + prefixlen: 64 + label: "eth0" + prefered: 3461 + valid: 7061 + cstamp: 2789057.25 + tstamp: 2789057.25 + - address: "fd29:db8::1234:5678:9abc:def0" + address_type: "site-local (ULA)" + family: "inet6" + scope: "universe" + flags: [] + prefixlen: 64 + label: "eth0" + prefered: 3461 + valid: 7061 + cstamp: 1720989931.1 + tstamp: 1720989931.1 + - address: "fe80::1234:5678:9abc:def0" + address_type: "link-local (LL)" + family: "inet6" + scope: "link" + flags: [ "permanent" ] + prefixlen: 64 + label: "eth0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 1720989931.1 + tstamp: 1720989931.1 + - name: "wg0" + speed: null + type: "none" + flags: [ "up", "pointopoint", "running", "noarp", "lower_up" ] + state: "unknown" + carrier: true + stats: + rx_bytes: + value: 458.44598 + unit: "MB" + tx_bytes: + value: 5.5895 + unit: "MB" + addresses: + - address: "10.1.0.1" + address_type: "private" + local: "10.1.0.1" + local_type: "private" + family: "inet" + scope: "universe" + flags: [ "permanent" ] + prefixlen: 24 + label: "wg0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 1720989931.1 + tstamp: 1720989931.1 + - address: "fd00:4711::1" + address_type: "site-local (ULA)" + family: "inet6" + scope: "global" + flags: [ "permanent" ] + prefixlen: 64 + label: "wg0" + prefered: 4294967295 + valid: 4294967295 + cstamp: 1720989931.1 + tstamp: 1720989931.1 devices: type: object properties: @@ -304,3 +646,11 @@ components: type: integer required: true example: 1 + detailed: + in: query + description: (Optional) Detailed interface/routing information + name: detailed + schema: + type: boolean + required: false + example: false diff --git a/src/api/docs/content/specs/padd.yaml b/src/api/docs/content/specs/padd.yaml new file mode 100644 index 00000000..ea7ead7b --- /dev/null +++ b/src/api/docs/content/specs/padd.yaml @@ -0,0 +1,248 @@ +openapi: 3.0.2 +components: + paths: + padd: + get: + summary: Get summarized data for PADD + tags: + - "PADD" + operationId: "get_padd" + parameters: + - in: query + description: (Optional) Return full data + name: full + schema: + type: boolean + required: false + example: true + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: 'padd.yaml#/components/schemas/padd' + - $ref: 'info.yaml#/components/schemas/system' + - $ref: 'info.yaml#/components/schemas/version' + - $ref: 'common.yaml#/components/schemas/took' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' + + schemas: + padd: + type: object + properties: + recent_blocked: + type: string + description: "Most recent blocked domain" + nullable: true + example: "bad.example.com" + top_domain: + type: string + description: "Most requested domain" + nullable: true + example: "good.example.com" + top_blocked: + type: string + description: "Most blocked domain" + nullable: true + example: "bad.example.com" + top_client: + type: string + description: "Most active client" + nullable: true + example: "localhost" + active_clients: + type: integer + description: "Number of active clients" + example: 22 + gravity_size: + type: integer + description: "Gravity list size" + example: 225382 + blocking: + type: string + description: "Blocking status" + example: "enabled" + queries: + type: object + properties: + total: + type: integer + description: "Total number of queries within the last 24 hours" + example: 92258 + blocked: + type: integer + description: "Number of blocked queries" + example: 4784 + percent_blocked: + type: number + description: "Percentage of blocked queries" + example: 5.18 + cache: + type: object + properties: + size: + type: integer + description: "Total cache size" + example: 10000 + inserted: + type: integer + description: "Number of inserted cache entries" + example: 233 + evicted: + type: integer + description: "Number of evicted cache entries" + example: 0 + iface: + type: object + description: "Default interfaces" + properties: + v4: + type: object + description: "IPv4 interface" + properties: + addr: + type: string + description: "Primary address" + nullable: true # there may be no IPv4 address + example: "192.168.2.11" + rx_bytes: + type: object + description: "Received bytes" + properties: + value: + type: number + example: 76.46 + unit: + type: string + example: "G" + tx_bytes: + type: object + description: "Transmitted bytes" + properties: + value: + type: number + example: 68.58 + unit: + type: string + example: "G" + num_addrs: + type: integer + description: "Number of addresses on the interface" + example: 1 + name: + type: string + description: "Interface name" + example: "eth0" + gw_addr: + type: string + description: "Gateway address" + nullable: true # there may be no IPv4 gateway + example: "192.168.2.1" + v6: + type: object + description: "IPv6 interface" + properties: + addr: + type: string + description: "Primary address" + nullable: true # there may be no IPv6 address + example: "fe80::b0e4:1b1e:7b7d:5855" + num_addrs: + type: integer + description: "Number of addresses on the interface" + example: 3 + name: + type: string + description: "Interface name" + example: "eth0" + gw_addr: + type: string + description: "Gateway address" + nullable: true # there may be no IPv6 gateway + example: "fe80::b0e4:1b1e:7b7d:1b1e" + node_name: + type: string + description: "Pi-hole host's name" + example: "pihole" + host_model: + type: string + description: "Host model" + example: "Raspberry Pi 3 Model B Plus Rev 1.3" + nullable: true + config: + type: object + description: "Pi-hole configuration (excerpt)" + properties: + dhcp_active: + type: boolean + description: "DHCP server status" + example: true + dhcp_start: + type: string + description: "DHCP start address" + example: "192.168.0.1" + dhcp_end: + type: string + description: "DHCP end address" + example: "192.168.0.254" + dhcp_ipv6: + type: boolean + description: "DHCPv6 server status" + example: false + dns_domain: + type: string + description: "DNS domain" + example: "lan" + dns_port: + type: integer + description: "DNS port" + example: 53 + dns_num_upstreams: + type: integer + description: "Number of upstream DNS servers" + example: 1 + dns_dnssec: + type: boolean + description: "DNSSEC status" + example: true + dns_revServer_active: + type: boolean + description: "Reverse DNS server status" + example: false + "%cpu": + type: number + description: "CPU usage" + example: 0.0 + "%mem": + type: number + description: "Memory usage" + example: 1.5 + pid: + type: integer + description: "FTL's process ID" + example: 1639 + sensors: + type: object + properties: + cpu_temp: + type: number + description: "CPU temperature" + nullable: true + example: 45.0 + hot_limit: + type: number + description: "Temperature limit" + example: 80.0 + unit: + type: string + description: "Temperature unit" + example: "C" diff --git a/src/api/docs/content/specs/search.yaml b/src/api/docs/content/specs/search.yaml index b5ad6fc9..94a068be 100644 --- a/src/api/docs/content/specs/search.yaml +++ b/src/api/docs/content/specs/search.yaml @@ -18,7 +18,6 @@ components: The optional parameters `N` and `partial` limit the maximum number of returned records and whether partial matches should be returned, respectively. There is a hard upper limit of `N` defined in FTL (currently set to 10,000) to ensure that the response is not too large. ABP matches are not returned when partial matching is requested. - Depending on the value of the config option webserver.api.searchAPIauth, local clients may not need to authenticate for this endpoint. International domains names (IDNs) are internally converted to punycode before matching. responses: '200': diff --git a/src/api/docs/content/specs/stats.yaml b/src/api/docs/content/specs/stats.yaml index b2dfba01..919ef7af 100644 --- a/src/api/docs/content/specs/stats.yaml +++ b/src/api/docs/content/specs/stats.yaml @@ -338,6 +338,10 @@ components: type: integer description: Number of queries replied to from cache or local configuration example: 9765 + frequency: + type: number + description: Average number of queries per second + example: 1.1 types: type: object description: Number of individual queries @@ -482,6 +486,10 @@ components: type: integer description: Type CACHE_STALE queries example: 0 + EXTERNAL_BLOCKED_EDE15: + type: integer + description: Type EXTERNAL_BLOCKED_EDE15 queries + example: 0 replies: type: object description: Number of individual replies @@ -560,6 +568,10 @@ components: type: integer description: Number of domain on your Pi-hole's gravity list example: 104756 + last_update: + type: integer + description: Unix timestamp of last gravity update (may be `0` if unknown) + example: 1725194639 upstreams: type: object properties: diff --git a/src/api/docs/content/specs/teleporter.yaml b/src/api/docs/content/specs/teleporter.yaml index 90132e2d..cc25e4ca 100644 --- a/src/api/docs/content/specs/teleporter.yaml +++ b/src/api/docs/content/specs/teleporter.yaml @@ -42,6 +42,50 @@ components: file: type: string format: binary + import: + type: object + nullable: true + properties: + config: + type: boolean + description: "Import Pi-hole configuration" + example: true + dhcp_leases: + type: boolean + description: "Import Pi-hole DHCP leases" + example: true + gravity: + type: object + properties: + group: + type: boolean + description: "Import Pi-hole's groups table" + example: true + adlist: + type: boolean + description: "Import Pi-hole's adlist table" + example: true + adlist_by_group: + type: boolean + description: "Import Pi-hole's table relating adlist entries to groups" + example: true + domainlist: + type: boolean + description: "Import Pi-hole's domainlist table" + example: true + domainlist_by_group: + type: boolean + description: "Import Pi-hole's table relating domainlist entries to groups" + example: true + client: + type: boolean + description: "Import Pi-hole's client table" + example: true + client_by_group: + type: boolean + description: "Import Pi-hole's table relating client entries to groups" + example: true + description: "A JSON object of files to import. If omitted, all files will be imported." responses: '200': description: OK @@ -106,4 +150,6 @@ components: value: processed: - etc/pihole/pihole.toml - - etc/pihole/gravity.db + - etc/pihole/gravity.db->group + - etc/pihole/gravity.db->adlist + - etc/pihole/gravity.db->adlist_by_group diff --git a/src/api/docs/docs.h b/src/api/docs/docs.h index 2884c88d..66c3ad70 100644 --- a/src/api/docs/docs.h +++ b/src/api/docs/docs.h @@ -132,6 +132,10 @@ static const unsigned char specs_action_yaml[] = { #include "hex/specs/action.yaml" }; +static const unsigned char specs_padd_yaml[] = { +#include "hex/specs/padd.yaml" +}; + struct { const char *path; const char *mime_type; @@ -168,6 +172,7 @@ struct { {"specs/stats.yaml", "text/plain", (const char*)specs_stats_yaml, sizeof(specs_stats_yaml)}, {"specs/teleporter.yaml", "text/plain", (const char*)specs_teleporter_yaml, sizeof(specs_teleporter_yaml)}, {"specs/action.yaml", "text/plain", (const char*)specs_action_yaml, sizeof(specs_action_yaml)}, + {"specs/padd.yaml", "text/plain", (const char*)specs_padd_yaml, sizeof(specs_padd_yaml)}, }; #endif // API_DOCS_H diff --git a/src/api/history.c b/src/api/history.c index 75a67544..dda62eb2 100644 --- a/src/api/history.c +++ b/src/api/history.c @@ -18,8 +18,6 @@ #include "overTime.h" // config struct #include "config/config.h" -// read_setupVarsconf() -#include "config/setupVars.h" // get_aliasclient_list() #include "database/aliasclients.h" diff --git a/src/api/info.c b/src/api/info.c index 581fc0b7..6d5acdf7 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -12,7 +12,7 @@ #include "webserver/http-common.h" #include "webserver/json_macros.h" #include "api/api.h" -// sysinfo() +// sysinfo(), get_nprocs_conf() #include // get_blockingstatus() #include "config/setupVars.h" @@ -157,9 +157,13 @@ int api_info_database(struct ftl_conn *api) JSON_SEND_OBJECT(json); } -static int get_system_obj(struct ftl_conn *api, cJSON *system) +int get_system_obj(struct ftl_conn *api, cJSON *system) { - const int nprocs = get_nprocs(); + // Use total number of processors + // This difference is important for virtualized systems where the number + // of available (= online) processors can be lower than the total number + // (= configured) of processors + const int nprocs = get_nprocs_conf(); struct sysinfo info; if(sysinfo(&info) != 0) return send_json_error(api, 500, "error", strerror(errno), NULL); @@ -465,7 +469,7 @@ static int get_hwmon_sensors(struct ftl_conn *api, cJSON *sensors) return 0; } -static cJSON *read_sys_property(const char *path) +cJSON *read_sys_property(const char *path) { if(!file_exists(path)) return cJSON_CreateNull(); @@ -540,10 +544,13 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl) const int db_groups = counters->database.groups; const int db_lists = counters->database.lists; const int db_clients = counters->database.clients; - const int db_allowed = counters->database.domains.allowed; - const int db_denied = counters->database.domains.denied; + const int db_allowed_exact = counters->database.domains.allowed.exact; + const int db_denied_exact = counters->database.domains.denied.exact; + const int db_allowed_regex = counters->database.domains.allowed.regex; + const int db_denied_regex = counters->database.domains.denied.regex; const int clients_total = counters->clients; const int privacylevel = config.misc.privacylevel.v.privacy_level; + const double qps = get_qps(); // unique_clients: count only clients that have been active within the most recent 24 hours int activeclients = 0; @@ -565,12 +572,18 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl) JSON_ADD_NUMBER_TO_OBJECT(database, "clients", db_clients); cJSON *domains = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(domains, "allowed", db_allowed); - JSON_ADD_NUMBER_TO_OBJECT(domains, "denied", db_denied); + JSON_ADD_NUMBER_TO_OBJECT(domains, "allowed", db_allowed_exact); + JSON_ADD_NUMBER_TO_OBJECT(domains, "denied", db_denied_exact); JSON_ADD_ITEM_TO_OBJECT(database, "domains", domains); + + cJSON *regex = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(regex, "allowed", db_allowed_regex); + JSON_ADD_NUMBER_TO_OBJECT(regex, "denied", db_denied_regex); + JSON_ADD_ITEM_TO_OBJECT(database, "regex", regex); JSON_ADD_ITEM_TO_OBJECT(ftl, "database", database); JSON_ADD_NUMBER_TO_OBJECT(ftl, "privacy_level", privacylevel); + JSON_ADD_NUMBER_TO_OBJECT(ftl, "query_frequency", qps); cJSON *clients = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(clients, "total",clients_total); @@ -641,16 +654,15 @@ int api_info_host(struct ftl_conn *api) JSON_SEND_OBJECT(json); } -int api_info_sensors(struct ftl_conn *api) +int get_sensors_obj(struct ftl_conn *api, cJSON *sensors, const bool add_list) { - cJSON *sensors = JSON_NEW_OBJECT(); - // Get sensors array cJSON *list = JSON_NEW_ARRAY(); int ret = get_hwmon_sensors(api, list); if (ret != 0) return ret; - JSON_ADD_ITEM_TO_OBJECT(sensors, "list", list); + if(add_list) + JSON_ADD_ITEM_TO_OBJECT(sensors, "list", list); // Loop over available sensors and try to identify the most suitable CPU temperature sensor int cpu_temp_sensor = -1; @@ -708,12 +720,25 @@ int api_info_sensors(struct ftl_conn *api) unit = "K"; JSON_REF_STR_IN_OBJECT(sensors, "unit", unit); + if(!add_list) + cJSON_Delete(list); + + return 0; +} + +int api_info_sensors(struct ftl_conn *api) +{ + cJSON *sensors = JSON_NEW_OBJECT(); + int ret = get_sensors_obj(api, sensors, true); + if (ret != 0) + return ret; + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "sensors", sensors); JSON_SEND_OBJECT(json); } -int api_info_version(struct ftl_conn *api) +int get_version_obj(struct ftl_conn *api, cJSON *version) { char *line = NULL; size_t len = 0; @@ -802,8 +827,6 @@ int api_info_version(struct ftl_conn *api) JSON_REF_STR_IN_OBJECT(ftl_local, "version", get_FTL_version()); JSON_REF_STR_IN_OBJECT(ftl_local, "date", GIT_DATE); - cJSON *version = JSON_NEW_OBJECT(); - cJSON *core = JSON_NEW_OBJECT(); JSON_ADD_NULL_IF_NOT_EXISTS(core_local, "branch"); JSON_ADD_NULL_IF_NOT_EXISTS(core_local, "version"); @@ -839,7 +862,14 @@ int api_info_version(struct ftl_conn *api) JSON_ADD_NULL_IF_NOT_EXISTS(docker, "remote"); JSON_ADD_ITEM_TO_OBJECT(version, "docker", docker); + return 0; +} + +int api_info_version(struct ftl_conn *api) +{ // Send reply + cJSON *version = JSON_NEW_OBJECT(); + get_version_obj(api, version); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "version", version); JSON_SEND_OBJECT(json); diff --git a/src/api/list.c b/src/api/list.c index 56d4e009..5a4563d6 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -19,6 +19,8 @@ #include "database/network-table.h" // valid_domain() #include "tools/gravity-parseList.h" +// parse_groupIDs() +#include "webserver/http-common.h" #include static int api_list_read(struct ftl_conn *api, @@ -96,19 +98,13 @@ static int api_list_read(struct ftl_conn *api, { if(table.group_ids != NULL) { - // Black magic at work here: We build a JSON array from - // the group_concat result delivered from the database, - // parse it as valid array and append it as row to the - // data - const size_t buflen = strlen(table.group_ids)+3u; - char *group_ids_str = calloc(buflen, sizeof(char)); - group_ids_str[0] = '['; - strcpy(group_ids_str+1u , table.group_ids); - group_ids_str[buflen-2u] = ']'; - group_ids_str[buflen-1u] = '\0'; - cJSON * group_ids = cJSON_Parse(group_ids_str); - free(group_ids_str); - JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids); + const int ret = parse_groupIDs(api, &table, row); + if(ret != 0) + { + JSON_DELETE(rows); + return ret; + } + } else { @@ -184,19 +180,9 @@ static int api_list_write(struct ftl_conn *api, tablerow row = { 0 }; // Check if valid JSON payload is available - if (api->payload.json == NULL) - { - if (api->payload.json_error == NULL) - return send_json_error(api, 400, - "bad_request", - "No request body data", - NULL); - else - return send_json_error(api, 400, - "bad_request", - "Invalid request body data (no valid JSON), error before hint", - api->payload.json_error); - } + const int json_ret = check_json_payload(api); + if(json_ret != 0) + return json_ret; bool spaces_allowed = false; bool allocated_json = false; @@ -475,7 +461,7 @@ static int api_list_write(struct ftl_conn *api, return send_json_error_free(api, 400, // 400 Bad Request "regex_error", "Regex validation failed", - regex_msg, true); + regex_msg, true, true); } // Try to add item(s) to table diff --git a/src/api/network.c b/src/api/network.c index 532a985e..67d80c57 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -24,292 +24,141 @@ #include "database/query-table.h" // config struct #include "config/config.h" +// PRIx64 +#include +#include +// IFA_LINK and friends +#include +// nlroutes(), nladdrs(), nllinks() +#include "tools/netlink.h" -static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) +int get_gateway(struct ftl_conn *api, cJSON * json, const bool detailed) { - // Get IPv4 default route gateway and associated interface - unsigned long dest_r = 0, gw_r = 0; - unsigned int flags = 0u; - int metric = 0, minmetric = __INT_MAX__; - FILE *file; - if((file = fopen("/proc/net/route", "r"))) + // Get routing information + cJSON *routes = JSON_NEW_ARRAY(); + nlroutes(routes, detailed); + + // Get interface information ... + cJSON *interfaces = JSON_NEW_ARRAY(); + nllinks(interfaces, detailed); + // ... and enrich them with addresses + nladdrs(interfaces, detailed); + + cJSON *gateway = JSON_NEW_ARRAY(); + // Search through routes for the default gateway + // They are the ones with "dst" == "default" + cJSON *route = NULL; + cJSON_ArrayForEach(route, routes) { - // Parse /proc/net/route - the kernel's IPv4 routing table - char buf[1024] = { 0 }; - while(fgets(buf, sizeof(buf), file)) + cJSON *dst = cJSON_GetObjectItem(route, "dst"); + if(dst != NULL && + cJSON_IsString(dst) && + strcmp(cJSON_GetStringValue(dst), "default") == 0) { - char iface_r[IF_NAMESIZE] = { 0 }; - if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) - continue; + cJSON *gwobj = JSON_NEW_OBJECT(); - // Only analyze routes which are UP and whose - // destinations are a gateway - if(!(flags & RTF_UP) || !(flags & RTF_GATEWAY)) - continue; + // Extract and add family + const char *family = cJSON_GetStringValue(cJSON_GetObjectItem(route, "family")); + JSON_REF_STR_IN_OBJECT(gwobj, "family", family); - // Only analyze "catch all" routes (destination 0.0.0.0) - if(dest_r != 0) - continue; + // Extract and add interface name + const char *iface_name = cJSON_GetStringValue(cJSON_GetObjectItem(route, "oif")); + JSON_COPY_STR_TO_OBJECT(gwobj, "interface", iface_name); - // Store default gateway, overwrite if we find a route with - // a lower metric - if(metric < minmetric) + // Extract and add gateway address + const char *gw_addr = cJSON_GetStringValue(cJSON_GetObjectItem(route, "gateway")); + JSON_COPY_STR_TO_OBJECT(gwobj, "address", gw_addr); + + // Extract and add local interface address + cJSON *local = JSON_NEW_ARRAY(); + cJSON *iface = NULL; + cJSON_ArrayForEach(iface, interfaces) { - minmetric = metric; - *gw = gw_r; - strcpy(iface, iface_r); + const char *ifname = cJSON_GetStringValue(cJSON_GetObjectItem(iface, "name")); + if(ifname != NULL && strcmp(ifname, iface_name) == 0) + { + cJSON *addr = NULL; + cJSON *addrs = cJSON_GetObjectItem(iface, "addresses"); + cJSON_ArrayForEach(addr, addrs) + { + // Skip addresses belonging to another address family + const char *ifamily = cJSON_GetStringValue(cJSON_GetObjectItem(addr, "family")); + if(ifamily == NULL || strcmp(ifamily, family) != 0) + continue; - log_debug(DEBUG_API, "Reading interfaces: flags: %u, addr: %s, iface: %s, metric: %i, minmetric: %i", - flags, inet_ntoa(*(struct in_addr *) gw), iface, metric, minmetric); + const char *addr_str = cJSON_GetStringValue(cJSON_GetObjectItem(addr, "address")); + if(addr_str != NULL) + JSON_COPY_STR_TO_ARRAY(local, addr_str); + } + break; + } } + + // Add local addresses array to gateway object + JSON_ADD_ITEM_TO_OBJECT(gwobj, "local", local); + + cJSON_AddItemToArray(gateway, gwobj); } - fclose(file); + } + + // Send gateway information + JSON_ADD_ITEM_TO_OBJECT(json, "gateway", gateway); + + if(detailed) + { + JSON_ADD_ITEM_TO_OBJECT(json, "routes", routes); + JSON_ADD_ITEM_TO_OBJECT(json, "interfaces", interfaces); } else - log_err("Cannot read /proc/net/route: %s", strerror(errno)); + { + // Free arrays + cJSON_Delete(routes); + cJSON_Delete(interfaces); + } - // Return success based on having found the default gateway's address - return gw != 0; + return 0; } int api_network_gateway(struct ftl_conn *api) { - in_addr_t gw = 0; - char iface[IF_NAMESIZE] = { 0 }; + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); - // Get default interface - getDefaultInterface(iface, &gw); - - // Generate JSON response cJSON *json = JSON_NEW_OBJECT(); - const char *gwaddr = inet_ntoa(*(struct in_addr *) &gw); - JSON_COPY_STR_TO_OBJECT(json, "address", gwaddr); - JSON_REF_STR_IN_OBJECT(json, "interface", iface); + get_gateway(api, json, detailed); + + JSON_SEND_OBJECT(json); +} + +int api_network_routes(struct ftl_conn *api) +{ + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); + + // Add routing information + cJSON *routes = JSON_NEW_ARRAY(); + nlroutes(routes, detailed); + cJSON *json = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(json, "routes", routes); JSON_SEND_OBJECT(json); } int api_network_interfaces(struct ftl_conn *api) { - cJSON *json = JSON_NEW_OBJECT(); - - // Get interface with default route - in_addr_t gw = 0; - char default_iface[IF_NAMESIZE] = { 0 }; - getDefaultInterface(default_iface, &gw); - - // Enumerate and list interfaces - // Loop over interfaces and extract information - DIR *dfd; - FILE *f; - struct dirent *dp; - size_t tx_sum = 0, rx_sum = 0; - char fname[64 + IF_NAMESIZE] = { 0 }; - char readbuffer[1024] = { 0 }; - - // Open /sys/class/net directory - if ((dfd = opendir("/sys/class/net")) == NULL) - { - log_err("API: Cannot access /sys/class/net"); - return 500; - } - - // Get IP addresses of all interfaces on this machine - struct ifaddrs *ifap = NULL; - if(getifaddrs(&ifap) == -1) - log_err("API: Cannot get interface addresses: %s", strerror(errno)); + // Get ?detailed parameter + bool detailed = false; + get_bool_var(api->request->query_string, "detailed", &detailed); cJSON *interfaces = JSON_NEW_ARRAY(); - // Walk /sys/class/net directory - while ((dp = readdir(dfd)) != NULL) - { - // Skip "." and ".." - if(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) - continue; + // Get links ... + nllinks(interfaces, detailed); + // ... and enrich them with addresses + nladdrs(interfaces, detailed); - // Create new interface record - cJSON *iface = JSON_NEW_OBJECT(); - - // Extract interface name - const char *iface_name = dp->d_name; - JSON_COPY_STR_TO_OBJECT(iface, "name", iface_name); - - // Is this the default interface? - const bool is_default_iface = strcmp(iface_name, default_iface) == 0; - JSON_ADD_BOOL_TO_OBJECT(iface, "default", is_default_iface); - - // Extract carrier status - bool carrier = false; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/carrier", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fgets(readbuffer, sizeof(readbuffer)-1, f) != NULL) - carrier = readbuffer[0] == '1'; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - JSON_ADD_BOOL_TO_OBJECT(iface, "carrier", carrier); - - // Extract link speed (may not be possible, e.g., for WiFi devices with dynamic link speeds) - int speed = -1; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/speed", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fscanf(f, "%i", &(speed)) != 1) - speed = -1; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - JSON_ADD_NUMBER_TO_OBJECT(iface, "speed", speed); - - // Get total transmitted bytes - ssize_t tx_bytes = -1; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/statistics/tx_bytes", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fscanf(f, "%zi", &(tx_bytes)) != 1) - tx_bytes = -1; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - - // Format transmitted bytes - double tx = 0.0; - char tx_unit[3] = { 0 }; - format_memory_size(tx_unit, tx_bytes, &tx); - if(tx_unit[0] != '\0') - tx_unit[1] = 'B'; - - // Add transmitted bytes to interface record - cJSON *tx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(tx_json, "num", tx); - JSON_COPY_STR_TO_OBJECT(tx_json, "unit", tx_unit); - JSON_ADD_ITEM_TO_OBJECT(iface, "tx", tx_json); - - // Get total received bytes - ssize_t rx_bytes = -1; - snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/statistics/rx_bytes", iface_name); - if((f = fopen(fname, "r")) != NULL) - { - if(fscanf(f, "%zi", &(rx_bytes)) != 1) - rx_bytes = -1; - fclose(f); - } - else - log_err("Cannot read %s: %s", fname, strerror(errno)); - - // Format received bytes - double rx = 0.0; - char rx_unit[3] = { 0 }; - format_memory_size(rx_unit, rx_bytes, &rx); - if(rx_unit[0] != '\0') - rx_unit[1] = 'B'; - - // Add received bytes to JSON object - cJSON *rx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(rx_json, "num", rx); - JSON_COPY_STR_TO_OBJECT(rx_json, "unit", rx_unit); - JSON_ADD_ITEM_TO_OBJECT(iface, "rx", rx_json); - - // Get IP address(es) of this interface - if(ifap) - { - // Walk through linked list of interface addresses - cJSON *ipv4 = JSON_NEW_ARRAY(); - cJSON *ipv6 = JSON_NEW_ARRAY(); - for(struct ifaddrs *ifa = ifap; ifa != NULL; ifa = ifa->ifa_next) - { - // Skip interfaces without an address and those - // not matching the current interface - if(ifa->ifa_addr == NULL || strcmp(ifa->ifa_name, iface_name) != 0) - continue; - - // If we reach this point, we found the correct interface - const sa_family_t family = ifa->ifa_addr->sa_family; - char host[NI_MAXHOST] = { 0 }; - if(family == AF_INET || family == AF_INET6) - { - // Get IP address - const int s = getnameinfo(ifa->ifa_addr, - (family == AF_INET) ? - sizeof(struct sockaddr_in) : - sizeof(struct sockaddr_in6), - host, NI_MAXHOST, - NULL, 0, NI_NUMERICHOST); - if (s != 0) - { - log_warn("API: getnameinfo() failed: %s\n", gai_strerror(s)); - continue; - } - - if(family == AF_INET) - { - JSON_COPY_STR_TO_ARRAY(ipv4, host); - } - else if(family == AF_INET6) - { - JSON_COPY_STR_TO_ARRAY(ipv6, host); - } - } - } - JSON_ADD_ITEM_TO_OBJECT(iface, "ipv4", ipv4); - JSON_ADD_ITEM_TO_OBJECT(iface, "ipv6", ipv6); - } - - // Sum up transmitted and received bytes - if(tx_bytes > 0) - tx_sum += tx_bytes; - if(rx_bytes > 0) - rx_sum += rx_bytes; - - // Add interface to array - JSON_ADD_ITEM_TO_ARRAY(interfaces, iface); - } - - freeifaddrs(ifap); - closedir(dfd); - - cJSON *sum = JSON_NEW_OBJECT(); - JSON_COPY_STR_TO_OBJECT(sum, "name", "sum"); - JSON_ADD_BOOL_TO_OBJECT(sum, "carrier", true); - JSON_ADD_NUMBER_TO_OBJECT(sum, "speed", 0); - - // Format transmitted bytes - double tx = 0.0; - char tx_unit[3] = { 0 }; - format_memory_size(tx_unit, tx_sum, &tx); - if(tx_unit[0] != '\0') - tx_unit[1] = 'B'; - - // Add transmitted bytes to interface record - cJSON *tx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(tx_json, "num", tx); - JSON_COPY_STR_TO_OBJECT(tx_json, "unit", tx_unit); - JSON_ADD_ITEM_TO_OBJECT(sum, "tx", tx_json); - - // Format received bytes - double rx = 0.0; - char rx_unit[3] = { 0 }; - format_memory_size(rx_unit, rx_sum, &rx); - if(rx_unit[0] != '\0') - rx_unit[1] = 'B'; - - // Add received bytes to JSON object - cJSON *rx_json = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(rx_json, "num", rx); - JSON_COPY_STR_TO_OBJECT(rx_json, "unit", rx_unit); - JSON_ADD_ITEM_TO_OBJECT(sum, "rx", rx_json); - - cJSON *ipv4 = JSON_NEW_ARRAY(); - cJSON *ipv6 = JSON_NEW_ARRAY(); - JSON_ADD_ITEM_TO_OBJECT(sum, "ipv4", ipv4); - JSON_ADD_ITEM_TO_OBJECT(sum, "ipv6", ipv6); - - // Add interface to array - JSON_ADD_ITEM_TO_ARRAY(interfaces, sum); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "interfaces", interfaces); JSON_SEND_OBJECT(json); } diff --git a/src/api/padd.c b/src/api/padd.c new file mode 100644 index 00000000..21a44cfa --- /dev/null +++ b/src/api/padd.c @@ -0,0 +1,308 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2019 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* API Implementation /api/dns +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "webserver/http-common.h" +#include "webserver/json_macros.h" +#include "api.h" +// lock_shm() and unlock_shm() +#include "shmem.h" +// counters +#include "datastructure.h" +// get_dnsmasq_metrics(&metrics) +#include "metrics.h" +// get_blockingstatus() +#include "config/config.h" +// uname() +#include +// nlroutes(), nladdrs(), nllinks() +#include "tools/netlink.h" +// struct proc_mem, getProcessMemory() +#include "procps.h" +// getcpu_percentage() +#include "daemon.h" + +int api_padd(struct ftl_conn *api) +{ + // Parse parameters + bool full = true; + if(api->request->query_string != NULL) + get_bool_var(api->request->query_string, "full", &full); + + cJSON *json = JSON_NEW_OBJECT(); + // Lock shared memory + lock_shm(); + + const int total = counters->queries; + const int blocked = get_blocked_count(); + const unsigned int active_clients = get_active_clients(); + const int num_gravity = counters->database.gravity; + + // If privacy level is set to hide domains, do not return the most + // recent blocked domain + if(config.misc.privacylevel.v.privacy_level < PRIVACY_HIDE_DOMAINS) + { + // Find most recently blocked query + for(int queryID = counters->queries - 1; queryID > 0 ; queryID--) + { + const queriesData *query = getQuery(queryID, true); + if(query == NULL) + continue; + + if(query->flags.blocked) + { + // Ask subroutine for domain. It may return "hidden" depending on + // the privacy settings at the time the query was made + const char *domain = getDomainString(query); + if(domain == NULL) + continue; + + JSON_COPY_STR_TO_OBJECT(json, "recent_blocked", domain); + break; + } + } + } + + // Unlock shared memory + unlock_shm(); + + // Add the number of active clients, the size of the gravity list + JSON_ADD_NUMBER_TO_OBJECT(json, "active_clients", active_clients); + JSON_ADD_NUMBER_TO_OBJECT(json, "gravity_size", num_gravity); + + cJSON *top_domains = get_top_domains(api, 1, false, true); + if(cJSON_GetArraySize(top_domains) == 0) + { + JSON_ADD_NULL_TO_OBJECT(json, "top_domain"); + } + else + { + cJSON *top_domain = cJSON_GetArrayItem(top_domains, 0); + const char *domain = cJSON_GetStringValue(top_domain); + JSON_COPY_STR_TO_OBJECT(json, "top_domain", domain); + } + cJSON_Delete(top_domains); + cJSON *top_blocked = get_top_domains(api, 1, true, true); + if(cJSON_GetArraySize(top_blocked) == 0) + { + JSON_ADD_NULL_TO_OBJECT(json, "top_blocked"); + } + else + { + cJSON *top_block = cJSON_GetArrayItem(top_blocked, 0); + const char *domain = cJSON_GetStringValue(top_block); + JSON_COPY_STR_TO_OBJECT(json, "top_blocked", domain); + } + cJSON *top_clients = get_top_clients(api, 1, false, true, false, true); + if(cJSON_GetArraySize(top_clients) == 0) + { + JSON_ADD_NULL_TO_OBJECT(json, "top_client"); + } + else + { + cJSON *top_client = cJSON_GetArrayItem(top_clients, 0); + const char *client = cJSON_GetStringValue(top_client); + JSON_COPY_STR_TO_OBJECT(json, "top_client", client); + } + + // Add a null entry if the domain is hidden or there is no recent + // blocked domain (e.g. when blocking is disabled) + JSON_ADD_NULL_IF_NOT_EXISTS(json, "recent_blocked"); + + // Calculate percentage of blocked queries + float percent_blocked = 0.0f; + // Avoid 1/0 condition + if(total > 0) + percent_blocked = 1e2f*blocked/total; + + // Add the blocking status + const char *blocking = get_blocking_status_str(get_blockingstatus()); + JSON_REF_STR_IN_OBJECT(json, "blocking", blocking); + + // Add query statistics + cJSON *queries = JSON_NEW_OBJECT(); + JSON_ADD_NUMBER_TO_OBJECT(queries, "total", total); + JSON_ADD_NUMBER_TO_OBJECT(queries, "blocked", blocked); + JSON_ADD_NUMBER_TO_OBJECT(queries, "percent_blocked", percent_blocked); + JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); + + // Add cache statistics + cJSON *cache = JSON_NEW_OBJECT(); + struct metrics metrics = { 0 }; + get_dnsmasq_metrics(&metrics); + JSON_ADD_NUMBER_TO_OBJECT(cache, "size", metrics.dns.cache.size); + JSON_ADD_NUMBER_TO_OBJECT(cache, "inserted", metrics.dns.cache.inserted); + JSON_ADD_NUMBER_TO_OBJECT(cache, "evicted", metrics.dns.cache.live_freed); + JSON_ADD_ITEM_TO_OBJECT(json, "cache", cache); + + // info/system + cJSON *system = JSON_NEW_OBJECT(); + get_system_obj(api, system); + JSON_ADD_ITEM_TO_OBJECT(json, "system", system); + + // info/host + struct utsname un = { 0 }; + uname(&un); + JSON_COPY_STR_TO_OBJECT(json, "node_name", un.nodename); + JSON_ADD_ITEM_TO_OBJECT(json, "host_model", read_sys_property("/sys/firmware/devicetree/base/model")); + + // Expensive calls, do only if full is requested + if(full) + { + // network/gateway + cJSON *gateway_ = JSON_NEW_OBJECT(); + get_gateway(api, gateway_, true); + + cJSON *gateway = cJSON_GetObjectItemCaseSensitive(gateway_, "gateway"); + cJSON *interfaces = cJSON_GetObjectItemCaseSensitive(gateway_, "interfaces"); + + // Loop over gateway and find first entry with family == "inet" + cJSON *entry = NULL; + const char *gw_v4_name = NULL, *gw_v6_name = NULL; + const char *gw_v4_addr = NULL, *gw_v6_addr = NULL; + cJSON_ArrayForEach(entry, gateway) + { + cJSON *family = cJSON_GetObjectItemCaseSensitive(entry, "family"); + if(gw_v4_name == NULL && strcmp(cJSON_GetStringValue(family), "inet") == 0) + { + gw_v4_name = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "interface")); + gw_v4_addr = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "address")); + } + if(gw_v6_name == NULL && strcmp(cJSON_GetStringValue(family), "inet6") == 0) + { + gw_v6_name = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "interface")); + gw_v6_addr = cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "address")); + } + + // Break if both addresses are found + if(gw_v4_name && gw_v6_name) + break; + } + + // If no IPv6 gateway is found, use the IPv4 gateway + if(gw_v6_name == NULL) + gw_v6_name = gw_v4_name; + + // Iterate over all interfaces until we find the one associated + // with the IPv4 gateway + cJSON *iface_v4 = JSON_NEW_OBJECT(); + cJSON *iface_v6 = JSON_NEW_OBJECT(); + unsigned int v4_addrs = 0, v6_addrs = 0; + cJSON_ArrayForEach(entry, interfaces) + { + if(strcmp(cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "name")), gw_v4_name) == 0) + { + // Add first interface address with family == inet + cJSON *addr = NULL; + cJSON *addrs = cJSON_GetObjectItemCaseSensitive(entry, "addresses"); + cJSON_ArrayForEach(addr, addrs) + { + cJSON *family = cJSON_GetObjectItemCaseSensitive(addr, "family"); + if(strcmp(cJSON_GetStringValue(family), "inet") == 0) + { + if(v4_addrs == 0) + { + cJSON *_addr = cJSON_GetObjectItemCaseSensitive(addr, "address"); + JSON_COPY_STR_TO_OBJECT(iface_v4, "addr", cJSON_GetStringValue(_addr)); + } + v4_addrs++; + } + } + + // Add NULL if no IPv4 address is found + if(v4_addrs == 0) + JSON_ADD_NULL_TO_OBJECT(iface_v4, "addr"); + + // Also add IPv4 interface statistics + cJSON *stats = cJSON_GetObjectItemCaseSensitive(entry, "stats"); + cJSON *rx_bytes = cJSON_GetObjectItemCaseSensitive(stats, "rx_bytes"); + JSON_ADD_ITEM_TO_OBJECT(iface_v4, "rx_bytes", cJSON_Duplicate(rx_bytes, true)); + cJSON *tx_bytes = cJSON_GetObjectItemCaseSensitive(stats, "tx_bytes"); + JSON_ADD_ITEM_TO_OBJECT(iface_v4, "tx_bytes", cJSON_Duplicate(tx_bytes, true)); + } + if(strcmp(cJSON_GetStringValue(cJSON_GetObjectItemCaseSensitive(entry, "name")), gw_v6_name) == 0) + { + // Add first interface address with family == inet + cJSON *addr = NULL; + cJSON *addrs = cJSON_GetObjectItemCaseSensitive(entry, "addresses"); + cJSON_ArrayForEach(addr, addrs) + { + cJSON *family = cJSON_GetObjectItemCaseSensitive(addr, "family"); + if(strcmp(cJSON_GetStringValue(family), "inet6") == 0) + { + if(v6_addrs == 0) + { + cJSON *_addr = cJSON_GetObjectItemCaseSensitive(addr, "address"); + JSON_COPY_STR_TO_OBJECT(iface_v6, "addr", cJSON_GetStringValue(_addr)); + } + v6_addrs++; + } + } + + // Add NULL if no IPv6 address is found + if(v6_addrs == 0) + JSON_ADD_NULL_TO_OBJECT(iface_v6, "addr"); + } + } + + // Add the number of addresses found + JSON_ADD_NUMBER_TO_OBJECT(iface_v4, "num_addrs", v4_addrs); + JSON_ADD_NUMBER_TO_OBJECT(iface_v6, "num_addrs", v6_addrs); + + // Add the interfaces to the gateway object + JSON_COPY_STR_TO_OBJECT(iface_v4, "name", gw_v4_name); + JSON_COPY_STR_TO_OBJECT(iface_v4, "gw_addr", gw_v4_addr); + JSON_COPY_STR_TO_OBJECT(iface_v6, "name", gw_v6_name); + JSON_COPY_STR_TO_OBJECT(iface_v6, "gw_addr", gw_v6_addr); + + // Create interface object + cJSON *iface = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(iface, "v4", iface_v4); + JSON_ADD_ITEM_TO_OBJECT(iface, "v6", iface_v6); + JSON_ADD_ITEM_TO_OBJECT(json, "iface", iface); + + // Free memory + cJSON_Delete(gateway_); + + // info/version + cJSON *version = JSON_NEW_OBJECT(); + get_version_obj(api, version); + JSON_ADD_ITEM_TO_OBJECT(json, "version", version); + } + + // subset of config + cJSON *jconfig = JSON_NEW_OBJECT(); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dhcp_active", config.dhcp.active.v.b); + JSON_ADD_ITEM_TO_OBJECT(jconfig, "dhcp_start", addJSONConfValue(config.dhcp.start.t, &config.dhcp.start.v)); + JSON_ADD_ITEM_TO_OBJECT(jconfig, "dhcp_end", addJSONConfValue(config.dhcp.end.t, &config.dhcp.end.v)); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dhcp_ipv6", config.dhcp.ipv6.v.b); + JSON_COPY_STR_TO_OBJECT(jconfig, "dns_domain", config.dns.domain.v.s); + JSON_ADD_NUMBER_TO_OBJECT(jconfig, "dns_port", config.dns.port.v.u16); + JSON_ADD_NUMBER_TO_OBJECT(jconfig, "dns_num_upstreams", cJSON_GetArraySize(config.dns.upstreams.v.json)); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dns_dnssec", config.dns.dnssec.v.b); + JSON_ADD_BOOL_TO_OBJECT(jconfig, "dns_revServer_active", cJSON_GetArraySize(config.dns.revServers.v.json) > 0); + JSON_ADD_ITEM_TO_OBJECT(json, "config", jconfig); + + // subset of info/ftl + struct proc_mem pmem = { 0 }; + struct proc_meminfo mem = { 0 }; + parse_proc_meminfo(&mem); + getProcessMemory(&pmem, mem.total); + JSON_ADD_NUMBER_TO_OBJECT(json, "%mem", pmem.VmRSS_percent); + JSON_ADD_NUMBER_TO_OBJECT(json, "%cpu", get_cpu_percentage()); + JSON_ADD_NUMBER_TO_OBJECT(json, "pid", getpid()); + + // info/sensors -> CPU temp sensor + cJSON *sensors = JSON_NEW_OBJECT(); + get_sensors_obj(api, sensors, false); + JSON_ADD_ITEM_TO_OBJECT(json, "sensors", sensors); + + JSON_SEND_OBJECT(json); +} diff --git a/src/api/queries.c b/src/api/queries.c index 82735726..20768774 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -22,7 +22,8 @@ // dbopen(false, ), dbclose() #include "database/common.h" -static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char *querystr, const int max_count) +#if 0 +static int add_strings_to_array(struct ftl_conn *api, cJSON *array1, cJSON *array2, const char *querystr, const int max_count) { sqlite3 *memdb = get_memdb(); @@ -44,11 +45,24 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char * sqlite3_errstr(rc)); } - // Loop through returned rows + // Loop through returned rows and add them to the array int counter = 0; while((rc = sqlite3_step(stmt)) == SQLITE_ROW && - (max_count < 0 || ++counter < max_count)) - JSON_COPY_STR_TO_ARRAY(array, (const char*)sqlite3_column_text(stmt, 0)); + (max_count < 0 || ++counter <= max_count)) + { + const char *array1_str = (const char*)sqlite3_column_text(stmt, 0); + if(array1_str != NULL && array1_str[0] != '\0') + // Only add non-empty strings + JSON_COPY_STR_TO_ARRAY(array1, array1_str); + if(array2 != NULL) + { + // We have a second array to fill (second column in the query) + const char *array2_str = (const char*)sqlite3_column_text(stmt, 1); + if(array2_str != NULL && array2_str[0] != '\0') + // Only add non-empty strings + JSON_COPY_STR_TO_ARRAY(array2, array2_str); + } + } // Acceptable return codes are either // - SQLITE_DONE: We read all lines, or @@ -67,60 +81,47 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char * return 0; } +#endif int api_queries_suggestions(struct ftl_conn *api) { - int rc; // Does the user request a custom number of records to be included? int count = 30; get_int_var(api->request->query_string, "count", &count); // Get domains - cJSON *domain = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, domain, "SELECT domain FROM domain_by_id", count); - if(rc != 0) + cJSON *domain = get_top_domains(api, count, false, true); + cJSON *blocked = get_top_domains(api, count, true, true); + // Add domains from both arrays, avoiding duplicates + cJSON *entry = NULL; + cJSON_ArrayForEach(entry, blocked) { - log_err("Cannot read domains from database"); - cJSON_Delete(domain); - return rc; + // Check if the domain is already in the list + bool found = false; + cJSON *entry2 = NULL; + cJSON_ArrayForEach(entry2, domain) + { + if(strcmp(cJSON_GetStringValue(entry), cJSON_GetStringValue(entry2)) == 0) + { + found = true; + break; + } + } + if(!found) + JSON_ADD_ITEM_TO_ARRAY(domain, cJSON_Duplicate(entry, true)); } + // Free the blocked list + cJSON_Delete(blocked); // Get clients, both by IP and names - // We have to call DISTINCT() here as multiple IPs can map to and name and - // vice versa - cJSON *client_ip = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, client_ip, "SELECT DISTINCT(ip) FROM client_by_id", count); - if(rc != 0) - { - log_err("Cannot read client IPs from database"); - cJSON_Delete(domain); - cJSON_Delete(client_ip); - return rc; - } - cJSON *client_name = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, client_name, "SELECT DISTINCT(name) FROM client_by_id", count); - if(rc != 0) - { - log_err("Cannot read client names from database"); - cJSON_Delete(domain); - cJSON_Delete(client_ip); - cJSON_Delete(client_name); - return rc; - } + cJSON *client_ip = get_top_clients(api, count, false, true, false, false); + cJSON *client_name = get_top_clients(api, count, false, true, true, false); + + // Delete duplicate entries from client_name + cJSON_unique_array(client_name); // Get upstreams - cJSON *upstream = JSON_NEW_ARRAY(); - rc = add_strings_to_array(api, upstream, "SELECT forward FROM forward_by_id", count); - if(rc != 0) - { - log_err("Cannot read forward from database"); - cJSON_Delete(domain); - cJSON_Delete(client_ip); - cJSON_Delete(client_name); - cJSON_Delete(upstream); - return rc; - } - + cJSON *upstream = get_top_upstreams(api, true); // Get types cJSON *type = JSON_NEW_ARRAY(); queriesData query = { 0 }; @@ -306,6 +307,11 @@ int api_queries(struct ftl_conn *api) bool cursor_set = false, where = false; double timestamp_from = 0.0, timestamp_until = 0.0; + // We use this boolean to memorize if we are filtering at all. It is used + // later to decide if we can short-circuit the query counting for + // performance reasons. + bool filtering = false; + // Filter-/sorting based on GET parameters? if(api->request->query_string != NULL) { @@ -330,11 +336,17 @@ int api_queries(struct ftl_conn *api) if(GET_STR("upstream", upstreamname, api->request->query_string) > 0) { if(strcmp(upstreamname, "blocklist") == 0) + { // Pseudo-upstream for blocked queries add_querystr_string(api, querystr, "q.status IN ", get_blocked_statuslist(), &where); + filtering = true; + } else if(strcmp(upstreamname, "cache") == 0) + { // Pseudo-upstream for cached queries add_querystr_string(api, querystr, "q.status IN ", get_cached_statuslist(), &where); + filtering = true; + } else { if(is_wildcard(upstreamname)) @@ -432,10 +444,14 @@ int api_queries(struct ftl_conn *api) // Encoded URI string: %5B = [ and %5D = ] if(GET_VAR(sort_col_id, sort_col, api->request->query_string) > 0) + { log_debug(DEBUG_API, "Sorting by column %s (%s)", sort_col, sort_dir); + } else + { log_warn("Sorting by column %d (%s) requested, but column name not found", sort_column, sort_dir); + } } // Column searching? @@ -505,11 +521,6 @@ int api_queries(struct ftl_conn *api) } } - // We use this boolean to memorize if we are filtering at all. It is used - // later to decide if we can short-circuit the query counting for - // performance reasons. - bool filtering = false; - // Regex filtering? regex_t *regex_domains = NULL; unsigned int N_regex_domains = 0; @@ -1010,6 +1021,7 @@ int api_queries(struct ftl_conn *api) case QUERY_EXTERNAL_BLOCKED_IP: case QUERY_EXTERNAL_BLOCKED_NULL: case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: case QUERY_RETRIED: case QUERY_RETRIED_DNSSEC: case QUERY_IN_PROGRESS: diff --git a/src/api/search.c b/src/api/search.c index 7358a612..09098351 100644 --- a/src/api/search.c +++ b/src/api/search.c @@ -15,6 +15,8 @@ #include "database/gravity-db.h" // match_regex() #include "regex_r.h" +// parse_groupIDs() +#include "webserver/http-common.h" #include #define MAX_SEARCH_RESULTS 10000u @@ -77,19 +79,9 @@ static int search_table(struct ftl_conn *api, const char *item, if(table.group_ids != NULL) { - // Black magic at work here: We build a JSON array from - // the group_concat result delivered from the database, - // parse it as valid array and append it as row to the - // data - const size_t buflen = strlen(table.group_ids)+3u; - char *group_ids_str = calloc(buflen, sizeof(char)); - group_ids_str[0] = '['; - strcpy(group_ids_str+1u , table.group_ids); - group_ids_str[buflen-2u] = ']'; - group_ids_str[buflen-1u] = '\0'; - cJSON * group_ids = cJSON_Parse(group_ids_str); - free(group_ids_str); - JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids); + const int ret = parse_groupIDs(api, &table, row); + if(ret != 0) + return ret; } else { diff --git a/src/api/stats.c b/src/api/stats.c index 86de44c5..ccaffc61 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -14,8 +14,6 @@ #include "api/api.h" #include "shmem.h" #include "datastructure.h" -// read_setupVarsconf() -#include "config/setupVars.h" // logging routines #include "log.h" // config struct @@ -27,6 +25,17 @@ // sqrt() #include +struct top_entries { + int count; + unsigned int responses; + in_port_t port; + size_t namepos; + size_t ippos; + double rtime; + double rtuncertainty; + +}; + /* qsort comparison function (count field), sort ASC static int __attribute__((pure)) cmpasc(const void *a, const void *b) { @@ -55,6 +64,20 @@ int __attribute__((pure)) cmpdesc(const void *a, const void *b) return 0; } +// qsort subroutine, sort DESC +static int __attribute__((pure)) cmpdesc_te(const void *a, const void *b) +{ + const struct top_entries *elem1 = (struct top_entries*)a; + const struct top_entries *elem2 = (struct top_entries*)b; + + if (elem1->count > elem2->count) + return -1; + else if (elem1->count < elem2->count) + return 1; + else + return 0; +} + static int get_query_types_obj(struct ftl_conn *api, cJSON *types) { for(unsigned int i = TYPE_A; i < TYPE_MAX; i++) @@ -69,29 +92,59 @@ static int get_query_types_obj(struct ftl_conn *api, cJSON *types) return 0; } +// shmem needs to be locked while calling this function +unsigned int get_active_clients(void) +{ + unsigned int activeclients = 0; + for(int clientID=0; clientID < counters->clients; clientID++) + { + // Get client pointer + const clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + + if(client->count > 0) + activeclients++; + } + + return activeclients; +} + int api_stats_summary(struct ftl_conn *api) { - const int blocked = get_blocked_count(); - const int forwarded = get_forwarded_count(); - const int cached = get_cached_count(); - const int total = counters->queries; - float percent_blocked = 0.0f; + // Lock shared memory + lock_shm(); + const int blocked = get_blocked_count(); + const int forwarded = get_forwarded_count(); + const int cached = get_cached_count(); + const int total = counters->queries; + const int num_gravity = counters->database.gravity; + const int num_clients = counters->clients; + const int num_domains = counters->domains; + + // Count clients that have been active within the most recent 24 hours + unsigned int activeclients = get_active_clients(); + + // Unlock shared memory + unlock_shm(); + + // Calculate percentage of blocked queries + float percent_blocked = 0.0f; // Avoid 1/0 condition if(total > 0) percent_blocked = 1e2f*blocked/total; - // Lock shared memory - lock_shm(); - cJSON *queries = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(queries, "total", total); JSON_ADD_NUMBER_TO_OBJECT(queries, "blocked", blocked); JSON_ADD_NUMBER_TO_OBJECT(queries, "percent_blocked", percent_blocked); - JSON_ADD_NUMBER_TO_OBJECT(queries, "unique_domains", counters->domains); + JSON_ADD_NUMBER_TO_OBJECT(queries, "unique_domains", num_domains); JSON_ADD_NUMBER_TO_OBJECT(queries, "forwarded", forwarded); JSON_ADD_NUMBER_TO_OBJECT(queries, "cached", cached); + JSON_ADD_NUMBER_TO_OBJECT(queries, "frequency", get_qps()); + cJSON *types = JSON_NEW_OBJECT(); int ret = get_query_types_obj(api, types); if(ret != 0) @@ -108,34 +161,23 @@ int api_stats_summary(struct ftl_conn *api) JSON_ADD_NUMBER_TO_OBJECT(replies, get_query_reply_str(reply), counters->reply[reply]); JSON_ADD_ITEM_TO_OBJECT(queries, "replies", replies); - // Count clients that have been active within the most recent 24 hours - unsigned int activeclients = 0; - for(int clientID=0; clientID < counters->clients; clientID++) - { - // Get client pointer - const clientsData* client = getClient(clientID, true); - if(client == NULL) - continue; - - if(client->count > 0) - activeclients++; - } - cJSON *clients = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(clients, "active", activeclients); - JSON_ADD_NUMBER_TO_OBJECT(clients, "total", counters->clients); + JSON_ADD_NUMBER_TO_OBJECT(clients, "total", num_clients); cJSON *gravity = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", counters->database.gravity); + JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", num_gravity); + JSON_ADD_NUMBER_TO_OBJECT(gravity, "last_update", gravity_last_updated()); cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries); JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients); JSON_ADD_ITEM_TO_OBJECT(json, "gravity", gravity); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } -int api_stats_top_domains(struct ftl_conn *api) +cJSON *get_top_domains(struct ftl_conn *api, const int count, + const bool blocked, const bool domains_only) { // Exit before processing any data if requested via config setting if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS) @@ -145,24 +187,150 @@ int api_stats_top_domains(struct ftl_conn *api) // Minimum structure is // {"top_domains":[]} - cJSON *json = JSON_NEW_OBJECT(); - cJSON *top_domains = JSON_NEW_ARRAY(); - JSON_ADD_ITEM_TO_OBJECT(json, "top_domains", top_domains); - JSON_SEND_OBJECT(json); + if(domains_only) + return cJSON_CreateArray(); + + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "domains", cJSON_CreateArray()); + cJSON_AddNumberToObject(json, "total_queries", -1); + cJSON_AddNumberToObject(json, "blocked_queries", -1); + return json; } + // Get domains which the user doesn't want to see + regex_t *regex_domains = NULL; + unsigned int N_regex_domains = 0; + compile_filter_regex(api, "webserver.api.excludeDomains", + config.webserver.api.excludeDomains.v.json, + ®ex_domains, &N_regex_domains); + // Lock shared memory lock_shm(); - // Allocate memory const int domains = counters->domains; - int *temparray = calloc(2*domains, sizeof(int)); - if(temparray == NULL) + const int total_queries = counters->queries; + const int blocked_count = get_blocked_count(); + struct top_entries *top_domains = calloc(domains, sizeof(struct top_entries)); + if(top_domains == NULL) { log_err("Memory allocation failed in %s()", __FUNCTION__); - return 0; + return NULL; } + unsigned int added_domains = 0u; + for(int domainID = 0; domainID < domains; domainID++) + { + // Get domain pointer + const domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + continue; + + const char *domain_name = getstr(domain->domainpos); + + // Hidden domain, probably due to privacy level. Skip this in the top lists + if(strcmp(domain_name, HIDDEN_DOMAIN) == 0) + continue; + + // Use either blocked or total count based on request string + top_domains[added_domains].count = blocked ? domain->blockedcount : domain->count - domain->blockedcount; + + // Get domain name + top_domains[added_domains].namepos = domain->domainpos; + + // Increment counter + added_domains++; + } + + // Unlock shared memory + unlock_shm(); + + // Sort temporary array + qsort(top_domains, added_domains, sizeof(*top_domains), cmpdesc_te); + + int n = 0; + cJSON *jtop_domains = cJSON_CreateArray(); + + // Lock shared memory + lock_shm(); + + for(unsigned int i = 0; i < added_domains; i++) + { + // Skip e.g. recycled domains + if(top_domains[i].namepos == 0) + continue; + + const char *domain = getstr(top_domains[i].namepos); + + // Skip this client if there is a filter on it + bool skip_domain = false; + if(N_regex_domains > 0) + { + // Iterate over all regex filters + for(unsigned int j = 0; j < N_regex_domains; j++) + { + // Check if the domain matches the regex + if(regexec(®ex_domains[j], domain, 0, NULL, 0) == 0) + { + // Domain matches + skip_domain = true; + break; + } + } + } + + if(skip_domain || top_domains[i].count < 1) + continue; + + if(domains_only) + { + cJSON_AddStringToArray(jtop_domains, domain); + } + else + { + cJSON *domain_item = cJSON_CreateObject(); + cJSON_AddStringToObject(domain_item, "domain", domain); + cJSON_AddNumberToObject(domain_item, "count", top_domains[i].count); + cJSON_AddItemToArray(jtop_domains, domain_item); + } + + // Only count entries that are actually sent and return when we have send enough data + if(++n >= count) + break; + } + + // Unlock shared memory + unlock_shm(); + + // Free temporary array + free(top_domains); + + // Free regexes + if(N_regex_domains > 0) + { + // Free individual regexes + for(unsigned int i = 0; i < N_regex_domains; i++) + regfree(®ex_domains[i]); + + // Free array of regex pointers + free(regex_domains); + } + + if(domains_only) + { + // Return the array of domains only + return jtop_domains; + } + + // else: Build and return full object + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "domains", jtop_domains); + cJSON_AddNumberToObject(json, "total_queries", total_queries); + cJSON_AddNumberToObject(json, "blocked_queries", blocked_count); + return json; +} + +int api_stats_top_domains(struct ftl_conn *api) +{ bool blocked = false; // Can be overwritten by query string int count = 10; // /api/stats/top_domains?blocked=true @@ -176,138 +344,14 @@ int api_stats_top_domains(struct ftl_conn *api) get_int_var(api->request->query_string, "count", &count); } - unsigned int added_domains = 0u; - for(int domainID = 0; domainID < domains; domainID++) - { - // Get domain pointer - const domainsData* domain = getDomain(domainID, true); - if(domain == NULL) - continue; - - // Add domain ID - temparray[2*added_domains + 0] = domainID; - - // Use either blocked or total count based on request string - temparray[2*added_domains + 1] = blocked ? domain->blockedcount : domain->count - domain->blockedcount; - - added_domains++; - } - - // Sort temporary array - qsort(temparray, added_domains, sizeof(int[2]), cmpdesc); - - // Get filter - const char* log_show = read_setupVarsconf("API_QUERY_LOG_SHOW"); - bool showpermitted = true, showblocked = true; - if(log_show != NULL) - { - if((strcmp(log_show, "permittedonly")) == 0) - showblocked = false; - else if((strcmp(log_show, "blockedonly")) == 0) - showpermitted = false; - else if((strcmp(log_show, "nothing")) == 0) - { - showpermitted = false; - showblocked = false; - } - } - clearSetupVarsArray(); - - // Get domains which the user doesn't want to see - regex_t *regex_domains = NULL; - unsigned int N_regex_domains = 0; - compile_filter_regex(api, "webserver.api.excludeDomains", - config.webserver.api.excludeDomains.v.json, - ®ex_domains, &N_regex_domains); - - int n = 0; - cJSON *top_domains = JSON_NEW_ARRAY(); - for(unsigned int i = 0; i < added_domains; i++) - { - // Get sorted index - const int domainID = temparray[2*i + 0]; - // Get domain pointer - const domainsData* domain = getDomain(domainID, true); - if(domain == NULL) - continue; - - // Get domain name - const char *domain_name = getstr(domain->domainpos); - - // Hidden domain, probably due to privacy level. Skip this in the top lists - if(strcmp(domain_name, HIDDEN_DOMAIN) == 0) - continue; - - // Skip this client if there is a filter on it - bool skip_domain = false; - if(N_regex_domains > 0) - { - // Iterate over all regex filters - for(unsigned int j = 0; j < N_regex_domains; j++) - { - // Check if the domain matches the regex - if(regexec(®ex_domains[j], domain_name, 0, NULL, 0) == 0) - { - // Domain matches - skip_domain = true; - break; - } - } - } - - if(skip_domain) - continue; - - int domain_count = -1; - if(blocked && showblocked && domain->blockedcount > 0) - { - domain_count = domain->blockedcount; - n++; - } - else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0) - { - domain_count = domain->count - domain->blockedcount; - n++; - } - if(domain_count > -1) - { - cJSON *domain_item = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(domain_item, "domain", domain_name); - JSON_ADD_NUMBER_TO_OBJECT(domain_item, "count", domain_count); - JSON_ADD_ITEM_TO_ARRAY(top_domains, domain_item); - } - - // Only count entries that are actually sent and return when we have send enough data - if(n >= count) - break; - } - free(temparray); - - // Free regexes - if(N_regex_domains > 0) - { - // Free individual regexes - for(unsigned int i = 0; i < N_regex_domains; i++) - regfree(®ex_domains[i]); - - // Free array of regex pointers - free(regex_domains); - } - - cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "domains", top_domains); - - const int blocked_count = get_blocked_count(); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); - JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); - - JSON_SEND_OBJECT_UNLOCK(json); + cJSON *json = get_top_domains(api, count, blocked, false); + JSON_SEND_OBJECT(json); } -int api_stats_top_clients(struct ftl_conn *api) +cJSON *get_top_clients(struct ftl_conn *api, const int count, + const bool blocked, const bool clients_only, + const bool names_only, const bool ip_if_no_name) { - int count = 10; - // Exit before processing any data if requested via config setting if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS) { @@ -316,31 +360,26 @@ int api_stats_top_clients(struct ftl_conn *api) // Minimum structure is // {"top_clients":[]} - cJSON *json = JSON_NEW_OBJECT(); - cJSON *top_clients = JSON_NEW_ARRAY(); - JSON_ADD_ITEM_TO_OBJECT(json, "top_clients", top_clients); - JSON_SEND_OBJECT(json); - } + if(clients_only) + return cJSON_CreateArray(); - bool blocked = false; // /api/stats/top_clients?blocked=true - if(api->request->query_string != NULL) - { - // Should blocked clients be shown? - get_bool_var(api->request->query_string, "blocked", &blocked); - - // Does the user request a non-default number of replies? - // Note: We do not accept zero query requests here - get_int_var(api->request->query_string, "count", &count); + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "clients", cJSON_CreateArray()); + cJSON_AddNumberToObject(json, "total_queries", -1); + cJSON_AddNumberToObject(json, "blocked_queries", -1); + return json; } // Lock shared memory lock_shm(); int clients = counters->clients; - int *temparray = calloc(2*clients, sizeof(int)); - if(temparray == NULL) + const int total_queries = counters->queries; + const int blocked_count = get_blocked_count(); + struct top_entries *top_clients = calloc(clients, sizeof(struct top_entries)); + if(top_clients == NULL) { - log_err("Memory allocation failed in api_stats_top_clients()"); + log_err("Memory allocation failed in %s()", __FUNCTION__); return 0; } @@ -352,17 +391,44 @@ int api_stats_top_clients(struct ftl_conn *api) // Skip invalid clients and also those managed by alias clients if(client == NULL || (!client->flags.aliasclient && client->aliasclient_id >= 0)) + { + log_debug(DEBUG_API, "Skipping client %i because %s", clientID, + client == NULL ? "it is invalid" : "it is an alias client"); continue; + } + + // Skip recycled clients + if(client->ippos == 0) + { + log_debug(DEBUG_API, "Skipping client %i because it is recycled", clientID); + continue; + } + + const char *client_ip = getstr(client->ippos); + // Hidden client, probably due to privacy level. Skip this in the top lists + if(strcmp(client_ip, HIDDEN_CLIENT) == 0) + { + log_debug(DEBUG_API, "Skipping client %i because it is hidden", clientID); + continue; + } - temparray[2*added_clients + 0] = clientID; // Use either blocked or total count based on request string - temparray[2*added_clients + 1] = blocked ? client->blockedcount : client->count; + top_clients[added_clients].count = blocked ? client->blockedcount : client->count; + + // Get client name and IP + top_clients[added_clients].ippos = client->ippos; + top_clients[added_clients].namepos = client->namepos; added_clients++; } + log_debug(DEBUG_API, "Found %u clients", added_clients); + + // Unlock shared memory + unlock_shm(); + // Sort temporary array - qsort(temparray, added_clients, sizeof(int[2]), cmpdesc); + qsort(top_clients, added_clients, sizeof(*top_clients), cmpdesc_te); // Get clients which the user doesn't want to see regex_t *regex_clients = NULL; @@ -372,24 +438,15 @@ int api_stats_top_clients(struct ftl_conn *api) ®ex_clients, &N_regex_clients); int n = 0; - cJSON *top_clients = JSON_NEW_ARRAY(); + cJSON *jtop_clients = JSON_NEW_ARRAY(); + + // Lock shared memory + lock_shm(); + for(unsigned int i = 0; i < added_clients; i++) { - // Get sorted indices and counter values (may be either total or blocked count) - const int clientID = temparray[2*i + 0]; - const int client_count = temparray[2*i + 1]; - // Get client pointer - const clientsData* client = getClient(clientID, true); - if(client == NULL) - continue; - - // Get IP and host name of client - const char *client_ip = getstr(client->ippos); - const char *client_name = getstr(client->namepos); - - // Hidden client, probably due to privacy level. Skip this in the top lists - if(strcmp(client_ip, HIDDEN_CLIENT) == 0) - continue; + const char *client_ip = getstr(top_clients[i].ippos); + const char *client_name = getstr(top_clients[i].namepos); // Skip this client if there is a filter on it bool skip_client = false; @@ -414,26 +471,48 @@ int api_stats_top_clients(struct ftl_conn *api) } } - if(skip_client) - continue; - - // Return this client if the client made at least one query - // within the most recent 24 hours - if(client_count > 0) + if(skip_client || top_clients[i].count < 1) { - cJSON *client_item = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(client_item, "name", client_name); - JSON_REF_STR_IN_OBJECT(client_item, "ip", client_ip); - JSON_ADD_NUMBER_TO_OBJECT(client_item, "count", client_count); - JSON_ADD_ITEM_TO_ARRAY(top_clients, client_item); - n++; + log_debug(DEBUG_API, "Skipping client %s because it %s", client_ip, + skip_client ? "matches a filter" : "has no queries"); + continue; } - if(n == count) + if(clients_only) + { + if(ip_if_no_name) + { + if(strlen(client_name) > 0) + cJSON_AddStringToArray(jtop_clients, client_name); + else + cJSON_AddStringToArray(jtop_clients, client_ip); + } + else if(names_only) + { + if(strlen(client_name) > 0) + cJSON_AddStringToArray(jtop_clients, client_name); + } + else + cJSON_AddStringToArray(jtop_clients, client_ip); + } + else + { + cJSON *client_item = cJSON_CreateObject(); + cJSON_AddStringToObject(client_item, "name", client_name); + cJSON_AddStringToObject(client_item, "ip", client_ip); + cJSON_AddNumberToObject(client_item, "count", top_clients[i].count); + cJSON_AddItemToArray(jtop_clients, client_item); + } + + if(++n == count) break; } + + // Unlock shared memory + unlock_shm(); + // Free temporary array - free(temparray); + free(top_clients); // Free regexes if(N_regex_clients > 0) @@ -446,22 +525,46 @@ int api_stats_top_clients(struct ftl_conn *api) free(regex_clients); } - cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "clients", top_clients); + if(clients_only) + { + // Return the array of clients only + return jtop_clients; + } - const int blocked_count = get_blocked_count(); - JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); - JSON_SEND_OBJECT_UNLOCK(json); + // else: Build and return full object + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "clients", jtop_clients); + cJSON_AddNumberToObject(json, "total_queries", total_queries); + cJSON_AddNumberToObject(json, "blocked_queries", blocked_count); + return json; } - -int api_stats_upstreams(struct ftl_conn *api) +int api_stats_top_clients(struct ftl_conn *api) +{ + bool blocked = false; // Can be overwritten by query string + int count = 10; + // /api/stats/top_clients?blocked=true + if(api->request->query_string != NULL) + { + // Should blocked clients be shown? + get_bool_var(api->request->query_string, "blocked", &blocked); + + // Does the user request a non-default number of replies? + // Note: We do not accept zero query requests here + get_int_var(api->request->query_string, "count", &count); + } + + cJSON *json = get_top_clients(api, count, blocked, false, false, false); + JSON_SEND_OBJECT(json); +} + +cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only) { - unsigned int totalcount = 0; const int upstreams = counters->upstreams; - int *temparray = calloc(2*upstreams, sizeof(int)); - if(temparray == NULL) + const int forwarded_count = get_forwarded_count(); + const int total_queries = counters->queries; + struct top_entries *top_upstreams = calloc(upstreams, sizeof(struct top_entries)); + if(top_upstreams == NULL) { log_err("Memory allocation failed in api_stats_upstreams()"); return 0; @@ -478,23 +581,34 @@ int api_stats_upstreams(struct ftl_conn *api) if(upstream == NULL) continue; - temparray[2*added_upstreams + 0] = upstreamID; - temparray[2*added_upstreams + 1] = upstream->count; - totalcount += upstream->count; + top_upstreams[added_upstreams].count = upstream->count; + top_upstreams[added_upstreams].ippos = upstream->ippos; + top_upstreams[added_upstreams].namepos = upstream->namepos; + top_upstreams[added_upstreams].port = upstream->port; + top_upstreams[added_upstreams].responses = upstream->responses; + top_upstreams[added_upstreams].rtime = upstream->rtime; + top_upstreams[added_upstreams].rtuncertainty = upstream->rtuncertainty; added_upstreams++; } + // Unlock shared memory + unlock_shm(); + // Sort temporary array in descending order - qsort(temparray, upstreams, sizeof(int[2]), cmpdesc); + qsort(top_upstreams, added_upstreams, sizeof(*top_upstreams), cmpdesc); // Loop over available forward destinations - cJSON *top_upstreams = JSON_NEW_ARRAY(); + cJSON *jtop_upstreams = JSON_NEW_ARRAY(); + + // Lock shared memory + lock_shm(); + for(int i = -2; i < (int)added_upstreams; i++) { int count = 0; const char* ip, *name; - int port = -1; + int port = -1; // Need signed data type here as -1 means: no port applicable double responsetime = 0.0, uncertainty = 0.0; if(i == -2) @@ -514,67 +628,80 @@ int api_stats_upstreams(struct ftl_conn *api) else { // Regular upstream destination - // Get sorted indices - const int upstreamID = temparray[2*i + 0]; - - // Get upstream pointer - const upstreamsData *upstream = getUpstream(upstreamID, true); - if(upstream == NULL) - continue; - - // Get IP and host name of upstream destination if available - ip = getstr(upstream->ippos); - name = getstr(upstream->namepos); - port = upstream->port; - - // Get percentage - count = upstream->count; + ip = getstr(top_upstreams[i].ippos); + name = getstr(top_upstreams[i].namepos); + port = top_upstreams[i].port; + count = top_upstreams[i].count; // Compute average response time and uncertainty (unit: seconds) - if(upstream->responses > 0) + if(top_upstreams[i].responses > 0) { // Simple average of the response times - responsetime = upstream->rtime / upstream->responses; + responsetime = top_upstreams[i].rtime / top_upstreams[i].responses; } - if(upstream->responses > 1) + if(top_upstreams[i].responses > 1) { // The actual value will be somewhere in a neighborhood around the mean value. // This neighborhood of values is the uncertainty in the mean. - uncertainty = sqrt(upstream->rtuncertainty / upstream->responses / (upstream->responses-1)); + uncertainty = sqrt(top_upstreams[i].rtuncertainty / top_upstreams[i].responses / (top_upstreams[i].responses-1)); } } // Send data: // - always if i < 0 (special upstreams: blocklist and cache) // - only if there are any queries for all others (i > 0) - if(count > 0 || i < 0) + if(count < 1 && i >= 0) + continue; + + if(upstreams_only) + { + cJSON_AddStringToArray(jtop_upstreams, name); + } + else { cJSON *upstream = JSON_NEW_OBJECT(); - JSON_REF_STR_IN_OBJECT(upstream, "ip", ip); - JSON_REF_STR_IN_OBJECT(upstream, "name", name); - JSON_ADD_NUMBER_TO_OBJECT(upstream, "port", port); - JSON_ADD_NUMBER_TO_OBJECT(upstream, "count", count); + cJSON_AddStringToObject(upstream, "ip", ip); + cJSON_AddStringToObject(upstream, "name", name); + cJSON_AddNumberToObject(upstream, "port", port); + cJSON_AddNumberToObject(upstream, "count", count); cJSON *statistics = JSON_NEW_OBJECT(); - JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", responsetime); - JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", uncertainty); - JSON_ADD_ITEM_TO_OBJECT(upstream, "statistics", statistics); - JSON_ADD_ITEM_TO_ARRAY(top_upstreams, upstream); + cJSON_AddNumberToObject(statistics, "response", responsetime); + cJSON_AddNumberToObject(statistics, "variance", uncertainty); + cJSON_AddItemToObject(upstream, "statistics", statistics); + cJSON_AddItemToArray(jtop_upstreams, upstream); } } - // Free temporary array - free(temparray); + // Unlock shared memory + unlock_shm(); - cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", top_upstreams); - const int forwarded_count = get_forwarded_count(); - JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_count); - JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); - JSON_SEND_OBJECT_UNLOCK(json); + // Free temporary array + free(top_upstreams); + + if(upstreams_only) + { + // Return the array of upstreams only + return jtop_upstreams; + } + + // else: Build and return full object + cJSON *json = cJSON_CreateObject(); + cJSON_AddItemToObject(json, "upstreams", jtop_upstreams); + cJSON_AddNumberToObject(json, "total_queries", total_queries); + cJSON_AddNumberToObject(json, "forwarded_queries", forwarded_count); + + return json; +} + +int api_stats_upstreams(struct ftl_conn *api) +{ + cJSON *json = get_top_upstreams(api, false); + JSON_SEND_OBJECT(json); } int api_stats_query_types(struct ftl_conn *api) { + // Lock shared memory lock_shm(); cJSON *types = JSON_NEW_OBJECT(); @@ -585,11 +712,14 @@ int api_stats_query_types(struct ftl_conn *api) return ret; } + // Unlock shared memory + unlock_shm(); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "types", types); // Send response - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } int api_stats_recentblocked(struct ftl_conn *api) @@ -621,7 +751,7 @@ int api_stats_recentblocked(struct ftl_conn *api) cJSON *blocked = JSON_NEW_ARRAY(); for(int queryID = counters->queries - 1; queryID > 0 ; queryID--) { - const queriesData* query = getQuery(queryID, true); + const queriesData *query = getQuery(queryID, true); if(query == NULL) continue; @@ -643,7 +773,10 @@ int api_stats_recentblocked(struct ftl_conn *api) break; } + // Unlock shared memory + unlock_shm(); + cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "blocked", blocked); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } diff --git a/src/api/stats_database.c b/src/api/stats_database.c index b4309f07..61213c1e 100644 --- a/src/api/stats_database.c +++ b/src/api/stats_database.c @@ -8,16 +8,16 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" -#include "../webserver/http-common.h" -#include "../webserver/json_macros.h" +#include "FTL.h" +#include "webserver/http-common.h" +#include "webserver/json_macros.h" #include "api.h" // querytypes[] -#include "../datastructure.h" +#include "datastructure.h" // logging routines #include "log.h" // db -#include "../database/common.h" +#include "database/common.h" // SQL Query type filters for the database #define FILTER_STATUS_NOT_BLOCKED "status IN (0,2,3,12,13,14,17)" @@ -514,13 +514,11 @@ int api_history_database_clients(struct ftl_conn *api) // Loop over clients and accumulate results cJSON *clients = JSON_NEW_OBJECT(); - unsigned int num_clients = 0; while((rc = sqlite3_step(stmt)) == SQLITE_ROW) { cJSON *item = JSON_NEW_OBJECT(); JSON_COPY_STR_TO_OBJECT(item, "name", sqlite3_column_text(stmt, 2)); JSON_ADD_ITEM_TO_OBJECT(clients, (const char*)sqlite3_column_text(stmt, 1), item); - num_clients++; } sqlite3_finalize(stmt); diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 12eb2ad1..16e15756 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -25,6 +25,12 @@ #include "database/common.h" // MAX_ROTATIONS #include "files.h" +//basename() +#include +// restart_ftl() +#include "signals.h" +// create_migration_target_v6() +#include "config/config.h" #define MAXFILESIZE (50u*1024*1024) @@ -68,14 +74,18 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; + cJSON *import; uint8_t *data; char *filename; size_t filesize; + struct { + bool file; + bool sid; + bool import; + } field; }; // Callback function for CivetWeb to determine which fields we want to receive -static bool is_file = false; -static bool is_sid = false; static int field_found(const char *key, const char *filename, char *path, @@ -85,17 +95,22 @@ static int field_found(const char *key, struct upload_data *data = (struct upload_data *)user_data; log_debug(DEBUG_API, "Found field: \"%s\", filename: \"%s\"", key, filename); - is_file = false; - is_sid = false; + // Set all fields to false + memset(&data->field, false, sizeof(data->field)); if(strcasecmp(key, "file") == 0 && filename && *filename) { data->filename = strdup(filename); - is_file = true; + data->field.file = true; return MG_FORM_FIELD_STORAGE_GET; } else if(strcasecmp(key, "sid") == 0) { - is_sid = true; + data->field.sid = true; + return MG_FORM_FIELD_STORAGE_GET; + } + else if(strcasecmp(key, "import") == 0) + { + data->field.import = true; return MG_FORM_FIELD_STORAGE_GET; } @@ -111,7 +126,7 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * struct upload_data *data = (struct upload_data *)user_data; log_debug(DEBUG_API, "Received field: \"%s\" (length %zu bytes)", key, valuelen); - if(is_file) + if(data->field.file) { if(data->filesize + valuelen > MAXFILESIZE) { @@ -129,7 +144,7 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * log_debug(DEBUG_API, "Received file (%zu bytes, buffer is now %zu bytes)", valuelen, data->filesize); } - else if(is_sid) + else if(data->field.sid) { // Allocate memory for the SID data->sid = calloc(valuelen + 1, sizeof(char)); @@ -138,6 +153,28 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * // Add terminating NULL byte (memcpy does not do this) data->sid[valuelen] = '\0'; } + else if(data->field.import) + { + // Try to parse the JSON data + const char *json_error = NULL; + cJSON *json = cJSON_ParseWithLengthOpts(value, valuelen, &json_error, false); + if(json == NULL) + { + log_err("Unable to parse JSON data in API request, error at: %.20s", json_error); + return MG_FORM_FIELD_HANDLE_ABORT; + } + + // Check if the JSON data is an object + if(!cJSON_IsObject(json)) + { + log_err("JSON data in API request is not an object"); + cJSON_Delete(json); + return MG_FORM_FIELD_HANDLE_ABORT; + } + + // Store the parsed JSON data + data->import = json; + } // If there is more data in this field, get the next chunk. // Otherwise: handle the next field. @@ -168,6 +205,11 @@ static int free_upload_data(struct upload_data *data) free(data->data); data->data = NULL; } + if(data->import) + { + cJSON_Delete(data->import); + data->import = NULL; + } return 0; } @@ -224,6 +266,9 @@ static int api_teleporter_POST(struct ftl_conn *api) NULL); } + // Ensure v6 migration directory exists + create_migration_target_v6(); + // Check if we received something that claims to be a ZIP archive // - filename should end in ".zip" // - the data itself @@ -262,7 +307,7 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) char hint[ERRBUF_SIZE]; memset(hint, 0, sizeof(hint)); cJSON *json_files = JSON_NEW_ARRAY(); - const char *error = read_teleporter_zip(data->data, data->filesize, hint, json_files); + const char *error = read_teleporter_zip(data->data, data->filesize, hint, data->import, json_files); if(error != NULL) { const size_t msglen = strlen(error) + strlen(hint) + 4; @@ -277,13 +322,17 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) free_upload_data(data); return send_json_error_free(api, 400, "bad_request", - "Invalid ZIP archive", - msg, true); + "Invalid request", + msg, true, true); } // Free allocated memory free_upload_data(data); + // Signal FTL we want to restart for re-import + api->ftl.restart_reason = "Teleporter (ZIP) import"; + api->ftl.restart = true; + // Send response cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); @@ -632,14 +681,49 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Parse JSON files in the TAR archive cJSON *imported_files = JSON_NEW_ARRAY(); + + // Check if the archive contains gravity tables + cJSON *gravity = data->import != NULL ? cJSON_GetObjectItemCaseSensitive(data->import, "gravity") : NULL; for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) { + // - if import is non-NULL we may skip some tables + if(data->import != NULL) + { + // - if import is non-NULL, but gravity is NULL we skip + // the import of gravity tables altogether + // - if import is non-NULL, and gravity is non-NULL, we + // import the file/table if it is in the object, a + // boolean and true + if(gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name)) + { + log_info("Skipping import of \"%s\" as it was not requested for import (JSON: %s, gravity: %s)", + teleporter_v5_files[i].filename, + data->import != NULL ? "yes" : "no", + gravity != NULL ? "yes" : "no"); + continue; + } + } + + // Import the JSON file size_t fileSize = 0u; cJSON *json = NULL; const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); - if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) + const char *json_error = NULL; + if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLengthOpts(file, fileSize, &json_error, false)) != NULL) + { if(import_json_table(json, &teleporter_v5_files[i])) JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); + } + else if(json_error != NULL) + { + log_err("Unable to parse JSON file \"%s\", error at: %.20s", + teleporter_v5_files[i].filename, json_error); + } + else + { + log_debug(DEBUG_CONFIG, "Unable to find file \"%s\" in TAR archive", + teleporter_v5_files[i].filename); + } } // Temporarily write further files to to disk so we can import them on restart @@ -648,23 +732,45 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat const char *destination; } extract_files[] = { { + // i = 0 .archive_name = "custom.list", .destination = DNSMASQ_CUSTOM_LIST_LEGACY },{ + // i = 1 .archive_name = "dhcp.leases", .destination = DHCPLEASESFILE },{ + // i = 2 .archive_name = "pihole-FTL.conf", .destination = GLOBALCONFFILE_LEGACY },{ + // i = 3 .archive_name = "setupVars.conf", .destination = config.files.setupVars.v.s + },{ + .archive_name = "dnsmasq.d/05-pihole-custom-cname.conf", + .destination = DNSMASQ_CNAMES } }; for(size_t i = 0; i < sizeof(extract_files) / sizeof(*extract_files); i++) { size_t fileSize = 0u; const char *file = find_file_in_tar(archive, archive_size, extract_files[i].archive_name, &fileSize); + + if(data->import != NULL && i == 1 && !JSON_KEY_TRUE(data->import, "dhcp_leases")) + { + log_info("Skipping import of \"%s\" as it was not requested for import", + extract_files[i].archive_name); + continue; + } + // all other values of i belong to config files + else if(data->import != NULL && !JSON_KEY_TRUE(data->import, "config")) + { + log_info("Skipping import of \"%s\" as it was not requested for import", + extract_files[i].archive_name); + continue; + } + if(file != NULL && fileSize > 0u) { // Write file to disk @@ -676,6 +782,12 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat log_err("Unable to open file \"%s\" for writing: %s", extract_files[i].destination, strerror(errno)); continue; } + + // Restrict permissions to owner read/write only + if(fchmod(fileno(fp), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", extract_files[i].destination, strerror(errno)); + + // Write file to disk if(fwrite(file, fileSize, 1, fp) != 1) { log_err("Unable to write file \"%s\": %s", extract_files[i].destination, strerror(errno)); @@ -705,8 +817,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // restore on restart for(unsigned int i = MAX_ROTATIONS; i > 0; i--) { - const char *fname = GLOBALTOMLPATH; - const char *filename = basename(fname); + char *fname = strdup(GLOBALTOMLPATH); + char *filename = basename(fname); // extra 6 bytes is enough space for up to 999 rotations ("/", ".", "\0", "999") const size_t buflen = strlen(filename) + strlen(BACKUP_DIR) + 6; char *path = calloc(buflen, sizeof(char)); @@ -715,12 +827,15 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Remove file (if it exists) if(remove(path) != 0 && errno != ENOENT) log_err("Unable to remove file \"%s\": %s", path, strerror(errno)); + + free(fname); } // Free allocated memory free_upload_data(data); // Signal FTL we want to restart for re-import + api->ftl.restart_reason = "Teleporter (TAR.GZ) import"; api->ftl.restart = true; // Send response diff --git a/src/api/theme.c b/src/api/theme.c index 0b4cf4dc..42529f01 100644 --- a/src/api/theme.c +++ b/src/api/theme.c @@ -11,7 +11,7 @@ // NULL #include // strcasecmp() -#include +#include #include "theme.h" diff --git a/src/args.c b/src/args.c index 29b24e2f..529bb488 100644 --- a/src/args.c +++ b/src/args.c @@ -66,6 +66,10 @@ #include "files.h" // resolveHostname() #include "resolve.h" +// ntp_client() +#include "ntp/ntp.h" +// check_capability() +#include "capabilities.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -102,6 +106,7 @@ const char** argv_dnsmasq = NULL; #define COL_BLUE "\x1b[94m" // bright foreground color #define COL_PURPLE "\x1b[95m" // bright foreground color #define COL_CYAN "\x1b[96m" // bright foreground color +#define CLI_OVER "\r\x1b[K" // go back to beginning of line and erase to end of line static bool __attribute__ ((pure)) is_term(void) { @@ -145,6 +150,16 @@ const char __attribute__ ((pure)) *cli_bold(void) return is_term() ? COL_BOLD : ""; } +const char __attribute__ ((pure)) *cli_underline(void) +{ + return is_term() ? COL_ULINE : ""; +} + +const char __attribute__ ((pure)) *cli_italics(void) +{ + return is_term() ? COL_ITALIC : ""; +} + // Resets font to normal const char __attribute__ ((pure)) *cli_normal(void) { @@ -161,7 +176,7 @@ static const char __attribute__ ((pure)) *cli_color(const char *color) const char __attribute__ ((pure)) *cli_over(void) { // \x1b[K is the ANSI escape sequence for "erase to end of line" - return is_term() ? "\r\x1b[K" : "\r"; + return is_term() ? CLI_OVER : "\r"; } static inline bool strEndsWith(const char *input, const char *end) @@ -197,6 +212,10 @@ void parse_args(int argc, char* argv[]) if(strEndsWith(argv[0], "luac")) exit(run_luac(argc, argv)); + // Special (undocumented) mode to test kernel signal handling + if(argc == 2 && strcmp(argv[1], "sigtest") == 0) + exit(sigtest()); + // If the binary name is "sqlite3" (e.g., symlink /usr/bin/sqlite3 -> /usr/bin/pihole-FTL), // we operate in drop-in mode and consume all arguments for the embedded SQLite3 engine // Also, we do this if the first argument is a file with ".db" ending @@ -305,6 +324,38 @@ void parse_args(int argc, char* argv[]) exit(write_teleporter_zip_to_disk() ? EXIT_SUCCESS : EXIT_FAILURE); } + // Create test NTP client + if((argc > 1 && argc < 5) && strcmp(argv[1], "ntp") == 0) + { + // Parse arguments + const bool update = (argc > 2 && strcmp(argv[2], "--update") == 0) || + (argc > 3 && strcmp(argv[3], "--update") == 0); + const char *server = "127.0.0.1"; + if(argc > 2 && strcmp(argv[2], "--update") != 0) + server = argv[2]; + + // Ensure we have the necessary capabilities + if(update && !check_capability(CAP_SYS_TIME)) + { + puts("Insufficient capabilities to run NTP client"); + const char *bold = cli_bold(); + const char *normal = cli_normal(); + printf("Try: %ssudo%s ", bold, normal); + for(int i = 0; i < argc; i++) + printf("%s ", argv[i]); + puts(""); + exit(EXIT_FAILURE); + } + + printf("Using NTP server: %s\n", server); + + // Enable stdout printing + cli_mode = true; + log_ctrl(false, true); + readFTLconf(&config, false); + exit(ntp_client(server, update, true) ? EXIT_SUCCESS : EXIT_FAILURE); + } + // Import teleporter archive through CLI if(argc == 3 && strcmp(argv[1], "--teleporter") == 0) { @@ -486,7 +537,7 @@ void parse_args(int argc, char* argv[]) // Enable stdout printing cli_mode = true; uint8_t checksum[SHA256_DIGEST_SIZE]; - if(!sha256sum(argv[2], checksum)) + if(!sha256sum(argv[2], checksum, false)) exit(EXIT_FAILURE); // Convert checksum to hex string @@ -498,8 +549,20 @@ void parse_args(int argc, char* argv[]) exit(EXIT_SUCCESS); } + // Checksum verification mode + if(argc == 2 && strcmp(argv[1], "verify") == 0) + { + // Enable stdout printing + cli_mode = true; + const bool match = verify_FTL(true); + printf("%s Binary integrity check: %s\n", + match ? cli_tick() : cli_cross() , + match ? "OK" : "FAILED"); + exit(match ? EXIT_SUCCESS : EXIT_FAILURE); + } + // Local reverse name resolver - if(argc == 3 && strcasecmp(argv[1], "ptr") == 0) + if((argc == 3 || argc == 4) && strcasecmp(argv[1], "ptr") == 0) { // Enable stdout printing cli_mode = true; @@ -507,7 +570,18 @@ void parse_args(int argc, char* argv[]) // Need to get dns.port and the resolver settings readFTLconf(&config, false); - char *name = resolveHostname(argv[2], true); + // TCP or UDP (default)? + const bool tcp = argc == 4 && strcasecmp(argv[3], "tcp") == 0; + + // Create a socket + struct sockaddr_in dest; + const int sock = create_socket(tcp, &dest); + char *name = resolveHostname(sock, tcp, &dest, argv[2], true, NULL); + + // Close the socket + close(sock); + + // Exit early if no name was found if(name == NULL) exit(EXIT_FAILURE); @@ -715,7 +789,12 @@ void parse_args(int argc, char* argv[]) printf("Branch: " GIT_BRANCH "\n"); printf("Commit: " GIT_HASH " (" GIT_DATE ")\n"); printf("Architecture: " FTL_ARCH "\n"); - printf("Compiler: " FTL_CC "\n\n"); + printf("Compiler: " FTL_CC "\n"); +#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) + printf("GLIBC version: %d.%d\n\n", __GLIBC__, __GLIBC_MINOR__); +#else + printf("GLIBC version: -\n\n"); +#endif // Print dnsmasq version and compile time options print_dnsmasq_version(yellow, green, bold, normal); @@ -750,11 +829,12 @@ void parse_args(int argc, char* argv[]) printf("\n"); printf("****************************** %s%sCivetWeb%s *****************************\n", yellow, bold, normal); -#ifdef MBEDTLS_VERSION_STRING_FULL - printf("Version: %s%s%s%s with %smbed TLS %s%s"MBEDTLS_VERSION_STRING"%s\n", +#ifdef HAVE_MBEDTLS + printf("Version: %s%s%s%s (modified by Pi-hole) with %smbed TLS %s%s"MBEDTLS_VERSION_STRING"%s\n", green, bold, mg_version(), normal, yellow, green, bold, normal); #else - printf("Version: %s%s%s%s\n", green, bold, mg_version(), normal); + printf("Version: %s%s%s%s%s (modified by Pi-hole) without %smbed TLS%s\n", + green, bold, mg_version(), normal, red, yellow, normal); #endif printf("Features: "); if(mg_check_feature(MG_FEATURES_FILES)) @@ -868,6 +948,7 @@ void parse_args(int argc, char* argv[]) if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0) { const char *bold = cli_bold(); + const char *uline = cli_underline(); const char *normal = cli_normal(); const char *blue = cli_color(COL_BLUE); const char *cyan = cli_color(COL_CYAN); @@ -958,12 +1039,17 @@ void parse_args(int argc, char* argv[]) printf("%sEmbedded GZIP un-/compressor:%s\n", yellow, normal); printf(" A simple but fast in-memory gzip compressor\n\n"); - printf(" Usage: %spihole-FTL --compress %sinfile %s[outfile]%s\n", green, cyan, purple, normal); - printf(" Usage: %spihole-FTL --uncompress %sinfile %s[outfile]%s\n\n", green, cyan, purple, normal); - printf(" - %sinfile%s is the file to be compressed.\n", cyan, normal); - printf(" - %s[outfile]%s is the optional target. If omitted, FTL will\n", purple, normal); - printf(" %s--compress%s: use the %sinfile%s and append %s.gz%s at the end\n", green, normal, cyan, normal, cyan, normal); - printf(" %s--uncompress%s: use the %sinfile%s and remove %s.gz%s at the end\n\n", green, normal, cyan, normal, cyan, normal); + printf(" Usage: %spihole-FTL --gzip %sinfile %s[outfile]%s\n\n", green, cyan, purple, normal); + printf(" - %sinfile%s is the file to be processed. If the filename ends\n", cyan, normal); + printf(" in %s.gz%s, FTL will uncompress, otherwise it will compress\n\n", yellow, normal); + printf(" - %s[outfile]%s is the optional target file.\n", purple, normal); + printf(" If omitted, FTL will try to derive the target file from\n"); + printf(" the source file.\n\n"); + printf(" Examples:\n"); + printf(" - %spihole-FTL --gzip %sfile.txt%s\n", green, cyan, normal); + printf(" compresses %sfile.txt%s to %sfile.txt%s.gz%s\n\n", cyan, normal, cyan, yellow, normal); + printf(" - %spihole-FTL --gzip %sfile.txt%s.gz%s\n", green, cyan, yellow, normal); + printf(" %sun%scompresses %sfile.txt%s.gz%s to %sfile.txt%s\n\n", uline, normal, cyan, yellow, normal, cyan, normal); printf("%sTeleporter:%s\n", yellow, normal); printf("\t%s--teleporter%s Create a Teleporter archive in the\n", green, normal); @@ -1000,9 +1086,25 @@ void parse_args(int argc, char* argv[]) printf(" Encoding: %spihole-FTL idn2 %sdomain%s\n", green, cyan, normal); printf(" Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal); + printf("%sNTP client:%s\n", yellow, normal); + printf(" Query an NTP server for the current time and print the\n"); + printf(" result in human-readable format. An optional %sserver%s may be\n", cyan, normal); + printf(" as argument. If the server is omitted, 127.0.0.1 is used.\n\n"); + printf(" The system time is updated on the system when the optional\n"); + printf(" %s--update%s flag is given.\n\n", purple, normal); + printf(" Usage: %spihole-FTL ntp %s[server]%s %s[--update]%s\n\n", green, cyan, normal, purple, normal); + + printf("%sSHA256 checksum tools:%s\n", yellow, normal); + printf(" Calculates the SHA256 checksum of a file. The checksum is\n"); + printf(" computed as described in FIPS-180-2 and uses streaming\n"); + printf(" to allow processing arbitrary large files with a small\n"); + printf(" memory footprint.\n\n"); + printf(" Usage: %spihole-FTL sha256sum %sfile%s\n\n", green, cyan, normal); + printf("%sOther:%s\n", yellow, normal); - printf("\t%sptr %sIP%s Resolve IP address to hostname\n", green, cyan, normal); - printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal); + printf("\t%sverify%s Verify the integrity of the FTL binary\n", green, normal); + printf("\t%sptr %sIP%s %s[tcp]%s Resolve IP address to hostname\n", green, cyan, normal, purple, normal); + printf("\t Append %stcp%s to use TCP instead of UDP\n", purple, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal); diff --git a/src/args.h b/src/args.h index fa188f3f..f9688917 100644 --- a/src/args.h +++ b/src/args.h @@ -24,6 +24,8 @@ const char *cli_done(void) __attribute__ ((pure)); const char *cli_bold(void) __attribute__ ((pure)); const char *cli_normal(void) __attribute__ ((pure)); const char *cli_over(void) __attribute__ ((pure)); +const char *cli_underline(void) __attribute__ ((pure)); +const char *cli_italics(void) __attribute__ ((pure)); void test_dnsmasq_options(int argc, const char *argv[]); diff --git a/src/capabilities.c b/src/capabilities.c index 79d0532e..57631edb 100644 --- a/src/capabilities.c +++ b/src/capabilities.c @@ -141,6 +141,13 @@ bool check_capabilities(void) log_warn("Required Linux capability CAP_CHOWN not available"); capabilities_okay = false; } + if (!(data->permitted & (1 << CAP_SYS_TIME)) || + !(data->effective & (1 << CAP_SYS_TIME))) + { + // Necessary for setting the system time in the NTP client + log_warn("Required Linux capability CAP_SYS_TIME not available"); + capabilities_okay = false; + } // Free allocated memory free(hdr); diff --git a/src/config/cli.c b/src/config/cli.c index d012d0d8..15b967a6 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -182,7 +182,7 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru // Free old password hash if it was allocated if(conf_item->t == CONF_STRING_ALLOCATED) - free(conf_item->v.s); + free(conf_item->v.s); // Store new password hash conf_item->v.s = pwhash; @@ -306,6 +306,21 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru } break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + const int edns_mode = get_edns_mode_val(value); + if(edns_mode != -1) + conf_item->v.edns_mode = edns_mode; + else + { + char *allowed = NULL; + CONFIG_ITEM_ARRAY(conf_item->a, allowed); + log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed); + free(allowed); + return false; + } + break; + } case CONF_STRUCT_IN_ADDR: { struct in_addr addr4 = { 0 }; @@ -342,10 +357,11 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru } case CONF_JSON_STRING_ARRAY: { - cJSON *elem = cJSON_Parse(value); + const char *json_error = NULL; + cJSON *elem = cJSON_ParseWithOpts(value, &json_error, 0); if(elem == NULL) { - log_err("Config setting %s is invalid: not valid JSON, error before: %s", conf_item->k, cJSON_GetErrorPtr()); + log_err("Config setting %s is invalid: not valid JSON, error at: %.20s", conf_item->k, json_error); return false; } if(!cJSON_IsArray(elem)) @@ -396,6 +412,14 @@ int set_config_from_CLI(const char *key, const char *value) return EXIT_FAILURE; } + // Return early if the user tries to change some settings but the config + // is in read-only mode + if(config.misc.readOnly.v.b) + { + printf("Config is in read-only mode, changes are not allowed (misc.readOnly = true)\n"); + return EXIT_FAILURE; + } + // Identify config option struct config newconf; duplicate_config(&newconf, &config); @@ -409,6 +433,7 @@ int set_config_from_CLI(const char *key, const char *value) if(strcmp(item->k, key) != 0) continue; + // Check if this is a read-only config option (forced by env var) if(item->f & FLAG_ENV_VAR) { log_err("Config option %s is read-only (set via environmental variable)", key); @@ -416,6 +441,14 @@ int set_config_from_CLI(const char *key, const char *value) return ENV_VAR_FORCED; } + // Check if this the special read-only config option + if(item->f & FLAG_READ_ONLY) + { + log_err("Config option %s can only be set in pihole.toml, not via the CLI", key); + free_config(&newconf); + return EXIT_FAILURE; + } + // This is the config option we are looking for new_item = item; diff --git a/src/config/cli.h b/src/config/cli.h index 4cf4cc31..c9398bcf 100644 --- a/src/config/cli.h +++ b/src/config/cli.h @@ -13,4 +13,4 @@ int set_config_from_CLI(const char *key, const char *value); int get_config_from_CLI(const char *key, const bool quiet); -#endif //CONFIG_CLI_H \ No newline at end of file +#endif //CONFIG_CLI_H diff --git a/src/config/config.c b/src/config/config.c index f5bb3fa1..434a4069 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -35,13 +35,18 @@ #include "config/env.h" // sha256sum() #include "files.h" +// restart_ftl() +#include "signals.h" +// Global variables struct config config = { 0 }; static bool config_initialized = false; uint8_t last_checksum[SHA256_DIGEST_SIZE] = { 0 }; // Private prototypes static bool port_in_use(const in_port_t port); +static void reset_config_default(struct conf_item *conf_item); +static void initConfig(struct config *conf); // Set debug flags from config struct to global debug_flags array // This is called whenever the config is reloaded and debug flags may have @@ -158,7 +163,10 @@ void free_config_path(char **paths) for(unsigned int i = 0; i < MAX_CONFIG_PATH_DEPTH; i++) if(paths[i] != NULL) + { free(paths[i]); + paths[i] = NULL; + } } bool __attribute__ ((pure)) check_paths_equal(char **paths1, char **paths2, unsigned int max_level) @@ -270,6 +278,7 @@ void duplicate_config(struct config *dst, struct config *src) case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: case CONF_STRUCT_IN_ADDR: case CONF_STRUCT_IN6_ADDR: case CONF_ALL_DEBUG_BOOL: @@ -306,6 +315,7 @@ bool compare_config_item(const enum conf_type t, const union conf_value *val1, c case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: case CONF_STRUCT_IN_ADDR: case CONF_STRUCT_IN6_ADDR: case CONF_ALL_DEBUG_BOOL: @@ -362,6 +372,7 @@ void free_config(struct config *conf) case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: case CONF_STRUCT_IN_ADDR: case CONF_STRUCT_IN6_ADDR: case CONF_ALL_DEBUG_BOOL: @@ -369,6 +380,8 @@ void free_config(struct config *conf) break; case CONF_STRING_ALLOCATED: free(copy_item->v.s); + copy_item->v.s = NULL; + copy_item->t = CONF_STRING; // not allocated anymore break; case CONF_JSON_STRING_ARRAY: cJSON_Delete(copy_item->v.json); @@ -377,7 +390,7 @@ void free_config(struct config *conf) } } -void initConfig(struct config *conf) +static void initConfig(struct config *conf) { if(config_initialized) return; @@ -395,47 +408,41 @@ void initConfig(struct config *conf) conf->dns.CNAMEdeepInspect.k = "dns.CNAMEdeepInspect"; conf->dns.CNAMEdeepInspect.h = "Use this option to control deep CNAME inspection. Disabling it might be beneficial for very low-end devices"; conf->dns.CNAMEdeepInspect.t = CONF_BOOL; - conf->dns.CNAMEdeepInspect.f = FLAG_ADVANCED_SETTING; conf->dns.CNAMEdeepInspect.d.b = true; conf->dns.CNAMEdeepInspect.c = validate_stub; // Only type-based checking conf->dns.blockESNI.k = "dns.blockESNI"; conf->dns.blockESNI.h = "Should _esni. subdomains be blocked by default? Encrypted Server Name Indication (ESNI) is certainly a good step into the right direction to enhance privacy on the web. It prevents on-path observers, including ISPs, coffee shop owners and firewalls, from intercepting the TLS Server Name Indication (SNI) extension by encrypting it. This prevents the SNI from being used to determine which websites users are visiting.\n ESNI will obviously cause issues for pixelserv-tls which will be unable to generate matching certificates on-the-fly when it cannot read the SNI. Cloudflare and Firefox are already enabling ESNI. According to the IEFT draft (link above), we can easily restore piselserv-tls's operation by replying NXDOMAIN to _esni. subdomains of blocked domains as this mimics a \"not configured for this domain\" behavior."; conf->dns.blockESNI.t = CONF_BOOL; - conf->dns.blockESNI.f = FLAG_ADVANCED_SETTING; conf->dns.blockESNI.d.b = true; conf->dns.blockESNI.c = validate_stub; // Only type-based checking conf->dns.EDNS0ECS.k = "dns.EDNS0ECS"; conf->dns.EDNS0ECS.h = "Should we overwrite the query source when client information is provided through EDNS0 client subnet (ECS) information? This allows Pi-hole to obtain client IPs even if they are hidden behind the NAT of a router. This feature has been requested and discussed on Discourse where further information how to use it can be found: https://discourse.pi-hole.net/t/support-for-add-subnet-option-from-dnsmasq-ecs-edns0-client-subnet/35940"; conf->dns.EDNS0ECS.t = CONF_BOOL; - conf->dns.EDNS0ECS.f = FLAG_ADVANCED_SETTING; conf->dns.EDNS0ECS.d.b = true; conf->dns.EDNS0ECS.c = validate_stub; // Only type-based checking conf->dns.ignoreLocalhost.k = "dns.ignoreLocalhost"; conf->dns.ignoreLocalhost.h = "Should FTL hide queries made by localhost?"; conf->dns.ignoreLocalhost.t = CONF_BOOL; - conf->dns.ignoreLocalhost.f = FLAG_ADVANCED_SETTING; conf->dns.ignoreLocalhost.d.b = false; conf->dns.ignoreLocalhost.c = validate_stub; // Only type-based checking conf->dns.showDNSSEC.k = "dns.showDNSSEC"; conf->dns.showDNSSEC.h = "Should FTL should analyze and show internally generated DNSSEC queries?"; conf->dns.showDNSSEC.t = CONF_BOOL; - conf->dns.showDNSSEC.f = FLAG_ADVANCED_SETTING; conf->dns.showDNSSEC.d.b = true; conf->dns.showDNSSEC.c = validate_stub; // Only type-based checking conf->dns.analyzeOnlyAandAAAA.k = "dns.analyzeOnlyAandAAAA"; conf->dns.analyzeOnlyAandAAAA.h = "Should FTL analyze *only* A and AAAA queries?"; conf->dns.analyzeOnlyAandAAAA.t = CONF_BOOL; - conf->dns.analyzeOnlyAandAAAA.f = FLAG_ADVANCED_SETTING; conf->dns.analyzeOnlyAandAAAA.d.b = false; conf->dns.analyzeOnlyAandAAAA.c = validate_stub; // Only type-based checking conf->dns.piholePTR.k = "dns.piholePTR"; - conf->dns.piholePTR.h = "Controls whether and how FTL will reply with for address for which a local interface exists."; + conf->dns.piholePTR.h = "Controls whether and how FTL will reply with for address for which a local interface exists. Changing this setting causes FTL to restart."; { struct enum_options piholePTR[] = { @@ -447,8 +454,8 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->dns.piholePTR.a, piholePTR); } conf->dns.piholePTR.t = CONF_ENUM_PTR_TYPE; - conf->dns.piholePTR.f = FLAG_ADVANCED_SETTING; conf->dns.piholePTR.d.ptr_type = PTR_PIHOLE; + conf->dns.piholePTR.f = FLAG_RESTART_FTL; conf->dns.piholePTR.c = validate_stub; // Only type-based checking conf->dns.replyWhenBusy.k = "dns.replyWhenBusy"; @@ -464,14 +471,12 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->dns.replyWhenBusy.a, replyWhenBusy); } conf->dns.replyWhenBusy.t = CONF_ENUM_BUSY_TYPE; - conf->dns.replyWhenBusy.f = FLAG_ADVANCED_SETTING; conf->dns.replyWhenBusy.d.busy_reply = BUSY_ALLOW; conf->dns.replyWhenBusy.c = validate_stub; // Only type-based checking conf->dns.blockTTL.k = "dns.blockTTL"; conf->dns.blockTTL.h = "FTL's internal TTL to be handed out for blocked queries in seconds. This settings allows users to select a value different from the dnsmasq config option local-ttl. This is useful in context of locally used hostnames that are known to stay constant over long times (printers, etc.).\n Note that large values may render whitelisting ineffective due to client-side caching of blocked queries."; conf->dns.blockTTL.t = CONF_UINT; - conf->dns.blockTTL.f = FLAG_ADVANCED_SETTING; conf->dns.blockTTL.d.ui = 2; conf->dns.blockTTL.c = validate_stub; // Only type-based checking @@ -479,21 +484,20 @@ void initConfig(struct config *conf) conf->dns.hosts.h = "Array of custom DNS records\n Example: hosts = [ \"127.0.0.1 mylocal\", \"192.168.0.1 therouter\" ]"; conf->dns.hosts.a = cJSON_CreateStringReference("Array of custom DNS records each one in HOSTS form: \"IP HOSTNAME\""); conf->dns.hosts.t = CONF_JSON_STRING_ARRAY; - conf->dns.hosts.f = FLAG_ADVANCED_SETTING; conf->dns.hosts.d.json = cJSON_CreateArray(); conf->dns.hosts.c = validate_dns_hosts; conf->dns.domainNeeded.k = "dns.domainNeeded"; conf->dns.domainNeeded.h = "If set, A and AAAA queries for plain names, without dots or domain parts, are never forwarded to upstream nameservers"; conf->dns.domainNeeded.t = CONF_BOOL; - conf->dns.domainNeeded.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.domainNeeded.f = FLAG_RESTART_FTL; conf->dns.domainNeeded.d.b = false; conf->dns.domainNeeded.c = validate_stub; // Only type-based checking conf->dns.expandHosts.k = "dns.expandHosts"; conf->dns.expandHosts.h = "If set, the domain is added to simple names (without a period) in /etc/hosts in the same way as for DHCP-derived names"; conf->dns.expandHosts.t = CONF_BOOL; - conf->dns.expandHosts.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.expandHosts.f = FLAG_RESTART_FTL; conf->dns.expandHosts.d.b = false; conf->dns.expandHosts.c = validate_stub; // Only type-based checking @@ -501,14 +505,14 @@ void initConfig(struct config *conf) conf->dns.domain.h = "The DNS domain used by your Pi-hole to expand hosts and for DHCP.\n\n Only if DHCP is enabled below: For DHCP, this has two effects; firstly it causes the DHCP server to return the domain to any hosts which request it, and secondly it sets the domain which it is legal for DHCP-configured hosts to claim. The intention is to constrain hostnames so that an untrusted host on the LAN cannot advertise its name via DHCP as e.g. \"google.com\" and capture traffic not meant for it. If no domain suffix is specified, then any DHCP hostname with a domain part (ie with a period) will be disallowed and logged. If a domain is specified, then hostnames with a domain part are allowed, provided the domain part matches the suffix. In addition, when a suffix is set then hostnames without a domain part have the suffix added as an optional domain part. For instance, we can set domain=mylab.com and have a machine whose DHCP hostname is \"laptop\". The IP address for that machine is available both as \"laptop\" and \"laptop.mylab.com\".\n\n You can disable setting a domain by setting this option to an empty string."; conf->dns.domain.a = cJSON_CreateStringReference(""); conf->dns.domain.t = CONF_STRING; - conf->dns.domain.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.domain.f = FLAG_RESTART_FTL; conf->dns.domain.d.s = (char*)"lan"; conf->dns.domain.c = validate_domain; conf->dns.bogusPriv.k = "dns.bogusPriv"; conf->dns.bogusPriv.h = "Should all reverse lookups for private IP ranges (i.e., 192.168.x.y, etc) which are not found in /etc/hosts or the DHCP leases file be answered with \"no such domain\" rather than being forwarded upstream?"; conf->dns.bogusPriv.t = CONF_BOOL; - conf->dns.bogusPriv.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.bogusPriv.f = FLAG_RESTART_FTL; conf->dns.bogusPriv.d.b = true; conf->dns.bogusPriv.c = validate_stub; // Only type-based checking @@ -523,7 +527,7 @@ void initConfig(struct config *conf) conf->dns.interface.h = "Interface to use for DNS (see also dnsmasq.listening.mode) and DHCP (if enabled)"; conf->dns.interface.a = cJSON_CreateStringReference("a valid interface name"); conf->dns.interface.t = CONF_STRING; - conf->dns.interface.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.interface.f = FLAG_RESTART_FTL; conf->dns.interface.d.s = (char*)""; conf->dns.interface.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -531,7 +535,7 @@ void initConfig(struct config *conf) conf->dns.hostRecord.h = "Add A, AAAA and PTR records to the DNS. This adds one or more names to the DNS with associated IPv4 (A) and IPv6 (AAAA) records"; conf->dns.hostRecord.a = cJSON_CreateStringReference("[,....],[],[][,]"); conf->dns.hostRecord.t = CONF_STRING; - conf->dns.hostRecord.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.hostRecord.f = FLAG_RESTART_FTL; conf->dns.hostRecord.d.s = (char*)""; conf->dns.hostRecord.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -549,7 +553,7 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->dns.listeningMode.a, listeningMode); } conf->dns.listeningMode.t = CONF_ENUM_LISTENING_MODE; - conf->dns.listeningMode.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.listeningMode.f = FLAG_RESTART_FTL; conf->dns.listeningMode.d.listeningMode = LISTEN_LOCAL; conf->dns.listeningMode.c = validate_stub; // Only type-based checking @@ -564,14 +568,14 @@ void initConfig(struct config *conf) conf->dns.cnameRecords.h = "List of CNAME records which indicate that is really . If the is given, it overwrites the value of local-ttl"; conf->dns.cnameRecords.a = cJSON_CreateStringReference("Array of CNAMEs each on in one of the following forms: \",[,]\""); conf->dns.cnameRecords.t = CONF_JSON_STRING_ARRAY; - conf->dns.cnameRecords.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.cnameRecords.f = FLAG_RESTART_FTL; conf->dns.cnameRecords.d.json = cJSON_CreateArray(); conf->dns.cnameRecords.c = validate_dns_cnames; conf->dns.port.k = "dns.port"; conf->dns.port.h = "Port used by the DNS server"; conf->dns.port.t = CONF_UINT16; - conf->dns.port.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.port.f = FLAG_RESTART_FTL; conf->dns.port.d.ui = 53u; conf->dns.port.c = validate_stub; // Only type-based checking @@ -579,17 +583,23 @@ void initConfig(struct config *conf) conf->dns.cache.size.k = "dns.cache.size"; conf->dns.cache.size.h = "Cache size of the DNS server. Note that expiring cache entries naturally make room for new insertions over time. Setting this number too high will have an adverse effect as not only more space is needed, but also lookup speed gets degraded in the 10,000+ range. dnsmasq may issue a warning when you go beyond 10,000+ cache entries."; conf->dns.cache.size.t = CONF_UINT; - conf->dns.cache.size.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.cache.size.f = FLAG_RESTART_FTL; conf->dns.cache.size.d.ui = 10000u; conf->dns.cache.size.c = validate_stub; // Only type-based checking conf->dns.cache.optimizer.k = "dns.cache.optimizer"; conf->dns.cache.optimizer.h = "Query cache optimizer: If a DNS name exists in the cache, but its time-to-live has expired only recently, the data will be used anyway (a refreshing from upstream is triggered). This can improve DNS query delays especially over unreliable Internet connections. This feature comes at the expense of possibly sometimes returning out-of-date data and less efficient cache utilization, since old data cannot be flushed when its TTL expires, so the cache becomes mostly least-recently-used. To mitigate issues caused by massively outdated DNS replies, the maximum overaging of cached records is limited. We strongly recommend staying below 86400 (1 day) with this option.\n Setting the TTL excess time to zero will serve stale cache data regardless how long it has expired. This is not recommended as it may lead to stale data being served for a long time. Setting this option to any negative value will disable this feature altogether."; conf->dns.cache.optimizer.t = CONF_INT; - conf->dns.cache.optimizer.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.cache.optimizer.f = FLAG_RESTART_FTL; conf->dns.cache.optimizer.d.i = 3600u; conf->dns.cache.optimizer.c = validate_stub; // Only type-based checking + conf->dns.cache.upstreamBlockedTTL.k = "dns.cache.upstreamBlockedTTL"; + conf->dns.cache.upstreamBlockedTTL.h = "This setting allows you to specify the TTL used for queries blocked upstream. Once the TTL expires, the query will be forwarded to the upstream server again to check if the block is still valid. Defaults to caching for one day (86400 seconds). Setting this value to zero disables caching of queries blocked upstream."; + conf->dns.cache.upstreamBlockedTTL.t = CONF_UINT; + conf->dns.cache.upstreamBlockedTTL.d.ui = 86400; + conf->dns.cache.upstreamBlockedTTL.c = validate_stub; // Only type-based checking + // sub-struct dns.blocking conf->dns.blocking.active.k = "dns.blocking.active"; conf->dns.blocking.active.h = "Should FTL block queries?"; @@ -614,6 +624,21 @@ void initConfig(struct config *conf) conf->dns.blocking.mode.d.blocking_mode = MODE_NULL; conf->dns.blocking.mode.c = validate_stub; // Only type-based checking + conf->dns.blocking.edns.k = "dns.blocking.edns"; + conf->dns.blocking.edns.h = "Should FTL enrich blocked replies with EDNS0 information?"; + { + struct enum_options blocking_edns[] = + { + { get_edns_mode_str(EDNS_MODE_NONE), "In NONE mode, no additional EDNS information is added to blocked queries" }, + { get_edns_mode_str(EDNS_MODE_CODE), "In CODE mode, blocked queries will be enriched with EDNS info-code BLOCKED (15)" }, + { get_edns_mode_str(EDNS_MODE_TEXT), "In TEXT mode, blocked queries will be enriched with EDNS info-code BLOCKED (15) and a text message describing the reason for the block" } + }; + CONFIG_ADD_ENUM_OPTIONS(conf->dns.blocking.edns.a, blocking_edns); + } + conf->dns.blocking.edns.t = CONF_ENUM_BLOCKING_EDNS_MODE; + conf->dns.blocking.edns.d.edns_mode = EDNS_MODE_TEXT; + conf->dns.blocking.edns.c = validate_stub; // Only type-based checking + conf->dns.revServers.k = "dns.revServers"; conf->dns.revServers.h = "Reverse server (former also called \"conditional forwarding\") feature\n Array of reverse servers each one in one of the following forms: \",[/],[#],\"\n\n Individual components:\n\n : either \"true\" or \"false\"\n\n [/]: Address range for the reverse server feature in CIDR notation. If the prefix length is omitted, either 32 (IPv4) or 128 (IPv6) are substituted (exact address match). This is almost certainly not what you want here.\n Example: \"192.168.0.0/24\" for the range 192.168.0.1 - 192.168.0.255\n\n [#]: Target server to be used for the reverse server feature\n Example: \"192.168.0.1#53\"\n\n : Domain used for the reverse server feature (e.g., \"fritz.box\")\n Example: \"fritz.box\""; conf->dns.revServers.a = cJSON_CreateStringReference("array of reverse servers each one in one of the following forms: \",[/],[#],\", e.g., \"true,192.168.0.0/24,192.168.0.1,fritz.box\""); @@ -652,7 +677,6 @@ void initConfig(struct config *conf) conf->dns.reply.host.force4.k = "dns.reply.host.force4"; conf->dns.reply.host.force4.h = "Use a specific IPv4 address for the Pi-hole host? By default, FTL determines the address of the interface a query arrived on and uses this address for replying to A queries with the most suitable address for the requesting client. This setting can be used to use a fixed, rather than the dynamically obtained, address when Pi-hole responds to the following names: [ \"pi.hole\", \"\", \"pi.hole.\", \".\" ]"; conf->dns.reply.host.force4.t = CONF_BOOL; - conf->dns.reply.host.force4.f = FLAG_ADVANCED_SETTING; conf->dns.reply.host.force4.d.b = false; conf->dns.reply.host.force4.c = validate_stub; // Only type-based checking @@ -660,14 +684,12 @@ void initConfig(struct config *conf) conf->dns.reply.host.v4.h = "Custom IPv4 address for the Pi-hole host"; conf->dns.reply.host.v4.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.host.v4.t = CONF_STRUCT_IN_ADDR; - conf->dns.reply.host.v4.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.host.v4.d.in_addr, 0, sizeof(struct in_addr)); conf->dns.reply.host.v4.c = validate_stub; // Only type-based checking conf->dns.reply.host.force6.k = "dns.reply.host.force6"; conf->dns.reply.host.force6.h = "Use a specific IPv6 address for the Pi-hole host? See description for the IPv4 variant above for further details."; conf->dns.reply.host.force6.t = CONF_BOOL; - conf->dns.reply.host.force6.f = FLAG_ADVANCED_SETTING; conf->dns.reply.host.force6.d.b = false; conf->dns.reply.host.force6.c = validate_stub; // Only type-based checking @@ -675,14 +697,12 @@ void initConfig(struct config *conf) conf->dns.reply.host.v6.h = "Custom IPv6 address for the Pi-hole host"; conf->dns.reply.host.v6.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.host.v6.t = CONF_STRUCT_IN6_ADDR; - conf->dns.reply.host.v6.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.host.v6.d.in6_addr, 0, sizeof(struct in6_addr)); conf->dns.reply.host.v6.c = validate_stub; // Only type-based checking conf->dns.reply.blocking.force4.k = "dns.reply.blocking.force4"; conf->dns.reply.blocking.force4.h = "Use a specific IPv4 address in IP blocking mode? By default, FTL determines the address of the interface a query arrived on and uses this address for replying to A queries with the most suitable address for the requesting client. This setting can be used to use a fixed, rather than the dynamically obtained, address when Pi-hole responds in the following cases: IP blocking mode is used and this query is to be blocked, regular expressions with the ;reply=IP regex extension."; conf->dns.reply.blocking.force4.t = CONF_BOOL; - conf->dns.reply.blocking.force4.f = FLAG_ADVANCED_SETTING; conf->dns.reply.blocking.force4.d.b = false; conf->dns.reply.blocking.force4.c = validate_stub; // Only type-based checking @@ -690,14 +710,12 @@ void initConfig(struct config *conf) conf->dns.reply.blocking.v4.h = "Custom IPv4 address for IP blocking mode"; conf->dns.reply.blocking.v4.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.blocking.v4.t = CONF_STRUCT_IN_ADDR; - conf->dns.reply.blocking.v4.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.blocking.v4.d.in_addr, 0, sizeof(struct in_addr)); conf->dns.reply.blocking.v4.c = validate_stub; // Only type-based checking conf->dns.reply.blocking.force6.k = "dns.reply.blocking.force6"; conf->dns.reply.blocking.force6.h = "Use a specific IPv6 address in IP blocking mode? See description for the IPv4 variant above for further details."; conf->dns.reply.blocking.force6.t = CONF_BOOL; - conf->dns.reply.blocking.force6.f = FLAG_ADVANCED_SETTING; conf->dns.reply.blocking.force6.d.b = false; conf->dns.reply.blocking.force6.c = validate_stub; // Only type-based checking @@ -705,7 +723,6 @@ void initConfig(struct config *conf) conf->dns.reply.blocking.v6.h = "Custom IPv6 address for IP blocking mode"; conf->dns.reply.blocking.v6.a = cJSON_CreateStringReference(" or empty string (\"\")"); conf->dns.reply.blocking.v6.t = CONF_STRUCT_IN6_ADDR; - conf->dns.reply.blocking.v6.f = FLAG_ADVANCED_SETTING; memset(&conf->dns.reply.blocking.v6.d.in6_addr, 0, sizeof(struct in6_addr)); conf->dns.reply.blocking.v6.c = validate_stub; // Only type-based checking @@ -745,7 +762,7 @@ void initConfig(struct config *conf) conf->dhcp.netmask.h = "The netmask used by your Pi-hole. For directly connected networks (i.e., networks on which the machine running Pi-hole has an interface) the netmask is optional and may be set to an empty string (\"\"): it will then be determined from the interface configuration itself. For networks which receive DHCP service via a relay agent, we cannot determine the netmask itself, so it should explicitly be specified, otherwise Pi-hole guesses based on the class (A, B or C) of the network address."; conf->dhcp.netmask.a = cJSON_CreateStringReference(" (e.g., \"255.255.255.0\") or empty string (\"\") for auto-discovery"); conf->dhcp.netmask.t = CONF_STRUCT_IN_ADDR; - conf->dhcp.netmask.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dhcp.netmask.f = FLAG_RESTART_FTL; memset(&conf->dhcp.netmask.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.netmask.c = validate_stub; // Only type-based checking @@ -753,7 +770,7 @@ void initConfig(struct config *conf) conf->dhcp.leaseTime.h = "If the lease time is given, then leases will be given for that length of time. If not given, the default lease time is one hour for IPv4 and one day for IPv6."; conf->dhcp.leaseTime.a = cJSON_CreateStringReference("The lease time can be in seconds, or minutes (e.g., \"45m\") or hours (e.g., \"1h\") or days (like \"2d\") or even weeks (\"1w\"). You may also use \"infinite\" as string but be aware of the drawbacks"); conf->dhcp.leaseTime.t = CONF_STRING; - conf->dhcp.leaseTime.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dhcp.leaseTime.f = FLAG_RESTART_FTL; conf->dhcp.leaseTime.d.s = (char*)""; conf->dhcp.leaseTime.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -785,15 +802,99 @@ void initConfig(struct config *conf) conf->dhcp.logging.d.b = false; conf->dhcp.logging.c = validate_stub; // Only type-based checking + conf->dhcp.ignoreUnknownClients.k = "dhcp.ignoreUnknownClients"; + conf->dhcp.ignoreUnknownClients.h = "Ignore unknown DHCP clients.\n If this option is set, Pi-hole ignores all clients which are not explicitly configured through dhcp.hosts. This can be useful to prevent unauthorized clients from getting an IP address from the DHCP server.\n It should be noted that this option is not a security feature, as clients can still assign themselves an IP address and use the network. It is merely a convenience feature to prevent unknown clients from getting a valid IP configuration assigned automatically.\n Note that you will need to configure new clients manually in dhcp.hosts before they can use the network when this feature is enabled."; + conf->dhcp.ignoreUnknownClients.t = CONF_BOOL; + conf->dhcp.ignoreUnknownClients.f = FLAG_RESTART_FTL; + conf->dhcp.ignoreUnknownClients.d.b = false; + conf->dhcp.ignoreUnknownClients.c = validate_stub; // Only type-based checking + conf->dhcp.hosts.k = "dhcp.hosts"; conf->dhcp.hosts.h = "Per host parameters for the DHCP server. This allows a machine with a particular hardware address to be always allocated the same hostname, IP address and lease time or to specify static DHCP leases"; conf->dhcp.hosts.a = cJSON_CreateStringReference("Array of static leases each on in one of the following forms: \"[][,id:|*][,set:][,tag:][,][,][,][,ignore]\""); conf->dhcp.hosts.t = CONF_JSON_STRING_ARRAY; - conf->dhcp.hosts.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dhcp.hosts.f = FLAG_RESTART_FTL; conf->dhcp.hosts.d.json = cJSON_CreateArray(); conf->dhcp.hosts.c = validate_stub; // Type-based checking + dnsmasq syntax checking + // struct ntp + conf->ntp.ipv4.active.k = "ntp.ipv4.active"; + conf->ntp.ipv4.active.h = "Should FTL act as network time protocol (NTP) server (IPv4)?"; + conf->ntp.ipv4.active.t = CONF_BOOL; + conf->ntp.ipv4.active.f = FLAG_RESTART_FTL; + conf->ntp.ipv4.active.d.b = true; + conf->ntp.ipv4.active.c = validate_stub; // Only type-based checking + + conf->ntp.ipv4.address.k = "ntp.ipv4.address"; + conf->ntp.ipv4.address.h = "IPv4 address to listen on for NTP requests"; + conf->ntp.ipv4.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard (0.0.0.0)"); + conf->ntp.ipv4.address.t = CONF_STRUCT_IN_ADDR; + conf->ntp.ipv4.address.f = FLAG_RESTART_FTL; + memset(&conf->ntp.ipv4.address.d.in_addr, 0, sizeof(struct in_addr)); + conf->ntp.ipv4.address.c = validate_stub; // Only type-based checking + + conf->ntp.ipv6.active.k = "ntp.ipv6.active"; + conf->ntp.ipv6.active.h = "Should FTL act as network time protocol (NTP) server (IPv6)?"; + conf->ntp.ipv6.active.t = CONF_BOOL; + conf->ntp.ipv6.active.f = FLAG_RESTART_FTL; + conf->ntp.ipv6.active.d.b = true; + conf->ntp.ipv6.active.c = validate_stub; // Only type-based checking + + conf->ntp.ipv6.address.k = "ntp.ipv6.address"; + conf->ntp.ipv6.address.h = "IPv6 address to listen on for NTP requests"; + conf->ntp.ipv6.address.a = cJSON_CreateStringReference(" or empty string (\"\") for wildcard (::)"); + conf->ntp.ipv6.address.t = CONF_STRUCT_IN6_ADDR; + conf->ntp.ipv6.address.f = FLAG_RESTART_FTL; + memset(&conf->ntp.ipv6.address.d.in6_addr, 0, sizeof(struct in6_addr)); + conf->ntp.ipv6.address.c = validate_stub; // Only type-based checking + + conf->ntp.sync.active.k = "ntp.sync.active"; + conf->ntp.sync.active.h = "Should FTL try to synchronize the system time with an upstream NTP server?"; + conf->ntp.sync.active.t = CONF_BOOL; + conf->ntp.sync.active.f = FLAG_RESTART_FTL; + conf->ntp.sync.active.d.b = true; + conf->ntp.sync.active.c = validate_stub; // Only type-based checking + + conf->ntp.sync.server.k = "ntp.sync.server"; + conf->ntp.sync.server.h = "NTP upstream server to sync with, e.g., \"pool.ntp.org\". Note that the NTP server should be located as close as possible to you in order to minimize the time offset possibly introduced by different routing paths."; + conf->ntp.sync.server.a = cJSON_CreateStringReference("valid NTP upstream server"); + conf->ntp.sync.server.t = CONF_STRING; + conf->ntp.sync.server.d.s = (char*)"pool.ntp.org"; + conf->ntp.sync.server.c = validate_stub; // Only type-based checking + + conf->ntp.sync.interval.k = "ntp.sync.interval"; + conf->ntp.sync.interval.h = "Interval in seconds between successive synchronization attempts with the NTP server"; + conf->ntp.sync.interval.t = CONF_UINT; + conf->ntp.sync.interval.d.ui = 3600; + conf->ntp.sync.interval.c = validate_stub; // Only type-based checking + + conf->ntp.sync.count.k = "ntp.sync.count"; + conf->ntp.sync.count.h = "Number of NTP syncs to perform and average before updating the system time"; + conf->ntp.sync.count.t = CONF_UINT; + conf->ntp.sync.count.d.ui = 8; + conf->ntp.sync.count.c = validate_stub; // Only type-based checking + + conf->ntp.sync.rtc.set.k = "ntp.sync.rtc.set"; + conf->ntp.sync.rtc.set.h = "Should FTL update a real-time clock (RTC) if available?"; + conf->ntp.sync.rtc.set.t = CONF_BOOL; + conf->ntp.sync.rtc.set.d.b = true; + conf->ntp.sync.rtc.set.c = validate_stub; // Only type-based checking + + conf->ntp.sync.rtc.device.k = "ntp.sync.rtc.device"; + conf->ntp.sync.rtc.device.h = "Path to the RTC device to update. Leave empty for auto-discovery"; + conf->ntp.sync.rtc.device.a = cJSON_CreateStringReference("Path to the RTC device, e.g., \"/dev/rtc0\""); + conf->ntp.sync.rtc.device.t = CONF_STRING; + conf->ntp.sync.rtc.device.d.s = (char*)""; + conf->ntp.sync.rtc.device.c = validate_stub; // Only type-based checking + + conf->ntp.sync.rtc.utc.k = "ntp.sync.rtc.utc"; + conf->ntp.sync.rtc.utc.h = "Should the RTC be set to UTC?"; + conf->ntp.sync.rtc.utc.t = CONF_BOOL; + conf->ntp.sync.rtc.utc.d.b = true; + conf->ntp.sync.rtc.utc.c = validate_stub; // Only type-based checking + + // struct resolver conf->resolver.resolveIPv6.k = "resolver.resolveIPv6"; conf->resolver.resolveIPv6.h = "Should FTL try to resolve IPv6 addresses to hostnames?"; @@ -810,7 +911,6 @@ void initConfig(struct config *conf) conf->resolver.networkNames.k = "resolver.networkNames"; conf->resolver.networkNames.h = "Control whether FTL should use the fallback option to try to obtain client names from checking the network table. This behavior can be disabled with this option.\n Assume an IPv6 client without a host names. However, the network table knows - though the client's MAC address - that this is the same device where we have a host name for another IP address (e.g., a DHCP server managed IPv4 address). In this case, we use the host name associated to the other address as this is the same device."; conf->resolver.networkNames.t = CONF_BOOL; - conf->resolver.networkNames.f = FLAG_ADVANCED_SETTING; conf->resolver.networkNames.d.b = true; conf->resolver.networkNames.c = validate_stub; // Only type-based checking @@ -827,7 +927,6 @@ void initConfig(struct config *conf) CONFIG_ADD_ENUM_OPTIONS(conf->resolver.refreshNames.a, refreshNames); } conf->resolver.refreshNames.t = CONF_ENUM_REFRESH_HOSTNAMES; - conf->resolver.refreshNames.f = FLAG_ADVANCED_SETTING; conf->resolver.refreshNames.d.refresh_hostnames = REFRESH_IPV4_ONLY; conf->resolver.refreshNames.c = validate_stub; // Only type-based checking @@ -865,7 +964,7 @@ void initConfig(struct config *conf) // loss). The gravity database is also not affected as it is only written // to on an individual basis (explicit API calls) and not continuously // (like the query database). - conf->database.useWAL.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->database.useWAL.f = FLAG_RESTART_FTL; conf->database.useWAL.d.b = true; conf->database.useWAL.c = validate_stub; // Only type-based checking @@ -873,14 +972,12 @@ void initConfig(struct config *conf) conf->database.network.parseARPcache.k = "database.network.parseARPcache"; conf->database.network.parseARPcache.h = "Should FTL analyze the local ARP cache? When disabled, client identification and the network table will stop working reliably."; conf->database.network.parseARPcache.t = CONF_BOOL; - conf->database.network.parseARPcache.f = FLAG_ADVANCED_SETTING; conf->database.network.parseARPcache.d.b = true; conf->database.network.parseARPcache.c = validate_stub; // Only type-based checking conf->database.network.expire.k = "database.network.expire"; conf->database.network.expire.h = "How long should IP addresses be kept in the network_addresses table [days]? IP addresses (and associated host names) older than the specified number of days are removed to avoid dead entries in the network overview table."; conf->database.network.expire.t = CONF_UINT; - conf->database.network.expire.f = FLAG_ADVANCED_SETTING; conf->database.network.expire.d.ui = conf->database.maxDBdays.d.ui; conf->database.network.expire.c = validate_stub; // Only type-based checking @@ -890,14 +987,14 @@ void initConfig(struct config *conf) conf->webserver.domain.h = "On which domain is the web interface served?"; conf->webserver.domain.a = cJSON_CreateStringReference(""); conf->webserver.domain.t = CONF_STRING; - conf->webserver.domain.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.domain.f = FLAG_RESTART_FTL; conf->webserver.domain.d.s = (char*)"pi.hole"; conf->webserver.domain.c = validate_domain; conf->webserver.acl.k = "webserver.acl"; conf->webserver.acl.h = "Webserver access control list (ACL) allowing for restrictions to be put on the list of IP addresses which have access to the web server. The ACL is a comma separated list of IP subnets, where each subnet is prepended by either a - or a + sign. A plus sign means allow, where a minus sign means deny. If a subnet mask is omitted, such as -1.2.3.4, this means to deny only that single IP address. If this value is not set (empty string), all accesses are allowed. Otherwise, the default setting is to deny all accesses. On each request the full list is traversed, and the last (!) match wins. IPv6 addresses may be specified in CIDR-form [a:b::c]/64.\n\n Example 1: acl = \"+127.0.0.1,+[::1]\"\n ---> deny all access, except from 127.0.0.1 and ::1,\n Example 2: acl = \"+192.168.0.0/16\"\n ---> deny all accesses, except from the 192.168.0.0/16 subnet,\n Example 3: acl = \"+[::]/0\" ---> allow only IPv6 access."; conf->webserver.acl.a = cJSON_CreateStringReference(""); - conf->webserver.acl.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.acl.f = FLAG_RESTART_FTL; conf->webserver.acl.t = CONF_STRING; conf->webserver.acl.d.s = (char*)""; conf->webserver.acl.c = validate_stub; // Type-based checking + civetweb syntax checking @@ -905,22 +1002,15 @@ void initConfig(struct config *conf) conf->webserver.port.k = "webserver.port"; conf->webserver.port.h = "Ports to be used by the webserver.\n Comma-separated list of ports to listen on. It is possible to specify an IP address to bind to. In this case, an IP address and a colon must be prepended to the port number. For example, to bind to the loopback interface on port 80 (IPv4) and to all interfaces port 8080 (IPv4), use \"127.0.0.1:80,8080\". \"[::]:80\" can be used to listen to IPv6 connections to port 80. IPv6 addresses of network interfaces can be specified as well, e.g. \"[::1]:80\" for the IPv6 loopback interface. [::]:80 will bind to port 80 IPv6 only.\n In order to use port 80 for all interfaces, both IPv4 and IPv6, use either the configuration \"80,[::]:80\" (create one socket for IPv4 and one for IPv6 only), or \"+80\" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 and IPv6 will only work if no network interface is specified. Depending on your operating system version and IPv6 network environment, some configurations might not work as expected, so you have to test to find the configuration most suitable for your needs. In case \"+80\" does not work for your environment, you need to use \"80,[::]:80\".\n If the port is TLS/SSL, a letter 's' must be appended, for example, \"80,443s\" will open port 80 and port 443, and connections on port 443 will be encrypted. For non-encrypted ports, it is allowed to append letter 'r' (as in redirect). Redirected ports will redirect all their traffic to the first configured SSL port. For example, if webserver.port is \"80r,443s\", then all HTTP traffic coming at port 80 will be redirected to HTTPS port 443. If this value is not set (empty string), the web server will not be started and, hence, the API will not be available."; conf->webserver.port.a = cJSON_CreateStringReference("comma-separated list of <[ip_address:]port>"); - conf->webserver.port.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.port.f = FLAG_RESTART_FTL; conf->webserver.port.t = CONF_STRING; conf->webserver.port.d.s = (char*)"80,[::]:80,443s,[::]:443s"; conf->webserver.port.c = validate_stub; // Type-based checking + civetweb syntax checking - conf->webserver.tls.rev_proxy.k = "webserver.tls.rev_proxy"; - conf->webserver.tls.rev_proxy.h = "Is Pi-hole running behind a reverse proxy? If yes, Pi-hole will not consider HTTP-only connections being insecure. This is useful if you are running Pi-hole in a trusted environment, for example, in a local network, and you are using a reverse proxy to provide TLS encryption, e.g., by using Traefik (docker). If you are using a reverse proxy, you can alternatively set webserver.tls.cert to the path of the TLS certificate file and let Pi-hole handle true end-to-end encryption."; - conf->webserver.tls.rev_proxy.f = FLAG_ADVANCED_SETTING; - conf->webserver.tls.rev_proxy.t = CONF_BOOL; - conf->webserver.tls.rev_proxy.d.b = false; - conf->webserver.tls.rev_proxy.c = validate_stub; // Only type-based checking - conf->webserver.tls.cert.k = "webserver.tls.cert"; conf->webserver.tls.cert.h = "Path to the TLS (SSL) certificate file. This option is only required when at least one of webserver.port is TLS. The file must be in PEM format, and it must have both, private key and certificate (the *.pem file created must contain a 'CERTIFICATE' section as well as a 'RSA PRIVATE KEY' section).\n The *.pem file can be created using\n cp server.crt server.pem\n cat server.key >> server.pem\n if you have these files instead"; conf->webserver.tls.cert.a = cJSON_CreateStringReference(""); - conf->webserver.tls.cert.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.tls.cert.f = FLAG_RESTART_FTL; conf->webserver.tls.cert.t = CONF_STRING; conf->webserver.tls.cert.d.s = (char*)"/etc/pihole/tls.pem"; conf->webserver.tls.cert.c = validate_filepath; @@ -942,7 +1032,7 @@ void initConfig(struct config *conf) conf->webserver.paths.webroot.h = "Server root on the host"; conf->webserver.paths.webroot.a = cJSON_CreateStringReference(""); conf->webserver.paths.webroot.t = CONF_STRING; - conf->webserver.paths.webroot.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.paths.webroot.f = FLAG_RESTART_FTL; conf->webserver.paths.webroot.d.s = (char*)"/var/www/html"; conf->webserver.paths.webroot.c = validate_filepath; @@ -950,7 +1040,7 @@ void initConfig(struct config *conf) conf->webserver.paths.webhome.h = "Sub-directory of the root containing the web interface"; conf->webserver.paths.webhome.a = cJSON_CreateStringReference(", both slashes are needed!"); conf->webserver.paths.webhome.t = CONF_STRING; - conf->webserver.paths.webhome.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.paths.webhome.f = FLAG_RESTART_FTL; conf->webserver.paths.webhome.d.s = (char*)"/admin/"; conf->webserver.paths.webhome.c = validate_filepath; @@ -977,29 +1067,16 @@ void initConfig(struct config *conf) conf->webserver.interface.theme.c = validate_stub; // Only type-based checking // sub-struct api - conf->webserver.api.searchAPIauth.k = "webserver.api.searchAPIauth"; - conf->webserver.api.searchAPIauth.h = "Do local clients need to authenticate to access the search API? This settings allows local clients to use pihole -q ... without authentication. Note that \"local\" in the sense of the option means only 127.0.0.1 and [::1]"; - conf->webserver.api.searchAPIauth.t = CONF_BOOL; - conf->webserver.api.searchAPIauth.d.b = false; - conf->webserver.api.searchAPIauth.c = validate_stub; // Only type-based checking - - conf->webserver.api.localAPIauth.k = "webserver.api.localAPIauth"; - conf->webserver.api.localAPIauth.h = "Do local clients need to authenticate to access the API? This settings allows local clients to use the API without authentication."; - conf->webserver.api.localAPIauth.t = CONF_BOOL; - conf->webserver.api.localAPIauth.d.b = true; - conf->webserver.api.localAPIauth.c = validate_stub; // Only type-based checking - conf->webserver.api.max_sessions.k = "webserver.api.max_sessions"; conf->webserver.api.max_sessions.h = "Number of concurrent sessions allowed for the API. If the number of sessions exceeds this value, no new sessions will be allowed until the number of sessions drops due to session expiration or logout. Note that the number of concurrent sessions is irrelevant if authentication is disabled as no sessions are used in this case."; conf->webserver.api.max_sessions.t = CONF_UINT16; conf->webserver.api.max_sessions.d.u16 = 16; - conf->webserver.api.max_sessions.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.api.max_sessions.f = FLAG_RESTART_FTL; conf->webserver.api.max_sessions.c = validate_stub; // Only type-based checking conf->webserver.api.prettyJSON.k = "webserver.api.prettyJSON"; conf->webserver.api.prettyJSON.h = "Should FTL prettify the API output (add extra spaces, newlines and indentation)?"; conf->webserver.api.prettyJSON.t = CONF_BOOL; - conf->webserver.api.prettyJSON.f = FLAG_ADVANCED_SETTING; conf->webserver.api.prettyJSON.d.b = false; conf->webserver.api.prettyJSON.c = validate_stub; // Only type-based checking @@ -1035,6 +1112,19 @@ void initConfig(struct config *conf) conf->webserver.api.app_pwhash.d.s = (char*)""; conf->webserver.api.app_pwhash.c = validate_stub; // Only type-based checking + conf->webserver.api.app_sudo.k = "webserver.api.app_sudo"; + conf->webserver.api.app_sudo.h = "Should application password API sessions be allowed to modify config settings?\n Setting this to true allows third-party applications using the application password to modify settings, e.g., the upstream DNS servers, DHCP server settings, or changing passwords. This setting should only be enabled if really needed and only if you trust the applications using the application password."; + conf->webserver.api.app_sudo.t = CONF_BOOL; + conf->webserver.api.app_sudo.d.b = false; + conf->webserver.api.app_sudo.c = validate_stub; // Only type-based checking + + conf->webserver.api.cli_pw.k = "webserver.api.cli_pw"; + conf->webserver.api.cli_pw.h = "Should FTL create a temporary CLI password? This password is stored in clear in /etc/pihole and can be used by the CLI (pihole ... commands) to authenticate against the API. Note that the password is only valid for the current session and regenerated on each FTL restart. Sessions initiated with this password cannot modify the Pi-hole configuration (change passwords, etc.) for security reasons but can still use the API to query data and manage lists."; + conf->webserver.api.cli_pw.t = CONF_BOOL; + conf->webserver.api.cli_pw.f = FLAG_RESTART_FTL; + conf->webserver.api.cli_pw.d.b = true; + conf->webserver.api.cli_pw.c = validate_stub; // Only type-based checking + conf->webserver.api.excludeClients.k = "webserver.api.excludeClients"; conf->webserver.api.excludeClients.h = "Array of clients to be excluded from certain API responses (regex):\n - Query Log (/api/queries)\n - Top Clients (/api/stats/top_clients)\n This setting accepts both IP addresses (IPv4 and IPv6) as well as hostnames.\n Note that backslashes \"\\\" need to be escaped, i.e. \"\\\\\" in this setting\n\n Example: [ \"^192\\\\.168\\\\.2\\\\.56$\", \"^fe80::341:[0-9a-f]*$\", \"^localhost$\" ]"; conf->webserver.api.excludeClients.a = cJSON_CreateStringReference("array of regular expressions describing clients"); @@ -1100,7 +1190,7 @@ void initConfig(struct config *conf) conf->files.pid.h = "The file which contains the PID of FTL's main process."; conf->files.pid.a = cJSON_CreateStringReference(""); conf->files.pid.t = CONF_STRING; - conf->files.pid.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.pid.f = FLAG_RESTART_FTL; conf->files.pid.d.s = (char*)"/run/pihole-FTL.pid"; conf->files.pid.c = validate_filepath; @@ -1108,7 +1198,6 @@ void initConfig(struct config *conf) conf->files.database.h = "The location of FTL's long-term database"; conf->files.database.a = cJSON_CreateStringReference(""); conf->files.database.t = CONF_STRING; - conf->files.database.f = FLAG_ADVANCED_SETTING; conf->files.database.d.s = (char*)"/etc/pihole/pihole-FTL.db"; conf->files.database.c = validate_filepath; @@ -1116,7 +1205,7 @@ void initConfig(struct config *conf) conf->files.gravity.h = "The location of Pi-hole's gravity database"; conf->files.gravity.a = cJSON_CreateStringReference(""); conf->files.gravity.t = CONF_STRING; - conf->files.gravity.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.gravity.f = FLAG_RESTART_FTL; conf->files.gravity.d.s = (char*)"/etc/pihole/gravity.db"; conf->files.gravity.c = validate_filepath; @@ -1124,7 +1213,7 @@ void initConfig(struct config *conf) conf->files.gravity_tmp.h = "A temporary directory where Pi-hole can store files during gravity updates. This directory must be writable by the user running gravity (typically pihole)."; conf->files.gravity_tmp.a = cJSON_CreateStringReference(""); conf->files.gravity_tmp.t = CONF_STRING; - conf->files.gravity_tmp.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.gravity_tmp.f = FLAG_RESTART_FTL; conf->files.gravity_tmp.d.s = (char*)"/tmp"; conf->files.gravity_tmp.c = validate_stub; // Only type-based checking @@ -1132,7 +1221,6 @@ void initConfig(struct config *conf) conf->files.macvendor.h = "The database containing MAC -> Vendor information for the network table"; conf->files.macvendor.a = cJSON_CreateStringReference(""); conf->files.macvendor.t = CONF_STRING; - conf->files.macvendor.f = FLAG_ADVANCED_SETTING; conf->files.macvendor.d.s = (char*)"/etc/pihole/macvendor.db"; conf->files.macvendor.c = validate_filepath; @@ -1140,7 +1228,6 @@ void initConfig(struct config *conf) conf->files.setupVars.h = "The old config file of Pi-hole used before v6.0"; conf->files.setupVars.a = cJSON_CreateStringReference(""); conf->files.setupVars.t = CONF_STRING; - conf->files.setupVars.f = FLAG_ADVANCED_SETTING; conf->files.setupVars.d.s = (char*)"/etc/pihole/setupVars.conf"; conf->files.setupVars.c = validate_filepath; @@ -1148,7 +1235,7 @@ void initConfig(struct config *conf) conf->files.pcap.h = "An optional file containing a pcap capture of the network traffic. This file is used for debugging purposes only. If you don't know what this is, you don't need it.\n Setting this to an empty string disables pcap recording. The file must be writable by the user running FTL (typically pihole). Failure to write to this file will prevent the DNS resolver from starting. The file is appended to if it already exists."; conf->files.pcap.a = cJSON_CreateStringReference(""); conf->files.pcap.t = CONF_STRING; - conf->files.pcap.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.pcap.f = FLAG_RESTART_FTL; conf->files.pcap.d.s = (char*)""; conf->files.pcap.c = validate_filepath_empty; @@ -1159,7 +1246,7 @@ void initConfig(struct config *conf) conf->files.log.webserver.h = "The log file used by the webserver"; conf->files.log.webserver.a = cJSON_CreateStringReference(""); conf->files.log.webserver.t = CONF_STRING; - conf->files.log.webserver.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.log.webserver.f = FLAG_RESTART_FTL; conf->files.log.webserver.d.s = (char*)"/var/log/pihole/webserver.log"; conf->files.log.webserver.c = validate_filepath; @@ -1167,7 +1254,7 @@ void initConfig(struct config *conf) conf->files.log.dnsmasq.h = "The log file used by the embedded dnsmasq DNS server"; conf->files.log.dnsmasq.a = cJSON_CreateStringReference(""); conf->files.log.dnsmasq.t = CONF_STRING; - conf->files.log.dnsmasq.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.log.dnsmasq.f = FLAG_RESTART_FTL; conf->files.log.dnsmasq.d.s = (char*)"/var/log/pihole/pihole.log"; conf->files.log.dnsmasq.c = validate_filepath_dash; @@ -1198,21 +1285,20 @@ void initConfig(struct config *conf) conf->misc.nice.k = "misc.nice"; conf->misc.nice.h = "Set niceness of pihole-FTL. Defaults to -10 and can be disabled altogether by setting a value of -999. The nice value is an attribute that can be used to influence the CPU scheduler to favor or disfavor a process in scheduling decisions. The range of the nice value varies across UNIX systems. On modern Linux, the range is -20 (high priority = not very nice to other processes) to +19 (low priority)."; conf->misc.nice.t = CONF_INT; - conf->misc.nice.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->misc.nice.f = FLAG_RESTART_FTL; conf->misc.nice.d.i = -10; conf->misc.nice.c = validate_stub; // Only type-based checking conf->misc.addr2line.k = "misc.addr2line"; conf->misc.addr2line.h = "Should FTL translate its own stack addresses into code lines during the bug backtrace? This improves the analysis of crashed significantly. It is recommended to leave the option enabled. This option should only be disabled when addr2line is known to not be working correctly on the machine because, in this case, the malfunctioning addr2line can prevent from generating any backtrace at all."; conf->misc.addr2line.t = CONF_BOOL; - conf->misc.addr2line.f = FLAG_ADVANCED_SETTING; conf->misc.addr2line.d.b = true; conf->misc.addr2line.c = validate_stub; // Only type-based checking conf->misc.etc_dnsmasq_d.k = "misc.etc_dnsmasq_d"; conf->misc.etc_dnsmasq_d.h = "Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/?"; conf->misc.etc_dnsmasq_d.t = CONF_BOOL; - conf->misc.etc_dnsmasq_d.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->misc.etc_dnsmasq_d.f = FLAG_RESTART_FTL; conf->misc.etc_dnsmasq_d.d.b = false; conf->misc.etc_dnsmasq_d.c = validate_stub; // Only type-based checking @@ -1220,7 +1306,7 @@ void initConfig(struct config *conf) conf->misc.dnsmasq_lines.h = "Additional lines to inject into the generated dnsmasq configuration.\n Warning: This is an advanced setting and should only be used with care. Incorrectly formatted or duplicated lines as well as lines conflicting with the automatic configuration of Pi-hole can break the embedded dnsmasq and will stop DNS resolution from working.\n Use this option with extra care."; conf->misc.dnsmasq_lines.a = cJSON_CreateStringReference("array of valid dnsmasq config line options"); conf->misc.dnsmasq_lines.t = CONF_JSON_STRING_ARRAY; - conf->misc.dnsmasq_lines.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->misc.dnsmasq_lines.f = FLAG_RESTART_FTL; conf->misc.dnsmasq_lines.d.json = cJSON_CreateArray(); conf->misc.dnsmasq_lines.c = validate_stub; // Type-based checking + dnsmasq syntax checking @@ -1231,6 +1317,13 @@ void initConfig(struct config *conf) conf->misc.extraLogging.d.b = false; conf->misc.extraLogging.c = validate_stub; // Only type-based checking + conf->misc.readOnly.k = "misc.readOnly"; + conf->misc.readOnly.h = "Put configuration into read-only mode. This will prevent any changes to the configuration file via the API or CLI. This setting useful when a configuration is to be forced/modified by some third-party application (like infrastructure-as-code providers) and should not be changed by any means."; + conf->misc.readOnly.t = CONF_BOOL; + conf->misc.readOnly.f = FLAG_READ_ONLY; + conf->misc.readOnly.d.b = false; + conf->misc.readOnly.c = validate_stub; // Only type-based checking + // sub-struct misc.check conf->misc.check.load.k = "misc.check.load"; conf->misc.check.load.h = "Pi-hole is very lightweight on resources. Nevertheless, this does not mean that you should run Pi-hole on a server that is otherwise extremely busy as queuing on the system can lead to unnecessary delays in DNS operation as the system becomes less and less usable as the system load increases because all resources are permanently in use. To account for this, FTL regularly checks the system load. To bring this to your attention, FTL warns about excessive load when the 15 minute system load average exceeds the number of cores.\n This check can be disabled with this setting."; @@ -1255,196 +1348,174 @@ void initConfig(struct config *conf) conf->debug.database.k = "debug.database"; conf->debug.database.h = "Print debugging information about database actions. This prints performed SQL statements as well as some general information such as the time it took to store the queries and how many have been saved to the database."; conf->debug.database.t = CONF_BOOL; - conf->debug.database.f = FLAG_ADVANCED_SETTING; conf->debug.database.d.b = false; conf->debug.database.c = validate_stub; // Only type-based checking conf->debug.networking.k = "debug.networking"; conf->debug.networking.h = "Prints a list of the detected interfaces on the startup of pihole-FTL. Also, prints whether these interfaces are IPv4 or IPv6 interfaces."; conf->debug.networking.t = CONF_BOOL; - conf->debug.networking.f = FLAG_ADVANCED_SETTING; conf->debug.networking.d.b = false; conf->debug.networking.c = validate_stub; // Only type-based checking conf->debug.locks.k = "debug.locks"; conf->debug.locks.h = "Print information about shared memory locks. Messages will be generated when waiting, obtaining, and releasing a lock."; conf->debug.locks.t = CONF_BOOL; - conf->debug.locks.f = FLAG_ADVANCED_SETTING; conf->debug.locks.d.b = false; conf->debug.locks.c = validate_stub; // Only type-based checking conf->debug.queries.k = "debug.queries"; conf->debug.queries.h = "Print extensive query information (domains, types, replies, etc.). This has always been part of the legacy debug mode of pihole-FTL."; conf->debug.queries.t = CONF_BOOL; - conf->debug.queries.f = FLAG_ADVANCED_SETTING; conf->debug.queries.d.b = false; conf->debug.queries.c = validate_stub; // Only type-based checking conf->debug.flags.k = "debug.flags"; conf->debug.flags.h = "Print flags of queries received by the DNS hooks. Only effective when DEBUG_QUERIES is enabled as well."; conf->debug.flags.t = CONF_BOOL; - conf->debug.flags.f = FLAG_ADVANCED_SETTING; conf->debug.flags.d.b = false; conf->debug.flags.c = validate_stub; // Only type-based checking conf->debug.shmem.k = "debug.shmem"; conf->debug.shmem.h = "Print information about shared memory buffers. Messages are either about creating or enlarging shmem objects or string injections."; conf->debug.shmem.t = CONF_BOOL; - conf->debug.shmem.f = FLAG_ADVANCED_SETTING; conf->debug.shmem.d.b = false; conf->debug.shmem.c = validate_stub; // Only type-based checking conf->debug.gc.k = "debug.gc"; conf->debug.gc.h = "Print information about garbage collection (GC): What is to be removed, how many have been removed and how long did GC take."; conf->debug.gc.t = CONF_BOOL; - conf->debug.gc.f = FLAG_ADVANCED_SETTING; conf->debug.gc.d.b = false; conf->debug.gc.c = validate_stub; // Only type-based checking conf->debug.arp.k = "debug.arp"; conf->debug.arp.h = "Print information about ARP table processing: How long did parsing take, whether read MAC addresses are valid, and if the macvendor.db file exists."; conf->debug.arp.t = CONF_BOOL; - conf->debug.arp.f = FLAG_ADVANCED_SETTING; conf->debug.arp.d.b = false; conf->debug.arp.c = validate_stub; // Only type-based checking conf->debug.regex.k = "debug.regex"; conf->debug.regex.h = "Controls if FTLDNS should print extended details about regex matching into FTL.log."; conf->debug.regex.t = CONF_BOOL; - conf->debug.regex.f = FLAG_ADVANCED_SETTING; conf->debug.regex.d.b = false; conf->debug.regex.c = validate_stub; // Only type-based checking conf->debug.api.k = "debug.api"; conf->debug.api.h = "Print extra debugging information concerning API calls. This includes the request, the request parameters, and the internal details about how the algorithms decide which data to present and in what form. This very verbose output should only be used when debugging specific API issues and can be helpful, e.g., when a client cannot connect due to an obscure API error. Furthermore, this setting enables logging of all API requests (auth log) and details about user authentication attempts."; conf->debug.api.t = CONF_BOOL; - conf->debug.api.f = FLAG_ADVANCED_SETTING; conf->debug.api.d.b = false; conf->debug.api.c = validate_stub; // Only type-based checking conf->debug.tls.k = "debug.tls"; conf->debug.tls.h = "Print extra debugging information about TLS connections. This includes the TLS version, the cipher suite, the certificate chain and much more. This very verbose output should only be used when debugging specific TLS issues and can be helpful, e.g., when a client cannot connect due to an obscure TLS error as modern browsers do not provide much information about the underlying TLS connection and most often give only very generic error messages without much/any underlying technical information."; conf->debug.tls.t = CONF_BOOL; - conf->debug.tls.f = FLAG_ADVANCED_SETTING; conf->debug.tls.d.b = false; conf->debug.tls.c = validate_stub; // Only type-based checking conf->debug.overtime.k = "debug.overtime"; conf->debug.overtime.h = "Print information about overTime memory operations, such as initializing or moving overTime slots."; conf->debug.overtime.t = CONF_BOOL; - conf->debug.overtime.f = FLAG_ADVANCED_SETTING; conf->debug.overtime.d.b = false; conf->debug.overtime.c = validate_stub; // Only type-based checking conf->debug.status.k = "debug.status"; conf->debug.status.h = "Print information about status changes for individual queries. This can be useful to identify unexpected unknown queries."; conf->debug.status.t = CONF_BOOL; - conf->debug.status.f = FLAG_ADVANCED_SETTING; conf->debug.status.d.b = false; conf->debug.status.c = validate_stub; // Only type-based checking conf->debug.caps.k = "debug.caps"; conf->debug.caps.h = "Print information about capabilities granted to the pihole-FTL process. The current capabilities are printed on receipt of SIGHUP, i.e., the current set of capabilities can be queried without restarting pihole-FTL (by setting DEBUG_CAPS=true and thereafter sending killall -HUP pihole-FTL)."; conf->debug.caps.t = CONF_BOOL; - conf->debug.caps.f = FLAG_ADVANCED_SETTING; conf->debug.caps.d.b = false; conf->debug.caps.c = validate_stub; // Only type-based checking conf->debug.dnssec.k = "debug.dnssec"; conf->debug.dnssec.h = "Print information about DNSSEC activity"; conf->debug.dnssec.t = CONF_BOOL; - conf->debug.dnssec.f = FLAG_ADVANCED_SETTING; conf->debug.dnssec.d.b = false; conf->debug.dnssec.c = validate_stub; // Only type-based checking conf->debug.vectors.k = "debug.vectors"; conf->debug.vectors.h = "FTL uses dynamically allocated vectors for various tasks. This config option enables extensive debugging information such as information about allocation, referencing, deletion, and appending."; conf->debug.vectors.t = CONF_BOOL; - conf->debug.vectors.f = FLAG_ADVANCED_SETTING; conf->debug.vectors.d.b = false; conf->debug.vectors.c = validate_stub; // Only type-based checking conf->debug.resolver.k = "debug.resolver"; conf->debug.resolver.h = "Extensive information about hostname resolution like which DNS servers are used in the first and second hostname resolving tries (only affecting internally generated PTR queries)."; conf->debug.resolver.t = CONF_BOOL; - conf->debug.resolver.f = FLAG_ADVANCED_SETTING; conf->debug.resolver.d.b = false; conf->debug.resolver.c = validate_stub; // Only type-based checking conf->debug.edns0.k = "debug.edns0"; conf->debug.edns0.h = "Print debugging information about received EDNS(0) data."; conf->debug.edns0.t = CONF_BOOL; - conf->debug.edns0.f = FLAG_ADVANCED_SETTING; conf->debug.edns0.d.b = false; conf->debug.edns0.c = validate_stub; // Only type-based checking conf->debug.clients.k = "debug.clients"; conf->debug.clients.h = "Log various important client events such as change of interface (e.g., client switching from WiFi to wired or VPN connection), as well as extensive reporting about how clients were assigned to its groups."; conf->debug.clients.t = CONF_BOOL; - conf->debug.clients.f = FLAG_ADVANCED_SETTING; conf->debug.clients.d.b = false; conf->debug.clients.c = validate_stub; // Only type-based checking conf->debug.aliasclients.k = "debug.aliasclients"; conf->debug.aliasclients.h = "Log information related to alias-client processing."; conf->debug.aliasclients.t = CONF_BOOL; - conf->debug.aliasclients.f = FLAG_ADVANCED_SETTING; conf->debug.aliasclients.d.b = false; conf->debug.aliasclients.c = validate_stub; // Only type-based checking conf->debug.events.k = "debug.events"; conf->debug.events.h = "Log information regarding FTL's embedded event handling queue."; conf->debug.events.t = CONF_BOOL; - conf->debug.events.f = FLAG_ADVANCED_SETTING; conf->debug.events.d.b = false; conf->debug.events.c = validate_stub; // Only type-based checking conf->debug.helper.k = "debug.helper"; conf->debug.helper.h = "Log information about script helpers, e.g., due to dhcp-script."; conf->debug.helper.t = CONF_BOOL; - conf->debug.helper.f = FLAG_ADVANCED_SETTING; conf->debug.helper.d.b = false; conf->debug.helper.c = validate_stub; // Only type-based checking conf->debug.config.k = "debug.config"; conf->debug.config.h = "Print config parsing details"; conf->debug.config.t = CONF_BOOL; - conf->debug.config.f = FLAG_ADVANCED_SETTING; conf->debug.config.d.b = false; conf->debug.config.c = validate_stub; // Only type-based checking conf->debug.inotify.k = "debug.inotify"; conf->debug.inotify.h = "Debug monitoring of /etc/pihole filesystem events"; conf->debug.inotify.t = CONF_BOOL; - conf->debug.inotify.f = FLAG_ADVANCED_SETTING; conf->debug.inotify.d.b = false; conf->debug.inotify.c = validate_stub; // Only type-based checking conf->debug.webserver.k = "debug.webserver"; conf->debug.webserver.h = "Debug monitoring of the webserver (CivetWeb) events"; conf->debug.webserver.t = CONF_BOOL; - conf->debug.webserver.f = FLAG_ADVANCED_SETTING; conf->debug.webserver.d.b = false; conf->debug.webserver.c = validate_stub; // Only type-based checking conf->debug.extra.k = "debug.extra"; conf->debug.extra.h = "Temporary flag that may print additional information. This debug flag is meant to be used whenever needed for temporary investigations. The logged content may change without further notice at any time."; conf->debug.extra.t = CONF_BOOL; - conf->debug.extra.f = FLAG_ADVANCED_SETTING; conf->debug.extra.d.b = false; conf->debug.extra.c = validate_stub; // Only type-based checking conf->debug.reserved.k = "debug.reserved"; conf->debug.reserved.h = "Reserved debug flag"; conf->debug.reserved.t = CONF_BOOL; - conf->debug.reserved.f = FLAG_ADVANCED_SETTING; conf->debug.reserved.d.b = false; conf->debug.reserved.c = validate_stub; // Only type-based checking + conf->debug.ntp.k = "debug.ntp"; + conf->debug.ntp.h = "Print information about NTP synchronization"; + conf->debug.ntp.t = CONF_BOOL; + conf->debug.ntp.d.b = false; + conf->debug.ntp.c = validate_stub; // Only type-based checking + conf->debug.all.k = "debug.all"; conf->debug.all.h = "Set all debug flags at once. This is a convenience option to enable all debug flags at once. Note that this option is not persistent, setting it to true will enable all *remaining* debug flags but unsetting it will disable *all* debug flags."; conf->debug.all.t = CONF_ALL_DEBUG_BOOL; - conf->debug.all.f = FLAG_ADVANCED_SETTING; conf->debug.all.d.b = false; conf->debug.all.c = validate_stub; // Only type-based checking @@ -1457,7 +1528,7 @@ void initConfig(struct config *conf) // Initialize config value with default one for all *except* the log file path if(conf_item != &conf->files.log.ftl) - reset_config(conf_item); + reset_config_default(conf_item); // Parse and split paths conf_item->p = gen_config_path(conf_item->k, '.'); @@ -1506,7 +1577,7 @@ void initConfig(struct config *conf) } } -void reset_config(struct conf_item *conf_item) +static void reset_config_default(struct conf_item *conf_item) { if(conf_item->t == CONF_JSON_STRING_ARRAY) { @@ -1565,13 +1636,26 @@ bool readFTLconf(struct config *conf, const bool rewrite) log_info("No config file nor backup available, using defaults"); + // If we reach this point, we could not read the TOML config file When + // this functions is invoked to run without rewriting, we are likely + // running interactively and do not want to migrate settings (yet): + // using defaults is fine in this case + if(!rewrite) + return false; + + // Check if MIGRATION_TARGET_V6 exists and is a directory + // Ideally, this directory should be created by the installer but users + // may have deleted it manually and it is necessary for restoring + // Teleporter files + create_migration_target_v6(); + // If no previous config file could be read, we are likely either running // for the first time or we are upgrading from a version prior to v6.0 // In this case, we try to read the legacy config files const char *path = ""; if((path = readFTLlegacy(conf)) != NULL) { - const char *target = "/etc/pihole/pihole-FTL.conf.bck"; + const char *target = MIGRATION_TARGET_V6"/pihole-FTL.conf"; log_info("Moving %s to %s", path, target); if(rename(path, target) != 0) log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); @@ -1626,8 +1710,8 @@ bool readFTLconf(struct config *conf, const bool rewrite) conf->webserver.port.v.s = ports; conf->webserver.port.t = CONF_STRING_ALLOCATED; - log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s", - http_port, https_port, have_ipv6 ? "enabled" : "disabled"); + log_info("Config initialized with webserver ports %d (HTTP) and %d (HTTPS), IPv6 support is %s", + http_port, https_port, have_ipv6 ? "enabled" : "disabled"); } // Initialize the TOML config file @@ -1648,7 +1732,6 @@ bool getLogFilePath(void) config.files.log.ftl.h = "The location of FTL's log file"; config.files.log.ftl.a = cJSON_CreateStringReference(""); config.files.log.ftl.t = CONF_STRING; - config.files.log.ftl.f = FLAG_ADVANCED_SETTING; config.files.log.ftl.d.s = (char*)"/var/log/pihole/FTL.log"; config.files.log.ftl.v.s = config.files.log.ftl.d.s; config.files.log.ftl.c = validate_filepath; @@ -1708,6 +1791,7 @@ const char * __attribute__ ((const)) get_conf_type_str(const enum conf_type type case CONF_ENUM_LISTENING_MODE: case CONF_ENUM_WEB_THEME: case CONF_ENUM_TEMP_UNIT: + case CONF_ENUM_BLOCKING_EDNS_MODE: return "enum (string)"; case CONF_ENUM_PRIVACY_LEVEL: return "enum (unsigned integer)"; @@ -1748,7 +1832,7 @@ void reread_config(void) // Create checksum of config file uint8_t checksum[SHA256_DIGEST_SIZE]; - if(!sha256sum(GLOBALTOMLPATH, checksum)) + if(!sha256sum(GLOBALTOMLPATH, checksum, false)) { log_err("Unable to create checksum of %s, not re-reading config file", GLOBALTOMLPATH); return; @@ -1812,12 +1896,7 @@ void reread_config(void) // If we need to restart FTL, we do so now if(restart) - { - log_info("Restarting FTL due to pihole.toml change"); - exit_code = RESTART_FTL_CODE; - // Send SIGTERM to FTL - kill(main_pid(), SIGTERM); - } + restart_ftl("pihole.toml change"); } // Very simple test of a port's availability by trying to bind a TCP socket to @@ -1850,3 +1929,32 @@ static bool port_in_use(const in_port_t port) close(sock); return false; } + +/** + * @brief Create a migration target directory for version 6. + * + * This function creates a directory for migration target version 6. If the directory + * already exists, it does nothing. The function also changes the ownership of the + * directory to the user running the FTL program. + * + * @return true if the directory creation and ownership change were successful, false otherwise. + */ +bool create_migration_target_v6(void) +{ + if(mkdir(MIGRATION_TARGET_V6, 0755) != 0 && errno != EEXIST) + { + log_err("Unable to create directory %s: %s", MIGRATION_TARGET_V6, strerror(errno)); + return false; + } + else + { + // Change ownership of the directory to the user running FTL + if(chown(MIGRATION_TARGET_V6, getuid(), getgid()) != 0) + { + log_err("Unable to change ownership of %s: %s", MIGRATION_TARGET_V6, strerror(errno)); + return false; + } + } + + return true; +} diff --git a/src/config/config.h b/src/config/config.h index 6cd4f05e..38523385 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -11,7 +11,7 @@ #define CONFIG_H // enum privacy_level -#include "../enums.h" +#include "enums.h" #include // typedef int16_t #include @@ -39,6 +39,9 @@ // Location of the legacy (pre-v6.0) config file #define GLOBALCONFFILE_LEGACY "/etc/pihole/pihole-FTL.conf" +// Migration target for the legacy (pre-v6.0) config file +#define MIGRATION_TARGET_V6 "/etc/pihole/migration_backup_v6" + union conf_value { bool b; // boolean value int i; // integer value @@ -57,6 +60,7 @@ union conf_value { enum listening_mode listeningMode; // enum listening_mode value enum web_theme web_theme; // enum web_theme value enum temp_unit temp_unit; // enum temp_unit value + enum edns_mode edns_mode; // enum edns_mode value struct in_addr in_addr; // struct in_addr value struct in6_addr in6_addr; // struct in6_addr value cJSON *json; // cJSON * value @@ -80,6 +84,7 @@ enum conf_type { CONF_ENUM_PRIVACY_LEVEL, CONF_ENUM_LISTENING_MODE, CONF_ENUM_WEB_THEME, + CONF_ENUM_BLOCKING_EDNS_MODE, CONF_ENUM_TEMP_UNIT, CONF_STRUCT_IN_ADDR, CONF_STRUCT_IN6_ADDR, @@ -90,12 +95,12 @@ enum conf_type { #define MAX_CONFIG_PATH_DEPTH 6 #define FLAG_RESTART_FTL (1 << 0) -#define FLAG_ADVANCED_SETTING (1 << 1) -#define FLAG_PSEUDO_ITEM (1 << 2) -#define FLAG_INVALIDATE_SESSIONS (1 << 3) -#define FLAG_WRITE_ONLY (1 << 4) -#define FLAG_ENV_VAR (1 << 5) -#define FLAG_CONF_IMPORTED (1 << 6) +#define FLAG_PSEUDO_ITEM (1 << 1) +#define FLAG_INVALIDATE_SESSIONS (1 << 2) +#define FLAG_WRITE_ONLY (1 << 3) +#define FLAG_ENV_VAR (1 << 4) +#define FLAG_CONF_IMPORTED (1 << 5) +#define FLAG_READ_ONLY (1 << 6) struct conf_item { const char *k; // item Key @@ -147,10 +152,12 @@ struct config { struct { struct conf_item size; struct conf_item optimizer; + struct conf_item upstreamBlockedTTL; } cache; struct { struct conf_item active; struct conf_item mode; + struct conf_item edns; } blocking; struct { struct conf_item mozillaCanary; @@ -187,9 +194,32 @@ struct config { struct conf_item rapidCommit; struct conf_item multiDNS; struct conf_item logging; + struct conf_item ignoreUnknownClients; struct conf_item hosts; } dhcp; + struct { + struct { + struct conf_item active; + struct conf_item address; + } ipv4; + struct { + struct conf_item active; + struct conf_item address; + } ipv6; + struct { + struct conf_item active; + struct conf_item server; + struct conf_item interval; + struct conf_item count; + struct { + struct conf_item set; + struct conf_item device; + struct conf_item utc; + } rtc; + } sync; + } ntp; + struct { struct conf_item resolveIPv4; struct conf_item resolveIPv6; @@ -217,7 +247,6 @@ struct config { struct conf_item restore; } session; struct { - struct conf_item rev_proxy; struct conf_item cert; } tls; struct { @@ -229,14 +258,14 @@ struct config { struct conf_item theme; } interface; struct { - struct conf_item localAPIauth; - struct conf_item searchAPIauth; struct conf_item max_sessions; struct conf_item prettyJSON; struct conf_item pwhash; struct conf_item password; // This is a pseudo-item struct conf_item totp_secret; // This is a write-only item struct conf_item app_pwhash; + struct conf_item app_sudo; + struct conf_item cli_pw; struct conf_item excludeClients; struct conf_item excludeDomains; struct conf_item maxHistory; @@ -273,6 +302,7 @@ struct config { struct conf_item etc_dnsmasq_d; struct conf_item dnsmasq_lines; struct conf_item extraLogging; + struct conf_item readOnly; struct { struct conf_item load; struct conf_item shmem; @@ -311,6 +341,7 @@ struct config { struct conf_item webserver; struct conf_item extra; struct conf_item reserved; + struct conf_item ntp; // all must be the last item in this struct struct conf_item all; } debug; @@ -324,8 +355,6 @@ extern struct config config; // Defined in config.c void set_debug_flags(struct config *conf); void set_all_debug(struct config *conf, const bool status); -void initConfig(struct config *conf); -void reset_config(struct conf_item *conf_item); bool readFTLconf(struct config *conf, const bool rewrite); bool getLogFilePath(void); struct conf_item *get_conf_item(struct config *conf, const unsigned int n); @@ -340,6 +369,7 @@ bool check_paths_equal(char **paths1, char **paths2, unsigned int max_level) __a const char *get_conf_type_str(const enum conf_type type) __attribute__ ((const)); void replace_config(struct config *newconf); void reread_config(void); +bool create_migration_target_v6(void); // Defined in toml_reader.c bool readDebugSettings(void); diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 8943c23f..e4c305ec 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -191,7 +191,7 @@ char *get_dnsmasq_line(const unsigned int lineno) static void write_config_header(FILE *fp, const char *description) { const time_t now = time(NULL); - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; get_timestr(timestring, now, false, false); fputs("# Pi-hole: A black hole for Internet advertisements\n", fp); fprintf(fp, "# (c) %u Pi-hole, LLC (https://pi-hole.net)\n", get_year(now)); @@ -398,8 +398,13 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# Use DNNSEC\n", pihole_conf); fputs("dnssec\n", pihole_conf); fputs("# 2017-02-02 root zone trust anchor\n", pihole_conf); + fputs("# https://www.iana.org/reports/2017/root-ksk-2017.pdf\n", pihole_conf); fputs("trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", pihole_conf); + fputs("# 2024-07-26 root zone trust anchor\n", pihole_conf); + fputs("# https://www.iana.org/reports/2024/root-ksk-2024.pdf\n", pihole_conf); + fputs("trust-anchor=.,38696,8,2,683D2D0ACB8C9B712A1948B27F741219298D0A450D612C483AF444A4C0FB2B16\n", + pihole_conf); fputs("\n", pihole_conf); } @@ -449,6 +454,8 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ } fputs("\n", pihole_conf); + // Add upstream DNS servers for reverse lookups + bool domain_revServer = false; const unsigned int revServers = cJSON_GetArraySize(conf->dns.revServers.v.json); for(unsigned int i = 0; i < revServers; i++) { @@ -485,8 +492,15 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ // If we have a reverse domain, we forward all queries to this domain to // the same destination if(strlen(domain) > 0) + { fprintf(pihole_conf, "server=/%s/%s\n", domain, target); + // Check if the configured domain is the same as the main domain + if(strlen(config.dns.domain.v.s) > 0 && + strcasecmp(domain, config.dns.domain.v.s) == 0) + domain_revServer = true; + } + // Forward unqualified names to the target only when the "never forward // non-FQDN" option is NOT ticked if(!conf->dns.domainNeeded.v.b) @@ -497,19 +511,14 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ free(copy); } - // When there is a Pi-hole domain set and "Never forward non-FQDNs" is - // ticked, we add `local=/domain/` to signal that this domain is purely - // local and FTL may answer queries from /etc/hosts or DHCP but should - // never forward queries on that domain to any upstream servers + // When "Never forward non-FQDNs" is ticked, we add `local=//` to signal + // that non-FQDNs queries should never be sent to any upstream servers if(conf->dns.domainNeeded.v.b) { fputs("# Never forward A or AAAA queries for plain names, without\n",pihole_conf); fputs("# dots or domain parts, to upstream nameservers. If the name\n", pihole_conf); - fputs("# is not known from /etc/hosts or DHCP a NXDOMAIN is returned\n", pihole_conf); - if(strlen(conf->dns.domain.v.s)) - fprintf(pihole_conf, "local=/%s/\n\n", conf->dns.domain.v.s); - else - fputs("\n", pihole_conf); + fputs("# is not known from /etc/hosts or DHCP, NXDOMAIN is returned\n", pihole_conf); + fputs("local=//\n\n", pihole_conf); } // Add domain to DNS server. It will also be used for DHCP if the DHCP @@ -517,7 +526,20 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ if(strlen(conf->dns.domain.v.s) > 0) { fputs("# DNS domain for both the DNS and DHCP server\n", pihole_conf); - fprintf(pihole_conf, "domain=%s\n\n", conf->dns.domain.v.s); + if(!domain_revServer) + { + fputs("# This DNS domain in purely local. FTL may answer queries from\n", pihole_conf); + fputs("# /etc/hosts or DHCP but should never forward queries on that\n", pihole_conf); + fputs("# domain to any upstream servers\n", pihole_conf); + fprintf(pihole_conf, "domain=%s\n", conf->dns.domain.v.s); + fprintf(pihole_conf, "local=/%s/\n\n", conf->dns.domain.v.s); + } + else + { + fputs("# This DNS domain is also used for reverse lookups\n", pihole_conf); + fputs("# (see server=//target above)\n", pihole_conf); + fprintf(pihole_conf, "domain=%s\n\n", conf->dns.domain.v.s); + } } if(conf->dhcp.active.v.b) @@ -582,6 +604,22 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("log-dhcp\n\n", pihole_conf); } + // Check if IPv4 NTP server is active and broadcast it as DHCP option + if(conf->ntp.ipv4.active.v.b) + { + fputs("# Add NTP server to DHCP\n", pihole_conf); + // The special address 0.0.0.0 is taken to mean "the + // address of the machine running the DHCP server" + fputs("dhcp-option=option:ntp-server,0.0.0.0\n\n", pihole_conf); + } + + // Add option to ignore unknown clients if enabled + if(conf->dhcp.ignoreUnknownClients.v.b) + { + fputs("# Ignore clients not configured below\n", pihole_conf); + fputs("dhcp-ignore=tag:!known\n\n", pihole_conf); + } + // Add per-host parameters if(cJSON_GetArraySize(conf->dhcp.hosts.v.json) > 0) { @@ -716,6 +754,12 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ { log_warn("New dnsmasq configuration is not valid (%s), config remains unchanged", errbuf); + if(debug_flags[DEBUG_ANY]) + { + log_debug(DEBUG_ANY, "Temporary dnsmasq config file left in place for debugging purposes"); + return false; + } + // Remove temporary config file if(remove(DNSMASQ_TEMP_CONF) != 0) { @@ -760,7 +804,6 @@ bool read_legacy_dhcp_static_config(void) { // Check if file exists, if not, there is nothing to do const char *path = DNSMASQ_STATIC_LEASES; - const char *target = DNSMASQ_STATIC_LEASES".bck"; if(!file_exists(path)) return true; @@ -810,11 +853,6 @@ bool read_legacy_dhcp_static_config(void) return false; } - // Move file to backup location - log_info("Moving %s to %s", path, target); - if(rename(path, target) != 0) - log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); - return true; } @@ -823,7 +861,6 @@ bool read_legacy_cnames_config(void) { // Check if file exists, if not, there is nothing to do const char *path = DNSMASQ_CNAMES; - const char *target = DNSMASQ_CNAMES".bck"; if(!file_exists(path)) return true; @@ -873,11 +910,6 @@ bool read_legacy_cnames_config(void) return false; } - // Move file to backup location - log_info("Moving %s to %s", path, target); - if(rename(path, target) != 0) - log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); - return true; } @@ -885,7 +917,7 @@ bool read_legacy_custom_hosts_config(void) { // Check if file exists, if not, there is nothing to do const char *path = DNSMASQ_CUSTOM_LIST_LEGACY; - const char *target = DNSMASQ_CUSTOM_LIST_LEGACY".bck"; + const char *target = DNSMASQ_CUSTOM_LIST_LEGACY_TARGET; if(!file_exists(path)) return true; diff --git a/src/config/dnsmasq_config.h b/src/config/dnsmasq_config.h index 77b1405c..df660790 100644 --- a/src/config/dnsmasq_config.h +++ b/src/config/dnsmasq_config.h @@ -24,11 +24,12 @@ bool write_custom_list(void); #define DNSMASQ_PH_CONFIG "/etc/pihole/dnsmasq.conf" #define DNSMASQ_TEMP_CONF "/etc/pihole/dnsmasq.conf.temp" -#define DNSMASQ_STATIC_LEASES "/etc/pihole/04-pihole-static-dhcp.conf" -#define DNSMASQ_CNAMES "/etc/pihole/05-pihole-custom-cname.conf" +#define DNSMASQ_STATIC_LEASES MIGRATION_TARGET_V6"/04-pihole-static-dhcp.conf" +#define DNSMASQ_CNAMES MIGRATION_TARGET_V6"/05-pihole-custom-cname.conf" #define DNSMASQ_HOSTSDIR "/etc/pihole/hosts" #define DNSMASQ_CUSTOM_LIST DNSMASQ_HOSTSDIR"/custom.list" #define DNSMASQ_CUSTOM_LIST_LEGACY "/etc/pihole/custom.list" +#define DNSMASQ_CUSTOM_LIST_LEGACY_TARGET MIGRATION_TARGET_V6"/custom.list" #define DHCPLEASESFILE "/etc/pihole/dhcp.leases" #endif //DNSMASQ_CONFIG_H diff --git a/src/config/env.c b/src/config/env.c index a58650fe..0981c619 100644 --- a/src/config/env.c +++ b/src/config/env.c @@ -19,6 +19,10 @@ #include "args.h" // suggest_closest() #include "config/suggest.h" +// LINE_MAX +#include +// openFTLtoml() +#include "config/toml_helper.h" struct env_item { bool used; @@ -161,16 +165,51 @@ void freeEnvVars(void) } } -bool readEnvValue(struct conf_item *conf_item, struct config *newconf) +bool __attribute__((nonnull(1,2,3))) readEnvValue(struct conf_item *conf_item, struct config *newconf, cJSON *forced_vars, bool *reset) { // First check if a environmental variable with the given key exists by // iterating over the list of FTLCONF_ variables struct env_item *item = getFTLenv(conf_item->e); - // Return early if this environment variable does not exist if(item == NULL) - return false; + { + // Environment variable does not exist + // Check if this was a forced setting before + // If so, we revert the config option to default + for(int i = 0; i < cJSON_GetArraySize(forced_vars); i++) + { + const char *forced_var = cJSON_GetArrayItem(forced_vars, i)->valuestring; + if(strcmp(forced_var, conf_item->k) == 0) + { + log_info("Resetting %s to default (not forced anymore)", conf_item->k); + + // Revert to default + if(conf_item->t == CONF_STRING_ALLOCATED) + { + // Free previously allocated string + free(conf_item->v.s); + // Make a duplicate of the default value + conf_item->v.s = strdup(conf_item->d.s); + } + else + { + // Revert to default value + memcpy(&conf_item->v, &conf_item->d, sizeof(conf_item->v)); + } + + // Mark this environment variable as reset to + // default + if(reset != NULL) + *reset = true; + break; + } + } + + // Return false as this setting is not forced by an environment + // variable + return false; + } // Mark this environment variable as used item->used = true; @@ -460,6 +499,25 @@ bool readEnvValue(struct conf_item *conf_item, struct config *newconf) } break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + const int edns_mode = get_edns_mode_val(envvar); + if(edns_mode != -1) + { + conf_item->v.edns_mode = edns_mode; + item->valid = true; + } + else + { + + item->error = "not an allowed option"; + item->allowed = conf_item->h; + log_warn("ENV %s is %s, allowed options are: %s", + conf_item->e, item->error, item->allowed); + item->valid = false; + } + break; + } case CONF_ENUM_PRIVACY_LEVEL: { int val = 0; @@ -561,3 +619,54 @@ bool readEnvValue(struct conf_item *conf_item, struct config *newconf) return true; } + +cJSON *read_forced_vars(const unsigned int version) +{ + // Create cJSON array to store forced variables + cJSON *env_vars = cJSON_CreateArray(); + + // Try to open default config file. Use fallback if not found + FILE *fp; + if((fp = openFTLtoml("r", version)) == NULL) + { + // Return empty cJSON array + return env_vars; + } + + // Read file line by line until we get to the end of the file where the + // statistics are stored, specifically, the line starting with + // "# X entr{y is,ies are} forced through environment" + char line[LINE_MAX] = { 0 }; + while(fgets(line, sizeof(line), fp) != NULL) + { + // Check if this is the line we are looking for + if(strncmp(line, "# ", 2) == 0) + { + // Check if this is the line we are looking for + if(strstr(line, "forced through environment:") != NULL) + break; + } + } + + // Read the next lines to extract the variables + while(fgets(line, sizeof(line), fp) != NULL) + { + // Check if this is the line we are looking for + if(strncmp(line, "# - ", 6) != 0) + { + // We are done, break out of the loop + break; + } + + // else: Add the variable to the cJSON array + // Trim the string (remove leading "# - " and trailing newline) + line[strcspn(line, "\n")] = '\0'; + cJSON_AddItemToArray(env_vars, cJSON_CreateString(line + 6)); + } + + // Close file and release exclusive lock + closeFTLtoml(fp); + + // Return cJSON array + return env_vars; +} diff --git a/src/config/env.h b/src/config/env.h index d26c7e75..b7b580af 100644 --- a/src/config/env.h +++ b/src/config/env.h @@ -23,6 +23,7 @@ int dist(const char *str); void getEnvVars(void); void freeEnvVars(void); void printFTLenv(void); -bool readEnvValue(struct conf_item *conf_item, struct config *newconf); +bool readEnvValue(struct conf_item *conf_item, struct config *newconf, cJSON *forced_vars, bool *reset) __attribute__((nonnull(1,2,3))); +cJSON *read_forced_vars(const unsigned int version); #endif //CONFIG_ENV_H diff --git a/src/config/inotify.c b/src/config/inotify.c index 17767350..dfb3837c 100644 --- a/src/config/inotify.c +++ b/src/config/inotify.c @@ -12,7 +12,7 @@ #include "log.h" #include // NAME_MAX -#include +#include #define WATCHDIR "/etc/pihole" diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 857bcc2b..fb698278 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -28,7 +28,7 @@ static pthread_mutex_t lock; // Private prototypes static char *parseFTLconf(FILE *fp, const char *key); static void releaseConfigMemory(void); -static char *getPath(FILE* fp, const char *option, char *ptr); +static char *__attribute__((nonnull(1,2,3), malloc, warn_unused_result)) getPath(FILE* fp, const char *option, char *ptr); static bool parseBool(const char *option, bool *ptr); static void readDebugingSettingsLegacy(FILE *fp); static void getBlockingModeLegacy(FILE *fp); @@ -66,6 +66,10 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) // No option set => use default log location if(buffer == NULL) { + // Free previously allocated memory (if any) + if(conf->files.log.ftl.t == CONF_STRING_ALLOCATED) + free(conf->files.log.ftl.v.s); + // Use standard path if no custom path was obtained from the config file conf->files.log.ftl.v.s = strdup("/var/log/pihole/FTL.log"); conf->files.log.ftl.t = CONF_STRING_ALLOCATED; @@ -77,9 +81,12 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) strerror(errno), errno); exit(EXIT_FAILURE); } + + fclose(fp); + return true; } // Use sscanf() to obtain filename from config file parameter only if buffer != NULL - else if(sscanf(buffer, "%127ms", &val_buffer) == 0) + else if((val_buffer = calloc(128, sizeof(char))) == NULL || sscanf(buffer, "%127s", val_buffer) == 0) { // Free previously allocated memory (if any) if(conf->files.log.ftl.t == CONF_STRING_ALLOCATED) @@ -89,9 +96,14 @@ bool getLogFilePathLegacy(struct config *conf, FILE *fp) conf->files.log.ftl.v.s = NULL; conf->files.log.ftl.t = CONF_STRING; log_info("Using syslog facility"); + + // Free buffer + if(val_buffer != NULL) + free(val_buffer); } - if(val_buffer) + // Set string if memory allocation was successful and a value was read + if(val_buffer != NULL && strlen(val_buffer) > 0) { // Free previously allocated memory (if any) if(conf->files.log.ftl.t == CONF_STRING_ALLOCATED) @@ -256,7 +268,7 @@ const char *readFTLlegacy(struct config *conf) buffer = parseFTLconf(fp, "DELAY_STARTUP"); unsigned int unum; - if(buffer != NULL && sscanf(buffer, "%u", &unum) && unum > 0 && unum <= 300) + if(buffer != NULL && sscanf(buffer, "%u", &unum) == 1 && unum > 0 && unum <= 300) conf->misc.delay_startup.v.ui = unum; // BLOCK_ESNI @@ -305,11 +317,6 @@ const char *readFTLlegacy(struct config *conf) if(buffer != NULL) conf->webserver.acl.v.s = strdup(buffer); - // API_AUTH_FOR_LOCALHOST - // defaults to: true - buffer = parseFTLconf(fp, "API_AUTH_FOR_LOCALHOST"); - parseBool(buffer, &conf->webserver.api.localAPIauth.v.b); - // API_SESSION_TIMEOUT // How long should a session be considered valid after login? // defaults to: 300 seconds @@ -589,35 +596,38 @@ const char *readFTLlegacy(struct config *conf) return path; } -static char* getPath(FILE* fp, const char *option, char *ptr) +static char *__attribute__((nonnull(1,2,3), malloc, warn_unused_result)) getPath(FILE* fp, const char *option, char *path_default) { // This subroutine is used to read paths from pihole-FTL.conf - // fp: File ptr to opened and readable config file - // option: Option string ("key") to try to read - // ptr: Location where read (or default) parameter is stored + // fp: File path to opened and readable config file + // option: Option string ("key") to try to read + // path_default: Location where read (or default) parameter is stored char *buffer = parseFTLconf(fp, option); errno = 0; // Use sscanf() to obtain filename from config file parameter only if buffer != NULL - if(buffer == NULL || sscanf(buffer, "%127ms", &ptr) != 1) - { - // Use standard path if no custom path was obtained from the config file - return ptr; - } + char *val_ptr = calloc(128, sizeof(char)); // Test if memory allocation was successful - if(ptr == NULL) + if(val_ptr == NULL) { - log_crit("Allocating memory for %s failed (%s, %i). Exiting.", option, strerror(errno), errno); + log_crit("Allocating memory for %s failed (%s, %i). Exiting.", + option, strerror(errno), errno); exit(EXIT_FAILURE); } - else if(strlen(ptr) == 0) + + if(buffer == NULL || sscanf(buffer, "%127s", val_ptr) != 1 || strlen(val_ptr) == 0) { + // Use standard path if no custom path was obtained from the config file log_info(" %s: Empty path is not possible, using default", option); + + strncpy(val_ptr, path_default, 127); + val_ptr[127] = '\0'; + return val_ptr; } - return ptr; + return val_ptr; } static char *parseFTLconf(FILE *fp, const char * key) @@ -703,7 +713,7 @@ void releaseConfigMemory(void) void init_config_mutex(void) { // Initialize the lock attributes - pthread_mutexattr_t lock_attr = {}; + pthread_mutexattr_t lock_attr; pthread_mutexattr_init(&lock_attr); // Initialize the lock diff --git a/src/config/password.c b/src/config/password.c index d97df6d2..ddae3856 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -39,6 +39,20 @@ // 2023, using 128 bits should be sufficient for the foreseeable future. #define SALT_LEN 16 // 16 bytes = 128 bits +// App password length +// The app password is a 256 bit password. This is a good balance between +// security and usability. It is long enough to be secure. +#define APPPW_LEN 32 // 32 bytes = 256 bits + +// CLI password file and memory +// We store the password in plain memory. This is not a security issue as the +// memory is only accessible to the user running the FTL process. Anyone with +// sufficient access to the memory (ptrace, swapfile) would also have access to +// the password file. Leaking the password after exit is not a concern as a new +// password is generated on every start. +#define CLI_PW_FILE "/etc/pihole/cli_pw" +static char *cli_password = NULL; + // Convert RAW data into hex representation // Two hexadecimal digits are generated for each input byte. void sha256_raw_to_hex(uint8_t *data, char *buffer) @@ -185,6 +199,14 @@ static char * __attribute__((malloc)) balloon_password(const char *password, // Build PHC string-like output (output string is 101 bytes long (measured)) char *output = calloc(128, sizeof(char)); + + if(output == NULL || salt_base64 == NULL || scratch_base64 == NULL) + { + log_err("Error while allocating memory for PHC string: %s", strerror(errno)); + goto clean_and_exit; + } + + // Generate PHC string int size = snprintf(output, 128, "$BALLOON-SHA256$v=1$s=%zu,t=%zu$%s$%s", s_cost, t_cost, @@ -199,11 +221,15 @@ static char * __attribute__((malloc)) balloon_password(const char *password, } clean_and_exit: - free(scratch); - free(salt_base64); - free(scratch_base64); + // Clean up + if(scratch != NULL) + free(scratch); + if(salt_base64 != NULL) + free(salt_base64); + if(scratch_base64 != NULL) + free(scratch_base64); - return output; + return output; // may be NULL on failure (unlikely) } // Parse a PHC string and return the parameters and hash @@ -314,11 +340,15 @@ char * __attribute__((malloc)) create_password(const char *password) enum password_result verify_login(const char *password) { + // Check if this is the CLI password + if(config.webserver.api.cli_pw.v.b && cli_password != NULL) + { + if(strcmp(cli_password, password) == 0) + return CLIPASSWORD_CORRECT; + } + enum password_result pw = verify_password(password, config.webserver.api.pwhash.v.s, true); - if(pw == PASSWORD_CORRECT) - log_debug(DEBUG_API, "Password correct"); - else - log_debug(DEBUG_API, "Password incorrect"); + log_debug(DEBUG_API, "Password %s correct", pw == PASSWORD_CORRECT ? "" : "not"); // Check if an application password is set and if it matches if(pw == PASSWORD_INCORRECT && @@ -410,7 +440,6 @@ enum password_result verify_password(const char *password, const char *pwhash, c config.webserver.api.pwhash.v.s = new_hash; config.webserver.api.pwhash.t = CONF_STRING_ALLOCATED; writeFTLtoml(true); - free(new_hash); } // Successful logins do not count against rate-limiting @@ -604,6 +633,16 @@ int run_performance_test(void) bool set_and_check_password(struct conf_item *conf_item, const char *password) { + // Check if the user wants to set an empty password but the password is + // already empty, or if the newly set password is the same as the old + // one + if((strlen(password) == 0 && strlen(config.webserver.api.pwhash.v.s) == 0) || + verify_password(password, config.webserver.api.pwhash.v.s, false) == PASSWORD_CORRECT) + { + log_debug(DEBUG_CONFIG, "Password unchanged, not updating"); + return true; + } + // Get password hash as allocated string (an empty string is hashed to an empty string) char *pwhash = strlen(password) > 0 ? create_password(password) : strdup(""); @@ -625,33 +664,46 @@ bool set_and_check_password(struct conf_item *conf_item, const char *password) // Set item conf_item->v.s = pwhash; + conf_item->t = CONF_STRING_ALLOCATED; log_debug(DEBUG_CONFIG, "Set %s to \"%s\"", conf_item->k, conf_item->v.s); return true; } -bool generate_app_password(char **password, char **pwhash) +bool generate_password(char **password, char **pwhash) { - // Generate a 128 bit random salt - // genrandom() returns cryptographically secure random data - uint8_t salt[SALT_LEN] = { 0 }; - if(getrandom(salt, sizeof(salt), 0) < 0) - { - log_err("getrandom() failed in generate_app_password()"); - return false; - } - // Generate a 256 bit random password - uint8_t password_raw[256/8] = { 0 }; + // genrandom() returns cryptographically secure random data + uint8_t password_raw[APPPW_LEN] = { 0 }; if(getrandom(password_raw, sizeof(password_raw), 0) < 0) { - log_err("getrandom() failed in generate_app_password()"); + log_err("getrandom() failed in generate_password()"); return false; } // Encode password as base64 *password = base64_encode(password_raw, sizeof(password_raw)); + if(*password == NULL) + { + log_err("Error while encoding password as base64"); + return false; + } + + if(pwhash == NULL) + { + // No password hash requested + return true; + } + + // Generate a 128 bit random salt + uint8_t salt[SALT_LEN] = { 0 }; + if(getrandom(salt, sizeof(salt), 0) < 0) + { + log_err("getrandom() failed in generate_password()"); + return false; + } + // Generate balloon PHC-encoded password hash *pwhash = balloon_password(*password, salt, true); @@ -666,3 +718,84 @@ bool generate_app_password(char **password, char **pwhash) return true; } + +bool create_cli_password(void) +{ + // Check if the CLI password is enabled + if(!config.webserver.api.cli_pw.v.b) + { + log_debug(DEBUG_API, "CLI password is not set"); + return true; + } + + // Generate a new CLI password hash + if(!generate_password(&cli_password, NULL)) + { + log_err("Failed to generate CLI password hash!"); + return false; + } + + // Store the CLI password in the corresponding file + FILE *file = fopen(CLI_PW_FILE, "w"); + if(file == NULL) + { + log_err("Failed to open CLI password file for writing: %s", strerror(errno)); + free(cli_password); + return false; + } + + // Write password + if(fputs(cli_password, file) == EOF) + { + log_err("Failed to write CLI password to file: %s", strerror(errno)); + fclose(file); + free(cli_password); + return false; + } + + // Close file + fclose(file); + + // Set file permissions to 0640 + if(chmod(CLI_PW_FILE, S_IRUSR | S_IWUSR | S_IRGRP) < 0) + { + log_err("Failed to set permissions on CLI password file: %s", strerror(errno)); + free(cli_password); + return false; + } + + log_debug(DEBUG_API, "CLI password set and stored in file"); + return true; +} + +bool remove_cli_password(void) +{ + // Remove the CLI password from memory (if allocated) + if(cli_password != NULL) + { + free(cli_password); + cli_password = NULL; + } + + // Empty the CLI password file + FILE *file = fopen(CLI_PW_FILE, "w"); + if(file == NULL) + { + log_err("Failed to open CLI password file for writing: %s", strerror(errno)); + return false; + } + + // Close file + fclose(file); + + // Remove the CLI password file from disk + // If the file does not exist, we returned above already + if(unlink(CLI_PW_FILE) < 0) + { + log_err("Failed to remove CLI password file: %s", strerror(errno)); + return false; + } + + log_debug(DEBUG_API, "CLI password removed"); + return true; +} diff --git a/src/config/password.h b/src/config/password.h index 063e5dcf..fd6fd332 100644 --- a/src/config/password.h +++ b/src/config/password.h @@ -20,13 +20,16 @@ enum password_result verify_login(const char *password); enum password_result verify_password(const char *password, const char *pwhash, const bool rate_limiting); int run_performance_test(void); bool set_and_check_password(struct conf_item *conf_item, const char *password); -bool generate_app_password(char **password, char **pwhash); +bool generate_password(char **password, char **pwhash); +bool create_cli_password(void); +bool remove_cli_password(void); enum password_result { PASSWORD_INCORRECT = 0, PASSWORD_CORRECT = 1, APPPASSWORD_CORRECT = 2, - NO_PASSWORD_SET = 3, + CLIPASSWORD_CORRECT = 3, + NO_PASSWORD_SET = 4, PASSWORD_RATE_LIMITED = -1 } __attribute__((packed)); diff --git a/src/config/setupVars.c b/src/config/setupVars.c index e0c54009..3a82beeb 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -30,7 +30,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co if(setupVarsValue == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + log_info("setupVars.conf:%s -> Not set", key); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -48,7 +48,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, conf_item->v.s); + log_info("setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, conf_item->v.s); } static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf_item) @@ -64,7 +64,7 @@ static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf if(setupVarsValue == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + log_info("setupVars.conf:%s -> Not set", key); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -75,7 +75,7 @@ static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf memset(&conf_item->v.in_addr, 0, sizeof(struct in_addr)); else if(inet_pton(AF_INET, setupVarsValue, &conf_item->v.in_addr) != 1) { - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Invalid IPv4 address: %s", key, setupVarsValue); + log_info("setupVars.conf:%s -> Invalid IPv4 address: %s", key, setupVarsValue); memset(&conf_item->v.in_addr, 0, sizeof(struct in_addr)); } @@ -83,7 +83,7 @@ static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, inet_ntoa(conf_item->v.in_addr)); + log_info("setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, inet_ntoa(conf_item->v.in_addr)); } static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf_item) @@ -100,7 +100,7 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf if(boolean == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + log_info("setupVars.conf:%s -> Not set", key); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -116,13 +116,12 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", - key, conf_item->k, conf_item->v.b ? "true" : "false"); + log_info("setupVars.conf:%s -> Setting %s to %s", + key, conf_item->k, conf_item->v.b ? "true" : "false"); } static void get_revServer_from_setupVars(void) { - bool active = false; char *cidr = NULL; char *target = NULL; char *domain = NULL; @@ -130,17 +129,21 @@ static void get_revServer_from_setupVars(void) if(active_str == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:REV_SERVER -> Not set"); + log_info("setupVars.conf:REV_SERVER -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); return; } - else + // Parameter present in setupVars.conf, check if either "true" or "false" + if(strcasecmp(active_str, "true") != 0 && strcasecmp(active_str, "false") != 0) { - // Parameter present in setupVars.conf - active = getSetupVarsBool(active_str); + log_info("setupVars.conf:REV_SERVER -> Invalid value: %s", active_str); + + clearSetupVarsArray(); + return; } + bool active = strcasecmp(active_str, "true") == 0; // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -151,6 +154,8 @@ static void get_revServer_from_setupVars(void) cidr = strdup(cidr_str); trim_whitespace(cidr); } + else + log_info("setupVars.conf:REV_SERVER_CIDR -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -161,6 +166,8 @@ static void get_revServer_from_setupVars(void) target = strdup(target_str); trim_whitespace(target); } + else + log_info("setupVars.conf:REV_SERVER_TARGET -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -171,25 +178,36 @@ static void get_revServer_from_setupVars(void) domain = strdup(domain_str); trim_whitespace(domain); } + else + log_info("setupVars.conf:REV_SERVER_DOMAIN -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); // Only add the entry if all values are present and active - if(active && cidr != NULL && target != NULL && domain != NULL) + if(cidr != NULL && target != NULL && domain != NULL) { // Build comma-separated string of all values - // 8 = 3 commas, "true", and null terminator - char *old = calloc(strlen(cidr) + strlen(target) + strlen(domain) + 8, sizeof(char)); - if(old) + // 9 = 3 commas, "true/false", and null terminator + char *old = calloc(strlen(cidr) + strlen(target) + strlen(domain) + 9, sizeof(char)); + if(old != NULL) { // Add to new config // active is always true as we only add active entries - sprintf(old, "true,%s,%s,%s", cidr, target, domain); + sprintf(old, "%s,%s,%s,%s", active ? "true" : "false", cidr, target, domain); cJSON_AddItemToArray(config.dns.revServers.v.json, cJSON_CreateString(old)); + + // Parameter present in setupVars.conf + log_info("setupVars.conf:REV_SERVER -> Setting %s to %s", + config.dns.revServers.k, old); free(old); } } + else + { + // Parameter not present in setupVars.conf + log_info("setupVars.conf:REV_SERVER_* -> Not set (found invalid/incomplete parameters)"); + } // Free memory if(cidr != NULL) @@ -262,8 +280,8 @@ static void get_conf_string_array_from_setupVars_regex(const char *key, struct c cJSON *item = cJSON_CreateString(regex2); cJSON_AddItemToArray(conf_item->v.json, item); - log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s[%u] = %s\n", - key, conf_item->k, i, item->valuestring); + log_info("setupVars.conf:%s -> Setting %s[%u] = %s\n", + key, conf_item->k, i, item->valuestring); // Free memory free(regex2); @@ -295,13 +313,12 @@ static void get_conf_upstream_servers_from_setupVars(struct conf_item *conf_item if(value != NULL) { - log_debug(DEBUG_CONFIG, "%s = %s\n", server_key, value); // Add string to our JSON array cJSON *item = cJSON_CreateString(value); cJSON_AddItemToArray(conf_item->v.json, item); - log_debug(DEBUG_CONFIG, "setupVars.conf:PIHOLE_DNS_%u -> Setting %s[%u] = %s\n", - j, conf_item->k, j, item->valuestring); + log_info("setupVars.conf:PIHOLE_DNS_%u -> Setting %s[%u] = %s", + j, conf_item->k, j, item->valuestring); } // Free memory, harmless to call if read_setupVarsconf() didn't return a result @@ -317,7 +334,7 @@ static void get_conf_temp_limit_from_setupVars(void) if(temp_limit == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_LIMIT -> Not set"); + log_info("setupVars.conf:TEMPERATURE_LIMIT -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -339,13 +356,13 @@ static void get_conf_temp_limit_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_LIMIT -> Setting %s to %f", + log_info("setupVars.conf:TEMPERATURE_LIMIT -> Setting %s to %f", config.webserver.api.temp.limit.k, config.webserver.api.temp.limit.v.d); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_LIMIT -> Not set (found invalid value)"); + log_info("setupVars.conf:TEMPERATURE_LIMIT -> Not set (found invalid value)"); } } @@ -357,7 +374,7 @@ static void get_conf_weblayout_from_setupVars(void) if(web_layout == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBUIBOXEDLAYOUT -> Not set"); + log_info("setupVars.conf:WEBUIBOXEDLAYOUT -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -365,16 +382,14 @@ static void get_conf_weblayout_from_setupVars(void) } // If the property is set to false and different than "boxed", the property - // is disabled. This is consistent with the code in AdminLTE when writing - // this code - if(strcasecmp(web_layout, "boxed") != 0) - config.webserver.interface.boxed.v.b = false; + // is disabled + config.webserver.interface.boxed.v.b = strcasecmp(web_layout, "boxed") == 0; // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBUIBOXEDLAYOUT -> Setting %s to %s", + log_info("setupVars.conf:WEBUIBOXEDLAYOUT -> Setting %s to %s", config.webserver.interface.boxed.k,config.webserver.interface.boxed.v.b ? "true" : "false"); } @@ -386,7 +401,7 @@ static void get_conf_webtheme_from_setupVars(void) if(webTheme == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBTHEME -> Not set"); + log_info("setupVars.conf:WEBTHEME -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -407,14 +422,14 @@ static void get_conf_webtheme_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBTHEME -> Setting %s to %s", - config.webserver.interface.theme.k, - get_web_theme_str(config.webserver.interface.theme.v.web_theme)); + log_info("setupVars.conf:WEBTHEME -> Setting %s to %s", + config.webserver.interface.theme.k, + get_web_theme_str(config.webserver.interface.theme.v.web_theme)); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:WEBTHEME -> Not set (found invalid value)"); + log_info("setupVars.conf:WEBTHEME -> Not set (found invalid value)"); } } @@ -426,7 +441,7 @@ static void get_conf_temp_unit_from_setupVars(void) if(temp_unit == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_UNIT -> Not set"); + log_info("setupVars.conf:TEMPERATURE_UNIT -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -447,14 +462,14 @@ static void get_conf_temp_unit_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_UNIT -> Setting %s to %s", - config.webserver.interface.theme.k, - get_temp_unit_str(config.webserver.api.temp.unit.v.temp_unit)); + log_info("setupVars.conf:TEMPERATURE_UNIT -> Setting %s to %s", + config.webserver.interface.theme.k, + get_temp_unit_str(config.webserver.api.temp.unit.v.temp_unit)); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:TEMPERATURE_UNIT -> Not set (found invalid value)"); + log_info("setupVars.conf:TEMPERATURE_UNIT -> Not set (found invalid value)"); } } @@ -466,7 +481,7 @@ static void get_conf_listeningMode_from_setupVars(void) if(listeningMode == NULL) { // Do not change default value, this value is not set in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:DNSMASQ_LISTENING -> Not set"); + log_info("setupVars.conf:DNSMASQ_LISTENING -> Not set"); // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -487,13 +502,13 @@ static void get_conf_listeningMode_from_setupVars(void) if(set) { // Parameter present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:DNSMASQ_LISTENING -> Setting %s to %s", - config.dns.listeningMode.k, get_listeningMode_str(config.dns.listeningMode.v.listeningMode)); + log_info("setupVars.conf:DNSMASQ_LISTENING -> Setting %s to %s", + config.dns.listeningMode.k, get_listeningMode_str(config.dns.listeningMode.v.listeningMode)); } else { // Parameter not present in setupVars.conf - log_debug(DEBUG_CONFIG, "setupVars.conf:DNSMASQ_LISTENING -> Not set (found invalid value)"); + log_info("setupVars.conf:DNSMASQ_LISTENING -> Not set (found invalid value)"); } } @@ -553,7 +568,7 @@ void importsetupVarsConf(void) get_conf_string_from_setupVars("DHCP_LEASETIME", &config.dhcp.leaseTime); // If the DHCP lease time is set to "24", it is interpreted as "24h". - // This is some relic from the past that may still be present in some + // This is some relict from the past that may still be present in some // setups if(strcmp(config.dhcp.leaseTime.v.s, "24") == 0) { @@ -573,20 +588,14 @@ void importsetupVarsConf(void) // Ports may be temporarily stored when importing a legacy Teleporter v5 file get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); - // Move the setupVars.conf file to setupVars.conf.old - char *old_setupVars = calloc(strlen(config.files.setupVars.v.s) + 5, sizeof(char)); - if(old_setupVars == NULL) - { - log_warn("Could not allocate memory for old_setupVars"); - return; - } - strcpy(old_setupVars, config.files.setupVars.v.s); - strcat(old_setupVars, ".old"); - if(rename(config.files.setupVars.v.s, old_setupVars) != 0) - log_warn("Could not move %s to %s", config.files.setupVars.v.s, old_setupVars); + // Move the setupVars.conf file to the migration directory + const char *setupVars_target = MIGRATION_TARGET_V6"/setupVars.conf"; + if(rename(config.files.setupVars.v.s, setupVars_target) != 0) + log_warn("Could not move %s to %s", config.files.setupVars.v.s, setupVars_target); else - log_info("Moved %s to %s", config.files.setupVars.v.s, old_setupVars); - free(old_setupVars); + log_info("Moved %s to %s", config.files.setupVars.v.s, setupVars_target); + + log_info("setupVars.conf migration complete"); } char* __attribute__((pure)) find_equals(char *s) diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 04ade29b..a4a402d7 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -23,6 +23,8 @@ #include // escape_json() #include "webserver/http-common.h" +// chown_pihole() +#include "files.h" // Open the TOML file for reading or writing FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode, const unsigned int version) @@ -59,8 +61,8 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m // Return early if opening failed if(!fp) { - log_info("Config %sfile %s not available: %s", - version > 0 ? "backup " : "", filename, strerror(errno)); + log_info("Config %sfile %s not available (%s): %s", + version > 0 ? "backup " : "", filename, mode, strerror(errno)); return NULL; } @@ -68,8 +70,8 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m if(flock(fileno(fp), LOCK_EX) != 0) { const int _e = errno; - log_err("Cannot open config file %s in exclusive mode: %s", - filename, strerror(errno)); + log_err("Cannot open config file %s in exclusive mode (%s): %s", + filename, mode, strerror(errno)); fclose(fp); errno = _e; return NULL; @@ -96,26 +98,8 @@ void closeFTLtoml(FILE *fp) // Chown file if we are root if(geteuid() == 0) - { - // Get UID and GID of user with name "pihole" - struct passwd *pwd = getpwnam("pihole"); - if(pwd == NULL) - { - log_warn("Cannot get UID and GID of user pihole: %s", strerror(errno)); - } - else - { - const uid_t pihole_uid = pwd->pw_uid; - const gid_t pihole_gid = pwd->pw_gid; - // Chown file to pihole user - if(chown(GLOBALTOMLPATH, pihole_uid, pihole_gid) != 0) - log_warn("Cannot chown "GLOBALTOMLPATH" to pihole:pihole (%u:%u): %s", - (unsigned int)pihole_uid, (unsigned int)pihole_gid, strerror(errno)); - else - log_debug(DEBUG_CONFIG, "Chown-ed "GLOBALTOMLPATH" to pihole:pihole (%u:%u)", - (unsigned int)pihole_uid, (unsigned int)pihole_gid); - } - } + chown_pihole(GLOBALTOMLPATH, NULL); + return; } @@ -209,7 +193,7 @@ void print_comment(FILE *fp, const char *str, const char *intro, const unsigned // If this the first line? If not, add a newline if (i > 0) fputc('\n', fp); - // Add intendation + // Add indentation for (unsigned int j = 0; j != 2*indent; ++j) fputc(' ', fp); // Start a new line @@ -378,6 +362,9 @@ void writeTOMLvalue(FILE * fp, const int indent, const enum conf_type t, union c case CONF_ENUM_TEMP_UNIT: printTOMLstring(fp, get_temp_unit_str(v->temp_unit), toml); break; + case CONF_ENUM_BLOCKING_EDNS_MODE: + printTOMLstring(fp, get_edns_mode_str(v->edns_mode), toml); + break; case CONF_STRUCT_IN_ADDR: { // Special case: 0.0.0.0 -> return empty string @@ -428,7 +415,7 @@ void writeTOMLvalue(FILE * fp, const int indent, const enum conf_type t, union c if(strlen(item->valuestring) == 0) continue; - // Add intendation (if we are indenting) + // Add indentation (if we are indenting) if(indent > -1) indentTOML(fp, indent + 1); @@ -559,7 +546,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_PTR_TYPE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int ptr_type = get_ptr_type_val(val.u.s); @@ -575,7 +562,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_BUSY_TYPE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int busy_reply = get_busy_reply_val(val.u.s); @@ -591,7 +578,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_BLOCKING_MODE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int blocking_mode = get_blocking_mode_val(val.u.s); @@ -607,7 +594,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_REFRESH_HOSTNAMES: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int refresh_hostnames = get_refresh_hostnames_val(val.u.s); @@ -623,7 +610,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_LISTENING_MODE: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int listeningMode = get_listeningMode_val(val.u.s); @@ -639,7 +626,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_WEB_THEME: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int web_theme = get_web_theme_val(val.u.s); @@ -655,7 +642,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t } case CONF_ENUM_TEMP_UNIT: { - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { const int temp_unit = get_temp_unit_val(val.u.s); @@ -669,6 +656,22 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t log_debug(DEBUG_CONFIG, "%s DOES NOT EXIST or is not a valid string", conf_item->k); break; } + case CONF_ENUM_BLOCKING_EDNS_MODE: + { + toml_datum_t val = toml_string_in(toml, key); + if(val.ok) + { + const int edns_mode = get_edns_mode_val(val.u.s); + free(val.u.s); + if(edns_mode != -1) + conf_item->v.edns_mode = edns_mode; + else + log_warn("Config setting %s is invalid, allowed options are: %s", conf_item->k, conf_item->h); + } + else + log_debug(DEBUG_CONFIG, "%s DOES NOT EXIST or is not a valid string", conf_item->k); + break; + } case CONF_ENUM_PRIVACY_LEVEL: { const toml_datum_t val = toml_int_in(toml, key); @@ -681,7 +684,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t case CONF_STRUCT_IN_ADDR: { struct in_addr addr4 = { 0 }; - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { if(strlen(val.u.s) == 0) @@ -702,7 +705,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t case CONF_STRUCT_IN6_ADDR: { struct in6_addr addr6 = { 0 }; - const toml_datum_t val = toml_string_in(toml, key); + toml_datum_t val = toml_string_in(toml, key); if(val.ok) { if(strlen(val.u.s) == 0) @@ -733,7 +736,7 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t for(unsigned int i = 0; i < nelem; i++) { // Get string from TOML - const toml_datum_t d = toml_string_at(array, i); + toml_datum_t d = toml_string_at(array, i); if(!d.ok) { log_warn("Config %s is an invalid array (found at index %u)", conf_item->k, i); diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 2c24e102..6f3f98e3 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -115,10 +115,14 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, return false; } + // First, get an array of keys of config items that have been forced + // through environment variables + cJSON *env_vars = read_forced_vars(version); + // Try to read debug config. This is done before the full config // parsing to allow for debug output further down // First try to read env variable, if this fails, read TOML - if(teleporter || !readEnvValue(&newconf->debug.config, newconf)) + if(teleporter || !readEnvValue(&newconf->debug.config, newconf, env_vars, NULL)) { toml_table_t *conf_debug = toml_table_in(toml, "debug"); if(conf_debug) @@ -140,12 +144,28 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, // First try to read this config option from an environment variable // Skip reading environment variables when importing from Teleporter // If this succeeds, skip searching the TOML file for this config item - if(!teleporter && readEnvValue(new_conf_item, newconf)) + bool reset = false; + if(!teleporter && readEnvValue(new_conf_item, newconf, env_vars, &reset)) { new_conf_item->f |= FLAG_ENV_VAR; continue; } + // Skip this variable if it has been reset (forced by + // environment variable before but not anymore) + if(reset) + { + if(new_conf_item->t == CONF_ALL_DEBUG_BOOL) + { + // Reset all debug flags to false if debug.all + // has been reset + set_all_debug(newconf, false); + set_debug_flags(newconf); + } + log_info("Skipping %s as it has been reset", new_conf_item->k); + continue; + } + // Get config path depth unsigned int level = config_path_depth(new_conf_item->p); @@ -200,6 +220,7 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, // Free memory allocated by the TOML parser and return success toml_free(toml); + cJSON_Delete(env_vars); return true; } diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index afc6a35b..57a2090a 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -27,6 +27,19 @@ extern uint8_t last_checksum[SHA256_DIGEST_SIZE]; bool writeFTLtoml(const bool verbose) { + // Return early without writing if we are in config read-only mode + if(config.misc.readOnly.v.b) + { + log_debug(DEBUG_CONFIG, "Config file is read-only, not writing"); + + // We need to (re-)calculate the checksum here as it'd otherwise + // be outdated (in non-read-only mode, it's calculated at the + // end of this function) + if(!sha256sum(GLOBALTOMLPATH, last_checksum, false)) + log_err("Unable to create checksum of %s", GLOBALTOMLPATH); + return true; + } + // Try to open a temporary config file for writing FILE *fp; if((fp = openFTLtoml("w", 0)) == NULL) @@ -39,7 +52,7 @@ bool writeFTLtoml(const bool verbose) fprintf(fp, "# Pi-hole configuration file (%s)\n", get_FTL_version()); fputs("# Encoding: UTF-8\n", fp); fputs("# This file is managed by pihole-FTL\n", fp); - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; get_timestr(timestring, time(NULL), false, false); fputs("# Last updated on ", fp); fputs(timestring, fp); @@ -47,7 +60,8 @@ bool writeFTLtoml(const bool verbose) // Iterate over configuration and store it into the file char *last_path = (char*)""; - unsigned int modified = 0, env_vars = 0; + unsigned int modified = 0; + cJSON *env_vars = cJSON_CreateArray(); for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++) { // Get pointer to memory location of this conf_item @@ -82,13 +96,6 @@ bool writeFTLtoml(const bool verbose) print_toml_allowed_values(conf_item->a, fp, 85, level-1); } - // Print info if this value is overwritten by an env var - if(conf_item->f & FLAG_ENV_VAR) - { - print_comment(fp, ">>> This config is overwritten by an environmental variable <<<", "", 85, level-1); - env_vars++; - } - // Write value indentTOML(fp, level-1); fprintf(fp, "%s = ", conf_item->p[level-1]); @@ -105,7 +112,12 @@ bool writeFTLtoml(const bool verbose) if(changed) { - fprintf(fp, " ### CHANGED, default = "); + + // Print info if this value is overwritten by an env var + if(conf_item->f & FLAG_ENV_VAR) + cJSON_AddItemToArray(env_vars, cJSON_CreateStringReference(conf_item->k)); + + fprintf(fp, " ### CHANGED%s, default = ", conf_item->f & FLAG_ENV_VAR ? " (env)" : ""); writeTOMLvalue(fp, -1, conf_item->t, &conf_item->d); modified++; } @@ -114,6 +126,29 @@ bool writeFTLtoml(const bool verbose) fputs("\n\n", fp); } + // Print config file statistics at the end of the file as comment + fputs("# Configuration statistics:\n", fp); + fprintf(fp, "# %zu total entries out of which %zu %s default\n", + CONFIG_ELEMENTS, CONFIG_ELEMENTS - modified, + CONFIG_ELEMENTS - modified == 1 ? "entry is" : "entries are"); + fprintf(fp, "# --> %u %s modified\n", + modified, modified == 1 ? "entry is" : "entries are"); + + const unsigned int num_env_vars = cJSON_GetArraySize(env_vars); + if(num_env_vars > 0) + { + fprintf(fp, "# %u %s forced through environment:\n", + num_env_vars, num_env_vars == 1 ? "entry is" : "entries are"); + + for(unsigned int i = 0; i < num_env_vars; i++) + { + const char *env_var = cJSON_GetArrayItem(env_vars, i)->valuestring; + fprintf(fp, "# - %s\n", env_var); + } + } + else + fputc('\n', fp); + // Log some statistics in verbose mode if(verbose || config.debug.config.v.b) { @@ -123,10 +158,13 @@ bool writeFTLtoml(const bool verbose) CONFIG_ELEMENTS - modified == 1 ? "entry is" : "entries are"); log_info(" - %u %s modified", modified, modified == 1 ? "entry is" : "entries are"); - log_info(" - %u %s forced through environment", env_vars, - env_vars == 1 ? "entry is" : "entries are"); + log_info(" - %u %s forced through environment", num_env_vars, + num_env_vars == 1 ? "entry is" : "entries are"); } + // Free cJSON array + cJSON_Delete(env_vars); + // Close file and release exclusive lock closeFTLtoml(fp); @@ -171,7 +209,7 @@ bool writeFTLtoml(const bool verbose) log_debug(DEBUG_CONFIG, "pihole.toml unchanged"); } - if(!sha256sum(GLOBALTOMLPATH, last_checksum)) + if(!sha256sum(GLOBALTOMLPATH, last_checksum, false)) log_err("Unable to create checksum of %s", GLOBALTOMLPATH); return true; diff --git a/src/config/validator.c b/src/config/validator.c index 543057d7..06b8affd 100644 --- a/src/config/validator.c +++ b/src/config/validator.c @@ -123,12 +123,6 @@ bool validate_dns_cnames(union conf_value *val, const char *key, char err[VALIDA return false; } - // Count the number of elements in the string - unsigned int elements = 1; - for(unsigned int j = 0; j < strlen(item->valuestring); j++) - if(item->valuestring[j] == ',') - elements++; - // Check if it's in the form ",[,][,]" // is optional and may be repeated char *str = strdup(item->valuestring); @@ -398,12 +392,6 @@ bool validate_dns_revServers(union conf_value *val, const char *key, char err[VA return false; } - // Count the number of elements in the string - unsigned int elements = 1; - for(unsigned int j = 0; j < strlen(item->valuestring); j++) - if(item->valuestring[j] == ',') - elements++; - // Check if it's in the form ",[/],[#]," // Mandatory elements are: , , , and // Optional elements are: [/] and [#] diff --git a/src/daemon.c b/src/daemon.c index 0345e046..9c0e641a 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -265,16 +265,19 @@ pid_t FTL_gettid(void) static void terminate_threads(void) { - struct timespec ts; // Terminate threads before closing database connections and finishing shared memory killed = true; // Try to join threads to ensure cancellation has succeeded log_info("Waiting for threads to join"); for(int i = 0; i < THREADS_MAX; i++) { + log_debug(DEBUG_EXTRA, "Joining %s thread (%d)", thread_names[i], i); // Skip threads that have never been started or which are already stopped - if(!thread_running[i]) + if(threads[i] == 0) + { + log_debug(DEBUG_EXTRA, "Skipping thread as it was never started"); continue; + } // Cancel thread if it is idle if(thread_cancellable[i]) @@ -282,9 +285,12 @@ static void terminate_threads(void) log_info("Thread %s (%d) is idle, terminating it.", thread_names[i], i); pthread_cancel(threads[i]); + continue; } // Cancel thread if we cannot set a timeout for joining + struct timespec ts; + memset(&ts, 0, sizeof(ts)); if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { log_info("Thread %s (%d) is busy, cancelling it (cannot set timeout).", @@ -297,8 +303,7 @@ static void terminate_threads(void) ts.tv_sec += 2; // Try to join thread and cancel it if it is still busy - const int s = pthread_timedjoin_np(threads[i], NULL, &ts); - if(s != 0) + if(pthread_timedjoin_np(threads[i], NULL, &ts) != 0) { log_info("Thread %s (%d) is still busy, cancelling it.", thread_names[i], i); @@ -326,13 +331,32 @@ void set_nice(void) // Set nice value const int ret = setpriority(which, pid, config.misc.nice.v.i); if(ret == -1) - // ERROR EPERM: The calling process attempted to increase its priority - // by supplying a negative value but has insufficient privileges. - // On Linux, the RLIMIT_NICE resource limit can be used to define a limit to - // which an unprivileged process's nice value can be raised. We are not - // affected by this limit when pihole-FTL is running with CAP_SYS_NICE - log_warn("Cannot set process priority to %d: %s. Process priority remains at %d", - config.misc.nice.v.i, strerror(errno), priority); + { + if(errno == EACCES || errno == EPERM) + { + // from man 2 setpriority: + // + // ERRORS + // [...] + // EACCES The caller attempted to set a lower nice value (i.e., a higher + // process priority), but did not have the required privilege (on + // Linux: did not have the CAP_SYS_NICE capability). + // + // EPERM A process was located, but its effective user ID did not match + // either the effective or the real user ID of the caller, and was + // not privileged (on Linux: did not have the CAP_SYS_NICE capabil‐ + // ity). + // [...] + log_warn("Insufficient permissions to set process priority to %d (CAP_SYS_NICE required), process priority remains at %d", + config.misc.nice.v.i, priority); + } + else + { + // Other error + log_warn("Cannot set process priority to %d: %s. Process priority remains at %d", + config.misc.nice.v.i, strerror(errno), priority); + } + } } } @@ -377,7 +401,10 @@ void cleanup(const int ret) char buffer[42] = { 0 }; format_time(buffer, 0, timer_elapsed_msec(EXIT_TIMER)); - log_info("########## FTL terminated after%s (code %i)! ##########", buffer, ret); + if(ret == RESTART_FTL_CODE) + log_info("########## FTL terminated after%s (internal restart)! ##########", buffer); + else + log_info("########## FTL terminated after%s (code %i)! ##########", buffer, ret); } static float last_clock = 0.0f; diff --git a/src/database/CMakeLists.txt b/src/database/CMakeLists.txt index 3a16bf9b..fda25a1a 100644 --- a/src/database/CMakeLists.txt +++ b/src/database/CMakeLists.txt @@ -18,7 +18,11 @@ set(sqlite3_sources ) add_library(sqlite3 OBJECT ${sqlite3_sources}) -target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare) +target_compile_options(sqlite3 PRIVATE -Wno-implicit-fallthrough -Wno-cast-function-type -Wno-sign-compare -Wno-implicit-function-declaration -Wno-int-conversion) + +if (CMAKE_C_COMPILER_ID STREQUAL "Clang") + target_compile_options(sqlite3 PRIVATE "-Wno-null-pointer-subtraction") +endif() set(database_sources common.c diff --git a/src/database/aliasclients.c b/src/database/aliasclients.c index f060e217..e258f5c6 100644 --- a/src/database/aliasclients.c +++ b/src/database/aliasclients.c @@ -8,15 +8,15 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" #include "aliasclients.h" #include "common.h" // global counters variable -#include "../shmem.h" +#include "shmem.h" // global config variable -#include "../config/config.h" +#include "config/config.h" // logging routines -#include "../log.h" +#include "log.h" // getAliasclientIDfromIP() #include "network-table.h" @@ -109,6 +109,7 @@ bool import_aliasclients(sqlite3 *db) // Loop until no further data is available int imported = 0; + const double now = double_time(); while((rc = sqlite3_step(stmt)) != SQLITE_DONE) { // Check if we ran into an error @@ -132,7 +133,7 @@ bool import_aliasclients(sqlite3 *db) } // Try to open existing client - const int clientID = findClientID(aliasclient_str, false, true); + const int clientID = findClientID(aliasclient_str, false, true, now); clientsData *client = getClient(clientID, true); if(client == NULL) diff --git a/src/database/aliasclients.h b/src/database/aliasclients.h index 8eb6d2f7..a03617fc 100644 --- a/src/database/aliasclients.h +++ b/src/database/aliasclients.h @@ -11,7 +11,7 @@ #define ALIASCLIENTS_TABLE_H // type clientsData -#include "../datastructure.h" +#include "datastructure.h" bool create_aliasclients_table(sqlite3 *db); diff --git a/src/database/common.c b/src/database/common.c index 2161fd38..90895469 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -103,6 +103,15 @@ sqlite3* _dbopen(const bool readonly, const bool create, const char *func, const return NULL; } + // If the database is opened in read-write mode, actually check if it is + // writable. If it is not, close the database and return an error + if(!readonly && sqlite3_db_readonly(db, NULL)) + { + log_err("Cannot open database in read-write mode"); + dbclose(&db); + return NULL; + } + // Explicitly set busy handler to value defined in FTL.h rc = sqlite3_busy_timeout(db, DATABASE_BUSY_TIMEOUT); if( rc != SQLITE_OK ) @@ -239,25 +248,23 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg) // Note: pArg is NULL and not used // See https://sqlite.org/rescode.html#extrc for details // concerning the return codes returned here - if(strncmp(zMsg, "file renamed while open: ", sizeof("file renamed while open: ")-1) == 0) + if(zMsg != NULL && strncmp(zMsg, "file renamed while open: ", sizeof("file renamed while open: ")-1) == 0) { // This happens when gravity.db is replaced while FTL is running // We can safely ignore this warning return; } - // Log backtrace if any debug flag is set - if(config.debug.extra.v.b) - generate_backtrace(); - if(iErrCode == SQLITE_WARNING) log_warn("SQLite3: %s (%d)", zMsg, iErrCode); else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA) + { // SQLITE_SCHEMA is returned when the database schema has changed // This is not necessarily an error, as sqlite3_step() will re-prepare // the statement and try again. If it cannot, it will return an error // and this will be handled over there. log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode); + } else log_err("SQLite3: %s (%d)", zMsg, iErrCode); } @@ -284,9 +291,9 @@ void db_init(void) } } - // Explicitly set permissions to 0644 - // 644 = u+w u+r g+w g+r o+r - const mode_t mode = S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP| S_IROTH; + // Explicitly set permissions to 0640 + // 640 = u+w u+r g+r + const mode_t mode = S_IWUSR | S_IRUSR | S_IRGRP; chmod_file(config.files.database.v.s, mode); // Open database @@ -564,10 +571,48 @@ void db_init(void) dbversion = db_get_int(db, DB_VERSION); } + // Update to version 18 if lower + if(dbversion < 18) + { + // Update to version 18: Add cli column to session table + log_info("Updating long-term database to version 18"); + if(!add_session_cli_column(db)) + { + log_info("Session table cannot be updated, database not available"); + dbclose(&db); + return; + } + // Get updated version + dbversion = db_get_int(db, DB_VERSION); + } + + // Update to version 19 if lower + if(dbversion < 19) + { + // Update to version 19: Add x_forwarded_for column to session table + log_info("Updating long-term database to version 19"); + if(!add_session_x_forwarded_for_column(db)) + { + log_info("Session table cannot be updated, database not available"); + dbclose(&db); + return; + } + // Get updated version + dbversion = db_get_int(db, DB_VERSION); + } + + /* * * * * * * * * * * * * IMPORTANT * * * * * * * * * * * * * + * If you add a new database version, check if the in-memory + * schema needs to be update as well (always recreated from + * scratch on every FTL (re)start). Also, ensure to update the + * MEMDB_VERSION in src/database/query-table.h as well as the + * expected database schema in the CI tests. + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + // Last check after all migrations, if this happens, it will cause the // CI to fail the tests if(dbversion != MEMDB_VERSION) - log_err("Database version %i does not match MEMDB_VERSION %i", dbversion, MEMDB_VERSION); + log_err("Expected query database version %d but found %d", MEMDB_VERSION, dbversion); lock_shm(); import_aliasclients(db); @@ -633,17 +678,6 @@ bool db_set_FTL_property(sqlite3 *db, const enum ftl_table_props ID, const int v return true; } -bool db_set_FTL_property_double(sqlite3 *db, const enum ftl_table_props ID, const double value) -{ - int ret = dbquery(db, "INSERT OR REPLACE INTO ftl (id, value) VALUES ( %u, %f );", ID, value); - if(ret != SQLITE_OK) - { - checkFTLDBrc(ret); - return false; - } - return true; -} - bool db_set_counter(sqlite3 *db, const enum counters_table_props ID, const int value) { int ret = dbquery(db, "INSERT OR REPLACE INTO counters (id, value) VALUES ( %u, %d );", ID, value); @@ -655,25 +689,6 @@ bool db_set_counter(sqlite3 *db, const enum counters_table_props ID, const int v return true; } -bool db_update_counters(sqlite3 *db, const int total, const int blocked) -{ - int ret = dbquery(db, "UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES); - if(ret != SQLITE_OK) - { - checkFTLDBrc(ret); - return false; - } - - ret = dbquery(db, "UPDATE counters SET value = value + %i WHERE id = %i;", total, DB_TOTALQUERIES); - if(ret != SQLITE_OK) - { - checkFTLDBrc(ret); - return false; - } - - return true; -} - int db_query_int(sqlite3 *db, const char* querystr) { log_debug(DEBUG_DATABASE, "dbquery: \"%s\"", querystr); @@ -713,6 +728,98 @@ int db_query_int(sqlite3 *db, const char* querystr) return result; } +int db_query_int_int(sqlite3 *db, const char* querystr, const int arg) +{ + log_debug(DEBUG_DATABASE, "db_query_int_arg: \"%s\"", querystr); + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("Encountered prepare error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + // Bind argument to prepared statement + if((rc = sqlite3_bind_int(stmt, 1, arg)) != SQLITE_OK) + { + log_err("Encountered bind error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + log_debug(DEBUG_DATABASE, " ---> Result %i (int)", result); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + log_debug(DEBUG_DATABASE, " ---> No data"); + } + else + { + log_err("Encountered step error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + +int db_query_int_str(sqlite3 *db, const char* querystr, const char *arg) +{ + log_debug(DEBUG_DATABASE, "db_query_int_str: \"%s\"", querystr); + + sqlite3_stmt* stmt; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + if( rc != SQLITE_BUSY ) + log_err("Encountered prepare error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + // Bind argument to prepared statement + if((rc = sqlite3_bind_text(stmt, 1, arg, -1, SQLITE_STATIC)) != SQLITE_OK) + { + log_err("Encountered bind error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + } + + rc = sqlite3_step(stmt); + int result; + + if( rc == SQLITE_ROW ) + { + result = sqlite3_column_int(stmt, 0); + log_debug(DEBUG_DATABASE, " ---> Result %i (int)", result); + } + else if( rc == SQLITE_DONE ) + { + // No rows available + result = DB_NODATA; + log_debug(DEBUG_DATABASE, " ---> No data"); + } + else + { + log_err("Encountered step error in db_query_int(\"%s\"): %s", + querystr, sqlite3_errstr(rc)); + return DB_FAILED; + } + + sqlite3_finalize(stmt); + return result; +} + double db_query_double(sqlite3 *db, const char* querystr) { log_debug(DEBUG_DATABASE, "dbquery: \"%s\"", querystr); diff --git a/src/database/common.h b/src/database/common.h index d2369185..46d863eb 100644 --- a/src/database/common.h +++ b/src/database/common.h @@ -33,10 +33,9 @@ int db_get_int(sqlite3* db, const enum ftl_table_props ID); int db_get_FTL_property(sqlite3* db, const enum ftl_table_props ID); double db_get_FTL_property_double(sqlite3* db, const enum ftl_table_props ID); bool db_set_FTL_property(sqlite3* db, const enum ftl_table_props ID, const int value); -bool db_set_FTL_property_double(sqlite3* db, const enum ftl_table_props ID, const double value); /// Execute a formatted SQL query and get the return code -int dbquery(sqlite3* db, const char *format, ...) __attribute__ ((format (gnu_printf, 2, 3)));; +int dbquery(sqlite3* db, const char *format, ...) __attribute__ ((format (printf, 2, 3)));; #define dbopen(readonly, create) _dbopen(readonly, create, __FUNCTION__, __LINE__, __FILE__) sqlite3 *_dbopen(const bool readonly, const bool create, const char *func, const int line, const char *file) __attribute__((warn_unused_result)); @@ -45,13 +44,14 @@ void _dbclose(sqlite3 **db, const char *func, const int line, const char *file); void piholeFTLDB_reopen(void); int db_query_int(sqlite3 *db, const char *querystr); +int db_query_int_int(sqlite3 *db, const char* querystr, const int arg); +int db_query_int_str(sqlite3 *db, const char* querystr, const char *arg); double db_query_double(sqlite3 *db, const char *querystr); int db_query_int_from_until(sqlite3 *db, const char* querystr, const double from, const double until); int db_query_int_from_until_type(sqlite3 *db, const char* querystr, const double from, const double until, const int type); void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg); bool db_set_counter(sqlite3 *db, const enum counters_table_props ID, const int value); -bool db_update_counters(sqlite3 *db, const int total, const int blocked); const char *get_sqlite3_version(void); extern bool DBdeleteoldqueries; diff --git a/src/database/database-thread.c b/src/database/database-thread.c index efdd1bd6..3a83e425 100644 --- a/src/database/database-thread.c +++ b/src/database/database-thread.c @@ -78,14 +78,11 @@ static bool analyze_database(sqlite3 *db) } #define DBOPEN_OR_AGAIN() { if(!db) db = dbopen(false, false); if(!db) { thread_sleepms(DB, 5000); continue; } } -#define BREAK_IF_KILLED() { if(killed) break; } #define DBCLOSE_OR_BREAK() { dbclose(&db); BREAK_IF_KILLED(); } void *DB_thread(void *val) { // Set thread name - thread_names[DB] = "database"; - thread_running[DB] = true; prctl(PR_SET_NAME, thread_names[DB], 0, 0, 0); // Save timestamp as we do not want to store immediately @@ -243,6 +240,5 @@ void *DB_thread(void *val) dbclose(&db); log_info("Terminating database thread"); - thread_running[DB] = false; return NULL; } diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index fef07869..282c80dc 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -8,29 +8,30 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" #include "sqlite3.h" #include "gravity-db.h" // struct config -#include "../config/config.h" +#include "config/config.h" // logging routines -#include "../log.h" +#include "log.h" // getstr() -#include "../shmem.h" +#include "shmem.h" // SQLite3 prepared statement vectors -#include "../vector.h" +#include "vector.h" // log_subnet_warning() // logg_inaccessible_adlist #include "message-table.h" // getMACfromIP() #include "network-table.h" // struct DNSCacheData -#include "../datastructure.h" +#include "datastructure.h" // reset_aliasclient() #include "aliasclients.h" - // Definition of struct regexData -#include "../regex_r.h" +#include "regex_r.h" +// file_readable() +#include "files.h" // Prefix of interface names in the client table #define INTERFACE_SEP ":" @@ -49,6 +50,17 @@ static sqlite3_stmt* table_stmt = NULL; bool gravityDB_opened = false; static bool gravity_abp_format = false; +// Variables memorizing the parent gravity database connection and prepared +// statements to avoid valgrind warnings about memory leaks +static sqlite3 *parent_gravity_db = NULL; +sqlite3_stmt_vec *parent_whitelist_stmt = NULL; +sqlite3_stmt_vec *parent_gravity_stmt = NULL; +sqlite3_stmt_vec *parent_antigravity_stmt = NULL; +sqlite3_stmt_vec *parent_blacklist_stmt = NULL; + +// Private prototypes +static bool gravityDB_open(void); + // Table names corresponding to the enum defined in gravity-db.h static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , "client", "group", "adlist", "denied_domains", "allowed_domains", "" }; @@ -85,12 +97,17 @@ void gravityDB_forked(void) // is clear that this in not what we want to do as this is a slow // process and many TCP queries could lead to a DoS attack. gravityDB_opened = false; + parent_gravity_db = gravity_db; gravity_db = NULL; // Also pretend we have not yet prepared the list statements + parent_whitelist_stmt = whitelist_stmt; whitelist_stmt = NULL; + parent_blacklist_stmt = blacklist_stmt; blacklist_stmt = NULL; + parent_gravity_stmt = gravity_stmt; gravity_stmt = NULL; + parent_antigravity_stmt = antigravity_stmt; antigravity_stmt = NULL; // Open the database @@ -132,7 +149,7 @@ static void gravity_check_ABP_format(void) } // Open gravity database -bool gravityDB_open(void) +static bool gravityDB_open(void) { struct stat st; if(stat(config.files.gravity.v.s, &st) != 0) @@ -954,6 +971,7 @@ void gravityDB_close(void) free_sqlite3_stmt_vec(&antigravity_stmt); // Close table + log_debug(DEBUG_ANY, "Closing gravity database"); sqlite3_close(gravity_db); gravity_db = NULL; gravityDB_opened = false; @@ -1045,7 +1063,7 @@ inline const char* gravityDB_getDomain(int *rowid) // Finalize statement of a gravity database transaction void gravityDB_finalizeTable(void) { - if(!gravityDB_opened) + if(!gravityDB_opened || table_stmt == NULL) return; // Finalize statement @@ -1094,12 +1112,6 @@ int gravityDB_count(const enum gravity_tables list) case ADLISTS_TABLE: querystr = "SELECT COUNT(1) FROM adlist WHERE enabled != 0"; break; - case DENIED_DOMAINS_TABLE: - querystr = "SELECT COUNT(1) FROM domainlist WHERE (type = 0 OR type = 2) AND enabled != 0"; - break; - case ALLOWED_DOMAINS_TABLE: - querystr = "SELECT COUNT(1) FROM domainlist WHERE (type = 1 OR type = 3) AND enabled != 0"; - break; case UNKNOWN_TABLE: log_err("List type %u unknown!", list); gravityDB_close(); @@ -2715,9 +2727,17 @@ bool gravity_updated(void) sqlite3 *db = NULL; sqlite3_stmt *query_stmt = NULL; + // Check if database is a readable file + if(file_readable(config.files.gravity.v.s) == false) + { + log_err("Cannot read gravity database at %s - file does not exist or is not readable", + config.files.gravity.v.s); + return false; + } + // Open database int rc = sqlite3_open_v2(config.files.gravity.v.s, &db, SQLITE_OPEN_READONLY, NULL); - if(db == NULL) + if(db == NULL || rc != SQLITE_OK) { log_err("gravity_updated(): %s - SQL error open: %s", config.files.gravity.v.s, sqlite3_errstr(rc)); return false; @@ -2770,3 +2790,8 @@ bool gravity_updated(void) return changed; } + +time_t __attribute__((pure)) gravity_last_updated(void) +{ + return last_updated > 0 ? (time_t)last_updated : 0; +} diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index a77deb72..996569e8 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -11,11 +11,9 @@ #define GRAVITY_H // clients data structure -#include "../datastructure.h" -// enum http_method -#include "../webserver/http-common.h" +#include "datastructure.h" // Definition of struct regexData -#include "../regex_r.h" +#include "regex_r.h" // Table row record, not all fields are used by all tables typedef struct { @@ -41,7 +39,6 @@ typedef struct { time_t date_updated; } tablerow; -bool gravityDB_open(void); bool gravityDB_reopen(void); void gravityDB_forked(void); void gravityDB_reload_groups(clientsData* client); @@ -73,4 +70,6 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups, const tablerow *row, const char **message); +time_t gravity_last_updated(void) __attribute__((pure)); + #endif //GRAVITY_H diff --git a/src/database/message-table.c b/src/database/message-table.c index f164542f..a563a939 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -29,6 +29,45 @@ #include "files.h" // get_memdb() #include "database/query-table.h" +// escape_html() +#include "webserver/http-common.h" +// GIT_HASH, FTL_ARCH +#include "version.h" + +// Number of arguments in a variadic macro +// Credit: https://stackoverflow.com/a/35693080/2087442 +#define PP_NARG(...) \ + PP_NARG_(__VA_ARGS__,PP_RSEQ_N()) +#define PP_NARG_(...) \ + PP_128TH_ARG(__VA_ARGS__) +#define PP_128TH_ARG( \ + _1, _2, _3, _4, _5, _6, _7, _8, _9,_10, \ + _11,_12,_13,_14,_15,_16,_17,_18,_19,_20, \ + _21,_22,_23,_24,_25,_26,_27,_28,_29,_30, \ + _31,_32,_33,_34,_35,_36,_37,_38,_39,_40, \ + _41,_42,_43,_44,_45,_46,_47,_48,_49,_50, \ + _51,_52,_53,_54,_55,_56,_57,_58,_59,_60, \ + _61,_62,_63,_64,_65,_66,_67,_68,_69,_70, \ + _71,_72,_73,_74,_75,_76,_77,_78,_79,_80, \ + _81,_82,_83,_84,_85,_86,_87,_88,_89,_90, \ + _91,_92,_93,_94,_95,_96,_97,_98,_99,_100, \ + _101,_102,_103,_104,_105,_106,_107,_108,_109,_110, \ + _111,_112,_113,_114,_115,_116,_117,_118,_119,_120, \ + _121,_122,_123,_124,_125,_126,_127,N,...) N +#define PP_RSEQ_N() \ + 127,126,125,124,123,122,121,120, \ + 119,118,117,116,115,114,113,112,111,110, \ + 109,108,107,106,105,104,103,102,101,100, \ + 99,98,97,96,95,94,93,92,91,90, \ + 89,88,87,86,85,84,83,82,81,80, \ + 79,78,77,76,75,74,73,72,71,70, \ + 69,68,67,66,65,64,63,62,61,60, \ + 59,58,57,56,55,54,53,52,51,50, \ + 49,48,47,46,45,44,43,42,41,40, \ + 39,38,37,36,35,34,33,32,31,30, \ + 29,28,27,26,25,24,23,22,21,20, \ + 19,18,17,16,15,14,13,12,11,10, \ + 9,8,7,6,5,4,3,2,1,0 static const char *get_message_type_str(const enum message_type type) { @@ -58,6 +97,12 @@ static const char *get_message_type_str(const enum message_type type) return "DISK_EXTENDED"; case CERTIFICATE_DOMAIN_MISMATCH_MESSAGE: return "CERTIFICATE_DOMAIN_MISMATCH"; + case CONNECTION_ERROR_MESSAGE: + return "CONNECTION_ERROR"; + case NTP_MESSAGE: + return "NTP"; + case VERIFY_MESSAGE: + return "VERIFY"; case MAX_MESSAGE: default: return "UNKNOWN"; @@ -90,15 +135,21 @@ static enum message_type get_message_type_from_string(const char *typestr) return DISK_MESSAGE_EXTENDED; else if (strcmp(typestr, "CERTIFICATE_DOMAIN_MISMATCH") == 0) return CERTIFICATE_DOMAIN_MISMATCH_MESSAGE; + else if (strcmp(typestr, "CONNECTION_ERROR") == 0) + return CONNECTION_ERROR_MESSAGE; + else if (strcmp(typestr, "NTP") == 0) + return NTP_MESSAGE; + else if (strcmp(typestr, "VERIFY") == 0) + return VERIFY_MESSAGE; else return MAX_MESSAGE; } static unsigned char message_blob_types[MAX_MESSAGE][5] = { - { // REGEX_MESSAGE: The message column contains the regex warning text + { // REGEX_MESSAGE: The message column contains the regex text (the erroring regex filter itself) SQLITE_TEXT, // regex type ("deny", "allow") - SQLITE_TEXT, // regex text (the erroring regex filter itself) + SQLITE_TEXT, // regex warning text SQLITE_INTEGER, // database index of regex (so the dashboard can show a link) SQLITE_NULL, // not used SQLITE_NULL // not used @@ -181,6 +232,30 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] = SQLITE_NULL, // not used SQLITE_NULL, // not used SQLITE_NULL // not used + }, + { + // CONNECTION_ERROR_MESSAGE: The message column contains the server address + SQLITE_TEXT, // reason + SQLITE_TEXT, // error message + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL // not used + }, + { + // NTP: The message column contains the warning/error + SQLITE_TEXT, // level (warning/error) + SQLITE_TEXT, // component (server/client) + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL // not used + }, + { + // VERIFY_MESSAGE: The message column contains the error + SQLITE_TEXT, // expected checksum + SQLITE_TEXT, // actual checksum + SQLITE_TEXT, // FTL commit hash + SQLITE_TEXT, // FTL architecture + SQLITE_NULL // not used } }; // Create message table in the database @@ -224,21 +299,59 @@ bool flush_message_table(void) return true; } -static int add_message(const enum message_type type, - const char *message, const int count,...) +static int _add_message(const enum message_type type, + const char *message, const size_t count, ...); +#define add_message(type, message, ...) _add_message(type, message, PP_NARG(__VA_ARGS__), __VA_ARGS__) +#define add_message_no_args(type, message) _add_message(type, message, 0) + +static int _add_message(const enum message_type type, + const char *message, const size_t count,...) { + // Log to database only if not in CLI mode + if(cli_mode) + return -1; + int rowid = -1; // Return early if database is known to be broken if(FTLDBerror()) - return rowid; + return -1; + + // Check if message type is known + if(type >= MAX_MESSAGE) + { + log_err("add_message(type=%u, message=%s) - Invalid message type with %zu arguments", + type, message, count); + return -1; + } + + // Check if number of arguments is valid + // Total number of arguments + if(count > 5) + { + log_err("add_message(type=%u, message=%s) - Too many arguments (%zu), expected at most 5", + type, message, count); + return -1; + } + // No arguments check + if(count == 0 && message_blob_types[type][0] != SQLITE_NULL) + { + log_err("add_message(type=%u, message=%s) - Invalid number of arguments: No arguments passed for message type requiring arguments", + type, message); + return -1; + } + // Non-zero arguments check + else if(count > 1 && message_blob_types[type][count - 2] == SQLITE_NULL) + { + log_err("add_message(type=%u, message=%s) - Invalid number of arguments: Too many (%zu) arguments passed for this message type", + type, message, count); + return -1; + } sqlite3 *db; // Open database connection if((db = dbopen(false, false)) == NULL) - { - log_err("add_message() - Failed to open DB"); - return rowid; - } + // Reason for failure is logged in dbopen() + return -1; // Ensure there are no duplicates when adding messages sqlite3_stmt* stmt = NULL; @@ -315,7 +428,7 @@ static int add_message(const enum message_type type, va_list ap; va_start(ap, count); - for (int j = 0; j < count; j++) + for (size_t j = 0; j < count; j++) { const unsigned char datatype = message_blob_types[type][j]; switch (datatype) @@ -343,7 +456,7 @@ static int add_message(const enum message_type type, // Bind message to prepared statement if(rc != SQLITE_OK) { - log_err("add_message(type=%u, message=%s) - Failed to bind argument %d (type %u): %s", + log_err("add_message(type=%u, message=%s) - Failed to bind argument %zu (type %u): %s", type, message, 3 + j, datatype, sqlite3_errstr(rc)); sqlite3_reset(stmt); sqlite3_finalize(stmt); @@ -441,14 +554,22 @@ static void format_regex_message(char *plain, const int sizeof_plain, char *html char *escaped_regex = escape_html(regex); char *escaped_warning = escape_html(warning); + // Return early if memory allocation failed + if(escaped_regex == NULL || escaped_warning == NULL) + { + if(escaped_regex != NULL) + free(escaped_regex); + if(escaped_warning != NULL) + free(escaped_warning); + return; + } + if(snprintf(html, sizeof_html, "Encountered an error when processing regex %s filter with ID %d: 
%s
Error message: 
%s
", - dbindex, type, dbindex, escaped_regex, escaped_warning)) + dbindex, type, dbindex, escaped_regex, escaped_warning) > sizeof_html) log_warn("format_regex_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_regex != NULL) - free(escaped_regex); - if(escaped_warning != NULL) - free(escaped_warning); + free(escaped_regex); + free(escaped_warning); } static void format_subnet_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *ip, const int matching_count, const char *names, const char *matching_ids, const char *chosen_match_text, const int chosen_match_id) @@ -467,17 +588,26 @@ static void format_subnet_message(char *plain, const int sizeof_plain, char *htm char *escaped_ids = escape_html(matching_ids); char *escaped_names = escape_html(names); + // Return early if memory allocation failed + if(escaped_ip == NULL || escaped_ids == NULL || escaped_names == NULL) + { + if(escaped_ip != NULL) + free(escaped_ip); + if(escaped_ids != NULL) + free(escaped_ids); + if(escaped_names != NULL) + free(escaped_names); + return; + } + if(snprintf(html, sizeof_html, "Client %s is managed by %i groups (IDs [%s]), all describing the same subnet:
%s
" "FTL chose the most recent entry (ID %i) to obtain the group configuration for this client.", escaped_ip, matching_count, escaped_ids, escaped_names, chosen_match_id) > sizeof_html) log_warn("format_subnet_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_ip != NULL) - free(escaped_ip); - if(escaped_ids != NULL) - free(escaped_ids); - if(escaped_names != NULL) - free(escaped_names); + free(escaped_ip); + free(escaped_ids); + free(escaped_names); } static void format_hostname_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *ip, const char *name, const int pos) @@ -517,14 +647,24 @@ static void format_hostname_message(char *plain, const int sizeof_plain, char *h char *escaped_ip = escape_html(ip); char *escaped_name = escape_html(namep); + // Return early if memory allocation failed + if(escaped_ip == NULL || escaped_name == NULL) + { + if(escaped_ip != NULL) + free(escaped_ip); + if(escaped_name != NULL) + free(escaped_name); + if(namep != NULL) + free(namep); + return; + } + if(snprintf(html, sizeof_html, "Host name of client %s => %s contains (at least) one invalid character (hex %02x) at position %i", escaped_ip, escaped_name, (unsigned char)name[pos], pos) > sizeof_html) log_warn("format_hostname_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_ip != NULL) - free(escaped_ip); - if(escaped_name != NULL) - free(escaped_name); + free(escaped_ip); + free(escaped_name); if(namep != NULL) free(namep); } @@ -540,11 +680,14 @@ static void format_dnsmasq_config_message(char *plain, const int sizeof_plain, c char *escaped_message = escape_html(message); + // Return early if memory allocation failed + if(escaped_message == NULL) + return; + if(snprintf(html, sizeof_html, "FTL failed to start due to %s.", escaped_message) > sizeof_html) log_warn("format_dnsmasq_config_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_message != NULL) - free(escaped_message); + free(escaped_message); } static void format_rate_limit_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *clientIP, const unsigned int count, const unsigned int interval, const time_t turnaround) @@ -559,12 +702,15 @@ static void format_rate_limit_message(char *plain, const int sizeof_plain, char char *escaped_clientIP = escape_html(clientIP); + // Return early if memory allocation failed + if(escaped_clientIP == NULL) + return; + if(snprintf(html, sizeof_html, "Client %s has been rate-limited for at least %lu second%s (current limit: %u queries per %u seconds)", escaped_clientIP, (unsigned long int)turnaround, turnaround == 1 ? "" : "s", count, interval) > sizeof_html) log_warn("format_rate_limit_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_clientIP != NULL) - free(escaped_clientIP); + free(escaped_clientIP); } static void format_dnsmasq_warn_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, const char *message) @@ -608,14 +754,22 @@ static void format_shmem_message(char *plain, const int sizeof_plain, char *html char *escaped_path = escape_html(path); char *escaped_msg = escape_html(msg); + // Return early if memory allocation failed + if(escaped_path == NULL || escaped_msg == NULL) + { + if(escaped_path != NULL) + free(escaped_path); + if(escaped_msg != NULL) + free(escaped_msg); + return; + } + if(snprintf(html, sizeof_html, "Shared memory shortage (%s) ahead: %d%% is used
%s", escaped_path, shmem, escaped_msg) > sizeof_html) log_warn("log_resource_shortage(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_path != NULL) - free(escaped_path); - if(escaped_msg != NULL) - free(escaped_msg); + free(escaped_path); + free(escaped_msg); } static void format_disk_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -632,10 +786,22 @@ static void format_disk_message(char *plain, const int sizeof_plain, char *html, char *escaped_path = escape_html(path); char *escaped_msg = escape_html(msg); + // Return early if memory allocation failed + if(escaped_path == NULL || escaped_msg == NULL) + { + if(escaped_path != NULL) + free(escaped_path); + if(escaped_msg != NULL) + free(escaped_msg); + return; + } if(snprintf(html, sizeof_html, "Disk shortage ahead: %d%% is used (%s) on partition containing the file %s", disk, escaped_msg, escaped_path) > sizeof_html) log_warn("format_disk_message(): Buffer too small to hold HTML message, warning truncated"); + + free(escaped_path); + free(escaped_msg); } static void format_disk_message_extended(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -653,16 +819,25 @@ static void format_disk_message_extended(char *plain, const int sizeof_plain, ch char *escaped_mnt_dir = escape_html(mnt_dir); char *escaped_msg = escape_html(msg); + // Return early if memory allocation failed + if(escaped_mnt_type == NULL || escaped_mnt_dir == NULL || escaped_msg == NULL) + { + if(escaped_mnt_type != NULL) + free(escaped_mnt_type); + if(escaped_mnt_dir != NULL) + free(escaped_mnt_dir); + if(escaped_msg != NULL) + free(escaped_msg); + return; + } + if(snprintf(html, sizeof_html, "Disk shortage ahead: %d%% is used (%s) on %s filesystem mounted at %s", disk, escaped_msg, escaped_mnt_type, escaped_mnt_dir) > sizeof_html) log_warn("format_disk_message_extended(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_mnt_type != NULL) - free(escaped_mnt_type); - if(escaped_mnt_dir != NULL) - free(escaped_mnt_dir); - if(escaped_msg != NULL) - free(escaped_msg); + free(escaped_mnt_type); + free(escaped_mnt_dir); + free(escaped_msg); } static void format_inaccessible_adlist_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -678,12 +853,15 @@ static void format_inaccessible_adlist_message(char *plain, const int sizeof_pla char *escaped_address = escape_html(address); + // Return early if memory allocation failed + if(escaped_address == NULL) + return; + if(snprintf(html, sizeof_html, "List with ID %d (%s) was inaccessible during last gravity run", dbindex, dbindex, escaped_address) > sizeof_html) log_warn("format_inaccessible_adlist_message(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_address != NULL) - free(escaped_address); + free(escaped_address); } static void format_certificate_domain_mismatch(char *plain, const int sizeof_plain, char *html, const int sizeof_html, @@ -699,13 +877,102 @@ static void format_certificate_domain_mismatch(char *plain, const int sizeof_pla char *escaped_certfile = escape_html(certfile); char *escaped_domain = escape_html(domain); + // Return early if memory allocation failed + if(escaped_certfile == NULL || escaped_domain == NULL) + { + if(escaped_certfile != NULL) + free(escaped_certfile); + if(escaped_domain != NULL) + free(escaped_domain); + return; + } + if(snprintf(html, sizeof_html, "SSL/TLS certificate %s does not match domain %s!", escaped_certfile, escaped_domain) > sizeof_html) log_warn("format_certificate_domain_mismatch(): Buffer too small to hold HTML message, warning truncated"); - if(escaped_certfile != NULL) - free(escaped_certfile); - if(escaped_domain != NULL) - free(escaped_domain); + free(escaped_certfile); + free(escaped_domain); +} + +static void format_connection_error(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *server, const char *reason, const char *error) +{ + if(snprintf(plain, sizeof_plain, "Connection error (%s): %s (%s)", server, reason, error) > sizeof_plain) + log_warn("format_connection_error(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + char *escaped_reason = escape_html(reason); + char *escaped_error = escape_html(error); + char *escaped_server = escape_html(server); + + // Return early if memory allocation failed + if(escaped_reason == NULL || escaped_error == NULL || escaped_server == NULL) + { + if(escaped_reason != NULL) + free(escaped_reason); + if(escaped_error != NULL) + free(escaped_error); + if(escaped_server != NULL) + free(escaped_server); + return; + } + + if(snprintf(html, sizeof_html, "Connection error (%s): %s (%s)", server, reason, error) > sizeof_html) + log_warn("format_connection_error(): Buffer too small to hold HTML message, warning truncated"); + + free(escaped_reason); + free(escaped_error); + free(escaped_server); +} + +static void format_ntp_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *message, const char *level, const char *who) +{ + if(snprintf(plain, sizeof_plain, "%s NTP %s: %s", level, who, message) > sizeof_plain) + log_warn("format_ntp_message(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + if(snprintf(html, sizeof_html, "%s in NTP %s:
%s
", level, who, message) > sizeof_html) + log_warn("format_ntp_message(): Buffer too small to hold HTML message, warning truncated"); +} + +static void format_verify_message(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *message, const char *expected, const char *actual, + const char *commit, const char *arch) +{ + if(snprintf(plain, sizeof_plain, "%s - expected \"%s\", but got \"%s\" - FTL commit is %s on %s", + message, expected, actual, commit, arch) > sizeof_plain) + log_warn("format_verify_message(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + char *escaped_message = escape_html(message); + char *escaped_expected = escape_html(expected); + char *escaped_actual = escape_html(actual); + char *escaped_commit = escape_html(commit); + char *escaped_arch = escape_html(arch); + + // Return early if memory allocation failed + if(escaped_message == NULL || escaped_expected == NULL || escaped_actual == NULL || escaped_commit == NULL || escaped_arch == NULL) + return; + + if(snprintf(html, sizeof_html, "%s
Expected: 
%s

Actual: 
%s

FTL commit is %s on %s", + escaped_message, escaped_expected, escaped_actual, escaped_commit, escaped_arch) > sizeof_html) + log_warn("format_verify_message(): Buffer too small to hold HTML message, warning truncated"); + + free(escaped_message); + free(escaped_expected); + free(escaped_actual); + free(escaped_commit); + free(escaped_arch); } int count_messages(const bool filter_dnsmasq_warnings) @@ -796,14 +1063,14 @@ bool format_messages(cJSON *array) // Generate messages char plain[1024] = { 0 }, html[2048] = { 0 }; - const int mtype = get_message_type_from_string(mtypestr); + const enum message_type mtype = get_message_type_from_string(mtypestr); switch(mtype) { case REGEX_MESSAGE: { - const char *warning = (const char*)sqlite3_column_text(stmt, 3); + const char *regex = (const char*)sqlite3_column_text(stmt, 3); const char *type = (const char*)sqlite3_column_text(stmt, 4); - const char *regex = (const char*)sqlite3_column_text(stmt, 5); + const char *warning = (const char*)sqlite3_column_text(stmt, 5); const int dbindex = sqlite3_column_int(stmt, 6); format_regex_message(plain, sizeof(plain), html, sizeof(html), @@ -942,6 +1209,49 @@ bool format_messages(cJSON *array) break; } + + case CONNECTION_ERROR_MESSAGE: + { + const char *server = (const char*)sqlite3_column_text(stmt, 3); + const char *reason = (const char*)sqlite3_column_text(stmt, 4); + const char *error = (const char*)sqlite3_column_text(stmt, 5); + + format_connection_error(plain, sizeof(plain), html, sizeof(html), + server, reason, error); + + break; + } + + case NTP_MESSAGE: + { + const char *message = (const char*)sqlite3_column_text(stmt, 3); + const char *level = (const char*)sqlite3_column_text(stmt, 4); + const char *who = (const char*)sqlite3_column_text(stmt, 5); + + format_ntp_message(plain, sizeof(plain), html, sizeof(html), + message, level, who); + + break; + } + + case VERIFY_MESSAGE: + { + const char *message = (const char*)sqlite3_column_text(stmt, 3); + const char *expected = (const char*)sqlite3_column_text(stmt, 4); + const char *actual = (const char*)sqlite3_column_text(stmt, 5); + const char *hash = (const char*)sqlite3_column_text(stmt, 6); + const char *arch = (const char*)sqlite3_column_text(stmt, 7); + + format_verify_message(plain, sizeof(plain), html, sizeof(html), + message, expected, actual, hash, arch); + + break; + } + + case MAX_MESSAGE: // Fall through + default: + log_warn("format_messages() - Unknown message type: %s", mtypestr); + break; } // Add the plain message @@ -992,14 +1302,8 @@ void logg_regex_warning(const char *type, const char *warning, const int dbindex // Log to FTL.log log_warn("%s", buf); - // Log to database only if not in CLI mode - if(cli_mode) - return; - // Add to database - const int rowid = add_message(REGEX_MESSAGE, warning, 3, type, regex, dbindex); - if(rowid == -1) - log_err("logg_regex_warning(): Failed to add message to database"); + add_message(REGEX_MESSAGE, regex, type, warning, dbindex); } void logg_subnet_warning(const char *ip, const int matching_count, const char *matching_ids, @@ -1017,10 +1321,8 @@ void logg_subnet_warning(const char *ip, const int matching_count, const char *m log_warn("%s", buf); // Log to database - const int rowid = add_message(SUBNET_MESSAGE, ip, 5, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); + add_message(SUBNET_MESSAGE, ip, matching_count, names, matching_ids, chosen_match_text, chosen_match_id); - if(rowid == -1) - log_err("logg_subnet_warning(): Failed to add message to database"); free(names); } @@ -1039,10 +1341,8 @@ void logg_hostname_warning(const char *ip, const char *name, const unsigned int log_warn("%s", buf); // Log to database - const int rowid = add_message(HOSTNAME_MESSAGE, ip, 2, name, (const int)pos); + add_message(HOSTNAME_MESSAGE, ip, name, (const int)pos); - if(rowid == -1) - log_err("logg_hostname_warning(): Failed to add message to database"); } void logg_fatal_dnsmasq_message(const char *message) @@ -1055,10 +1355,8 @@ void logg_fatal_dnsmasq_message(const char *message) log_crit("%s", buf); // Log to database - const int rowid = add_message(DNSMASQ_CONFIG_MESSAGE, message, 0); + add_message_no_args(DNSMASQ_CONFIG_MESSAGE, message); - if(rowid == -1) - log_err("logg_fatal_dnsmasq_message(): Failed to add message to database"); } void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit_count) @@ -1073,10 +1371,8 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit log_info("%s", buf); // Log to database - const int rowid = add_message(RATE_LIMIT_MESSAGE, clientIP, 3, config.dns.rateLimit.count.v.ui, config.dns.rateLimit.interval.v.ui, turnaround); + add_message(RATE_LIMIT_MESSAGE, clientIP, config.dns.rateLimit.count.v.ui, config.dns.rateLimit.interval.v.ui, turnaround); - if(rowid == -1) - log_err("logg_rate_limit_message(): Failed to add message to database"); } void logg_warn_dnsmasq_message(char *message) @@ -1089,10 +1385,8 @@ void logg_warn_dnsmasq_message(char *message) log_warn("%s", buf); // Log to database - const int rowid = add_message(DNSMASQ_WARN_MESSAGE, message, 0); + add_message_no_args(DNSMASQ_WARN_MESSAGE, message); - if(rowid == -1) - log_err("logg_warn_dnsmasq_message(): Failed to add message to database"); } void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg) @@ -1108,10 +1402,9 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = add_message(LOAD_MESSAGE, "excessive load", 2, load, nprocs); + add_message(LOAD_MESSAGE, "excessive load", load, nprocs); + - if(rowid == -1) - log_err("log_resource_shortage(): Failed to add message to database"); } else if(shmem > -1) { @@ -1121,16 +1414,32 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = add_message(SHMEM_MESSAGE, path, 2, shmem, msg); + add_message(SHMEM_MESSAGE, path, shmem, msg); + - if(rowid == -1) - log_err("log_resource_shortage(): Failed to add message to database"); } else if(disk > -1) { // Get filesystem details for this path struct mntent *fsdetails = get_filesystem_details(path); + // Log filesystem details if in debug mode + if(config.debug.gc.v.b) + { + if(fsdetails != NULL) + { + log_debug(DEBUG_GC, "Disk details for path \"%s\":", path); + log_debug(DEBUG_GC, " Device or server for filesystem: %s", fsdetails->mnt_fsname); + log_debug(DEBUG_GC, " Directory mounted on: %s", fsdetails->mnt_dir); + log_debug(DEBUG_GC, " Type of filesystem: %s", fsdetails->mnt_type); + log_debug(DEBUG_GC, " Comma-separated options for fs: %s", fsdetails->mnt_opts); + log_debug(DEBUG_GC, " Dump frequency (in days): %d", fsdetails->mnt_freq); + log_debug(DEBUG_GC, " Pass number for `fsck': %d", fsdetails->mnt_passno); + } + else + log_debug(DEBUG_GC, "Failed to get filesystem details for path \"%s\"", path); + } + // Create plain message if(fsdetails != NULL) format_disk_message_extended(buf, sizeof(buf), NULL, 0, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir); @@ -1141,12 +1450,11 @@ void log_resource_shortage(const double load, const int nprocs, const int shmem, log_warn("%s", buf); // Log to database - const int rowid = fsdetails != NULL ? - add_message(DISK_MESSAGE_EXTENDED, path, 4, disk, fsdetails->mnt_type, fsdetails->mnt_dir) : - add_message(DISK_MESSAGE, path, 2, disk, msg); + fsdetails != NULL ? + add_message(DISK_MESSAGE_EXTENDED, path, disk, msg, fsdetails->mnt_type, fsdetails->mnt_dir) : + add_message(DISK_MESSAGE, path, disk, msg); + - if(rowid == -1) - log_err("log_resource_shortage(): Failed to add message to database"); } } @@ -1160,10 +1468,8 @@ void logg_inaccessible_adlist(const int dbindex, const char *address) log_warn("%s", buf); // Log to database - const int rowid = add_message(INACCESSIBLE_ADLIST_MESSAGE, address, 1, dbindex); + add_message(INACCESSIBLE_ADLIST_MESSAGE, address, dbindex); - if(rowid == -1) - log_err("logg_inaccessible_adlist(): Failed to add message to database"); } void log_certificate_domain_mismatch(const char *certfile, const char *domain) @@ -1176,8 +1482,54 @@ void log_certificate_domain_mismatch(const char *certfile, const char *domain) log_warn("%s", buf); // Log to database - const int rowid = add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, 1, domain); + add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, domain); + +} + +void log_connection_error(const char *server, const char *reason, const char *error) +{ + // Create message + char buf[2048]; + format_connection_error(buf, sizeof(buf), NULL, 0, server, reason, error); + + // Log to FTL.log + log_warn("%s", buf); + + // Log to database + add_message(CONNECTION_ERROR_MESSAGE, server, reason, error); + +} + +void log_ntp_message(const bool error, const bool server, const char *message) +{ + const char *who = server ? "server" : "client"; + const char *level = error ? "Error" : "Warning"; + + // Create message + char buf[2048]; + format_ntp_message(buf, sizeof(buf), NULL, 0, message, level, who); + + // Log to FTL.log + if(error) + log_err("%s", buf); + else + log_warn("%s", buf); + + // Log to database + add_message(NTP_MESSAGE, message, level, who); + +} + +void log_verify_message(const char *expected, const char *actual) +{ + // Create message + char buf[2048]; + snprintf(buf, sizeof(buf), "Corrupt binary detected - this may lead to unexpected behaviour!"); + + // Log to FTL.log + log_crit("%s", buf); + + // Log to database + add_message(VERIFY_MESSAGE, buf, expected, actual, GIT_HASH, FTL_ARCH); - if(rowid == -1) - log_err("log_certificate_domain_mismatch(): Failed to add message to database"); } diff --git a/src/database/message-table.h b/src/database/message-table.h index 14956bf5..d230f066 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -29,5 +29,8 @@ void logg_warn_dnsmasq_message(char *message); void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg); void logg_inaccessible_adlist(const int dbindex, const char *address); void log_certificate_domain_mismatch(const char *certfile, const char *domain); +void log_connection_error(const char *server, const char *reason, const char *error); +void log_ntp_message(const bool error, const bool server, const char *message); +void log_verify_message(const char *expected, const char *actual); #endif //MESSAGETABLE_H diff --git a/src/database/network-table.c b/src/database/network-table.c index 1e2a8817..686e1ed4 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -8,21 +8,21 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" #include "network-table.h" #include "common.h" -#include "../shmem.h" -#include "../log.h" +#include "shmem.h" +#include "log.h" // timer_elapsed_msec() -#include "../timers.h" -#include "../config/config.h" -#include "../datastructure.h" +#include "timers.h" +#include "config/config.h" +#include "datastructure.h" // struct config -#include "../config/config.h" +#include "config/config.h" // resolve_this_name() -#include "../resolve.h" +#include "resolve.h" // killed -#include "../signals.h" +#include "signals.h" // Private prototypes static char *getMACVendor(const char *hwaddr) __attribute__ ((malloc)); @@ -210,23 +210,13 @@ static int find_device_by_recent_ip(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return -1; - char *querystr = NULL; - int ret = asprintf(&querystr, - "SELECT network_id FROM network_addresses " - "WHERE ip = \'%s\' AND " - "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " - "ORDER BY lastSeen DESC LIMIT 1;", ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_recent_ip(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT network_id FROM network_addresses " + "WHERE ip = ?1 AND " + "lastSeen > (cast(strftime('%%s', 'now') as int)-86400) " + "ORDER BY lastSeen DESC LIMIT 1;"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - querystr = NULL; + int network_id = db_query_int_str(db, querystr, ipaddr); if(network_id == DB_FAILED) { @@ -252,20 +242,10 @@ static int find_device_by_mock_hwaddr(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'ip-%s\';", ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_mock_hwaddr(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE hwaddr = concat('ip-',?1)"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, ipaddr); } // Try to find device by hardware address @@ -275,20 +255,10 @@ static int find_device_by_hwaddr(sqlite3 *db, const char hwaddr[]) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, "SELECT id FROM network WHERE hwaddr = \'%s\' COLLATE NOCASE;", hwaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_hwaddr(\"%s\"): %i", - hwaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE hwaddr = ?1 COLLATE NOCASE;"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, hwaddr); } // Try to find device by RECENT mock hardware address (generated from IP address) @@ -298,24 +268,12 @@ static int find_recent_device_by_mock_hwaddr(sqlite3 *db, const char *ipaddr) if(FTLDBerror()) return DB_FAILED; - char *querystr = NULL; - int ret = asprintf(&querystr, - "SELECT id FROM network WHERE " - "hwaddr = \'ip-%s\' AND " - "firstSeen > (cast(strftime('%%s', 'now') as int)-3600);", - ipaddr); - if(querystr == NULL || ret < 0) - { - log_warn("Memory allocation failed in find_device_by_recent_mock_hwaddr(\"%s\"): %i", - ipaddr, ret); - return -1; - } + const char *querystr = "SELECT id FROM network WHERE " + "hwaddr = concat('ip-',?1) AND " + "firstSeen > (cast(strftime('%%s', 'now') as int)-3600)"; // Perform SQL query - int network_id = db_query_int(db, querystr); - free(querystr); - - return network_id; + return db_query_int_str(db, querystr, ipaddr); } // Store hostname of device identified by dbID @@ -510,8 +468,8 @@ static int add_netDB_network_address(sqlite3 *db, const int network_id, const ch } // Insert a new record into the network table -static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time_t lastQuery, - unsigned int numQueriesARP, const char *macVendor) +static int insert_netDB_device(sqlite3 *db, const char *hwaddr, const time_t firstSeen, const time_t lastQuery, + const unsigned int numQueriesARP, const char *macVendor) { // Return early if database is known to be broken if(FTLDBerror()) @@ -526,29 +484,29 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if(rc != SQLITE_OK) { log_err("insert_netDB_device(\"%s\", %lu, %lu, %u, \"%s\") - SQL error prepare (%i): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); checkFTLDBrc(rc); return rc; } log_debug(DEBUG_DATABASE, "dbquery: \"%s\" with arguments ?1-?5 = (\"%s\", %lu, %lu, %u, \"%s\")", - querystr, hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor); + querystr, hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor); // Bind hwaddr to prepared statement (1st argument) if((rc = sqlite3_bind_text(query_stmt, 1, hwaddr, -1, SQLITE_STATIC)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\", %lu, %lu, %u, \"%s\"): Failed to bind hwaddr (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; } - // Bind now to prepared statement (2nd argument) - if((rc = sqlite3_bind_int(query_stmt, 2, now)) != SQLITE_OK) + // Bind firstSeen to prepared statement (2nd argument) + if((rc = sqlite3_bind_int(query_stmt, 2, firstSeen)) != SQLITE_OK) { - log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind now (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind firstSeen (error %d): %s", + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -558,7 +516,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_int(query_stmt, 3, lastQuery)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind lastQuery (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -568,7 +526,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_int(query_stmt, 4, numQueriesARP)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind numQueriesARP (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -578,7 +536,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if((rc = sqlite3_bind_text(query_stmt, 5, macVendor, -1, SQLITE_STATIC)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to bind macVendor (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -588,7 +546,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if ((rc = sqlite3_step(query_stmt)) != SQLITE_DONE) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to step (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -598,7 +556,7 @@ static int insert_netDB_device(sqlite3 *db, const char *hwaddr, time_t now, time if ((rc = sqlite3_finalize(query_stmt)) != SQLITE_OK) { log_err("insert_netDB_device(\"%s\",%lu, %lu, %u, \"%s\"): Failed to finalize (error %d): %s", - hwaddr, (unsigned long)now, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); + hwaddr, (unsigned long)firstSeen, (unsigned long)lastQuery, numQueriesARP, macVendor, rc, sqlite3_errstr(rc)); sqlite3_reset(query_stmt); checkFTLDBrc(rc); return rc; @@ -901,9 +859,10 @@ static bool add_FTL_clients_to_network_table(sqlite3 *db, const enum arp_status // Add new device to database const time_t lastQuery = client->lastQuery; - const unsigned int numQueriesARP = client->numQueriesARP; + const time_t firstSeen = client->firstSeen; + const unsigned int numQueries = client->count; unlock_shm(); - insert_netDB_device(db, hwaddr, now, lastQuery, numQueriesARP, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueries, macVendor); lock_shm(); // Reacquire client pointer (if may have changed when unlocking above) @@ -1136,30 +1095,9 @@ static bool add_local_interfaces_to_network_table(sqlite3 *db, time_t now, unsig int lastQuery = 0, firstSeen = now, numQueries = 0; if(mockID >= 0) { - char *querystr = NULL; - if(asprintf(&querystr, "SELECT lastQuery from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - lastQuery = db_query_int(db, querystr); - free(querystr); - - if(asprintf(&querystr, "SELECT firstSeen from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - firstSeen = db_query_int(db, querystr); - free(querystr); - - if(asprintf(&querystr, "SELECT numQueries from network where id = %i", mockID) < 10) - { - free(macVendor); - return false; - } - numQueries = db_query_int(db, querystr); - free(querystr); + lastQuery = db_query_int_int(db, "SELECT lastQuery from network where id = ?1", mockID); + firstSeen = db_query_int_int(db, "SELECT firstSeen from network where id = ?1", mockID); + numQueries = db_query_int_int(db, "SELECT numQueries from network where id = ?1", mockID); } // Add new device to database @@ -1251,7 +1189,7 @@ void parse_neighbor_cache(sqlite3* db) char *linebuffer = NULL; size_t linebuffersize = 0u; unsigned int entries = 0u, additional_entries = 0u; - time_t now = time(NULL); + const time_t now = time(NULL); // Start ARP timer if(config.debug.arp.v.b) @@ -1298,6 +1236,7 @@ void parse_neighbor_cache(sqlite3* db) if((arpfp = popen(cmd, "r")) == NULL) { log_warn("Command \"%s\" failed: %s", cmd, strerror(errno)); + free(client_status); return; } @@ -1330,8 +1269,11 @@ void parse_neighbor_cache(sqlite3* db) { // This line is incomplete, remember this to skip // mock-device creation after ARP processing + // both false = do not create a new record if the client + // is unknown (only DNS requesting clients + // do this), the now value is ignored lock_shm(); - int clientID = findClientID(ip, false, false); + int clientID = findClientID(ip, false, false, 0.0); unlock_shm(); if(clientID >= 0 && clientID < clients) client_status[clientID] = CLIENT_ARP_INCOMPLETE; @@ -1364,16 +1306,19 @@ void parse_neighbor_cache(sqlite3* db) // If we reach this point, we can check if this client // is known to pihole-FTL - // false = do not create a new record if the client is - // unknown (only DNS requesting clients do this) + // both false = do not create a new record if the client + // is unknown (only DNS requesting clients + // do this), the now value is ignored lock_shm(); - int clientID = findClientID(ip, false, false); + int clientID = findClientID(ip, false, false, 0.0); - // Get hostname of this client if the client is known + // Set default values for a new device, may be updated + // below if the client is known to pihole-FTL char *hostname = NULL; bool client_valid = false; time_t lastQuery = 0; - unsigned int numQueries = 0; + time_t firstSeen = now; + unsigned int numQueries = 0, totalQueries = 0; // This client is known (by its IP address) to pihole-FTL if // findClientID() returned a non-negative index @@ -1383,14 +1328,21 @@ void parse_neighbor_cache(sqlite3* db) if(!client) continue; + // Client is known to Pi-hole, update properties + // with their real values client_valid = true; hostname = strdup(getstr(client->namepos)); + firstSeen = client->firstSeen; lastQuery = client->lastQuery; numQueries = client->numQueriesARP; + totalQueries = client->count; client_status[clientID] = CLIENT_ARP_COMPLETE; } else { + // Client is not known to Pi-hole, create a + // mock-device with the default values set above + // and an empty hostname hostname = strdup(""); } unlock_shm(); @@ -1405,6 +1357,15 @@ void parse_neighbor_cache(sqlite3* db) // and the ARP entry just came a bit delayed (reported by at least one user) dbID = find_recent_device_by_mock_hwaddr(db, ip); + // Exception for the case where the device is + // not yet in the database: Use total count of + // queries as the number of queries for the new + // device instead of the special ARP cache + // counter to add also the number of queries in + // the DNS history imported from the long-term + // database + numQueries = totalQueries; + if(dbID == DB_NODATA) { // Device not known AND no recent mock-device found ---> create new device record @@ -1412,7 +1373,7 @@ void parse_neighbor_cache(sqlite3* db) hwaddr, ip, hostname, macVendor); // Create new record (INSERT) - insert_netDB_device(db, hwaddr, now, lastQuery, numQueries, macVendor); + insert_netDB_device(db, hwaddr, firstSeen, lastQuery, numQueries, macVendor); lock_shm(); clientsData *client = getClient(clientID, true); @@ -1790,33 +1751,56 @@ void updateMACVendorRecords(sqlite3 *db) // Get vendor for MAC char *vendor = getMACVendor(hwaddr); + + // Free allocated memory free(hwaddr); hwaddr = NULL; - // Prepare UPDATE statement - char *updatestr = NULL; - if(asprintf(&updatestr, "UPDATE network SET macVendor = \'%s\' WHERE id = %i", vendor, id) < 1) + // Prepare statement + sqlite3_stmt *stmt2 = NULL; + const char *updatestr = "UPDATE network SET macVendor = ?1 WHERE id = ?2"; + rc = sqlite3_prepare_v2(db, updatestr, -1, &stmt2, NULL); + if(rc != SQLITE_OK) { - log_err("updateMACVendorRecords() - Allocation error"); + log_err("updateMACVendorRecords() - SQL error prepare \"%s\": %s", updatestr, sqlite3_errstr(rc)); + checkFTLDBrc(rc); free(vendor); break; } - // Execute prepared statement - char *zErrMsg = NULL; - rc = sqlite3_exec(db, updatestr, NULL, NULL, &zErrMsg); - if(rc != SQLITE_OK) + // Bind vendor to prepared statement + if((rc = sqlite3_bind_text(stmt2, 1, vendor, -1, SQLITE_STATIC)) != SQLITE_OK) { - log_err("updateMACVendorRecords() - SQL exec error: \"%s\": %s", updatestr, zErrMsg); + log_err("updateMACVendorRecords() - Failed to bind vendor: %s", sqlite3_errstr(rc)); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); + free(vendor); + break; + } + + // Bind id to prepared statement + if((rc = sqlite3_bind_int(stmt2, 2, id)) != SQLITE_OK) + { + log_err("updateMACVendorRecords() - Failed to bind id: %s", sqlite3_errstr(rc)); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); + free(vendor); + break; + } + + // Execute statement + rc = sqlite3_step(stmt2); + if(rc != SQLITE_DONE) + { + log_err("updateMACVendorRecords() - SQL error step: %s", sqlite3_errstr(rc)); checkFTLDBrc(rc); - sqlite3_free(zErrMsg); - free(updatestr); + sqlite3_reset(stmt2); + sqlite3_finalize(stmt2); free(vendor); break; } // Free allocated memory - free(updatestr); free(vendor); } if(rc != SQLITE_DONE) @@ -2004,11 +1988,12 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) // Return early if database is known to be broken if(FTLDBerror()) return NULL; + log_debug(DEBUG_RESOLVER, "Trying to obtain host name of \"%s\" from network_addresses table", ipaddr); // Check if we want to resolve host names if(!resolve_this_name(ipaddr)) { - log_debug(DEBUG_DATABASE, "getNameFromIP(\"%s\") - configured to not resolve host name", ipaddr); + log_debug(DEBUG_RESOLVER, "getNameFromIP(\"%s\") - configured to not resolve host name", ipaddr); return NULL; } @@ -2055,6 +2040,8 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return NULL; } + log_debug(DEBUG_RESOLVER, "Check for a host name associated with IP address %s", ipaddr); + char *name = NULL; rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) @@ -2062,7 +2049,7 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) // Database record found (result might be empty) name = strdup((char*)sqlite3_column_text(stmt, 0)); - log_debug(DEBUG_DATABASE, "Found database host name (same address) %s -> %s", ipaddr, name); + log_debug(DEBUG_RESOLVER, "Found database host name (same address) %s -> %s", ipaddr, name); } else if(rc != SQLITE_DONE) { @@ -2083,6 +2070,8 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return name; } + log_debug(DEBUG_RESOLVER, " ---> not found"); + // Nothing found for the exact IP address // Check for a host name associated with the same device (but another IP address) querystr = "SELECT name FROM network_addresses " @@ -2113,6 +2102,8 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return NULL; } + log_debug(DEBUG_RESOLVER, "Checking for a host name associated with the same device (but another IP address)"); + rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) { @@ -2152,8 +2143,6 @@ char *__attribute__((malloc)) getNameFromMAC(const char *client) if(FTLDBerror()) return NULL; - log_debug(DEBUG_DATABASE,"Looking up host name for %s", client); - // Open pihole-FTL.db database file sqlite3 *db = NULL; if((db = dbopen(false, false)) == NULL) @@ -2191,6 +2180,8 @@ char *__attribute__((malloc)) getNameFromMAC(const char *client) return NULL; } + log_debug(DEBUG_RESOLVER, "Check for a host name associated with MAC address %s", client); + char *name = NULL; rc = sqlite3_step(stmt); if(rc == SQLITE_ROW) @@ -2261,11 +2252,8 @@ char *__attribute__((malloc)) getIfaceFromIP(sqlite3 *db, const char *ipaddr) return NULL; } - if(config.debug.resolver.v.b) - { - log_debug(DEBUG_RESOLVER, "getIfaceFromIP(): \"%s\" with ? = \"%s\"", - querystr, ipaddr); - } + log_debug(DEBUG_DATABASE, "getIfaceFromIP(): \"%s\" with ? = \"%s\"", + querystr, ipaddr); // Bind ipaddr to prepared statement if((rc = sqlite3_bind_text(stmt, 1, ipaddr, -1, SQLITE_STATIC)) != SQLITE_OK) diff --git a/src/database/query-table.c b/src/database/query-table.c index f66d5152..140805c8 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -21,12 +21,28 @@ #include "overTime.h" #include "database/common.h" #include "timers.h" +// runGC() +#include "gc.h" static sqlite3 *_memdb = NULL; +static bool store_in_database = false; static double new_last_timestamp = 0; static unsigned int new_total = 0, new_blocked = 0; static unsigned long last_mem_db_idx = 0, last_disk_db_idx = 0; static unsigned int mem_db_num = 0, disk_db_num = 0; +static sqlite3_stmt *query_stmt = NULL; +static sqlite3_stmt *domain_stmt = NULL; +static sqlite3_stmt *client_stmt = NULL; +static sqlite3_stmt *forward_stmt = NULL; +static sqlite3_stmt *addinfo_stmt = NULL; +static sqlite3_stmt **stmts[] = { &query_stmt, + &domain_stmt, + &client_stmt, + &forward_stmt, + &addinfo_stmt }; + +// Private prototypes +static void load_queries_from_disk(void); // Return the maximum ID of the in-memory database unsigned long __attribute__((pure)) get_max_db_idx(void) @@ -155,6 +171,60 @@ bool init_memory_database(void) } } + // Prepare persistent insertion/replace statements + rc = sqlite3_prepare_v3(_memdb, "REPLACE INTO query_storage VALUES "\ + "(?1," \ + "?2," \ + "?3," \ + "?4," \ + "(SELECT id FROM domain_by_id WHERE domain = ?5)," \ + "(SELECT id FROM client_by_id WHERE ip = ?6 AND name = ?7)," \ + "(SELECT id FROM forward_by_id WHERE forward = ?8)," \ + "(SELECT id FROM addinfo_by_id WHERE type = ?9 AND content = ?10)," + "?11," \ + "?12," \ + "?13," \ + "?14)", -1, SQLITE_PREPARE_PERSISTENT, &query_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(query_storage) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO domain_by_id (domain) VALUES (?)", + -1, SQLITE_PREPARE_PERSISTENT, &domain_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(domain_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO client_by_id (ip,name) VALUES (?,?)", + -1, SQLITE_PREPARE_PERSISTENT, &client_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(client_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO forward_by_id (forward) VALUES (?)", + -1, SQLITE_PREPARE_PERSISTENT, &forward_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(forward_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + rc = sqlite3_prepare_v3(_memdb, "INSERT OR IGNORE INTO addinfo_by_id (type,content) VALUES (?,?)", + -1, SQLITE_PREPARE_PERSISTENT, &addinfo_stmt, NULL); + if( rc != SQLITE_OK ) + { + log_err("init_memory_database(addinfo_by_id) - SQL error step: %s", sqlite3_errstr(rc)); + return false; + } + + load_queries_from_disk(); + // Everything went well return true; } @@ -166,6 +236,15 @@ void close_memory_database(void) if(_memdb == NULL) return; + // Finalize all statements + for(unsigned int i = 0; i < ArraySize(stmts); i++) + { + if(*stmts[i] == NULL) + continue; + sqlite3_finalize(*stmts[i]); + *stmts[i] = NULL; + } + // Detach disk database if(!detach_database(_memdb, NULL, "disk")) log_err("close_memory_database(): Failed to detach disk database"); @@ -402,14 +481,6 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename) rc = sqlite3_step(stmt); if( rc == SQLITE_ROW ) num = sqlite3_column_int(stmt, 0); - else - { - log_err("get_number_of_queries_in_DB(%s): Step error: %s", - tablename, sqlite3_errstr(rc)); - free(querystr); - sqlite3_finalize(stmt); - return false; - } sqlite3_finalize(stmt); free(querystr); @@ -504,10 +575,14 @@ bool import_queries_from_disk(void) // Export in-memory queries to disk - either due to periodic dumping (final = // false) or because of a shutdown (final = true) +// When final is false, we only export queries that are older than REPLY_TIMEOUT +// seconds. This is to give queries some time to complete before they are +// exported to disk. When final is true, we export all queries (nothing is going +// to be added to the in-memory database anymore). bool export_queries_to_disk(bool final) { bool okay = false; - const double time = double_time() - (final ? 0.0 : 30.0); + const double time = double_time() - (final ? 0.0 : REPLY_TIMEOUT); const char *querystr = "INSERT INTO disk.query_storage SELECT * FROM query_storage WHERE id > ? AND timestamp < ?"; log_debug(DEBUG_DATABASE, "Storing queries on disk WHERE id > %lu (max is %lu) and timestamp < %f", @@ -562,7 +637,6 @@ bool export_queries_to_disk(bool final) // Finalize statement sqlite3_finalize(stmt); - // Update last_disk_db_idx // Prepare SQLite3 statement log_debug(DEBUG_DATABASE, "Accessing in-memory database"); @@ -603,6 +677,35 @@ bool export_queries_to_disk(bool final) log_debug(DEBUG_DATABASE, "Exported %i rows to disk.%s", sqlite3_changes(memdb), subtable_names[i]); } + /* + * If there are any insertions, we: + * 1. Insert (or replace) the last timestamp into the `disk.ftl` table. + * 2. Update the total queries counter in the `disk.counters` table. + * 3. Update the blocked queries counter in the `disk.counters` table. + * + * Note that new_total does not need to match the total number of + * insertions here as storing queries to the database happens + * time-delayed. In the end, the total number of queries will be + * correct (after final synchronization during FTL shutdown). + */ + if(insertions > 0) + { + if((rc = dbquery(memdb, "INSERT OR REPLACE INTO disk.ftl (id, value) VALUES ( %i, %f );", DB_LASTTIMESTAMP, new_last_timestamp)) != SQLITE_OK) + log_err("export_queries_to_disk(): Cannot update timestamp: %s", sqlite3_errstr(rc)); + + if((rc = dbquery(memdb, "UPDATE disk.counters SET value = value + %u WHERE id = %i;", new_total, DB_TOTALQUERIES)) != SQLITE_OK) + log_err("export_queries_to_disk(): Cannot update total queries counter: %s", sqlite3_errstr(rc)); + else + // Success + new_total = 0; + + if((rc = dbquery(memdb, "UPDATE disk.counters SET value = value + %u WHERE id = %i;", new_blocked, DB_BLOCKEDQUERIES)) != SQLITE_OK) + log_err("export_queries_to_disk(): Cannot update blocked queries counter: %s", sqlite3_errstr(rc)); + else + // Success + new_blocked = 0; + } + // End transaction if((rc = sqlite3_exec(memdb, "END TRANSACTION", NULL, NULL, NULL)) != SQLITE_OK) { @@ -616,17 +719,6 @@ bool export_queries_to_disk(bool final) // All temp queries were stored to disk, update the IDs last_disk_db_idx += insertions; - if(insertions > 0) - { - sqlite3 *db = dbopen(false, false); - if(db != NULL) - { - db_set_FTL_property_double(db, DB_LASTTIMESTAMP, new_last_timestamp); - db_update_counters(db, new_total, new_blocked); - dbclose(&db); - } - } - log_debug(DEBUG_DATABASE, "Exported %u rows for disk.query_storage (took %.1f ms, last SQLite ID %lu)", insertions, timer_elapsed_msec(DATABASE_WRITE_TIMER), last_disk_db_idx); @@ -670,8 +762,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime) mintime, sqlite3_errstr(rc)); // Update number of queries in in-memory database - sqlite3 *memdb = get_memdb(); - const int new_num = get_number_of_queries_in_DB(memdb, "query_storage"); + const int new_num = get_number_of_queries_in_DB(NULL, "query_storage"); log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i", sqlite3_changes(db), (mem_db_num - new_num), new_num); mem_db_num = new_num; @@ -1046,13 +1137,13 @@ void DB_read_queries(void) (buffer = (const char *)sqlite3_column_text(stmt, 6)) != NULL) { // Get IP address and port of upstream destination - char serv_addr[INET6_ADDRSTRLEN] = { 0 }; + char serv_addr[INET6_ADDRSTRLEN + 16] = { 0 }; unsigned int serv_port = 53; // We limit the number of bytes written into the serv_addr buffer // to prevent buffer overflows. If there is no port available in // the database, we skip extracting them and use the default port sscanf(buffer, "%"xstr(INET6_ADDRSTRLEN)"[^#]#%u", serv_addr, &serv_port); - serv_addr[INET6_ADDRSTRLEN-1] = '\0'; + serv_addr[INET6_ADDRSTRLEN + 15] = '\0'; upstreamID = findUpstreamID(serv_addr, (in_port_t)serv_port); } @@ -1074,7 +1165,7 @@ void DB_read_queries(void) // Obtain IDs only after filtering which queries we want to keep const int timeidx = getOverTimeID(queryTimeStamp); const int domainID = findDomainID(domainname, true); - const int clientID = findClientID(clientIP, true, false); + const int clientID = findClientID(clientIP, true, false, queryTimeStamp); // Set index for this query const int queryIndex = counters->queries; @@ -1182,9 +1273,10 @@ void DB_read_queries(void) case QUERY_GRAVITY: // Blocked by gravity case QUERY_REGEX: // Blocked by regex denylist case QUERY_DENYLIST: // Blocked by exact denylist - case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by external provider - case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by external provider - case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by external provider + case QUERY_EXTERNAL_BLOCKED_IP: // Blocked upstream + case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked upstream + case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked upstream + case QUERY_EXTERNAL_BLOCKED_EDE15: // Blocked upstream case QUERY_GRAVITY_CNAME: // Blocked by gravity (inside CNAME path) case QUERY_REGEX_CNAME: // Blocked by regex denylist (inside CNAME path) case QUERY_DENYLIST_CNAME: // Blocked by exact denylist (inside CNAME path) @@ -1232,6 +1324,7 @@ void DB_read_queries(void) log_info(" %d queries parsed...", counters->queries); } + // Release shared memory unlock_shm(); if( rc != SQLITE_DONE ) @@ -1279,16 +1372,6 @@ bool queries_to_database(void) int rc; unsigned int added = 0, updated = 0; sqlite3_int64 idx = 0; - sqlite3_stmt *query_stmt = NULL; - sqlite3_stmt *domain_stmt = NULL; - sqlite3_stmt *client_stmt = NULL; - sqlite3_stmt *forward_stmt = NULL; - sqlite3_stmt *addinfo_stmt = NULL; - sqlite3_stmt **stmts[] = { &query_stmt, - &domain_stmt, - &client_stmt, - &forward_stmt, - &addinfo_stmt }; // Skip, we never store nor count queries recorded while have been in // maximum privacy mode in the database @@ -1302,83 +1385,39 @@ bool queries_to_database(void) log_debug(DEBUG_DATABASE, "Not storing query in database as there are none"); return true; } - - // Start preparing query - sqlite3 *memdb = get_memdb(); - rc = sqlite3_prepare_v3(memdb, "REPLACE INTO query_storage VALUES "\ - "(?1," \ - "?2," \ - "?3," \ - "?4," \ - "(SELECT id FROM domain_by_id WHERE domain = ?5)," \ - "(SELECT id FROM client_by_id WHERE ip = ?6 AND name = ?7)," \ - "(SELECT id FROM forward_by_id WHERE forward = ?8)," \ - "(SELECT id FROM addinfo_by_id WHERE type = ?9 AND content = ?10)," - "?11," \ - "?12," \ - "?13," \ - "?14)", -1, SQLITE_PREPARE_PERSISTENT, &query_stmt, NULL); - if( rc != SQLITE_OK ) + if(!store_in_database) { - log_err("queries_to_database(query_storage) - SQL error step: %s", sqlite3_errstr(rc)); - return false; + log_debug(DEBUG_DATABASE, "Not storing query in database as this is disabled"); + return true; } - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO domain_by_id (domain) VALUES (?)", - -1, SQLITE_PREPARE_PERSISTENT, &domain_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(domain_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO client_by_id (ip,name) VALUES (?,?)", - -1, SQLITE_PREPARE_PERSISTENT, &client_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(client_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO forward_by_id (forward) VALUES (?)", - -1, SQLITE_PREPARE_PERSISTENT, &forward_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(forward_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - rc = sqlite3_prepare_v3(memdb, "INSERT OR IGNORE INTO addinfo_by_id (type,content) VALUES (?,?)", - -1, SQLITE_PREPARE_PERSISTENT, &addinfo_stmt, NULL); - if( rc != SQLITE_OK ) - { - log_err("queries_to_database(addinfo_by_id) - SQL error step: %s", sqlite3_errstr(rc)); - return false; - } - - // Loop over recent queries and store new or changed ones in the in-memory database - const unsigned int min_iter = counters->queries - 1; - unsigned int max_iter = min_iter > DB_QUERY_MAX_ITER ? min_iter - DB_QUERY_MAX_ITER : 0; - for(unsigned int queryID = min_iter; queryID > max_iter; queryID--) + // Loop over recent queries and store new or changed ones in the + // in-memory database + // The upper bound is the last query in the array, the lower bound is + // indirectly given by the first query older than 30 seconds - we do not + // expect replies to still arrive after 30 seconds - they are anyway + // useless as the client will have already timed out tis particular + // query and retried or failed + const double limit_timestamp = double_time() - REPLY_TIMEOUT; + for(unsigned int queryID = counters->queries - 1; queryID > 0; queryID--) { // Get query pointer queriesData *query = getQuery(queryID, true); if(query == NULL) { // Encountered memory error, skip query - log_err("Memory error in queries_to_database()"); + log_err("Memory error in queries_to_database() when trying to access query %u", queryID); break; } + // Skip too old queries (see note above the loop) + if(query->timestamp < limit_timestamp) + break; + // Skip queries which have not changed since the last iteration if(!query->flags.database.changed) continue; - // Update max_iter in case we have changes queries very close to - // the end of the iteration interval - if(min_iter - max_iter < 10) - max_iter = max_iter > DB_QUERY_MAX_ITER ? max_iter - DB_QUERY_MAX_ITER : 0; - // Explicitly set ID to match what is in the on-disk database if(query->db > -1) { @@ -1599,15 +1638,8 @@ bool queries_to_database(void) query->flags.database.changed = false; } - // Finalize all statements - for(unsigned int i = 0; i < ArraySize(stmts); i++) - { - sqlite3_finalize(*stmts[i]); - *stmts[i] = NULL; - } - // Update number of queries in in-memory database - mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage"); + mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage"); if(config.debug.database.v.b && updated + added > 0) { @@ -1617,3 +1649,22 @@ bool queries_to_database(void) return true; } + +static void load_queries_from_disk(void) +{ + // Compensate for possible jumps in time + runGC(time(NULL), NULL, false); + + // Skip if we are not supposed to load queries from disk + if(!config.database.DBimport.v.b) + return; + + // Try to import queries from long-term database if available + import_queries_from_disk(); + DB_read_queries(); + + // Log some information about the imported queries (if any) + log_counter_info(); + + store_in_database = true; +} diff --git a/src/database/query-table.h b/src/database/query-table.h index 9fecb9ea..50d824f6 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -23,7 +23,7 @@ "client TEXT NOT NULL, " \ "forward TEXT );" -#define MEMDB_VERSION 17 +#define MEMDB_VERSION 19 #define CREATE_QUERY_STORAGE_TABLE "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \ "timestamp INTEGER NOT NULL, " \ "type INTEGER NOT NULL, " \ diff --git a/src/database/session-table.c b/src/database/session-table.c index 0147263a..0ee28e3c 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -65,6 +65,48 @@ bool add_session_app_column(sqlite3 *db) return true; } +bool add_session_cli_column(sqlite3 *db) +{ + // Start transaction of database update + SQL_bool(db, "BEGIN TRANSACTION;"); + + // Create session table + SQL_bool(db, "ALTER TABLE session ADD COLUMN cli BOOL;"); + + // Update database version to 18 + if(!db_set_FTL_property(db, DB_VERSION, 18)) + { + log_err("add_session_cli_column(): Failed to update database version!"); + return false; + } + + // Finish transaction + SQL_bool(db, "COMMIT"); + + return true; +} + +bool add_session_x_forwarded_for_column(sqlite3 *db) +{ + // Start transaction of database update + SQL_bool(db, "BEGIN TRANSACTION;"); + + // Create session table + SQL_bool(db, "ALTER TABLE session ADD COLUMN x_forwarded_for TEXT;"); + + // Update database version to 18 + if(!db_set_FTL_property(db, DB_VERSION, 19)) + { + log_err("add_session_x_forwarded_for_column(): Failed to update database version!"); + return false; + } + + // Finish transaction + SQL_bool(db, "COMMIT"); + + return true; +} + // Store all session in database bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) { @@ -83,7 +125,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) // Insert session into database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(db, "INSERT INTO session (login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(db, "INSERT INTO session (login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli, x_forwarded_for) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", sqlite3_errmsg(db), sqlite3_errcode(db)); @@ -105,63 +147,77 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_bind_int64(stmt, 1, sess->login_at) != SQLITE_OK) { log_err("Cannot bind login_at = %ld in backup_db_sessions(): %s (%d)", - (long int)sess->login_at, sqlite3_errmsg(db), sqlite3_errcode(db)); + (long int)sess->login_at, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 2: valid_until if(sqlite3_bind_int64(stmt, 2, sess->valid_until) != SQLITE_OK) { log_err("Cannot bind valid_until = %ld in backup_db_sessions(): %s (%d)", - (long int)sess->valid_until, sqlite3_errmsg(db), sqlite3_errcode(db)); + (long int)sess->valid_until, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 3: remote_addr if(sqlite3_bind_text(stmt, 3, sess->remote_addr, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind remote_addr = %s in backup_db_sessions(): %s (%d)", - sess->remote_addr, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->remote_addr, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 4: user_agent if(sqlite3_bind_text(stmt, 4, sess->user_agent, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind user_agent = %s in backup_db_sessions(): %s (%d)", - sess->user_agent, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->user_agent, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 5: sid if(sqlite3_bind_text(stmt, 5, sess->sid, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind sid = %s in backup_db_sessions(): %s (%d)", - sess->sid, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->sid, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 6: csrf if(sqlite3_bind_text(stmt, 6, sess->csrf, -1, SQLITE_STATIC) != SQLITE_OK) { log_err("Cannot bind csrf = %s in backup_db_sessions(): %s (%d)", - sess->csrf, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->csrf, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 7: tls_login if(sqlite3_bind_int(stmt, 7, sess->tls.login ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind tls_login = %d in backup_db_sessions(): %s (%d)", - sess->tls.login ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->tls.login ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 8: tls_mixed - if(sqlite3_bind_int(stmt, 8, sess->tls.mixed ? 1: 0) != SQLITE_OK) + if(sqlite3_bind_int(stmt, 8, sess->tls.mixed ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind tls_mixed = %d in backup_db_sessions(): %s (%d)", - sess->tls.mixed ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->tls.mixed ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } // 9: app - if(sqlite3_bind_int(stmt, 8, sess->app ? 1: 0) != SQLITE_OK) + if(sqlite3_bind_int(stmt, 9, sess->app ? 1 : 0) != SQLITE_OK) { log_err("Cannot bind app = %d in backup_db_sessions(): %s (%d)", - sess->app ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + sess->app ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + return false; + } + // 10: cli + if(sqlite3_bind_int(stmt, 10, sess->cli ? 1 : 0) != SQLITE_OK) + { + log_err("Cannot bind cli = %d in backup_db_sessions(): %s (%d)", + sess->cli ? 1 : 0, sqlite3_errmsg(db), sqlite3_errcode(db)); + return false; + } + // 11: x_forwarded_for + if(sqlite3_bind_text(stmt, 11, sess->x_forwarded_for, -1, SQLITE_STATIC) != SQLITE_OK) + { + log_err("Cannot bind x_forwarded_for = %s in backup_db_sessions(): %s (%d)", + sess->x_forwarded_for, sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -169,7 +225,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_step(stmt) != SQLITE_DONE) { log_err("SQL error in backup_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -177,7 +233,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_clear_bindings(stmt) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -185,7 +241,7 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) if(sqlite3_reset(stmt) != SQLITE_OK) { log_err("SQL error in backup_db_sessions(): %s (%d)", - sqlite3_errmsg(db), sqlite3_errcode(db)); + sqlite3_errmsg(db), sqlite3_errcode(db)); return false; } @@ -225,7 +281,7 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // Get all sessions from database sqlite3_stmt *stmt = NULL; - if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) + if(sqlite3_prepare_v2(memdb, "SELECT login_at, valid_until, remote_addr, user_agent, sid, csrf, tls_login, tls_mixed, app, cli, x_forwarded_for FROM disk.session;", -1, &stmt, 0) != SQLITE_OK) { log_err("SQL error in restore_db_sessions(): %s (%d)", sqlite3_errmsg(memdb), sqlite3_errcode(memdb)); @@ -284,9 +340,20 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) // 8: tls_mixed sess->tls.mixed = sqlite3_column_int(stmt, 7) == 1 ? true : false; - // 8: app + // 9: app sess->app = sqlite3_column_int(stmt, 8) == 1 ? true : false; + // 10: cli + sess->cli = sqlite3_column_int(stmt, 9) == 1 ? true : false; + + // 11: x_forwarded_for + const char *x_forwarded_for = (const char *)sqlite3_column_text(stmt, 10); + if(x_forwarded_for != NULL) + { + strncpy(sess->x_forwarded_for, x_forwarded_for, sizeof(sess->x_forwarded_for)-1); + sess->x_forwarded_for[sizeof(sess->x_forwarded_for)-1] = '\0'; + } + // Mark session as used sess->used = true; diff --git a/src/database/session-table.h b/src/database/session-table.h index 2e9ff296..0aa79b19 100644 --- a/src/database/session-table.h +++ b/src/database/session-table.h @@ -16,6 +16,8 @@ bool create_session_table(sqlite3 *db); bool add_session_app_column(sqlite3 *db); +bool add_session_cli_column(sqlite3 *db); +bool add_session_x_forwarded_for_column(sqlite3 *db); bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions); bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions); diff --git a/src/database/shell.c b/src/database/shell.c index ead6aa9f..d573e8d4 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -582,6 +582,9 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm); #ifndef HAVE_CONSOLE_IO_H # include "console_io.h" #endif +#if defined(_MSC_VER) +# pragma warning(disable : 4204) +#endif #ifndef SQLITE_CIO_NO_TRANSLATE # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT @@ -603,11 +606,6 @@ zSkipValidUtf8(const char *z, int nAccept, long ccm); # define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */ #endif -#if CIO_WIN_WC_XLATE -/* Character used to represent a known-incomplete UTF-8 char group (�) */ -static WCHAR cBadGroup = 0xfffd; -#endif - #if CIO_WIN_WC_XLATE static HANDLE handleOfFile(FILE *pf){ int fileDesc = _fileno(pf); @@ -680,6 +678,10 @@ static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){ # endif } +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) +# endif + # if CIO_WIN_WC_XLATE /* Define console modes for use with the Windows Console API. */ # define SHELL_CONI_MODE \ @@ -1230,6 +1232,10 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ +#if defined(_MSC_VER) +# pragma warning(default : 4204) +#endif + #undef SHELL_INVALID_FILE_PTR /************************* End ../ext/consio/console_io.c ********************/ @@ -1252,6 +1258,9 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ * setOutputStream(FILE *pf) * This is normally the stream that CLI normal output goes to. * For the stand-alone CLI, it is stdout with no .output redirect. + * + * The ?putz(z) forms are required for the Fiddle builds for string literal + * output, in aid of enforcing format string to argument correspondence. */ # define sputz(s,z) fPutsUtf8(z,s) # define sputf fPrintfUtf8 @@ -1263,12 +1272,18 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ #else /* For Fiddle, all console handling and emit redirection is omitted. */ -# define sputz(fp,z) fputs(z,fp) -# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) -# define oputz(z) fputs(z,stdout) +/* These next 3 macros are for emitting formatted output. When complaints + * from the WASM build are issued for non-formatted output, (when a mere + * string literal is to be emitted, the ?putz(z) forms should be used. + * (This permits compile-time checking of format string / argument mismatch.) + */ # define oputf(fmt, ...) printf(fmt,__VA_ARGS__) -# define eputz(z) fputs(z,stderr) # define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) +# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) +/* These next 3 macros are for emitting simple string literals. */ +# define oputz(z) fputs(z,stdout) +# define eputz(z) fputs(z,stderr) +# define sputz(fp,z) fputs(z,fp) # define oputb(buf,na) fwrite(buf,1,na,stdout) #endif @@ -5713,16 +5728,20 @@ SQLITE_EXTENSION_INIT1 ** index is ix. The 0th member is given by smBase. The sequence members ** progress per ix increment by smStep. */ -static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix){ - if( ix>=(sqlite3_uint64)LLONG_MAX ){ +static sqlite3_int64 genSeqMember( + sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix +){ + static const sqlite3_uint64 mxI64 = + ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; + if( ix>=mxI64 ){ /* Get ix into signed i64 range. */ - ix -= (sqlite3_uint64)LLONG_MAX; + ix -= mxI64; /* With 2's complement ALU, this next can be 1 step, but is split into * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (LLONG_MAX/2) * smStep; - smBase += (LLONG_MAX - LLONG_MAX/2) * smStep; + smBase += (mxI64/2) * smStep; + smBase += (mxI64 - mxI64/2) * smStep; } /* Under UBSAN (or on 1's complement machines), must do this last term * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ @@ -5982,13 +6001,13 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** parameter. (idxStr is not used in this implementation.) idxNum ** is a bitmask showing which constraints are available: ** -** 1: start=VALUE -** 2: stop=VALUE -** 4: step=VALUE -** -** Also, if bit 8 is set, that means that the series should be output -** in descending order rather than in ascending order. If bit 16 is -** set, then output must appear in ascending order. +** 0x01: start=VALUE +** 0x02: stop=VALUE +** 0x04: step=VALUE +** 0x08: descending order +** 0x10: ascending order +** 0x20: LIMIT VALUE +** 0x40: OFFSET VALUE ** ** This routine should initialize the cursor and position it so that it ** is pointing at the first row, or pointing off the end of the table @@ -6002,26 +6021,44 @@ static int seriesFilter( series_cursor *pCur = (series_cursor *)pVtabCursor; int i = 0; (void)idxStrUnused; - if( idxNum & 1 ){ + if( idxNum & 0x01 ){ pCur->ss.iBase = sqlite3_value_int64(argv[i++]); }else{ pCur->ss.iBase = 0; } - if( idxNum & 2 ){ + if( idxNum & 0x02 ){ pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); }else{ pCur->ss.iTerm = 0xffffffff; } - if( idxNum & 4 ){ + if( idxNum & 0x04 ){ pCur->ss.iStep = sqlite3_value_int64(argv[i++]); if( pCur->ss.iStep==0 ){ pCur->ss.iStep = 1; }else if( pCur->ss.iStep<0 ){ - if( (idxNum & 16)==0 ) idxNum |= 8; + if( (idxNum & 0x10)==0 ) idxNum |= 0x08; } }else{ pCur->ss.iStep = 1; } + if( idxNum & 0x20 ){ + sqlite3_int64 iLimit = sqlite3_value_int64(argv[i++]); + sqlite3_int64 iTerm; + if( idxNum & 0x40 ){ + sqlite3_int64 iOffset = sqlite3_value_int64(argv[i++]); + if( iOffset>0 ){ + pCur->ss.iBase += pCur->ss.iStep*iOffset; + } + } + if( iLimit>=0 ){ + iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; + if( pCur->ss.iStep<0 ){ + if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + }else{ + if( iTermss.iTerm ) pCur->ss.iTerm = iTerm; + } + } + } for(i=0; iss.isReversing = pCur->ss.iStep > 0; }else{ pCur->ss.isReversing = pCur->ss.iStep < 0; @@ -6052,10 +6089,13 @@ static int seriesFilter( ** ** The query plan is represented by bits in idxNum: ** -** (1) start = $value -- constraint exists -** (2) stop = $value -- constraint exists -** (4) step = $value -- constraint exists -** (8) output in descending order +** 0x01 start = $value -- constraint exists +** 0x02 stop = $value -- constraint exists +** 0x04 step = $value -- constraint exists +** 0x08 output is in descending order +** 0x10 output is in ascending order +** 0x20 LIMIT $value -- constraint exists +** 0x40 OFFSET $value -- constraint exists */ static int seriesBestIndex( sqlite3_vtab *pVTab, @@ -6063,10 +6103,12 @@ static int seriesBestIndex( ){ int i, j; /* Loop over constraints */ int idxNum = 0; /* The query plan bitmask */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES int bStartSeen = 0; /* EQ constraint seen on the START column */ +#endif int unusableMask = 0; /* Mask of unusable constraints */ int nArg = 0; /* Number of arguments that seriesFilter() expects */ - int aIdx[3]; /* Constraints on start, stop, and step */ + int aIdx[5]; /* Constraints on start, stop, step, LIMIT, OFFSET */ const struct sqlite3_index_constraint *pConstraint; /* This implementation assumes that the start, stop, and step columns @@ -6074,28 +6116,54 @@ static int seriesBestIndex( assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); - aIdx[0] = aIdx[1] = aIdx[2] = -1; + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = -1; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ int iCol; /* 0 for start, 1 for stop, 2 for step */ int iMask; /* bitmask for those column */ + int op = pConstraint->op; + if( op>=SQLITE_INDEX_CONSTRAINT_LIMIT + && op<=SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + if( pConstraint->usable==0 ){ + /* do nothing */ + }else if( op==SQLITE_INDEX_CONSTRAINT_LIMIT ){ + aIdx[3] = i; + idxNum |= 0x20; + }else{ + assert( op==SQLITE_INDEX_CONSTRAINT_OFFSET ); + aIdx[4] = i; + idxNum |= 0x40; + } + continue; + } if( pConstraint->iColumniColumn - SERIES_COLUMN_START; assert( iCol>=0 && iCol<=2 ); iMask = 1 << iCol; - if( iCol==0 ) bStartSeen = 1; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( iCol==0 && op==SQLITE_INDEX_CONSTRAINT_EQ ){ + bStartSeen = 1; + } +#endif if( pConstraint->usable==0 ){ unusableMask |= iMask; continue; - }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + }else if( op==SQLITE_INDEX_CONSTRAINT_EQ ){ idxNum |= iMask; aIdx[iCol] = i; } } - for(i=0; i<3; i++){ + if( aIdx[3]==0 ){ + /* Ignore OFFSET if LIMIT is omitted */ + idxNum &= ~0x60; + aIdx[4] = 0; + } + for(i=0; i<5; i++){ if( (j = aIdx[i])>=0 ){ pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; + pIdxInfo->aConstraintUsage[j].omit = + !SQLITE_SERIES_CONSTRAINT_VERIFY || i>=3; } } /* The current generate_column() implementation requires at least one @@ -6116,19 +6184,22 @@ static int seriesBestIndex( ** this plan is unusable */ return SQLITE_CONSTRAINT; } - if( (idxNum & 3)==3 ){ + if( (idxNum & 0x03)==0x03 ){ /* Both start= and stop= boundaries are available. This is the ** the preferred case */ pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); pIdxInfo->estimatedRows = 1000; if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn==0 ){ if( pIdxInfo->aOrderBy[0].desc ){ - idxNum |= 8; + idxNum |= 0x08; }else{ - idxNum |= 16; + idxNum |= 0x10; } pIdxInfo->orderByConsumed = 1; } + }else if( (idxNum & 0x21)==0x21 ){ + /* We have start= and LIMIT */ + pIdxInfo->estimatedRows = 2500; }else{ /* If either boundary is missing, we have to generate a huge span ** of numbers. Make this case very expensive so that the query @@ -7455,7 +7526,9 @@ static int writeFile( #if !defined(_WIN32) && !defined(WIN32) if( S_ISLNK(mode) ){ const char *zTo = (const char*)sqlite3_value_text(pData); - if( zTo==0 || symlink(zTo, zFile)<0 ) return 1; + if( zTo==0 ) return 1; + unlink(zFile); + if( symlink(zTo, zFile)<0 ) return 1; }else #endif { @@ -7541,13 +7614,19 @@ static int writeFile( return 1; } #else - /* Legacy unix */ - struct timeval times[2]; - times[0].tv_usec = times[1].tv_usec = 0; - times[0].tv_sec = time(0); - times[1].tv_sec = mtime; - if( utimes(zFile, times) ){ - return 1; + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } } #endif } @@ -11619,7 +11698,7 @@ static void sqlarUncompressFunc( sqlite3_value **argv ){ uLong nData; - uLongf sz; + sqlite3_int64 sz; assert( argc==2 ); sz = sqlite3_value_int(argv[1]); @@ -11627,14 +11706,15 @@ static void sqlarUncompressFunc( if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ + uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); Bytef *pOut = sqlite3_malloc(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); - }else if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){ + }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ sqlite3_result_error(context, "error in uncompress()", -1); }else{ - sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT); + sqlite3_result_blob(context, pOut, szf, SQLITE_TRANSIENT); } sqlite3_free(pOut); } @@ -12464,7 +12544,7 @@ static int expertFilter( pCsr->pData = 0; if( rc==SQLITE_OK ){ rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, - "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName ); } @@ -13338,7 +13418,7 @@ struct IdxRemCtx { }; /* -** Implementation of scalar function rem(). +** Implementation of scalar function sqlite_expert_rem(). */ static void idxRemFunc( sqlite3_context *pCtx, @@ -13351,7 +13431,7 @@ static void idxRemFunc( assert( argc==2 ); iSlot = sqlite3_value_int(argv[0]); - assert( iSlot<=p->nSlot ); + assert( iSlotnSlot ); pSlot = &p->aSlot[iSlot]; switch( pSlot->eType ){ @@ -13462,7 +13542,8 @@ static int idxPopulateOneStat1( const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); zCols = idxAppendText(&rc, zCols, - "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl ); zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); } @@ -13595,13 +13676,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ if( rc==SQLITE_OK ){ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = sqlite3_create_function( - dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 ); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 ); } @@ -13653,6 +13734,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); } + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); return rc; } @@ -13786,7 +13870,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_stmt *pSql = 0; rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" - " AND sql NOT LIKE 'CREATE VIRTUAL %%'" + " AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid" ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); @@ -13988,6 +14072,1124 @@ void sqlite3_expert_destroy(sqlite3expert *p){ /************************* End ../ext/expert/sqlite3expert.c ********************/ +/************************* Begin ../ext/intck/sqlite3intck.h ******************/ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +/* #include "sqlite3.h" */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ +typedef struct sqlite3_intck sqlite3_intck; + +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); + +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); + +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_step(sqlite3_intck *pCk); + +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); + +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ + +/************************* End ../ext/intck/sqlite3intck.h ********************/ +/************************* Begin ../ext/intck/sqlite3intck.c ******************/ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* #include "sqlite3intck.h" */ +#include +#include + +#include +#include + +/* +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. +*/ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + return zRet; +} + +/* +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. +*/ +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); + } + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +/* +** Return true if argument c is an ascii whitespace character. +*/ +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); +*/ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" + + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } + } + intckFinalize(p, pStmt); + + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); + return zRet; +} + +/* +** Open a new integrity-check object. +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } + } + + *ppOut = pNew; + return rc; +} + +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } +} + +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; + } + } + + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } + } + } + } + + return p->rc; +} + +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +/* +** Return the error code and message. +*/ +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; +} + +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; +} + +/************************* End ../ext/intck/sqlite3intck.c ********************/ + #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 #else @@ -14338,6 +15540,15 @@ int sqlite3_recover_finish(sqlite3_recover*); typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; +typedef struct DbdataBuffer DbdataBuffer; + +/* +** Buffer type. +*/ +struct DbdataBuffer { + u8 *aBuf; + sqlite3_int64 nBuf; +}; /* Cursor object */ struct DbdataCursor { @@ -14354,7 +15565,7 @@ struct DbdataCursor { sqlite3_int64 iRowid; /* Only for the sqlite_dbdata table */ - u8 *pRec; /* Buffer containing current record */ + DbdataBuffer rec; sqlite3_int64 nRec; /* Size of pRec[] in bytes */ sqlite3_int64 nHdr; /* Size of header in bytes */ int iField; /* Current field number */ @@ -14399,6 +15610,31 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +/* +** Ensure the buffer passed as the first argument is at least nMin bytes +** in size. If an error occurs while attempting to resize the buffer, +** SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int dbdataBufferSize(DbdataBuffer *pBuf, sqlite3_int64 nMin){ + if( nMin>pBuf->nBuf ){ + sqlite3_int64 nNew = nMin+16384; + u8 *aNew = (u8*)sqlite3_realloc64(pBuf->aBuf, nNew); + + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->aBuf = aNew; + pBuf->nBuf = nNew; + } + return SQLITE_OK; +} + +/* +** Release the allocation managed by buffer pBuf. +*/ +static void dbdataBufferFree(DbdataBuffer *pBuf){ + sqlite3_free(pBuf->aBuf); + memset(pBuf, 0, sizeof(*pBuf)); +} + /* ** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual ** table. @@ -14539,9 +15775,9 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ pCsr->iField = 0; pCsr->bOnePage = 0; sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; + dbdataBufferFree(&pCsr->rec); pCsr->aPage = 0; + pCsr->nRec = 0; } /* @@ -14683,67 +15919,88 @@ static void dbdataValue( u8 *pData, sqlite3_int64 nData ){ - if( eType>=0 && dbdataValueBytes(eType)<=nData ){ - switch( eType ){ - case 0: - case 10: - case 11: - sqlite3_result_null(pCtx); - break; - - case 8: - sqlite3_result_int(pCtx, 0); - break; - case 9: - sqlite3_result_int(pCtx, 1); - break; - - case 1: case 2: case 3: case 4: case 5: case 6: case 7: { - sqlite3_uint64 v = (signed char)pData[0]; - pData++; - switch( eType ){ - case 7: - case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 4: v = (v<<8) + pData[0]; pData++; - case 3: v = (v<<8) + pData[0]; pData++; - case 2: v = (v<<8) + pData[0]; pData++; - } - - if( eType==7 ){ - double r; - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(pCtx, r); - }else{ - sqlite3_result_int64(pCtx, (sqlite3_int64)v); - } - break; - } - - default: { - int n = ((eType-12) / 2); - if( eType % 2 ){ - switch( enc ){ -#ifndef SQLITE_OMIT_UTF16 - case SQLITE_UTF16BE: - sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; - case SQLITE_UTF16LE: - sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; -#endif - default: - sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); - break; + if( eType>=0 ){ + if( dbdataValueBytes(eType)<=nData ){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; } - }else{ - sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + switch( enc ){ + #ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + #endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } + }else{ + if( eType==7 ){ + sqlite3_result_double(pCtx, 0.0); + }else if( eType<7 ){ + sqlite3_result_int(pCtx, 0); + }else if( eType%2 ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + }else{ + sqlite3_result_blob(pCtx, "", 0, SQLITE_STATIC); } } } } +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ @@ -14772,6 +16029,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } } if( pTab->bPtr ){ @@ -14789,7 +16049,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } }else{ /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ + assert( pCsr->rec.aBuf!=0 || pCsr->nRec==0 ); + if( pCsr->nRec==0 ){ int bHasRowid = 0; int nPointer = 0; sqlite3_int64 nPayload = 0; @@ -14816,23 +16077,24 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ + if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); + iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; + if( nPayload==0 ) nPayload = 1; } /* If this is a leaf intkey cell, load the rowid */ @@ -14867,13 +16129,12 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Allocate space for payload. And a bit more to catch small buffer ** overruns caused by attempting to read a varint or similar from ** near the end of a corrupt record. */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES); - pCsr->nRec = nPayload; + rc = dbdataBufferSize(&pCsr->rec, nPayload+DBDATA_PADDING_BYTES); + if( rc!=SQLITE_OK ) return rc; + assert( nPayload!=0 ); /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + memcpy(pCsr->rec.aBuf, &pCsr->aPage[iOff], nLocal); iOff += nLocal; /* Load content from overflow pages */ @@ -14891,19 +16152,22 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ nCopy = U-4; if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + memcpy(&pCsr->rec.aBuf[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } + nPayload -= nRem; } + memset(&pCsr->rec.aBuf[nPayload], 0, DBDATA_PADDING_BYTES); + pCsr->nRec = nPayload; - iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr); + iHdr = dbdataGetVarintU32(pCsr->rec.aBuf, &nHdr); if( nHdr>nPayload ) nHdr = 0; pCsr->nHdr = nHdr; - pCsr->pHdrPtr = &pCsr->pRec[iHdr]; - pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; + pCsr->pHdrPtr = &pCsr->rec.aBuf[iHdr]; + pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nHdr]; pCsr->iField = (bHasRowid ? -1 : 0); } } @@ -14911,14 +16175,16 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ + if( pCsr->pHdrPtr>=&pCsr->rec.aBuf[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ bNextPage = 1; }else{ int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); szField = dbdataValueBytes(iType); - if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))pPtr = &pCsr->pRec[pCsr->nRec]; + if( (pCsr->nRec - (pCsr->pPtr - pCsr->rec.aBuf))pPtr = &pCsr->rec.aBuf[pCsr->nRec]; }else{ pCsr->pPtr += szField; } @@ -14928,20 +16194,18 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( bNextPage ){ sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); pCsr->aPage = 0; - pCsr->pRec = 0; + pCsr->nRec = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; }else{ - if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->rec.aBuf[pCsr->nHdr] ){ return SQLITE_OK; } /* Advance to the next cell. The next iteration of the loop will load ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; + pCsr->nRec = 0; pCsr->iCell++; } } @@ -15131,12 +16395,12 @@ static int dbdataColumn( case DBDATA_COLUMN_VALUE: { if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); - }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){ + }else if( &pCsr->rec.aBuf[pCsr->nRec] >= pCsr->pPtr ){ sqlite3_int64 iType; dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( ctx, pCsr->enc, iType, pCsr->pPtr, - &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + &pCsr->rec.aBuf[pCsr->nRec] - pCsr->pPtr ); } break; @@ -15573,8 +16837,8 @@ static int recoverError( va_start(ap, zFmt); if( zFmt ){ z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); } + va_end(ap); sqlite3_free(p->zErrMsg); p->zErrMsg = z; p->errCode = errCode; @@ -16399,7 +17663,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zTbl, iRoot); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } @@ -20621,6 +21885,7 @@ static void exec_prepared_stmt_columnar( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ) return; nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; nAlloc = nColumn*4; if( nAlloc<=0 ) nAlloc = 1; azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); @@ -20706,7 +21971,6 @@ static void exec_prepared_stmt_columnar( if( n>p->actualWidth[j] ) p->actualWidth[j] = n; } if( seenInterrupt ) goto columnar_end; - if( nColumn==0 ) goto columnar_end; switch( p->cMode ){ case MODE_Column: { colSep = " "; @@ -21589,6 +22853,7 @@ static const char *(azHelp[]) = { ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", + ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif @@ -24498,6 +25763,40 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } #endif /* SQLITE_SHELL_HAVE_RECOVER */ +/* +** Implementation of ".intck STEPS_PER_UNLOCK" command. +*/ +static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ + sqlite3_intck *p = 0; + int rc = SQLITE_OK; + + rc = sqlite3_intck_open(pState->db, "main", &p); + if( rc==SQLITE_OK ){ + i64 nStep = 0; + i64 nError = 0; + const char *zErr = 0; + while( SQLITE_OK==sqlite3_intck_step(p) ){ + const char *zMsg = sqlite3_intck_message(p); + if( zMsg ){ + oputf("%s\n", zMsg); + nError++; + } + nStep++; + if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){ + sqlite3_intck_unlock(p); + } + } + rc = sqlite3_intck_error(p, &zErr); + if( zErr ){ + eputf("%s\n", zErr); + } + sqlite3_intck_close(p); + + oputf("%lld steps, %lld errors\n", nStep, nError); + } + + return rc; +} /* * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it. @@ -24741,6 +26040,45 @@ static int outputDumpWarning(ShellState *p, const char *zLike){ return rc; } +/* +** Fault-Simulator state and logic. +*/ +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; + +/* +** This is the fault-sim callback +*/ +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; + } + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + oputf("FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); + } + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + oputf("FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; + } + return faultsim_state.iErr; +} + /* ** If an input line begins with "." then invoke this routine to ** process that line. @@ -25232,7 +26570,8 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = sqlite3_mprintf( "SELECT sql FROM sqlite_schema AS o " "WHERE (%s) AND sql NOT NULL" - " AND type IN ('index','trigger','view')", + " AND type IN ('index','trigger','view') " + "ORDER BY type COLLATE NOCASE DESC", zLike ); run_table_dump_query(p, zSql); @@ -25555,16 +26894,15 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* within this schema (may default to "main") */ + char *zSchema = 0; /* Schema of zTable */ char *zFile = 0; /* Name of file to extra content from */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ - int nByte; /* Number of bytes in an SQL string */ + i64 nByte; /* Number of bytes in an SQL string */ int i, j; /* Loop counters */ int needCommit; /* True to COMMIT or ROLLBACK at end */ int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql; /* An SQL statement */ - char *zFullTabName; /* Table name with schema if applicable */ + char *zSql = 0; /* An SQL statement */ ImportCtx sCtx; /* Reader context */ char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ int eVerbose = 0; /* Larger for more console output */ @@ -25698,24 +27036,14 @@ static int do_meta_command(char *zLine, ShellState *p){ while( (nSkip--)>0 ){ while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} } - if( zSchema!=0 ){ - zFullTabName = sqlite3_mprintf("\"%w\".\"%w\"", zSchema, zTable); - }else{ - zFullTabName = sqlite3_mprintf("\"%w\"", zTable); - } - zSql = sqlite3_mprintf("SELECT * FROM %s", zFullTabName); - if( zSql==0 || zFullTabName==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - nByte = strlen30(zSql); - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) ){ + /* Table does not exist. Create it. */ sqlite3 *dbCols = 0; char *zRenames = 0; char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE %s", zFullTabName); + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); while( xRead(&sCtx) ){ zAutoColumn(sCtx.z, &dbCols, 0); if( sCtx.cTerm!=sCtx.cColSep ) break; @@ -25730,43 +27058,68 @@ static int do_meta_command(char *zLine, ShellState *p){ assert(dbCols==0); if( zColDefs==0 ){ eputf("%s: empty file\n", sCtx.zFile); - import_fail: - sqlite3_free(zCreate); - sqlite3_free(zSql); - sqlite3_free(zFullTabName); import_cleanup(&sCtx); rc = 1; goto meta_command_exit; } zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } if( eVerbose>=1 ){ oputf("%s\n", zCreate); } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); - if( rc ){ - eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - goto import_fail; - } sqlite3_free(zCreate); zCreate = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc ){ + eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } } - if( rc ){ - if (pStmt) sqlite3_finalize(pStmt); - eputf("Error: %s\n", sqlite3_errmsg(p->db)); - goto import_fail; - } - sqlite3_free(zSql); - nCol = sqlite3_column_count(pStmt); - sqlite3_finalize(pStmt); - pStmt = 0; - if( nCol==0 ) return 0; /* no columns, no error */ - zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zFullTabName); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } j = strlen30(zSql); for(i=1; i=2 ){ oputf("Insert using: %s\n", zSql); } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ eputf("Error: %s\n", sqlite3_errmsg(p->db)); if (pStmt) sqlite3_finalize(pStmt); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zSql); - sqlite3_free(zFullTabName); needCommit = sqlite3_get_autocommit(p->db); if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); do{ @@ -25955,6 +27311,21 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ + i64 iArg = 0; + if( nArg==2 ){ + iArg = integerValue(azArg[1]); + if( iArg==0 ) iArg = -1; + } + if( (nArg!=1 && nArg!=2) || iArg<0 ){ + eputf("%s","Usage: .intck STEPS_PER_UNLOCK\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + rc = intckDatabaseCmd(p, iArg); + }else + #ifdef SQLITE_ENABLE_IOTRACE if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); @@ -27628,7 +28999,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /*{"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, - /*{"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ + {"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..." }, {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, @@ -27861,6 +29232,76 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_test_control(testctrl, &rc2); break; + case SQLITE_TESTCTRL_FAULT_INSTALL: { + int kk; + int bShowHelp = nArg<=2; + isOk = 3; + for(kk=2; kk0 ) faultsim_state.eVerbose--; + }else if( cli_strcmp(z,"-id")==0 && kk+1=0 ){ @@ -28750,6 +30191,7 @@ static const char zOptions[] = " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" " -nonce STRING set the safe-mode escape nonce\n" + " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -pcachetrace trace all page cache operations\n" @@ -28782,7 +30224,7 @@ static void usage(int showDetail){ }else{ eputz("Use the -help option for additional information\n"); } - exit(1); + exit(0); } /* @@ -29040,6 +30482,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 0; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + int val = 0; + sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW, &val); + assert( val==0 ); }else if( cli_strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) const char *zSize; @@ -29315,6 +30761,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ /* already handled */ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + /* already handled */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ @@ -29477,6 +30925,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( data.expert.pExpert ){ + expertFinish(&data, 1, 0); + } +#endif free(azCmd); set_table_name(&data, 0); if( data.db ){ @@ -29543,7 +30996,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - printf("fiddle_db_arg(%p)\n", (const void*)arg); + oputf("fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -29569,12 +31022,22 @@ const char * fiddle_db_filename(const char * zDbName){ /* ** Completely wipes out the contents of the currently-opened database -** but leaves its storage intact for reuse. +** but leaves its storage intact for reuse. If any transactions are +** active, they are forcibly rolled back. */ void fiddle_reset_db(void){ if( globalDb ){ - int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); - if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); + int rc; + while( sqlite3_txn_state(globalDb,0)>0 ){ + /* + ** Resolve problem reported in + ** https://sqlite.org/forum/forumpost/0b41a25d65 + */ + oputz("Rolling back in-progress transaction.\n"); + sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); + } + rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); } } diff --git a/src/database/sqlite3-ext.c b/src/database/sqlite3-ext.c index f52ef029..b065576e 100644 --- a/src/database/sqlite3-ext.c +++ b/src/database/sqlite3-ext.c @@ -22,9 +22,9 @@ // free() #include // logging routines -#include "../log.h" +#include "log.h" // struct config -#include "../config/config.h" +#include "config/config.h" // isMAC() #include "network-table.h" @@ -215,4 +215,4 @@ int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const str } return rc; -} \ No newline at end of file +} diff --git a/src/database/sqlite3-ext.h b/src/database/sqlite3-ext.h index 2636eb5d..18eabd8c 100644 --- a/src/database/sqlite3-ext.h +++ b/src/database/sqlite3-ext.h @@ -8,5 +8,10 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#ifndef SQLITE3_EXT_H +#define SQLITE3_EXT_H + // Initialization point for SQLite3 extensions extern int sqlite3_pihole_extensions_init(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pApi); + +#endif // SQLITE3_EXT_H diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index a2718dcc..98809f1b 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.45.1. By combining all the individual C code files into this +** version 3.46.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** e876e51a0ed5c5b3126f52e532044363a014. +** c9c2ab54ba1f5f46360f1b4f35d849cd3f08. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.1" -#define SQLITE_VERSION_NUMBER 3045001 -#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257cc467a" +#define SQLITE_VERSION "3.46.1" +#define SQLITE_VERSION_NUMBER 3046001 +#define SQLITE_SOURCE_ID "2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -733,6 +733,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ SQLITE_API int sqlite3_exec( @@ -1075,11 +1077,11 @@ struct sqlite3_file { ** ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, @@ -2454,6 +2456,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2485,6 +2503,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -3599,8 +3618,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -7181,6 +7200,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -8651,7 +8676,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -10230,24 +10255,45 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **

  • ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. **

  • -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** ** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** ** ^For the purposes of comparing virtual table output values to see if the ** values are same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" @@ -12292,6 +12338,30 @@ SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const c */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -13096,8 +13166,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -14295,6 +14365,8 @@ struct fts5_api { # define SQLITE_OMIT_ALTERTABLE #endif +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -14587,8 +14659,8 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_TRUEFALSE 170 #define TK_ISNOT 171 #define TK_FUNCTION 172 -#define TK_UMINUS 173 -#define TK_UPLUS 174 +#define TK_UPLUS 173 +#define TK_UMINUS 174 #define TK_TRUTH 175 #define TK_REGISTER 176 #define TK_VECTOR 177 @@ -14597,8 +14669,9 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_ASTERISK 180 #define TK_SPAN 181 #define TK_ERROR 182 -#define TK_SPACE 183 -#define TK_ILLEGAL 184 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -14860,7 +14933,7 @@ typedef INT16_TYPE LogEst; # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ - (defined(__APPLE__) && defined(__POWERPC__)) || \ + (defined(__APPLE__) && defined(__ppc__)) || \ (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else @@ -15097,6 +15170,7 @@ SQLITE_PRIVATE u32 sqlite3TreeTrace; ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated +** 0x00080000 NOT NULL strength reduction */ /* @@ -15127,7 +15201,7 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; ** 0x00000010 Display sqlite3_index_info xBestIndex calls ** 0x00000020 Range an equality scan metrics ** 0x00000040 IN operator decisions -** 0x00000080 WhereLoop cost adjustements +** 0x00000080 WhereLoop cost adjustments ** 0x00000100 ** 0x00000200 Covering index decisions ** 0x00000400 OR optimization @@ -16276,6 +16350,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -16546,12 +16621,12 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Vacuum 5 #define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ #define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ -#define OP_Init 8 /* jump, synopsis: Start at P2 */ +#define OP_Init 8 /* jump0, synopsis: Start at P2 */ #define OP_Goto 9 /* jump */ #define OP_Gosub 10 /* jump */ -#define OP_InitCoroutine 11 /* jump */ -#define OP_Yield 12 /* jump */ -#define OP_MustBeInt 13 /* jump */ +#define OP_InitCoroutine 11 /* jump0 */ +#define OP_Yield 12 /* jump0 */ +#define OP_MustBeInt 13 /* jump0 */ #define OP_Jump 14 /* jump */ #define OP_Once 15 /* jump */ #define OP_If 16 /* jump */ @@ -16559,22 +16634,22 @@ typedef struct VdbeOpList VdbeOpList; #define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ #define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ -#define OP_SeekLT 21 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 22 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 23 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLT 21 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump0, synopsis: key=r[P3@P4] */ #define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ #define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ #define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ #define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ #define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_SeekRowid 30 /* jump0, synopsis: intkey=r[P3] */ #define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 32 /* jump */ -#define OP_IfSmaller 33 /* jump */ +#define OP_Last 32 /* jump0 */ +#define OP_IfSizeBetween 33 /* jump */ #define OP_SorterSort 34 /* jump */ #define OP_Sort 35 /* jump */ -#define OP_Rewind 36 /* jump */ +#define OP_Rewind 36 /* jump0 */ #define OP_SorterNext 37 /* jump */ #define OP_Prev 38 /* jump */ #define OP_Next 39 /* jump */ @@ -16586,7 +16661,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ #define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ #define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 48 /* jump */ +#define OP_Program 48 /* jump0 */ #define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ @@ -16616,7 +16691,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Null 75 /* synopsis: r[P2..P3]=NULL */ #define OP_SoftNull 76 /* synopsis: r[P1]=NULL */ #define OP_Blob 77 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1) */ #define OP_Move 79 /* synopsis: r[P2@P3]=r[P1@P3] */ #define OP_Copy 80 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ #define OP_SCopy 81 /* synopsis: r[P2]=r[P1] */ @@ -16740,14 +16815,15 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */ +#define OPFLG_JUMP0 0x80 /* jump0: P2 might be zero */ #define OPFLG_INITIALIZER {\ /* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ -/* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x49, 0x49, 0x49,\ -/* 24 */ 0x49, 0x01, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,\ -/* 32 */ 0x41, 0x01, 0x41, 0x41, 0x41, 0x01, 0x41, 0x41,\ +/* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ +/* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ +/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x41, 0x41,\ /* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ -/* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 48 */ 0x81, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ /* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x41,\ /* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ /* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ @@ -16907,6 +16983,8 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); + SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); #ifdef SQLITE_ENABLE_BYTECODE_VTAB SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); @@ -17494,6 +17572,10 @@ struct FuncDefHash { }; #define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) +#if defined(SQLITE_USER_AUTHENTICATION) +# warning "The SQLITE_USER_AUTHENTICATION extension is deprecated. \ + See ext/userauth/user-auth.txt for details." +#endif #ifdef SQLITE_USER_AUTHENTICATION /* ** Information held in the "sqlite3" database connection object and used @@ -17797,7 +17879,7 @@ struct sqlite3 { #define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ #define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ #define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ @@ -18370,8 +18452,7 @@ struct Table { #define TF_HasStored 0x00000040 /* Has one or more STORED columns */ #define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ #define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ -#define TF_StatsUsed 0x00000100 /* Query planner decisions affected by - ** Index.aiRowLogEst[] values */ +#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */ #define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ #define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ @@ -18427,6 +18508,15 @@ struct Table { #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -19162,10 +19252,12 @@ struct IdList { ** ** Union member validity: ** -** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc -** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy -** u2.pIBIndex fg.isIndexedBy && !fg.isCte -** u2.pCteUse fg.isCte && !fg.isIndexedBy +** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc +** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy +** u1.nRow !fg.isTabFunc && !fg.isIndexedBy +** +** u2.pIBIndex fg.isIndexedBy && !fg.isCte +** u2.pCteUse fg.isCte && !fg.isIndexedBy */ struct SrcItem { Schema *pSchema; /* Schema to which this item is fixed */ @@ -19193,6 +19285,7 @@ struct SrcItem { unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ + unsigned rowidUsed :1; /* The ROWID of this table is referenced */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ union { @@ -19203,6 +19296,7 @@ struct SrcItem { union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ + u32 nRow; /* Number of rows in a VALUES clause */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ @@ -19267,7 +19361,7 @@ struct SrcList { #define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ #define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ - /* 0x2000 not currently used */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ @@ -19346,6 +19440,7 @@ struct NameContext { #define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ #define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ #define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */ #define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ /* @@ -19369,6 +19464,7 @@ struct Upsert { Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + u8 isDup; /* True if 2nd or later with same pUpsertIdx */ /* Above this point is the parse tree for the ON CONFLICT clauses. ** The next group of fields stores intermediate data. */ void *pToFree; /* Free memory when deleting the Upsert object */ @@ -19458,11 +19554,12 @@ struct Select { #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ -#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ /* True if S exists and has SF_NestedFrom */ #define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) @@ -19702,6 +19799,7 @@ struct Parse { u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasWith; /* True if statement contains WITH */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -20139,6 +20237,11 @@ struct Sqlite3Config { #endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ @@ -20376,6 +20479,9 @@ struct Window { ** due to the SQLITE_SUBTYPE flag */ }; +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow); +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal); + #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); @@ -20595,10 +20701,13 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); # define EXP754 (((u64)0x7ff)<<52) # define MAN754 ((((u64)1)<<52)-1) # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) SQLITE_PRIVATE int sqlite3IsNaN(double); +SQLITE_PRIVATE int sqlite3IsOverflow(double); #else -# define IsNaN(X) 0 -# define sqlite3IsNaN(X) 0 +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* @@ -20690,6 +20799,7 @@ SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3DequoteToken(Token*); +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*); @@ -20720,7 +20830,7 @@ SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*) SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3*,void*); -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse*, Expr*); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); @@ -20943,12 +21053,10 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse*,Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); -SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif @@ -21133,7 +21241,9 @@ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3*); SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); +#if !defined(SQLITE_OMIT_BLOB_LITERAL) SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +#endif SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -21444,7 +21554,7 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8); SQLITE_PRIVATE Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3*,Upsert*); SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); -SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*); SQLITE_PRIVATE void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert*); @@ -21834,6 +21944,9 @@ static const char * const sqlite3azCompileOpt[] = { "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), # endif #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", +#endif #ifdef SQLITE_ALLOW_URI_AUTHORITY "ALLOW_URI_AUTHORITY", #endif @@ -22853,6 +22966,9 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { #endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ #endif 0, /* bLocaltimeFault */ 0, /* xAltLocaltime */ @@ -24174,13 +24290,14 @@ struct DateTime { int tz; /* Timezone offset in minutes */ double s; /* Seconds */ char validJD; /* True (1) if iJD is valid */ - char rawS; /* Raw numeric value stored in s */ char validYMD; /* True (1) if Y,M,D are valid */ char validHMS; /* True (1) if h,m,s are valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ - char isError; /* An overflow has occurred */ - char useSubsec; /* Display subsecond precision */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -24278,6 +24395,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -24290,7 +24409,6 @@ static int parseTimezone(const char *zDate, DateTime *p){ p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -24334,7 +24452,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -24381,15 +24498,40 @@ static void computeJD(DateTime *p){ p->validJD = 1; if( p->validHMS ){ p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); - if( p->validTZ ){ + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } +/* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + /* ** Parse dates of the form ** @@ -24428,12 +24570,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -24443,6 +24589,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; @@ -24581,7 +24730,7 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME @@ -24713,7 +24862,7 @@ static int toLocaltime( p->validHMS = 1; p->validJD = 0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->isError = 0; return SQLITE_OK; } @@ -24733,12 +24882,12 @@ static const struct { float rLimit; /* Maximum NNN value for this transform */ float rXform; /* Constant used for this transform */ } aXformType[] = { - { 6, "second", 4.6427e+14, 1.0 }, - { 6, "minute", 7.7379e+12, 60.0 }, - { 4, "hour", 1.2897e+11, 3600.0 }, - { 3, "day", 5373485.0, 86400.0 }, - { 5, "month", 176546.0, 2592000.0 }, - { 4, "year", 14713.0, 31536000.0 }, + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 30.0*86400.0 }, + /* 5 */ { 4, "year", 14713.0, 365.0*86400.0 }, }; /* @@ -24770,14 +24919,20 @@ static void autoAdjustDate(DateTime *p){ ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written @@ -24808,6 +24963,37 @@ static int parseModifier( } break; } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } case 'j': { /* ** julianday @@ -24834,7 +25020,9 @@ static int parseModifier( ** show local time. */ if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ - rc = toLocaltime(p, pCtx); + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -24859,7 +25047,7 @@ static int parseModifier( } #ifndef SQLITE_OMIT_LOCALTIME else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ - if( p->tzSet==0 ){ + if( p->isUtc==0 ){ i64 iOrigJD; /* Original localtime */ i64 iGuess; /* Guess at the corresponding utc time */ int cnt = 0; /* Safety to prevent infinite loop */ @@ -24882,7 +25070,8 @@ static int parseModifier( memset(p, 0, sizeof(*p)); p->iJD = iGuess; p->validJD = 1; - p->tzSet = 1; + p->isUtc = 1; + p->isLocal = 0; } rc = SQLITE_OK; } @@ -24902,7 +25091,7 @@ static int parseModifier( && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -24942,7 +25131,7 @@ static int parseModifier( p->h = p->m = 0; p->s = 0.0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; @@ -25013,6 +25202,7 @@ static int parseModifier( x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); computeJD(p); p->validHMS = 0; p->validYMD = 0; @@ -25059,11 +25249,12 @@ static int parseModifier( z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; + if( n<3 || n>10 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; + p->nFloor = 0; for(i=0; iM += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } case 5: { /* Special processing to add years */ int y = (int)r; - assert( strcmp(aXformType[i].zName,"year")==0 ); + assert( strcmp(aXformType[5].zName,"year")==0 ); computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); p->Y += y; + computeFloor(p); p->validJD = 0; r -= (int)r; break; @@ -25339,22 +25533,83 @@ static void dateFunc( } } +/* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "am" or "pm" +** %P "AM" or "PM" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 Sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -25391,7 +25646,7 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='d' ? "%02d" : "%2d", x.D); break; } - case 'f': { + case 'f': { /* Fractional seconds. (Non-standard) */ double s = x.s; if( s>59.999 ) s = 59.999; sqlite3_str_appendf(&sRes, "%06.3f", s); @@ -25401,6 +25656,21 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); break; } + case 'G': /* Fall thru */ + case 'g': { + DateTime y = x; + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); + }else{ + sqlite3_str_appendf(&sRes, "%04d", y.Y); + } + break; + } case 'H': case 'k': { sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); @@ -25414,25 +25684,11 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); break; } - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ - DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( cf=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); - }else{ - sqlite3_str_appendf(&sRes,"%03d",nDay+1); - } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); break; } - case 'J': { + case 'J': { /* Julian day number. (Non-standard) */ sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); break; } @@ -25475,13 +25731,33 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); break; } - case 'u': /* Fall thru */ - case 'w': { - char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; if( c=='0' && cf=='u' ) c = '7'; sqlite3_str_appendchar(&sRes, 1, c); break; } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); + break; + } case 'Y': { sqlite3_str_appendf(&sRes,"%04d",x.Y); break; @@ -25628,9 +25904,7 @@ static void timediffFunc( d1.iJD = d2.iJD - d1.iJD; d1.iJD += (u64)1486995408 * (u64)100000; } - d1.validYMD = 0; - d1.validHMS = 0; - d1.validTZ = 0; + clearYMD_HMS_TZ(&d1); computeYMD_HMS(&d1); sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", @@ -25699,6 +25973,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -25714,6 +26018,9 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), @@ -30129,6 +30436,24 @@ static void sqlite3MallocAlarm(int nByte){ sqlite3_mutex_enter(mem0.mutex); } +#ifdef SQLITE_DEBUG +/* +** This routine is called whenever an out-of-memory condition is seen, +** It's only purpose to to serve as a breakpoint for gdb or similar +** code debuggers when working on out-of-memory conditions, for example +** caused by PRAGMA hard_heap_limit=N. +*/ +static SQLITE_NOINLINE void test_oom_breakpoint(u64 n){ + static u64 nOomFault = 0; + nOomFault += n; + /* The assert() is never reached in a human lifetime. It is here mostly + ** to prevent code optimizers from optimizing out this function. */ + assert( (nOomFault>>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -30155,6 +30480,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); *pp = 0; return; } @@ -30443,6 +30769,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); return 0; } } @@ -31309,6 +31636,7 @@ SQLITE_API void sqlite3_str_vappendf( if( xtype==etFLOAT ){ iRound = -precision; }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; iRound = precision; }else{ iRound = precision+1; @@ -31344,13 +31672,14 @@ SQLITE_API void sqlite3_str_vappendf( } exp = s.iDP-1; - if( xtype==etGENERIC && precision>0 ) precision--; /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -31666,9 +31995,13 @@ SQLITE_API void sqlite3_str_vappendf( sqlite3_str_appendall(pAccum, pItem->zAlias); }else{ Select *pSel = pItem->pSelect; - assert( pSel!=0 ); + assert( pSel!=0 ); /* Because of tag-20240424-1 */ if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } @@ -32445,8 +32778,10 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) x.printfFlags |= SQLITE_PRINTF_INTERNAL; sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", - pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", + pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, + pItem->colUsed, + pItem->fg.rowidUsed ? "+rowid" : ""); } if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){ sqlite3_str_appendf(&x, " FULL-OUTER-JOIN"); @@ -32486,12 +32821,14 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); } if( pItem->pSelect ){ + sqlite3TreeViewPush(&pView, i+1nSrc); if( pItem->pTab ){ Table *pTab = pItem->pTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); + sqlite3TreeViewPop(&pView); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); @@ -32595,7 +32932,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); if( p->pLimit->pRight ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewItem(pView, "OFFSET", 0); sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); sqlite3TreeViewPop(&pView); } @@ -34640,6 +34977,19 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is NaN or +Inf or -Inf. +*/ +SQLITE_PRIVATE int sqlite3IsOverflow(double x){ + int rc; /* The value return */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsOvfl(y); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -34883,6 +35233,44 @@ SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ sqlite3Dequote(p->u.zToken); } +/* +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. +*/ +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse *pParse, Expr *p){ + assert( p!=0 || pParse->db->mallocFailed ); + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); + int iValue; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } + } + }while( *pIn++ ); + if( bHex ) p->op = TK_INTEGER; + + /* tag-20240227-a: If after dequoting, the number is an integer that + ** fits in 32 bits, then it must be converted into EP_IntValue. Other + ** parts of the code expect this. See also tag-20240227-b. */ + if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){ + p->u.iValue = iValue; + p->flags |= EP_IntValue; + } + } +} + /* ** If the input token p is quoted, try to adjust the token to remove ** the quotes. This is not always possible: @@ -35199,6 +35587,9 @@ do_atof_calc: u64 s2; rr[0] = (double)s; s2 = (u64)rr[0]; +#if defined(_MSC_VER) && _MSC_VER<1700 + if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } +#endif rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); if( e>0 ){ while( e>=100 ){ @@ -35641,7 +36032,7 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou assert( p->n>0 ); assert( p->nzBuf) ); p->iDP = p->n + exp; - if( iRound<0 ){ + if( iRound<=0 ){ iRound = p->iDP - iRound; if( iRound==0 && p->zBuf[i+1]>='5' ){ iRound = 1; @@ -36819,7 +37210,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), /* 32 */ "Last" OpHelp(""), - /* 33 */ "IfSmaller" OpHelp(""), + /* 33 */ "IfSizeBetween" OpHelp(""), /* 34 */ "SorterSort" OpHelp(""), /* 35 */ "Sort" OpHelp(""), /* 36 */ "Rewind" OpHelp(""), @@ -36864,7 +37255,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 75 */ "Null" OpHelp("r[P2..P3]=NULL"), /* 76 */ "SoftNull" OpHelp("r[P1]=NULL"), /* 77 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1)"), /* 79 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), /* 80 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), /* 81 */ "SCopy" OpHelp("r[P2]=r[P1]"), @@ -39262,8 +39653,12 @@ static int unixLogErrorAtLine( ** available, the error message will often be an empty string. Not a ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -44361,12 +44756,19 @@ static int unixOpen( rc = SQLITE_READONLY_DIRECTORY; }else if( errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } } } if( fd<0 ){ @@ -53262,6 +53664,14 @@ SQLITE_API unsigned char *sqlite3_serialize( pOut = 0; }else{ sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } if( piSize ) *piSize = sz; if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ pOut = 0; @@ -63785,7 +64195,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; @@ -69809,6 +70219,7 @@ struct IntegrityCk { StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* @@ -70283,8 +70694,47 @@ int corruptPageError(int lineno, MemPage *p){ # define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) #endif +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** @@ -70361,6 +70811,8 @@ static int hasSharedCacheTableLock( iTab = iRoot; } + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ @@ -70494,6 +70946,8 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); @@ -70561,6 +71015,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -70599,6 +71055,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -75212,9 +75671,12 @@ static int accessPayload( if( pCur->aOverflow==0 || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -75224,6 +75686,12 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. @@ -75705,6 +76173,23 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -75733,18 +76218,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } @@ -75797,6 +76271,7 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } @@ -76263,10 +76738,10 @@ SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; @@ -77088,7 +77563,10 @@ static int fillInCell( n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; assert( nSrc<=nPayload ); testcase( nSrcaData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } @@ -78418,7 +78896,7 @@ static int balance_nonroot( memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ if( NEVER(limitaiOvfl[0]) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } limit = pOld->aiOvfl[0]; @@ -79061,7 +79539,7 @@ static int anotherValidCursor(BtCursor *pCur){ && pOther->eState==CURSOR_VALID && pOther->pPage==pCur->pPage ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pCur->pPage); } } return SQLITE_OK; @@ -79121,7 +79599,7 @@ static int balance(BtCursor *pCur){ /* The page being written is not a root page, and there is currently ** more than one reference to it. This only happens if the page is one ** of its own ancestor pages. Corruption. */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -79285,7 +79763,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -79313,7 +79791,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd || pCur->info.pPayload < pPage->aData + pPage->cellOffset ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCur->info.nLocal==nTotal ){ /* The entire cell is local */ @@ -79394,7 +79872,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** Which can only happen if the SQLITE_NoSchemaError flag was set when ** the schema was loaded. This cannot be asserted though, as a user might ** set the flag, load the schema, and then unset the flag. */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } @@ -79517,7 +79995,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( pPage->nFree<0 ){ if( NEVER(pCur->eState>CURSOR_INVALID) ){ /* ^^^^^--- due to the moveToRoot() call above */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ rc = btreeComputeFreeSpace(pPage); } @@ -79534,7 +80012,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; szNew = p->pBt->nPreformatSize; - if( szNew<4 ) szNew = 4; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); @@ -79556,7 +80037,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -79583,10 +80064,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( oldCell+szNew > pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } memcpy(oldCell, newCell, szNew); return SQLITE_OK; @@ -79596,7 +80077,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } @@ -79626,7 +80107,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -79688,7 +80169,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ @@ -79713,7 +80194,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 if( nRem>nIn ){ if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } ovflIn = get4byte(&pSrc->info.pPayload[nIn]); } @@ -79809,7 +80290,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); if( rc || pCur->eState!=CURSOR_VALID ) return rc; }else{ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } assert( pCur->eState==CURSOR_VALID ); @@ -79818,14 +80299,14 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ iCellIdx = pCur->ix; pPage = pCur->pPage; if( pPage->nCell<=iCellIdx ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } pCell = findCell(pPage, iCellIdx); if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCell<&pPage->aCellIdx[pPage->nCell] ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must @@ -79916,7 +80397,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ n = pCur->pPage->pgno; } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; @@ -80032,7 +80513,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); if( pgnoRoot>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgnoRoot); } pgnoRoot++; @@ -80080,7 +80561,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -80170,14 +80651,14 @@ static int clearDatabasePage( assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno); } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; } hdr = pPage->hdrOffset; @@ -80281,7 +80762,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); if( iTable>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(iTable); } rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -80875,6 +81356,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -80986,6 +81470,7 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header @@ -81085,6 +81570,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -81098,7 +81584,9 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( int bPartial = 0; /* True if not checking all btrees */ int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); /* aRoot[0]==0 means this is a partial check */ if( aRoot[0]==0 ){ @@ -81171,15 +81659,18 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); - } + if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } #endif - sCheck.v0 = aRoot[i]; - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; @@ -83234,6 +83725,13 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ } } +/* +** Set the iIdx'th entry of array aMem[] to contain integer value val. +*/ +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){ + sqlite3VdbeMemSetInt64(&aMem[iIdx], val); +} + /* A no-op destructor */ SQLITE_PRIVATE void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } @@ -83922,14 +84420,20 @@ static int valueFromExpr( } /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + ** case when the value is -9223372036854775808. Except - do not do this + ** for hexadecimal literals. */ + if( op==TK_UMINUS ){ + Expr *pLeft = pExpr->pLeft; + if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){ + if( ExprHasProperty(pLeft, EP_IntValue) + || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X' + ){ + pExpr = pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + } } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ @@ -83938,12 +84442,26 @@ static int valueFromExpr( if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + i64 iVal; + if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){ + sqlite3VdbeMemSetInt64(pVal, iVal*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + if( affinity==SQLITE_AFF_BLOB ){ + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + pVal->flags = MEM_Real; + }else if( op==TK_INTEGER ){ + /* This case is required by -9223372036854775808 and other strings + ** that look like integers but cannot be handled by the + ** sqlite3DecOrHexToI64() call above. */ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + } }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } @@ -84213,17 +84731,17 @@ SQLITE_PRIVATE int sqlite3Stat4Column( sqlite3_value **ppVal /* OUT: Extracted value */ ){ u32 t = 0; /* a column type code */ - int nHdr; /* Size of the header in the record */ - int iHdr; /* Next unread header byte */ - int iField; /* Next unread data byte */ - int szField = 0; /* Size of the current data field */ + u32 nHdr; /* Size of the header in the record */ + u32 iHdr; /* Next unread header byte */ + i64 iField; /* Next unread data byte */ + u32 szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ assert( iCol>0 ); iHdr = getVarint32(a, nHdr); - if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; iField = nHdr; for(i=0; i<=iCol; i++){ iHdr += getVarint32(&a[iHdr], t); @@ -85258,6 +85776,15 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } + + /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes + ** might */ + assert( pOp->p2>0 + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 ); + + /* Jumps never go off the end of the bytecode array */ + assert( pOp->p2nOp + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 ); break; } } @@ -87665,7 +88192,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFk(p, 0); } /* If the auto-commit flag is set and this is the only active writer @@ -88379,6 +88906,23 @@ static void serialGet( pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } } +static int serialGet7( + const unsigned char *buf, /* Buffer to deserialize from */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + if( IsNaN(x) ){ + pMem->flags = MEM_Null; + return 1; + } + pMem->flags = MEM_Real; + return 0; +} SQLITE_PRIVATE void sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ @@ -88818,17 +89362,15 @@ SQLITE_PRIVATE int sqlite3IntFloatCompare(i64 i, double r){ return (xr); }else{ i64 y; - double s; if( r<-9223372036854775808.0 ) return +1; if( r>=9223372036854775808.0 ) return -1; y = (i64)r; if( iy ) return +1; - s = (double)i; - testcase( doubleLt(s,r) ); - testcase( doubleLt(r,s) ); - testcase( doubleEq(r,s) ); - return (sr); + testcase( doubleLt(((double)i),r) ); + testcase( doubleLt(r,((double)i)) ); + testcase( doubleEq(r,((double)i)) ); + return (((double)i)r); } } @@ -89058,7 +89600,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + serialGet7(&aKey1[d1], &mem1); rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); @@ -89083,14 +89625,18 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else{ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - if( mem1.u.ru.r ){ + if( serialGet7(&aKey1[d1], &mem1) ){ + rc = -1; /* mem1 is a NaN */ + }else if( mem1.u.ru.r ){ rc = -1; }else if( mem1.u.r>pRhs->u.r ){ rc = +1; + }else{ + assert( rc==0 ); } }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } @@ -89160,7 +89706,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0 && serial_type!=10); + if( serial_type==0 + || serial_type==10 + || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0) + ){ + assert( rc==0 ); + }else{ + rc = 1; + } } if( rc!=0 ){ @@ -89620,7 +90173,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ @@ -89640,7 +90194,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff */ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( iVar>=32 ){ v->expmask |= 0x80000000; }else{ @@ -92225,7 +92780,6 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } if( flags & SQLITE_SCANSTAT_COMPLEX ){ idx = iScan; - pScan = &p->aScan[idx]; }else{ /* If the COMPLEX flag is clear, then this function must ignore any ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ @@ -92238,6 +92792,8 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } } if( idx>=p->nScan ) return 1; + assert( pScan==0 || pScan==&p->aScan[idx] ); + pScan = &p->aScan[idx]; switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { @@ -93686,7 +94242,7 @@ case OP_Return: { /* in1 */ ** ** See also: EndCoroutine */ -case OP_InitCoroutine: { /* jump */ +case OP_InitCoroutine: { /* jump0 */ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); @@ -93709,7 +94265,9 @@ jump_to_p2: ** ** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. -** After the jump, register P1 becomes undefined. +** After the jump, the value register P1 is left with a value +** such that subsequent OP_Yields go back to the this same +** OP_EndCoroutine instruction. ** ** See also: InitCoroutine */ @@ -93721,8 +94279,8 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); + pIn1->u.i = (int)(pOp - p->aOp) - 1; pOp = &aOp[pCaller->p2 - 1]; - pIn1->flags = MEM_Undefined; break; } @@ -93739,7 +94297,7 @@ case OP_EndCoroutine: { /* in1 */ ** ** See also: InitCoroutine */ -case OP_Yield: { /* in1, jump */ +case OP_Yield: { /* in1, jump0 */ int pcDest; pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); @@ -94069,19 +94627,15 @@ case OP_Blob: { /* out2 */ break; } -/* Opcode: Variable P1 P2 * P4 * -** Synopsis: r[P2]=parameter(P1,P4) +/* Opcode: Variable P1 P2 * * * +** Synopsis: r[P2]=parameter(P1) ** ** Transfer the values of bound parameter P1 into register P2 -** -** If the parameter is named, then its name appears in P4. -** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); - assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -94602,7 +95156,7 @@ case OP_AddImm: { /* in1 */ ** without data loss, then jump immediately to P2, or if P2==0 ** raise an SQLITE_MISMATCH exception. */ -case OP_MustBeInt: { /* jump, in1 */ +case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); @@ -94643,7 +95197,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** @@ -94858,7 +95412,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ - if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags1 & MEM_Str)!=0 ){ + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); testcase( pIn1->flags & MEM_IntReal ); @@ -94867,7 +95423,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags3 & MEM_Str)!=0 ){ + pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); testcase( pIn3->flags & MEM_IntReal ); @@ -96211,11 +96769,16 @@ case OP_MakeRecord: { switch( len ){ default: zPayload[7] = (u8)(v&0xff); v >>= 8; zPayload[6] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; zPayload[4] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 1: zPayload[0] = (u8)(v&0xff); } zPayload += len; @@ -97134,7 +97697,8 @@ case OP_SequenceTest: { ** is the only cursor opcode that works with a pseudo-table. ** ** P3 is the number of fields in the records that will be stored by -** the pseudo-table. +** the pseudo-table. If P2 is 0 or negative then the pseudo-cursor +** will return NULL for every column. */ case OP_OpenPseudo: { VdbeCursor *pCx; @@ -97277,10 +97841,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group, ncycle */ -case OP_SeekLE: /* jump, in3, group, ncycle */ -case OP_SeekGE: /* jump, in3, group, ncycle */ -case OP_SeekGT: { /* jump, in3, group, ncycle */ +case OP_SeekLT: /* jump0, in3, group, ncycle */ +case OP_SeekLE: /* jump0, in3, group, ncycle */ +case OP_SeekGE: /* jump0, in3, group, ncycle */ +case OP_SeekGT: { /* jump0, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -97947,7 +98511,7 @@ case OP_Found: { /* jump, in3, ncycle */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3, ncycle */ +case OP_SeekRowid: { /* jump0, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98706,7 +99270,7 @@ case OP_NullRow: { ** configured to use Prev, not Next. */ case OP_SeekEnd: /* ncycle */ -case OP_Last: { /* jump, ncycle */ +case OP_Last: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98740,28 +99304,38 @@ case OP_Last: { /* jump, ncycle */ break; } -/* Opcode: IfSmaller P1 P2 P3 * * +/* Opcode: IfSizeBetween P1 P2 P3 P4 * ** -** Estimate the number of rows in the table P1. Jump to P2 if that -** estimate is less than approximately 2**(0.1*P3). +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. +** +** Jump to P2 if X is in between P3 and P4, inclusive. */ -case OP_IfSmaller: { /* jump */ +case OP_IfSizeBetween: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; i64 sz; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640*2 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ sz = sqlite3BtreeRowCountEst(pCrsr); - if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); } + res = sz>=pOp->p3 && sz<=pOp->p4.i; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; break; @@ -98814,7 +99388,7 @@ case OP_Sort: { /* jump ncycle */ ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump, ncycle */ +case OP_Rewind: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -99461,11 +100035,18 @@ case OP_CreateBtree: { /* out2 */ break; } -/* Opcode: SqlExec * * * P4 * +/* Opcode: SqlExec P1 P2 * P4 * ** ** Run the SQL statement or statements specified in the P4 string. -** Disable Auth and Trace callbacks while those statements are running if -** P1 is true. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** */ case OP_SqlExec: { char *zErr; @@ -99473,6 +100054,7 @@ case OP_SqlExec: { sqlite3_xauth xAuth; #endif u8 mTrace; + int savedAnalysisLimit; sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; @@ -99481,18 +100063,23 @@ case OP_SqlExec: { xAuth = db->xAuth; #endif mTrace = db->mTrace; - if( pOp->p1 ){ + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = 0; #endif db->mTrace = 0; } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = xAuth; #endif db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; if( zErr || rc ){ sqlite3VdbeError(p, "%s", zErr); sqlite3_free(zErr); @@ -99644,11 +100231,11 @@ case OP_DropTrigger: { /* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +** register (P1+1) the text of an error message describing any problems. +** If no problems are found, store a NULL in register (P1+1). ** -** The register P3 contains one less than the maximum number of allowed errors. -** At most reg(P3) errors will be reported. +** The register (P1) contains one less than the maximum number of allowed +** errors. At most reg(P1) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** @@ -99668,19 +100255,21 @@ case OP_IntegrityCk: { Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); + assert( pOp->p4type==P4_INTARRAY ); nRoot = pOp->p2; aRoot = pOp->p4.ai; assert( nRoot>0 ); + assert( aRoot!=0 ); assert( aRoot[0]==(Pgno)nRoot ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pnErr = &aMem[pOp->p3]; + assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p1]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1+1]; assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, - (int)pnErr->u.i+1, &nErr, &z); + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], + &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); @@ -99807,7 +100396,9 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** P1 contains the address of the memory cell that contains the first memory ** cell in an array of values used as arguments to the sub-program. P2 ** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address +** exception using the RAISE() function. P2 might be zero, if there is +** no possibility that an IGNORE exception will be raised. +** Register P3 contains the address ** of a memory cell in this (the parent) VM that is used to allocate the ** memory required by the sub-vdbe at runtime. ** @@ -99815,7 +100406,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** ** If P5 is non-zero, then recursive program invocation is enabled. */ -case OP_Program: { /* jump */ +case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ int nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ @@ -101364,7 +101955,7 @@ case OP_Filter: { /* jump */ ** error is encountered. */ case OP_Trace: -case OP_Init: { /* jump */ +case OP_Init: { /* jump0 */ int i; #ifndef SQLITE_OMIT_TRACE char *zTrace; @@ -105265,10 +105856,10 @@ static int bytecodevtabColumn( #ifdef SQLITE_ENABLE_STMT_SCANSTATUS case 9: /* nexec */ - sqlite3_result_int(ctx, pOp->nExec); + sqlite3_result_int64(ctx, pOp->nExec); break; case 10: /* ncycle */ - sqlite3_result_int(ctx, pOp->nCycle); + sqlite3_result_int64(ctx, pOp->nCycle); break; #else case 9: /* nexec */ @@ -106212,6 +106803,8 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( db->mallocFailed ){ @@ -106359,7 +106952,7 @@ static void extendFJMatch( static SQLITE_NOINLINE int isValidSchemaTableName( const char *zTab, /* Name as it appears in the SQL */ Table *pTab, /* The schema table we are trying to match */ - Schema *pSchema /* non-NULL if a database qualifier is present */ + const char *zDb /* non-NULL if a database qualifier is present */ ){ const char *zLegacy; assert( pTab!=0 ); @@ -106370,7 +106963,7 @@ static SQLITE_NOINLINE int isValidSchemaTableName( if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ return 1; } - if( pSchema==0 ) return 0; + if( zDb==0 ) return 0; if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; }else{ @@ -106410,7 +107003,7 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ @@ -106427,6 +107020,7 @@ static int lookupName( Table *pTab = 0; /* Table holding the row */ Column *pCol; /* A column of pTab */ ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ @@ -106552,7 +107146,7 @@ static int lookupName( } }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ if( pTab->tnum!=1 ) continue; - if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ @@ -106599,8 +107193,37 @@ static int lookupName( } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && ALWAYS(pMatch!=0) + && ALWAYS(pMatch->pTab!=0) + && (pMatch->pTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ @@ -106629,7 +107252,8 @@ static int lookupName( if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 && ALWAYS(zTab==0 - || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -106726,13 +107350,18 @@ static int lookupName( ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; + cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pTab!=0 && IsView(pMatch->pTab) ){ + eNewExprOp = TK_NULL; + } +#endif if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -106886,6 +107515,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } @@ -106919,8 +107552,12 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && cnt==1 && pMatch!=0 ){ - pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; + } } pExpr->op = eNewExprOp; @@ -107097,6 +107734,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** resolved. This prevents "column" from being counted as having been ** referenced, which might prevent a SELECT from being erroneously ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ case TK_NOTNULL: case TK_ISNULL: { @@ -107107,19 +107757,36 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ anRef[i] = p->nRef; } sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ - testcase( ExprHasProperty(pExpr, EP_OuterON) ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->u.iValue = (pExpr->op==TK_NOTNULL); - pExpr->flags |= EP_IntValue; - pExpr->op = TK_INTEGER; - - for(i=0, p=pNC; p && ipNext, i++){ - p->nRef = anRef[i]; - } - sqlite3ExprDelete(pParse->db, pExpr->pLeft); - pExpr->pLeft = 0; + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; } + + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ + } + } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; return WRC_Prune; } @@ -107133,7 +107800,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ */ case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; @@ -107142,7 +107808,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zDb = 0; zTable = 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zColumn = pExpr->u.zToken; + pRight = pExpr; }else{ Expr *pLeft = pExpr->pLeft; testcase( pNC->ncFlags & NC_IdxExpr ); @@ -107161,14 +107827,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); zTable = pLeft->u.zToken; - zColumn = pRight->u.zToken; assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names @@ -107344,11 +108009,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif } } -#ifndef SQLITE_OMIT_WINDOWFUNC - else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ is_agg = 1; } -#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ if( pExpr->pLeft ){ @@ -107418,6 +108081,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -107426,6 +108090,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pExpr->x.pSelect->selFlags |= SF_Correlated; } pNC->ncFlags |= NC_Subquery; } @@ -107951,6 +108616,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; + assert( pItem->zName!=0 || pItem->pSelect!=0 );/* Test of tag-20240424-1*/ if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; @@ -108019,7 +108685,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; ipSrc->nSrc; i++){ @@ -108210,6 +108878,9 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ SQLITE_PRIVATE int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ @@ -108218,7 +108889,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -108232,7 +108903,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif sqlite3WalkExprNN(&w, pExpr); @@ -108249,10 +108920,10 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* @@ -108558,9 +109229,10 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } } return pExpr; @@ -109254,11 +109926,12 @@ SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ @@ -109274,7 +109947,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -109706,6 +110379,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); assert( db!=0 ); +exprDeleteRestart: assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -109721,7 +110395,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); @@ -109736,6 +110409,19 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } #endif } + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } + } } if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbNNFreeNN(db, p); @@ -109768,11 +110454,11 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** ** The pExpr might be deleted immediately on an OOM error. ** -** The deferred delete is (currently) implemented by adding the -** pExpr to the pParse->pConstExpr list with a register number of 0. +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. */ -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -110208,17 +110894,19 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int fla pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; + pNewItem->regResult = pOldItem->regResult; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nTabRef++; @@ -110684,6 +111372,54 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ return pExpr; } +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -110712,6 +111448,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from ** the ON or USING clauses of an outer join disqualifies the expression @@ -110731,6 +111468,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ){ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; @@ -110759,9 +111498,11 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_IF_NULL_ROW: case TK_REGISTER: case TK_DOT: + case TK_RAISE: testcase( pExpr->op==TK_REGISTER ); testcase( pExpr->op==TK_IF_NULL_ROW ); testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; case TK_VARIABLE: @@ -110783,15 +111524,15 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -110803,9 +111544,15 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* @@ -110821,8 +111568,24 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ ** can be added to the pParse->pConstExpr list and evaluated once when ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -110830,9 +111593,26 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* @@ -110850,7 +111630,10 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** -** (2) pExpr cannot use subqueries or non-deterministic functions. +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. ** ** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. ** (Is there some way to relax this constraint?) @@ -110879,7 +111662,8 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( Expr *pExpr, /* The constraint */ const SrcList *pSrcList, /* Complete FROM clause */ - int iSrc /* Which element of pSrcList to use */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ ){ const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ @@ -110904,7 +111688,8 @@ SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( } } } - return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } @@ -110989,7 +111774,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprLi */ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -111079,9 +111864,12 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); @@ -111234,13 +112022,13 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -111509,7 +112297,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( if( eType==0 && (inFlags & IN_INDEX_NOOP_OK) && ExprUseXList(pX) - && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ pParse->nTab--; /* Back out the allocation of the unused cursor */ iTab = -1; /* Cursor is not allocated */ @@ -111792,7 +112580,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ sqlite3VdbeChangeToNoop(v, addrOnce-1); sqlite3VdbeChangeToNoop(v, addrOnce); ExprClearProperty(pExpr, EP_Subrtn); @@ -112956,12 +113744,6 @@ expr_code_doover: assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); - pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ - sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); - } return target; } case TK_REGISTER: { @@ -113135,7 +113917,9 @@ expr_code_doover: } #endif - if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ /* SQL functions can be expensive. So try to avoid running them ** multiple times if we know they always give the same result */ return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -113166,7 +113950,7 @@ expr_code_doover: } for(i=0; ia[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -113308,8 +114092,9 @@ expr_code_doover: if( !ExprHasProperty(pExpr, EP_Collate) ){ /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called ** "SOFT-COLLATE" that is added to constraints that are pushed down - ** from outer queries into sub-queries by the push-down optimization. - ** Clear subtypes as subtypes may not cross a subquery boundary. + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. */ assert( pExpr->pLeft ); sqlite3ExprCode(pParse, pExpr->pLeft, target); @@ -113633,7 +114418,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ if( ConstFactorOk(pParse) && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ *pReg = 0; r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -113697,7 +114482,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ sqlite3ExprCodeCopy(pParse, pExpr, target); @@ -113756,7 +114541,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ @@ -114907,9 +115692,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } }else{ @@ -114918,9 +115702,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } } @@ -116679,7 +117462,7 @@ static int renameResolveTrigger(Parse *pParse){ /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ @@ -117621,7 +118404,12 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } @@ -118540,7 +119328,7 @@ static void statGet( if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); #ifdef SQLITE_ENABLE_STAT4 - assert( p->current.anEq[i] ); + assert( p->current.anEq[i] || p->nRow==0 ); #endif } sqlite3ResultStrAccum(context, &sStat); @@ -118725,7 +119513,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -118749,9 +119537,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -118790,41 +119583,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: + /* Implementation of the following: ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -118931,6 +119719,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -118954,6 +119748,13 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + if( doOnce ){ int mxCol = nCol; Index *pX; @@ -119006,7 +119807,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } @@ -120755,7 +121556,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION +#if SQLITE_USER_AUTHENTICATION && !defined(SQLITE_OMIT_SHARED_CACHE) if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelrc = SQLITE_ERROR; pParse->nErr++; return; } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; @@ -123428,11 +124229,11 @@ SQLITE_PRIVATE void sqlite3EndTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -123489,13 +124290,10 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Test for cycles in generated columns and illegal expressions ** in CHECK constraints and in DEFAULT clauses. */ if( p->tabFlags & TF_HasGenerated ){ - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, - sqlite3MPrintf(db, "PRAGMA \"%w\".integrity_check(%Q)", - db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } /* Add the table to the in-memory representation of the database. @@ -123572,9 +124370,12 @@ SQLITE_PRIVATE void sqlite3CreateView( ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -123630,8 +124431,9 @@ create_view_fail: #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ @@ -123754,7 +124556,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; } SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable!=0 ); @@ -128947,13 +129749,13 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ double r1, r2; const char *zVal; r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!.15g", r1); + sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); if( r1!=r2 ){ sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!.20e", r1); + sqlite3_str_appendf(pStr, "%!0.20e", r1); } } break; @@ -129255,7 +130057,7 @@ static void replaceFunc( } if( zPattern[0]==0 ){ assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); - sqlite3_result_value(context, argv[0]); + sqlite3_result_text(context, (const char*)zStr, nStr, SQLITE_TRANSIENT); return; } nPattern = sqlite3_value_bytes(argv[1]); @@ -129738,7 +130540,7 @@ static void sumFinalize(sqlite3_context *context){ if( p->approx ){ if( p->ovrfl ){ sqlite3_result_error(context,"integer overflow",-1); - }else if( !sqlite3IsNaN(p->rErr) ){ + }else if( !sqlite3IsOverflow(p->rErr) ){ sqlite3_result_double(context, p->rSum+p->rErr); }else{ sqlite3_result_double(context, p->rSum); @@ -129755,7 +130557,7 @@ static void avgFinalize(sqlite3_context *context){ double r; if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -129769,7 +130571,7 @@ static void totalFinalize(sqlite3_context *context){ if( p ){ if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -130052,6 +130854,8 @@ static void groupConcatValue(sqlite3_context *context){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); @@ -132666,6 +133470,196 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->addrFillSub - 1); + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->pSelect = pLeft; + p->fg.viaCoroutine = 1; + p->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + p->regReturn = ++pParse->nMem; + p->iCursor = -1; + p->u1.nRow = 2; + sqlite3VdbeAddOp3(v,OP_InitCoroutine,p->regReturn,0,p->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, p->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + p->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + assert( p!=0 ); + if( p->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, p->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -133002,25 +133996,40 @@ SQLITE_PRIVATE void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - assert( db->pParse==pParse ); - if( rc || pParse->nErr ) goto insert_cleanup; - assert( db->mallocFailed==0 ); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + dest.iSDParm = pItem->regReturn; + regFromSelect = pItem->regResult; + nColumn = pItem->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -133175,7 +134184,7 @@ SQLITE_PRIVATE void sqlite3Insert( pNx->iDataCur = iDataCur; pNx->iIdxCur = iIdxCur; if( pNx->pUpsertTarget ){ - if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ goto insert_cleanup; } } @@ -135067,7 +136076,10 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif @@ -137742,6 +138754,34 @@ static const PragmaName aPragmaName[] = { /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen emperically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or @@ -139387,7 +140427,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - if( sqlite3GetInt32(zRight, &mxErr) ){ + if( sqlite3GetInt32(pValue->z, &mxErr) ){ if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -139404,7 +140444,6 @@ SQLITE_PRIVATE void sqlite3Pragma( Hash *pTbls; /* Set of all tables in the schema */ int *aRoot; /* Array of root page numbers of all btrees */ int cnt = 0; /* Number of entries in aRoot[] */ - int mxIdx = 0; /* Maximum number of indexes for any table */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -139426,7 +140465,6 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pObjTab && pObjTab!=pTab ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } if( cnt==0 ) continue; if( pObjTab ) cnt++; @@ -139446,11 +140484,11 @@ SQLITE_PRIVATE void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - sqlite3TouchRegister(pParse, 8+mxIdx); + sqlite3TouchRegister(pParse, 8+cnt); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, @@ -139460,6 +140498,36 @@ SQLITE_PRIVATE void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ @@ -139474,31 +140542,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int mxCol; /* Maximum non-virtual column number */ if( pObjTab && pObjTab!=pTab ) continue; - if( !IsOrdinaryTable(pTab) ){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_vtab *pVTab; - int a1; - if( !IsVirtual(pTab) ) continue; - if( pTab->nCol<=0 ){ - const char *zMod = pTab->u.vtab.azArg[0]; - if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; - } - sqlite3ViewGetColumnNames(pParse, pTab); - if( pTab->u.vtab.p==0 ) continue; - pVTab = pTab->u.vtab.p->pVtab; - if( NEVER(pVTab==0) ) continue; - if( NEVER(pVTab->pModule==0) ) continue; - if( pVTab->pModule->iVersion<4 ) continue; - if( pVTab->pModule->xIntegrity==0 ) continue; - sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); - pTab->nTabRef++; - sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); - a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, a1); -#endif - continue; - } + if( !IsOrdinaryTable(pTab) ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; r2 = 0; @@ -139633,6 +140677,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** is REAL, we have to load the actual data using OP_Column ** to reliably determine if the value is a NULL. */ sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); VdbeCoverage(v); } @@ -139806,23 +140851,43 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); - if( !isQuick ){ - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, addr); - } - if( pPk ){ - sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); - } + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( pObjTab && pObjTab!=pTab ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; + } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); @@ -140086,44 +141151,63 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (2) The table is an ordinary table, not a virtual table or view. ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more - ** since the last time ANALYZE was run. + ** (3) The table name does not begin with "sqlite_". + ** + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. + ** + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -140135,6 +141219,10 @@ SQLITE_PRIVATE void sqlite3Pragma( LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -140142,6 +141230,14 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -140150,23 +141246,61 @@ SQLITE_PRIVATE void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; - /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ - break; + szThreshold = -1; /* Always analyze if any index lacks statistics */ } } - if( szThreshold ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", @@ -140176,11 +141310,27 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddrnConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; j = pConstraint->iColumn - pTab->iHidden; assert( j < 2 ); seen[j] = i+1; @@ -140459,12 +141609,13 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; - j = seen[1]-1; - pIdxInfo->aConstraintUsage[j].argvIndex = 2; - pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } return SQLITE_OK; } @@ -140484,6 +141635,7 @@ static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ int i; sqlite3_finalize(pCsr->pPragma); pCsr->pPragma = 0; + pCsr->iRowid = 0; for(i=0; iazArg); i++){ sqlite3_free(pCsr->azArg[i]); pCsr->azArg[i] = 0; @@ -141284,7 +142436,13 @@ SQLITE_PRIVATE void *sqlite3ParserAddCleanup( void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ - ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } if( pCleanup ){ pCleanup->pNext = pParse->pCleanup; pParse->pCleanup = pCleanup; @@ -143406,9 +144564,16 @@ static void generateSortTail( int addrExplain; /* Address of OP_Explain instruction */ #endif - ExplainQueryPlan2(addrExplain, (pParse, 0, - "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") - ); + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); @@ -143446,7 +144611,6 @@ static void generateSortTail( regRow = sqlite3GetTempRange(pParse, nColumn); } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; @@ -143686,11 +144850,7 @@ static const char *columnTypeImpl( ** data for the result-set column of the sub-select. */ if( iColpEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif + && (!ViewCanHaveRowid || iCol>=0) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see @@ -144055,8 +145215,7 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( NameContext sNC; assert( pSelect!=0 ); - testcase( (pSelect->selFlags & SF_Resolved)==0 ); - assert( (pSelect->selFlags & SF_Resolved)!=0 || IN_RENAME_OBJECT ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed || IN_RENAME_OBJECT ) return; @@ -144067,17 +145226,22 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; i64 n; + int m = 0; + Select *pS2 = pSelect; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } if( pCol->affinity<=SQLITE_AFF_NONE ){ pCol->affinity = aff; } - if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ - int m = 0; - Select *pS2; - for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); } if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ @@ -144107,12 +145271,12 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( } } if( zType ){ - i64 m = sqlite3Strlen30(zType); + const i64 k = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; } } @@ -146509,7 +147673,7 @@ static void constInsert( ){ int i; assert( pColumn->op==TK_COLUMN ); - assert( sqlite3ExprIsConstant(pValue) ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; @@ -146567,10 +147731,10 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ pLeft = pExpr->pLeft; assert( pRight!=0 ); assert( pLeft!=0 ); - if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ constInsert(pConst,pRight,pLeft,pExpr); } - if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ constInsert(pConst,pLeft,pRight,pExpr); } } @@ -146791,6 +147955,18 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** ** (1) (** This restriction was removed on 2017-09-29. We used to @@ -146856,15 +148032,19 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING ** clause and the subquery. ** -** Without this restriction, the push-down optimization might move -** the ON/USING filter expression from the left side of a RIGHT JOIN -** over to the right side, which leads to incorrect answers. See -** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). ** ** (10) The inner query is not the right-hand table of a RIGHT JOIN. ** ** (11) The subquery is not a VALUES clause ** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -146975,7 +148155,19 @@ static int pushDownWhereTerms( } #endif - if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -147602,12 +148794,14 @@ SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } @@ -147875,7 +149069,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pNestedFrom = pFrom->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); - assert( VisibleRowid(pTab)==0 ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; @@ -147907,7 +149101,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pUsing = 0; } - nAdd = pTab->nCol + (VisibleRowid(pTab) && (selFlags&SF_NestedFrom)); + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; for(j=0; ja[pNew->nExpr-1]; assert( pX->zEName==0 ); if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - if( pNestedFrom ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ @@ -148106,8 +149302,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; - testcase( (p->selFlags & SF_Resolved)==0 ); - assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT ); + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; @@ -148177,6 +149372,8 @@ SQLITE_PRIVATE void sqlite3SelectPrep( */ static void printAggInfo(AggInfo *pAggInfo){ int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); for(ii=0; iinColumn; ii++){ struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; sqlite3DebugPrintf( @@ -149367,7 +150564,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Generate code for all sub-queries in the FROM clause */ pSub = pItem->pSelect; - if( pSub==0 ) continue; + if( pSub==0 || pItem->addrFillSub!=0 ) continue; /* The code for a subquery should only be generated once. */ assert( pItem->addrFillSub==0 ); @@ -149398,7 +150595,7 @@ SQLITE_PRIVATE int sqlite3Select( #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); } /* Convert unused result columns of the subquery into simple NULL @@ -150279,6 +151476,12 @@ select_end: sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } +#endif for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; if( pExpr==0 ) continue; @@ -151460,6 +152663,72 @@ static ExprList *sqlite3ExpandReturning( return pNew; } +/* If the Expr node is a subquery or an EXISTS operator or an IN operator that +** uses a subquery, and if the subquery is SF_Correlated, then mark the +** expression as EP_VarSelect. +*/ +static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){ + UNUSED_PARAMETER(NotUsed); + if( ExprUseXSelect(pExpr) + && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0 + ){ + testcase( ExprHasProperty(pExpr, EP_VarSelect) ); + ExprSetProperty(pExpr, EP_VarSelect); + } + return WRC_Continue; +} + + +/* +** If the SELECT references the table pWalker->u.pTab, then do two things: +** +** (1) Mark the SELECT as as SF_Correlated. +** (2) Set pWalker->eCode to non-zero so that the caller will know +** that (1) has happened. +*/ +static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ + int i; + SrcList *pSrc; + assert( pSelect!=0 ); + pSrc = pSelect->pSrc; + assert( pSrc!=0 ); + for(i=0; inSrc; i++){ + if( pSrc->a[i].pTab==pWalker->u.pTab ){ + testcase( pSelect->selFlags & SF_Correlated ); + pSelect->selFlags |= SF_Correlated; + pWalker->eCode = 1; + break; + } + } + return WRC_Continue; +} + +/* +** Scan the expression list that is the argument to RETURNING looking +** for subqueries that depend on the table which is being modified in the +** statement that is hosting the RETURNING clause (pTab). Mark all such +** subqueries as SF_Correlated. If the subqueries are part of an +** expression, mark the expression as EP_VarSelect. +** +** https://sqlite.org/forum/forumpost/2c83569ce8945d39 +*/ +static void sqlite3ProcessReturningSubqueries( + ExprList *pEList, + Table *pTab +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3ReturningSubqueryCorrelated; + w.u.pTab = pTab; + sqlite3WalkExprList(&w, pEList); + if( w.eCode ){ + w.xExprCallback = sqlite3ReturningSubqueryVarSelect; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkExprList(&w, pEList); + } +} + /* ** Generate code for the RETURNING trigger. Unlike other triggers ** that invoke a subprogram in the bytecode, the code for RETURNING @@ -151496,6 +152765,7 @@ static void codeReturningTrigger( sSelect.pSrc = &sFrom; sFrom.nSrc = 1; sFrom.a[0].pTab = pTab; + sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ sFrom.a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); if( pParse->nErr==0 ){ @@ -151522,6 +152792,7 @@ static void codeReturningTrigger( int i; int nCol = pNew->nExpr; int reg = pParse->nMem+1; + sqlite3ProcessReturningSubqueries(pNew, pTab); pParse->nMem += nCol+2; pReturning->iRetReg = reg; for(i=0; ipUpsertIdx = pIdx; + if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){ + /* Really this should be an error. The isDup ON CONFLICT clause will + ** never fire. But this problem was not discovered until three years + ** after multi-CONFLICT upsert was added, and so we silently ignore + ** the problem to prevent breaking applications that might actually + ** have redundant ON CONFLICT clauses. */ + pUpsert->isDup = 1; + } break; } if( pUpsert->pUpsertIdx==0 ){ @@ -153589,9 +154872,13 @@ SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ Upsert *pNext; if( NEVER(pUpsert==0) ) return 0; pNext = pUpsert->pNextUpsert; - if( pNext==0 ) return 1; - if( pNext->pUpsertTarget==0 ) return 1; - if( pNext->pUpsertIdx==0 ) return 1; + while( 1 /*exit-by-return*/ ){ + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + if( !pNext->isDup ) return 0; + pNext = pNext->pNextUpsert; + } return 0; } @@ -154716,6 +156003,8 @@ static int vtabCallConstructor( db->pVtabCtx = &sCtx; pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + assert( pTab!=0 ); + assert( pTab->nTabRef>1 || rc!=SQLITE_OK ); sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); @@ -154738,7 +156027,7 @@ static int vtabCallConstructor( pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + *pzErr = sqlite3MPrintf(db, zFormat, zModuleName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -154916,12 +156205,30 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; Parse sParse; int initBusy; + int i; + const unsigned char *z; + static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif + + /* Verify that the first two keywords in the CREATE TABLE statement + ** really are "CREATE" and "TABLE". If this is not the case, then + ** sqlite3_declare_vtab() is being misused. + */ + z = (const unsigned char*)zCreateTable; + for(i=0; aKeyword[i]; i++){ + int tokenType = 0; + do{ z += sqlite3GetToken(z, &tokenType); }while( tokenType==TK_SPACE ); + if( tokenType!=aKeyword[i] ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error"); + return SQLITE_ERROR; + } + } + sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ @@ -154929,6 +156236,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( IsVirtual(pTab) ); @@ -154942,11 +156250,10 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ initBusy = db->init.busy; db->init.busy = 0; sParse.nQueryLoop = 1; - if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) - && ALWAYS(sParse.pNewTable!=0) - && ALWAYS(!db->mallocFailed) - && IsOrdinaryTable(sParse.pNewTable) - ){ + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){ + assert( sParse.pNewTable!=0 ); + assert( !db->mallocFailed ); + assert( IsOrdinaryTable(sParse.pNewTable) ); assert( sParse.zErrMsg==0 ); if( !pTab->aCol ){ Table *pNew = sParse.pNewTable; @@ -157441,6 +158748,27 @@ static SQLITE_NOINLINE void filterPullDown( } } +/* +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one +** row for each key generated for the index. +*/ +static int whereLoopIsOneRow(WhereLoop *pLoop){ + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + ){ + int ii; + for(ii=0; iiu.btree.nEq; ii++){ + if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){ + return 0; + } + } + return 1; + } + return 0; +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -157519,7 +158847,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + VdbeComment((v, "init LEFT JOIN match flag")); } /* Compute a safe address to jump to if we discover that the table for @@ -158188,7 +159516,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } /* Record the instruction used to terminate the loop. */ - if( pLoop->wsFlags & WHERE_ONEROW ){ + if( (pLoop->wsFlags & WHERE_ONEROW) + || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) + ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -158578,6 +159908,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ iLoop = (pIdx ? 1 : 2); do{ @@ -158828,7 +160164,16 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( pRJ->regReturn); for(k=0; ka[k].pWLoop->iTab == pWInfo->a[k].iFrom ); + pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; mAll |= pWInfo->a[k].pWLoop->maskSelf; + if( pRight->fg.viaCoroutine ){ + sqlite3VdbeAddOp3( + v, OP_Null, 0, pRight->regResult, + pRight->regResult + pRight->pSelect->pEList->nExpr-1 + ); + } sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); iIdxCur = pWInfo->a[k].iIdxCur; if( iIdxCur ){ @@ -159885,7 +161230,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aiColumn[i]!=XN_EXPR ) continue; assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 - && pExpr->op!=TK_STRING + && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr) ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR; @@ -160534,6 +161879,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec continue; } if( pWC->a[ii].leftCursor!=iCsr ) return; + if( pWC->a[ii].prereqRight!=0 ) return; } /* Check condition (5). Return early if it is not met. */ @@ -160548,12 +161894,14 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec /* All conditions are met. Add the terms to the where-clause object. */ assert( p->pLimit->op==TK_LIMIT ); - whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, - iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); - if( p->iOffset>0 ){ + if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){ whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight, iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET); } + if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, + iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); + } } } @@ -161071,6 +162419,42 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ return 0; } +/* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + assert( pTerm->eOperator & WO_IN ); + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; +} + /* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). @@ -161121,16 +162505,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; + }else{ + CollSeq *pColl; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - assert(pX->pLeft); - pColl = sqlite3ExprCompareCollSeq(pParse, pX); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } @@ -161482,9 +162874,13 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", @@ -161502,9 +162898,13 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -161518,8 +162918,8 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define whereTraceIndexInfoInputs(A) -#define whereTraceIndexInfoOutputs(A) +#define whereTraceIndexInfoInputs(A,B) +#define whereTraceIndexInfoOutputs(A,B) #endif /* @@ -161703,7 +163103,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -161745,7 +163145,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** if they go out of sync. */ if( IsView(pTable) ){ - extraCols = ALLBITS; + extraCols = ALLBITS & ~idxCols; }else{ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); } @@ -161972,7 +163372,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTermpExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -162098,7 +163498,7 @@ static sqlite3_index_info *allocateIndexInfo( Expr *pE2; /* Skip over constant terms in the ORDER BY clause */ - if( sqlite3ExprIsConstant(pExpr) ){ + if( sqlite3ExprIsConstant(0, pExpr) ){ continue; } @@ -162133,7 +163533,7 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n ){ nOrderBy = n; - if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ eDistinct = 1; @@ -162210,7 +163610,7 @@ static sqlite3_index_info *allocateIndexInfo( pIdxInfo->nConstraint = j; for(i=j=0; ia[i].pExpr; - if( sqlite3ExprIsConstant(pExpr) ) continue; + if( sqlite3ExprIsConstant(0, pExpr) ) continue; assert( pExpr->op==TK_COLUMN || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN && pExpr->iColumn==pExpr->pLeft->iColumn) ); @@ -162262,11 +163662,11 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; - whereTraceIndexInfoInputs(p); + whereTraceIndexInfoInputs(p, pTab); pParse->db->nSchemaLock++; rc = pVtab->pModule->xBestIndex(pVtab, p); pParse->db->nSchemaLock--; - whereTraceIndexInfoOutputs(p); + whereTraceIndexInfoOutputs(p, pTab); if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ @@ -163744,7 +165144,9 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered || pProbe->bLowQual ){ if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); + if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ + opMask &= ~(WO_EQ|WO_IN|WO_IS); + } } assert( pNew->u.btree.nEqnColumn ); @@ -164011,10 +165413,13 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ assert( pSrc->pTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior @@ -164025,7 +165430,15 @@ static int whereLoopAddBtreeIndex( }else{ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; } - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } @@ -164131,7 +165544,9 @@ static int indexMightHelpWithOrderBy( for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( NEVER(pExpr==0) ) continue; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) + && pExpr->iTable==iCursor + ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; @@ -164388,7 +165803,7 @@ static void wherePartIdxExpr( u8 aff; if( pLeft->op!=TK_COLUMN ) return; - if( !sqlite3ExprIsConstant(pRight) ) return; + if( !sqlite3ExprIsConstant(0, pRight) ) return; if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; if( pLeft->iColumn<0 ) return; aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; @@ -164661,7 +166076,9 @@ static int whereLoopAddBtree( " according to whereIsCoveringIndex()\n", pProbe->zName)); } } - }else if( m==0 ){ + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ WHERETRACE(0x200, ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); @@ -164737,7 +166154,7 @@ static int whereLoopAddBtree( ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the ** plan */ - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; } #ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); @@ -164759,6 +166176,21 @@ static int isLimitTerm(WhereTerm *pTerm){ && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; } +/* +** Return true if the first nCons constraints in the pUsage array are +** marked as in-use (have argvIndex>0). False otherwise. +*/ +static int allConstraintsUsed( + struct sqlite3_index_constraint_usage *aUsage, + int nCons +){ + int ii; + for(ii=0; iipNew->iTab. This @@ -164899,13 +166331,20 @@ static int whereLoopAddVirtualOne( *pbIn = 1; assert( (mExclude & WO_IN)==0 ); } + /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET + ** terms. And if there are any, they should follow all other terms. */ assert( pbRetryLimit || !isLimitTerm(pTerm) ); - if( isLimitTerm(pTerm) && *pbIn ){ + assert( !isLimitTerm(pTerm) || i>=nConstraint-2 ); + assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) ); + + if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){ /* If there is an IN(...) term handled as an == (separate call to ** xFilter for each value on the RHS of the IN) and a LIMIT or - ** OFFSET term handled as well, the plan is unusable. Set output - ** variable *pbRetryLimit to true to tell the caller to retry with - ** LIMIT and OFFSET disabled. */ + ** OFFSET term handled as well, the plan is unusable. Similarly, + ** if there is a LIMIT/OFFSET and there are other unused terms, + ** the plan cannot be used. In these cases set variable *pbRetryLimit + ** to true to tell the caller to retry with LIMIT and OFFSET + ** disabled. */ if( pIdxInfo->needToFreeIdxStr ){ sqlite3_free(pIdxInfo->idxStr); pIdxInfo->idxStr = 0; @@ -165762,7 +167201,7 @@ static i8 wherePathSatisfiesOrderBy( if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } @@ -166231,10 +167670,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } - if( pWInfo->pSelect->pOrderBy - && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ - pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; - } + /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */ + assert( pWInfo->pSelect->pOrderBy==0 + || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr ); }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -166277,7 +167715,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ @@ -166285,6 +167722,83 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -166453,6 +167967,10 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** the right-most table of a subquery that was flattened into the ** main query and that subquery was the right-hand operand of an ** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. ** ** For example, given: ** @@ -166573,7 +168091,7 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; Table *pTab = pItem->pTab; if( (pTab->tabFlags & TF_HasStat1)==0 ) break; - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; if( i>=1 && (pLoop->wsFlags & reqFlags)==reqFlags /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ @@ -166594,6 +168112,58 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + /* ** The index pIdx is used by a query and contains one or more expressions. ** In other words pIdx is an index on an expression. iIdxCur is the cursor @@ -166619,33 +168189,19 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( for(i=0; inColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } - if( sqlite3ExprIsConstant(pExpr) ) continue; - if( pExpr->op==TK_FUNCTION ){ + if( sqlite3ExprIsConstant(0,pExpr) ) continue; + if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){ /* Functions that might set a subtype should not be replaced by the ** value taken from an expression index since the index omits the ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ - int n; - FuncDef *pDef; - sqlite3 *db = pParse->db; - assert( ExprUseXList(pExpr) ); - n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; - pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); - if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ - continue; - } + continue; } p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; @@ -166660,7 +168216,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } @@ -166828,6 +168384,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pOrderBy && pOrderBy->nExpr>=BMS ){ pOrderBy = 0; wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ } /* The number of tables in the FROM clause is limited by the number of @@ -166910,7 +168467,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } - ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + if( ALWAYS(pWInfo->pSelect) + && (pWInfo->pSelect->selFlags & SF_MultiValue)==0 + ){ + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + } }else{ /* Assign a bit from the bitmask to every term in the FROM clause. ** @@ -167063,6 +168624,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ + whereInterstageHeuristic(pWInfo); wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } @@ -167123,10 +168685,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. */ notReady = ~(Bitmask)0; - if( pWInfo->nLevel>=2 - && pResultSet!=0 /* these two combine to guarantee */ - && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ ){ notReady = whereOmitNoopJoin(pWInfo, notReady); nTabList = pWInfo->nLevel; @@ -167446,26 +169008,6 @@ whereBeginError: } #endif -#ifdef SQLITE_DEBUG -/* -** Return true if cursor iCur is opened by instruction k of the -** bytecode. Used inside of assert() only. -*/ -static int cursorIsOpen(Vdbe *v, int iCur, int k){ - while( k>=0 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); - if( pOp->p1!=iCur ) continue; - if( pOp->opcode==OP_Close ) return 0; - if( pOp->opcode==OP_OpenRead ) return 1; - if( pOp->opcode==OP_OpenWrite ) return 1; - if( pOp->opcode==OP_OpenDup ) return 1; - if( pOp->opcode==OP_OpenAutoindex ) return 1; - if( pOp->opcode==OP_OpenEphemeral ) return 1; - } - return 0; -} -#endif /* SQLITE_DEBUG */ - /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -167612,7 +169154,15 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); if( (ws & WHERE_IDX_ONLY)==0 ){ - assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + SrcItem *pSrc = &pTabList->a[pLevel->iFrom]; + assert( pLevel->iTabCur==pSrc->iCursor ); + if( pSrc->fg.viaCoroutine ){ + int m, n; + n = pSrc->regResult; + assert( pSrc->pTab!=0 ); + m = pSrc->pTab->nCol; + sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); + } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } if( (ws & WHERE_INDEXED) @@ -167662,6 +169212,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); + assert( pTabItem->regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, pTabItem->regResult, 0); continue; @@ -167756,16 +169307,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** reference. Verify that this is harmless - that the ** table being referenced really is open. */ -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - || pOp->opcode==OP_Offset - ); -#else - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - ); -#endif + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + } } }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; @@ -168966,7 +170511,7 @@ SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ ** variable values in the expression tree. */ static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ - if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( 0==sqlite3ExprIsConstant(0,pExpr) ){ if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); sqlite3ExprDelete(pParse->db, pExpr); pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); @@ -171036,9 +172581,9 @@ static void updateDeleteLimitError( break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } @@ -171058,6 +172603,14 @@ static void updateDeleteLimitError( return pSelect; } + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. + */ + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } + /* Construct a new Expr object from a single token */ static Expr *tokenExpr(Parse *pParse, int op, Token t){ @@ -171307,8 +172860,8 @@ static void updateDeleteLimitError( #define TK_TRUEFALSE 170 #define TK_ISNOT 171 #define TK_FUNCTION 172 -#define TK_UMINUS 173 -#define TK_UPLUS 174 +#define TK_UPLUS 173 +#define TK_UMINUS 174 #define TK_TRUTH 175 #define TK_REGISTER 176 #define TK_VECTOR 177 @@ -171317,8 +172870,9 @@ static void updateDeleteLimitError( #define TK_ASTERISK 180 #define TK_SPAN 181 #define TK_ERROR 182 -#define TK_SPACE 183 -#define TK_ILLEGAL 184 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 #endif /**************** End token definitions ***************************************/ @@ -171359,6 +172913,9 @@ static void updateDeleteLimitError( ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser ** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -171372,37 +172929,39 @@ static void updateDeleteLimitError( ** YY_NO_ACTION The yy_action[] code for no-op ** YY_MIN_REDUCE Minimum value for reduce actions ** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 319 +#define YYNOCODE 322 #define YYACTIONTYPE unsigned short int #define YYWILDCARD 101 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - TriggerStep* yy33; - Window* yy41; - Select* yy47; - SrcList* yy131; - struct TrigEvent yy180; - struct {int value; int mask;} yy231; - IdList* yy254; - u32 yy285; - ExprList* yy322; - Cte* yy385; - int yy394; - Upsert* yy444; - u8 yy516; - With* yy521; - const char* yy522; - Expr* yy528; - OnOrUsing yy561; - struct FrameBound yy595; + ExprList* yy14; + With* yy59; + Cte* yy67; + Upsert* yy122; + IdList* yy132; + int yy144; + const char* yy168; + SrcList* yy203; + Window* yy211; + OnOrUsing yy269; + struct TrigEvent yy286; + struct {int value; int mask;} yy383; + u32 yy391; + TriggerStep* yy427; + Expr* yy454; + u8 yy462; + struct FrameBound yy509; + Select* yy555; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -171412,24 +172971,29 @@ typedef union { #define sqlite3ParserARG_PARAM #define sqlite3ParserARG_FETCH #define sqlite3ParserARG_STORE +#define YYREALLOC parserStackRealloc +#define YYFREE sqlite3_free +#define YYDYNSTACK 1 #define sqlite3ParserCTX_SDECL Parse *pParse; #define sqlite3ParserCTX_PDECL ,Parse *pParse #define sqlite3ParserCTX_PARAM ,pParse #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 579 -#define YYNRULE 405 -#define YYNRULE_WITH_ACTION 340 -#define YYNTOKEN 185 -#define YY_MAX_SHIFT 578 -#define YY_MIN_SHIFTREDUCE 838 -#define YY_MAX_SHIFTREDUCE 1242 -#define YY_ERROR_ACTION 1243 -#define YY_ACCEPT_ACTION 1244 -#define YY_NO_ACTION 1245 -#define YY_MIN_REDUCE 1246 -#define YY_MAX_REDUCE 1650 +#define YYNSTATE 583 +#define YYNRULE 409 +#define YYNRULE_WITH_ACTION 344 +#define YYNTOKEN 186 +#define YY_MAX_SHIFT 582 +#define YY_MIN_SHIFTREDUCE 845 +#define YY_MAX_SHIFTREDUCE 1253 +#define YY_ERROR_ACTION 1254 +#define YY_ACCEPT_ACTION 1255 +#define YY_NO_ACTION 1256 +#define YY_MIN_REDUCE 1257 +#define YY_MAX_REDUCE 1665 +#define YY_MIN_DSTRCTR 205 +#define YY_MAX_DSTRCTR 319 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -171445,6 +173009,22 @@ typedef union { # define yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -171496,619 +173076,630 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2100) +#define YY_ACTTAB_COUNT (2142) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 572, 210, 572, 119, 116, 231, 572, 119, 116, 231, - /* 10 */ 572, 1317, 379, 1296, 410, 566, 566, 566, 572, 411, - /* 20 */ 380, 1317, 1279, 42, 42, 42, 42, 210, 1529, 72, - /* 30 */ 72, 974, 421, 42, 42, 495, 305, 281, 305, 975, - /* 40 */ 399, 72, 72, 126, 127, 81, 1217, 1217, 1054, 1057, - /* 50 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 480, 411, - /* 60 */ 1244, 1, 1, 578, 2, 1248, 554, 119, 116, 231, - /* 70 */ 319, 484, 147, 484, 528, 119, 116, 231, 533, 1330, - /* 80 */ 419, 527, 143, 126, 127, 81, 1217, 1217, 1054, 1057, - /* 90 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 119, 116, - /* 100 */ 231, 329, 123, 123, 123, 123, 122, 122, 121, 121, - /* 110 */ 121, 120, 117, 448, 286, 286, 286, 286, 446, 446, - /* 120 */ 446, 1568, 378, 1570, 1193, 377, 1164, 569, 1164, 569, - /* 130 */ 411, 1568, 541, 261, 228, 448, 102, 146, 453, 318, - /* 140 */ 563, 242, 123, 123, 123, 123, 122, 122, 121, 121, - /* 150 */ 121, 120, 117, 448, 126, 127, 81, 1217, 1217, 1054, - /* 160 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 143, - /* 170 */ 296, 1193, 341, 452, 121, 121, 121, 120, 117, 448, - /* 180 */ 128, 1193, 1194, 1193, 149, 445, 444, 572, 120, 117, - /* 190 */ 448, 125, 125, 125, 125, 118, 123, 123, 123, 123, - /* 200 */ 122, 122, 121, 121, 121, 120, 117, 448, 458, 114, - /* 210 */ 13, 13, 550, 123, 123, 123, 123, 122, 122, 121, - /* 220 */ 121, 121, 120, 117, 448, 424, 318, 563, 1193, 1194, - /* 230 */ 1193, 150, 1225, 411, 1225, 125, 125, 125, 125, 123, - /* 240 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, - /* 250 */ 448, 469, 344, 1041, 1041, 1055, 1058, 126, 127, 81, - /* 260 */ 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, - /* 270 */ 125, 125, 1282, 526, 224, 1193, 572, 411, 226, 519, - /* 280 */ 177, 83, 84, 123, 123, 123, 123, 122, 122, 121, - /* 290 */ 121, 121, 120, 117, 448, 1010, 16, 16, 1193, 134, - /* 300 */ 134, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, - /* 310 */ 124, 124, 125, 125, 125, 125, 123, 123, 123, 123, - /* 320 */ 122, 122, 121, 121, 121, 120, 117, 448, 1045, 550, - /* 330 */ 1193, 375, 1193, 1194, 1193, 254, 1438, 401, 508, 505, - /* 340 */ 504, 112, 564, 570, 4, 929, 929, 435, 503, 342, - /* 350 */ 464, 330, 362, 396, 1238, 1193, 1194, 1193, 567, 572, - /* 360 */ 123, 123, 123, 123, 122, 122, 121, 121, 121, 120, - /* 370 */ 117, 448, 286, 286, 371, 1581, 1607, 445, 444, 155, - /* 380 */ 411, 449, 72, 72, 1289, 569, 1222, 1193, 1194, 1193, - /* 390 */ 86, 1224, 273, 561, 547, 520, 520, 572, 99, 1223, - /* 400 */ 6, 1281, 476, 143, 126, 127, 81, 1217, 1217, 1054, - /* 410 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 554, - /* 420 */ 13, 13, 1031, 511, 1225, 1193, 1225, 553, 110, 110, - /* 430 */ 224, 572, 1239, 177, 572, 429, 111, 199, 449, 573, - /* 440 */ 449, 432, 1555, 1019, 327, 555, 1193, 272, 289, 370, - /* 450 */ 514, 365, 513, 259, 72, 72, 547, 72, 72, 361, - /* 460 */ 318, 563, 1613, 123, 123, 123, 123, 122, 122, 121, - /* 470 */ 121, 121, 120, 117, 448, 1019, 1019, 1021, 1022, 28, - /* 480 */ 286, 286, 1193, 1194, 1193, 1159, 572, 1612, 411, 904, - /* 490 */ 192, 554, 358, 569, 554, 940, 537, 521, 1159, 437, - /* 500 */ 415, 1159, 556, 1193, 1194, 1193, 572, 548, 548, 52, - /* 510 */ 52, 216, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, - /* 520 */ 1044, 124, 124, 125, 125, 125, 125, 1193, 478, 136, - /* 530 */ 136, 411, 286, 286, 1493, 509, 122, 122, 121, 121, - /* 540 */ 121, 120, 117, 448, 1010, 569, 522, 219, 545, 545, - /* 550 */ 318, 563, 143, 6, 536, 126, 127, 81, 1217, 1217, - /* 560 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 570 */ 1557, 123, 123, 123, 123, 122, 122, 121, 121, 121, - /* 580 */ 120, 117, 448, 489, 1193, 1194, 1193, 486, 283, 1270, - /* 590 */ 960, 254, 1193, 375, 508, 505, 504, 1193, 342, 574, - /* 600 */ 1193, 574, 411, 294, 503, 960, 879, 193, 484, 318, - /* 610 */ 563, 386, 292, 382, 123, 123, 123, 123, 122, 122, - /* 620 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 630 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 640 */ 125, 411, 396, 1139, 1193, 872, 101, 286, 286, 1193, - /* 650 */ 1194, 1193, 375, 1096, 1193, 1194, 1193, 1193, 1194, 1193, - /* 660 */ 569, 459, 33, 375, 235, 126, 127, 81, 1217, 1217, - /* 670 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 680 */ 1437, 962, 572, 230, 961, 123, 123, 123, 123, 122, - /* 690 */ 122, 121, 121, 121, 120, 117, 448, 1159, 230, 1193, - /* 700 */ 158, 1193, 1194, 1193, 1556, 13, 13, 303, 960, 1233, - /* 710 */ 1159, 154, 411, 1159, 375, 1584, 1177, 5, 371, 1581, - /* 720 */ 431, 1239, 3, 960, 123, 123, 123, 123, 122, 122, - /* 730 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 740 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 750 */ 125, 411, 210, 571, 1193, 1032, 1193, 1194, 1193, 1193, - /* 760 */ 390, 855, 156, 1555, 376, 404, 1101, 1101, 492, 572, - /* 770 */ 469, 344, 1322, 1322, 1555, 126, 127, 81, 1217, 1217, - /* 780 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 790 */ 130, 572, 13, 13, 532, 123, 123, 123, 123, 122, - /* 800 */ 122, 121, 121, 121, 120, 117, 448, 304, 572, 457, - /* 810 */ 229, 1193, 1194, 1193, 13, 13, 1193, 1194, 1193, 1300, - /* 820 */ 467, 1270, 411, 1320, 1320, 1555, 1015, 457, 456, 436, - /* 830 */ 301, 72, 72, 1268, 123, 123, 123, 123, 122, 122, - /* 840 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 850 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 860 */ 125, 411, 384, 1076, 1159, 286, 286, 421, 314, 280, - /* 870 */ 280, 287, 287, 461, 408, 407, 1539, 1159, 569, 572, - /* 880 */ 1159, 1196, 569, 409, 569, 126, 127, 81, 1217, 1217, - /* 890 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 900 */ 457, 1485, 13, 13, 1541, 123, 123, 123, 123, 122, - /* 910 */ 122, 121, 121, 121, 120, 117, 448, 202, 572, 462, - /* 920 */ 1587, 578, 2, 1248, 843, 844, 845, 1563, 319, 409, - /* 930 */ 147, 6, 411, 257, 256, 255, 208, 1330, 9, 1196, - /* 940 */ 264, 72, 72, 1436, 123, 123, 123, 123, 122, 122, - /* 950 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 960 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 970 */ 125, 572, 286, 286, 572, 1213, 411, 577, 315, 1248, - /* 980 */ 421, 371, 1581, 356, 319, 569, 147, 495, 529, 1644, - /* 990 */ 397, 935, 495, 1330, 71, 71, 934, 72, 72, 242, - /* 1000 */ 1328, 105, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, - /* 1010 */ 124, 125, 125, 125, 125, 123, 123, 123, 123, 122, - /* 1020 */ 122, 121, 121, 121, 120, 117, 448, 1117, 286, 286, - /* 1030 */ 1422, 452, 1528, 1213, 443, 286, 286, 1492, 1355, 313, - /* 1040 */ 478, 569, 1118, 454, 351, 495, 354, 1266, 569, 209, - /* 1050 */ 572, 418, 179, 572, 1031, 242, 385, 1119, 523, 123, - /* 1060 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, - /* 1070 */ 448, 1020, 108, 72, 72, 1019, 13, 13, 915, 572, - /* 1080 */ 1498, 572, 286, 286, 98, 530, 1537, 452, 916, 1334, - /* 1090 */ 1329, 203, 411, 286, 286, 569, 152, 211, 1498, 1500, - /* 1100 */ 426, 569, 56, 56, 57, 57, 569, 1019, 1019, 1021, - /* 1110 */ 447, 572, 411, 531, 12, 297, 126, 127, 81, 1217, - /* 1120 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1130 */ 125, 572, 411, 867, 15, 15, 126, 127, 81, 1217, - /* 1140 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1150 */ 125, 373, 529, 264, 44, 44, 126, 115, 81, 1217, - /* 1160 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1170 */ 125, 1498, 478, 1271, 417, 123, 123, 123, 123, 122, - /* 1180 */ 122, 121, 121, 121, 120, 117, 448, 205, 1213, 495, - /* 1190 */ 430, 867, 468, 322, 495, 123, 123, 123, 123, 122, - /* 1200 */ 122, 121, 121, 121, 120, 117, 448, 572, 557, 1140, - /* 1210 */ 1642, 1422, 1642, 543, 572, 123, 123, 123, 123, 122, - /* 1220 */ 122, 121, 121, 121, 120, 117, 448, 572, 1422, 572, - /* 1230 */ 13, 13, 542, 323, 1325, 411, 334, 58, 58, 349, - /* 1240 */ 1422, 1170, 326, 286, 286, 549, 1213, 300, 895, 530, - /* 1250 */ 45, 45, 59, 59, 1140, 1643, 569, 1643, 565, 417, - /* 1260 */ 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, - /* 1270 */ 125, 125, 125, 125, 1367, 373, 500, 290, 1193, 512, - /* 1280 */ 1366, 427, 394, 394, 393, 275, 391, 896, 1138, 852, - /* 1290 */ 478, 258, 1422, 1170, 463, 1159, 12, 331, 428, 333, - /* 1300 */ 1117, 460, 236, 258, 325, 460, 544, 1544, 1159, 1098, - /* 1310 */ 491, 1159, 324, 1098, 440, 1118, 335, 516, 123, 123, - /* 1320 */ 123, 123, 122, 122, 121, 121, 121, 120, 117, 448, - /* 1330 */ 1119, 318, 563, 1138, 572, 1193, 1194, 1193, 112, 564, - /* 1340 */ 201, 4, 238, 433, 935, 490, 285, 228, 1517, 934, - /* 1350 */ 170, 560, 572, 142, 1516, 567, 572, 60, 60, 572, - /* 1360 */ 416, 572, 441, 572, 535, 302, 875, 8, 487, 572, - /* 1370 */ 237, 572, 416, 572, 485, 61, 61, 572, 449, 62, - /* 1380 */ 62, 332, 63, 63, 46, 46, 47, 47, 361, 572, - /* 1390 */ 561, 572, 48, 48, 50, 50, 51, 51, 572, 295, - /* 1400 */ 64, 64, 482, 295, 539, 412, 471, 1031, 572, 538, - /* 1410 */ 318, 563, 65, 65, 66, 66, 409, 475, 572, 1031, - /* 1420 */ 572, 14, 14, 875, 1020, 110, 110, 409, 1019, 572, - /* 1430 */ 474, 67, 67, 111, 455, 449, 573, 449, 98, 317, - /* 1440 */ 1019, 132, 132, 133, 133, 572, 1561, 572, 974, 409, - /* 1450 */ 6, 1562, 68, 68, 1560, 6, 975, 572, 6, 1559, - /* 1460 */ 1019, 1019, 1021, 6, 346, 218, 101, 531, 53, 53, - /* 1470 */ 69, 69, 1019, 1019, 1021, 1022, 28, 1586, 1181, 451, - /* 1480 */ 70, 70, 290, 87, 215, 31, 1363, 394, 394, 393, - /* 1490 */ 275, 391, 350, 109, 852, 107, 572, 112, 564, 483, - /* 1500 */ 4, 1212, 572, 239, 153, 572, 39, 236, 1299, 325, - /* 1510 */ 112, 564, 1298, 4, 567, 572, 32, 324, 572, 54, - /* 1520 */ 54, 572, 1135, 353, 398, 165, 165, 567, 166, 166, - /* 1530 */ 572, 291, 355, 572, 17, 357, 572, 449, 77, 77, - /* 1540 */ 1313, 55, 55, 1297, 73, 73, 572, 238, 470, 561, - /* 1550 */ 449, 472, 364, 135, 135, 170, 74, 74, 142, 163, - /* 1560 */ 163, 374, 561, 539, 572, 321, 572, 886, 540, 137, - /* 1570 */ 137, 339, 1353, 422, 298, 237, 539, 572, 1031, 572, - /* 1580 */ 340, 538, 101, 369, 110, 110, 162, 131, 131, 164, - /* 1590 */ 164, 1031, 111, 368, 449, 573, 449, 110, 110, 1019, - /* 1600 */ 157, 157, 141, 141, 572, 111, 572, 449, 573, 449, - /* 1610 */ 412, 288, 1019, 572, 882, 318, 563, 572, 219, 572, - /* 1620 */ 241, 1012, 477, 263, 263, 894, 893, 140, 140, 138, - /* 1630 */ 138, 1019, 1019, 1021, 1022, 28, 139, 139, 525, 455, - /* 1640 */ 76, 76, 78, 78, 1019, 1019, 1021, 1022, 28, 1181, - /* 1650 */ 451, 572, 1083, 290, 112, 564, 1575, 4, 394, 394, - /* 1660 */ 393, 275, 391, 572, 1023, 852, 572, 479, 345, 263, - /* 1670 */ 101, 567, 882, 1376, 75, 75, 1421, 501, 236, 260, - /* 1680 */ 325, 112, 564, 359, 4, 101, 43, 43, 324, 49, - /* 1690 */ 49, 901, 902, 161, 449, 101, 977, 978, 567, 1079, - /* 1700 */ 1349, 260, 965, 932, 263, 114, 561, 1095, 517, 1095, - /* 1710 */ 1083, 1094, 865, 1094, 151, 933, 1144, 114, 238, 1361, - /* 1720 */ 558, 449, 1023, 559, 1426, 1278, 170, 1269, 1257, 142, - /* 1730 */ 1601, 1256, 1258, 561, 1594, 1031, 496, 278, 213, 1346, - /* 1740 */ 310, 110, 110, 939, 311, 312, 237, 11, 234, 111, - /* 1750 */ 221, 449, 573, 449, 293, 395, 1019, 1408, 337, 1403, - /* 1760 */ 1396, 338, 1031, 299, 343, 1413, 1412, 481, 110, 110, - /* 1770 */ 506, 402, 225, 1296, 206, 367, 111, 1358, 449, 573, - /* 1780 */ 449, 412, 1359, 1019, 1489, 1488, 318, 563, 1019, 1019, - /* 1790 */ 1021, 1022, 28, 562, 207, 220, 80, 564, 389, 4, - /* 1800 */ 1597, 1357, 552, 1356, 1233, 181, 267, 232, 1536, 1534, - /* 1810 */ 455, 1230, 420, 567, 82, 1019, 1019, 1021, 1022, 28, - /* 1820 */ 86, 217, 85, 1494, 190, 175, 183, 465, 185, 466, - /* 1830 */ 36, 1409, 186, 187, 188, 499, 449, 244, 37, 99, - /* 1840 */ 400, 1415, 1414, 488, 1417, 194, 473, 403, 561, 1483, - /* 1850 */ 248, 92, 1505, 494, 198, 279, 112, 564, 250, 4, - /* 1860 */ 348, 497, 405, 352, 1259, 251, 252, 515, 1316, 434, - /* 1870 */ 1315, 1314, 94, 567, 1307, 886, 1306, 1031, 226, 406, - /* 1880 */ 1611, 1610, 438, 110, 110, 1580, 1286, 524, 439, 308, - /* 1890 */ 266, 111, 1285, 449, 573, 449, 449, 309, 1019, 366, - /* 1900 */ 1284, 1609, 265, 1566, 1565, 442, 372, 1381, 561, 129, - /* 1910 */ 550, 1380, 10, 1470, 383, 106, 316, 551, 100, 35, - /* 1920 */ 534, 575, 212, 1339, 381, 387, 1187, 1338, 274, 276, - /* 1930 */ 1019, 1019, 1021, 1022, 28, 277, 413, 1031, 576, 1254, - /* 1940 */ 388, 1521, 1249, 110, 110, 167, 1522, 168, 148, 1520, - /* 1950 */ 1519, 111, 306, 449, 573, 449, 222, 223, 1019, 839, - /* 1960 */ 169, 79, 450, 214, 414, 233, 320, 145, 1093, 1091, - /* 1970 */ 328, 182, 171, 1212, 918, 184, 240, 336, 243, 1107, - /* 1980 */ 189, 172, 173, 423, 425, 88, 180, 191, 89, 90, - /* 1990 */ 1019, 1019, 1021, 1022, 28, 91, 174, 1110, 245, 1106, - /* 2000 */ 246, 159, 18, 247, 347, 1099, 263, 195, 1227, 493, - /* 2010 */ 249, 196, 38, 854, 498, 368, 253, 360, 897, 197, - /* 2020 */ 502, 93, 19, 20, 507, 884, 363, 510, 95, 307, - /* 2030 */ 160, 96, 518, 97, 1175, 1060, 1146, 40, 21, 227, - /* 2040 */ 176, 1145, 282, 284, 969, 200, 963, 114, 262, 1165, - /* 2050 */ 22, 23, 24, 1161, 1169, 25, 1163, 1150, 34, 26, - /* 2060 */ 1168, 546, 27, 204, 101, 103, 104, 1074, 7, 1061, - /* 2070 */ 1059, 1063, 1116, 1064, 1115, 268, 269, 29, 41, 270, - /* 2080 */ 1024, 866, 113, 30, 568, 392, 1183, 144, 178, 1182, - /* 2090 */ 271, 928, 1245, 1245, 1245, 1245, 1245, 1245, 1245, 1602, + /* 0 */ 576, 128, 125, 232, 1622, 549, 576, 1290, 1281, 576, + /* 10 */ 328, 576, 1300, 212, 576, 128, 125, 232, 578, 412, + /* 20 */ 578, 391, 1542, 51, 51, 523, 405, 1293, 529, 51, + /* 30 */ 51, 983, 51, 51, 81, 81, 1107, 61, 61, 984, + /* 40 */ 1107, 1292, 380, 135, 136, 90, 1228, 1228, 1063, 1066, + /* 50 */ 1053, 1053, 133, 133, 134, 134, 134, 134, 1577, 412, + /* 60 */ 287, 287, 7, 287, 287, 422, 1050, 1050, 1064, 1067, + /* 70 */ 289, 556, 492, 573, 524, 561, 573, 497, 561, 482, + /* 80 */ 530, 262, 229, 135, 136, 90, 1228, 1228, 1063, 1066, + /* 90 */ 1053, 1053, 133, 133, 134, 134, 134, 134, 128, 125, + /* 100 */ 232, 1506, 132, 132, 132, 132, 131, 131, 130, 130, + /* 110 */ 130, 129, 126, 450, 1204, 1255, 1, 1, 582, 2, + /* 120 */ 1259, 1571, 420, 1582, 379, 320, 1174, 153, 1174, 1584, + /* 130 */ 412, 378, 1582, 543, 1341, 330, 111, 570, 570, 570, + /* 140 */ 293, 1054, 132, 132, 132, 132, 131, 131, 130, 130, + /* 150 */ 130, 129, 126, 450, 135, 136, 90, 1228, 1228, 1063, + /* 160 */ 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, 287, + /* 170 */ 287, 1204, 1205, 1204, 255, 287, 287, 510, 507, 506, + /* 180 */ 137, 455, 573, 212, 561, 447, 446, 505, 573, 1616, + /* 190 */ 561, 134, 134, 134, 134, 127, 400, 243, 132, 132, + /* 200 */ 132, 132, 131, 131, 130, 130, 130, 129, 126, 450, + /* 210 */ 282, 471, 345, 132, 132, 132, 132, 131, 131, 130, + /* 220 */ 130, 130, 129, 126, 450, 574, 155, 936, 936, 454, + /* 230 */ 227, 521, 1236, 412, 1236, 134, 134, 134, 134, 132, + /* 240 */ 132, 132, 132, 131, 131, 130, 130, 130, 129, 126, + /* 250 */ 450, 130, 130, 130, 129, 126, 450, 135, 136, 90, + /* 260 */ 1228, 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, + /* 270 */ 134, 134, 128, 125, 232, 450, 576, 412, 397, 1249, + /* 280 */ 180, 92, 93, 132, 132, 132, 132, 131, 131, 130, + /* 290 */ 130, 130, 129, 126, 450, 381, 387, 1204, 383, 81, + /* 300 */ 81, 135, 136, 90, 1228, 1228, 1063, 1066, 1053, 1053, + /* 310 */ 133, 133, 134, 134, 134, 134, 132, 132, 132, 132, + /* 320 */ 131, 131, 130, 130, 130, 129, 126, 450, 131, 131, + /* 330 */ 130, 130, 130, 129, 126, 450, 556, 1204, 302, 319, + /* 340 */ 567, 121, 568, 480, 4, 555, 1149, 1657, 1628, 1657, + /* 350 */ 45, 128, 125, 232, 1204, 1205, 1204, 1250, 571, 1169, + /* 360 */ 132, 132, 132, 132, 131, 131, 130, 130, 130, 129, + /* 370 */ 126, 450, 1169, 287, 287, 1169, 1019, 576, 422, 1019, + /* 380 */ 412, 451, 1602, 582, 2, 1259, 573, 44, 561, 95, + /* 390 */ 320, 110, 153, 565, 1204, 1205, 1204, 522, 522, 1341, + /* 400 */ 81, 81, 7, 44, 135, 136, 90, 1228, 1228, 1063, + /* 410 */ 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, 295, + /* 420 */ 1149, 1658, 1040, 1658, 1204, 1147, 319, 567, 119, 119, + /* 430 */ 343, 466, 331, 343, 287, 287, 120, 556, 451, 577, + /* 440 */ 451, 1169, 1169, 1028, 319, 567, 438, 573, 210, 561, + /* 450 */ 1339, 1451, 546, 531, 1169, 1169, 1598, 1169, 1169, 416, + /* 460 */ 319, 567, 243, 132, 132, 132, 132, 131, 131, 130, + /* 470 */ 130, 130, 129, 126, 450, 1028, 1028, 1030, 1031, 35, + /* 480 */ 44, 1204, 1205, 1204, 472, 287, 287, 1328, 412, 1307, + /* 490 */ 372, 1595, 359, 225, 454, 1204, 195, 1328, 573, 1147, + /* 500 */ 561, 1333, 1333, 274, 576, 1188, 576, 340, 46, 196, + /* 510 */ 537, 217, 135, 136, 90, 1228, 1228, 1063, 1066, 1053, + /* 520 */ 1053, 133, 133, 134, 134, 134, 134, 19, 19, 19, + /* 530 */ 19, 412, 581, 1204, 1259, 511, 1204, 319, 567, 320, + /* 540 */ 944, 153, 425, 491, 430, 943, 1204, 488, 1341, 1450, + /* 550 */ 532, 1277, 1204, 1205, 1204, 135, 136, 90, 1228, 1228, + /* 560 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 570 */ 575, 132, 132, 132, 132, 131, 131, 130, 130, 130, + /* 580 */ 129, 126, 450, 287, 287, 528, 287, 287, 372, 1595, + /* 590 */ 1204, 1205, 1204, 1204, 1205, 1204, 573, 486, 561, 573, + /* 600 */ 889, 561, 412, 1204, 1205, 1204, 886, 40, 22, 22, + /* 610 */ 220, 243, 525, 1449, 132, 132, 132, 132, 131, 131, + /* 620 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 630 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 640 */ 134, 412, 180, 454, 1204, 879, 255, 287, 287, 510, + /* 650 */ 507, 506, 372, 1595, 1568, 1331, 1331, 576, 889, 505, + /* 660 */ 573, 44, 561, 559, 1207, 135, 136, 90, 1228, 1228, + /* 670 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 680 */ 81, 81, 422, 576, 377, 132, 132, 132, 132, 131, + /* 690 */ 131, 130, 130, 130, 129, 126, 450, 297, 287, 287, + /* 700 */ 460, 1204, 1205, 1204, 1204, 534, 19, 19, 448, 448, + /* 710 */ 448, 573, 412, 561, 230, 436, 1187, 535, 319, 567, + /* 720 */ 363, 432, 1207, 1435, 132, 132, 132, 132, 131, 131, + /* 730 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 740 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 750 */ 134, 412, 211, 949, 1169, 1041, 1110, 1110, 494, 547, + /* 760 */ 547, 1204, 1205, 1204, 7, 539, 1570, 1169, 376, 576, + /* 770 */ 1169, 5, 1204, 486, 3, 135, 136, 90, 1228, 1228, + /* 780 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 790 */ 576, 513, 19, 19, 427, 132, 132, 132, 132, 131, + /* 800 */ 131, 130, 130, 130, 129, 126, 450, 305, 1204, 433, + /* 810 */ 225, 1204, 385, 19, 19, 273, 290, 371, 516, 366, + /* 820 */ 515, 260, 412, 538, 1568, 549, 1024, 362, 437, 1204, + /* 830 */ 1205, 1204, 902, 1552, 132, 132, 132, 132, 131, 131, + /* 840 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 850 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 860 */ 134, 412, 1435, 514, 1281, 1204, 1205, 1204, 1204, 1205, + /* 870 */ 1204, 903, 48, 342, 1568, 1568, 1279, 1627, 1568, 911, + /* 880 */ 576, 129, 126, 450, 110, 135, 136, 90, 1228, 1228, + /* 890 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 900 */ 265, 576, 459, 19, 19, 132, 132, 132, 132, 131, + /* 910 */ 131, 130, 130, 130, 129, 126, 450, 1345, 204, 576, + /* 920 */ 459, 458, 50, 47, 19, 19, 49, 434, 1105, 573, + /* 930 */ 497, 561, 412, 428, 108, 1224, 1569, 1554, 376, 205, + /* 940 */ 550, 550, 81, 81, 132, 132, 132, 132, 131, 131, + /* 950 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 960 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 970 */ 134, 480, 576, 1204, 576, 1541, 412, 1435, 969, 315, + /* 980 */ 1659, 398, 284, 497, 969, 893, 1569, 1569, 376, 376, + /* 990 */ 1569, 461, 376, 1224, 459, 80, 80, 81, 81, 497, + /* 1000 */ 374, 114, 90, 1228, 1228, 1063, 1066, 1053, 1053, 133, + /* 1010 */ 133, 134, 134, 134, 134, 132, 132, 132, 132, 131, + /* 1020 */ 131, 130, 130, 130, 129, 126, 450, 1204, 1505, 576, + /* 1030 */ 1204, 1205, 1204, 1366, 316, 486, 281, 281, 497, 431, + /* 1040 */ 557, 288, 288, 402, 1340, 471, 345, 298, 429, 573, + /* 1050 */ 576, 561, 81, 81, 573, 374, 561, 971, 386, 132, + /* 1060 */ 132, 132, 132, 131, 131, 130, 130, 130, 129, 126, + /* 1070 */ 450, 231, 117, 81, 81, 287, 287, 231, 287, 287, + /* 1080 */ 576, 1511, 576, 1336, 1204, 1205, 1204, 139, 573, 556, + /* 1090 */ 561, 573, 412, 561, 441, 456, 969, 213, 558, 1511, + /* 1100 */ 1513, 1550, 969, 143, 143, 145, 145, 1368, 314, 478, + /* 1110 */ 444, 970, 412, 850, 851, 852, 135, 136, 90, 1228, + /* 1120 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1130 */ 134, 357, 412, 397, 1148, 304, 135, 136, 90, 1228, + /* 1140 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1150 */ 134, 1575, 323, 6, 862, 7, 135, 124, 90, 1228, + /* 1160 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1170 */ 134, 409, 408, 1511, 212, 132, 132, 132, 132, 131, + /* 1180 */ 131, 130, 130, 130, 129, 126, 450, 411, 118, 1204, + /* 1190 */ 116, 10, 352, 265, 355, 132, 132, 132, 132, 131, + /* 1200 */ 131, 130, 130, 130, 129, 126, 450, 576, 324, 306, + /* 1210 */ 576, 306, 1250, 469, 158, 132, 132, 132, 132, 131, + /* 1220 */ 131, 130, 130, 130, 129, 126, 450, 207, 1224, 1126, + /* 1230 */ 65, 65, 470, 66, 66, 412, 447, 446, 882, 531, + /* 1240 */ 335, 258, 257, 256, 1127, 1233, 1204, 1205, 1204, 327, + /* 1250 */ 1235, 874, 159, 576, 16, 480, 1085, 1040, 1234, 1128, + /* 1260 */ 136, 90, 1228, 1228, 1063, 1066, 1053, 1053, 133, 133, + /* 1270 */ 134, 134, 134, 134, 1029, 576, 81, 81, 1028, 1040, + /* 1280 */ 922, 576, 463, 1236, 576, 1236, 1224, 502, 107, 1435, + /* 1290 */ 923, 6, 576, 410, 1498, 882, 1029, 480, 21, 21, + /* 1300 */ 1028, 332, 1380, 334, 53, 53, 497, 81, 81, 874, + /* 1310 */ 1028, 1028, 1030, 445, 259, 19, 19, 533, 132, 132, + /* 1320 */ 132, 132, 131, 131, 130, 130, 130, 129, 126, 450, + /* 1330 */ 551, 301, 1028, 1028, 1030, 107, 532, 545, 121, 568, + /* 1340 */ 1188, 4, 1126, 1576, 449, 576, 462, 7, 1282, 418, + /* 1350 */ 462, 350, 1435, 576, 518, 571, 544, 1127, 121, 568, + /* 1360 */ 442, 4, 1188, 464, 533, 1180, 1223, 9, 67, 67, + /* 1370 */ 487, 576, 1128, 303, 410, 571, 54, 54, 451, 576, + /* 1380 */ 123, 944, 576, 417, 576, 333, 943, 1379, 576, 236, + /* 1390 */ 565, 576, 1574, 564, 68, 68, 7, 576, 451, 362, + /* 1400 */ 419, 182, 69, 69, 541, 70, 70, 71, 71, 540, + /* 1410 */ 565, 72, 72, 484, 55, 55, 473, 1180, 296, 1040, + /* 1420 */ 56, 56, 296, 493, 541, 119, 119, 410, 1573, 542, + /* 1430 */ 569, 418, 7, 120, 1244, 451, 577, 451, 465, 1040, + /* 1440 */ 1028, 576, 1557, 552, 476, 119, 119, 527, 259, 121, + /* 1450 */ 568, 240, 4, 120, 576, 451, 577, 451, 576, 477, + /* 1460 */ 1028, 576, 156, 576, 57, 57, 571, 576, 286, 229, + /* 1470 */ 410, 336, 1028, 1028, 1030, 1031, 35, 59, 59, 219, + /* 1480 */ 983, 60, 60, 220, 73, 73, 74, 74, 984, 451, + /* 1490 */ 75, 75, 1028, 1028, 1030, 1031, 35, 96, 216, 291, + /* 1500 */ 552, 565, 1188, 318, 395, 395, 394, 276, 392, 576, + /* 1510 */ 485, 859, 474, 1311, 410, 541, 576, 417, 1530, 1144, + /* 1520 */ 540, 399, 1188, 292, 237, 1153, 326, 38, 23, 576, + /* 1530 */ 1040, 576, 20, 20, 325, 299, 119, 119, 164, 76, + /* 1540 */ 76, 1529, 121, 568, 120, 4, 451, 577, 451, 203, + /* 1550 */ 576, 1028, 141, 141, 142, 142, 576, 322, 39, 571, + /* 1560 */ 341, 1021, 110, 264, 239, 901, 900, 423, 242, 908, + /* 1570 */ 909, 370, 173, 77, 77, 43, 479, 1310, 264, 62, + /* 1580 */ 62, 369, 451, 1028, 1028, 1030, 1031, 35, 1601, 1192, + /* 1590 */ 453, 1092, 238, 291, 565, 163, 1309, 110, 395, 395, + /* 1600 */ 394, 276, 392, 986, 987, 859, 481, 346, 264, 110, + /* 1610 */ 1032, 489, 576, 1188, 503, 1088, 261, 261, 237, 576, + /* 1620 */ 326, 121, 568, 1040, 4, 347, 1376, 413, 325, 119, + /* 1630 */ 119, 948, 319, 567, 351, 78, 78, 120, 571, 451, + /* 1640 */ 577, 451, 79, 79, 1028, 354, 356, 576, 360, 1092, + /* 1650 */ 110, 576, 974, 942, 264, 123, 457, 358, 239, 576, + /* 1660 */ 519, 451, 939, 1104, 123, 1104, 173, 576, 1032, 43, + /* 1670 */ 63, 63, 1324, 565, 168, 168, 1028, 1028, 1030, 1031, + /* 1680 */ 35, 576, 169, 169, 1308, 872, 238, 157, 1589, 576, + /* 1690 */ 86, 86, 365, 89, 568, 375, 4, 1103, 941, 1103, + /* 1700 */ 123, 576, 1040, 1389, 64, 64, 1188, 1434, 119, 119, + /* 1710 */ 571, 576, 82, 82, 563, 576, 120, 165, 451, 577, + /* 1720 */ 451, 413, 1362, 1028, 144, 144, 319, 567, 576, 1374, + /* 1730 */ 562, 498, 279, 451, 83, 83, 1439, 576, 166, 166, + /* 1740 */ 576, 1289, 554, 576, 1280, 565, 576, 12, 576, 1268, + /* 1750 */ 457, 146, 146, 1267, 576, 1028, 1028, 1030, 1031, 35, + /* 1760 */ 140, 140, 1269, 167, 167, 1609, 160, 160, 1359, 150, + /* 1770 */ 150, 149, 149, 311, 1040, 576, 312, 147, 147, 313, + /* 1780 */ 119, 119, 222, 235, 576, 1188, 396, 576, 120, 576, + /* 1790 */ 451, 577, 451, 1192, 453, 1028, 508, 291, 148, 148, + /* 1800 */ 1421, 1612, 395, 395, 394, 276, 392, 85, 85, 859, + /* 1810 */ 87, 87, 84, 84, 553, 576, 294, 576, 1426, 338, + /* 1820 */ 339, 1425, 237, 300, 326, 1416, 1409, 1028, 1028, 1030, + /* 1830 */ 1031, 35, 325, 344, 403, 483, 226, 1307, 52, 52, + /* 1840 */ 58, 58, 368, 1371, 1502, 566, 1501, 121, 568, 221, + /* 1850 */ 4, 208, 268, 209, 390, 1244, 1549, 1188, 1372, 1370, + /* 1860 */ 1369, 1547, 239, 184, 571, 233, 421, 1241, 95, 218, + /* 1870 */ 173, 1507, 193, 43, 91, 94, 178, 186, 467, 188, + /* 1880 */ 468, 1422, 13, 189, 190, 191, 501, 451, 245, 108, + /* 1890 */ 238, 401, 1428, 1427, 1430, 475, 404, 1496, 197, 565, + /* 1900 */ 14, 490, 249, 101, 1518, 496, 349, 280, 251, 201, + /* 1910 */ 353, 499, 252, 406, 1270, 253, 517, 1327, 1326, 435, + /* 1920 */ 1325, 1318, 103, 893, 1296, 413, 227, 407, 1040, 1626, + /* 1930 */ 319, 567, 1625, 1297, 119, 119, 439, 367, 1317, 1295, + /* 1940 */ 1624, 526, 120, 440, 451, 577, 451, 1594, 309, 1028, + /* 1950 */ 310, 373, 266, 267, 457, 1580, 1579, 443, 138, 1394, + /* 1960 */ 552, 1393, 11, 1483, 384, 115, 317, 1350, 109, 536, + /* 1970 */ 42, 579, 382, 214, 1349, 388, 1198, 389, 275, 277, + /* 1980 */ 278, 1028, 1028, 1030, 1031, 35, 580, 1265, 414, 1260, + /* 1990 */ 170, 415, 183, 1534, 1535, 1533, 171, 154, 307, 1532, + /* 2000 */ 846, 223, 224, 88, 452, 215, 172, 321, 234, 1102, + /* 2010 */ 152, 1188, 1100, 329, 185, 174, 1223, 925, 187, 241, + /* 2020 */ 337, 244, 1116, 192, 175, 176, 424, 426, 97, 194, + /* 2030 */ 98, 99, 100, 177, 1119, 1115, 246, 247, 161, 24, + /* 2040 */ 248, 348, 1238, 264, 1108, 250, 495, 199, 198, 15, + /* 2050 */ 861, 500, 369, 254, 504, 509, 512, 200, 102, 25, + /* 2060 */ 179, 361, 26, 364, 104, 891, 308, 162, 105, 904, + /* 2070 */ 520, 106, 1185, 1069, 1155, 17, 228, 27, 1154, 283, + /* 2080 */ 285, 263, 978, 202, 972, 123, 28, 1175, 29, 30, + /* 2090 */ 1179, 1171, 31, 1173, 1160, 41, 32, 206, 548, 33, + /* 2100 */ 110, 1178, 1083, 8, 112, 1070, 113, 1068, 1072, 34, + /* 2110 */ 1073, 560, 1125, 269, 1124, 270, 36, 18, 1194, 1033, + /* 2120 */ 873, 151, 122, 37, 393, 271, 272, 572, 181, 1193, + /* 2130 */ 1256, 1256, 1256, 935, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2140 */ 1256, 1617, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 193, 193, 193, 274, 275, 276, 193, 274, 275, 276, - /* 10 */ 193, 223, 219, 225, 206, 210, 211, 212, 193, 19, - /* 20 */ 219, 233, 216, 216, 217, 216, 217, 193, 295, 216, - /* 30 */ 217, 31, 193, 216, 217, 193, 228, 213, 230, 39, - /* 40 */ 206, 216, 217, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 193, 19, - /* 60 */ 185, 186, 187, 188, 189, 190, 253, 274, 275, 276, - /* 70 */ 195, 193, 197, 193, 261, 274, 275, 276, 253, 204, - /* 80 */ 238, 204, 81, 43, 44, 45, 46, 47, 48, 49, - /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 274, 275, - /* 100 */ 276, 262, 102, 103, 104, 105, 106, 107, 108, 109, - /* 110 */ 110, 111, 112, 113, 239, 240, 239, 240, 210, 211, - /* 120 */ 212, 314, 315, 314, 59, 316, 86, 252, 88, 252, - /* 130 */ 19, 314, 315, 256, 257, 113, 25, 72, 296, 138, - /* 140 */ 139, 266, 102, 103, 104, 105, 106, 107, 108, 109, + /* 0 */ 194, 276, 277, 278, 216, 194, 194, 217, 194, 194, + /* 10 */ 194, 194, 224, 194, 194, 276, 277, 278, 204, 19, + /* 20 */ 206, 202, 297, 217, 218, 205, 207, 217, 205, 217, + /* 30 */ 218, 31, 217, 218, 217, 218, 29, 217, 218, 39, + /* 40 */ 33, 217, 220, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 312, 19, + /* 60 */ 240, 241, 316, 240, 241, 194, 46, 47, 48, 49, + /* 70 */ 22, 254, 65, 253, 254, 255, 253, 194, 255, 194, + /* 80 */ 263, 258, 259, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 276, 277, + /* 100 */ 278, 285, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 113, 59, 186, 187, 188, 189, 190, + /* 120 */ 191, 310, 239, 317, 318, 196, 86, 198, 88, 317, + /* 130 */ 19, 319, 317, 318, 205, 264, 25, 211, 212, 213, + /* 140 */ 205, 121, 102, 103, 104, 105, 106, 107, 108, 109, /* 150 */ 110, 111, 112, 113, 43, 44, 45, 46, 47, 48, - /* 160 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 81, - /* 170 */ 292, 59, 292, 298, 108, 109, 110, 111, 112, 113, - /* 180 */ 69, 116, 117, 118, 72, 106, 107, 193, 111, 112, - /* 190 */ 113, 54, 55, 56, 57, 58, 102, 103, 104, 105, - /* 200 */ 106, 107, 108, 109, 110, 111, 112, 113, 120, 25, - /* 210 */ 216, 217, 145, 102, 103, 104, 105, 106, 107, 108, - /* 220 */ 109, 110, 111, 112, 113, 231, 138, 139, 116, 117, - /* 230 */ 118, 164, 153, 19, 155, 54, 55, 56, 57, 102, + /* 160 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 240, + /* 170 */ 241, 116, 117, 118, 119, 240, 241, 122, 123, 124, + /* 180 */ 69, 298, 253, 194, 255, 106, 107, 132, 253, 141, + /* 190 */ 255, 54, 55, 56, 57, 58, 207, 268, 102, 103, + /* 200 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + /* 210 */ 214, 128, 129, 102, 103, 104, 105, 106, 107, 108, + /* 220 */ 109, 110, 111, 112, 113, 134, 25, 136, 137, 300, + /* 230 */ 165, 166, 153, 19, 155, 54, 55, 56, 57, 102, /* 240 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 250 */ 113, 128, 129, 46, 47, 48, 49, 43, 44, 45, + /* 250 */ 113, 108, 109, 110, 111, 112, 113, 43, 44, 45, /* 260 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 270 */ 56, 57, 216, 193, 25, 59, 193, 19, 165, 166, - /* 280 */ 193, 67, 24, 102, 103, 104, 105, 106, 107, 108, - /* 290 */ 109, 110, 111, 112, 113, 73, 216, 217, 59, 216, - /* 300 */ 217, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 270 */ 56, 57, 276, 277, 278, 113, 194, 19, 22, 23, + /* 280 */ 194, 67, 24, 102, 103, 104, 105, 106, 107, 108, + /* 290 */ 109, 110, 111, 112, 113, 220, 250, 59, 252, 217, + /* 300 */ 218, 43, 44, 45, 46, 47, 48, 49, 50, 51, /* 310 */ 52, 53, 54, 55, 56, 57, 102, 103, 104, 105, - /* 320 */ 106, 107, 108, 109, 110, 111, 112, 113, 121, 145, - /* 330 */ 59, 193, 116, 117, 118, 119, 273, 204, 122, 123, - /* 340 */ 124, 19, 20, 134, 22, 136, 137, 19, 132, 127, - /* 350 */ 128, 129, 24, 22, 23, 116, 117, 118, 36, 193, + /* 320 */ 106, 107, 108, 109, 110, 111, 112, 113, 106, 107, + /* 330 */ 108, 109, 110, 111, 112, 113, 254, 59, 205, 138, + /* 340 */ 139, 19, 20, 194, 22, 263, 22, 23, 231, 25, + /* 350 */ 72, 276, 277, 278, 116, 117, 118, 101, 36, 76, /* 360 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 370 */ 112, 113, 239, 240, 311, 312, 215, 106, 107, 241, - /* 380 */ 19, 59, 216, 217, 223, 252, 115, 116, 117, 118, - /* 390 */ 151, 120, 26, 71, 193, 308, 309, 193, 149, 128, - /* 400 */ 313, 216, 269, 81, 43, 44, 45, 46, 47, 48, - /* 410 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 253, - /* 420 */ 216, 217, 100, 95, 153, 59, 155, 261, 106, 107, - /* 430 */ 25, 193, 101, 193, 193, 231, 114, 25, 116, 117, - /* 440 */ 118, 113, 304, 121, 193, 204, 59, 119, 120, 121, - /* 450 */ 122, 123, 124, 125, 216, 217, 193, 216, 217, 131, - /* 460 */ 138, 139, 230, 102, 103, 104, 105, 106, 107, 108, + /* 370 */ 112, 113, 89, 240, 241, 92, 73, 194, 194, 73, + /* 380 */ 19, 59, 188, 189, 190, 191, 253, 81, 255, 151, + /* 390 */ 196, 25, 198, 71, 116, 117, 118, 311, 312, 205, + /* 400 */ 217, 218, 316, 81, 43, 44, 45, 46, 47, 48, + /* 410 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 270, + /* 420 */ 22, 23, 100, 25, 59, 101, 138, 139, 106, 107, + /* 430 */ 127, 128, 129, 127, 240, 241, 114, 254, 116, 117, + /* 440 */ 118, 76, 76, 121, 138, 139, 263, 253, 264, 255, + /* 450 */ 205, 275, 87, 19, 89, 89, 194, 92, 92, 199, + /* 460 */ 138, 139, 268, 102, 103, 104, 105, 106, 107, 108, /* 470 */ 109, 110, 111, 112, 113, 153, 154, 155, 156, 157, - /* 480 */ 239, 240, 116, 117, 118, 76, 193, 23, 19, 25, - /* 490 */ 22, 253, 23, 252, 253, 108, 87, 204, 89, 261, - /* 500 */ 198, 92, 261, 116, 117, 118, 193, 306, 307, 216, - /* 510 */ 217, 150, 43, 44, 45, 46, 47, 48, 49, 50, - /* 520 */ 51, 52, 53, 54, 55, 56, 57, 59, 193, 216, - /* 530 */ 217, 19, 239, 240, 283, 23, 106, 107, 108, 109, - /* 540 */ 110, 111, 112, 113, 73, 252, 253, 142, 308, 309, - /* 550 */ 138, 139, 81, 313, 145, 43, 44, 45, 46, 47, + /* 480 */ 81, 116, 117, 118, 129, 240, 241, 224, 19, 226, + /* 490 */ 314, 315, 23, 25, 300, 59, 22, 234, 253, 101, + /* 500 */ 255, 236, 237, 26, 194, 183, 194, 152, 72, 22, + /* 510 */ 145, 150, 43, 44, 45, 46, 47, 48, 49, 50, + /* 520 */ 51, 52, 53, 54, 55, 56, 57, 217, 218, 217, + /* 530 */ 218, 19, 189, 59, 191, 23, 59, 138, 139, 196, + /* 540 */ 135, 198, 232, 283, 232, 140, 59, 287, 205, 275, + /* 550 */ 116, 205, 116, 117, 118, 43, 44, 45, 46, 47, /* 560 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 570 */ 307, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 580 */ 111, 112, 113, 281, 116, 117, 118, 285, 23, 193, - /* 590 */ 25, 119, 59, 193, 122, 123, 124, 59, 127, 203, - /* 600 */ 59, 205, 19, 268, 132, 25, 23, 22, 193, 138, - /* 610 */ 139, 249, 204, 251, 102, 103, 104, 105, 106, 107, + /* 570 */ 194, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 580 */ 111, 112, 113, 240, 241, 194, 240, 241, 314, 315, + /* 590 */ 116, 117, 118, 116, 117, 118, 253, 194, 255, 253, + /* 600 */ 59, 255, 19, 116, 117, 118, 23, 22, 217, 218, + /* 610 */ 142, 268, 205, 275, 102, 103, 104, 105, 106, 107, /* 620 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 630 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 640 */ 57, 19, 22, 23, 59, 23, 25, 239, 240, 116, - /* 650 */ 117, 118, 193, 11, 116, 117, 118, 116, 117, 118, - /* 660 */ 252, 269, 22, 193, 15, 43, 44, 45, 46, 47, + /* 640 */ 57, 19, 194, 300, 59, 23, 119, 240, 241, 122, + /* 650 */ 123, 124, 314, 315, 194, 236, 237, 194, 117, 132, + /* 660 */ 253, 81, 255, 205, 59, 43, 44, 45, 46, 47, /* 670 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 680 */ 273, 143, 193, 118, 143, 102, 103, 104, 105, 106, - /* 690 */ 107, 108, 109, 110, 111, 112, 113, 76, 118, 59, - /* 700 */ 241, 116, 117, 118, 304, 216, 217, 292, 143, 60, - /* 710 */ 89, 241, 19, 92, 193, 193, 23, 22, 311, 312, - /* 720 */ 231, 101, 22, 143, 102, 103, 104, 105, 106, 107, + /* 680 */ 217, 218, 194, 194, 194, 102, 103, 104, 105, 106, + /* 690 */ 107, 108, 109, 110, 111, 112, 113, 294, 240, 241, + /* 700 */ 120, 116, 117, 118, 59, 194, 217, 218, 211, 212, + /* 710 */ 213, 253, 19, 255, 194, 19, 23, 254, 138, 139, + /* 720 */ 24, 232, 117, 194, 102, 103, 104, 105, 106, 107, /* 730 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 740 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 750 */ 57, 19, 193, 193, 59, 23, 116, 117, 118, 59, - /* 760 */ 201, 21, 241, 304, 193, 206, 127, 128, 129, 193, - /* 770 */ 128, 129, 235, 236, 304, 43, 44, 45, 46, 47, + /* 750 */ 57, 19, 264, 108, 76, 23, 127, 128, 129, 311, + /* 760 */ 312, 116, 117, 118, 316, 87, 306, 89, 308, 194, + /* 770 */ 92, 22, 59, 194, 22, 43, 44, 45, 46, 47, /* 780 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 790 */ 22, 193, 216, 217, 193, 102, 103, 104, 105, 106, - /* 800 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 193, - /* 810 */ 193, 116, 117, 118, 216, 217, 116, 117, 118, 226, - /* 820 */ 80, 193, 19, 235, 236, 304, 23, 211, 212, 231, - /* 830 */ 204, 216, 217, 205, 102, 103, 104, 105, 106, 107, + /* 790 */ 194, 95, 217, 218, 265, 102, 103, 104, 105, 106, + /* 800 */ 107, 108, 109, 110, 111, 112, 113, 232, 59, 113, + /* 810 */ 25, 59, 194, 217, 218, 119, 120, 121, 122, 123, + /* 820 */ 124, 125, 19, 145, 194, 194, 23, 131, 232, 116, + /* 830 */ 117, 118, 35, 194, 102, 103, 104, 105, 106, 107, /* 840 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 850 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 860 */ 57, 19, 193, 123, 76, 239, 240, 193, 253, 239, - /* 870 */ 240, 239, 240, 244, 106, 107, 193, 89, 252, 193, - /* 880 */ 92, 59, 252, 254, 252, 43, 44, 45, 46, 47, + /* 860 */ 57, 19, 194, 66, 194, 116, 117, 118, 116, 117, + /* 870 */ 118, 74, 242, 294, 194, 194, 206, 23, 194, 25, + /* 880 */ 194, 111, 112, 113, 25, 43, 44, 45, 46, 47, /* 890 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 900 */ 284, 161, 216, 217, 193, 102, 103, 104, 105, 106, - /* 910 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 244, - /* 920 */ 187, 188, 189, 190, 7, 8, 9, 309, 195, 254, - /* 930 */ 197, 313, 19, 127, 128, 129, 262, 204, 22, 117, - /* 940 */ 24, 216, 217, 273, 102, 103, 104, 105, 106, 107, + /* 900 */ 24, 194, 194, 217, 218, 102, 103, 104, 105, 106, + /* 910 */ 107, 108, 109, 110, 111, 112, 113, 241, 232, 194, + /* 920 */ 212, 213, 242, 242, 217, 218, 242, 130, 11, 253, + /* 930 */ 194, 255, 19, 265, 149, 59, 306, 194, 308, 232, + /* 940 */ 309, 310, 217, 218, 102, 103, 104, 105, 106, 107, /* 950 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 960 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 970 */ 57, 193, 239, 240, 193, 59, 19, 188, 253, 190, - /* 980 */ 193, 311, 312, 16, 195, 252, 197, 193, 19, 301, - /* 990 */ 302, 135, 193, 204, 216, 217, 140, 216, 217, 266, - /* 1000 */ 204, 159, 45, 46, 47, 48, 49, 50, 51, 52, + /* 970 */ 57, 194, 194, 59, 194, 239, 19, 194, 25, 254, + /* 980 */ 303, 304, 23, 194, 25, 126, 306, 306, 308, 308, + /* 990 */ 306, 271, 308, 117, 286, 217, 218, 217, 218, 194, + /* 1000 */ 194, 159, 45, 46, 47, 48, 49, 50, 51, 52, /* 1010 */ 53, 54, 55, 56, 57, 102, 103, 104, 105, 106, - /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 12, 239, 240, - /* 1030 */ 193, 298, 238, 117, 253, 239, 240, 238, 259, 260, - /* 1040 */ 193, 252, 27, 193, 77, 193, 79, 204, 252, 262, - /* 1050 */ 193, 299, 300, 193, 100, 266, 278, 42, 204, 102, + /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 59, 239, 194, + /* 1030 */ 116, 117, 118, 260, 254, 194, 240, 241, 194, 233, + /* 1040 */ 205, 240, 241, 205, 239, 128, 129, 270, 265, 253, + /* 1050 */ 194, 255, 217, 218, 253, 194, 255, 143, 280, 102, /* 1060 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 1070 */ 113, 117, 159, 216, 217, 121, 216, 217, 63, 193, - /* 1080 */ 193, 193, 239, 240, 115, 116, 193, 298, 73, 240, - /* 1090 */ 238, 231, 19, 239, 240, 252, 22, 24, 211, 212, - /* 1100 */ 263, 252, 216, 217, 216, 217, 252, 153, 154, 155, - /* 1110 */ 253, 193, 19, 144, 213, 268, 43, 44, 45, 46, + /* 1070 */ 113, 118, 159, 217, 218, 240, 241, 118, 240, 241, + /* 1080 */ 194, 194, 194, 239, 116, 117, 118, 22, 253, 254, + /* 1090 */ 255, 253, 19, 255, 233, 194, 143, 24, 263, 212, + /* 1100 */ 213, 194, 143, 217, 218, 217, 218, 261, 262, 271, + /* 1110 */ 254, 143, 19, 7, 8, 9, 43, 44, 45, 46, /* 1120 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1130 */ 57, 193, 19, 59, 216, 217, 43, 44, 45, 46, + /* 1130 */ 57, 16, 19, 22, 23, 294, 43, 44, 45, 46, /* 1140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1150 */ 57, 193, 19, 24, 216, 217, 43, 44, 45, 46, + /* 1150 */ 57, 312, 194, 214, 21, 316, 43, 44, 45, 46, /* 1160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1170 */ 57, 284, 193, 208, 209, 102, 103, 104, 105, 106, - /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 286, 59, 193, - /* 1190 */ 232, 117, 291, 193, 193, 102, 103, 104, 105, 106, - /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 193, 204, 22, - /* 1210 */ 23, 193, 25, 66, 193, 102, 103, 104, 105, 106, - /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 193, 193, 193, - /* 1230 */ 216, 217, 85, 193, 238, 19, 16, 216, 217, 238, - /* 1240 */ 193, 94, 193, 239, 240, 231, 117, 268, 35, 116, - /* 1250 */ 216, 217, 216, 217, 22, 23, 252, 25, 208, 209, + /* 1170 */ 57, 106, 107, 286, 194, 102, 103, 104, 105, 106, + /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 207, 158, 59, + /* 1190 */ 160, 22, 77, 24, 79, 102, 103, 104, 105, 106, + /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 194, 194, 229, + /* 1210 */ 194, 231, 101, 80, 22, 102, 103, 104, 105, 106, + /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 288, 59, 12, + /* 1230 */ 217, 218, 293, 217, 218, 19, 106, 107, 59, 19, + /* 1240 */ 16, 127, 128, 129, 27, 115, 116, 117, 118, 194, + /* 1250 */ 120, 59, 22, 194, 24, 194, 123, 100, 128, 42, /* 1260 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 1270 */ 54, 55, 56, 57, 193, 193, 19, 5, 59, 66, - /* 1280 */ 193, 263, 10, 11, 12, 13, 14, 74, 101, 17, - /* 1290 */ 193, 46, 193, 146, 193, 76, 213, 77, 263, 79, - /* 1300 */ 12, 260, 30, 46, 32, 264, 87, 193, 89, 29, - /* 1310 */ 263, 92, 40, 33, 232, 27, 193, 108, 102, 103, + /* 1270 */ 54, 55, 56, 57, 117, 194, 217, 218, 121, 100, + /* 1280 */ 63, 194, 245, 153, 194, 155, 117, 19, 115, 194, + /* 1290 */ 73, 214, 194, 256, 161, 116, 117, 194, 217, 218, + /* 1300 */ 121, 77, 194, 79, 217, 218, 194, 217, 218, 117, + /* 1310 */ 153, 154, 155, 254, 46, 217, 218, 144, 102, 103, /* 1320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - /* 1330 */ 42, 138, 139, 101, 193, 116, 117, 118, 19, 20, - /* 1340 */ 255, 22, 70, 130, 135, 65, 256, 257, 193, 140, - /* 1350 */ 78, 63, 193, 81, 193, 36, 193, 216, 217, 193, - /* 1360 */ 115, 193, 263, 193, 145, 268, 59, 48, 193, 193, - /* 1370 */ 98, 193, 115, 193, 291, 216, 217, 193, 59, 216, - /* 1380 */ 217, 161, 216, 217, 216, 217, 216, 217, 131, 193, - /* 1390 */ 71, 193, 216, 217, 216, 217, 216, 217, 193, 260, - /* 1400 */ 216, 217, 19, 264, 85, 133, 244, 100, 193, 90, - /* 1410 */ 138, 139, 216, 217, 216, 217, 254, 244, 193, 100, - /* 1420 */ 193, 216, 217, 116, 117, 106, 107, 254, 121, 193, - /* 1430 */ 115, 216, 217, 114, 162, 116, 117, 118, 115, 244, - /* 1440 */ 121, 216, 217, 216, 217, 193, 309, 193, 31, 254, - /* 1450 */ 313, 309, 216, 217, 309, 313, 39, 193, 313, 309, - /* 1460 */ 153, 154, 155, 313, 193, 150, 25, 144, 216, 217, - /* 1470 */ 216, 217, 153, 154, 155, 156, 157, 0, 1, 2, - /* 1480 */ 216, 217, 5, 149, 150, 22, 193, 10, 11, 12, - /* 1490 */ 13, 14, 193, 158, 17, 160, 193, 19, 20, 116, - /* 1500 */ 22, 25, 193, 24, 22, 193, 24, 30, 226, 32, - /* 1510 */ 19, 20, 226, 22, 36, 193, 53, 40, 193, 216, - /* 1520 */ 217, 193, 23, 193, 25, 216, 217, 36, 216, 217, - /* 1530 */ 193, 99, 193, 193, 22, 193, 193, 59, 216, 217, - /* 1540 */ 193, 216, 217, 193, 216, 217, 193, 70, 129, 71, - /* 1550 */ 59, 129, 193, 216, 217, 78, 216, 217, 81, 216, - /* 1560 */ 217, 193, 71, 85, 193, 133, 193, 126, 90, 216, - /* 1570 */ 217, 152, 258, 61, 152, 98, 85, 193, 100, 193, - /* 1580 */ 23, 90, 25, 121, 106, 107, 23, 216, 217, 216, - /* 1590 */ 217, 100, 114, 131, 116, 117, 118, 106, 107, 121, - /* 1600 */ 216, 217, 216, 217, 193, 114, 193, 116, 117, 118, - /* 1610 */ 133, 22, 121, 193, 59, 138, 139, 193, 142, 193, - /* 1620 */ 141, 23, 23, 25, 25, 120, 121, 216, 217, 216, - /* 1630 */ 217, 153, 154, 155, 156, 157, 216, 217, 19, 162, - /* 1640 */ 216, 217, 216, 217, 153, 154, 155, 156, 157, 1, - /* 1650 */ 2, 193, 59, 5, 19, 20, 318, 22, 10, 11, - /* 1660 */ 12, 13, 14, 193, 59, 17, 193, 23, 23, 25, - /* 1670 */ 25, 36, 117, 193, 216, 217, 193, 23, 30, 25, - /* 1680 */ 32, 19, 20, 23, 22, 25, 216, 217, 40, 216, - /* 1690 */ 217, 7, 8, 23, 59, 25, 83, 84, 36, 23, - /* 1700 */ 193, 25, 23, 23, 25, 25, 71, 153, 145, 155, - /* 1710 */ 117, 153, 23, 155, 25, 23, 97, 25, 70, 193, - /* 1720 */ 193, 59, 117, 236, 193, 193, 78, 193, 193, 81, - /* 1730 */ 141, 193, 193, 71, 193, 100, 288, 287, 242, 255, - /* 1740 */ 255, 106, 107, 108, 255, 255, 98, 243, 297, 114, - /* 1750 */ 214, 116, 117, 118, 245, 191, 121, 271, 293, 267, - /* 1760 */ 267, 246, 100, 246, 245, 271, 271, 293, 106, 107, - /* 1770 */ 220, 271, 229, 225, 249, 219, 114, 259, 116, 117, - /* 1780 */ 118, 133, 259, 121, 219, 219, 138, 139, 153, 154, - /* 1790 */ 155, 156, 157, 280, 249, 243, 19, 20, 245, 22, - /* 1800 */ 196, 259, 140, 259, 60, 297, 141, 297, 200, 200, - /* 1810 */ 162, 38, 200, 36, 294, 153, 154, 155, 156, 157, - /* 1820 */ 151, 150, 294, 283, 22, 43, 234, 18, 237, 200, - /* 1830 */ 270, 272, 237, 237, 237, 18, 59, 199, 270, 149, - /* 1840 */ 246, 272, 272, 200, 234, 234, 246, 246, 71, 246, - /* 1850 */ 199, 158, 290, 62, 22, 200, 19, 20, 199, 22, - /* 1860 */ 289, 221, 221, 200, 200, 199, 199, 115, 218, 64, - /* 1870 */ 218, 218, 22, 36, 227, 126, 227, 100, 165, 221, - /* 1880 */ 224, 224, 24, 106, 107, 312, 218, 305, 113, 282, - /* 1890 */ 91, 114, 220, 116, 117, 118, 59, 282, 121, 218, - /* 1900 */ 218, 218, 200, 317, 317, 82, 221, 265, 71, 148, - /* 1910 */ 145, 265, 22, 277, 200, 158, 279, 140, 147, 25, - /* 1920 */ 146, 202, 248, 250, 249, 247, 13, 250, 194, 194, - /* 1930 */ 153, 154, 155, 156, 157, 6, 303, 100, 192, 192, - /* 1940 */ 246, 213, 192, 106, 107, 207, 213, 207, 222, 213, - /* 1950 */ 213, 114, 222, 116, 117, 118, 214, 214, 121, 4, - /* 1960 */ 207, 213, 3, 22, 303, 15, 163, 16, 23, 23, - /* 1970 */ 139, 151, 130, 25, 20, 142, 24, 16, 144, 1, - /* 1980 */ 142, 130, 130, 61, 37, 53, 300, 151, 53, 53, - /* 1990 */ 153, 154, 155, 156, 157, 53, 130, 116, 34, 1, - /* 2000 */ 141, 5, 22, 115, 161, 68, 25, 68, 75, 41, - /* 2010 */ 141, 115, 24, 20, 19, 131, 125, 23, 28, 22, - /* 2020 */ 67, 22, 22, 22, 67, 59, 24, 96, 22, 67, - /* 2030 */ 23, 149, 22, 25, 23, 23, 23, 22, 34, 141, - /* 2040 */ 37, 97, 23, 23, 116, 22, 143, 25, 34, 75, - /* 2050 */ 34, 34, 34, 88, 75, 34, 86, 23, 22, 34, - /* 2060 */ 93, 24, 34, 25, 25, 142, 142, 23, 44, 23, - /* 2070 */ 23, 23, 23, 11, 23, 25, 22, 22, 22, 141, - /* 2080 */ 23, 23, 22, 22, 25, 15, 1, 23, 25, 1, - /* 2090 */ 141, 135, 319, 319, 319, 319, 319, 319, 319, 141, - /* 2100 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2110 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2120 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2130 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2140 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2150 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2160 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2170 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2180 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2190 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2200 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2210 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2220 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2230 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2240 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2250 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2260 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2270 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2280 */ 319, 319, 319, 319, 319, + /* 1330 */ 232, 270, 153, 154, 155, 115, 116, 66, 19, 20, + /* 1340 */ 183, 22, 12, 312, 254, 194, 262, 316, 209, 210, + /* 1350 */ 266, 239, 194, 194, 108, 36, 85, 27, 19, 20, + /* 1360 */ 265, 22, 183, 245, 144, 94, 25, 48, 217, 218, + /* 1370 */ 293, 194, 42, 270, 256, 36, 217, 218, 59, 194, + /* 1380 */ 25, 135, 194, 115, 194, 161, 140, 194, 194, 15, + /* 1390 */ 71, 194, 312, 63, 217, 218, 316, 194, 59, 131, + /* 1400 */ 301, 302, 217, 218, 85, 217, 218, 217, 218, 90, + /* 1410 */ 71, 217, 218, 19, 217, 218, 245, 146, 262, 100, + /* 1420 */ 217, 218, 266, 265, 85, 106, 107, 256, 312, 90, + /* 1430 */ 209, 210, 316, 114, 60, 116, 117, 118, 194, 100, + /* 1440 */ 121, 194, 194, 145, 115, 106, 107, 19, 46, 19, + /* 1450 */ 20, 24, 22, 114, 194, 116, 117, 118, 194, 245, + /* 1460 */ 121, 194, 164, 194, 217, 218, 36, 194, 258, 259, + /* 1470 */ 256, 194, 153, 154, 155, 156, 157, 217, 218, 150, + /* 1480 */ 31, 217, 218, 142, 217, 218, 217, 218, 39, 59, + /* 1490 */ 217, 218, 153, 154, 155, 156, 157, 149, 150, 5, + /* 1500 */ 145, 71, 183, 245, 10, 11, 12, 13, 14, 194, + /* 1510 */ 116, 17, 129, 227, 256, 85, 194, 115, 194, 23, + /* 1520 */ 90, 25, 183, 99, 30, 97, 32, 22, 22, 194, + /* 1530 */ 100, 194, 217, 218, 40, 152, 106, 107, 23, 217, + /* 1540 */ 218, 194, 19, 20, 114, 22, 116, 117, 118, 257, + /* 1550 */ 194, 121, 217, 218, 217, 218, 194, 133, 53, 36, + /* 1560 */ 23, 23, 25, 25, 70, 120, 121, 61, 141, 7, + /* 1570 */ 8, 121, 78, 217, 218, 81, 23, 227, 25, 217, + /* 1580 */ 218, 131, 59, 153, 154, 155, 156, 157, 0, 1, + /* 1590 */ 2, 59, 98, 5, 71, 23, 227, 25, 10, 11, + /* 1600 */ 12, 13, 14, 83, 84, 17, 23, 23, 25, 25, + /* 1610 */ 59, 194, 194, 183, 23, 23, 25, 25, 30, 194, + /* 1620 */ 32, 19, 20, 100, 22, 194, 194, 133, 40, 106, + /* 1630 */ 107, 108, 138, 139, 194, 217, 218, 114, 36, 116, + /* 1640 */ 117, 118, 217, 218, 121, 194, 194, 194, 23, 117, + /* 1650 */ 25, 194, 23, 23, 25, 25, 162, 194, 70, 194, + /* 1660 */ 145, 59, 23, 153, 25, 155, 78, 194, 117, 81, + /* 1670 */ 217, 218, 194, 71, 217, 218, 153, 154, 155, 156, + /* 1680 */ 157, 194, 217, 218, 194, 23, 98, 25, 321, 194, + /* 1690 */ 217, 218, 194, 19, 20, 194, 22, 153, 23, 155, + /* 1700 */ 25, 194, 100, 194, 217, 218, 183, 194, 106, 107, + /* 1710 */ 36, 194, 217, 218, 237, 194, 114, 243, 116, 117, + /* 1720 */ 118, 133, 194, 121, 217, 218, 138, 139, 194, 194, + /* 1730 */ 194, 290, 289, 59, 217, 218, 194, 194, 217, 218, + /* 1740 */ 194, 194, 140, 194, 194, 71, 194, 244, 194, 194, + /* 1750 */ 162, 217, 218, 194, 194, 153, 154, 155, 156, 157, + /* 1760 */ 217, 218, 194, 217, 218, 194, 217, 218, 257, 217, + /* 1770 */ 218, 217, 218, 257, 100, 194, 257, 217, 218, 257, + /* 1780 */ 106, 107, 215, 299, 194, 183, 192, 194, 114, 194, + /* 1790 */ 116, 117, 118, 1, 2, 121, 221, 5, 217, 218, + /* 1800 */ 273, 197, 10, 11, 12, 13, 14, 217, 218, 17, + /* 1810 */ 217, 218, 217, 218, 140, 194, 246, 194, 273, 295, + /* 1820 */ 247, 273, 30, 247, 32, 269, 269, 153, 154, 155, + /* 1830 */ 156, 157, 40, 246, 273, 295, 230, 226, 217, 218, + /* 1840 */ 217, 218, 220, 261, 220, 282, 220, 19, 20, 244, + /* 1850 */ 22, 250, 141, 250, 246, 60, 201, 183, 261, 261, + /* 1860 */ 261, 201, 70, 299, 36, 299, 201, 38, 151, 150, + /* 1870 */ 78, 285, 22, 81, 296, 296, 43, 235, 18, 238, + /* 1880 */ 201, 274, 272, 238, 238, 238, 18, 59, 200, 149, + /* 1890 */ 98, 247, 274, 274, 235, 247, 247, 247, 235, 71, + /* 1900 */ 272, 201, 200, 158, 292, 62, 291, 201, 200, 22, + /* 1910 */ 201, 222, 200, 222, 201, 200, 115, 219, 219, 64, + /* 1920 */ 219, 228, 22, 126, 221, 133, 165, 222, 100, 225, + /* 1930 */ 138, 139, 225, 219, 106, 107, 24, 219, 228, 219, + /* 1940 */ 219, 307, 114, 113, 116, 117, 118, 315, 284, 121, + /* 1950 */ 284, 222, 201, 91, 162, 320, 320, 82, 148, 267, + /* 1960 */ 145, 267, 22, 279, 201, 158, 281, 251, 147, 146, + /* 1970 */ 25, 203, 250, 249, 251, 248, 13, 247, 195, 195, + /* 1980 */ 6, 153, 154, 155, 156, 157, 193, 193, 305, 193, + /* 1990 */ 208, 305, 302, 214, 214, 214, 208, 223, 223, 214, + /* 2000 */ 4, 215, 215, 214, 3, 22, 208, 163, 15, 23, + /* 2010 */ 16, 183, 23, 139, 151, 130, 25, 20, 142, 24, + /* 2020 */ 16, 144, 1, 142, 130, 130, 61, 37, 53, 151, + /* 2030 */ 53, 53, 53, 130, 116, 1, 34, 141, 5, 22, + /* 2040 */ 115, 161, 75, 25, 68, 141, 41, 115, 68, 24, + /* 2050 */ 20, 19, 131, 125, 67, 67, 96, 22, 22, 22, + /* 2060 */ 37, 23, 22, 24, 22, 59, 67, 23, 149, 28, + /* 2070 */ 22, 25, 23, 23, 23, 22, 141, 34, 97, 23, + /* 2080 */ 23, 34, 116, 22, 143, 25, 34, 75, 34, 34, + /* 2090 */ 75, 88, 34, 86, 23, 22, 34, 25, 24, 34, + /* 2100 */ 25, 93, 23, 44, 142, 23, 142, 23, 23, 22, + /* 2110 */ 11, 25, 23, 25, 23, 22, 22, 22, 1, 23, + /* 2120 */ 23, 23, 22, 22, 15, 141, 141, 25, 25, 1, + /* 2130 */ 322, 322, 322, 135, 322, 322, 322, 322, 322, 322, + /* 2140 */ 322, 141, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2150 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2160 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2170 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2180 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2190 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2200 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2210 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2220 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2230 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2240 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2250 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2260 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2270 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2280 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2290 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2300 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2310 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2320 */ 322, 322, 322, 322, 322, 322, 322, 322, }; -#define YY_SHIFT_COUNT (578) +#define YY_SHIFT_COUNT (582) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2088) +#define YY_SHIFT_MAX (2128) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1648, 1477, 1272, 322, 322, 1, 1319, 1478, 1491, 1837, - /* 10 */ 1837, 1837, 471, 0, 0, 214, 1093, 1837, 1837, 1837, - /* 20 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 30 */ 1837, 271, 271, 1219, 1219, 216, 88, 1, 1, 1, - /* 40 */ 1, 1, 40, 111, 258, 361, 469, 512, 583, 622, - /* 50 */ 693, 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, - /* 60 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, - /* 70 */ 1093, 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1635, - /* 80 */ 1662, 1777, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 90 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 100 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 110 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 120 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 130 */ 1837, 137, 181, 181, 181, 181, 181, 181, 181, 94, - /* 140 */ 430, 66, 65, 112, 366, 533, 533, 740, 1257, 533, - /* 150 */ 533, 79, 79, 533, 412, 412, 412, 77, 412, 123, - /* 160 */ 113, 113, 113, 22, 22, 2100, 2100, 328, 328, 328, - /* 170 */ 239, 468, 468, 468, 468, 1015, 1015, 409, 366, 1187, - /* 180 */ 1232, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 200 */ 533, 969, 621, 621, 533, 642, 788, 788, 1133, 1133, - /* 210 */ 822, 822, 67, 1193, 2100, 2100, 2100, 2100, 2100, 2100, - /* 220 */ 2100, 1307, 954, 954, 585, 472, 640, 387, 695, 538, - /* 230 */ 541, 700, 533, 533, 533, 533, 533, 533, 533, 533, - /* 240 */ 533, 533, 222, 533, 533, 533, 533, 533, 533, 533, - /* 250 */ 533, 533, 533, 533, 533, 1213, 1213, 1213, 533, 533, - /* 260 */ 533, 565, 533, 533, 533, 916, 1147, 533, 533, 1288, - /* 270 */ 533, 533, 533, 533, 533, 533, 533, 533, 639, 1280, - /* 280 */ 209, 1129, 1129, 1129, 1129, 580, 209, 209, 1209, 768, - /* 290 */ 917, 649, 1315, 1334, 405, 1334, 1383, 249, 1315, 1315, - /* 300 */ 249, 1315, 405, 1383, 1441, 464, 1245, 1417, 1417, 1417, - /* 310 */ 1323, 1323, 1323, 1323, 184, 184, 1335, 1476, 856, 1482, - /* 320 */ 1744, 1744, 1665, 1665, 1773, 1773, 1665, 1669, 1671, 1802, - /* 330 */ 1782, 1809, 1809, 1809, 1809, 1665, 1817, 1690, 1671, 1671, - /* 340 */ 1690, 1802, 1782, 1690, 1782, 1690, 1665, 1817, 1693, 1791, - /* 350 */ 1665, 1817, 1832, 1665, 1817, 1665, 1817, 1832, 1752, 1752, - /* 360 */ 1752, 1805, 1850, 1850, 1832, 1752, 1749, 1752, 1805, 1752, - /* 370 */ 1752, 1713, 1858, 1775, 1775, 1832, 1665, 1799, 1799, 1823, - /* 380 */ 1823, 1761, 1765, 1890, 1665, 1757, 1761, 1771, 1774, 1690, - /* 390 */ 1894, 1913, 1913, 1929, 1929, 1929, 2100, 2100, 2100, 2100, - /* 400 */ 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, - /* 410 */ 2100, 207, 1220, 331, 620, 967, 806, 1074, 1499, 1432, - /* 420 */ 1463, 1479, 1419, 1422, 1557, 1512, 1598, 1599, 1644, 1645, - /* 430 */ 1654, 1660, 1555, 1505, 1684, 1462, 1670, 1563, 1619, 1593, - /* 440 */ 1676, 1679, 1613, 1680, 1554, 1558, 1689, 1692, 1605, 1589, - /* 450 */ 1955, 1959, 1941, 1803, 1950, 1951, 1945, 1946, 1831, 1820, - /* 460 */ 1842, 1948, 1948, 1952, 1833, 1954, 1834, 1961, 1978, 1838, - /* 470 */ 1851, 1948, 1852, 1922, 1947, 1948, 1836, 1932, 1935, 1936, - /* 480 */ 1942, 1866, 1881, 1964, 1859, 1998, 1996, 1980, 1888, 1843, - /* 490 */ 1937, 1981, 1939, 1933, 1968, 1869, 1896, 1988, 1993, 1995, - /* 500 */ 1884, 1891, 1997, 1953, 1999, 2000, 1994, 2001, 1957, 1966, - /* 510 */ 2002, 1931, 1990, 2006, 1962, 2003, 2007, 2004, 1882, 2010, - /* 520 */ 2011, 2012, 2008, 2013, 2015, 1944, 1898, 2019, 2020, 1928, - /* 530 */ 2014, 2023, 1903, 2022, 2016, 2017, 2018, 2021, 1965, 1974, - /* 540 */ 1970, 2024, 1979, 1967, 2025, 2034, 2036, 2037, 2038, 2039, - /* 550 */ 2028, 1923, 1924, 2044, 2022, 2046, 2047, 2048, 2049, 2050, - /* 560 */ 2051, 2054, 2062, 2055, 2056, 2057, 2058, 2060, 2061, 2059, - /* 570 */ 1956, 1938, 1949, 1958, 2063, 2064, 2070, 2085, 2088, + /* 0 */ 1792, 1588, 1494, 322, 322, 399, 306, 1319, 1339, 1430, + /* 10 */ 1828, 1828, 1828, 580, 399, 399, 399, 399, 399, 0, + /* 20 */ 0, 214, 1093, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 30 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1130, 1130, + /* 40 */ 365, 365, 55, 278, 436, 713, 713, 201, 201, 201, + /* 50 */ 201, 40, 111, 258, 361, 469, 512, 583, 622, 693, + /* 60 */ 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, 1093, + /* 70 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, + /* 80 */ 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1523, 1602, + /* 90 */ 1674, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 100 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 110 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 120 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 130 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 140 */ 137, 181, 181, 181, 181, 181, 181, 181, 96, 222, + /* 150 */ 143, 477, 713, 1133, 1268, 713, 713, 79, 79, 713, + /* 160 */ 770, 83, 65, 65, 65, 288, 162, 162, 2142, 2142, + /* 170 */ 696, 696, 696, 238, 474, 474, 474, 474, 1217, 1217, + /* 180 */ 678, 477, 324, 398, 713, 713, 713, 713, 713, 713, + /* 190 */ 713, 713, 713, 713, 713, 713, 713, 713, 713, 713, + /* 200 */ 713, 713, 713, 1220, 366, 366, 713, 917, 283, 283, + /* 210 */ 434, 434, 605, 605, 1298, 2142, 2142, 2142, 2142, 2142, + /* 220 */ 2142, 2142, 1179, 1157, 1157, 487, 527, 585, 645, 749, + /* 230 */ 914, 968, 752, 713, 713, 713, 713, 713, 713, 713, + /* 240 */ 713, 713, 713, 303, 713, 713, 713, 713, 713, 713, + /* 250 */ 713, 713, 713, 713, 713, 713, 797, 797, 797, 713, + /* 260 */ 713, 713, 959, 713, 713, 713, 1169, 1271, 713, 713, + /* 270 */ 1330, 713, 713, 713, 713, 713, 713, 713, 713, 629, + /* 280 */ 7, 91, 876, 876, 876, 876, 953, 91, 91, 1246, + /* 290 */ 1065, 1106, 1374, 1329, 1348, 468, 1348, 1394, 785, 1329, + /* 300 */ 1329, 785, 1329, 468, 1394, 859, 854, 1402, 1449, 1449, + /* 310 */ 1449, 1173, 1173, 1173, 1173, 1355, 1355, 1030, 1341, 405, + /* 320 */ 1230, 1795, 1795, 1711, 1711, 1829, 1829, 1711, 1717, 1719, + /* 330 */ 1850, 1833, 1860, 1860, 1860, 1860, 1711, 1868, 1740, 1719, + /* 340 */ 1719, 1740, 1850, 1833, 1740, 1833, 1740, 1711, 1868, 1745, + /* 350 */ 1843, 1711, 1868, 1887, 1711, 1868, 1711, 1868, 1887, 1801, + /* 360 */ 1801, 1801, 1855, 1900, 1900, 1887, 1801, 1797, 1801, 1855, + /* 370 */ 1801, 1801, 1761, 1912, 1830, 1830, 1887, 1711, 1862, 1862, + /* 380 */ 1875, 1875, 1810, 1815, 1940, 1711, 1807, 1810, 1821, 1823, + /* 390 */ 1740, 1945, 1963, 1963, 1974, 1974, 1974, 2142, 2142, 2142, + /* 400 */ 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, + /* 410 */ 2142, 2142, 20, 1224, 256, 1111, 1115, 1114, 1192, 1496, + /* 420 */ 1424, 1505, 1427, 355, 1383, 1537, 1506, 1538, 1553, 1583, + /* 430 */ 1584, 1591, 1625, 541, 1445, 1562, 1450, 1572, 1515, 1428, + /* 440 */ 1532, 1592, 1629, 1520, 1630, 1639, 1510, 1544, 1662, 1675, + /* 450 */ 1551, 48, 1996, 2001, 1983, 1844, 1993, 1994, 1986, 1989, + /* 460 */ 1874, 1863, 1885, 1991, 1991, 1995, 1876, 1997, 1877, 2004, + /* 470 */ 2021, 1881, 1894, 1991, 1895, 1965, 1990, 1991, 1878, 1975, + /* 480 */ 1977, 1978, 1979, 1903, 1918, 2002, 1896, 2034, 2033, 2017, + /* 490 */ 1925, 1880, 1976, 2018, 1980, 1967, 2005, 1904, 1932, 2025, + /* 500 */ 2030, 2032, 1921, 1928, 2035, 1987, 2036, 2037, 2038, 2040, + /* 510 */ 1988, 2006, 2039, 1960, 2041, 2042, 1999, 2023, 2044, 2043, + /* 520 */ 1919, 2048, 2049, 2050, 2046, 2051, 2053, 1981, 1935, 2056, + /* 530 */ 2057, 1966, 2047, 2061, 1941, 2060, 2052, 2054, 2055, 2058, + /* 540 */ 2003, 2012, 2007, 2059, 2015, 2008, 2062, 2071, 2073, 2074, + /* 550 */ 2072, 2075, 2065, 1962, 1964, 2079, 2060, 2082, 2084, 2085, + /* 560 */ 2087, 2086, 2089, 2088, 2091, 2093, 2099, 2094, 2095, 2096, + /* 570 */ 2097, 2100, 2101, 2102, 1998, 1984, 1985, 2000, 2103, 2098, + /* 580 */ 2109, 2117, 2128, }; -#define YY_REDUCE_COUNT (410) -#define YY_REDUCE_MIN (-271) -#define YY_REDUCE_MAX (1753) +#define YY_REDUCE_COUNT (411) +#define YY_REDUCE_MIN (-275) +#define YY_REDUCE_MAX (1798) static const short yy_reduce_ofst[] = { - /* 0 */ -125, 733, 789, 241, 293, -123, -193, -191, -183, -187, - /* 10 */ 166, 238, 133, -207, -199, -267, -176, -6, 204, 489, - /* 20 */ 576, 598, -175, 686, 860, 615, 725, 1014, 778, 781, - /* 30 */ 857, 616, 887, 87, 240, -192, 408, 626, 796, 843, - /* 40 */ 854, 1004, -271, -271, -271, -271, -271, -271, -271, -271, - /* 50 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 60 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 70 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, 80, - /* 80 */ 83, 313, 886, 888, 918, 938, 1021, 1034, 1036, 1141, - /* 90 */ 1159, 1163, 1166, 1168, 1170, 1176, 1178, 1180, 1184, 1196, - /* 100 */ 1198, 1205, 1215, 1225, 1227, 1236, 1252, 1254, 1264, 1303, - /* 110 */ 1309, 1312, 1322, 1325, 1328, 1337, 1340, 1343, 1353, 1371, - /* 120 */ 1373, 1384, 1386, 1411, 1413, 1420, 1424, 1426, 1458, 1470, - /* 130 */ 1473, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 140 */ -271, -271, 138, 459, 396, -158, 470, 302, -212, 521, - /* 150 */ 201, -195, -92, 559, 630, 632, 630, -271, 632, 901, - /* 160 */ 63, 407, 670, -271, -271, -271, -271, 161, 161, 161, - /* 170 */ 251, 335, 847, 979, 1097, 537, 588, 618, 628, 688, - /* 180 */ 688, -166, -161, 674, 787, 794, 799, 852, 996, -122, - /* 190 */ 837, -120, 1018, 1035, 415, 1047, 1001, 958, 1082, 400, - /* 200 */ 1099, 779, 1137, 1142, 263, 1083, 1145, 1150, 1041, 1139, - /* 210 */ 965, 1050, 362, 849, 752, 629, 675, 1162, 1173, 1090, - /* 220 */ 1195, -194, 56, 185, -135, 232, 522, 560, 571, 601, - /* 230 */ 617, 669, 683, 711, 850, 893, 1000, 1040, 1049, 1081, - /* 240 */ 1087, 1101, 392, 1114, 1123, 1155, 1161, 1175, 1271, 1293, - /* 250 */ 1299, 1330, 1339, 1342, 1347, 593, 1282, 1286, 1350, 1359, - /* 260 */ 1368, 1314, 1480, 1483, 1507, 1085, 1338, 1526, 1527, 1487, - /* 270 */ 1531, 560, 1532, 1534, 1535, 1538, 1539, 1541, 1448, 1450, - /* 280 */ 1496, 1484, 1485, 1489, 1490, 1314, 1496, 1496, 1504, 1536, - /* 290 */ 1564, 1451, 1486, 1492, 1509, 1493, 1465, 1515, 1494, 1495, - /* 300 */ 1517, 1500, 1519, 1474, 1550, 1543, 1548, 1556, 1565, 1566, - /* 310 */ 1518, 1523, 1542, 1544, 1525, 1545, 1513, 1553, 1552, 1604, - /* 320 */ 1508, 1510, 1608, 1609, 1520, 1528, 1612, 1540, 1559, 1560, - /* 330 */ 1592, 1591, 1595, 1596, 1597, 1629, 1638, 1594, 1569, 1570, - /* 340 */ 1600, 1568, 1610, 1601, 1611, 1603, 1643, 1651, 1562, 1571, - /* 350 */ 1655, 1659, 1640, 1663, 1666, 1664, 1667, 1641, 1650, 1652, - /* 360 */ 1653, 1647, 1656, 1657, 1658, 1668, 1672, 1681, 1649, 1682, - /* 370 */ 1683, 1573, 1582, 1607, 1615, 1685, 1702, 1586, 1587, 1642, - /* 380 */ 1646, 1673, 1675, 1636, 1714, 1637, 1677, 1674, 1678, 1694, - /* 390 */ 1719, 1734, 1735, 1746, 1747, 1750, 1633, 1661, 1686, 1738, - /* 400 */ 1728, 1733, 1736, 1737, 1740, 1726, 1730, 1742, 1743, 1748, - /* 410 */ 1753, + /* 0 */ -71, 194, 343, 835, -180, -177, 838, -194, -188, -185, + /* 10 */ -183, 82, 183, -65, 133, 245, 346, 407, 458, -178, + /* 20 */ 75, -275, -4, 310, 312, 489, 575, 596, 463, 686, + /* 30 */ 707, 725, 780, 1098, 856, 778, 1059, 1090, 708, 887, + /* 40 */ 86, 448, 980, 630, 680, 681, 684, 796, 801, 796, + /* 50 */ 801, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 60 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 70 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 80 */ -261, -261, -261, -261, -261, -261, -261, -261, 391, 886, + /* 90 */ 888, 1013, 1016, 1081, 1087, 1151, 1159, 1177, 1185, 1188, + /* 100 */ 1190, 1194, 1197, 1203, 1247, 1260, 1264, 1267, 1269, 1273, + /* 110 */ 1315, 1322, 1335, 1337, 1356, 1362, 1418, 1425, 1453, 1457, + /* 120 */ 1465, 1473, 1487, 1495, 1507, 1517, 1521, 1534, 1543, 1546, + /* 130 */ 1549, 1552, 1554, 1560, 1581, 1590, 1593, 1595, 1621, 1623, + /* 140 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 150 */ -261, -186, -117, 260, 263, 460, 631, -74, 497, -181, + /* 160 */ -261, 939, 176, 274, 338, 676, -261, -261, -261, -261, + /* 170 */ -212, -212, -212, -184, 149, 777, 1061, 1103, 265, 419, + /* 180 */ -254, 670, 677, 677, -11, -129, 184, 488, 736, 789, + /* 190 */ 805, 844, 403, 529, 579, 668, 783, 841, 1158, 1112, + /* 200 */ 806, 861, 1095, 846, 839, 1031, -189, 1077, 1080, 1116, + /* 210 */ 1084, 1156, 1139, 1221, 46, 1099, 1037, 1118, 1171, 1214, + /* 220 */ 1210, 1258, -210, -190, -176, -115, 117, 262, 376, 490, + /* 230 */ 511, 520, 618, 639, 743, 901, 907, 958, 1014, 1055, + /* 240 */ 1108, 1193, 1244, 720, 1248, 1277, 1324, 1347, 1417, 1431, + /* 250 */ 1432, 1440, 1451, 1452, 1463, 1478, 1286, 1350, 1369, 1490, + /* 260 */ 1498, 1501, 773, 1509, 1513, 1528, 1292, 1367, 1535, 1536, + /* 270 */ 1477, 1542, 376, 1547, 1550, 1555, 1559, 1568, 1571, 1441, + /* 280 */ 1443, 1474, 1511, 1516, 1519, 1522, 773, 1474, 1474, 1503, + /* 290 */ 1567, 1594, 1484, 1527, 1556, 1570, 1557, 1524, 1573, 1545, + /* 300 */ 1548, 1576, 1561, 1587, 1540, 1575, 1606, 1611, 1622, 1624, + /* 310 */ 1626, 1582, 1597, 1598, 1599, 1601, 1603, 1563, 1608, 1605, + /* 320 */ 1604, 1564, 1566, 1655, 1660, 1578, 1579, 1665, 1586, 1607, + /* 330 */ 1610, 1642, 1641, 1645, 1646, 1647, 1679, 1688, 1644, 1618, + /* 340 */ 1619, 1648, 1628, 1659, 1649, 1663, 1650, 1700, 1702, 1612, + /* 350 */ 1615, 1706, 1708, 1689, 1709, 1712, 1713, 1715, 1691, 1698, + /* 360 */ 1699, 1701, 1693, 1704, 1707, 1705, 1714, 1703, 1718, 1710, + /* 370 */ 1720, 1721, 1632, 1634, 1664, 1666, 1729, 1751, 1635, 1636, + /* 380 */ 1692, 1694, 1716, 1722, 1684, 1763, 1685, 1723, 1724, 1727, + /* 390 */ 1730, 1768, 1783, 1784, 1793, 1794, 1796, 1683, 1686, 1690, + /* 400 */ 1782, 1779, 1780, 1781, 1785, 1788, 1774, 1775, 1786, 1787, + /* 410 */ 1789, 1798, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1648, 1648, 1648, 1478, 1243, 1354, 1243, 1243, 1243, 1478, - /* 10 */ 1478, 1478, 1243, 1384, 1384, 1531, 1276, 1243, 1243, 1243, - /* 20 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1477, 1243, - /* 30 */ 1243, 1243, 1243, 1564, 1564, 1243, 1243, 1243, 1243, 1243, - /* 40 */ 1243, 1243, 1243, 1393, 1243, 1400, 1243, 1243, 1243, 1243, - /* 50 */ 1243, 1479, 1480, 1243, 1243, 1243, 1530, 1532, 1495, 1407, - /* 60 */ 1406, 1405, 1404, 1513, 1372, 1398, 1391, 1395, 1474, 1475, - /* 70 */ 1473, 1626, 1480, 1479, 1243, 1394, 1442, 1458, 1441, 1243, - /* 80 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 90 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 100 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 110 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 120 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 130 */ 1243, 1450, 1457, 1456, 1455, 1464, 1454, 1451, 1444, 1443, - /* 140 */ 1445, 1446, 1243, 1243, 1267, 1243, 1243, 1264, 1318, 1243, - /* 150 */ 1243, 1243, 1243, 1243, 1550, 1549, 1243, 1447, 1243, 1276, - /* 160 */ 1435, 1434, 1433, 1461, 1448, 1460, 1459, 1538, 1600, 1599, - /* 170 */ 1496, 1243, 1243, 1243, 1243, 1243, 1243, 1564, 1243, 1243, - /* 180 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 190 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 200 */ 1243, 1374, 1564, 1564, 1243, 1276, 1564, 1564, 1375, 1375, - /* 210 */ 1272, 1272, 1378, 1243, 1545, 1345, 1345, 1345, 1345, 1354, - /* 220 */ 1345, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 230 */ 1243, 1243, 1243, 1243, 1243, 1243, 1535, 1533, 1243, 1243, - /* 240 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 250 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 260 */ 1243, 1243, 1243, 1243, 1243, 1350, 1243, 1243, 1243, 1243, - /* 270 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1593, 1243, 1508, - /* 280 */ 1332, 1350, 1350, 1350, 1350, 1352, 1333, 1331, 1344, 1277, - /* 290 */ 1250, 1640, 1410, 1399, 1351, 1399, 1637, 1397, 1410, 1410, - /* 300 */ 1397, 1410, 1351, 1637, 1293, 1615, 1288, 1384, 1384, 1384, - /* 310 */ 1374, 1374, 1374, 1374, 1378, 1378, 1476, 1351, 1344, 1243, - /* 320 */ 1640, 1640, 1360, 1360, 1639, 1639, 1360, 1496, 1623, 1419, - /* 330 */ 1321, 1327, 1327, 1327, 1327, 1360, 1261, 1397, 1623, 1623, - /* 340 */ 1397, 1419, 1321, 1397, 1321, 1397, 1360, 1261, 1512, 1634, - /* 350 */ 1360, 1261, 1486, 1360, 1261, 1360, 1261, 1486, 1319, 1319, - /* 360 */ 1319, 1308, 1243, 1243, 1486, 1319, 1293, 1319, 1308, 1319, - /* 370 */ 1319, 1582, 1243, 1490, 1490, 1486, 1360, 1574, 1574, 1387, - /* 380 */ 1387, 1392, 1378, 1481, 1360, 1243, 1392, 1390, 1388, 1397, - /* 390 */ 1311, 1596, 1596, 1592, 1592, 1592, 1645, 1645, 1545, 1608, - /* 400 */ 1276, 1276, 1276, 1276, 1608, 1295, 1295, 1277, 1277, 1276, - /* 410 */ 1608, 1243, 1243, 1243, 1243, 1243, 1243, 1603, 1243, 1540, - /* 420 */ 1497, 1364, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 430 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1551, 1243, - /* 440 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1424, - /* 450 */ 1243, 1246, 1542, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 460 */ 1243, 1401, 1402, 1365, 1243, 1243, 1243, 1243, 1243, 1243, - /* 470 */ 1243, 1416, 1243, 1243, 1243, 1411, 1243, 1243, 1243, 1243, - /* 480 */ 1243, 1243, 1243, 1243, 1636, 1243, 1243, 1243, 1243, 1243, - /* 490 */ 1243, 1511, 1510, 1243, 1243, 1362, 1243, 1243, 1243, 1243, - /* 500 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1291, - /* 510 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 520 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 530 */ 1243, 1243, 1243, 1389, 1243, 1243, 1243, 1243, 1243, 1243, - /* 540 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1579, 1379, - /* 550 */ 1243, 1243, 1243, 1243, 1627, 1243, 1243, 1243, 1243, 1243, - /* 560 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1619, - /* 570 */ 1335, 1425, 1243, 1428, 1265, 1243, 1255, 1243, 1243, + /* 0 */ 1663, 1663, 1663, 1491, 1254, 1367, 1254, 1254, 1254, 1254, + /* 10 */ 1491, 1491, 1491, 1254, 1254, 1254, 1254, 1254, 1254, 1397, + /* 20 */ 1397, 1544, 1287, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 30 */ 1254, 1254, 1254, 1254, 1254, 1490, 1254, 1254, 1254, 1254, + /* 40 */ 1578, 1578, 1254, 1254, 1254, 1254, 1254, 1563, 1562, 1254, + /* 50 */ 1254, 1254, 1406, 1254, 1413, 1254, 1254, 1254, 1254, 1254, + /* 60 */ 1492, 1493, 1254, 1254, 1254, 1543, 1545, 1508, 1420, 1419, + /* 70 */ 1418, 1417, 1526, 1385, 1411, 1404, 1408, 1487, 1488, 1486, + /* 80 */ 1641, 1493, 1492, 1254, 1407, 1455, 1471, 1454, 1254, 1254, + /* 90 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 100 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 110 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 120 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 130 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 140 */ 1463, 1470, 1469, 1468, 1477, 1467, 1464, 1457, 1456, 1458, + /* 150 */ 1459, 1278, 1254, 1275, 1329, 1254, 1254, 1254, 1254, 1254, + /* 160 */ 1460, 1287, 1448, 1447, 1446, 1254, 1474, 1461, 1473, 1472, + /* 170 */ 1551, 1615, 1614, 1509, 1254, 1254, 1254, 1254, 1254, 1254, + /* 180 */ 1578, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 190 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 200 */ 1254, 1254, 1254, 1387, 1578, 1578, 1254, 1287, 1578, 1578, + /* 210 */ 1388, 1388, 1283, 1283, 1391, 1558, 1358, 1358, 1358, 1358, + /* 220 */ 1367, 1358, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 230 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1548, 1546, 1254, + /* 240 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 250 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 260 */ 1254, 1254, 1254, 1254, 1254, 1254, 1363, 1254, 1254, 1254, + /* 270 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1608, 1254, + /* 280 */ 1521, 1343, 1363, 1363, 1363, 1363, 1365, 1344, 1342, 1357, + /* 290 */ 1288, 1261, 1655, 1423, 1412, 1364, 1412, 1652, 1410, 1423, + /* 300 */ 1423, 1410, 1423, 1364, 1652, 1304, 1630, 1299, 1397, 1397, + /* 310 */ 1397, 1387, 1387, 1387, 1387, 1391, 1391, 1489, 1364, 1357, + /* 320 */ 1254, 1655, 1655, 1373, 1373, 1654, 1654, 1373, 1509, 1638, + /* 330 */ 1432, 1332, 1338, 1338, 1338, 1338, 1373, 1272, 1410, 1638, + /* 340 */ 1638, 1410, 1432, 1332, 1410, 1332, 1410, 1373, 1272, 1525, + /* 350 */ 1649, 1373, 1272, 1499, 1373, 1272, 1373, 1272, 1499, 1330, + /* 360 */ 1330, 1330, 1319, 1254, 1254, 1499, 1330, 1304, 1330, 1319, + /* 370 */ 1330, 1330, 1596, 1254, 1503, 1503, 1499, 1373, 1588, 1588, + /* 380 */ 1400, 1400, 1405, 1391, 1494, 1373, 1254, 1405, 1403, 1401, + /* 390 */ 1410, 1322, 1611, 1611, 1607, 1607, 1607, 1660, 1660, 1558, + /* 400 */ 1623, 1287, 1287, 1287, 1287, 1623, 1306, 1306, 1288, 1288, + /* 410 */ 1287, 1623, 1254, 1254, 1254, 1254, 1254, 1254, 1618, 1254, + /* 420 */ 1553, 1510, 1377, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 430 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1564, + /* 440 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 450 */ 1254, 1437, 1254, 1257, 1555, 1254, 1254, 1254, 1254, 1254, + /* 460 */ 1254, 1254, 1254, 1414, 1415, 1378, 1254, 1254, 1254, 1254, + /* 470 */ 1254, 1254, 1254, 1429, 1254, 1254, 1254, 1424, 1254, 1254, + /* 480 */ 1254, 1254, 1254, 1254, 1254, 1254, 1651, 1254, 1254, 1254, + /* 490 */ 1254, 1254, 1254, 1524, 1523, 1254, 1254, 1375, 1254, 1254, + /* 500 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 510 */ 1254, 1302, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 520 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 530 */ 1254, 1254, 1254, 1254, 1254, 1402, 1254, 1254, 1254, 1254, + /* 540 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 550 */ 1593, 1392, 1254, 1254, 1254, 1254, 1642, 1254, 1254, 1254, + /* 560 */ 1254, 1352, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 570 */ 1254, 1254, 1254, 1634, 1346, 1438, 1254, 1441, 1276, 1254, + /* 580 */ 1266, 1254, 1254, }; /********** End of lemon-generated parsing tables *****************************/ @@ -172301,8 +173892,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* TRUEFALSE => nothing */ 0, /* ISNOT => nothing */ 0, /* FUNCTION => nothing */ - 0, /* UMINUS => nothing */ 0, /* UPLUS => nothing */ + 0, /* UMINUS => nothing */ 0, /* TRUTH => nothing */ 0, /* REGISTER => nothing */ 0, /* VECTOR => nothing */ @@ -172311,6 +173902,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASTERISK => nothing */ 0, /* SPAN => nothing */ 0, /* ERROR => nothing */ + 0, /* QNUMBER => nothing */ 0, /* SPACE => nothing */ 0, /* ILLEGAL => nothing */ }; @@ -172353,14 +173945,9 @@ struct yyParser { #endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct yyParser yyParser; @@ -172574,8 +174161,8 @@ static const char *const yyTokenName[] = { /* 170 */ "TRUEFALSE", /* 171 */ "ISNOT", /* 172 */ "FUNCTION", - /* 173 */ "UMINUS", - /* 174 */ "UPLUS", + /* 173 */ "UPLUS", + /* 174 */ "UMINUS", /* 175 */ "TRUTH", /* 176 */ "REGISTER", /* 177 */ "VECTOR", @@ -172584,142 +174171,145 @@ static const char *const yyTokenName[] = { /* 180 */ "ASTERISK", /* 181 */ "SPAN", /* 182 */ "ERROR", - /* 183 */ "SPACE", - /* 184 */ "ILLEGAL", - /* 185 */ "input", - /* 186 */ "cmdlist", - /* 187 */ "ecmd", - /* 188 */ "cmdx", - /* 189 */ "explain", - /* 190 */ "cmd", - /* 191 */ "transtype", - /* 192 */ "trans_opt", - /* 193 */ "nm", - /* 194 */ "savepoint_opt", - /* 195 */ "create_table", - /* 196 */ "create_table_args", - /* 197 */ "createkw", - /* 198 */ "temp", - /* 199 */ "ifnotexists", - /* 200 */ "dbnm", - /* 201 */ "columnlist", - /* 202 */ "conslist_opt", - /* 203 */ "table_option_set", - /* 204 */ "select", - /* 205 */ "table_option", - /* 206 */ "columnname", - /* 207 */ "carglist", - /* 208 */ "typetoken", - /* 209 */ "typename", - /* 210 */ "signed", - /* 211 */ "plus_num", - /* 212 */ "minus_num", - /* 213 */ "scanpt", - /* 214 */ "scantok", - /* 215 */ "ccons", - /* 216 */ "term", - /* 217 */ "expr", - /* 218 */ "onconf", - /* 219 */ "sortorder", - /* 220 */ "autoinc", - /* 221 */ "eidlist_opt", - /* 222 */ "refargs", - /* 223 */ "defer_subclause", - /* 224 */ "generated", - /* 225 */ "refarg", - /* 226 */ "refact", - /* 227 */ "init_deferred_pred_opt", - /* 228 */ "conslist", - /* 229 */ "tconscomma", - /* 230 */ "tcons", - /* 231 */ "sortlist", - /* 232 */ "eidlist", - /* 233 */ "defer_subclause_opt", - /* 234 */ "orconf", - /* 235 */ "resolvetype", - /* 236 */ "raisetype", - /* 237 */ "ifexists", - /* 238 */ "fullname", - /* 239 */ "selectnowith", - /* 240 */ "oneselect", - /* 241 */ "wqlist", - /* 242 */ "multiselect_op", - /* 243 */ "distinct", - /* 244 */ "selcollist", - /* 245 */ "from", - /* 246 */ "where_opt", - /* 247 */ "groupby_opt", - /* 248 */ "having_opt", - /* 249 */ "orderby_opt", - /* 250 */ "limit_opt", - /* 251 */ "window_clause", - /* 252 */ "values", - /* 253 */ "nexprlist", - /* 254 */ "sclp", - /* 255 */ "as", - /* 256 */ "seltablist", - /* 257 */ "stl_prefix", - /* 258 */ "joinop", - /* 259 */ "on_using", - /* 260 */ "indexed_by", - /* 261 */ "exprlist", - /* 262 */ "xfullname", - /* 263 */ "idlist", - /* 264 */ "indexed_opt", - /* 265 */ "nulls", - /* 266 */ "with", - /* 267 */ "where_opt_ret", - /* 268 */ "setlist", - /* 269 */ "insert_cmd", - /* 270 */ "idlist_opt", - /* 271 */ "upsert", - /* 272 */ "returning", - /* 273 */ "filter_over", - /* 274 */ "likeop", - /* 275 */ "between_op", - /* 276 */ "in_op", - /* 277 */ "paren_exprlist", - /* 278 */ "case_operand", - /* 279 */ "case_exprlist", - /* 280 */ "case_else", - /* 281 */ "uniqueflag", - /* 282 */ "collate", - /* 283 */ "vinto", - /* 284 */ "nmnum", - /* 285 */ "trigger_decl", - /* 286 */ "trigger_cmd_list", - /* 287 */ "trigger_time", - /* 288 */ "trigger_event", - /* 289 */ "foreach_clause", - /* 290 */ "when_clause", - /* 291 */ "trigger_cmd", - /* 292 */ "trnm", - /* 293 */ "tridxby", - /* 294 */ "database_kw_opt", - /* 295 */ "key_opt", - /* 296 */ "add_column_fullname", - /* 297 */ "kwcolumn_opt", - /* 298 */ "create_vtab", - /* 299 */ "vtabarglist", - /* 300 */ "vtabarg", - /* 301 */ "vtabargtoken", - /* 302 */ "lp", - /* 303 */ "anylist", - /* 304 */ "wqitem", - /* 305 */ "wqas", - /* 306 */ "windowdefn_list", - /* 307 */ "windowdefn", - /* 308 */ "window", - /* 309 */ "frame_opt", - /* 310 */ "part_opt", - /* 311 */ "filter_clause", - /* 312 */ "over_clause", - /* 313 */ "range_or_rows", - /* 314 */ "frame_bound", - /* 315 */ "frame_bound_s", - /* 316 */ "frame_bound_e", - /* 317 */ "frame_exclude_opt", - /* 318 */ "frame_exclude", + /* 183 */ "QNUMBER", + /* 184 */ "SPACE", + /* 185 */ "ILLEGAL", + /* 186 */ "input", + /* 187 */ "cmdlist", + /* 188 */ "ecmd", + /* 189 */ "cmdx", + /* 190 */ "explain", + /* 191 */ "cmd", + /* 192 */ "transtype", + /* 193 */ "trans_opt", + /* 194 */ "nm", + /* 195 */ "savepoint_opt", + /* 196 */ "create_table", + /* 197 */ "create_table_args", + /* 198 */ "createkw", + /* 199 */ "temp", + /* 200 */ "ifnotexists", + /* 201 */ "dbnm", + /* 202 */ "columnlist", + /* 203 */ "conslist_opt", + /* 204 */ "table_option_set", + /* 205 */ "select", + /* 206 */ "table_option", + /* 207 */ "columnname", + /* 208 */ "carglist", + /* 209 */ "typetoken", + /* 210 */ "typename", + /* 211 */ "signed", + /* 212 */ "plus_num", + /* 213 */ "minus_num", + /* 214 */ "scanpt", + /* 215 */ "scantok", + /* 216 */ "ccons", + /* 217 */ "term", + /* 218 */ "expr", + /* 219 */ "onconf", + /* 220 */ "sortorder", + /* 221 */ "autoinc", + /* 222 */ "eidlist_opt", + /* 223 */ "refargs", + /* 224 */ "defer_subclause", + /* 225 */ "generated", + /* 226 */ "refarg", + /* 227 */ "refact", + /* 228 */ "init_deferred_pred_opt", + /* 229 */ "conslist", + /* 230 */ "tconscomma", + /* 231 */ "tcons", + /* 232 */ "sortlist", + /* 233 */ "eidlist", + /* 234 */ "defer_subclause_opt", + /* 235 */ "orconf", + /* 236 */ "resolvetype", + /* 237 */ "raisetype", + /* 238 */ "ifexists", + /* 239 */ "fullname", + /* 240 */ "selectnowith", + /* 241 */ "oneselect", + /* 242 */ "wqlist", + /* 243 */ "multiselect_op", + /* 244 */ "distinct", + /* 245 */ "selcollist", + /* 246 */ "from", + /* 247 */ "where_opt", + /* 248 */ "groupby_opt", + /* 249 */ "having_opt", + /* 250 */ "orderby_opt", + /* 251 */ "limit_opt", + /* 252 */ "window_clause", + /* 253 */ "values", + /* 254 */ "nexprlist", + /* 255 */ "mvalues", + /* 256 */ "sclp", + /* 257 */ "as", + /* 258 */ "seltablist", + /* 259 */ "stl_prefix", + /* 260 */ "joinop", + /* 261 */ "on_using", + /* 262 */ "indexed_by", + /* 263 */ "exprlist", + /* 264 */ "xfullname", + /* 265 */ "idlist", + /* 266 */ "indexed_opt", + /* 267 */ "nulls", + /* 268 */ "with", + /* 269 */ "where_opt_ret", + /* 270 */ "setlist", + /* 271 */ "insert_cmd", + /* 272 */ "idlist_opt", + /* 273 */ "upsert", + /* 274 */ "returning", + /* 275 */ "filter_over", + /* 276 */ "likeop", + /* 277 */ "between_op", + /* 278 */ "in_op", + /* 279 */ "paren_exprlist", + /* 280 */ "case_operand", + /* 281 */ "case_exprlist", + /* 282 */ "case_else", + /* 283 */ "uniqueflag", + /* 284 */ "collate", + /* 285 */ "vinto", + /* 286 */ "nmnum", + /* 287 */ "trigger_decl", + /* 288 */ "trigger_cmd_list", + /* 289 */ "trigger_time", + /* 290 */ "trigger_event", + /* 291 */ "foreach_clause", + /* 292 */ "when_clause", + /* 293 */ "trigger_cmd", + /* 294 */ "trnm", + /* 295 */ "tridxby", + /* 296 */ "database_kw_opt", + /* 297 */ "key_opt", + /* 298 */ "add_column_fullname", + /* 299 */ "kwcolumn_opt", + /* 300 */ "create_vtab", + /* 301 */ "vtabarglist", + /* 302 */ "vtabarg", + /* 303 */ "vtabargtoken", + /* 304 */ "lp", + /* 305 */ "anylist", + /* 306 */ "wqitem", + /* 307 */ "wqas", + /* 308 */ "withnm", + /* 309 */ "windowdefn_list", + /* 310 */ "windowdefn", + /* 311 */ "window", + /* 312 */ "frame_opt", + /* 313 */ "part_opt", + /* 314 */ "filter_clause", + /* 315 */ "over_clause", + /* 316 */ "range_or_rows", + /* 317 */ "frame_bound", + /* 318 */ "frame_bound_s", + /* 319 */ "frame_bound_e", + /* 320 */ "frame_exclude_opt", + /* 321 */ "frame_exclude", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -172822,351 +174412,363 @@ static const char *const yyRuleName[] = { /* 92 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", /* 93 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", /* 94 */ "values ::= VALUES LP nexprlist RP", - /* 95 */ "values ::= values COMMA LP nexprlist RP", - /* 96 */ "distinct ::= DISTINCT", - /* 97 */ "distinct ::= ALL", - /* 98 */ "distinct ::=", - /* 99 */ "sclp ::=", - /* 100 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 101 */ "selcollist ::= sclp scanpt STAR", - /* 102 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 103 */ "as ::= AS nm", - /* 104 */ "as ::=", - /* 105 */ "from ::=", - /* 106 */ "from ::= FROM seltablist", - /* 107 */ "stl_prefix ::= seltablist joinop", - /* 108 */ "stl_prefix ::=", - /* 109 */ "seltablist ::= stl_prefix nm dbnm as on_using", - /* 110 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", - /* 111 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", - /* 112 */ "seltablist ::= stl_prefix LP select RP as on_using", - /* 113 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", - /* 114 */ "dbnm ::=", - /* 115 */ "dbnm ::= DOT nm", - /* 116 */ "fullname ::= nm", - /* 117 */ "fullname ::= nm DOT nm", - /* 118 */ "xfullname ::= nm", - /* 119 */ "xfullname ::= nm DOT nm", - /* 120 */ "xfullname ::= nm DOT nm AS nm", - /* 121 */ "xfullname ::= nm AS nm", - /* 122 */ "joinop ::= COMMA|JOIN", - /* 123 */ "joinop ::= JOIN_KW JOIN", - /* 124 */ "joinop ::= JOIN_KW nm JOIN", - /* 125 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 126 */ "on_using ::= ON expr", - /* 127 */ "on_using ::= USING LP idlist RP", - /* 128 */ "on_using ::=", - /* 129 */ "indexed_opt ::=", - /* 130 */ "indexed_by ::= INDEXED BY nm", - /* 131 */ "indexed_by ::= NOT INDEXED", - /* 132 */ "orderby_opt ::=", - /* 133 */ "orderby_opt ::= ORDER BY sortlist", - /* 134 */ "sortlist ::= sortlist COMMA expr sortorder nulls", - /* 135 */ "sortlist ::= expr sortorder nulls", - /* 136 */ "sortorder ::= ASC", - /* 137 */ "sortorder ::= DESC", - /* 138 */ "sortorder ::=", - /* 139 */ "nulls ::= NULLS FIRST", - /* 140 */ "nulls ::= NULLS LAST", - /* 141 */ "nulls ::=", - /* 142 */ "groupby_opt ::=", - /* 143 */ "groupby_opt ::= GROUP BY nexprlist", - /* 144 */ "having_opt ::=", - /* 145 */ "having_opt ::= HAVING expr", - /* 146 */ "limit_opt ::=", - /* 147 */ "limit_opt ::= LIMIT expr", - /* 148 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 149 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 150 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", - /* 151 */ "where_opt ::=", - /* 152 */ "where_opt ::= WHERE expr", - /* 153 */ "where_opt_ret ::=", - /* 154 */ "where_opt_ret ::= WHERE expr", - /* 155 */ "where_opt_ret ::= RETURNING selcollist", - /* 156 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", - /* 157 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", - /* 158 */ "setlist ::= setlist COMMA nm EQ expr", - /* 159 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 160 */ "setlist ::= nm EQ expr", - /* 161 */ "setlist ::= LP idlist RP EQ expr", - /* 162 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 163 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", - /* 164 */ "upsert ::=", - /* 165 */ "upsert ::= RETURNING selcollist", - /* 166 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", - /* 167 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", - /* 168 */ "upsert ::= ON CONFLICT DO NOTHING returning", - /* 169 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", - /* 170 */ "returning ::= RETURNING selcollist", - /* 171 */ "insert_cmd ::= INSERT orconf", - /* 172 */ "insert_cmd ::= REPLACE", - /* 173 */ "idlist_opt ::=", - /* 174 */ "idlist_opt ::= LP idlist RP", - /* 175 */ "idlist ::= idlist COMMA nm", - /* 176 */ "idlist ::= nm", - /* 177 */ "expr ::= LP expr RP", - /* 178 */ "expr ::= ID|INDEXED|JOIN_KW", - /* 179 */ "expr ::= nm DOT nm", - /* 180 */ "expr ::= nm DOT nm DOT nm", - /* 181 */ "term ::= NULL|FLOAT|BLOB", - /* 182 */ "term ::= STRING", - /* 183 */ "term ::= INTEGER", - /* 184 */ "expr ::= VARIABLE", - /* 185 */ "expr ::= expr COLLATE ID|STRING", - /* 186 */ "expr ::= CAST LP expr AS typetoken RP", - /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", - /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", - /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", - /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", - /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", - /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", - /* 193 */ "term ::= CTIME_KW", - /* 194 */ "expr ::= LP nexprlist COMMA expr RP", - /* 195 */ "expr ::= expr AND expr", - /* 196 */ "expr ::= expr OR expr", - /* 197 */ "expr ::= expr LT|GT|GE|LE expr", - /* 198 */ "expr ::= expr EQ|NE expr", - /* 199 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 200 */ "expr ::= expr PLUS|MINUS expr", - /* 201 */ "expr ::= expr STAR|SLASH|REM expr", - /* 202 */ "expr ::= expr CONCAT expr", - /* 203 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 204 */ "expr ::= expr likeop expr", - /* 205 */ "expr ::= expr likeop expr ESCAPE expr", - /* 206 */ "expr ::= expr ISNULL|NOTNULL", - /* 207 */ "expr ::= expr NOT NULL", - /* 208 */ "expr ::= expr IS expr", - /* 209 */ "expr ::= expr IS NOT expr", - /* 210 */ "expr ::= expr IS NOT DISTINCT FROM expr", - /* 211 */ "expr ::= expr IS DISTINCT FROM expr", - /* 212 */ "expr ::= NOT expr", - /* 213 */ "expr ::= BITNOT expr", - /* 214 */ "expr ::= PLUS|MINUS expr", - /* 215 */ "expr ::= expr PTR expr", - /* 216 */ "between_op ::= BETWEEN", - /* 217 */ "between_op ::= NOT BETWEEN", - /* 218 */ "expr ::= expr between_op expr AND expr", - /* 219 */ "in_op ::= IN", - /* 220 */ "in_op ::= NOT IN", - /* 221 */ "expr ::= expr in_op LP exprlist RP", - /* 222 */ "expr ::= LP select RP", - /* 223 */ "expr ::= expr in_op LP select RP", - /* 224 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 225 */ "expr ::= EXISTS LP select RP", - /* 226 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 227 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 228 */ "case_exprlist ::= WHEN expr THEN expr", - /* 229 */ "case_else ::= ELSE expr", - /* 230 */ "case_else ::=", - /* 231 */ "case_operand ::=", - /* 232 */ "exprlist ::=", - /* 233 */ "nexprlist ::= nexprlist COMMA expr", - /* 234 */ "nexprlist ::= expr", - /* 235 */ "paren_exprlist ::=", - /* 236 */ "paren_exprlist ::= LP exprlist RP", - /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 238 */ "uniqueflag ::= UNIQUE", - /* 239 */ "uniqueflag ::=", - /* 240 */ "eidlist_opt ::=", - /* 241 */ "eidlist_opt ::= LP eidlist RP", - /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 243 */ "eidlist ::= nm collate sortorder", - /* 244 */ "collate ::=", - /* 245 */ "collate ::= COLLATE ID|STRING", - /* 246 */ "cmd ::= DROP INDEX ifexists fullname", - /* 247 */ "cmd ::= VACUUM vinto", - /* 248 */ "cmd ::= VACUUM nm vinto", - /* 249 */ "vinto ::= INTO expr", - /* 250 */ "vinto ::=", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 260 */ "trigger_time ::= BEFORE|AFTER", - /* 261 */ "trigger_time ::= INSTEAD OF", - /* 262 */ "trigger_time ::=", - /* 263 */ "trigger_event ::= DELETE|INSERT", - /* 264 */ "trigger_event ::= UPDATE", - /* 265 */ "trigger_event ::= UPDATE OF idlist", - /* 266 */ "when_clause ::=", - /* 267 */ "when_clause ::= WHEN expr", - /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 270 */ "trnm ::= nm DOT nm", - /* 271 */ "tridxby ::= INDEXED BY nm", - /* 272 */ "tridxby ::= NOT INDEXED", - /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 276 */ "trigger_cmd ::= scanpt select scanpt", - /* 277 */ "expr ::= RAISE LP IGNORE RP", - /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 279 */ "raisetype ::= ROLLBACK", - /* 280 */ "raisetype ::= ABORT", - /* 281 */ "raisetype ::= FAIL", - /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 284 */ "cmd ::= DETACH database_kw_opt expr", - /* 285 */ "key_opt ::=", - /* 286 */ "key_opt ::= KEY expr", - /* 287 */ "cmd ::= REINDEX", - /* 288 */ "cmd ::= REINDEX nm dbnm", - /* 289 */ "cmd ::= ANALYZE", - /* 290 */ "cmd ::= ANALYZE nm dbnm", - /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 294 */ "add_column_fullname ::= fullname", - /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 296 */ "cmd ::= create_vtab", - /* 297 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 299 */ "vtabarg ::=", - /* 300 */ "vtabargtoken ::= ANY", - /* 301 */ "vtabargtoken ::= lp anylist RP", - /* 302 */ "lp ::= LP", - /* 303 */ "with ::= WITH wqlist", - /* 304 */ "with ::= WITH RECURSIVE wqlist", - /* 305 */ "wqas ::= AS", - /* 306 */ "wqas ::= AS MATERIALIZED", - /* 307 */ "wqas ::= AS NOT MATERIALIZED", - /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 309 */ "wqlist ::= wqitem", - /* 310 */ "wqlist ::= wqlist COMMA wqitem", - /* 311 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 312 */ "windowdefn ::= nm AS LP window RP", - /* 313 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 314 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 315 */ "window ::= ORDER BY sortlist frame_opt", - /* 316 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 317 */ "window ::= nm frame_opt", - /* 318 */ "frame_opt ::=", - /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 320 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 321 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 322 */ "frame_bound_s ::= frame_bound", - /* 323 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 324 */ "frame_bound_e ::= frame_bound", - /* 325 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 326 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 327 */ "frame_bound ::= CURRENT ROW", - /* 328 */ "frame_exclude_opt ::=", - /* 329 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 330 */ "frame_exclude ::= NO OTHERS", - /* 331 */ "frame_exclude ::= CURRENT ROW", - /* 332 */ "frame_exclude ::= GROUP|TIES", - /* 333 */ "window_clause ::= WINDOW windowdefn_list", - /* 334 */ "filter_over ::= filter_clause over_clause", - /* 335 */ "filter_over ::= over_clause", - /* 336 */ "filter_over ::= filter_clause", - /* 337 */ "over_clause ::= OVER LP window RP", - /* 338 */ "over_clause ::= OVER nm", - /* 339 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 340 */ "input ::= cmdlist", - /* 341 */ "cmdlist ::= cmdlist ecmd", - /* 342 */ "cmdlist ::= ecmd", - /* 343 */ "ecmd ::= SEMI", - /* 344 */ "ecmd ::= cmdx SEMI", - /* 345 */ "ecmd ::= explain cmdx SEMI", - /* 346 */ "trans_opt ::=", - /* 347 */ "trans_opt ::= TRANSACTION", - /* 348 */ "trans_opt ::= TRANSACTION nm", - /* 349 */ "savepoint_opt ::= SAVEPOINT", - /* 350 */ "savepoint_opt ::=", - /* 351 */ "cmd ::= create_table create_table_args", - /* 352 */ "table_option_set ::= table_option", - /* 353 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 354 */ "columnlist ::= columnname carglist", - /* 355 */ "nm ::= ID|INDEXED|JOIN_KW", - /* 356 */ "nm ::= STRING", - /* 357 */ "typetoken ::= typename", - /* 358 */ "typename ::= ID|STRING", - /* 359 */ "signed ::= plus_num", - /* 360 */ "signed ::= minus_num", - /* 361 */ "carglist ::= carglist ccons", - /* 362 */ "carglist ::=", - /* 363 */ "ccons ::= NULL onconf", - /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 365 */ "ccons ::= AS generated", - /* 366 */ "conslist_opt ::= COMMA conslist", - /* 367 */ "conslist ::= conslist tconscomma tcons", - /* 368 */ "conslist ::= tcons", - /* 369 */ "tconscomma ::=", - /* 370 */ "defer_subclause_opt ::= defer_subclause", - /* 371 */ "resolvetype ::= raisetype", - /* 372 */ "selectnowith ::= oneselect", - /* 373 */ "oneselect ::= values", - /* 374 */ "sclp ::= selcollist COMMA", - /* 375 */ "as ::= ID|STRING", - /* 376 */ "indexed_opt ::= indexed_by", - /* 377 */ "returning ::=", - /* 378 */ "expr ::= term", - /* 379 */ "likeop ::= LIKE_KW|MATCH", - /* 380 */ "case_operand ::= expr", - /* 381 */ "exprlist ::= nexprlist", - /* 382 */ "nmnum ::= plus_num", - /* 383 */ "nmnum ::= nm", - /* 384 */ "nmnum ::= ON", - /* 385 */ "nmnum ::= DELETE", - /* 386 */ "nmnum ::= DEFAULT", - /* 387 */ "plus_num ::= INTEGER|FLOAT", - /* 388 */ "foreach_clause ::=", - /* 389 */ "foreach_clause ::= FOR EACH ROW", - /* 390 */ "trnm ::= nm", - /* 391 */ "tridxby ::=", - /* 392 */ "database_kw_opt ::= DATABASE", - /* 393 */ "database_kw_opt ::=", - /* 394 */ "kwcolumn_opt ::=", - /* 395 */ "kwcolumn_opt ::= COLUMNKW", - /* 396 */ "vtabarglist ::= vtabarg", - /* 397 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 398 */ "vtabarg ::= vtabarg vtabargtoken", - /* 399 */ "anylist ::=", - /* 400 */ "anylist ::= anylist LP anylist RP", - /* 401 */ "anylist ::= anylist ANY", - /* 402 */ "with ::=", - /* 403 */ "windowdefn_list ::= windowdefn", - /* 404 */ "window ::= frame_opt", + /* 95 */ "oneselect ::= mvalues", + /* 96 */ "mvalues ::= values COMMA LP nexprlist RP", + /* 97 */ "mvalues ::= mvalues COMMA LP nexprlist RP", + /* 98 */ "distinct ::= DISTINCT", + /* 99 */ "distinct ::= ALL", + /* 100 */ "distinct ::=", + /* 101 */ "sclp ::=", + /* 102 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 103 */ "selcollist ::= sclp scanpt STAR", + /* 104 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 105 */ "as ::= AS nm", + /* 106 */ "as ::=", + /* 107 */ "from ::=", + /* 108 */ "from ::= FROM seltablist", + /* 109 */ "stl_prefix ::= seltablist joinop", + /* 110 */ "stl_prefix ::=", + /* 111 */ "seltablist ::= stl_prefix nm dbnm as on_using", + /* 112 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", + /* 113 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", + /* 114 */ "seltablist ::= stl_prefix LP select RP as on_using", + /* 115 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", + /* 116 */ "dbnm ::=", + /* 117 */ "dbnm ::= DOT nm", + /* 118 */ "fullname ::= nm", + /* 119 */ "fullname ::= nm DOT nm", + /* 120 */ "xfullname ::= nm", + /* 121 */ "xfullname ::= nm DOT nm", + /* 122 */ "xfullname ::= nm DOT nm AS nm", + /* 123 */ "xfullname ::= nm AS nm", + /* 124 */ "joinop ::= COMMA|JOIN", + /* 125 */ "joinop ::= JOIN_KW JOIN", + /* 126 */ "joinop ::= JOIN_KW nm JOIN", + /* 127 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 128 */ "on_using ::= ON expr", + /* 129 */ "on_using ::= USING LP idlist RP", + /* 130 */ "on_using ::=", + /* 131 */ "indexed_opt ::=", + /* 132 */ "indexed_by ::= INDEXED BY nm", + /* 133 */ "indexed_by ::= NOT INDEXED", + /* 134 */ "orderby_opt ::=", + /* 135 */ "orderby_opt ::= ORDER BY sortlist", + /* 136 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 137 */ "sortlist ::= expr sortorder nulls", + /* 138 */ "sortorder ::= ASC", + /* 139 */ "sortorder ::= DESC", + /* 140 */ "sortorder ::=", + /* 141 */ "nulls ::= NULLS FIRST", + /* 142 */ "nulls ::= NULLS LAST", + /* 143 */ "nulls ::=", + /* 144 */ "groupby_opt ::=", + /* 145 */ "groupby_opt ::= GROUP BY nexprlist", + /* 146 */ "having_opt ::=", + /* 147 */ "having_opt ::= HAVING expr", + /* 148 */ "limit_opt ::=", + /* 149 */ "limit_opt ::= LIMIT expr", + /* 150 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 151 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 152 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 153 */ "where_opt ::=", + /* 154 */ "where_opt ::= WHERE expr", + /* 155 */ "where_opt_ret ::=", + /* 156 */ "where_opt_ret ::= WHERE expr", + /* 157 */ "where_opt_ret ::= RETURNING selcollist", + /* 158 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 159 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 160 */ "setlist ::= setlist COMMA nm EQ expr", + /* 161 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 162 */ "setlist ::= nm EQ expr", + /* 163 */ "setlist ::= LP idlist RP EQ expr", + /* 164 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 165 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 166 */ "upsert ::=", + /* 167 */ "upsert ::= RETURNING selcollist", + /* 168 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 169 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 170 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 171 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 172 */ "returning ::= RETURNING selcollist", + /* 173 */ "insert_cmd ::= INSERT orconf", + /* 174 */ "insert_cmd ::= REPLACE", + /* 175 */ "idlist_opt ::=", + /* 176 */ "idlist_opt ::= LP idlist RP", + /* 177 */ "idlist ::= idlist COMMA nm", + /* 178 */ "idlist ::= nm", + /* 179 */ "expr ::= LP expr RP", + /* 180 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 181 */ "expr ::= nm DOT nm", + /* 182 */ "expr ::= nm DOT nm DOT nm", + /* 183 */ "term ::= NULL|FLOAT|BLOB", + /* 184 */ "term ::= STRING", + /* 185 */ "term ::= INTEGER", + /* 186 */ "expr ::= VARIABLE", + /* 187 */ "expr ::= expr COLLATE ID|STRING", + /* 188 */ "expr ::= CAST LP expr AS typetoken RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", + /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 193 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", + /* 194 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 195 */ "term ::= CTIME_KW", + /* 196 */ "expr ::= LP nexprlist COMMA expr RP", + /* 197 */ "expr ::= expr AND expr", + /* 198 */ "expr ::= expr OR expr", + /* 199 */ "expr ::= expr LT|GT|GE|LE expr", + /* 200 */ "expr ::= expr EQ|NE expr", + /* 201 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 202 */ "expr ::= expr PLUS|MINUS expr", + /* 203 */ "expr ::= expr STAR|SLASH|REM expr", + /* 204 */ "expr ::= expr CONCAT expr", + /* 205 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 206 */ "expr ::= expr likeop expr", + /* 207 */ "expr ::= expr likeop expr ESCAPE expr", + /* 208 */ "expr ::= expr ISNULL|NOTNULL", + /* 209 */ "expr ::= expr NOT NULL", + /* 210 */ "expr ::= expr IS expr", + /* 211 */ "expr ::= expr IS NOT expr", + /* 212 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 213 */ "expr ::= expr IS DISTINCT FROM expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= PLUS|MINUS expr", + /* 217 */ "expr ::= expr PTR expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::=", + /* 234 */ "exprlist ::=", + /* 235 */ "nexprlist ::= nexprlist COMMA expr", + /* 236 */ "nexprlist ::= expr", + /* 237 */ "paren_exprlist ::=", + /* 238 */ "paren_exprlist ::= LP exprlist RP", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "eidlist_opt ::=", + /* 243 */ "eidlist_opt ::= LP eidlist RP", + /* 244 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 245 */ "eidlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM vinto", + /* 250 */ "cmd ::= VACUUM nm vinto", + /* 251 */ "vinto ::= INTO expr", + /* 252 */ "vinto ::=", + /* 253 */ "cmd ::= PRAGMA nm dbnm", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 256 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 257 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 258 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 259 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 260 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 261 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 262 */ "trigger_time ::= BEFORE|AFTER", + /* 263 */ "trigger_time ::= INSTEAD OF", + /* 264 */ "trigger_time ::=", + /* 265 */ "trigger_event ::= DELETE|INSERT", + /* 266 */ "trigger_event ::= UPDATE", + /* 267 */ "trigger_event ::= UPDATE OF idlist", + /* 268 */ "when_clause ::=", + /* 269 */ "when_clause ::= WHEN expr", + /* 270 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 271 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 272 */ "trnm ::= nm DOT nm", + /* 273 */ "tridxby ::= INDEXED BY nm", + /* 274 */ "tridxby ::= NOT INDEXED", + /* 275 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 276 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 277 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 278 */ "trigger_cmd ::= scanpt select scanpt", + /* 279 */ "expr ::= RAISE LP IGNORE RP", + /* 280 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 281 */ "raisetype ::= ROLLBACK", + /* 282 */ "raisetype ::= ABORT", + /* 283 */ "raisetype ::= FAIL", + /* 284 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 285 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 286 */ "cmd ::= DETACH database_kw_opt expr", + /* 287 */ "key_opt ::=", + /* 288 */ "key_opt ::= KEY expr", + /* 289 */ "cmd ::= REINDEX", + /* 290 */ "cmd ::= REINDEX nm dbnm", + /* 291 */ "cmd ::= ANALYZE", + /* 292 */ "cmd ::= ANALYZE nm dbnm", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 294 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 295 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 296 */ "add_column_fullname ::= fullname", + /* 297 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 298 */ "cmd ::= create_vtab", + /* 299 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 300 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 301 */ "vtabarg ::=", + /* 302 */ "vtabargtoken ::= ANY", + /* 303 */ "vtabargtoken ::= lp anylist RP", + /* 304 */ "lp ::= LP", + /* 305 */ "with ::= WITH wqlist", + /* 306 */ "with ::= WITH RECURSIVE wqlist", + /* 307 */ "wqas ::= AS", + /* 308 */ "wqas ::= AS MATERIALIZED", + /* 309 */ "wqas ::= AS NOT MATERIALIZED", + /* 310 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", + /* 311 */ "withnm ::= nm", + /* 312 */ "wqlist ::= wqitem", + /* 313 */ "wqlist ::= wqlist COMMA wqitem", + /* 314 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 315 */ "windowdefn ::= nm AS LP window RP", + /* 316 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 317 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 318 */ "window ::= ORDER BY sortlist frame_opt", + /* 319 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 320 */ "window ::= nm frame_opt", + /* 321 */ "frame_opt ::=", + /* 322 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 323 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 324 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 325 */ "frame_bound_s ::= frame_bound", + /* 326 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 327 */ "frame_bound_e ::= frame_bound", + /* 328 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 329 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 330 */ "frame_bound ::= CURRENT ROW", + /* 331 */ "frame_exclude_opt ::=", + /* 332 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 333 */ "frame_exclude ::= NO OTHERS", + /* 334 */ "frame_exclude ::= CURRENT ROW", + /* 335 */ "frame_exclude ::= GROUP|TIES", + /* 336 */ "window_clause ::= WINDOW windowdefn_list", + /* 337 */ "filter_over ::= filter_clause over_clause", + /* 338 */ "filter_over ::= over_clause", + /* 339 */ "filter_over ::= filter_clause", + /* 340 */ "over_clause ::= OVER LP window RP", + /* 341 */ "over_clause ::= OVER nm", + /* 342 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 343 */ "term ::= QNUMBER", + /* 344 */ "input ::= cmdlist", + /* 345 */ "cmdlist ::= cmdlist ecmd", + /* 346 */ "cmdlist ::= ecmd", + /* 347 */ "ecmd ::= SEMI", + /* 348 */ "ecmd ::= cmdx SEMI", + /* 349 */ "ecmd ::= explain cmdx SEMI", + /* 350 */ "trans_opt ::=", + /* 351 */ "trans_opt ::= TRANSACTION", + /* 352 */ "trans_opt ::= TRANSACTION nm", + /* 353 */ "savepoint_opt ::= SAVEPOINT", + /* 354 */ "savepoint_opt ::=", + /* 355 */ "cmd ::= create_table create_table_args", + /* 356 */ "table_option_set ::= table_option", + /* 357 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 358 */ "columnlist ::= columnname carglist", + /* 359 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 360 */ "nm ::= STRING", + /* 361 */ "typetoken ::= typename", + /* 362 */ "typename ::= ID|STRING", + /* 363 */ "signed ::= plus_num", + /* 364 */ "signed ::= minus_num", + /* 365 */ "carglist ::= carglist ccons", + /* 366 */ "carglist ::=", + /* 367 */ "ccons ::= NULL onconf", + /* 368 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 369 */ "ccons ::= AS generated", + /* 370 */ "conslist_opt ::= COMMA conslist", + /* 371 */ "conslist ::= conslist tconscomma tcons", + /* 372 */ "conslist ::= tcons", + /* 373 */ "tconscomma ::=", + /* 374 */ "defer_subclause_opt ::= defer_subclause", + /* 375 */ "resolvetype ::= raisetype", + /* 376 */ "selectnowith ::= oneselect", + /* 377 */ "oneselect ::= values", + /* 378 */ "sclp ::= selcollist COMMA", + /* 379 */ "as ::= ID|STRING", + /* 380 */ "indexed_opt ::= indexed_by", + /* 381 */ "returning ::=", + /* 382 */ "expr ::= term", + /* 383 */ "likeop ::= LIKE_KW|MATCH", + /* 384 */ "case_operand ::= expr", + /* 385 */ "exprlist ::= nexprlist", + /* 386 */ "nmnum ::= plus_num", + /* 387 */ "nmnum ::= nm", + /* 388 */ "nmnum ::= ON", + /* 389 */ "nmnum ::= DELETE", + /* 390 */ "nmnum ::= DEFAULT", + /* 391 */ "plus_num ::= INTEGER|FLOAT", + /* 392 */ "foreach_clause ::=", + /* 393 */ "foreach_clause ::= FOR EACH ROW", + /* 394 */ "trnm ::= nm", + /* 395 */ "tridxby ::=", + /* 396 */ "database_kw_opt ::= DATABASE", + /* 397 */ "database_kw_opt ::=", + /* 398 */ "kwcolumn_opt ::=", + /* 399 */ "kwcolumn_opt ::= COLUMNKW", + /* 400 */ "vtabarglist ::= vtabarg", + /* 401 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 402 */ "vtabarg ::= vtabarg vtabargtoken", + /* 403 */ "anylist ::=", + /* 404 */ "anylist ::= anylist LP anylist RP", + /* 405 */ "anylist ::= anylist ANY", + /* 406 */ "with ::=", + /* 407 */ "windowdefn_list ::= windowdefn", + /* 408 */ "window ::= frame_opt", }; #endif /* NDEBUG */ -#if YYSTACKDEPTH<=0 +#if YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); int newSize; int idx; yyStackEntry *pNew; - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystack = pNew; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; } +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -173186,24 +174788,14 @@ SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL) #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; #endif yypParser->yytos = yypParser->yystack; yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Parser_ENGINEALWAYSONSTACK @@ -173257,97 +174849,98 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 204: /* select */ - case 239: /* selectnowith */ - case 240: /* oneselect */ - case 252: /* values */ + case 205: /* select */ + case 240: /* selectnowith */ + case 241: /* oneselect */ + case 253: /* values */ + case 255: /* mvalues */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy47)); +sqlite3SelectDelete(pParse->db, (yypminor->yy555)); } break; - case 216: /* term */ - case 217: /* expr */ - case 246: /* where_opt */ - case 248: /* having_opt */ - case 267: /* where_opt_ret */ - case 278: /* case_operand */ - case 280: /* case_else */ - case 283: /* vinto */ - case 290: /* when_clause */ - case 295: /* key_opt */ - case 311: /* filter_clause */ + case 217: /* term */ + case 218: /* expr */ + case 247: /* where_opt */ + case 249: /* having_opt */ + case 269: /* where_opt_ret */ + case 280: /* case_operand */ + case 282: /* case_else */ + case 285: /* vinto */ + case 292: /* when_clause */ + case 297: /* key_opt */ + case 314: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy528)); +sqlite3ExprDelete(pParse->db, (yypminor->yy454)); } break; - case 221: /* eidlist_opt */ - case 231: /* sortlist */ - case 232: /* eidlist */ - case 244: /* selcollist */ - case 247: /* groupby_opt */ - case 249: /* orderby_opt */ - case 253: /* nexprlist */ - case 254: /* sclp */ - case 261: /* exprlist */ - case 268: /* setlist */ - case 277: /* paren_exprlist */ - case 279: /* case_exprlist */ - case 310: /* part_opt */ + case 222: /* eidlist_opt */ + case 232: /* sortlist */ + case 233: /* eidlist */ + case 245: /* selcollist */ + case 248: /* groupby_opt */ + case 250: /* orderby_opt */ + case 254: /* nexprlist */ + case 256: /* sclp */ + case 263: /* exprlist */ + case 270: /* setlist */ + case 279: /* paren_exprlist */ + case 281: /* case_exprlist */ + case 313: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy322)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 238: /* fullname */ - case 245: /* from */ - case 256: /* seltablist */ - case 257: /* stl_prefix */ - case 262: /* xfullname */ + case 239: /* fullname */ + case 246: /* from */ + case 258: /* seltablist */ + case 259: /* stl_prefix */ + case 264: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy131)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy203)); } break; - case 241: /* wqlist */ + case 242: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy521)); +sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; - case 251: /* window_clause */ - case 306: /* windowdefn_list */ + case 252: /* window_clause */ + case 309: /* windowdefn_list */ { -sqlite3WindowListDelete(pParse->db, (yypminor->yy41)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy211)); } break; - case 263: /* idlist */ - case 270: /* idlist_opt */ + case 265: /* idlist */ + case 272: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy254)); +sqlite3IdListDelete(pParse->db, (yypminor->yy132)); } break; - case 273: /* filter_over */ - case 307: /* windowdefn */ - case 308: /* window */ - case 309: /* frame_opt */ - case 312: /* over_clause */ + case 275: /* filter_over */ + case 310: /* windowdefn */ + case 311: /* window */ + case 312: /* frame_opt */ + case 315: /* over_clause */ { -sqlite3WindowDelete(pParse->db, (yypminor->yy41)); +sqlite3WindowDelete(pParse->db, (yypminor->yy211)); } break; - case 286: /* trigger_cmd_list */ - case 291: /* trigger_cmd */ + case 288: /* trigger_cmd_list */ + case 293: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy33)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy427)); } break; - case 288: /* trigger_event */ + case 290: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy180).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy286).b); } break; - case 314: /* frame_bound */ - case 315: /* frame_bound_s */ - case 316: /* frame_bound_e */ + case 317: /* frame_bound */ + case 318: /* frame_bound_s */ + case 319: /* frame_bound_e */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy595).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy509).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -173381,9 +174974,26 @@ static void yy_pop_parser_stack(yyParser *pParser){ */ SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); #endif } @@ -173566,7 +175176,7 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); /******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ sqlite3ParserCTX_STORE @@ -173610,25 +175220,19 @@ static void yy_shift( assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ if( yyGrowStack(yypParser) ){ yypParser->yytos--; yyStackOverflow(yypParser); return; } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); } -#endif if( yyNewState > YY_MAX_SHIFT ){ yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } - yytos = yypParser->yytos; yytos->stateno = yyNewState; yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; @@ -173638,411 +175242,415 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 189, /* (0) explain ::= EXPLAIN */ - 189, /* (1) explain ::= EXPLAIN QUERY PLAN */ - 188, /* (2) cmdx ::= cmd */ - 190, /* (3) cmd ::= BEGIN transtype trans_opt */ - 191, /* (4) transtype ::= */ - 191, /* (5) transtype ::= DEFERRED */ - 191, /* (6) transtype ::= IMMEDIATE */ - 191, /* (7) transtype ::= EXCLUSIVE */ - 190, /* (8) cmd ::= COMMIT|END trans_opt */ - 190, /* (9) cmd ::= ROLLBACK trans_opt */ - 190, /* (10) cmd ::= SAVEPOINT nm */ - 190, /* (11) cmd ::= RELEASE savepoint_opt nm */ - 190, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ - 195, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ - 197, /* (14) createkw ::= CREATE */ - 199, /* (15) ifnotexists ::= */ - 199, /* (16) ifnotexists ::= IF NOT EXISTS */ - 198, /* (17) temp ::= TEMP */ - 198, /* (18) temp ::= */ - 196, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ - 196, /* (20) create_table_args ::= AS select */ - 203, /* (21) table_option_set ::= */ - 203, /* (22) table_option_set ::= table_option_set COMMA table_option */ - 205, /* (23) table_option ::= WITHOUT nm */ - 205, /* (24) table_option ::= nm */ - 206, /* (25) columnname ::= nm typetoken */ - 208, /* (26) typetoken ::= */ - 208, /* (27) typetoken ::= typename LP signed RP */ - 208, /* (28) typetoken ::= typename LP signed COMMA signed RP */ - 209, /* (29) typename ::= typename ID|STRING */ - 213, /* (30) scanpt ::= */ - 214, /* (31) scantok ::= */ - 215, /* (32) ccons ::= CONSTRAINT nm */ - 215, /* (33) ccons ::= DEFAULT scantok term */ - 215, /* (34) ccons ::= DEFAULT LP expr RP */ - 215, /* (35) ccons ::= DEFAULT PLUS scantok term */ - 215, /* (36) ccons ::= DEFAULT MINUS scantok term */ - 215, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ - 215, /* (38) ccons ::= NOT NULL onconf */ - 215, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ - 215, /* (40) ccons ::= UNIQUE onconf */ - 215, /* (41) ccons ::= CHECK LP expr RP */ - 215, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ - 215, /* (43) ccons ::= defer_subclause */ - 215, /* (44) ccons ::= COLLATE ID|STRING */ - 224, /* (45) generated ::= LP expr RP */ - 224, /* (46) generated ::= LP expr RP ID */ - 220, /* (47) autoinc ::= */ - 220, /* (48) autoinc ::= AUTOINCR */ - 222, /* (49) refargs ::= */ - 222, /* (50) refargs ::= refargs refarg */ - 225, /* (51) refarg ::= MATCH nm */ - 225, /* (52) refarg ::= ON INSERT refact */ - 225, /* (53) refarg ::= ON DELETE refact */ - 225, /* (54) refarg ::= ON UPDATE refact */ - 226, /* (55) refact ::= SET NULL */ - 226, /* (56) refact ::= SET DEFAULT */ - 226, /* (57) refact ::= CASCADE */ - 226, /* (58) refact ::= RESTRICT */ - 226, /* (59) refact ::= NO ACTION */ - 223, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ - 223, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - 227, /* (62) init_deferred_pred_opt ::= */ - 227, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ - 227, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ - 202, /* (65) conslist_opt ::= */ - 229, /* (66) tconscomma ::= COMMA */ - 230, /* (67) tcons ::= CONSTRAINT nm */ - 230, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ - 230, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ - 230, /* (70) tcons ::= CHECK LP expr RP onconf */ - 230, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ - 233, /* (72) defer_subclause_opt ::= */ - 218, /* (73) onconf ::= */ - 218, /* (74) onconf ::= ON CONFLICT resolvetype */ - 234, /* (75) orconf ::= */ - 234, /* (76) orconf ::= OR resolvetype */ - 235, /* (77) resolvetype ::= IGNORE */ - 235, /* (78) resolvetype ::= REPLACE */ - 190, /* (79) cmd ::= DROP TABLE ifexists fullname */ - 237, /* (80) ifexists ::= IF EXISTS */ - 237, /* (81) ifexists ::= */ - 190, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ - 190, /* (83) cmd ::= DROP VIEW ifexists fullname */ - 190, /* (84) cmd ::= select */ - 204, /* (85) select ::= WITH wqlist selectnowith */ - 204, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ - 204, /* (87) select ::= selectnowith */ - 239, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ - 242, /* (89) multiselect_op ::= UNION */ - 242, /* (90) multiselect_op ::= UNION ALL */ - 242, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ - 240, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ - 240, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ - 252, /* (94) values ::= VALUES LP nexprlist RP */ - 252, /* (95) values ::= values COMMA LP nexprlist RP */ - 243, /* (96) distinct ::= DISTINCT */ - 243, /* (97) distinct ::= ALL */ - 243, /* (98) distinct ::= */ - 254, /* (99) sclp ::= */ - 244, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - 244, /* (101) selcollist ::= sclp scanpt STAR */ - 244, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - 255, /* (103) as ::= AS nm */ - 255, /* (104) as ::= */ - 245, /* (105) from ::= */ - 245, /* (106) from ::= FROM seltablist */ - 257, /* (107) stl_prefix ::= seltablist joinop */ - 257, /* (108) stl_prefix ::= */ - 256, /* (109) seltablist ::= stl_prefix nm dbnm as on_using */ - 256, /* (110) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ - 256, /* (111) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ - 256, /* (112) seltablist ::= stl_prefix LP select RP as on_using */ - 256, /* (113) seltablist ::= stl_prefix LP seltablist RP as on_using */ - 200, /* (114) dbnm ::= */ - 200, /* (115) dbnm ::= DOT nm */ - 238, /* (116) fullname ::= nm */ - 238, /* (117) fullname ::= nm DOT nm */ - 262, /* (118) xfullname ::= nm */ - 262, /* (119) xfullname ::= nm DOT nm */ - 262, /* (120) xfullname ::= nm DOT nm AS nm */ - 262, /* (121) xfullname ::= nm AS nm */ - 258, /* (122) joinop ::= COMMA|JOIN */ - 258, /* (123) joinop ::= JOIN_KW JOIN */ - 258, /* (124) joinop ::= JOIN_KW nm JOIN */ - 258, /* (125) joinop ::= JOIN_KW nm nm JOIN */ - 259, /* (126) on_using ::= ON expr */ - 259, /* (127) on_using ::= USING LP idlist RP */ - 259, /* (128) on_using ::= */ - 264, /* (129) indexed_opt ::= */ - 260, /* (130) indexed_by ::= INDEXED BY nm */ - 260, /* (131) indexed_by ::= NOT INDEXED */ - 249, /* (132) orderby_opt ::= */ - 249, /* (133) orderby_opt ::= ORDER BY sortlist */ - 231, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - 231, /* (135) sortlist ::= expr sortorder nulls */ - 219, /* (136) sortorder ::= ASC */ - 219, /* (137) sortorder ::= DESC */ - 219, /* (138) sortorder ::= */ - 265, /* (139) nulls ::= NULLS FIRST */ - 265, /* (140) nulls ::= NULLS LAST */ - 265, /* (141) nulls ::= */ - 247, /* (142) groupby_opt ::= */ - 247, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 248, /* (144) having_opt ::= */ - 248, /* (145) having_opt ::= HAVING expr */ - 250, /* (146) limit_opt ::= */ - 250, /* (147) limit_opt ::= LIMIT expr */ - 250, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - 250, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - 190, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 246, /* (151) where_opt ::= */ - 246, /* (152) where_opt ::= WHERE expr */ - 267, /* (153) where_opt_ret ::= */ - 267, /* (154) where_opt_ret ::= WHERE expr */ - 267, /* (155) where_opt_ret ::= RETURNING selcollist */ - 267, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - 190, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - 268, /* (158) setlist ::= setlist COMMA nm EQ expr */ - 268, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - 268, /* (160) setlist ::= nm EQ expr */ - 268, /* (161) setlist ::= LP idlist RP EQ expr */ - 190, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - 190, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 271, /* (164) upsert ::= */ - 271, /* (165) upsert ::= RETURNING selcollist */ - 271, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - 271, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - 271, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - 271, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - 272, /* (170) returning ::= RETURNING selcollist */ - 269, /* (171) insert_cmd ::= INSERT orconf */ - 269, /* (172) insert_cmd ::= REPLACE */ - 270, /* (173) idlist_opt ::= */ - 270, /* (174) idlist_opt ::= LP idlist RP */ - 263, /* (175) idlist ::= idlist COMMA nm */ - 263, /* (176) idlist ::= nm */ - 217, /* (177) expr ::= LP expr RP */ - 217, /* (178) expr ::= ID|INDEXED|JOIN_KW */ - 217, /* (179) expr ::= nm DOT nm */ - 217, /* (180) expr ::= nm DOT nm DOT nm */ - 216, /* (181) term ::= NULL|FLOAT|BLOB */ - 216, /* (182) term ::= STRING */ - 216, /* (183) term ::= INTEGER */ - 217, /* (184) expr ::= VARIABLE */ - 217, /* (185) expr ::= expr COLLATE ID|STRING */ - 217, /* (186) expr ::= CAST LP expr AS typetoken RP */ - 217, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ - 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - 217, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ - 217, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - 216, /* (193) term ::= CTIME_KW */ - 217, /* (194) expr ::= LP nexprlist COMMA expr RP */ - 217, /* (195) expr ::= expr AND expr */ - 217, /* (196) expr ::= expr OR expr */ - 217, /* (197) expr ::= expr LT|GT|GE|LE expr */ - 217, /* (198) expr ::= expr EQ|NE expr */ - 217, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 217, /* (200) expr ::= expr PLUS|MINUS expr */ - 217, /* (201) expr ::= expr STAR|SLASH|REM expr */ - 217, /* (202) expr ::= expr CONCAT expr */ - 274, /* (203) likeop ::= NOT LIKE_KW|MATCH */ - 217, /* (204) expr ::= expr likeop expr */ - 217, /* (205) expr ::= expr likeop expr ESCAPE expr */ - 217, /* (206) expr ::= expr ISNULL|NOTNULL */ - 217, /* (207) expr ::= expr NOT NULL */ - 217, /* (208) expr ::= expr IS expr */ - 217, /* (209) expr ::= expr IS NOT expr */ - 217, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ - 217, /* (211) expr ::= expr IS DISTINCT FROM expr */ - 217, /* (212) expr ::= NOT expr */ - 217, /* (213) expr ::= BITNOT expr */ - 217, /* (214) expr ::= PLUS|MINUS expr */ - 217, /* (215) expr ::= expr PTR expr */ - 275, /* (216) between_op ::= BETWEEN */ - 275, /* (217) between_op ::= NOT BETWEEN */ - 217, /* (218) expr ::= expr between_op expr AND expr */ - 276, /* (219) in_op ::= IN */ - 276, /* (220) in_op ::= NOT IN */ - 217, /* (221) expr ::= expr in_op LP exprlist RP */ - 217, /* (222) expr ::= LP select RP */ - 217, /* (223) expr ::= expr in_op LP select RP */ - 217, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ - 217, /* (225) expr ::= EXISTS LP select RP */ - 217, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ - 279, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 279, /* (228) case_exprlist ::= WHEN expr THEN expr */ - 280, /* (229) case_else ::= ELSE expr */ - 280, /* (230) case_else ::= */ - 278, /* (231) case_operand ::= */ - 261, /* (232) exprlist ::= */ - 253, /* (233) nexprlist ::= nexprlist COMMA expr */ - 253, /* (234) nexprlist ::= expr */ - 277, /* (235) paren_exprlist ::= */ - 277, /* (236) paren_exprlist ::= LP exprlist RP */ - 190, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 281, /* (238) uniqueflag ::= UNIQUE */ - 281, /* (239) uniqueflag ::= */ - 221, /* (240) eidlist_opt ::= */ - 221, /* (241) eidlist_opt ::= LP eidlist RP */ - 232, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - 232, /* (243) eidlist ::= nm collate sortorder */ - 282, /* (244) collate ::= */ - 282, /* (245) collate ::= COLLATE ID|STRING */ - 190, /* (246) cmd ::= DROP INDEX ifexists fullname */ - 190, /* (247) cmd ::= VACUUM vinto */ - 190, /* (248) cmd ::= VACUUM nm vinto */ - 283, /* (249) vinto ::= INTO expr */ - 283, /* (250) vinto ::= */ - 190, /* (251) cmd ::= PRAGMA nm dbnm */ - 190, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 190, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 190, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 190, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 211, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - 212, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - 190, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 285, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 287, /* (260) trigger_time ::= BEFORE|AFTER */ - 287, /* (261) trigger_time ::= INSTEAD OF */ - 287, /* (262) trigger_time ::= */ - 288, /* (263) trigger_event ::= DELETE|INSERT */ - 288, /* (264) trigger_event ::= UPDATE */ - 288, /* (265) trigger_event ::= UPDATE OF idlist */ - 290, /* (266) when_clause ::= */ - 290, /* (267) when_clause ::= WHEN expr */ - 286, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 286, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - 292, /* (270) trnm ::= nm DOT nm */ - 293, /* (271) tridxby ::= INDEXED BY nm */ - 293, /* (272) tridxby ::= NOT INDEXED */ - 291, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 291, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 291, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 291, /* (276) trigger_cmd ::= scanpt select scanpt */ - 217, /* (277) expr ::= RAISE LP IGNORE RP */ - 217, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - 236, /* (279) raisetype ::= ROLLBACK */ - 236, /* (280) raisetype ::= ABORT */ - 236, /* (281) raisetype ::= FAIL */ - 190, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - 190, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 190, /* (284) cmd ::= DETACH database_kw_opt expr */ - 295, /* (285) key_opt ::= */ - 295, /* (286) key_opt ::= KEY expr */ - 190, /* (287) cmd ::= REINDEX */ - 190, /* (288) cmd ::= REINDEX nm dbnm */ - 190, /* (289) cmd ::= ANALYZE */ - 190, /* (290) cmd ::= ANALYZE nm dbnm */ - 190, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 190, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 190, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 296, /* (294) add_column_fullname ::= fullname */ - 190, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 190, /* (296) cmd ::= create_vtab */ - 190, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - 298, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 300, /* (299) vtabarg ::= */ - 301, /* (300) vtabargtoken ::= ANY */ - 301, /* (301) vtabargtoken ::= lp anylist RP */ - 302, /* (302) lp ::= LP */ - 266, /* (303) with ::= WITH wqlist */ - 266, /* (304) with ::= WITH RECURSIVE wqlist */ - 305, /* (305) wqas ::= AS */ - 305, /* (306) wqas ::= AS MATERIALIZED */ - 305, /* (307) wqas ::= AS NOT MATERIALIZED */ - 304, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - 241, /* (309) wqlist ::= wqitem */ - 241, /* (310) wqlist ::= wqlist COMMA wqitem */ - 306, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 307, /* (312) windowdefn ::= nm AS LP window RP */ - 308, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (315) window ::= ORDER BY sortlist frame_opt */ - 308, /* (316) window ::= nm ORDER BY sortlist frame_opt */ - 308, /* (317) window ::= nm frame_opt */ - 309, /* (318) frame_opt ::= */ - 309, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 309, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 313, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ - 315, /* (322) frame_bound_s ::= frame_bound */ - 315, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ - 316, /* (324) frame_bound_e ::= frame_bound */ - 316, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 314, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ - 314, /* (327) frame_bound ::= CURRENT ROW */ - 317, /* (328) frame_exclude_opt ::= */ - 317, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 318, /* (330) frame_exclude ::= NO OTHERS */ - 318, /* (331) frame_exclude ::= CURRENT ROW */ - 318, /* (332) frame_exclude ::= GROUP|TIES */ - 251, /* (333) window_clause ::= WINDOW windowdefn_list */ - 273, /* (334) filter_over ::= filter_clause over_clause */ - 273, /* (335) filter_over ::= over_clause */ - 273, /* (336) filter_over ::= filter_clause */ - 312, /* (337) over_clause ::= OVER LP window RP */ - 312, /* (338) over_clause ::= OVER nm */ - 311, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ - 185, /* (340) input ::= cmdlist */ - 186, /* (341) cmdlist ::= cmdlist ecmd */ - 186, /* (342) cmdlist ::= ecmd */ - 187, /* (343) ecmd ::= SEMI */ - 187, /* (344) ecmd ::= cmdx SEMI */ - 187, /* (345) ecmd ::= explain cmdx SEMI */ - 192, /* (346) trans_opt ::= */ - 192, /* (347) trans_opt ::= TRANSACTION */ - 192, /* (348) trans_opt ::= TRANSACTION nm */ - 194, /* (349) savepoint_opt ::= SAVEPOINT */ - 194, /* (350) savepoint_opt ::= */ - 190, /* (351) cmd ::= create_table create_table_args */ - 203, /* (352) table_option_set ::= table_option */ - 201, /* (353) columnlist ::= columnlist COMMA columnname carglist */ - 201, /* (354) columnlist ::= columnname carglist */ - 193, /* (355) nm ::= ID|INDEXED|JOIN_KW */ - 193, /* (356) nm ::= STRING */ - 208, /* (357) typetoken ::= typename */ - 209, /* (358) typename ::= ID|STRING */ - 210, /* (359) signed ::= plus_num */ - 210, /* (360) signed ::= minus_num */ - 207, /* (361) carglist ::= carglist ccons */ - 207, /* (362) carglist ::= */ - 215, /* (363) ccons ::= NULL onconf */ - 215, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - 215, /* (365) ccons ::= AS generated */ - 202, /* (366) conslist_opt ::= COMMA conslist */ - 228, /* (367) conslist ::= conslist tconscomma tcons */ - 228, /* (368) conslist ::= tcons */ - 229, /* (369) tconscomma ::= */ - 233, /* (370) defer_subclause_opt ::= defer_subclause */ - 235, /* (371) resolvetype ::= raisetype */ - 239, /* (372) selectnowith ::= oneselect */ - 240, /* (373) oneselect ::= values */ - 254, /* (374) sclp ::= selcollist COMMA */ - 255, /* (375) as ::= ID|STRING */ - 264, /* (376) indexed_opt ::= indexed_by */ - 272, /* (377) returning ::= */ - 217, /* (378) expr ::= term */ - 274, /* (379) likeop ::= LIKE_KW|MATCH */ - 278, /* (380) case_operand ::= expr */ - 261, /* (381) exprlist ::= nexprlist */ - 284, /* (382) nmnum ::= plus_num */ - 284, /* (383) nmnum ::= nm */ - 284, /* (384) nmnum ::= ON */ - 284, /* (385) nmnum ::= DELETE */ - 284, /* (386) nmnum ::= DEFAULT */ - 211, /* (387) plus_num ::= INTEGER|FLOAT */ - 289, /* (388) foreach_clause ::= */ - 289, /* (389) foreach_clause ::= FOR EACH ROW */ - 292, /* (390) trnm ::= nm */ - 293, /* (391) tridxby ::= */ - 294, /* (392) database_kw_opt ::= DATABASE */ - 294, /* (393) database_kw_opt ::= */ - 297, /* (394) kwcolumn_opt ::= */ - 297, /* (395) kwcolumn_opt ::= COLUMNKW */ - 299, /* (396) vtabarglist ::= vtabarg */ - 299, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ - 300, /* (398) vtabarg ::= vtabarg vtabargtoken */ - 303, /* (399) anylist ::= */ - 303, /* (400) anylist ::= anylist LP anylist RP */ - 303, /* (401) anylist ::= anylist ANY */ - 266, /* (402) with ::= */ - 306, /* (403) windowdefn_list ::= windowdefn */ - 308, /* (404) window ::= frame_opt */ + 190, /* (0) explain ::= EXPLAIN */ + 190, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 189, /* (2) cmdx ::= cmd */ + 191, /* (3) cmd ::= BEGIN transtype trans_opt */ + 192, /* (4) transtype ::= */ + 192, /* (5) transtype ::= DEFERRED */ + 192, /* (6) transtype ::= IMMEDIATE */ + 192, /* (7) transtype ::= EXCLUSIVE */ + 191, /* (8) cmd ::= COMMIT|END trans_opt */ + 191, /* (9) cmd ::= ROLLBACK trans_opt */ + 191, /* (10) cmd ::= SAVEPOINT nm */ + 191, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 191, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 196, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 198, /* (14) createkw ::= CREATE */ + 200, /* (15) ifnotexists ::= */ + 200, /* (16) ifnotexists ::= IF NOT EXISTS */ + 199, /* (17) temp ::= TEMP */ + 199, /* (18) temp ::= */ + 197, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + 197, /* (20) create_table_args ::= AS select */ + 204, /* (21) table_option_set ::= */ + 204, /* (22) table_option_set ::= table_option_set COMMA table_option */ + 206, /* (23) table_option ::= WITHOUT nm */ + 206, /* (24) table_option ::= nm */ + 207, /* (25) columnname ::= nm typetoken */ + 209, /* (26) typetoken ::= */ + 209, /* (27) typetoken ::= typename LP signed RP */ + 209, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + 210, /* (29) typename ::= typename ID|STRING */ + 214, /* (30) scanpt ::= */ + 215, /* (31) scantok ::= */ + 216, /* (32) ccons ::= CONSTRAINT nm */ + 216, /* (33) ccons ::= DEFAULT scantok term */ + 216, /* (34) ccons ::= DEFAULT LP expr RP */ + 216, /* (35) ccons ::= DEFAULT PLUS scantok term */ + 216, /* (36) ccons ::= DEFAULT MINUS scantok term */ + 216, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + 216, /* (38) ccons ::= NOT NULL onconf */ + 216, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 216, /* (40) ccons ::= UNIQUE onconf */ + 216, /* (41) ccons ::= CHECK LP expr RP */ + 216, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + 216, /* (43) ccons ::= defer_subclause */ + 216, /* (44) ccons ::= COLLATE ID|STRING */ + 225, /* (45) generated ::= LP expr RP */ + 225, /* (46) generated ::= LP expr RP ID */ + 221, /* (47) autoinc ::= */ + 221, /* (48) autoinc ::= AUTOINCR */ + 223, /* (49) refargs ::= */ + 223, /* (50) refargs ::= refargs refarg */ + 226, /* (51) refarg ::= MATCH nm */ + 226, /* (52) refarg ::= ON INSERT refact */ + 226, /* (53) refarg ::= ON DELETE refact */ + 226, /* (54) refarg ::= ON UPDATE refact */ + 227, /* (55) refact ::= SET NULL */ + 227, /* (56) refact ::= SET DEFAULT */ + 227, /* (57) refact ::= CASCADE */ + 227, /* (58) refact ::= RESTRICT */ + 227, /* (59) refact ::= NO ACTION */ + 224, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 224, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 228, /* (62) init_deferred_pred_opt ::= */ + 228, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 228, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 203, /* (65) conslist_opt ::= */ + 230, /* (66) tconscomma ::= COMMA */ + 231, /* (67) tcons ::= CONSTRAINT nm */ + 231, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 231, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + 231, /* (70) tcons ::= CHECK LP expr RP onconf */ + 231, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 234, /* (72) defer_subclause_opt ::= */ + 219, /* (73) onconf ::= */ + 219, /* (74) onconf ::= ON CONFLICT resolvetype */ + 235, /* (75) orconf ::= */ + 235, /* (76) orconf ::= OR resolvetype */ + 236, /* (77) resolvetype ::= IGNORE */ + 236, /* (78) resolvetype ::= REPLACE */ + 191, /* (79) cmd ::= DROP TABLE ifexists fullname */ + 238, /* (80) ifexists ::= IF EXISTS */ + 238, /* (81) ifexists ::= */ + 191, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 191, /* (83) cmd ::= DROP VIEW ifexists fullname */ + 191, /* (84) cmd ::= select */ + 205, /* (85) select ::= WITH wqlist selectnowith */ + 205, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + 205, /* (87) select ::= selectnowith */ + 240, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + 243, /* (89) multiselect_op ::= UNION */ + 243, /* (90) multiselect_op ::= UNION ALL */ + 243, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + 241, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 241, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 253, /* (94) values ::= VALUES LP nexprlist RP */ + 241, /* (95) oneselect ::= mvalues */ + 255, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + 255, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + 244, /* (98) distinct ::= DISTINCT */ + 244, /* (99) distinct ::= ALL */ + 244, /* (100) distinct ::= */ + 256, /* (101) sclp ::= */ + 245, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + 245, /* (103) selcollist ::= sclp scanpt STAR */ + 245, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + 257, /* (105) as ::= AS nm */ + 257, /* (106) as ::= */ + 246, /* (107) from ::= */ + 246, /* (108) from ::= FROM seltablist */ + 259, /* (109) stl_prefix ::= seltablist joinop */ + 259, /* (110) stl_prefix ::= */ + 258, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + 258, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + 258, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + 258, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + 258, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 201, /* (116) dbnm ::= */ + 201, /* (117) dbnm ::= DOT nm */ + 239, /* (118) fullname ::= nm */ + 239, /* (119) fullname ::= nm DOT nm */ + 264, /* (120) xfullname ::= nm */ + 264, /* (121) xfullname ::= nm DOT nm */ + 264, /* (122) xfullname ::= nm DOT nm AS nm */ + 264, /* (123) xfullname ::= nm AS nm */ + 260, /* (124) joinop ::= COMMA|JOIN */ + 260, /* (125) joinop ::= JOIN_KW JOIN */ + 260, /* (126) joinop ::= JOIN_KW nm JOIN */ + 260, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + 261, /* (128) on_using ::= ON expr */ + 261, /* (129) on_using ::= USING LP idlist RP */ + 261, /* (130) on_using ::= */ + 266, /* (131) indexed_opt ::= */ + 262, /* (132) indexed_by ::= INDEXED BY nm */ + 262, /* (133) indexed_by ::= NOT INDEXED */ + 250, /* (134) orderby_opt ::= */ + 250, /* (135) orderby_opt ::= ORDER BY sortlist */ + 232, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + 232, /* (137) sortlist ::= expr sortorder nulls */ + 220, /* (138) sortorder ::= ASC */ + 220, /* (139) sortorder ::= DESC */ + 220, /* (140) sortorder ::= */ + 267, /* (141) nulls ::= NULLS FIRST */ + 267, /* (142) nulls ::= NULLS LAST */ + 267, /* (143) nulls ::= */ + 248, /* (144) groupby_opt ::= */ + 248, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 249, /* (146) having_opt ::= */ + 249, /* (147) having_opt ::= HAVING expr */ + 251, /* (148) limit_opt ::= */ + 251, /* (149) limit_opt ::= LIMIT expr */ + 251, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + 251, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + 191, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 247, /* (153) where_opt ::= */ + 247, /* (154) where_opt ::= WHERE expr */ + 269, /* (155) where_opt_ret ::= */ + 269, /* (156) where_opt_ret ::= WHERE expr */ + 269, /* (157) where_opt_ret ::= RETURNING selcollist */ + 269, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 191, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 270, /* (160) setlist ::= setlist COMMA nm EQ expr */ + 270, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 270, /* (162) setlist ::= nm EQ expr */ + 270, /* (163) setlist ::= LP idlist RP EQ expr */ + 191, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 191, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 273, /* (166) upsert ::= */ + 273, /* (167) upsert ::= RETURNING selcollist */ + 273, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 273, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 273, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + 273, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 274, /* (172) returning ::= RETURNING selcollist */ + 271, /* (173) insert_cmd ::= INSERT orconf */ + 271, /* (174) insert_cmd ::= REPLACE */ + 272, /* (175) idlist_opt ::= */ + 272, /* (176) idlist_opt ::= LP idlist RP */ + 265, /* (177) idlist ::= idlist COMMA nm */ + 265, /* (178) idlist ::= nm */ + 218, /* (179) expr ::= LP expr RP */ + 218, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + 218, /* (181) expr ::= nm DOT nm */ + 218, /* (182) expr ::= nm DOT nm DOT nm */ + 217, /* (183) term ::= NULL|FLOAT|BLOB */ + 217, /* (184) term ::= STRING */ + 217, /* (185) term ::= INTEGER */ + 218, /* (186) expr ::= VARIABLE */ + 218, /* (187) expr ::= expr COLLATE ID|STRING */ + 218, /* (188) expr ::= CAST LP expr AS typetoken RP */ + 218, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 218, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + 218, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 218, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 218, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + 218, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 217, /* (195) term ::= CTIME_KW */ + 218, /* (196) expr ::= LP nexprlist COMMA expr RP */ + 218, /* (197) expr ::= expr AND expr */ + 218, /* (198) expr ::= expr OR expr */ + 218, /* (199) expr ::= expr LT|GT|GE|LE expr */ + 218, /* (200) expr ::= expr EQ|NE expr */ + 218, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 218, /* (202) expr ::= expr PLUS|MINUS expr */ + 218, /* (203) expr ::= expr STAR|SLASH|REM expr */ + 218, /* (204) expr ::= expr CONCAT expr */ + 276, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + 218, /* (206) expr ::= expr likeop expr */ + 218, /* (207) expr ::= expr likeop expr ESCAPE expr */ + 218, /* (208) expr ::= expr ISNULL|NOTNULL */ + 218, /* (209) expr ::= expr NOT NULL */ + 218, /* (210) expr ::= expr IS expr */ + 218, /* (211) expr ::= expr IS NOT expr */ + 218, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + 218, /* (213) expr ::= expr IS DISTINCT FROM expr */ + 218, /* (214) expr ::= NOT expr */ + 218, /* (215) expr ::= BITNOT expr */ + 218, /* (216) expr ::= PLUS|MINUS expr */ + 218, /* (217) expr ::= expr PTR expr */ + 277, /* (218) between_op ::= BETWEEN */ + 277, /* (219) between_op ::= NOT BETWEEN */ + 218, /* (220) expr ::= expr between_op expr AND expr */ + 278, /* (221) in_op ::= IN */ + 278, /* (222) in_op ::= NOT IN */ + 218, /* (223) expr ::= expr in_op LP exprlist RP */ + 218, /* (224) expr ::= LP select RP */ + 218, /* (225) expr ::= expr in_op LP select RP */ + 218, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + 218, /* (227) expr ::= EXISTS LP select RP */ + 218, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + 281, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 281, /* (230) case_exprlist ::= WHEN expr THEN expr */ + 282, /* (231) case_else ::= ELSE expr */ + 282, /* (232) case_else ::= */ + 280, /* (233) case_operand ::= */ + 263, /* (234) exprlist ::= */ + 254, /* (235) nexprlist ::= nexprlist COMMA expr */ + 254, /* (236) nexprlist ::= expr */ + 279, /* (237) paren_exprlist ::= */ + 279, /* (238) paren_exprlist ::= LP exprlist RP */ + 191, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 283, /* (240) uniqueflag ::= UNIQUE */ + 283, /* (241) uniqueflag ::= */ + 222, /* (242) eidlist_opt ::= */ + 222, /* (243) eidlist_opt ::= LP eidlist RP */ + 233, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + 233, /* (245) eidlist ::= nm collate sortorder */ + 284, /* (246) collate ::= */ + 284, /* (247) collate ::= COLLATE ID|STRING */ + 191, /* (248) cmd ::= DROP INDEX ifexists fullname */ + 191, /* (249) cmd ::= VACUUM vinto */ + 191, /* (250) cmd ::= VACUUM nm vinto */ + 285, /* (251) vinto ::= INTO expr */ + 285, /* (252) vinto ::= */ + 191, /* (253) cmd ::= PRAGMA nm dbnm */ + 191, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 191, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 191, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 191, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 212, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + 213, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + 191, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 287, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 289, /* (262) trigger_time ::= BEFORE|AFTER */ + 289, /* (263) trigger_time ::= INSTEAD OF */ + 289, /* (264) trigger_time ::= */ + 290, /* (265) trigger_event ::= DELETE|INSERT */ + 290, /* (266) trigger_event ::= UPDATE */ + 290, /* (267) trigger_event ::= UPDATE OF idlist */ + 292, /* (268) when_clause ::= */ + 292, /* (269) when_clause ::= WHEN expr */ + 288, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 288, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + 294, /* (272) trnm ::= nm DOT nm */ + 295, /* (273) tridxby ::= INDEXED BY nm */ + 295, /* (274) tridxby ::= NOT INDEXED */ + 293, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 293, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 293, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 293, /* (278) trigger_cmd ::= scanpt select scanpt */ + 218, /* (279) expr ::= RAISE LP IGNORE RP */ + 218, /* (280) expr ::= RAISE LP raisetype COMMA nm RP */ + 237, /* (281) raisetype ::= ROLLBACK */ + 237, /* (282) raisetype ::= ABORT */ + 237, /* (283) raisetype ::= FAIL */ + 191, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + 191, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 191, /* (286) cmd ::= DETACH database_kw_opt expr */ + 297, /* (287) key_opt ::= */ + 297, /* (288) key_opt ::= KEY expr */ + 191, /* (289) cmd ::= REINDEX */ + 191, /* (290) cmd ::= REINDEX nm dbnm */ + 191, /* (291) cmd ::= ANALYZE */ + 191, /* (292) cmd ::= ANALYZE nm dbnm */ + 191, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 191, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 191, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 298, /* (296) add_column_fullname ::= fullname */ + 191, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 191, /* (298) cmd ::= create_vtab */ + 191, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + 300, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 302, /* (301) vtabarg ::= */ + 303, /* (302) vtabargtoken ::= ANY */ + 303, /* (303) vtabargtoken ::= lp anylist RP */ + 304, /* (304) lp ::= LP */ + 268, /* (305) with ::= WITH wqlist */ + 268, /* (306) with ::= WITH RECURSIVE wqlist */ + 307, /* (307) wqas ::= AS */ + 307, /* (308) wqas ::= AS MATERIALIZED */ + 307, /* (309) wqas ::= AS NOT MATERIALIZED */ + 306, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + 308, /* (311) withnm ::= nm */ + 242, /* (312) wqlist ::= wqitem */ + 242, /* (313) wqlist ::= wqlist COMMA wqitem */ + 309, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 310, /* (315) windowdefn ::= nm AS LP window RP */ + 311, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (318) window ::= ORDER BY sortlist frame_opt */ + 311, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + 311, /* (320) window ::= nm frame_opt */ + 312, /* (321) frame_opt ::= */ + 312, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 312, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 316, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + 318, /* (325) frame_bound_s ::= frame_bound */ + 318, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + 319, /* (327) frame_bound_e ::= frame_bound */ + 319, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 317, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + 317, /* (330) frame_bound ::= CURRENT ROW */ + 320, /* (331) frame_exclude_opt ::= */ + 320, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 321, /* (333) frame_exclude ::= NO OTHERS */ + 321, /* (334) frame_exclude ::= CURRENT ROW */ + 321, /* (335) frame_exclude ::= GROUP|TIES */ + 252, /* (336) window_clause ::= WINDOW windowdefn_list */ + 275, /* (337) filter_over ::= filter_clause over_clause */ + 275, /* (338) filter_over ::= over_clause */ + 275, /* (339) filter_over ::= filter_clause */ + 315, /* (340) over_clause ::= OVER LP window RP */ + 315, /* (341) over_clause ::= OVER nm */ + 314, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + 217, /* (343) term ::= QNUMBER */ + 186, /* (344) input ::= cmdlist */ + 187, /* (345) cmdlist ::= cmdlist ecmd */ + 187, /* (346) cmdlist ::= ecmd */ + 188, /* (347) ecmd ::= SEMI */ + 188, /* (348) ecmd ::= cmdx SEMI */ + 188, /* (349) ecmd ::= explain cmdx SEMI */ + 193, /* (350) trans_opt ::= */ + 193, /* (351) trans_opt ::= TRANSACTION */ + 193, /* (352) trans_opt ::= TRANSACTION nm */ + 195, /* (353) savepoint_opt ::= SAVEPOINT */ + 195, /* (354) savepoint_opt ::= */ + 191, /* (355) cmd ::= create_table create_table_args */ + 204, /* (356) table_option_set ::= table_option */ + 202, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + 202, /* (358) columnlist ::= columnname carglist */ + 194, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + 194, /* (360) nm ::= STRING */ + 209, /* (361) typetoken ::= typename */ + 210, /* (362) typename ::= ID|STRING */ + 211, /* (363) signed ::= plus_num */ + 211, /* (364) signed ::= minus_num */ + 208, /* (365) carglist ::= carglist ccons */ + 208, /* (366) carglist ::= */ + 216, /* (367) ccons ::= NULL onconf */ + 216, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + 216, /* (369) ccons ::= AS generated */ + 203, /* (370) conslist_opt ::= COMMA conslist */ + 229, /* (371) conslist ::= conslist tconscomma tcons */ + 229, /* (372) conslist ::= tcons */ + 230, /* (373) tconscomma ::= */ + 234, /* (374) defer_subclause_opt ::= defer_subclause */ + 236, /* (375) resolvetype ::= raisetype */ + 240, /* (376) selectnowith ::= oneselect */ + 241, /* (377) oneselect ::= values */ + 256, /* (378) sclp ::= selcollist COMMA */ + 257, /* (379) as ::= ID|STRING */ + 266, /* (380) indexed_opt ::= indexed_by */ + 274, /* (381) returning ::= */ + 218, /* (382) expr ::= term */ + 276, /* (383) likeop ::= LIKE_KW|MATCH */ + 280, /* (384) case_operand ::= expr */ + 263, /* (385) exprlist ::= nexprlist */ + 286, /* (386) nmnum ::= plus_num */ + 286, /* (387) nmnum ::= nm */ + 286, /* (388) nmnum ::= ON */ + 286, /* (389) nmnum ::= DELETE */ + 286, /* (390) nmnum ::= DEFAULT */ + 212, /* (391) plus_num ::= INTEGER|FLOAT */ + 291, /* (392) foreach_clause ::= */ + 291, /* (393) foreach_clause ::= FOR EACH ROW */ + 294, /* (394) trnm ::= nm */ + 295, /* (395) tridxby ::= */ + 296, /* (396) database_kw_opt ::= DATABASE */ + 296, /* (397) database_kw_opt ::= */ + 299, /* (398) kwcolumn_opt ::= */ + 299, /* (399) kwcolumn_opt ::= COLUMNKW */ + 301, /* (400) vtabarglist ::= vtabarg */ + 301, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + 302, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 305, /* (403) anylist ::= */ + 305, /* (404) anylist ::= anylist LP anylist RP */ + 305, /* (405) anylist ::= anylist ANY */ + 268, /* (406) with ::= */ + 309, /* (407) windowdefn_list ::= windowdefn */ + 311, /* (408) window ::= frame_opt */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -174143,316 +175751,320 @@ static const signed char yyRuleInfoNRhs[] = { -9, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -10, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ -4, /* (94) values ::= VALUES LP nexprlist RP */ - -5, /* (95) values ::= values COMMA LP nexprlist RP */ - -1, /* (96) distinct ::= DISTINCT */ - -1, /* (97) distinct ::= ALL */ - 0, /* (98) distinct ::= */ - 0, /* (99) sclp ::= */ - -5, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - -3, /* (101) selcollist ::= sclp scanpt STAR */ - -5, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - -2, /* (103) as ::= AS nm */ - 0, /* (104) as ::= */ - 0, /* (105) from ::= */ - -2, /* (106) from ::= FROM seltablist */ - -2, /* (107) stl_prefix ::= seltablist joinop */ - 0, /* (108) stl_prefix ::= */ - -5, /* (109) seltablist ::= stl_prefix nm dbnm as on_using */ - -6, /* (110) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ - -8, /* (111) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ - -6, /* (112) seltablist ::= stl_prefix LP select RP as on_using */ - -6, /* (113) seltablist ::= stl_prefix LP seltablist RP as on_using */ - 0, /* (114) dbnm ::= */ - -2, /* (115) dbnm ::= DOT nm */ - -1, /* (116) fullname ::= nm */ - -3, /* (117) fullname ::= nm DOT nm */ - -1, /* (118) xfullname ::= nm */ - -3, /* (119) xfullname ::= nm DOT nm */ - -5, /* (120) xfullname ::= nm DOT nm AS nm */ - -3, /* (121) xfullname ::= nm AS nm */ - -1, /* (122) joinop ::= COMMA|JOIN */ - -2, /* (123) joinop ::= JOIN_KW JOIN */ - -3, /* (124) joinop ::= JOIN_KW nm JOIN */ - -4, /* (125) joinop ::= JOIN_KW nm nm JOIN */ - -2, /* (126) on_using ::= ON expr */ - -4, /* (127) on_using ::= USING LP idlist RP */ - 0, /* (128) on_using ::= */ - 0, /* (129) indexed_opt ::= */ - -3, /* (130) indexed_by ::= INDEXED BY nm */ - -2, /* (131) indexed_by ::= NOT INDEXED */ - 0, /* (132) orderby_opt ::= */ - -3, /* (133) orderby_opt ::= ORDER BY sortlist */ - -5, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - -3, /* (135) sortlist ::= expr sortorder nulls */ - -1, /* (136) sortorder ::= ASC */ - -1, /* (137) sortorder ::= DESC */ - 0, /* (138) sortorder ::= */ - -2, /* (139) nulls ::= NULLS FIRST */ - -2, /* (140) nulls ::= NULLS LAST */ - 0, /* (141) nulls ::= */ - 0, /* (142) groupby_opt ::= */ - -3, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 0, /* (144) having_opt ::= */ - -2, /* (145) having_opt ::= HAVING expr */ - 0, /* (146) limit_opt ::= */ - -2, /* (147) limit_opt ::= LIMIT expr */ - -4, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - -4, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - -6, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 0, /* (151) where_opt ::= */ - -2, /* (152) where_opt ::= WHERE expr */ - 0, /* (153) where_opt_ret ::= */ - -2, /* (154) where_opt_ret ::= WHERE expr */ - -2, /* (155) where_opt_ret ::= RETURNING selcollist */ - -4, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - -9, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - -5, /* (158) setlist ::= setlist COMMA nm EQ expr */ - -7, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - -3, /* (160) setlist ::= nm EQ expr */ - -5, /* (161) setlist ::= LP idlist RP EQ expr */ - -7, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - -8, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 0, /* (164) upsert ::= */ - -2, /* (165) upsert ::= RETURNING selcollist */ - -12, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - -9, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - -5, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - -8, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - -2, /* (170) returning ::= RETURNING selcollist */ - -2, /* (171) insert_cmd ::= INSERT orconf */ - -1, /* (172) insert_cmd ::= REPLACE */ - 0, /* (173) idlist_opt ::= */ - -3, /* (174) idlist_opt ::= LP idlist RP */ - -3, /* (175) idlist ::= idlist COMMA nm */ - -1, /* (176) idlist ::= nm */ - -3, /* (177) expr ::= LP expr RP */ - -1, /* (178) expr ::= ID|INDEXED|JOIN_KW */ - -3, /* (179) expr ::= nm DOT nm */ - -5, /* (180) expr ::= nm DOT nm DOT nm */ - -1, /* (181) term ::= NULL|FLOAT|BLOB */ - -1, /* (182) term ::= STRING */ - -1, /* (183) term ::= INTEGER */ - -1, /* (184) expr ::= VARIABLE */ - -3, /* (185) expr ::= expr COLLATE ID|STRING */ - -6, /* (186) expr ::= CAST LP expr AS typetoken RP */ - -5, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - -8, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ - -4, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - -6, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - -9, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ - -5, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - -1, /* (193) term ::= CTIME_KW */ - -5, /* (194) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (195) expr ::= expr AND expr */ - -3, /* (196) expr ::= expr OR expr */ - -3, /* (197) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (198) expr ::= expr EQ|NE expr */ - -3, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (200) expr ::= expr PLUS|MINUS expr */ - -3, /* (201) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (202) expr ::= expr CONCAT expr */ - -2, /* (203) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (204) expr ::= expr likeop expr */ - -5, /* (205) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (206) expr ::= expr ISNULL|NOTNULL */ - -3, /* (207) expr ::= expr NOT NULL */ - -3, /* (208) expr ::= expr IS expr */ - -4, /* (209) expr ::= expr IS NOT expr */ - -6, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ - -5, /* (211) expr ::= expr IS DISTINCT FROM expr */ - -2, /* (212) expr ::= NOT expr */ - -2, /* (213) expr ::= BITNOT expr */ - -2, /* (214) expr ::= PLUS|MINUS expr */ - -3, /* (215) expr ::= expr PTR expr */ - -1, /* (216) between_op ::= BETWEEN */ - -2, /* (217) between_op ::= NOT BETWEEN */ - -5, /* (218) expr ::= expr between_op expr AND expr */ - -1, /* (219) in_op ::= IN */ - -2, /* (220) in_op ::= NOT IN */ - -5, /* (221) expr ::= expr in_op LP exprlist RP */ - -3, /* (222) expr ::= LP select RP */ - -5, /* (223) expr ::= expr in_op LP select RP */ - -5, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (225) expr ::= EXISTS LP select RP */ - -5, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (228) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (229) case_else ::= ELSE expr */ - 0, /* (230) case_else ::= */ - 0, /* (231) case_operand ::= */ - 0, /* (232) exprlist ::= */ - -3, /* (233) nexprlist ::= nexprlist COMMA expr */ - -1, /* (234) nexprlist ::= expr */ - 0, /* (235) paren_exprlist ::= */ - -3, /* (236) paren_exprlist ::= LP exprlist RP */ - -12, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (238) uniqueflag ::= UNIQUE */ - 0, /* (239) uniqueflag ::= */ - 0, /* (240) eidlist_opt ::= */ - -3, /* (241) eidlist_opt ::= LP eidlist RP */ - -5, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (243) eidlist ::= nm collate sortorder */ - 0, /* (244) collate ::= */ - -2, /* (245) collate ::= COLLATE ID|STRING */ - -4, /* (246) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (247) cmd ::= VACUUM vinto */ - -3, /* (248) cmd ::= VACUUM nm vinto */ - -2, /* (249) vinto ::= INTO expr */ - 0, /* (250) vinto ::= */ - -3, /* (251) cmd ::= PRAGMA nm dbnm */ - -5, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (260) trigger_time ::= BEFORE|AFTER */ - -2, /* (261) trigger_time ::= INSTEAD OF */ - 0, /* (262) trigger_time ::= */ - -1, /* (263) trigger_event ::= DELETE|INSERT */ - -1, /* (264) trigger_event ::= UPDATE */ - -3, /* (265) trigger_event ::= UPDATE OF idlist */ - 0, /* (266) when_clause ::= */ - -2, /* (267) when_clause ::= WHEN expr */ - -3, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (270) trnm ::= nm DOT nm */ - -3, /* (271) tridxby ::= INDEXED BY nm */ - -2, /* (272) tridxby ::= NOT INDEXED */ - -9, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (276) trigger_cmd ::= scanpt select scanpt */ - -4, /* (277) expr ::= RAISE LP IGNORE RP */ - -6, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (279) raisetype ::= ROLLBACK */ - -1, /* (280) raisetype ::= ABORT */ - -1, /* (281) raisetype ::= FAIL */ - -4, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (284) cmd ::= DETACH database_kw_opt expr */ - 0, /* (285) key_opt ::= */ - -2, /* (286) key_opt ::= KEY expr */ - -1, /* (287) cmd ::= REINDEX */ - -3, /* (288) cmd ::= REINDEX nm dbnm */ - -1, /* (289) cmd ::= ANALYZE */ - -3, /* (290) cmd ::= ANALYZE nm dbnm */ - -6, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (294) add_column_fullname ::= fullname */ - -8, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (296) cmd ::= create_vtab */ - -4, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (299) vtabarg ::= */ - -1, /* (300) vtabargtoken ::= ANY */ - -3, /* (301) vtabargtoken ::= lp anylist RP */ - -1, /* (302) lp ::= LP */ - -2, /* (303) with ::= WITH wqlist */ - -3, /* (304) with ::= WITH RECURSIVE wqlist */ - -1, /* (305) wqas ::= AS */ - -2, /* (306) wqas ::= AS MATERIALIZED */ - -3, /* (307) wqas ::= AS NOT MATERIALIZED */ - -6, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (309) wqlist ::= wqitem */ - -3, /* (310) wqlist ::= wqlist COMMA wqitem */ - -3, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (312) windowdefn ::= nm AS LP window RP */ - -5, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (315) window ::= ORDER BY sortlist frame_opt */ - -5, /* (316) window ::= nm ORDER BY sortlist frame_opt */ - -2, /* (317) window ::= nm frame_opt */ - 0, /* (318) frame_opt ::= */ - -3, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (322) frame_bound_s ::= frame_bound */ - -2, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (324) frame_bound_e ::= frame_bound */ - -2, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (327) frame_bound ::= CURRENT ROW */ - 0, /* (328) frame_exclude_opt ::= */ - -2, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (330) frame_exclude ::= NO OTHERS */ - -2, /* (331) frame_exclude ::= CURRENT ROW */ - -1, /* (332) frame_exclude ::= GROUP|TIES */ - -2, /* (333) window_clause ::= WINDOW windowdefn_list */ - -2, /* (334) filter_over ::= filter_clause over_clause */ - -1, /* (335) filter_over ::= over_clause */ - -1, /* (336) filter_over ::= filter_clause */ - -4, /* (337) over_clause ::= OVER LP window RP */ - -2, /* (338) over_clause ::= OVER nm */ - -5, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (340) input ::= cmdlist */ - -2, /* (341) cmdlist ::= cmdlist ecmd */ - -1, /* (342) cmdlist ::= ecmd */ - -1, /* (343) ecmd ::= SEMI */ - -2, /* (344) ecmd ::= cmdx SEMI */ - -3, /* (345) ecmd ::= explain cmdx SEMI */ - 0, /* (346) trans_opt ::= */ - -1, /* (347) trans_opt ::= TRANSACTION */ - -2, /* (348) trans_opt ::= TRANSACTION nm */ - -1, /* (349) savepoint_opt ::= SAVEPOINT */ - 0, /* (350) savepoint_opt ::= */ - -2, /* (351) cmd ::= create_table create_table_args */ - -1, /* (352) table_option_set ::= table_option */ - -4, /* (353) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (354) columnlist ::= columnname carglist */ - -1, /* (355) nm ::= ID|INDEXED|JOIN_KW */ - -1, /* (356) nm ::= STRING */ - -1, /* (357) typetoken ::= typename */ - -1, /* (358) typename ::= ID|STRING */ - -1, /* (359) signed ::= plus_num */ - -1, /* (360) signed ::= minus_num */ - -2, /* (361) carglist ::= carglist ccons */ - 0, /* (362) carglist ::= */ - -2, /* (363) ccons ::= NULL onconf */ - -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (365) ccons ::= AS generated */ - -2, /* (366) conslist_opt ::= COMMA conslist */ - -3, /* (367) conslist ::= conslist tconscomma tcons */ - -1, /* (368) conslist ::= tcons */ - 0, /* (369) tconscomma ::= */ - -1, /* (370) defer_subclause_opt ::= defer_subclause */ - -1, /* (371) resolvetype ::= raisetype */ - -1, /* (372) selectnowith ::= oneselect */ - -1, /* (373) oneselect ::= values */ - -2, /* (374) sclp ::= selcollist COMMA */ - -1, /* (375) as ::= ID|STRING */ - -1, /* (376) indexed_opt ::= indexed_by */ - 0, /* (377) returning ::= */ - -1, /* (378) expr ::= term */ - -1, /* (379) likeop ::= LIKE_KW|MATCH */ - -1, /* (380) case_operand ::= expr */ - -1, /* (381) exprlist ::= nexprlist */ - -1, /* (382) nmnum ::= plus_num */ - -1, /* (383) nmnum ::= nm */ - -1, /* (384) nmnum ::= ON */ - -1, /* (385) nmnum ::= DELETE */ - -1, /* (386) nmnum ::= DEFAULT */ - -1, /* (387) plus_num ::= INTEGER|FLOAT */ - 0, /* (388) foreach_clause ::= */ - -3, /* (389) foreach_clause ::= FOR EACH ROW */ - -1, /* (390) trnm ::= nm */ - 0, /* (391) tridxby ::= */ - -1, /* (392) database_kw_opt ::= DATABASE */ - 0, /* (393) database_kw_opt ::= */ - 0, /* (394) kwcolumn_opt ::= */ - -1, /* (395) kwcolumn_opt ::= COLUMNKW */ - -1, /* (396) vtabarglist ::= vtabarg */ - -3, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (398) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (399) anylist ::= */ - -4, /* (400) anylist ::= anylist LP anylist RP */ - -2, /* (401) anylist ::= anylist ANY */ - 0, /* (402) with ::= */ - -1, /* (403) windowdefn_list ::= windowdefn */ - -1, /* (404) window ::= frame_opt */ + -1, /* (95) oneselect ::= mvalues */ + -5, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + -5, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + -1, /* (98) distinct ::= DISTINCT */ + -1, /* (99) distinct ::= ALL */ + 0, /* (100) distinct ::= */ + 0, /* (101) sclp ::= */ + -5, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (103) selcollist ::= sclp scanpt STAR */ + -5, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (105) as ::= AS nm */ + 0, /* (106) as ::= */ + 0, /* (107) from ::= */ + -2, /* (108) from ::= FROM seltablist */ + -2, /* (109) stl_prefix ::= seltablist joinop */ + 0, /* (110) stl_prefix ::= */ + -5, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + -6, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + -8, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + -6, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + -6, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 0, /* (116) dbnm ::= */ + -2, /* (117) dbnm ::= DOT nm */ + -1, /* (118) fullname ::= nm */ + -3, /* (119) fullname ::= nm DOT nm */ + -1, /* (120) xfullname ::= nm */ + -3, /* (121) xfullname ::= nm DOT nm */ + -5, /* (122) xfullname ::= nm DOT nm AS nm */ + -3, /* (123) xfullname ::= nm AS nm */ + -1, /* (124) joinop ::= COMMA|JOIN */ + -2, /* (125) joinop ::= JOIN_KW JOIN */ + -3, /* (126) joinop ::= JOIN_KW nm JOIN */ + -4, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (128) on_using ::= ON expr */ + -4, /* (129) on_using ::= USING LP idlist RP */ + 0, /* (130) on_using ::= */ + 0, /* (131) indexed_opt ::= */ + -3, /* (132) indexed_by ::= INDEXED BY nm */ + -2, /* (133) indexed_by ::= NOT INDEXED */ + 0, /* (134) orderby_opt ::= */ + -3, /* (135) orderby_opt ::= ORDER BY sortlist */ + -5, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (137) sortlist ::= expr sortorder nulls */ + -1, /* (138) sortorder ::= ASC */ + -1, /* (139) sortorder ::= DESC */ + 0, /* (140) sortorder ::= */ + -2, /* (141) nulls ::= NULLS FIRST */ + -2, /* (142) nulls ::= NULLS LAST */ + 0, /* (143) nulls ::= */ + 0, /* (144) groupby_opt ::= */ + -3, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (146) having_opt ::= */ + -2, /* (147) having_opt ::= HAVING expr */ + 0, /* (148) limit_opt ::= */ + -2, /* (149) limit_opt ::= LIMIT expr */ + -4, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (153) where_opt ::= */ + -2, /* (154) where_opt ::= WHERE expr */ + 0, /* (155) where_opt_ret ::= */ + -2, /* (156) where_opt_ret ::= WHERE expr */ + -2, /* (157) where_opt_ret ::= RETURNING selcollist */ + -4, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (160) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (162) setlist ::= nm EQ expr */ + -5, /* (163) setlist ::= LP idlist RP EQ expr */ + -7, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (166) upsert ::= */ + -2, /* (167) upsert ::= RETURNING selcollist */ + -12, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (172) returning ::= RETURNING selcollist */ + -2, /* (173) insert_cmd ::= INSERT orconf */ + -1, /* (174) insert_cmd ::= REPLACE */ + 0, /* (175) idlist_opt ::= */ + -3, /* (176) idlist_opt ::= LP idlist RP */ + -3, /* (177) idlist ::= idlist COMMA nm */ + -1, /* (178) idlist ::= nm */ + -3, /* (179) expr ::= LP expr RP */ + -1, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (181) expr ::= nm DOT nm */ + -5, /* (182) expr ::= nm DOT nm DOT nm */ + -1, /* (183) term ::= NULL|FLOAT|BLOB */ + -1, /* (184) term ::= STRING */ + -1, /* (185) term ::= INTEGER */ + -1, /* (186) expr ::= VARIABLE */ + -3, /* (187) expr ::= expr COLLATE ID|STRING */ + -6, /* (188) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -8, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + -4, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -9, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + -5, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (195) term ::= CTIME_KW */ + -5, /* (196) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (197) expr ::= expr AND expr */ + -3, /* (198) expr ::= expr OR expr */ + -3, /* (199) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (200) expr ::= expr EQ|NE expr */ + -3, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (202) expr ::= expr PLUS|MINUS expr */ + -3, /* (203) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (204) expr ::= expr CONCAT expr */ + -2, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (206) expr ::= expr likeop expr */ + -5, /* (207) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (208) expr ::= expr ISNULL|NOTNULL */ + -3, /* (209) expr ::= expr NOT NULL */ + -3, /* (210) expr ::= expr IS expr */ + -4, /* (211) expr ::= expr IS NOT expr */ + -6, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (213) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (214) expr ::= NOT expr */ + -2, /* (215) expr ::= BITNOT expr */ + -2, /* (216) expr ::= PLUS|MINUS expr */ + -3, /* (217) expr ::= expr PTR expr */ + -1, /* (218) between_op ::= BETWEEN */ + -2, /* (219) between_op ::= NOT BETWEEN */ + -5, /* (220) expr ::= expr between_op expr AND expr */ + -1, /* (221) in_op ::= IN */ + -2, /* (222) in_op ::= NOT IN */ + -5, /* (223) expr ::= expr in_op LP exprlist RP */ + -3, /* (224) expr ::= LP select RP */ + -5, /* (225) expr ::= expr in_op LP select RP */ + -5, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (227) expr ::= EXISTS LP select RP */ + -5, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (230) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (231) case_else ::= ELSE expr */ + 0, /* (232) case_else ::= */ + 0, /* (233) case_operand ::= */ + 0, /* (234) exprlist ::= */ + -3, /* (235) nexprlist ::= nexprlist COMMA expr */ + -1, /* (236) nexprlist ::= expr */ + 0, /* (237) paren_exprlist ::= */ + -3, /* (238) paren_exprlist ::= LP exprlist RP */ + -12, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (240) uniqueflag ::= UNIQUE */ + 0, /* (241) uniqueflag ::= */ + 0, /* (242) eidlist_opt ::= */ + -3, /* (243) eidlist_opt ::= LP eidlist RP */ + -5, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (245) eidlist ::= nm collate sortorder */ + 0, /* (246) collate ::= */ + -2, /* (247) collate ::= COLLATE ID|STRING */ + -4, /* (248) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (249) cmd ::= VACUUM vinto */ + -3, /* (250) cmd ::= VACUUM nm vinto */ + -2, /* (251) vinto ::= INTO expr */ + 0, /* (252) vinto ::= */ + -3, /* (253) cmd ::= PRAGMA nm dbnm */ + -5, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (262) trigger_time ::= BEFORE|AFTER */ + -2, /* (263) trigger_time ::= INSTEAD OF */ + 0, /* (264) trigger_time ::= */ + -1, /* (265) trigger_event ::= DELETE|INSERT */ + -1, /* (266) trigger_event ::= UPDATE */ + -3, /* (267) trigger_event ::= UPDATE OF idlist */ + 0, /* (268) when_clause ::= */ + -2, /* (269) when_clause ::= WHEN expr */ + -3, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (272) trnm ::= nm DOT nm */ + -3, /* (273) tridxby ::= INDEXED BY nm */ + -2, /* (274) tridxby ::= NOT INDEXED */ + -9, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (278) trigger_cmd ::= scanpt select scanpt */ + -4, /* (279) expr ::= RAISE LP IGNORE RP */ + -6, /* (280) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (281) raisetype ::= ROLLBACK */ + -1, /* (282) raisetype ::= ABORT */ + -1, /* (283) raisetype ::= FAIL */ + -4, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (286) cmd ::= DETACH database_kw_opt expr */ + 0, /* (287) key_opt ::= */ + -2, /* (288) key_opt ::= KEY expr */ + -1, /* (289) cmd ::= REINDEX */ + -3, /* (290) cmd ::= REINDEX nm dbnm */ + -1, /* (291) cmd ::= ANALYZE */ + -3, /* (292) cmd ::= ANALYZE nm dbnm */ + -6, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (296) add_column_fullname ::= fullname */ + -8, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (298) cmd ::= create_vtab */ + -4, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (301) vtabarg ::= */ + -1, /* (302) vtabargtoken ::= ANY */ + -3, /* (303) vtabargtoken ::= lp anylist RP */ + -1, /* (304) lp ::= LP */ + -2, /* (305) with ::= WITH wqlist */ + -3, /* (306) with ::= WITH RECURSIVE wqlist */ + -1, /* (307) wqas ::= AS */ + -2, /* (308) wqas ::= AS MATERIALIZED */ + -3, /* (309) wqas ::= AS NOT MATERIALIZED */ + -6, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + -1, /* (311) withnm ::= nm */ + -1, /* (312) wqlist ::= wqitem */ + -3, /* (313) wqlist ::= wqlist COMMA wqitem */ + -3, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (315) windowdefn ::= nm AS LP window RP */ + -5, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (318) window ::= ORDER BY sortlist frame_opt */ + -5, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + -2, /* (320) window ::= nm frame_opt */ + 0, /* (321) frame_opt ::= */ + -3, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (325) frame_bound_s ::= frame_bound */ + -2, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (327) frame_bound_e ::= frame_bound */ + -2, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (330) frame_bound ::= CURRENT ROW */ + 0, /* (331) frame_exclude_opt ::= */ + -2, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (333) frame_exclude ::= NO OTHERS */ + -2, /* (334) frame_exclude ::= CURRENT ROW */ + -1, /* (335) frame_exclude ::= GROUP|TIES */ + -2, /* (336) window_clause ::= WINDOW windowdefn_list */ + -2, /* (337) filter_over ::= filter_clause over_clause */ + -1, /* (338) filter_over ::= over_clause */ + -1, /* (339) filter_over ::= filter_clause */ + -4, /* (340) over_clause ::= OVER LP window RP */ + -2, /* (341) over_clause ::= OVER nm */ + -5, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (343) term ::= QNUMBER */ + -1, /* (344) input ::= cmdlist */ + -2, /* (345) cmdlist ::= cmdlist ecmd */ + -1, /* (346) cmdlist ::= ecmd */ + -1, /* (347) ecmd ::= SEMI */ + -2, /* (348) ecmd ::= cmdx SEMI */ + -3, /* (349) ecmd ::= explain cmdx SEMI */ + 0, /* (350) trans_opt ::= */ + -1, /* (351) trans_opt ::= TRANSACTION */ + -2, /* (352) trans_opt ::= TRANSACTION nm */ + -1, /* (353) savepoint_opt ::= SAVEPOINT */ + 0, /* (354) savepoint_opt ::= */ + -2, /* (355) cmd ::= create_table create_table_args */ + -1, /* (356) table_option_set ::= table_option */ + -4, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (358) columnlist ::= columnname carglist */ + -1, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (360) nm ::= STRING */ + -1, /* (361) typetoken ::= typename */ + -1, /* (362) typename ::= ID|STRING */ + -1, /* (363) signed ::= plus_num */ + -1, /* (364) signed ::= minus_num */ + -2, /* (365) carglist ::= carglist ccons */ + 0, /* (366) carglist ::= */ + -2, /* (367) ccons ::= NULL onconf */ + -4, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (369) ccons ::= AS generated */ + -2, /* (370) conslist_opt ::= COMMA conslist */ + -3, /* (371) conslist ::= conslist tconscomma tcons */ + -1, /* (372) conslist ::= tcons */ + 0, /* (373) tconscomma ::= */ + -1, /* (374) defer_subclause_opt ::= defer_subclause */ + -1, /* (375) resolvetype ::= raisetype */ + -1, /* (376) selectnowith ::= oneselect */ + -1, /* (377) oneselect ::= values */ + -2, /* (378) sclp ::= selcollist COMMA */ + -1, /* (379) as ::= ID|STRING */ + -1, /* (380) indexed_opt ::= indexed_by */ + 0, /* (381) returning ::= */ + -1, /* (382) expr ::= term */ + -1, /* (383) likeop ::= LIKE_KW|MATCH */ + -1, /* (384) case_operand ::= expr */ + -1, /* (385) exprlist ::= nexprlist */ + -1, /* (386) nmnum ::= plus_num */ + -1, /* (387) nmnum ::= nm */ + -1, /* (388) nmnum ::= ON */ + -1, /* (389) nmnum ::= DELETE */ + -1, /* (390) nmnum ::= DEFAULT */ + -1, /* (391) plus_num ::= INTEGER|FLOAT */ + 0, /* (392) foreach_clause ::= */ + -3, /* (393) foreach_clause ::= FOR EACH ROW */ + -1, /* (394) trnm ::= nm */ + 0, /* (395) tridxby ::= */ + -1, /* (396) database_kw_opt ::= DATABASE */ + 0, /* (397) database_kw_opt ::= */ + 0, /* (398) kwcolumn_opt ::= */ + -1, /* (399) kwcolumn_opt ::= COLUMNKW */ + -1, /* (400) vtabarglist ::= vtabarg */ + -3, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (403) anylist ::= */ + -4, /* (404) anylist ::= anylist LP anylist RP */ + -2, /* (405) anylist ::= anylist ANY */ + 0, /* (406) with ::= */ + -1, /* (407) windowdefn_list ::= windowdefn */ + -1, /* (408) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -174504,16 +176116,16 @@ static YYACTIONTYPE yy_reduce( { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy394);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy144);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy394 = TK_DEFERRED;} +{yymsp[1].minor.yy144 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 321: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==321); -{yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/} + case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324); +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); @@ -174536,7 +176148,7 @@ static YYACTIONTYPE yy_reduce( break; case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy394,0,0,yymsp[-2].minor.yy394); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy144,0,0,yymsp[-2].minor.yy144); } break; case 14: /* createkw ::= CREATE */ @@ -174548,40 +176160,40 @@ static YYACTIONTYPE yy_reduce( case 62: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==62); case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); - case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 244: /* collate ::= */ yytestcase(yyruleno==244); -{yymsp[1].minor.yy394 = 0;} + case 100: /* distinct ::= */ yytestcase(yyruleno==100); + case 246: /* collate ::= */ yytestcase(yyruleno==246); +{yymsp[1].minor.yy144 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy394 = 1;} +{yymsp[-2].minor.yy144 = 1;} break; case 17: /* temp ::= TEMP */ -{yymsp[0].minor.yy394 = pParse->db->init.busy==0;} +{yymsp[0].minor.yy144 = pParse->db->init.busy==0;} break; case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy285,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy391,0); } break; case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy47); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy47); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy555); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 21: /* table_option_set ::= */ -{yymsp[1].minor.yy285 = 0;} +{yymsp[1].minor.yy391 = 0;} break; case 22: /* table_option_set ::= table_option_set COMMA table_option */ -{yylhsminor.yy285 = yymsp[-2].minor.yy285|yymsp[0].minor.yy285;} - yymsp[-2].minor.yy285 = yylhsminor.yy285; +{yylhsminor.yy391 = yymsp[-2].minor.yy391|yymsp[0].minor.yy391;} + yymsp[-2].minor.yy391 = yylhsminor.yy391; break; case 23: /* table_option ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy285 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy391 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy285 = 0; + yymsp[-1].minor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -174589,20 +176201,20 @@ static YYACTIONTYPE yy_reduce( case 24: /* table_option ::= nm */ { if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ - yylhsminor.yy285 = TF_Strict; + yylhsminor.yy391 = TF_Strict; }else{ - yylhsminor.yy285 = 0; + yylhsminor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } - yymsp[0].minor.yy285 = yylhsminor.yy285; + yymsp[0].minor.yy391 = yylhsminor.yy391; break; case 25: /* columnname ::= nm typetoken */ {sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} break; case 26: /* typetoken ::= */ case 65: /* conslist_opt ::= */ yytestcase(yyruleno==65); - case 104: /* as ::= */ yytestcase(yyruleno==104); + case 106: /* as ::= */ yytestcase(yyruleno==106); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; case 27: /* typetoken ::= typename LP signed RP */ @@ -174621,7 +176233,7 @@ static YYACTIONTYPE yy_reduce( case 30: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy522 = yyLookaheadToken.z; + yymsp[1].minor.yy168 = yyLookaheadToken.z; } break; case 31: /* scantok ::= */ @@ -174635,17 +176247,17 @@ static YYACTIONTYPE yy_reduce( {pParse->constraintName = yymsp[0].minor.yy0;} break; case 33: /* ccons ::= DEFAULT scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy528,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 34: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy528,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; case 35: /* ccons ::= DEFAULT PLUS scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy528,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 36: /* ccons ::= DEFAULT MINUS scantok term */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy528, 0); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy454, 0); sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; @@ -174660,151 +176272,151 @@ static YYACTIONTYPE yy_reduce( } break; case 38: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy394);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy144);} break; case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy394,yymsp[0].minor.yy394,yymsp[-2].minor.yy394);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy144,yymsp[0].minor.yy144,yymsp[-2].minor.yy144);} break; case 40: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy394,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 41: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy528,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} break; case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy322,yymsp[0].minor.yy394);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy144);} break; case 43: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy394);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy144);} break; case 44: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 45: /* generated ::= LP expr RP */ -{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy528,0);} +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy454,0);} break; case 46: /* generated ::= LP expr RP ID */ -{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy528,&yymsp[0].minor.yy0);} +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy454,&yymsp[0].minor.yy0);} break; case 48: /* autoinc ::= AUTOINCR */ -{yymsp[0].minor.yy394 = 1;} +{yymsp[0].minor.yy144 = 1;} break; case 49: /* refargs ::= */ -{ yymsp[1].minor.yy394 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yymsp[1].minor.yy144 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 50: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy394 = (yymsp[-1].minor.yy394 & ~yymsp[0].minor.yy231.mask) | yymsp[0].minor.yy231.value; } +{ yymsp[-1].minor.yy144 = (yymsp[-1].minor.yy144 & ~yymsp[0].minor.yy383.mask) | yymsp[0].minor.yy383.value; } break; case 51: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy231.value = 0; yymsp[-1].minor.yy231.mask = 0x000000; } +{ yymsp[-1].minor.yy383.value = 0; yymsp[-1].minor.yy383.mask = 0x000000; } break; case 52: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy231.value = 0; yymsp[-2].minor.yy231.mask = 0x000000; } +{ yymsp[-2].minor.yy383.value = 0; yymsp[-2].minor.yy383.mask = 0x000000; } break; case 53: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy231.value = yymsp[0].minor.yy394; yymsp[-2].minor.yy231.mask = 0x0000ff; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144; yymsp[-2].minor.yy383.mask = 0x0000ff; } break; case 54: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy231.value = yymsp[0].minor.yy394<<8; yymsp[-2].minor.yy231.mask = 0x00ff00; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144<<8; yymsp[-2].minor.yy383.mask = 0x00ff00; } break; case 55: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy394 = OE_SetNull; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetNull; /* EV: R-33326-45252 */} break; case 56: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy394 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 57: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy394 = OE_Cascade; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Cascade; /* EV: R-33326-45252 */} break; case 58: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy394 = OE_Restrict; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Restrict; /* EV: R-33326-45252 */} break; case 59: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy394 = OE_None; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_None; /* EV: R-33326-45252 */} break; case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy394 = 0;} +{yymsp[-2].minor.yy144 = 0;} break; case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); - case 171: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==171); -{yymsp[-1].minor.yy394 = yymsp[0].minor.yy394;} + case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173); +{yymsp[-1].minor.yy144 = yymsp[0].minor.yy144;} break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 217: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==217); - case 220: /* in_op ::= NOT IN */ yytestcase(yyruleno==220); - case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245); -{yymsp[-1].minor.yy394 = 1;} + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); +{yymsp[-1].minor.yy144 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy394 = 0;} +{yymsp[-1].minor.yy144 = 0;} break; case 66: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy322,yymsp[0].minor.yy394,yymsp[-2].minor.yy394,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy144,yymsp[-2].minor.yy144,0);} break; case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy322,yymsp[0].minor.yy394,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 70: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy528,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy454,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} break; case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy322, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[-1].minor.yy394); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy394); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy144); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy144); } break; case 73: /* onconf ::= */ case 75: /* orconf ::= */ yytestcase(yyruleno==75); -{yymsp[1].minor.yy394 = OE_Default;} +{yymsp[1].minor.yy144 = OE_Default;} break; case 74: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy394 = yymsp[0].minor.yy394;} +{yymsp[-2].minor.yy144 = yymsp[0].minor.yy144;} break; case 77: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy394 = OE_Ignore;} +{yymsp[0].minor.yy144 = OE_Ignore;} break; case 78: /* resolvetype ::= REPLACE */ - case 172: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==172); -{yymsp[0].minor.yy394 = OE_Replace;} + case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174); +{yymsp[0].minor.yy144 = OE_Replace;} break; case 79: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy131, 0, yymsp[-1].minor.yy394); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 0, yymsp[-1].minor.yy144); } break; case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[0].minor.yy47, yymsp[-7].minor.yy394, yymsp[-5].minor.yy394); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy555, yymsp[-7].minor.yy144, yymsp[-5].minor.yy144); } break; case 83: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy131, 1, yymsp[-1].minor.yy394); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 1, yymsp[-1].minor.yy144); } break; case 84: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy47, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy47); + sqlite3Select(pParse, yymsp[0].minor.yy555, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 85: /* select ::= WITH wqlist selectnowith */ -{yymsp[-2].minor.yy47 = attachWithToSelect(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy521);} +{yymsp[-2].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ -{yymsp[-3].minor.yy47 = attachWithToSelect(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy521);} +{yymsp[-3].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 87: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy47; + Select *p = yymsp[0].minor.yy555; if( p ){ parserDoubleLinkSelect(pParse, p); } @@ -174812,8 +176424,8 @@ static YYACTIONTYPE yy_reduce( break; case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy47; - Select *pLhs = yymsp[-2].minor.yy47; + Select *pRhs = yymsp[0].minor.yy555; + Select *pLhs = yymsp[-2].minor.yy555; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -174823,148 +176435,145 @@ static YYACTIONTYPE yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy394; + pRhs->op = (u8)yymsp[-1].minor.yy144; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy394!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy144!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy47 = pRhs; + yymsp[-2].minor.yy555 = pRhs; } break; case 89: /* multiselect_op ::= UNION */ case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); -{yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-OP*/} +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-OP*/} break; case 90: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy394 = TK_ALL;} +{yymsp[-1].minor.yy144 = TK_ALL;} break; case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yymsp[-8].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy322,yymsp[-5].minor.yy131,yymsp[-4].minor.yy528,yymsp[-3].minor.yy322,yymsp[-2].minor.yy528,yymsp[-1].minor.yy322,yymsp[-7].minor.yy394,yymsp[0].minor.yy528); + yymsp[-8].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy203,yymsp[-4].minor.yy454,yymsp[-3].minor.yy14,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[-7].minor.yy144,yymsp[0].minor.yy454); } break; case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-9].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy322,yymsp[-6].minor.yy131,yymsp[-5].minor.yy528,yymsp[-4].minor.yy322,yymsp[-3].minor.yy528,yymsp[-1].minor.yy322,yymsp[-8].minor.yy394,yymsp[0].minor.yy528); - if( yymsp[-9].minor.yy47 ){ - yymsp[-9].minor.yy47->pWinDefn = yymsp[-2].minor.yy41; + yymsp[-9].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy14,yymsp[-6].minor.yy203,yymsp[-5].minor.yy454,yymsp[-4].minor.yy14,yymsp[-3].minor.yy454,yymsp[-1].minor.yy14,yymsp[-8].minor.yy144,yymsp[0].minor.yy454); + if( yymsp[-9].minor.yy555 ){ + yymsp[-9].minor.yy555->pWinDefn = yymsp[-2].minor.yy211; }else{ - sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy41); + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy211); } } break; case 94: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy322,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0); } break; - case 95: /* values ::= values COMMA LP nexprlist RP */ + case 95: /* oneselect ::= mvalues */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy47; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy322,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy47 = pRight; - }else{ - yymsp[-4].minor.yy47 = pLeft; - } + sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy555); } break; - case 96: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy394 = SF_Distinct;} - break; - case 97: /* distinct ::= ALL */ -{yymsp[0].minor.yy394 = SF_All;} - break; - case 99: /* sclp ::= */ - case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); - case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 232: /* exprlist ::= */ yytestcase(yyruleno==232); - case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235); - case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240); -{yymsp[1].minor.yy322 = 0;} - break; - case 100: /* selcollist ::= sclp scanpt expr scanpt as */ + case 96: /* mvalues ::= values COMMA LP nexprlist RP */ + case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97); { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy322, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy322,yymsp[-3].minor.yy522,yymsp[-1].minor.yy522); + yymsp[-4].minor.yy555 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy555, yymsp[-1].minor.yy14); } break; - case 101: /* selcollist ::= sclp scanpt STAR */ + case 98: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy144 = SF_Distinct;} + break; + case 99: /* distinct ::= ALL */ +{yymsp[0].minor.yy144 = SF_All;} + break; + case 101: /* sclp ::= */ + case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134); + case 144: /* groupby_opt ::= */ yytestcase(yyruleno==144); + case 234: /* exprlist ::= */ yytestcase(yyruleno==234); + case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237); + case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242); +{yymsp[1].minor.yy14 = 0;} + break; + case 102: /* selcollist ::= sclp scanpt expr scanpt as */ +{ + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy14,yymsp[-3].minor.yy168,yymsp[-1].minor.yy168); +} + break; + case 103: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); - yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, p); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, p); } break; - case 102: /* selcollist ::= sclp scanpt nm DOT STAR */ + case 104: /* selcollist ::= sclp scanpt nm DOT STAR */ { Expr *pRight, *pLeft, *pDot; pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, pDot); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, pDot); } break; - case 103: /* as ::= AS nm */ - case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115); - case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256); - case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257); + case 105: /* as ::= AS nm */ + case 117: /* dbnm ::= DOT nm */ yytestcase(yyruleno==117); + case 258: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==258); + case 259: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==259); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 105: /* from ::= */ - case 108: /* stl_prefix ::= */ yytestcase(yyruleno==108); -{yymsp[1].minor.yy131 = 0;} + case 107: /* from ::= */ + case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110); +{yymsp[1].minor.yy203 = 0;} break; - case 106: /* from ::= FROM seltablist */ + case 108: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy131 = yymsp[0].minor.yy131; - sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy131); + yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; + sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy203); } break; - case 107: /* stl_prefix ::= seltablist joinop */ + case 109: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy131 && yymsp[-1].minor.yy131->nSrc>0) ) yymsp[-1].minor.yy131->a[yymsp[-1].minor.yy131->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy394; + if( ALWAYS(yymsp[-1].minor.yy203 && yymsp[-1].minor.yy203->nSrc>0) ) yymsp[-1].minor.yy203->a[yymsp[-1].minor.yy203->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy144; } break; - case 109: /* seltablist ::= stl_prefix nm dbnm as on_using */ + case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */ { - yymsp[-4].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy131,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); + yymsp[-4].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy203,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); } break; - case 110: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ { - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy561); - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy131, &yymsp[-1].minor.yy0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-1].minor.yy0); } break; - case 111: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ { - yymsp[-7].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy131,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); - sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy131, yymsp[-3].minor.yy322); + yymsp[-7].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy203,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy203, yymsp[-3].minor.yy14); } break; - case 112: /* seltablist ::= stl_prefix LP select RP as on_using */ + case 114: /* seltablist ::= stl_prefix LP select RP as on_using */ { - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy47,&yymsp[0].minor.yy561); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy555,&yymsp[0].minor.yy269); } break; - case 113: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ + case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ { - if( yymsp[-5].minor.yy131==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy561.pOn==0 && yymsp[0].minor.yy561.pUsing==0 ){ - yymsp[-5].minor.yy131 = yymsp[-3].minor.yy131; - }else if( ALWAYS(yymsp[-3].minor.yy131!=0) && yymsp[-3].minor.yy131->nSrc==1 ){ - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); - if( yymsp[-5].minor.yy131 ){ - SrcItem *pNew = &yymsp[-5].minor.yy131->a[yymsp[-5].minor.yy131->nSrc-1]; - SrcItem *pOld = yymsp[-3].minor.yy131->a; + if( yymsp[-5].minor.yy203==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy269.pOn==0 && yymsp[0].minor.yy269.pUsing==0 ){ + yymsp[-5].minor.yy203 = yymsp[-3].minor.yy203; + }else if( ALWAYS(yymsp[-3].minor.yy203!=0) && yymsp[-3].minor.yy203->nSrc==1 ){ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + if( yymsp[-5].minor.yy203 ){ + SrcItem *pNew = &yymsp[-5].minor.yy203->a[yymsp[-5].minor.yy203->nSrc-1]; + SrcItem *pOld = yymsp[-3].minor.yy203->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; @@ -174980,153 +176589,153 @@ static YYACTIONTYPE yy_reduce( pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy131); + sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy203); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy131); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy131,0,0,0,0,SF_NestedFrom,0); - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy561); + sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy203); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy203,0,0,0,0,SF_NestedFrom,0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy269); } } break; - case 114: /* dbnm ::= */ - case 129: /* indexed_opt ::= */ yytestcase(yyruleno==129); + case 116: /* dbnm ::= */ + case 131: /* indexed_opt ::= */ yytestcase(yyruleno==131); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 116: /* fullname ::= nm */ + case 118: /* fullname ::= nm */ { - yylhsminor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); - if( IN_RENAME_OBJECT && yylhsminor.yy131 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy131->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy131 = yylhsminor.yy131; + yymsp[0].minor.yy203 = yylhsminor.yy203; break; - case 117: /* fullname ::= nm DOT nm */ + case 119: /* fullname ::= nm DOT nm */ { - yylhsminor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - if( IN_RENAME_OBJECT && yylhsminor.yy131 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy131->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + yymsp[-2].minor.yy203 = yylhsminor.yy203; break; - case 118: /* xfullname ::= nm */ -{yymsp[0].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + case 120: /* xfullname ::= nm */ +{yymsp[0].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} break; - case 119: /* xfullname ::= nm DOT nm */ -{yymsp[-2].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 121: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 120: /* xfullname ::= nm DOT nm AS nm */ + case 122: /* xfullname ::= nm DOT nm AS nm */ { - yymsp[-4].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy131 ) yymsp[-4].minor.yy131->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-4].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy203 ) yymsp[-4].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 121: /* xfullname ::= nm AS nm */ + case 123: /* xfullname ::= nm AS nm */ { - yymsp[-2].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy131 ) yymsp[-2].minor.yy131->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy203 ) yymsp[-2].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 122: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy394 = JT_INNER; } + case 124: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy144 = JT_INNER; } break; - case 123: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 125: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 124: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 126: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 125: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 127: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 126: /* on_using ::= ON expr */ -{yymsp[-1].minor.yy561.pOn = yymsp[0].minor.yy528; yymsp[-1].minor.yy561.pUsing = 0;} + case 128: /* on_using ::= ON expr */ +{yymsp[-1].minor.yy269.pOn = yymsp[0].minor.yy454; yymsp[-1].minor.yy269.pUsing = 0;} break; - case 127: /* on_using ::= USING LP idlist RP */ -{yymsp[-3].minor.yy561.pOn = 0; yymsp[-3].minor.yy561.pUsing = yymsp[-1].minor.yy254;} + case 129: /* on_using ::= USING LP idlist RP */ +{yymsp[-3].minor.yy269.pOn = 0; yymsp[-3].minor.yy269.pUsing = yymsp[-1].minor.yy132;} break; - case 128: /* on_using ::= */ -{yymsp[1].minor.yy561.pOn = 0; yymsp[1].minor.yy561.pUsing = 0;} + case 130: /* on_using ::= */ +{yymsp[1].minor.yy269.pOn = 0; yymsp[1].minor.yy269.pUsing = 0;} break; - case 130: /* indexed_by ::= INDEXED BY nm */ + case 132: /* indexed_by ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 131: /* indexed_by ::= NOT INDEXED */ + case 133: /* indexed_by ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 133: /* orderby_opt ::= ORDER BY sortlist */ - case 143: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==143); -{yymsp[-2].minor.yy322 = yymsp[0].minor.yy322;} + case 135: /* orderby_opt ::= ORDER BY sortlist */ + case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145); +{yymsp[-2].minor.yy14 = yymsp[0].minor.yy14;} break; - case 134: /* sortlist ::= sortlist COMMA expr sortorder nulls */ + case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322,yymsp[-2].minor.yy528); - sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy322,yymsp[-1].minor.yy394,yymsp[0].minor.yy394); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 135: /* sortlist ::= expr sortorder nulls */ + case 137: /* sortlist ::= expr sortorder nulls */ { - yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy528); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy322,yymsp[-1].minor.yy394,yymsp[0].minor.yy394); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy454); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 136: /* sortorder ::= ASC */ -{yymsp[0].minor.yy394 = SQLITE_SO_ASC;} + case 138: /* sortorder ::= ASC */ +{yymsp[0].minor.yy144 = SQLITE_SO_ASC;} break; - case 137: /* sortorder ::= DESC */ -{yymsp[0].minor.yy394 = SQLITE_SO_DESC;} + case 139: /* sortorder ::= DESC */ +{yymsp[0].minor.yy144 = SQLITE_SO_DESC;} break; - case 138: /* sortorder ::= */ - case 141: /* nulls ::= */ yytestcase(yyruleno==141); -{yymsp[1].minor.yy394 = SQLITE_SO_UNDEFINED;} + case 140: /* sortorder ::= */ + case 143: /* nulls ::= */ yytestcase(yyruleno==143); +{yymsp[1].minor.yy144 = SQLITE_SO_UNDEFINED;} break; - case 139: /* nulls ::= NULLS FIRST */ -{yymsp[-1].minor.yy394 = SQLITE_SO_ASC;} + case 141: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_ASC;} break; - case 140: /* nulls ::= NULLS LAST */ -{yymsp[-1].minor.yy394 = SQLITE_SO_DESC;} + case 142: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_DESC;} break; - case 144: /* having_opt ::= */ - case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); - case 151: /* where_opt ::= */ yytestcase(yyruleno==151); - case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 230: /* case_else ::= */ yytestcase(yyruleno==230); - case 231: /* case_operand ::= */ yytestcase(yyruleno==231); - case 250: /* vinto ::= */ yytestcase(yyruleno==250); -{yymsp[1].minor.yy528 = 0;} + case 146: /* having_opt ::= */ + case 148: /* limit_opt ::= */ yytestcase(yyruleno==148); + case 153: /* where_opt ::= */ yytestcase(yyruleno==153); + case 155: /* where_opt_ret ::= */ yytestcase(yyruleno==155); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 233: /* case_operand ::= */ yytestcase(yyruleno==233); + case 252: /* vinto ::= */ yytestcase(yyruleno==252); +{yymsp[1].minor.yy454 = 0;} break; - case 145: /* having_opt ::= HAVING expr */ - case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); - case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 229: /* case_else ::= ELSE expr */ yytestcase(yyruleno==229); - case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249); -{yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;} + case 147: /* having_opt ::= HAVING expr */ + case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154); + case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251); +{yymsp[-1].minor.yy454 = yymsp[0].minor.yy454;} break; - case 147: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy528,0);} + case 149: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,0);} break; - case 148: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 150: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 149: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy528,yymsp[-2].minor.yy528);} + case 151: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,yymsp[-2].minor.yy454);} break; - case 150: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy131, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy131,yymsp[0].minor.yy528,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy203, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy203,yymsp[0].minor.yy454,0,0); } break; - case 155: /* where_opt_ret ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322); yymsp[-1].minor.yy528 = 0;} + case 157: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-1].minor.yy454 = 0;} break; - case 156: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322); yymsp[-3].minor.yy528 = yymsp[-2].minor.yy528;} + case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-3].minor.yy454 = yymsp[-2].minor.yy454;} break; - case 157: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy131, &yymsp[-4].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy322,"set list"); - if( yymsp[-1].minor.yy131 ){ - SrcList *pFromClause = yymsp[-1].minor.yy131; + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy14,"set list"); + if( yymsp[-1].minor.yy203 ){ + SrcList *pFromClause = yymsp[-1].minor.yy203; if( pFromClause->nSrc>1 ){ Select *pSubquery; Token as; @@ -175135,92 +176744,92 @@ static YYACTIONTYPE yy_reduce( as.z = 0; pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); } - yymsp[-5].minor.yy131 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy131, pFromClause); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy203, pFromClause); } - sqlite3Update(pParse,yymsp[-5].minor.yy131,yymsp[-2].minor.yy322,yymsp[0].minor.yy528,yymsp[-6].minor.yy394,0,0,0); + sqlite3Update(pParse,yymsp[-5].minor.yy203,yymsp[-2].minor.yy14,yymsp[0].minor.yy454,yymsp[-6].minor.yy144,0,0,0); } break; - case 158: /* setlist ::= setlist COMMA nm EQ expr */ + case 160: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[0].minor.yy528); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 159: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ + case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy322 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy322, yymsp[-3].minor.yy254, yymsp[0].minor.yy528); + yymsp[-6].minor.yy14 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy14, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 160: /* setlist ::= nm EQ expr */ + case 162: /* setlist ::= nm EQ expr */ { - yylhsminor.yy322 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy528); - sqlite3ExprListSetName(pParse, yylhsminor.yy322, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yylhsminor.yy14, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy322 = yylhsminor.yy322; + yymsp[-2].minor.yy14 = yylhsminor.yy14; break; - case 161: /* setlist ::= LP idlist RP EQ expr */ + case 163: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy254, yymsp[0].minor.yy528); + yymsp[-4].minor.yy14 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 162: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy131, yymsp[-1].minor.yy47, yymsp[-2].minor.yy254, yymsp[-5].minor.yy394, yymsp[0].minor.yy444); + sqlite3Insert(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy555, yymsp[-2].minor.yy132, yymsp[-5].minor.yy144, yymsp[0].minor.yy122); } break; - case 163: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ { - sqlite3Insert(pParse, yymsp[-4].minor.yy131, 0, yymsp[-3].minor.yy254, yymsp[-6].minor.yy394, 0); + sqlite3Insert(pParse, yymsp[-4].minor.yy203, 0, yymsp[-3].minor.yy132, yymsp[-6].minor.yy144, 0); } break; - case 164: /* upsert ::= */ -{ yymsp[1].minor.yy444 = 0; } + case 166: /* upsert ::= */ +{ yymsp[1].minor.yy122 = 0; } break; - case 165: /* upsert ::= RETURNING selcollist */ -{ yymsp[-1].minor.yy444 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy322); } + case 167: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy122 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy14); } break; - case 166: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ -{ yymsp[-11].minor.yy444 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy322,yymsp[-6].minor.yy528,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528,yymsp[0].minor.yy444);} + case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy14,yymsp[-6].minor.yy454,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,yymsp[0].minor.yy122);} break; - case 167: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ -{ yymsp[-8].minor.yy444 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy322,yymsp[-3].minor.yy528,0,0,yymsp[0].minor.yy444); } + case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy14,yymsp[-3].minor.yy454,0,0,yymsp[0].minor.yy122); } break; - case 168: /* upsert ::= ON CONFLICT DO NOTHING returning */ -{ yymsp[-4].minor.yy444 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } break; - case 169: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ -{ yymsp[-7].minor.yy444 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528,0);} + case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,0);} break; - case 170: /* returning ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322);} + case 172: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14);} break; - case 173: /* idlist_opt ::= */ -{yymsp[1].minor.yy254 = 0;} + case 175: /* idlist_opt ::= */ +{yymsp[1].minor.yy132 = 0;} break; - case 174: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy254 = yymsp[-1].minor.yy254;} + case 176: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy132 = yymsp[-1].minor.yy132;} break; - case 175: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy254 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy254,&yymsp[0].minor.yy0);} + case 177: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy132 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy132,&yymsp[0].minor.yy0);} break; - case 176: /* idlist ::= nm */ -{yymsp[0].minor.yy254 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + case 178: /* idlist ::= nm */ +{yymsp[0].minor.yy132 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 177: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy528 = yymsp[-1].minor.yy528;} + case 179: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy454 = yymsp[-1].minor.yy454;} break; - case 178: /* expr ::= ID|INDEXED|JOIN_KW */ -{yymsp[0].minor.yy528=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 180: /* expr ::= ID|INDEXED|JOIN_KW */ +{yymsp[0].minor.yy454=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 179: /* expr ::= nm DOT nm */ + case 181: /* expr ::= nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); - yylhsminor.yy528 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy528 = yylhsminor.yy528; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 180: /* expr ::= nm DOT nm DOT nm */ + case 182: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); @@ -175229,27 +176838,27 @@ static YYACTIONTYPE yy_reduce( if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, 0, temp1); } - yylhsminor.yy528 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 181: /* term ::= NULL|FLOAT|BLOB */ - case 182: /* term ::= STRING */ yytestcase(yyruleno==182); -{yymsp[0].minor.yy528=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 183: /* term ::= NULL|FLOAT|BLOB */ + case 184: /* term ::= STRING */ yytestcase(yyruleno==184); +{yymsp[0].minor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 183: /* term ::= INTEGER */ + case 185: /* term ::= INTEGER */ { - yylhsminor.yy528 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); - if( yylhsminor.yy528 ) yylhsminor.yy528->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); + yylhsminor.yy454 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + if( yylhsminor.yy454 ) yylhsminor.yy454->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } - yymsp[0].minor.yy528 = yylhsminor.yy528; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 184: /* expr ::= VARIABLE */ + case 186: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy528 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy528, n); + yymsp[0].minor.yy454 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy454, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -175258,194 +176867,203 @@ static YYACTIONTYPE yy_reduce( assert( t.n>=2 ); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy528 = 0; + yymsp[0].minor.yy454 = 0; }else{ - yymsp[0].minor.yy528 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy528 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy528->iTable); + yymsp[0].minor.yy454 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy454 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy454->iTable); } } } break; - case 185: /* expr ::= expr COLLATE ID|STRING */ + case 187: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy528 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy528, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy454 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy454, &yymsp[0].minor.yy0, 1); } break; - case 186: /* expr ::= CAST LP expr AS typetoken RP */ + case 188: /* expr ::= CAST LP expr AS typetoken RP */ { - yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy528, yymsp[-3].minor.yy528, 0); + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy454, yymsp[-3].minor.yy454, 0); } break; - case 187: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy394); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy144); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 188: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy322, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy394); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-1].minor.yy322); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy14, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy144); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-1].minor.yy14); } - yymsp[-7].minor.yy528 = yylhsminor.yy528; + yymsp[-7].minor.yy454 = yylhsminor.yy454; break; - case 189: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-3].minor.yy528 = yylhsminor.yy528; + yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy14, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-5].minor.yy528 = yylhsminor.yy528; + yymsp[-5].minor.yy454 = yylhsminor.yy454; break; - case 191: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy322, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy394); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-2].minor.yy322); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy14, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-2].minor.yy14); } - yymsp[-8].minor.yy528 = yylhsminor.yy528; + yymsp[-8].minor.yy454 = yylhsminor.yy454; break; - case 192: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 193: /* term ::= CTIME_KW */ + case 195: /* term ::= CTIME_KW */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } - yymsp[0].minor.yy528 = yylhsminor.yy528; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 194: /* expr ::= LP nexprlist COMMA expr RP */ + case 196: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; if( ALWAYS(pList->nExpr) ){ - yymsp[-4].minor.yy528->flags |= pList->a[0].pExpr->flags & EP_Propagate; + yymsp[-4].minor.yy454->flags |= pList->a[0].pExpr->flags & EP_Propagate; } }else{ sqlite3ExprListDelete(pParse->db, pList); } } break; - case 195: /* expr ::= expr AND expr */ -{yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 197: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy454=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 196: /* expr ::= expr OR expr */ - case 197: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==202); -{yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 198: /* expr ::= expr OR expr */ + case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204); +{yymsp[-2].minor.yy454=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 203: /* likeop ::= NOT LIKE_KW|MATCH */ + case 205: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 204: /* expr ::= expr likeop expr */ + case 206: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy528); - yymsp[-2].minor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); - if( bNot ) yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy528, 0); - if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy454); + yymsp[-2].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy454, 0); + if( yymsp[-2].minor.yy454 ) yymsp[-2].minor.yy454->flags |= EP_InfixFunc; } break; - case 205: /* expr ::= expr likeop expr ESCAPE expr */ + case 207: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); - if( bNot ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ) yymsp[-4].minor.yy454->flags |= EP_InfixFunc; } break; - case 206: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);} + case 208: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy454,0);} break; - case 207: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);} + case 209: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy454,0);} break; - case 208: /* expr ::= expr IS expr */ + case 210: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-2].minor.yy454, TK_ISNULL); } break; - case 209: /* expr ::= expr IS NOT expr */ + case 211: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL); + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-3].minor.yy454, TK_NOTNULL); } break; - case 210: /* expr ::= expr IS NOT DISTINCT FROM expr */ + case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */ { - yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL); + yymsp[-5].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-5].minor.yy454, TK_ISNULL); } break; - case 211: /* expr ::= expr IS DISTINCT FROM expr */ + case 213: /* expr ::= expr IS DISTINCT FROM expr */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-4].minor.yy454, TK_NOTNULL); } break; - case 212: /* expr ::= NOT expr */ - case 213: /* expr ::= BITNOT expr */ yytestcase(yyruleno==213); -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/} + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy454, 0);/*A-overwrites-B*/} break; - case 214: /* expr ::= PLUS|MINUS expr */ + case 216: /* expr ::= PLUS|MINUS expr */ { - yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0); - /*A-overwrites-B*/ + Expr *p = yymsp[0].minor.yy454; + u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + yymsp[-1].minor.yy454 = p; + }else{ + yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } } break; - case 215: /* expr ::= expr PTR expr */ + case 217: /* expr ::= expr PTR expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528); - yylhsminor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy454); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); } - yymsp[-2].minor.yy528 = yylhsminor.yy528; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 216: /* between_op ::= BETWEEN */ - case 219: /* in_op ::= IN */ yytestcase(yyruleno==219); -{yymsp[0].minor.yy394 = 0;} + case 218: /* between_op ::= BETWEEN */ + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yymsp[0].minor.yy144 = 0;} break; - case 218: /* expr ::= expr between_op expr AND expr */ + case 220: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 221: /* expr ::= expr in_op LP exprlist RP */ + case 223: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy322==0 ){ + if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -175454,208 +177072,208 @@ static YYACTIONTYPE yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy394 ? "true" : "false"); - if( yymsp[-4].minor.yy528 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy528); + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy144 ? "true" : "false"); + if( yymsp[-4].minor.yy454 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy454); }else{ - Expr *pRHS = yymsp[-1].minor.yy322->a[0].pExpr; - if( yymsp[-1].minor.yy322->nExpr==1 && sqlite3ExprIsConstant(pRHS) && yymsp[-4].minor.yy528->op!=TK_VECTOR ){ - yymsp[-1].minor.yy322->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + if( yymsp[-1].minor.yy14->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy454->op!=TK_VECTOR ){ + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy528, pRHS); - }else if( yymsp[-1].minor.yy322->nExpr==1 && pRHS->op==TK_SELECT ){ - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pRHS->x.pSelect); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy454, pRHS); + }else if( yymsp[-1].minor.yy14->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pRHS->x.pSelect); pRHS->x.pSelect = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); }else{ - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528==0 ){ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); - }else if( yymsp[-4].minor.yy528->pLeft->op==TK_VECTOR ){ - int nExpr = yymsp[-4].minor.yy528->pLeft->x.pList->nExpr; - Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy322); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + }else if( yymsp[-4].minor.yy454->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy454->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy14); if( pSelectRHS ){ parserDoubleLinkSelect(pParse, pSelectRHS); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelectRHS); } }else{ - yymsp[-4].minor.yy528->x.pList = yymsp[-1].minor.yy322; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy528); + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); } } - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } } break; - case 222: /* expr ::= LP select RP */ + case 224: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy454, yymsp[-1].minor.yy555); } break; - case 223: /* expr ::= expr in_op LP select RP */ + case 225: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47); - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, yymsp[-1].minor.yy555); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 224: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy322 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy322); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pSelect); - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[0].minor.yy14 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy14); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelect); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 225: /* expr ::= EXISTS LP select RP */ + case 227: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47); + p = yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy555); } break; - case 226: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = yymsp[-1].minor.yy528 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528) : yymsp[-2].minor.yy322; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy528); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy454 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy322); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy528); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } } break; - case 227: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[0].minor.yy454); } break; - case 228: /* case_exprlist ::= WHEN expr THEN expr */ + case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, yymsp[0].minor.yy454); } break; - case 233: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);} + case 235: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy454);} break; - case 234: /* nexprlist ::= expr */ -{yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/} + case 236: /* nexprlist ::= expr */ +{yymsp[0].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy454); /*A-overwrites-Y*/} break; - case 236: /* paren_exprlist ::= LP exprlist RP */ - case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241); -{yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;} + case 238: /* paren_exprlist ::= LP exprlist RP */ + case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243); +{yymsp[-2].minor.yy14 = yymsp[-1].minor.yy14;} break; - case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy528, SQLITE_SO_ASC, yymsp[-8].minor.yy394, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy144, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy454, SQLITE_SO_ASC, yymsp[-8].minor.yy144, SQLITE_IDXTYPE_APPDEF); if( IN_RENAME_OBJECT && pParse->pNewIndex ){ sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); } } break; - case 238: /* uniqueflag ::= UNIQUE */ - case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280); -{yymsp[0].minor.yy394 = OE_Abort;} + case 240: /* uniqueflag ::= UNIQUE */ + case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282); +{yymsp[0].minor.yy144 = OE_Abort;} break; - case 239: /* uniqueflag ::= */ -{yymsp[1].minor.yy394 = OE_None;} + case 241: /* uniqueflag ::= */ +{yymsp[1].minor.yy144 = OE_None;} break; - case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); + yymsp[-4].minor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); } break; - case 243: /* eidlist ::= nm collate sortorder */ + case 245: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/ + yymsp[-2].minor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); /*A-overwrites-Y*/ } break; - case 246: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);} + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy203, yymsp[-1].minor.yy144);} break; - case 247: /* cmd ::= VACUUM vinto */ -{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);} + case 249: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy454);} break; - case 248: /* cmd ::= VACUUM nm vinto */ -{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);} + case 250: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy454);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 253: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 256: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 257: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 260: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy427, &all); } break; - case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy144, yymsp[-4].minor.yy286.a, yymsp[-4].minor.yy286.b, yymsp[-2].minor.yy203, yymsp[0].minor.yy454, yymsp[-10].minor.yy144, yymsp[-8].minor.yy144); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 260: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ } + case 262: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 261: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy394 = TK_INSTEAD;} + case 263: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy144 = TK_INSTEAD;} break; - case 262: /* trigger_time ::= */ -{ yymsp[1].minor.yy394 = TK_BEFORE; } + case 264: /* trigger_time ::= */ +{ yymsp[1].minor.yy144 = TK_BEFORE; } break; - case 263: /* trigger_event ::= DELETE|INSERT */ - case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264); -{yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;} + case 265: /* trigger_event ::= DELETE|INSERT */ + case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); +{yymsp[0].minor.yy286.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy286.b = 0;} break; - case 265: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;} + case 267: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy286.a = TK_UPDATE; yymsp[-2].minor.yy286.b = yymsp[0].minor.yy132;} break; - case 266: /* when_clause ::= */ - case 285: /* key_opt ::= */ yytestcase(yyruleno==285); -{ yymsp[1].minor.yy528 = 0; } + case 268: /* when_clause ::= */ + case 287: /* key_opt ::= */ yytestcase(yyruleno==287); +{ yymsp[1].minor.yy454 = 0; } break; - case 267: /* when_clause ::= WHEN expr */ - case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286); -{ yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; } + case 269: /* when_clause ::= WHEN expr */ + case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288); +{ yymsp[-1].minor.yy454 = yymsp[0].minor.yy454; } break; - case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy33!=0 ); - yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33; - yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33; + assert( yymsp[-2].minor.yy427!=0 ); + yymsp[-2].minor.yy427->pLast->pNext = yymsp[-1].minor.yy427; + yymsp[-2].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy33!=0 ); - yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33; + assert( yymsp[-1].minor.yy427!=0 ); + yymsp[-1].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 270: /* trnm ::= nm DOT nm */ + case 272: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -175663,367 +177281,377 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 271: /* tridxby ::= INDEXED BY nm */ + case 273: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 272: /* tridxby ::= NOT INDEXED */ + case 274: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ -{yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);} - yymsp[-8].minor.yy33 = yylhsminor.yy33; + case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy203, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454, yymsp[-7].minor.yy144, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-8].minor.yy427 = yylhsminor.yy427; break; - case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/ + yylhsminor.yy427 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy132,yymsp[-2].minor.yy555,yymsp[-6].minor.yy144,yymsp[-1].minor.yy122,yymsp[-7].minor.yy168,yymsp[0].minor.yy168);/*yylhsminor.yy427-overwrites-yymsp[-6].minor.yy144*/ } - yymsp[-7].minor.yy33 = yylhsminor.yy33; + yymsp[-7].minor.yy427 = yylhsminor.yy427; break; - case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);} - yymsp[-5].minor.yy33 = yylhsminor.yy33; + case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy454, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-5].minor.yy427 = yylhsminor.yy427; break; - case 276: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/} - yymsp[-2].minor.yy33 = yylhsminor.yy33; + case 278: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy427 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy555, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); /*yylhsminor.yy427-overwrites-yymsp[-1].minor.yy555*/} + yymsp[-2].minor.yy427 = yylhsminor.yy427; break; - case 277: /* expr ::= RAISE LP IGNORE RP */ + case 279: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy528 ){ - yymsp[-3].minor.yy528->affExpr = OE_Ignore; + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy454 ){ + yymsp[-3].minor.yy454->affExpr = OE_Ignore; } } break; - case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 280: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy528 ) { - yymsp[-5].minor.yy528->affExpr = (char)yymsp[-3].minor.yy394; + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy454 ) { + yymsp[-5].minor.yy454->affExpr = (char)yymsp[-3].minor.yy144; } } break; - case 279: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy394 = OE_Rollback;} + case 281: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy144 = OE_Rollback;} break; - case 281: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy394 = OE_Fail;} + case 283: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy144 = OE_Fail;} break; - case 282: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 284: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy203,yymsp[-1].minor.yy144); } break; - case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528); + sqlite3Attach(pParse, yymsp[-3].minor.yy454, yymsp[-1].minor.yy454, yymsp[0].minor.yy454); } break; - case 284: /* cmd ::= DETACH database_kw_opt expr */ + case 286: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy528); + sqlite3Detach(pParse, yymsp[0].minor.yy454); } break; - case 287: /* cmd ::= REINDEX */ + case 289: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 288: /* cmd ::= REINDEX nm dbnm */ + case 290: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 289: /* cmd ::= ANALYZE */ + case 291: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 290: /* cmd ::= ANALYZE nm dbnm */ + case 292: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy203,&yymsp[0].minor.yy0); } break; - case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { - sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0); + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0); } break; - case 294: /* add_column_fullname ::= fullname */ + case 296: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy203); } break; - case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { - sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy203, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 296: /* cmd ::= create_vtab */ + case 298: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 297: /* cmd ::= create_vtab LP vtabarglist RP */ + case 299: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy144); } break; - case 299: /* vtabarg ::= */ + case 301: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 300: /* vtabargtoken ::= ANY */ - case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301); - case 302: /* lp ::= LP */ yytestcase(yyruleno==302); + case 302: /* vtabargtoken ::= ANY */ + case 303: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==303); + case 304: /* lp ::= LP */ yytestcase(yyruleno==304); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 303: /* with ::= WITH wqlist */ - case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); } + case 305: /* with ::= WITH wqlist */ + case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy59, 1); } break; - case 305: /* wqas ::= AS */ -{yymsp[0].minor.yy516 = M10d_Any;} + case 307: /* wqas ::= AS */ +{yymsp[0].minor.yy462 = M10d_Any;} break; - case 306: /* wqas ::= AS MATERIALIZED */ -{yymsp[-1].minor.yy516 = M10d_Yes;} + case 308: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy462 = M10d_Yes;} break; - case 307: /* wqas ::= AS NOT MATERIALIZED */ -{yymsp[-2].minor.yy516 = M10d_No;} + case 309: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy462 = M10d_No;} break; - case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ { - yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/ + yymsp[-5].minor.yy67 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy555, yymsp[-3].minor.yy462); /*A-overwrites-X*/ } break; - case 309: /* wqlist ::= wqitem */ + case 311: /* withnm ::= nm */ +{pParse->bHasWith = 1;} + break; + case 312: /* wqlist ::= wqitem */ { - yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/ + yymsp[0].minor.yy59 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy67); /*A-overwrites-X*/ } break; - case 310: /* wqlist ::= wqlist COMMA wqitem */ + case 313: /* wqlist ::= wqlist COMMA wqitem */ { - yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385); + yymsp[-2].minor.yy59 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy59, yymsp[0].minor.yy67); } break; - case 311: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { - assert( yymsp[0].minor.yy41!=0 ); - sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41); - yymsp[0].minor.yy41->pNextWin = yymsp[-2].minor.yy41; - yylhsminor.yy41 = yymsp[0].minor.yy41; + assert( yymsp[0].minor.yy211!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy211); + yymsp[0].minor.yy211->pNextWin = yymsp[-2].minor.yy211; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-2].minor.yy41 = yylhsminor.yy41; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 312: /* windowdefn ::= nm AS LP window RP */ + case 315: /* windowdefn ::= nm AS LP window RP */ { - if( ALWAYS(yymsp[-1].minor.yy41) ){ - yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + if( ALWAYS(yymsp[-1].minor.yy211) ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); } - yylhsminor.yy41 = yymsp[-1].minor.yy41; + yylhsminor.yy211 = yymsp[-1].minor.yy211; } - yymsp[-4].minor.yy41 = yylhsminor.yy41; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 313: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { - yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0); + yymsp[-4].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, 0); } break; - case 314: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy41 = yylhsminor.yy41; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 315: /* window ::= ORDER BY sortlist frame_opt */ + case 318: /* window ::= ORDER BY sortlist frame_opt */ { - yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0); + yymsp[-3].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, 0); } break; - case 316: /* window ::= nm ORDER BY sortlist frame_opt */ + case 319: /* window ::= nm ORDER BY sortlist frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy41 = yylhsminor.yy41; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 317: /* window ::= nm frame_opt */ + case 320: /* window ::= nm frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, 0, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy41 = yylhsminor.yy41; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 318: /* frame_opt ::= */ + case 321: /* frame_opt ::= */ { - yymsp[1].minor.yy41 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); + yymsp[1].minor.yy211 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 319: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { - yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy394, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy516); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy144, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy462); } - yymsp[-2].minor.yy41 = yylhsminor.yy41; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 320: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { - yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy394, yymsp[-3].minor.yy595.eType, yymsp[-3].minor.yy595.pExpr, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, yymsp[0].minor.yy516); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy144, yymsp[-3].minor.yy509.eType, yymsp[-3].minor.yy509.pExpr, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, yymsp[0].minor.yy462); } - yymsp[-5].minor.yy41 = yylhsminor.yy41; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 322: /* frame_bound_s ::= frame_bound */ - case 324: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==324); -{yylhsminor.yy595 = yymsp[0].minor.yy595;} - yymsp[0].minor.yy595 = yylhsminor.yy595; + case 325: /* frame_bound_s ::= frame_bound */ + case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327); +{yylhsminor.yy509 = yymsp[0].minor.yy509;} + yymsp[0].minor.yy509 = yylhsminor.yy509; break; - case 323: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 325: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==325); - case 327: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==327); -{yylhsminor.yy595.eType = yymsp[-1].major; yylhsminor.yy595.pExpr = 0;} - yymsp[-1].minor.yy595 = yylhsminor.yy595; + case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328); + case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330); +{yylhsminor.yy509.eType = yymsp[-1].major; yylhsminor.yy509.pExpr = 0;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 326: /* frame_bound ::= expr PRECEDING|FOLLOWING */ -{yylhsminor.yy595.eType = yymsp[0].major; yylhsminor.yy595.pExpr = yymsp[-1].minor.yy528;} - yymsp[-1].minor.yy595 = yylhsminor.yy595; + case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy509.eType = yymsp[0].major; yylhsminor.yy509.pExpr = yymsp[-1].minor.yy454;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 328: /* frame_exclude_opt ::= */ -{yymsp[1].minor.yy516 = 0;} + case 331: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy462 = 0;} break; - case 329: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ -{yymsp[-1].minor.yy516 = yymsp[0].minor.yy516;} + case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy462 = yymsp[0].minor.yy462;} break; - case 330: /* frame_exclude ::= NO OTHERS */ - case 331: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==331); -{yymsp[-1].minor.yy516 = yymsp[-1].major; /*A-overwrites-X*/} + case 333: /* frame_exclude ::= NO OTHERS */ + case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334); +{yymsp[-1].minor.yy462 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 332: /* frame_exclude ::= GROUP|TIES */ -{yymsp[0].minor.yy516 = yymsp[0].major; /*A-overwrites-X*/} + case 335: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy462 = yymsp[0].major; /*A-overwrites-X*/} break; - case 333: /* window_clause ::= WINDOW windowdefn_list */ -{ yymsp[-1].minor.yy41 = yymsp[0].minor.yy41; } + case 336: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy211 = yymsp[0].minor.yy211; } break; - case 334: /* filter_over ::= filter_clause over_clause */ + case 337: /* filter_over ::= filter_clause over_clause */ { - if( yymsp[0].minor.yy41 ){ - yymsp[0].minor.yy41->pFilter = yymsp[-1].minor.yy528; + if( yymsp[0].minor.yy211 ){ + yymsp[0].minor.yy211->pFilter = yymsp[-1].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy528); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } - yylhsminor.yy41 = yymsp[0].minor.yy41; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-1].minor.yy41 = yylhsminor.yy41; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 335: /* filter_over ::= over_clause */ + case 338: /* filter_over ::= over_clause */ { - yylhsminor.yy41 = yymsp[0].minor.yy41; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[0].minor.yy41 = yylhsminor.yy41; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 336: /* filter_over ::= filter_clause */ + case 339: /* filter_over ::= filter_clause */ { - yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yylhsminor.yy41 ){ - yylhsminor.yy41->eFrmType = TK_FILTER; - yylhsminor.yy41->pFilter = yymsp[0].minor.yy528; + yylhsminor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy211 ){ + yylhsminor.yy211->eFrmType = TK_FILTER; + yylhsminor.yy211->pFilter = yymsp[0].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy528); + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy454); } } - yymsp[0].minor.yy41 = yylhsminor.yy41; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 337: /* over_clause ::= OVER LP window RP */ + case 340: /* over_clause ::= OVER LP window RP */ { - yymsp[-3].minor.yy41 = yymsp[-1].minor.yy41; - assert( yymsp[-3].minor.yy41!=0 ); + yymsp[-3].minor.yy211 = yymsp[-1].minor.yy211; + assert( yymsp[-3].minor.yy211!=0 ); } break; - case 338: /* over_clause ::= OVER nm */ + case 341: /* over_clause ::= OVER nm */ { - yymsp[-1].minor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yymsp[-1].minor.yy41 ){ - yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + yymsp[-1].minor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy211 ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); } } break; - case 339: /* filter_clause ::= FILTER LP WHERE expr RP */ -{ yymsp[-4].minor.yy528 = yymsp[-1].minor.yy528; } + case 342: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy454 = yymsp[-1].minor.yy454; } + break; + case 343: /* term ::= QNUMBER */ +{ + yylhsminor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); + sqlite3DequoteNumber(pParse, yylhsminor.yy454); +} + yymsp[0].minor.yy454 = yylhsminor.yy454; break; default: - /* (340) input ::= cmdlist */ yytestcase(yyruleno==340); - /* (341) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==341); - /* (342) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=342); - /* (343) ecmd ::= SEMI */ yytestcase(yyruleno==343); - /* (344) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==344); - /* (345) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=345); - /* (346) trans_opt ::= */ yytestcase(yyruleno==346); - /* (347) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==347); - /* (348) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==348); - /* (349) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==349); - /* (350) savepoint_opt ::= */ yytestcase(yyruleno==350); - /* (351) cmd ::= create_table create_table_args */ yytestcase(yyruleno==351); - /* (352) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=352); - /* (353) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==353); - /* (354) columnlist ::= columnname carglist */ yytestcase(yyruleno==354); - /* (355) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==355); - /* (356) nm ::= STRING */ yytestcase(yyruleno==356); - /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); - /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); - /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); - /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); - /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); - /* (362) carglist ::= */ yytestcase(yyruleno==362); - /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); - /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); - /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); - /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); - /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); - /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); - /* (369) tconscomma ::= */ yytestcase(yyruleno==369); - /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); - /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); - /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); - /* (373) oneselect ::= values */ yytestcase(yyruleno==373); - /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); - /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); - /* (376) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=376); - /* (377) returning ::= */ yytestcase(yyruleno==377); - /* (378) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=378); - /* (379) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==379); - /* (380) case_operand ::= expr */ yytestcase(yyruleno==380); - /* (381) exprlist ::= nexprlist */ yytestcase(yyruleno==381); - /* (382) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=382); - /* (383) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=383); - /* (384) nmnum ::= ON */ yytestcase(yyruleno==384); - /* (385) nmnum ::= DELETE */ yytestcase(yyruleno==385); - /* (386) nmnum ::= DEFAULT */ yytestcase(yyruleno==386); - /* (387) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==387); - /* (388) foreach_clause ::= */ yytestcase(yyruleno==388); - /* (389) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==389); - /* (390) trnm ::= nm */ yytestcase(yyruleno==390); - /* (391) tridxby ::= */ yytestcase(yyruleno==391); - /* (392) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==392); - /* (393) database_kw_opt ::= */ yytestcase(yyruleno==393); - /* (394) kwcolumn_opt ::= */ yytestcase(yyruleno==394); - /* (395) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==395); - /* (396) vtabarglist ::= vtabarg */ yytestcase(yyruleno==396); - /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==397); - /* (398) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==398); - /* (399) anylist ::= */ yytestcase(yyruleno==399); - /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400); - /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401); - /* (402) with ::= */ yytestcase(yyruleno==402); - /* (403) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=403); - /* (404) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=404); + /* (344) input ::= cmdlist */ yytestcase(yyruleno==344); + /* (345) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==345); + /* (346) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=346); + /* (347) ecmd ::= SEMI */ yytestcase(yyruleno==347); + /* (348) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==348); + /* (349) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=349); + /* (350) trans_opt ::= */ yytestcase(yyruleno==350); + /* (351) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==351); + /* (352) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==352); + /* (353) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==353); + /* (354) savepoint_opt ::= */ yytestcase(yyruleno==354); + /* (355) cmd ::= create_table create_table_args */ yytestcase(yyruleno==355); + /* (356) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==357); + /* (358) columnlist ::= columnname carglist */ yytestcase(yyruleno==358); + /* (359) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==359); + /* (360) nm ::= STRING */ yytestcase(yyruleno==360); + /* (361) typetoken ::= typename */ yytestcase(yyruleno==361); + /* (362) typename ::= ID|STRING */ yytestcase(yyruleno==362); + /* (363) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); + /* (364) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); + /* (365) carglist ::= carglist ccons */ yytestcase(yyruleno==365); + /* (366) carglist ::= */ yytestcase(yyruleno==366); + /* (367) ccons ::= NULL onconf */ yytestcase(yyruleno==367); + /* (368) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==368); + /* (369) ccons ::= AS generated */ yytestcase(yyruleno==369); + /* (370) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==370); + /* (371) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==371); + /* (372) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) tconscomma ::= */ yytestcase(yyruleno==373); + /* (374) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=374); + /* (375) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=375); + /* (376) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) oneselect ::= values */ yytestcase(yyruleno==377); + /* (378) sclp ::= selcollist COMMA */ yytestcase(yyruleno==378); + /* (379) as ::= ID|STRING */ yytestcase(yyruleno==379); + /* (380) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) returning ::= */ yytestcase(yyruleno==381); + /* (382) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==383); + /* (384) case_operand ::= expr */ yytestcase(yyruleno==384); + /* (385) exprlist ::= nexprlist */ yytestcase(yyruleno==385); + /* (386) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=386); + /* (387) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=387); + /* (388) nmnum ::= ON */ yytestcase(yyruleno==388); + /* (389) nmnum ::= DELETE */ yytestcase(yyruleno==389); + /* (390) nmnum ::= DEFAULT */ yytestcase(yyruleno==390); + /* (391) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==391); + /* (392) foreach_clause ::= */ yytestcase(yyruleno==392); + /* (393) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==393); + /* (394) trnm ::= nm */ yytestcase(yyruleno==394); + /* (395) tridxby ::= */ yytestcase(yyruleno==395); + /* (396) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==396); + /* (397) database_kw_opt ::= */ yytestcase(yyruleno==397); + /* (398) kwcolumn_opt ::= */ yytestcase(yyruleno==398); + /* (399) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==399); + /* (400) vtabarglist ::= vtabarg */ yytestcase(yyruleno==400); + /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==401); + /* (402) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==402); + /* (403) anylist ::= */ yytestcase(yyruleno==403); + /* (404) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==404); + /* (405) anylist ::= anylist ANY */ yytestcase(yyruleno==405); + /* (406) with ::= */ yytestcase(yyruleno==406); + /* (407) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=407); + /* (408) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=408); break; /********** End reduce actions ************************************************/ }; @@ -176210,19 +177838,12 @@ SQLITE_PRIVATE void sqlite3Parser( (int)(yypParser->yytos - yypParser->yystack)); } #endif -#if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - break; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); break; } } -#endif } yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ @@ -177293,27 +178914,58 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; @@ -177478,10 +179130,13 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ if( tokenType>=TK_WINDOW ){ assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER ); #else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER + ); #endif /* SQLITE_OMIT_WINDOWFUNC */ if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; @@ -177514,7 +179169,7 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else{ + }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; x.n = n; @@ -178865,6 +180520,18 @@ SQLITE_API int sqlite3_config(int op, ...){ } #endif /* SQLITE_OMIT_DESERIALIZE */ + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + default: { rc = SQLITE_ERROR; break; @@ -188495,22 +190162,24 @@ static int fts3IntegrityMethod( char **pzErr /* Write error message here */ ){ Fts3Table *p = (Fts3Table*)pVtab; - int rc; + int rc = SQLITE_OK; int bOk = 0; UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); - assert( rc!=SQLITE_CORRUPT_VTAB || bOk==0 ); - if( rc!=SQLITE_OK && rc!=SQLITE_CORRUPT_VTAB ){ + assert( rc!=SQLITE_CORRUPT_VTAB ); + if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" " FTS%d table %s.%s: %s", p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); - }else if( bOk==0 ){ + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; } sqlite3Fts3SegmentsClose(p); - return SQLITE_OK; + return rc; } @@ -200172,7 +201841,12 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -201078,7 +202752,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } @@ -203739,7 +205413,6 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ } } - /* Append formatted text (not to exceed N bytes) to the JsonString. */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ @@ -203797,6 +205470,40 @@ static void jsonAppendSeparator(JsonString *p){ jsonAppendChar(p, ','); } +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. +*/ +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && cnUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; + } +} + /* Append the N-byte string in zIn to the end of the JsonString string ** under construction. Enclose the string in double-quotes ("...") and ** escape any double-quotes or backslash characters contained within the @@ -203856,35 +205563,14 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ } c = z[0]; if( c=='"' || c=='\\' ){ - json_simple_escape: if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ p->zBuf[p->nUsed++] = c; }else{ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - assert( c>=0 && cnUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; + jsonAppendControlChar(p, c); } z++; N--; @@ -204585,7 +206271,10 @@ static u32 jsonbValidityCheck( if( !jsonIsOk[z[j]] && z[j]!='\'' ){ if( z[j]=='"' ){ if( x==JSONB_TEXTJ ) return j+1; - }else if( z[j]!='\\' || j+1>=k ){ + }else if( z[j]<=0x1f ){ + /* Control characters in JSON5 string literals are ok */ + if( x==JSONB_TEXTJ ) return j+1; + }else if( NEVER(z[j]!='\\') || j+1>=k ){ return j+1; }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ j++; @@ -204783,6 +206472,7 @@ json_parse_restart: case '[': { /* Parse array */ iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); iStart = pParse->nBlob; if( pParse->oom ) return -1; @@ -204879,9 +206569,14 @@ json_parse_restart: return -1; } }else if( c<=0x1f ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; + if( c==0 ){ + pParse->iErr = j; + return -1; + } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; }else if( c=='"' ){ opcode = JSONB_TEXT5; } @@ -205097,6 +206792,7 @@ json_parse_restart: return i+4; } /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through } default: { u32 k; @@ -205181,6 +206877,10 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; memset(&px, 0, sizeof(px)); jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); @@ -205361,7 +207061,7 @@ static u32 jsonTranslateBlobToText( zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); while( sz2>0 ){ - for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); if( k>=sz2 ){ @@ -205376,6 +207076,13 @@ static u32 jsonTranslateBlobToText( sz2--; continue; } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } assert( zIn[0]=='\\' ); assert( sz2>=1 ); if( sz2<2 ){ @@ -205478,6 +207185,112 @@ static u32 jsonTranslateBlobToText( return i+n+sz; } +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jjnIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } +} + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, ']'); + i = iEnd; + break; + } + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, '}'); + i = iEnd; + break; + } + default: { + i = jsonTranslateBlobToText(pParse, i, pOut); + break; + } + } + return i; +} + + /* Return true if the input pJson ** ** For performance reasons, this routine does not do a detailed check of the @@ -206506,8 +208319,9 @@ rebuild_from_cache: } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; if( p->nJson==0 ) goto json_pfa_malformed; - if( NEVER(p->zJson==0) ) goto json_pfa_oom; + assert( p->zJson!=0 ); if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; @@ -206673,10 +208487,10 @@ static void jsonDebugPrintBlob( if( sz==0 && x<=JSONB_FALSE ){ sqlite3_str_append(pOut, "\n", 1); }else{ - u32 i; + u32 j; sqlite3_str_appendall(pOut, ": \""); - for(i=iStart+n; iaBlob[i]; + for(j=iStart+n; jaBlob[j]; if( c<0x20 || c>=0x7f ) c = '.'; sqlite3_str_append(pOut, (char*)&c, 1); } @@ -206727,11 +208541,12 @@ static void jsonParseFunc( if( p==0 ) return; if( argc==1 ){ jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); - sqlite3_result_text64(ctx, out.zText, out.nChar, SQLITE_DYNAMIC, SQLITE_UTF8); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); }else{ jsonShowParse(p); } jsonParseFree(p); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -206830,13 +208645,6 @@ static void jsonArrayLengthFunc( jsonParseFree(p); } -/* True if the string is all digits */ -static int jsonAllDigits(const char *z, int n){ - int i; - for(i=0; i $[NUMBER] // Not PG. Purely for convenience */ jsonStringInit(&jx, ctx); - if( jsonAllDigits(zPath, nPath) ){ + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ jsonAppendRawNZ(&jx, "[", 1); jsonAppendRaw(&jx, zPath, nPath); jsonAppendRawNZ(&jx, "]", 2); @@ -207395,6 +209203,40 @@ json_type_done: jsonParseFree(p); } +/* +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. +** +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). +*/ +static void jsonPrettyFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; + }else{ + x.szIndent = (u32)strlen(x.zIndent); + } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); +} + /* ** json_valid(JSON) ** json_valid(JSON, FLAGS) @@ -208084,6 +209926,9 @@ static int jsonEachColumn( case JEACH_VALUE: { u32 i = jsonSkipLabel(p); jsonReturnFromBlob(&p->sParse, i, ctx, 1); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } break; } case JEACH_TYPE: { @@ -208130,9 +209975,9 @@ static int jsonEachColumn( case JEACH_JSON: { if( p->sParse.zJson==0 ){ sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); + SQLITE_TRANSIENT); }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); } break; } @@ -208406,6 +210251,8 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), @@ -209158,11 +211005,9 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ ** Clear the Rtree.pNodeBlob object */ static void nodeBlobReset(Rtree *pRtree){ - if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ - sqlite3_blob *pBlob = pRtree->pNodeBlob; - pRtree->pNodeBlob = 0; - sqlite3_blob_close(pBlob); - } + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); } /* @@ -209206,7 +211051,6 @@ static int nodeAcquire( &pRtree->pNodeBlob); } if( rc ){ - nodeBlobReset(pRtree); *ppNode = 0; /* If unable to open an sqlite3_blob on the desired row, that can only ** be because the shadow tables hold erroneous data. */ @@ -209266,6 +211110,7 @@ static int nodeAcquire( } *ppNode = pNode; }else{ + nodeBlobReset(pRtree); if( pNode ){ pRtree->nNodeRef--; sqlite3_free(pNode); @@ -209410,6 +211255,7 @@ static void nodeGetCoord( int iCoord, /* Which coordinate to extract */ RtreeCoord *pCoord /* OUT: Space to write result to */ ){ + assert( iCellzData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -209599,7 +211445,9 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ sqlite3_finalize(pCsr->pReadAux); sqlite3_free(pCsr); pRtree->nCursor--; - nodeBlobReset(pRtree); + if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){ + nodeBlobReset(pRtree); + } return SQLITE_OK; } @@ -210184,7 +212032,11 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc==SQLITE_OK && ALWAYS(p) ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + if( p->iCell>=NCELL(pNode) ){ + rc = SQLITE_ABORT; + }else{ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } } return rc; } @@ -210202,6 +212054,7 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( rc ) return rc; if( NEVER(p==0) ) return SQLITE_OK; + if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else if( i<=pRtree->nDim2 ){ @@ -210299,6 +212152,8 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_OK; } +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); + /* ** Rtree virtual table module xFilter method. */ @@ -210328,7 +212183,8 @@ static int rtreeFilter( i64 iNode = 0; int eType = sqlite3_value_numeric_type(argv[0]); if( eType==SQLITE_INTEGER - || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) ){ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); }else{ @@ -211684,7 +213540,7 @@ constraint: static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; assert( pRtree->inWrTrans==0 ); - pRtree->inWrTrans++; + pRtree->inWrTrans = 1; return SQLITE_OK; } @@ -211698,6 +213554,9 @@ static int rtreeEndTransaction(sqlite3_vtab *pVtab){ nodeBlobReset(pRtree); return SQLITE_OK; } +static int rtreeRollback(sqlite3_vtab *pVtab){ + return rtreeEndTransaction(pVtab); +} /* ** The xRename method for rtree module virtual tables. @@ -211816,7 +213675,7 @@ static sqlite3_module rtreeModule = { rtreeBeginTransaction, /* xBegin - begin transaction */ rtreeEndTransaction, /* xSync - sync transaction */ rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeRollback, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ @@ -215235,7 +217094,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -215250,7 +217109,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; ipTblIter, &p->zErrmsg); pIter->zTbl = 0; + pIter->zDataTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); @@ -219203,7 +221097,7 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ u32 volatile *ptr; p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; + iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]); } } return iRet; @@ -226674,14 +228568,14 @@ static int sessionChangesetNextOne( p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + /* If the iterator is already at the end of the changeset, return DONE. */ if( p->in.iNext>=p->in.nData ){ return SQLITE_DONE; } - sessionDiscardData(&p->in); - p->in.iCurrent = p->in.iNext; - op = p->in.aData[p->in.iNext++]; while( op=='T' || op=='P' ){ if( pbNew ) *pbNew = 1; @@ -228416,6 +230310,7 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + SessionBuffer rec; sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ @@ -228714,108 +230609,128 @@ static int sessionChangesetExtendRecord( } /* -** Add all changes in the changeset traversed by the iterator passed as -** the first argument to the changegroup hash tables. +** Locate or create a SessionTable object that may be used to add the +** change currently pointed to by iterator pIter to changegroup pGrp. +** If successful, set output variable (*ppTab) to point to the table +** object and return SQLITE_OK. Otherwise, if some error occurs, return +** an SQLite error code and leave (*ppTab) set to NULL. */ -static int sessionChangesetToHash( - sqlite3_changeset_iter *pIter, /* Iterator to read from */ - sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ - int bRebase /* True if hash table is for rebasing */ +static int sessionChangesetFindTable( + sqlite3_changegroup *pGrp, + const char *zTab, + sqlite3_changeset_iter *pIter, + SessionTable **ppTab ){ - u8 *aRec; - int nRec; int rc = SQLITE_OK; SessionTable *pTab = 0; - SessionBuffer rec = {0, 0, 0}; + int nTab = (int)strlen(zTab); + u8 *abPK = 0; + int nCol = 0; - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ - const char *zNew; - int nCol; - int op; - int iHash; - int bIndirect; - SessionChange *pChange; - SessionChange *pExist = 0; - SessionChange **pp; + *ppTab = 0; + sqlite3changeset_pk(pIter, &abPK, &nCol); - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - break; + /* Search the list for an existing table */ + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; + } + + /* If one was not found above, create a new table now */ + if( !pTab ){ + SessionTable **ppNew; + + pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nTab+1); + if( !pTab ){ + return SQLITE_NOMEM; } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zTab, nTab+1); - sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); - if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ - /* Search the list for a matching table */ - int nNew = (int)strlen(zNew); - u8 *abPK; - - sqlite3changeset_pk(pIter, &abPK, 0); - for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; - } - if( !pTab ){ - SessionTable **ppTab; - - pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nNew+1); - if( !pTab ){ - rc = SQLITE_NOMEM; - break; - } - memset(pTab, 0, sizeof(SessionTable)); - pTab->nCol = nCol; - pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); - pTab->zName = (char*)&pTab->abPK[nCol]; - memcpy(pTab->zName, zNew, nNew+1); - - if( pGrp->db ){ - pTab->nCol = 0; - rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); - sqlite3_free(pTab); - break; - } - } - - /* The new object must be linked on to the end of the list, not - ** simply added to the start of it. This is to ensure that the - ** tables within the output of sqlite3changegroup_output() are in - ** the right order. */ - for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); - *ppTab = pTab; - } - - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ - rc = SQLITE_SCHEMA; - break; + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + return rc; } } - if( nColnCol ){ - assert( pGrp->db ); - rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, &rec); - if( rc ) break; - aRec = rec.aBuf; - nRec = rec.nBuf; - } + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppNew=&pGrp->pList; *ppNew; ppNew=&(*ppNew)->pNext); + *ppNew = pTab; + } - if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ - rc = SQLITE_NOMEM; - break; - } + /* Check that the table is compatible. */ + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + rc = SQLITE_SCHEMA; + } + + *ppTab = pTab; + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + int rc = SQLITE_OK; + int nCol = 0; + int op = 0; + int iHash = 0; + int bIndirect = 0; + SessionChange *pChange = 0; + SessionChange *pExist = 0; + SessionChange **pp = 0; + SessionTable *pTab = 0; + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + const char *zTab = 0; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK && nColnCol ){ + SessionBuffer *pBuf = &pGrp->rec; + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); + aRec = pBuf->aBuf; + nRec = pBuf->nBuf; + assert( pGrp->db ); + } + + if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. */ iHash = sessionChangeHash( pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); - - /* Search for existing entry. If found, remove it from the hash table. - ** Code below may link it back in. - */ for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; @@ -228830,19 +230745,41 @@ static int sessionChangesetToHash( break; } } + } + if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); - if( rc ) break; - if( pChange ){ - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; - pTab->nEntry++; - } + } + if( rc==SQLITE_OK && pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ + int bRebase /* True if hash table is for rebasing */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + + while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ + rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + if( rc!=SQLITE_OK ) break; } - sqlite3_free(rec.aBuf); if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -228970,6 +230907,23 @@ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void return rc; } +/* +** Add a single change to a changeset-group. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter +){ + if( pIter->in.iCurrent==pIter->in.iNext + || pIter->rc!=SQLITE_OK + || pIter->bInvert + ){ + /* Iterator does not point to any valid entry or is an INVERT iterator. */ + return SQLITE_ERROR; + } + return sessionOneChangeToHash(pGrp, pIter, 0); +} + /* ** Obtain a buffer containing a changeset representing the concatenation ** of all changesets added to the group so far. @@ -229019,6 +230973,7 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); + sqlite3_free(pGrp->rec.aBuf); sqlite3_free(pGrp); } } @@ -229420,6 +231375,7 @@ SQLITE_API int sqlite3rebaser_rebase_strm( SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p){ if( p ){ sessionDeleteTable(0, p->grp.pList); + sqlite3_free(p->grp.rec.aBuf); sqlite3_free(p); } } @@ -229517,8 +231473,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -231114,6 +233070,9 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser ** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser ** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context +** fts5YYREALLOC Name of the realloc() function to use +** fts5YYFREE Name of the free() function to use +** fts5YYDYNSTACK True if stack space should be extended on heap ** fts5YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** fts5YYNSTATE the combined number of states. @@ -231127,6 +233086,8 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** fts5YY_NO_ACTION The fts5yy_action[] code for no-op ** fts5YY_MIN_REDUCE Minimum value for reduce actions ** fts5YY_MAX_REDUCE Maximum value for reduce actions +** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 @@ -231153,6 +233114,9 @@ typedef union { #define sqlite3Fts5ParserARG_PARAM ,pParse #define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; #define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#define fts5YYREALLOC realloc +#define fts5YYFREE free +#define fts5YYDYNSTACK 0 #define sqlite3Fts5ParserCTX_SDECL #define sqlite3Fts5ParserCTX_PDECL #define sqlite3Fts5ParserCTX_PARAM @@ -231170,6 +233134,8 @@ typedef union { #define fts5YY_NO_ACTION 82 #define fts5YY_MIN_REDUCE 83 #define fts5YY_MAX_REDUCE 110 +#define fts5YY_MIN_DSTRCTR 16 +#define fts5YY_MAX_DSTRCTR 24 /************* End control #defines *******************************************/ #define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) @@ -231185,6 +233151,22 @@ typedef union { # define fts5yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if fts5YYSTACKDEPTH<=0 || fts5YYDYNSTACK +# define fts5YYGROWABLESTACK 1 +#else +# define fts5YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if fts5YYSTACKDEPTH<=0 +# undef fts5YYSTACKDEPTH +# define fts5YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -231345,14 +233327,9 @@ struct fts5yyParser { #endif sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */ -#if fts5YYSTACKDEPTH<=0 - int fts5yystksz; /* Current side of the stack */ - fts5yyStackEntry *fts5yystack; /* The parser's stack */ - fts5yyStackEntry fts5yystk0; /* First stack entry */ -#else - fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ - fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ -#endif + fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ + fts5yyStackEntry *fts5yystack; /* The parser stack */ + fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct fts5yyParser fts5yyParser; @@ -231459,37 +233436,45 @@ static const char *const fts5yyRuleName[] = { #endif /* NDEBUG */ -#if fts5YYSTACKDEPTH<=0 +#if fts5YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int fts5yyGrowStack(fts5yyParser *p){ + int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack); int newSize; int idx; fts5yyStackEntry *pNew; - newSize = p->fts5yystksz*2 + 100; - idx = p->fts5yytos ? (int)(p->fts5yytos - p->fts5yystack) : 0; - if( p->fts5yystack==&p->fts5yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->fts5yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->fts5yytos - p->fts5yystack); + if( p->fts5yystack==p->fts5yystk0 ){ + pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); + pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->fts5yystack = pNew; - p->fts5yytos = &p->fts5yystack[idx]; + p->fts5yystack = pNew; + p->fts5yytos = &p->fts5yystack[idx]; #ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", - fts5yyTracePrompt, p->fts5yystksz, newSize); - } -#endif - p->fts5yystksz = newSize; + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", + fts5yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->fts5yystackEnd = &p->fts5yystack[newSize-1]; + return 0; } +#endif /* fts5YYGROWABLESTACK */ + +#if !fts5YYGROWABLESTACK +/* For builds that do no have a growable stack, fts5yyGrowStack always +** returns an error. +*/ +# define fts5yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -231509,24 +233494,14 @@ static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PD #ifdef fts5YYTRACKMAXSTACKDEPTH fts5yypParser->fts5yyhwm = 0; #endif -#if fts5YYSTACKDEPTH<=0 - fts5yypParser->fts5yytos = NULL; - fts5yypParser->fts5yystack = NULL; - fts5yypParser->fts5yystksz = 0; - if( fts5yyGrowStack(fts5yypParser) ){ - fts5yypParser->fts5yystack = &fts5yypParser->fts5yystk0; - fts5yypParser->fts5yystksz = 1; - } -#endif + fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0; + fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; #ifndef fts5YYNOERRORRECOVERY fts5yypParser->fts5yyerrcnt = -1; #endif fts5yypParser->fts5yytos = fts5yypParser->fts5yystack; fts5yypParser->fts5yystack[0].stateno = 0; fts5yypParser->fts5yystack[0].major = 0; -#if fts5YYSTACKDEPTH>0 - fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK @@ -231640,9 +233615,26 @@ static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ */ static void sqlite3Fts5ParserFinalize(void *p){ fts5yyParser *pParser = (fts5yyParser*)p; - while( pParser->fts5yytos>pParser->fts5yystack ) fts5yy_pop_parser_stack(pParser); -#if fts5YYSTACKDEPTH<=0 - if( pParser->fts5yystack!=&pParser->fts5yystk0 ) free(pParser->fts5yystack); + + /* In-lined version of calling fts5yy_pop_parser_stack() for each + ** element left in the stack */ + fts5yyStackEntry *fts5yytos = pParser->fts5yytos; + while( fts5yytos>pParser->fts5yystack ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + if( fts5yytos->major>=fts5YY_MIN_DSTRCTR ){ + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); + } + fts5yytos--; + } + +#if fts5YYGROWABLESTACK + if( pParser->fts5yystack!=pParser->fts5yystk0 ) fts5YYFREE(pParser->fts5yystack); #endif } @@ -231869,25 +233861,19 @@ static void fts5yy_shift( assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); } #endif -#if fts5YYSTACKDEPTH>0 - if( fts5yypParser->fts5yytos>fts5yypParser->fts5yystackEnd ){ - fts5yypParser->fts5yytos--; - fts5yyStackOverflow(fts5yypParser); - return; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz] ){ + fts5yytos = fts5yypParser->fts5yytos; + if( fts5yytos>fts5yypParser->fts5yystackEnd ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yypParser->fts5yytos--; fts5yyStackOverflow(fts5yypParser); return; } + fts5yytos = fts5yypParser->fts5yytos; + assert( fts5yytos <= fts5yypParser->fts5yystackEnd ); } -#endif if( fts5yyNewState > fts5YY_MAX_SHIFT ){ fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; } - fts5yytos = fts5yypParser->fts5yytos; fts5yytos->stateno = fts5yyNewState; fts5yytos->major = fts5yyMajor; fts5yytos->minor.fts5yy0 = fts5yyMinor; @@ -232324,19 +234310,12 @@ static void sqlite3Fts5Parser( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); } #endif -#if fts5YYSTACKDEPTH>0 if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){ - fts5yyStackOverflow(fts5yypParser); - break; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz-1] ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yyStackOverflow(fts5yypParser); break; } } -#endif } fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM); }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ @@ -235013,7 +236992,11 @@ static int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } @@ -237141,6 +239124,7 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -245375,23 +247359,26 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ int ii; Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; for(ii=0; iinIter; ii++){ Fts5Iter *p = pT->apIter[ii]; if( p->base.bEof==0 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidpIndex, p, bFrom, iFrom); + fts5MultiIterNext(pIndex, p, bFrom, iFrom); while( bFrom && p->base.bEof==0 && p->base.iRowidpIndex->rc==SQLITE_OK + && pIndex->rc==SQLITE_OK ){ - fts5MultiIterNext(p->pIndex, p, 0, 0); + fts5MultiIterNext(pIndex, p, 0, 0); } } } - fts5IterSetOutputsTokendata(pIter); + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } } /* @@ -249305,6 +251292,7 @@ static int fts5UpdateMethod( rc = SQLITE_ERROR; }else{ rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); @@ -250479,14 +252467,16 @@ static int sqlite3Fts5GetTokenizer( if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ rc = pMod->x.xCreate( pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok ); pConfig->pTokApi = &pMod->x; if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + if( pzErr && rc!=SQLITE_NOMEM ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } }else{ pConfig->ePattern = sqlite3Fts5TokenizerPattern( pMod->x.xCreate, pConfig->pTok @@ -250545,7 +252535,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257cc467a", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33", -1, SQLITE_TRANSIENT); } /* @@ -250580,18 +252570,25 @@ static int fts5IntegrityMethod( assert( pzErr!=0 && *pzErr==0 ); UNUSED_PARAM(isQuick); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); - if( (rc&0xff)==SQLITE_CORRUPT ){ - *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", - zSchema, zTabname); - }else if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unable to validate the inverted index for" - " FTS5 table %s.%s: %s", - zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } } - sqlite3Fts5IndexCloseReader(pTab->p.pIndex); - return SQLITE_OK; + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; + + return rc; } static int fts5Init(sqlite3 *db){ @@ -252023,7 +254020,7 @@ static int fts5AsciiCreate( int i; memset(p, 0, sizeof(AsciiTokenizer)); memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); - for(i=0; rc==SQLITE_OK && ibFold = 1; pNew->iFoldParam = 0; - for(i=0; rc==SQLITE_OK && iiFoldParam!=0 && pNew->bFold==0 ){ rc = SQLITE_ERROR; diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index 4fdfde00..f64ca017 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.1" -#define SQLITE_VERSION_NUMBER 3045001 -#define SQLITE_SOURCE_ID "2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257cc467a" +#define SQLITE_VERSION "3.46.1" +#define SQLITE_VERSION_NUMBER 3046001 +#define SQLITE_SOURCE_ID "2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -420,6 +420,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **

  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ SQLITE_API int sqlite3_exec( @@ -762,11 +764,11 @@ struct sqlite3_file { ** ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, @@ -2141,6 +2143,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2172,6 +2190,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -3286,8 +3305,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -6868,6 +6887,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -8338,7 +8363,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -9917,24 +9942,45 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **

  • ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. **

  • -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** ** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** ** ^For the purposes of comparing virtual table output values to see if the ** values are same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" @@ -11979,6 +12025,30 @@ SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const c */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -12783,8 +12853,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken diff --git a/src/datastructure.c b/src/datastructure.c index 6a48c9a5..7d2504fa 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -69,7 +69,7 @@ int findQueryID(const int id) // Check UUIDs of queries for(int i = start; i >= until; i--) { - const queriesData* query = getQuery(i, true); + const queriesData *query = getQuery(i, true); // Check if the returned pointer is valid before trying to access it if(query == NULL) @@ -239,7 +239,8 @@ static int get_next_free_clientID(void) return counters->clients; } -int _findClientID(const char *clientIP, const bool count, const bool aliasclient, int line, const char *func, const char *file) +int _findClientID(const char *clientIP, const bool count, const bool aliasclient, + const double now, int line, const char *func, const char *file) { // Compare content of client against known client IP addresses for(int clientID=0; clientID < counters->clients; clientID++) @@ -308,7 +309,7 @@ int _findClientID(const char *clientIP, const bool count, const bool aliasclient // some time after adding a client to ensure we pick up possible // group configuration though hostname, MAC address or interface client->reread_groups = 0u; - client->firstSeen = time(NULL); + client->firstSeen = now; // Interface is not yet known client->ifacepos = 0; // Set all MAC address bytes to zero @@ -434,7 +435,8 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q // Initialize cache entry dns_cache->magic = MAGICBYTE; - dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->blocking_status = QUERY_UNKNOWN; + dns_cache->expires = 0; dns_cache->domainID = domainID; dns_cache->clientID = clientID; dns_cache->query_type = query_type; @@ -461,7 +463,7 @@ bool isValidIPv6(const char *addr) // Privacy-level sensitive subroutine that returns the domain name // only when appropriate for the requested query -const char *getDomainString(const queriesData* query) +const char *getDomainString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->domainID < 0) @@ -485,7 +487,7 @@ const char *getDomainString(const queriesData* query) // Privacy-level sensitive subroutine that returns the domain name // only when appropriate for the requested query -const char *getCNAMEDomainString(const queriesData* query) +const char *getCNAMEDomainString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->CNAME_domainID < 0) @@ -509,7 +511,7 @@ const char *getCNAMEDomainString(const queriesData* query) // Privacy-level sensitive subroutine that returns the client IP // only when appropriate for the requested query -const char *getClientIPString(const queriesData* query) +const char *getClientIPString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->clientID < 0) @@ -533,7 +535,7 @@ const char *getClientIPString(const queriesData* query) // Privacy-level sensitive subroutine that returns the client host name // only when appropriate for the requested query -const char *getClientNameString(const queriesData* query) +const char *getClientNameString(const queriesData *query) { // Check if the returned pointer is valid before trying to access it if(query == NULL || query->clientID < 0) @@ -569,7 +571,9 @@ void FTL_reset_per_client_domain_data(void) continue; // Reset blocking status - dns_cache->blocking_status = UNKNOWN_BLOCKED; + dns_cache->blocking_status = QUERY_UNKNOWN; + // Reset expiry + dns_cache->expires = 0; // Reset domainlist ID dns_cache->list_id = -1; } @@ -590,11 +594,13 @@ void FTL_reload_all_domainlists(void) counters->database.groups = gravityDB_count(GROUPS_TABLE); counters->database.clients = gravityDB_count(CLIENTS_TABLE); counters->database.lists = gravityDB_count(ADLISTS_TABLE); - counters->database.domains.allowed = gravityDB_count(DENIED_DOMAINS_TABLE); - counters->database.domains.denied = gravityDB_count(ALLOWED_DOMAINS_TABLE); + counters->database.domains.allowed.exact = gravityDB_count(EXACT_WHITELIST_TABLE); + counters->database.domains.denied.exact = gravityDB_count(EXACT_BLACKLIST_TABLE); + counters->database.domains.allowed.regex = gravityDB_count(REGEX_ALLOW_TABLE); + counters->database.domains.denied.regex = gravityDB_count(REGEX_DENY_TABLE); // Read and compile possible regex filters - // only after having called gravityDB_open() + // only after having called gravityDB_reopen() read_regex_from_database(); // Check for inaccessible adlist URLs @@ -699,6 +705,8 @@ const char * __attribute__ ((const)) get_query_status_str(const enum query_statu return "SPECIAL_DOMAIN"; case QUERY_CACHE_STALE: return "CACHE_STALE"; + case QUERY_EXTERNAL_BLOCKED_EDE15: + return "EXTERNAL_BLOCKED_EDE15"; case QUERY_STATUS_MAX: default: return "INVALID"; @@ -834,6 +842,22 @@ int __attribute__ ((pure)) get_blocking_mode_val(const char *blocking_mode) return -1; } +const char * __attribute__ ((const)) get_blocking_status_str(const enum blocking_status blocking) +{ + switch(blocking) + { + case BLOCKING_ENABLED: + return "enabled"; + case BLOCKING_DISABLED: + return "disabled"; + case DNS_FAILED: + return "failure"; + case BLOCKING_UNKNOWN: + default: + return "unknown"; + } +} + bool __attribute__ ((const)) is_blocked(const enum query_status status) { switch (status) @@ -855,6 +879,7 @@ bool __attribute__ ((const)) is_blocked(const enum query_status status) case QUERY_EXTERNAL_BLOCKED_IP: case QUERY_EXTERNAL_BLOCKED_NULL: case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: case QUERY_GRAVITY_CNAME: case QUERY_REGEX_CNAME: case QUERY_DENYLIST_CNAME: @@ -952,6 +977,7 @@ bool __attribute__ ((const)) is_cached(const enum query_status status) case QUERY_EXTERNAL_BLOCKED_IP: case QUERY_EXTERNAL_BLOCKED_NULL: case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: case QUERY_GRAVITY_CNAME: case QUERY_REGEX_CNAME: case QUERY_DENYLIST_CNAME: @@ -1002,6 +1028,8 @@ static const char* __attribute__ ((const)) query_status_str(const enum query_sta return "SPECIAL_DOMAIN"; case QUERY_CACHE_STALE: return "CACHE_STALE"; + case QUERY_EXTERNAL_BLOCKED_EDE15: + return "EXTERNAL_BLOCKED_EDE15"; case QUERY_STATUS_MAX: return NULL; } @@ -1046,6 +1074,59 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c return; } + // Memorize this in the DNS cache if blocked due to the response + // We do not cache intermittent statuses as they are subject to change + if(!init && + new_status != QUERY_UNKNOWN && + new_status != QUERY_DBBUSY && + new_status != QUERY_IN_PROGRESS && + new_status != QUERY_RETRIED && + new_status != QUERY_RETRIED_DNSSEC) + { + const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); + DNSCacheData *dns_cache = getDNSCache(cacheID, true); + if(dns_cache != NULL && dns_cache->blocking_status != new_status) + { + // Memorize blocking status DNS cache for the domain/client combination + dns_cache->blocking_status = new_status; + + // Set expiration time for this cache entry (if applicable) + // We set this only if not already set to avoid extending the TTL of an + // existing entry + if(config.dns.cache.upstreamBlockedTTL.v.ui > 0 && + dns_cache->expires == 0 && + (new_status == QUERY_EXTERNAL_BLOCKED_NXRA || + new_status == QUERY_EXTERNAL_BLOCKED_NULL || + new_status == QUERY_EXTERNAL_BLOCKED_IP || + new_status == QUERY_EXTERNAL_BLOCKED_EDE15)) + { + // Set expiration time for this cache entry + dns_cache->expires = time(NULL) + config.dns.cache.upstreamBlockedTTL.v.ui; + } + + if(config.debug.queries.v.b) + { + // Debug logging + const char *qtype = get_query_type_str(dns_cache->query_type, NULL, NULL); + const char *domain = getDomainString(query); + const char *clientstr = getClientIPString(query); + const char *statusstr = get_query_status_str(new_status); + + if(dns_cache->expires > 0) + { + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s -> %s, expires in %lis", + qtype, clientstr, domain, statusstr, + (long)(dns_cache->expires - time(NULL))); + } + else + { + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s -> %s, no expiry", + qtype, clientstr, domain, statusstr); + } + } + } + } + // else: update global counters, ... if(!init) { @@ -1200,3 +1281,30 @@ int __attribute__ ((pure)) get_temp_unit_val(const char *temp_unit) // Invalid value return -1; } + +const char * __attribute__ ((const)) get_edns_mode_str(const enum edns_mode edns_mode) +{ + switch(edns_mode) + { + case EDNS_MODE_NONE: + return "NONE"; + case EDNS_MODE_CODE: + return "CODE"; + case EDNS_MODE_TEXT: + return "TEXT"; + } + return NULL; +} + +int __attribute__ ((pure)) get_edns_mode_val(const char *edns_mode) +{ + if(strcasecmp(edns_mode, "NONE") == 0) + return EDNS_MODE_NONE; + else if(strcasecmp(edns_mode, "CODE") == 0) + return EDNS_MODE_CODE; + else if(strcasecmp(edns_mode, "TEXT") == 0) + return EDNS_MODE_TEXT; + + // Invalid value + return -1; +} diff --git a/src/datastructure.h b/src/datastructure.h index 43b2a5c1..3e204939 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -93,7 +93,7 @@ typedef struct { size_t ippos; size_t namepos; size_t ifacepos; - time_t firstSeen; + double firstSeen; double lastQuery; } clientsData; @@ -108,12 +108,16 @@ typedef struct { typedef struct { unsigned char magic; - enum domain_client_status blocking_status; + struct { + bool allowed :1; + } flags; + enum query_status blocking_status; enum reply_type force_reply; enum query_type query_type; int domainID; int clientID; int list_id; + time_t expires; char *cname_target; } DNSCacheData; @@ -124,8 +128,8 @@ int findQueryID(const int id); int _findUpstreamID(const char *upstream, const in_port_t port, int line, const char *func, const char *file); #define findDomainID(domain, count) _findDomainID(domain, count, __LINE__, __FUNCTION__, __FILE__) int _findDomainID(const char *domain, const bool count, int line, const char *func, const char *file); -#define findClientID(client, count, aliasclient) _findClientID(client, count, aliasclient, __LINE__, __FUNCTION__, __FILE__) -int _findClientID(const char *client, const bool count, const bool aliasclient, int line, const char *func, const char *file); +#define findClientID(client, count, aliasclient, now) _findClientID(client, count, aliasclient, now, __LINE__, __FUNCTION__, __FILE__) +int _findClientID(const char *client, const bool count, const bool aliasclient, const double now, int line, const char *func, const char *file); #define findCacheID(domainID, clientID, query_type, create_new) _findCacheID(domainID, clientID, query_type, create_new, __FUNCTION__, __LINE__, __FILE__) int _findCacheID(const int domainID, const int clientID, const enum query_type query_type, const bool create_new, const char *func, const int line, const char *file); bool isValidIPv4(const char *addr); @@ -145,10 +149,10 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c void FTL_reload_all_domainlists(void); void FTL_reset_per_client_domain_data(void); -const char *getDomainString(const queriesData* query); -const char *getCNAMEDomainString(const queriesData* query); -const char *getClientIPString(const queriesData* query); -const char *getClientNameString(const queriesData* query); +const char *getDomainString(const queriesData *query); +const char *getCNAMEDomainString(const queriesData *query); +const char *getClientIPString(const queriesData *query); +const char *getClientNameString(const queriesData *query); void change_clientcount(clientsData *client, int total, int blocked, int overTimeIdx, int overTimeMod); const char *get_query_type_str(const enum query_type type, const queriesData *query, char buffer[20]); @@ -159,6 +163,7 @@ const char *get_refresh_hostnames_str(const enum refresh_hostnames refresh) __at int get_refresh_hostnames_val(const char *refresh_hostnames) __attribute__ ((pure)); const char *get_blocking_mode_str(const enum blocking_mode mode) __attribute__ ((const)); int get_blocking_mode_val(const char *blocking_mode) __attribute__ ((pure)); +const char * __attribute__ ((const)) get_blocking_status_str(const enum blocking_status blocking); const char *get_ptr_type_str(const enum ptr_type piholePTR) __attribute__ ((const)); int get_ptr_type_val(const char *piholePTR) __attribute__ ((pure)); const char *get_busy_reply_str(const enum busy_reply replyWhenBusy) __attribute__ ((const)); @@ -167,10 +172,12 @@ const char * get_listeningMode_str(const enum listening_mode listeningMode) __at int get_listeningMode_val(const char *listeningMode) __attribute__ ((pure)); const char * __attribute__ ((const)) get_temp_unit_str(const enum temp_unit temp_unit); int __attribute__ ((pure)) get_temp_unit_val(const char *temp_unit); +const char * __attribute__ ((const)) get_edns_mode_str(const enum edns_mode edns_mode); +int __attribute__ ((pure)) get_edns_mode_val(const char *edns_mode); // Pointer getter functions #define getQuery(queryID, checkMagic) _getQuery(queryID, checkMagic, __LINE__, __FUNCTION__, __FILE__) -queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file); +queriesData *_getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file); #define getClient(clientID, checkMagic) _getClient(clientID, checkMagic, __LINE__, __FUNCTION__, __FILE__) clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file); #define getDomain(domainID, checkMagic) _getDomain(domainID, checkMagic, __LINE__, __FUNCTION__, __FILE__) diff --git a/src/dnsmasq/CMakeLists.txt b/src/dnsmasq/CMakeLists.txt index 2497bc72..927ed572 100644 --- a/src/dnsmasq/CMakeLists.txt +++ b/src/dnsmasq/CMakeLists.txt @@ -65,5 +65,9 @@ set(sources add_library(dnsmasq OBJECT ${sources}) target_compile_definitions(dnsmasq PRIVATE VERSION=\"${DNSMASQ_VERSION}\") target_compile_definitions(dnsmasq PRIVATE CONFFILE=\"/etc/pihole/dnsmasq.conf\") -target_compile_options(dnsmasq PRIVATE -Wno-maybe-uninitialized) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(dnsmasq PRIVATE -Wno-maybe-uninitialized -Wno-sign-compare) +elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") + target_compile_options(dnsmasq PRIVATE -Wno-gnu-variable-sized-type-not-at-end -Wno-sign-compare -Wno-deprecated-non-prototype) +endif() target_include_directories(dnsmasq PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/lua) diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index 144468a3..b04f964f 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -31,7 +31,7 @@ #define FORWARD_TEST 1000 /* try all servers every 1000 queries */ #define FORWARD_TIME 600 /* or 10 minutes */ #define UDP_TEST_TIME 60 /* How often to reset our idea of max packet size. */ -#define SERVERS_LOGGED 30 /* Only log this many servers when logging state */ +#define SERVERS_LOGGED 300 /* Only log this many servers when logging state */ #define LOCALS_LOGGED 8 /* Only log this many local addresses when logging state */ #define LEASE_RETRY 60 /* on error, retry writing leasefile after LEASE_RETRY seconds */ #define CACHESIZ 150 /* default cache size */ diff --git a/src/dnsmasq/dhcp.c b/src/dnsmasq/dhcp.c index b65facd8..e70e011c 100644 --- a/src/dnsmasq/dhcp.c +++ b/src/dnsmasq/dhcp.c @@ -162,7 +162,7 @@ void dhcp_packet(time_t now, int pxe_fd) #elif defined(HAVE_BSD_NETWORK) char control[CMSG_SPACE(sizeof(struct sockaddr_dl))]; #endif - } control_u; + } control_u = { 0 }; struct dhcp_bridge *bridge, *alias; msg.msg_controllen = sizeof(control_u); diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 3107cc3f..2c1c6f14 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -28,12 +28,13 @@ #include "signals.h" // FTL_fork_and_bind_sockets() #include "main.h" +// log_debug() +#include "log.h" struct daemon *daemon; static volatile pid_t pid = 0; static volatile int pipewrite; -volatile char FTL_terminate = 0; static void set_dns_listeners(void); static void set_tftp_listeners(void); @@ -1068,12 +1069,8 @@ int main_dnsmasq (int argc, char **argv) /* Using inotify, have to select a resolv file at startup */ poll_resolv(1, 0, now); #endif - - /*** Pi-hole modification ***/ - FTL_terminate = killed; - /****************************/ - while (!FTL_terminate) + while (!killed) { int timeout = fast_retry(now); @@ -1316,6 +1313,10 @@ int main_dnsmasq (int argc, char **argv) static void sig_handler(int sig) { + /**** Pi-hole modification ****/ + send_event(pipewrite, EVENT_SIGNAL, sig, NULL); + /******************************/ + if (pid == 0) { /* ignore anything other than TERM during startup @@ -1537,6 +1538,7 @@ static void async_event(int pipe, time_t now) { lease_prune(NULL, now); lease_update_file(now); + lease_update_dns(0); } #ifdef HAVE_DHCP6 else if (daemon->doing_ra) @@ -1574,6 +1576,12 @@ static void async_event(int pipe, time_t now) my_syslog(LOG_WARNING, _("script process exited with status %d"), ev.data); break; + /**** Pi-hole modification ****/ + case EVENT_SIGNAL: + log_debug(DEBUG_ANY, "dnsmasq received signal %d", ev.data); + break; + /**************************** */ + case EVENT_EXEC_ERR: my_syslog(LOG_ERR, _("failed to execute %s: %s"), daemon->lease_change_command, strerror(ev.data)); @@ -1668,7 +1676,7 @@ static void async_event(int pipe, time_t now) flush_log(); /*** Pi-hole modification ***/ // exit(EC_GOOD); - FTL_terminate = 1; + killed = 1; /*** Pi-hole modification ***/ } } diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index a16c83b4..ce8d1d09 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -200,6 +200,9 @@ struct event_desc { #define EVENT_SCRIPT_LOG 25 #define EVENT_TIME 26 +// Pi-hole +#define EVENT_SIGNAL 255 + /* Exit codes. */ #define EC_GOOD 0 #define EC_BADCONF 1 diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 2176c231..c2824d85 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" +#include "dnsmasq_interface.h" static struct frec *get_new_frec(time_t now, struct server *serv, int force); static struct frec *lookup_frec(unsigned short id, int fd, void *hash, int *firstp, int *lastp); @@ -36,7 +36,7 @@ int send_from(int fd, int nowild, char *packet, size_t len, union mysockaddr *to, union all_addr *source, unsigned int iface) { - struct msghdr msg; + struct msghdr msg = { 0 }; struct iovec iov[1]; union { struct cmsghdr align; /* this ensures alignment */ @@ -46,7 +46,7 @@ int send_from(int fd, int nowild, char *packet, size_t len, char control[CMSG_SPACE(sizeof(struct in_addr))]; #endif char control6[CMSG_SPACE(sizeof(struct in6_pktinfo))]; - } control_u; + } control_u = { 0 }; iov[0].iov_base = packet; iov[0].iov_len = len; @@ -105,7 +105,12 @@ int send_from(int fd, int nowild, char *packet, size_t len, #ifdef HAVE_LINUX_NETWORK /* If interface is still in DAD, EINVAL results - ignore that. */ if (errno != EINVAL) - my_syslog(LOG_ERR, _("failed to send packet: %s"), strerror(errno)); + { + my_syslog(LOG_ERR, _("failed to send packet: %s"), strerror(errno)); + /********** Pi-hole modification **********/ + FTL_connection_error("failed to send UDP reply", to); + /******************************************/ + } #endif return 0; } @@ -567,6 +572,12 @@ static int forward_query(int udpfd, union mysockaddr *udpaddr, break; forward->forwardall++; } + /**** Pi-hole modification ****/ + else + { + FTL_connection_error("failed to send UDP request", &srv->addr); + } + /******************************/ } if (++start == last) @@ -703,6 +714,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server size_t plen; /******** Pi-hole modification ********/ unsigned char *pheader_copy = NULL; + unsigned char ede_data[MAX_EDE_DATA] = { 0 }; + size_t ede_len = 0; /**************************************/ (void)ad_reqd; @@ -770,7 +783,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server } } - FTL_header_analysis(header->hb4, rcode, server, daemon->log_display_id); + FTL_header_analysis(header->hb4, server, daemon->log_display_id); /* RFC 4035 sect 4.6 para 3 */ if (!is_sign && !option_bool(OPT_DNSSEC_PROXY)) @@ -879,7 +892,7 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server // Generate DNS packet for reply, a possibly existing pseudo header // will be restored later inside resize_packet() - n = FTL_make_answer(header, ((char *) header) + 65536, n, &ede); + n = FTL_make_answer(header, ((char *) header) + 65536, n, ede_data, &ede_len); } } @@ -919,13 +932,18 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server // pheader_copy instead of pheader if(pheader_copy) free(pheader_copy); - /**************************************/ - if (pheader && ede != EDE_UNSET) + if (pheader && (ede != EDE_UNSET || ede_len > 0)) { - u16 swap = htons((u16)ede); - n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 1); + if (ede_len > 0) + n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, ede_data, ede_len, do_bit, 1); + else + { + u16 swap = htons((u16)ede); + n = add_pseudoheader(header, n, limit, daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 1); + } } + /**************************************/ if (RCODE(header) == NXDOMAIN) server->nxdomain_replies++; @@ -1195,7 +1213,7 @@ void reply_query(int fd, time_t now) server = daemon->serverarray[c]; - FTL_header_analysis(header->hb4, RCODE(header), server, daemon->log_display_id); + FTL_header_analysis(header->hb4, server, daemon->log_display_id); if (RCODE(header) != REFUSED) daemon->serverarray[first]->last_server = c; @@ -1919,8 +1937,9 @@ void receive_query(struct listener *listen, time_t now) if(piholeblocked) { // Generate DNS packet for reply - int ede = EDE_UNSET; - n = FTL_make_answer(header, ((char *) header) + udp_size, n, &ede); + unsigned char ede_data[MAX_EDE_DATA] = { 0 }; + size_t ede_len = 0; + n = FTL_make_answer(header, ((char *) header) + udp_size, n, ede_data, &ede_len); // The pseudoheader may contain important information such as EDNS0 version important for // some DNS resolvers (such as systemd-resolved) to work properly. We should not discard them. @@ -1930,10 +1949,9 @@ void receive_query(struct listener *listen, time_t now) if (have_pseudoheader) { - u16 swap = htons(ede); - if (ede != EDE_UNSET) // Add EDNS0 option EDE if applicable + if (ede_len > 0) // Add EDNS0 option EDE if applicable n = add_pseudoheader(header, n, ((unsigned char *) header) + udp_size, - daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + daemon->edns_pktsz, EDNS0_OPTION_EDE, ede_data, ede_len, do_bit, 0); else n = add_pseudoheader(header, n, ((unsigned char *) header) + udp_size, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); @@ -2087,12 +2105,19 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, data_sent = 1; else if (errno == ETIMEDOUT || errno == EHOSTUNREACH) timedout = 1; + /**** Pi-hole modification ****/ + if (errno != 0) + FTL_connection_error("failed to send TCP(fast-open) packet", &serv->addr); + /******************************/ #endif /* If fastopen failed due to lack of reply, then there's no point in trying again in non-FASTOPEN mode. */ if (timedout || (!data_sent && connect(serv->tcpfd, &serv->addr.sa, sa_len(&serv->addr)) == -1)) { + /**** Pi-hole modification ****/ + FTL_connection_error("failed to send TCP(connect) packet", &serv->addr); + /******************************/ close(serv->tcpfd); serv->tcpfd = -1; continue; @@ -2107,6 +2132,10 @@ static ssize_t tcp_talk(int first, int last, int start, unsigned char *packet, !read_write(serv->tcpfd, &c2, 1, 1) || !read_write(serv->tcpfd, payload, (rsize = (c1 << 8) | c2), 1)) { + /**** Pi-hole modification ****/ + FTL_connection_error("failed to send TCP(read_write) packet", &serv->addr); + /******************************/ + close(serv->tcpfd); serv->tcpfd = -1; /* We get data then EOF, reopen connection to same server, @@ -2144,7 +2173,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si unsigned char *packet = NULL; struct dns_header *new_header = NULL; - FTL_header_analysis(header->hb4, RCODE(header), server, daemon->log_display_id); + FTL_header_analysis(header->hb4, server, daemon->log_display_id); while (1) { @@ -2448,18 +2477,18 @@ unsigned char *tcp_request(int confd, time_t now, // Interface name is known from before forking if(piholeblocked) { - int ede = EDE_UNSET; + unsigned char ede_data[MAX_EDE_DATA] = { 0 }; + size_t ede_len = 0; stale = 0; // Generate DNS packet for reply - m = FTL_make_answer(header, ((char *) header) + 65536, size, &ede); + m = FTL_make_answer(header, ((char *) header) + 65536, size, ede_data, &ede_len); // The pseudoheader may contain important information such as EDNS0 version important for // some DNS resolvers (such as systemd-resolved) to work properly. We should not discard them. if (have_pseudoheader && m > 0) { - u16 swap = htons(ede); - if (ede != -1) // Add EDNS0 option EDE if applicable + if (ede_len > 0) // Add EDNS0 option EDE if applicable m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, - daemon->edns_pktsz, EDNS0_OPTION_EDE, (unsigned char *)&swap, 2, do_bit, 0); + daemon->edns_pktsz, EDNS0_OPTION_EDE, ede_data, ede_len, do_bit, 0); else m = add_pseudoheader(header, m, ((unsigned char *) header) + 65536, daemon->edns_pktsz, 0, NULL, 0, do_bit, 0); diff --git a/src/dnsmasq/helper.c b/src/dnsmasq/helper.c index a59a0a78..65727ba4 100644 --- a/src/dnsmasq/helper.c +++ b/src/dnsmasq/helper.c @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../log.h" +#include "log.h" #ifdef HAVE_SCRIPT diff --git a/src/dnsmasq/log.c b/src/dnsmasq/log.c index eb0f2cef..f040163f 100644 --- a/src/dnsmasq/log.c +++ b/src/dnsmasq/log.c @@ -90,7 +90,7 @@ int log_start(struct passwd *ent_pw, int errfd) if (!log_reopen(daemon->log_file)) { send_event(errfd, EVENT_LOG_ERR, errno, daemon->log_file ? daemon->log_file : ""); - _exit(0); + die(_("failed to open log file: %s"), strerror(errno), 1); // Pi-hole modification } /* if queuing is inhibited, make sure we allocate diff --git a/src/dnsmasq/network.c b/src/dnsmasq/network.c index 60799d48..4d35478b 100644 --- a/src/dnsmasq/network.c +++ b/src/dnsmasq/network.c @@ -15,8 +15,8 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" -#include "../log.h" +#include "dnsmasq_interface.h" +#include "log.h" #ifdef HAVE_LINUX_NETWORK @@ -1861,3 +1861,46 @@ void newaddress(time_t now) relay->iface_index = 0; #endif } + + +static int callback_v4(struct in_addr local, int if_index, char *label, + struct in_addr netmask, struct in_addr broadcast, void *vparam) + { + log_info("callback_v4"); + // Log the interface information + log_info("Interface: %s", label); + log_info("IP Address: %s", inet_ntoa(local)); + log_info("Netmask: %s", inet_ntoa(netmask)); + log_info("Broadcast: %s", inet_ntoa(broadcast)); + log_info("Interface Index: %d", if_index); + return 1; + } + + +static int callback_v6(struct in6_addr *local, int prefix, + int scope, int if_index, int flags, + int preferred, int valid, void *vparam) + { + log_info("callback_v6"); + // Log the interface information + char ip[INET6_ADDRSTRLEN]; + inet_ntop(AF_INET6, local, ip, INET6_ADDRSTRLEN); + log_info("IP Address: %s", ip); + log_info("Prefix: %d", prefix); + log_info("Scope: %d", scope); + log_info("Interface Index: %d", if_index); + log_info("Flags: %d", flags); + log_info("Preferred: %d", preferred); + log_info("Valid: %d", valid); + return 1; + } + +extern int iface_enumerate(int family, void *parm, int (*callback)()); +void test_enumerate(void) +{ + log_info("test_enumerate 4"); + iface_enumerate(AF_INET, NULL, callback_v4); + log_info("test_enumerate 6"); + iface_enumerate(AF_INET6, NULL, callback_v6); + log_info("test_enumerate done"); +}; diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 249a6f35..d075ad0a 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -20,7 +20,7 @@ #include /* Pi-hole modification */ -#include "../log.h" +#include "log.h" /************************/ static volatile int mem_recover = 0; @@ -1193,10 +1193,10 @@ static char *domain_rev4(int from_file, char *server, struct in_addr *addr4, int return _("error"); } - if (sdetails.orig_hostinfo) - freeaddrinfo(sdetails.orig_hostinfo); } } + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); return NULL; } @@ -1280,11 +1280,10 @@ static char *domain_rev6(int from_file, char *server, struct in6_addr *addr6, in if (!add_update_server(flags, &serv_addr, &source_addr, interface, domain, NULL)) return _("error"); } - - if (sdetails.orig_hostinfo) - freeaddrinfo(sdetails.orig_hostinfo); } } + if (sdetails.orig_hostinfo) + freeaddrinfo(sdetails.orig_hostinfo); return NULL; } diff --git a/src/dnsmasq/rfc1035.c b/src/dnsmasq/rfc1035.c index 06d3067c..f16ae343 100644 --- a/src/dnsmasq/rfc1035.c +++ b/src/dnsmasq/rfc1035.c @@ -15,7 +15,7 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" +#include "dnsmasq_interface.h" int extract_name(struct dns_header *header, size_t plen, unsigned char **pp, char *name, int isExtract, int extrabytes) @@ -792,6 +792,17 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t flags |= F_RR; else insert = 0; /* NOTE: do not cache data from CNAME queries. */ + + /*********** Pi-hole modification ***********/ + if(FTL_check_reply(RCODE(header), flags, NULL, daemon->log_display_id)) + { + // Found while processing a reply from upstream. We prevent cache insertion here + // This query is to be blocked as we found a blocked + // domain while walking the CNAME path. Log to pihole.log here + log_query(F_UPSTREAM, name, NULL, "blocked due to upstream response (header)", 0); + return 99; + } + /********************************************/ cname_loop1: if (!(p1 = skip_questions(header, qlen))) @@ -868,7 +879,7 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t return 2; // ****************************** Pi-hole modification ****************************** - const char *src = cpp != NULL ? cpp->flags & F_BIGNAME ? cpp->name.bname->name : cpp->name.sname : NULL; + const char *src = cpp != NULL ? cache_get_name(cpp) : NULL; if(FTL_CNAME(name, src, daemon->log_display_id)) { // Found while processing a reply from upstream. We prevent cache insertion here @@ -1017,6 +1028,15 @@ int extract_addresses(struct dns_header *header, size_t qlen, char *name, time_t log_query((flags & (F_IPV4 | F_IPV6)) | F_IPSET, nftsets->domain, &addr, *nftsets_cur, 0); #endif } + + /*********** Pi-hole modification ***********/ + if(FTL_check_reply(RCODE(header), flags, &addr, daemon->log_display_id)) + { + // Found while processing a reply from upstream + log_query(F_UPSTREAM, name, NULL, "blocked due to upstream response (answer)", 0); + return 99; + } + /********************************************/ if (insert) { @@ -2047,7 +2067,7 @@ size_t answer_request(struct dns_header *header, char *limit, size_t qlen, log_query(stale_flag | (crecp->flags & ~F_REVERSE), name, &crecp->addr, record_source(crecp->uid), 0); // ****************************** Pi-hole modification ****************************** - const char *src = crecp != NULL ? crecp->flags & F_BIGNAME ? crecp->name.bname->name : crecp->name.sname : NULL; + const char *src = crecp != NULL ? cache_get_name(crecp) : NULL; if(FTL_CNAME(name, src, daemon->log_display_id)) { // Served from cache. This can happen if a domain hidden in the CNAME path diff --git a/src/dnsmasq/util.c b/src/dnsmasq/util.c index c5273812..53afbf9d 100644 --- a/src/dnsmasq/util.c +++ b/src/dnsmasq/util.c @@ -34,6 +34,10 @@ #include #endif +/****** Pi-hole modification ******/ +extern int is_shm_fd(const int fd); +/**********************************/ + /* SURF random number generator */ static u32 seed[32]; @@ -815,6 +819,11 @@ void close_fds(long max_fd, int spare1, int spare2, int spare3) fd == spare1 || fd == spare2 || fd == spare3) continue; + /****** Pi-hole modification ******/ + if(is_shm_fd(fd)) + continue; + /**********************************/ + close(fd); } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 68cd310d..9eec8cda 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -58,21 +58,22 @@ #include "config/config.h" // FTL_fork_and_bind_sockets() #include "main.h" +// ntp_server_start() +#include "ntp/ntp.h" // Private prototypes static void print_flags(const unsigned int flags); -#define query_set_reply(flags, type, addr, query, response) _query_set_reply(flags, type, addr, query, response, __FILE__, __LINE__) -static void _query_set_reply(const unsigned int flags, const enum reply_type reply, const union all_addr *addr, queriesData* query, +#define query_set_reply(flags, reply, addr, query, response) _query_set_reply(flags, reply, addr, query, response, __FILE__, __LINE__) +static void _query_set_reply(const unsigned int flags, const enum reply_type reply, const union all_addr *addr, queriesData *query, const struct timeval response, const char *file, const int line); #define FTL_check_blocking(queryID, domainID, clientID) _FTL_check_blocking(queryID, domainID, clientID, __FILE__, __LINE__) static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const char* file, const int line); -static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr, const queriesData *query, const domainsData *domain); -static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const enum query_status new_status); +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, const enum query_status new_status); static void FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, unsigned short port, const int id, const char* file, const int line); -static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, const int id, const char* file, const int line); +static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, unsigned short type, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); static void FTL_dnssec(const char *result, const union all_addr *addr, const int id, const char* file, const int line); -static void mysockaddr_extract_ip_port(union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port); +static void mysockaddr_extract_ip_port(const union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port); static void alladdr_extract_ip(union all_addr *addr, const sa_family_t family, char ip[ADDRSTRLEN+1]); static void check_pihole_PTR(char *domain); #define query_set_dnssec(query, dnssec) _query_set_dnssec(query, dnssec, __FILE__, __LINE__) @@ -81,11 +82,11 @@ static char *get_ptrname(struct in_addr *addr); static const char *check_dnsmasq_name(const char *name); // Static blocking metadata -static bool adbit = false; +static bool adbit = false, rabit = false; static const char *blockingreason = ""; static enum reply_type force_next_DNS_reply = REPLY_UNKNOWN; +static enum query_status cacheStatus = QUERY_UNKNOWN; static int last_regex_idx = -1; -static struct ptr_record *pihole_ptr = NULL; static char *pihole_suffix = NULL; static char *hostname_suffix = NULL; static char *cname_target = NULL; @@ -180,11 +181,13 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * // otherwise, flags will be F_UPSTREAM and the type is not set // (== 0) else - FTL_reply(flags, name, addr, arg, id, path, line); + FTL_reply(flags, name, addr, arg, type, id, path, line); } // This is inspired by make_local_answer() -size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char *file, const int line) +size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, + unsigned char ede_data[MAX_EDE_DATA], size_t *ede_len, + const char *file, const int line) { log_debug(DEBUG_FLAGS, "FTL_make_answer() called from %s:%d", short_path(file), line); // Exit early if there are no questions in this query @@ -198,10 +201,7 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len return 0; // Debug logging - if(*ede != EDE_UNSET) - log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\", EDE: %s (%d)", name, edestr(*ede), *ede); - else - log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\", EDE: N/A", name); + log_debug(DEBUG_QUERIES, "Preparing reply for \"%s\"", name); // Get question type int qtype, flags = 0; @@ -249,9 +249,6 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len // Debug logging log_debug(DEBUG_QUERIES, "Forced DNS reply to REFUSED"); - - // Set EDE code to blocked - *ede = EDE_BLOCKED; } else if(force_next_DNS_reply == REPLY_IP) { @@ -326,11 +323,103 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len force_next_DNS_reply = REPLY_UNKNOWN; } + // Derive EDE code and text from cacheStatus + int ede_code = EDE_UNSET; + const char *ede_text = NULL; + switch(cacheStatus) + { + case QUERY_UNKNOWN: +// case QUERY_CACHE: + case QUERY_FORWARDED: + case QUERY_RETRIED: + case QUERY_RETRIED_DNSSEC: + case QUERY_IN_PROGRESS: + case QUERY_DBBUSY: + case QUERY_CACHE_STALE: + case QUERY_STATUS_MAX: + // Not going through this function + break; + case QUERY_GRAVITY: + ede_code = EDE_BLOCKED; + ede_text = "gravity"; + break; + case QUERY_GRAVITY_CNAME: + ede_code = EDE_BLOCKED; + ede_text = "gravity (CNAME)"; + break; + case QUERY_DENYLIST: + ede_code = EDE_BLOCKED; + ede_text = "denylist"; + break; + case QUERY_DENYLIST_CNAME: + ede_code = EDE_BLOCKED; + ede_text = "denylist (CNAME)"; + break; + case QUERY_REGEX: + ede_code = EDE_BLOCKED; + ede_text = "regex"; + break; + case QUERY_REGEX_CNAME: + ede_code = EDE_BLOCKED; + ede_text = "regex (CNAME)"; + break; + case QUERY_SPECIAL_DOMAIN: + ede_code = EDE_BLOCKED; + ede_text = "special"; + break; + case QUERY_EXTERNAL_BLOCKED_NXRA: + ede_code = EDE_BLOCKED; + ede_text = "upstream NXRA"; + break; + case QUERY_EXTERNAL_BLOCKED_NULL: + ede_code = EDE_BLOCKED; + ede_text = "upstream NULL"; + break; + case QUERY_EXTERNAL_BLOCKED_IP: + ede_code = EDE_BLOCKED; + ede_text = "upstream IP"; + break; + case QUERY_EXTERNAL_BLOCKED_EDE15: + ede_code = EDE_BLOCKED; + ede_text = "upstream EDE 15"; + break; + case QUERY_CACHE: + ede_code = EDE_SYNTHESIZED; + ede_text = "synthesized"; + break; + } + + // Reset global DNS cache status + cacheStatus = QUERY_UNKNOWN; + + // Debug logging + log_debug(DEBUG_QUERIES, "Setting EDE: %s (%d) + \"%s\"", + ede_code != EDE_UNSET ? edestr(ede_code) : "---", ede_code, ede_text ? ede_text : "---"); + + if(ede_code != EDE_UNSET && config.dns.blocking.edns.v.edns_mode > EDNS_MODE_NONE) + { + // Set EDE INFO-CODE (network byte order) + uint16_t swap = htons(ede_code); + memcpy(ede_data, &swap, sizeof(swap)); + *ede_len = sizeof(swap); + + // Set EDE INFO-TEXT (if available) + if(ede_text && config.dns.blocking.edns.v.edns_mode > EDNS_MODE_CODE) + { + size_t extra_len = strlen(ede_text); + // Truncate if necessary + if(extra_len > MAX_EDE_DATA - *ede_len) + extra_len = MAX_EDE_DATA - *ede_len; + memcpy(ede_data + *ede_len, ede_text, extra_len); + *ede_len += extra_len; + } + } + // Debug logging print_flags(flags); // Setup reply header - setup_reply(header, flags, *ede); + setup_reply(header, flags, ede_code); // Add NEG flag when replying with NXDOMAIN or NODATA. This is necessary // to get proper logging in pihole.log At the same time, we cannot add @@ -464,6 +553,9 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len if (trunc) header->hb3 |= HB3_TC; + // Unset the blocking reason + blockingreason = ""; + return p - (unsigned char *)header; } @@ -589,6 +681,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, "interface-local IP address" : "NODATA due to missing iface address"); + cacheStatus = QUERY_CACHE; return true; } else @@ -649,7 +742,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, const int queryID = counters->queries; // Find client IP - const int clientID = findClientID(clientIP, true, false); + const int clientID = findClientID(clientIP, true, false, querytimestamp); // Get client pointer clientsData* client = getClient(clientID, true); @@ -663,6 +756,9 @@ bool _FTL_new_query(const unsigned int flags, const char *name, return false; } + // Update rolling window of queries per second + update_qps(querytimestamp); + // Interface name is only available for regular queries, not for // automatically generated DNSSEC queries const char *interface = internal_query ? "-" : next_iface.name; @@ -730,7 +826,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, const int domainID = findDomainID(domainString, true); // Save everything - queriesData* query = getQuery(queryID, false); + queriesData *query = getQuery(queryID, false); if(query == NULL) { // Encountered memory error, skip query @@ -844,13 +940,17 @@ bool _FTL_new_query(const unsigned int flags, const char *name, if(config.debug.arp.v.b) { if(client->hwlen == 6) + { log_debug(DEBUG_ARP, "find_mac(\"%s\") returned hardware address " "%02X:%02X:%02X:%02X:%02X:%02X", clientIP, client->hwaddr[0], client->hwaddr[1], client->hwaddr[2], client->hwaddr[3], client->hwaddr[4], client->hwaddr[5]); + } else + { log_debug(DEBUG_ARP, "find_mac(\"%s\") returned %i bytes of data", clientIP, client->hwlen); + } } } @@ -998,7 +1098,7 @@ void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_fam // MUSL defines it differently than GNU C uint8_t bytes[2]; memcpy(&bytes, &iface->addr.in6.sin6_addr, 2); - // Global Unicast Address (2000::/3, RFC 4291) + // Global Unicast Address (2000::/3, RFC 4291) isGUA = (bytes[0] & 0x70) == 0x20; // Unique Local Address (fc00::/7, RFC 4193) isULA = (bytes[0] & 0xfe) == 0xfc; @@ -1055,9 +1155,22 @@ void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_fam static void check_pihole_PTR(char *domain) { - // Return early if Pi-hole PTR is not available - if(pihole_ptr == NULL) - return; + // Iterate through the already configured PTR entries in dnsmasq's + // structure and check if we already have a PTR record for this address + // This avoids adding work into defining PTR records that have already + // been added but also overwriting PTR records manually added by users + // using custom dnsmasq config lines like "ptr-record=," + for(struct ptr_record *ptr = daemon->ptr; ptr; ptr = ptr->next) + { + log_debug(DEBUG_EXTRA, "Known PTR record %p: %s -> %s (next = %p)", ptr, ptr->name, ptr->ptr, ptr->next); + + if(ptr->name != NULL && strcmp(ptr->name, domain) == 0) + { + // We already have a PTR record for this address + log_debug(DEBUG_QUERIES, "PTR record for %s exists", domain); + return; + } + } // Convert PTR request into numeric form union all_addr addr = {{ 0 }}; @@ -1079,43 +1192,49 @@ static void check_pihole_PTR(char *domain) for (struct irec *iface = daemon->interfaces; iface != NULL; iface = iface->next) { const sa_family_t family = iface->addr.sa.sa_family; - if((family == AF_INET && flags == F_IPV4 && iface->addr.in.sin_addr.s_addr == addr.addr4.s_addr) || - (family == AF_INET6 && flags == F_IPV6 && IN6_ARE_ADDR_EQUAL(&iface->addr.in6.sin6_addr, &addr.addr6))) + // If the family matches but the address doesn't, we skip this address + if(!(family == AF_INET && flags == F_IPV4 && iface->addr.in.sin_addr.s_addr == addr.addr4.s_addr) && + !(family == AF_INET6 && flags == F_IPV6 && IN6_ARE_ADDR_EQUAL(&iface->addr.in6.sin6_addr, &addr.addr6))) + continue; + + // If we reached this point, we have a match between the address the client + struct ptr_record *pihole_ptr = calloc(1, sizeof(struct ptr_record)); + pihole_ptr->name = strdup(domain); + if(family == AF_INET) { - // The last PTR record in daemon->ptr is reserved for Pi-hole - free(pihole_ptr->name); - pihole_ptr->name = strdup(domain); - if(family == AF_INET) - { - // IPv4 supports conditional domains - struct in_addr addrv4 = { 0 }; - addrv4.s_addr = iface->addr.in.sin_addr.s_addr; - pihole_ptr->ptr = get_ptrname(&addrv4); - } - else - { - // IPv6 does not support conditional domains - pihole_ptr->ptr = get_ptrname(NULL); - } - - // Debug logging - log_debug(DEBUG_QUERIES, "Generating PTR response: %s -> %s", pihole_ptr->name, pihole_ptr->ptr); - - return; + // IPv4 supports conditional domains + pihole_ptr->ptr = get_ptrname(&iface->addr.in.sin_addr); } + else + { + // IPv6 does not support conditional domains + pihole_ptr->ptr = get_ptrname(NULL); + } + + // If we have a PTR record, we add it to the list + if(daemon->ptr != NULL) + { + // Iterate to the last PTR entry in dnsmasq's structure + struct ptr_record *ptr; + for(ptr = daemon->ptr; ptr && ptr->next; ptr = ptr->next); + + // Add our record after the last existing ptr-record + ptr->next = pihole_ptr; + } + else + { + // We do not have any PTR records yet, so we add our + // record as the first one + daemon->ptr = pihole_ptr; + } + + // Debug logging + log_debug(DEBUG_QUERIES, "Generating PTR record (%p): %s -> %s", pihole_ptr, pihole_ptr->name, pihole_ptr->ptr); + + return; } } -inline static void set_dnscache_blockingstatus(DNSCacheData * dns_cache, clientsData *client, - enum domain_client_status new_status, const char *domain) -{ - // Memorize blocking status DNS cache for the domain/client combination - dns_cache->blocking_status = new_status; - - const char *clientip = client ? getstr(client->ippos) : "N/A"; - log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s", clientip, domain, blockingreason); -} - static bool check_domain_blocked(const char *domain, const int clientID, clientsData *client, queriesData *query, DNSCacheData *dns_cache, enum query_status *new_status, bool *db_okay) @@ -1132,9 +1251,6 @@ static bool check_domain_blocked(const char *domain, const int clientID, *new_status = QUERY_DENYLIST; blockingreason = "exactly denied"; - // Mark domain as exactly denied for this client - set_dnscache_blockingstatus(dns_cache, client, DENYLIST_BLOCKED, domain); - // We block this domain return true; } @@ -1154,6 +1270,11 @@ static bool check_domain_blocked(const char *domain, const int clientID, // ... dns_cache->list_id = -1 * (list_id + 2); + // Mark query as allowed to prevent further checks such as CNAME + // inspection. This ensures antigravity matches have similar effects + // than explicitly allowed domains. + query->flags.allowed = true; + return false; } @@ -1165,9 +1286,6 @@ static bool check_domain_blocked(const char *domain, const int clientID, *new_status = QUERY_GRAVITY; blockingreason = "gravity blocked"; - // Mark domain as gravity blocked for this client - set_dnscache_blockingstatus(dns_cache, client, GRAVITY_BLOCKED, domain); - log_debug(DEBUG_QUERIES, "Blocking query due to gravity match (list ID %i)", list_id); // Store ID of the matching gravity list @@ -1225,9 +1343,6 @@ static bool check_domain_blocked(const char *domain, const int clientID, *new_status = QUERY_REGEX; blockingreason = "regex denied"; - // Mark domain as regex matched for this client - set_dnscache_blockingstatus(dns_cache, client, REGEX_BLOCKED, domain); - // Regex may be overwriting reply type for this domain if(dns_cache->force_reply != REPLY_UNKNOWN) force_next_DNS_reply = dns_cache->force_reply; @@ -1315,8 +1430,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // already stored in the query, we have to re-lookup the cache ID. // This can happen when a CNAME chain is followed and analyzed const int cacheID = query->domainID == domainID && query->clientID == clientID ? - query->cacheID : - findCacheID(domainID, clientID, query->type, true); + query->cacheID : findCacheID(domainID, clientID, query->type, true); DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache == NULL) { @@ -1324,23 +1438,38 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c return false; } + // If this cache record can expire, check if it is still valid + if(dns_cache->expires > 0 && dns_cache->expires < time(NULL)) + { + // This cache record is expired, we have to re-check + log_debug(DEBUG_QUERIES, "DNS cache record expired"); + dns_cache->blocking_status = QUERY_UNKNOWN; + dns_cache->flags.allowed = false; + dns_cache->expires = 0; + dns_cache->list_id = -1; + } + + // Memorize blocking status DNS cache for the domain/client combination + cacheStatus = dns_cache->blocking_status; + log_debug(DEBUG_QUERIES, "Set global cache status to %d", cacheStatus); + // Skip the entire chain of tests if we already know the answer for this // particular client - unsigned char blockingStatus = dns_cache->blocking_status; char *domainstr = (char*)getstr(domain->domainpos); - switch(blockingStatus) + switch(dns_cache->blocking_status) { - case UNKNOWN_BLOCKED: + case QUERY_UNKNOWN: // New domain/client combination. // We have to go through all the tests below log_debug(DEBUG_QUERIES, "%s is not known", domainstr); break; - case DENYLIST_BLOCKED: + case QUERY_DENYLIST: + case QUERY_DENYLIST_CNAME: // Known as exactly denied, we return this result early, skipping // all the lengthy tests below - blockingreason = "exactly denied"; + blockingreason = dns_cache->blocking_status == QUERY_DENYLIST ? "exactly denied" : "exactly denied (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); // Do not block if the entire query is to be permitted @@ -1353,10 +1482,11 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } break; - case GRAVITY_BLOCKED: + case QUERY_GRAVITY: + case QUERY_GRAVITY_CNAME: // Known as gravity blocked, we return this result early, skipping // all the lengthy tests below - blockingreason = "gravity blocked"; + blockingreason = dns_cache->blocking_status == QUERY_GRAVITY ? "gravity blocked" : "gravity blocked (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); // Do not block if the entire query is to be permitted @@ -1369,10 +1499,11 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } break; - case REGEX_BLOCKED: + case QUERY_REGEX: + case QUERY_REGEX_CNAME: // Known as regex denied, we return this result early, skipping all // the lengthy tests below - blockingreason = "regex denied"; + blockingreason = dns_cache->blocking_status == QUERY_REGEX ? "regex denied" : "regex denied (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s (cache regex ID: %i)", domainstr, blockingreason, dns_cache->list_id); @@ -1387,17 +1518,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c } break; - case ALLOWED: - // Known as allowed, we return this result early, skipping all the - // lengthy tests below - log_debug(DEBUG_QUERIES, "%s is known as not to be blocked (allowed)", domainstr); - - query->flags.allowed = true; - - return false; - break; - - case SPECIAL_DOMAIN: + case QUERY_SPECIAL_DOMAIN: // Known as a special domain, we return this result early, skipping // all the lengthy tests below blockingreason = "special domain"; @@ -1408,10 +1529,71 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c return true; break; - case NOT_BLOCKED: - // Known as not blocked, we return this result early, skipping all + case QUERY_EXTERNAL_BLOCKED_IP: + case QUERY_EXTERNAL_BLOCKED_NULL: + case QUERY_EXTERNAL_BLOCKED_NXRA: + case QUERY_EXTERNAL_BLOCKED_EDE15: + + switch(dns_cache->blocking_status) + { + case QUERY_UNKNOWN: + case QUERY_GRAVITY: + case QUERY_DENYLIST: + case QUERY_REGEX: + case QUERY_FORWARDED: + case QUERY_CACHE: + case QUERY_GRAVITY_CNAME: + case QUERY_REGEX_CNAME: + case QUERY_DENYLIST_CNAME: + case QUERY_RETRIED: + case QUERY_RETRIED_DNSSEC: + case QUERY_IN_PROGRESS: + case QUERY_DBBUSY: + case QUERY_SPECIAL_DOMAIN: + case QUERY_CACHE_STALE: + case QUERY_STATUS_MAX: + // Cannot happen + break; + case QUERY_EXTERNAL_BLOCKED_IP: + blockingreason = "blocked upstream with known address"; + break; + case QUERY_EXTERNAL_BLOCKED_NULL: + blockingreason = "blocked upstream with NULL address"; + break; + case QUERY_EXTERNAL_BLOCKED_EDE15: + blockingreason = "blocked upstream with EDE15"; + break; + case QUERY_EXTERNAL_BLOCKED_NXRA: + blockingreason = "blocked upstream with NXRA address"; + break; + } + + // Known as upstream blocked, we return this result + // early, skipping all the lengthy tests below + log_debug(DEBUG_QUERIES, "%s is known as %s (expires in %lus)", + domainstr, blockingreason, (unsigned long)(dns_cache->expires - time(NULL))); + + force_next_DNS_reply = dns_cache->force_reply; + query_blocked(query, domain, client, dns_cache->blocking_status); + return true; + break; + + case QUERY_CACHE: + case QUERY_FORWARDED: + case QUERY_RETRIED: + case QUERY_RETRIED_DNSSEC: + case QUERY_IN_PROGRESS: + case QUERY_DBBUSY: + case QUERY_CACHE_STALE: + case QUERY_STATUS_MAX: + // Known as not to be blocked, possibly even explicitly + // allowed - we return this result early, skipping all // the lengthy tests below - log_debug(DEBUG_QUERIES, "%s is known as not to be blocked", domainstr); + log_debug(DEBUG_QUERIES, "%s is known as not to be blocked%s", domainstr, + dns_cache->flags.allowed ? " (allowed)" : ""); + + if(dns_cache->flags.allowed) + query->flags.allowed = true; return false; break; @@ -1443,7 +1625,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed && special_domain(query, domainstr)) { // Set DNS cache properties - dns_cache->blocking_status = SPECIAL_DOMAIN; + dns_cache->blocking_status = QUERY_SPECIAL_DOMAIN; + cacheStatus = dns_cache->blocking_status; dns_cache->force_reply = force_next_DNS_reply; // Adjust counters @@ -1451,6 +1634,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // Debug output log_debug(DEBUG_QUERIES, "Special domain: %s is %s", domainstr, blockingreason); + free(domainstr); return true; } @@ -1464,10 +1648,13 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c // (defaulting to true) if(config.dns.blockESNI.v.b && !query->flags.allowed && blockDomain == NOT_FOUND && - strlen(domainstr) > 6 && strncasecmp(domainstr, "_esni.", 6u) == 0) + strlen(domainstr) > 6 && strncasecmp(domainstr, "_esni.", 6u) == 0) { blockDomain = check_domain_blocked(domainstr + 6u, clientID, client, query, dns_cache, &new_status, &db_okay); + // Update DNS cache status + cacheStatus = dns_cache->blocking_status; + if(blockDomain) { // Truncate "_esni." from queried domain if the parenting domain was @@ -1502,13 +1689,14 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c { // Explicitly mark as not blocked to skip the entire gravity/blacklist // chain when the same client asks for the same domain in the future. - // Store domain as whitelisted if this is the case - dns_cache->blocking_status = query->flags.allowed ? ALLOWED : NOT_BLOCKED; + // Store domain as allowed if this is the case + dns_cache->flags.allowed = query->flags.allowed; // Debug output // client is guaranteed to be non-NULL above - log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s (domainlist ID: %i)", getstr(client->ippos), - domainstr, query->flags.allowed ? "whitelisted" : "not blocked", dns_cache->list_id); + log_debug(DEBUG_QUERIES, "DNS cache: %s/%s/%s is %s (domainlist ID: %i)", + get_query_type_str(query->type, NULL, NULL), getstr(client->ippos), + domainstr, query->flags.allowed ? "allowed" : "not blocked", dns_cache->list_id); } free(domainstr); @@ -1543,7 +1731,7 @@ bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file // Get query pointer so we can later extract the client requesting this domain for // the per-client blocking evaluation - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Nothing to be done here @@ -1703,7 +1891,7 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { free(upstreamIP); @@ -1718,7 +1906,10 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio upstreamsData *upstream = getUpstream(upstreamID, true); if(upstream != NULL) + { upstream->count++; + upstream->lastQuery = now; + } // Proceed only if // - current query has not been marked as replied to so far @@ -1792,7 +1983,7 @@ void FTL_dnsmasq_reload(void) // This function is called by the dnsmasq code on receive of SIGHUP // *before* clearing the cache and re-reading the lists if(reload++ > 0) - log_info("Received SIGHUP, flushing cache and re-reading config"); + log_info("Flushing cache and re-reading config"); // Gravity database updates // - (Re-)open gravity database connection @@ -1824,7 +2015,7 @@ static void alladdr_extract_ip(union all_addr *addr, const sa_family_t family, c inet_ntop(family, addr, ip, ADDRSTRLEN); } -static void mysockaddr_extract_ip_port(union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port) +static void mysockaddr_extract_ip_port(const union mysockaddr *server, char ip[ADDRSTRLEN+1], in_port_t *port) { // Extract IP address inet_ntop(server->sa.sa_family, @@ -1892,7 +2083,7 @@ static void update_upstream(queriesData *query, const int id) } static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, - const char *arg, const int id, const char* file, const int line) + const char *arg, unsigned short type, const int id, const char* file, const int line) { const double now = double_time(); // If domain is "pi.hole", we skip this query @@ -1950,7 +2141,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al const char *answer = arg; // Determine returned address (if applicable) char dest[ADDRSTRLEN]; dest[0] = '\0'; - if(addr) + if(addr && flags & (F_IPV4 | F_IPV6)) { inet_ntop((flags & F_IPV4) ? AF_INET : AF_INET6, addr, dest, ADDRSTRLEN); answer = dest; // Overwrite answer with human-readable IP address @@ -1991,11 +2182,23 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(!name || strlen(name) == 0) dispname = "."; + // Swap display name with answer if this is a reverse query + // Check for reverse query by looking at the query type not only + // the flag as some PTR queries are not flagged (DNS-SD) + if(flags & F_REVERSE || type == T_PTR) + { + const char *tmp = dispname; + dispname = answer; + answer = tmp; + } + if(cached || last_server.sa.sa_family == 0) + { // Log cache or upstream reply from unknown source log_debug(DEBUG_QUERIES, "**** got %s%s reply: %s is %s (ID %i, %s:%i)", stale ? "stale ": "", cached ? "cache" : "upstream", dispname, answer, id, file, line); + } else { char ip[ADDRSTRLEN+1] = { 0 }; @@ -2009,7 +2212,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al } // Get and check query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Nothing to be done here @@ -2074,17 +2277,6 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(!is_blocked(query->status)) query_set_status(query, qs); - // Detect if returned IP indicates that this query was blocked - const enum query_status new_status = detect_blocked_IP(flags, addr, query, domain); - - // Update status of this query if detected as external blocking - if(new_status != query->status) - { - clientsData *client = getClient(query->clientID, true); - if(client != NULL) - query_blocked(query, domain, client, new_status); - } - // Save reply type and update individual reply counters query_set_reply(flags, 0, addr, query, response); @@ -2138,10 +2330,11 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al upstream->rtuncertainty += (mean - query->response)*(mean - query->response); // Only proceed if query is not already known - // to have been blocked by Quad9 + // to have been blocked upstream if(query->status == QUERY_EXTERNAL_BLOCKED_IP || query->status == QUERY_EXTERNAL_BLOCKED_NULL || - query->status == QUERY_EXTERNAL_BLOCKED_NXRA) + query->status == QUERY_EXTERNAL_BLOCKED_NXRA || + query->status == QUERY_EXTERNAL_BLOCKED_EDE15) { unlock_shm(); return; @@ -2175,37 +2368,37 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // Save reply type and update individual reply counters query_set_reply(reply_flags, 0, addr, query, response); - // Further checks if this is an IP address - if(addr) - { - // Detect if returned IP indicates that this query was blocked - const enum query_status new_status = detect_blocked_IP(flags, addr, query, domain); - - // Update status of this query if detected as external blocking - if(new_status != query->status) - { - clientsData *client = getClient(query->clientID, true); - if(client != NULL) - query_blocked(query, domain, client, new_status); - } - } - // Mark query for updating in the database query->flags.database.changed = true; } - else if(flags & F_REVERSE) + else if(flags & F_REVERSE || type == T_PTR) { // isExactMatch is not used here as the PTR is special. // Example: - // Question: PTR 8.8.8.8 + // Question: PTR -x 8.8.8.8 // will lead to: // domain->domain = 8.8.8.8.in-addr.arpa - // and will return - // name = google-public-dns-a.google.com + // name = 8.8.8.8 (derived above from addr) + // answer = dns.google // Hence, isExactMatch is always false + // DNS-SD example: + // Question: PTR _http._tcp.local + // will lead to: + // domain->domain = obs.cr + // name = (null) + // answer = obs.cr + + // if flags does not contain F_REVERSE, it is not a reverse + // query, e.g. DNS-SD + unsigned int pflags = flags; + if(!(flags & F_REVERSE)) + pflags |= F_RRNAME; // Save reply type and update individual reply counters - query_set_reply(flags, 0, addr, query, response); + query_set_reply(pflags, 0, addr, query, response); + + // Hereby, this query is now fully determined + query->flags.complete = true; // Mark query for updating in the database query->flags.database.changed = true; @@ -2216,7 +2409,8 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al } else if(config.debug.flags.v.b) { - log_warn("Unknown upstream REPLY"); + log_warn("Unknown upstream REPLY, exact: %s, type: %u", + isExactMatch ? "true" : "false", type); } if(query && option_bool(OPT_DNSSEC_PROXY)) @@ -2236,15 +2430,10 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al unlock_shm(); } -static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr, const queriesData *query, const domainsData *domain) +static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr) { // Compare returned IP against list of known blocking splash pages - if (!addr) - { - return query->status; - } - // First, we check if we want to skip this result even before comparing against the known IPs if(flags & F_HOSTS || flags & F_REVERSE) { @@ -2252,18 +2441,18 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni // count gravity.list blocked queries as externally blocked. // Also: Do not mark responses of PTR requests as externally blocked. const char *cause = (flags & F_HOSTS) ? "origin is HOSTS" : "query is PTR"; - log_debug(DEBUG_QUERIES, "Skipping detection of external blocking IP for ID %i as %s", query->id, cause); + log_debug(DEBUG_QUERIES, "Skipping detection of external blocking IP as %s", cause); // Return early, do not compare against known blocking page IP addresses below - return query->status; + return QUERY_UNKNOWN; } // If received one of the following IPs as reply, OpenDNS // (Cisco Umbrella) blocked this query - // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses- + // See https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses // for a full list of these IP addresses - in_addr_t ipv4Addr = ntohl(addr->addr4.s_addr); - in_addr_t ipv6Addr = ntohl(addr->addr6.s6_addr32[3]); + const in_addr_t ipv4Addr = flags & F_IPV4 ? ntohl(addr->addr4.s_addr) : 0; + const in_addr_t ipv6Addr = flags & F_IPV6 ? ntohl(addr->addr6.s6_addr32[3]) : 0; // Check for IP block 146.112.61.104 - 146.112.61.110 if((flags & F_IPV4) && ipv4Addr >= 0x92703d68 && ipv4Addr <= 0x92703d6e) { @@ -2271,14 +2460,15 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET, addr, answer, ADDRSTRLEN); - log_debug(DEBUG_QUERIES, "Upstream responded with known blocking page (IPv4), ID %i:\n\t\"%s\" -> \"%s\"", - query->id, getstr(domain->domainpos), answer); + blockingreason = "blocked upstream with known address (IPv4)"; + cacheStatus = QUERY_EXTERNAL_BLOCKED_IP; + log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } // Update status return QUERY_EXTERNAL_BLOCKED_IP; } - // Check for IP block :ffff:146.112.61.104 - :ffff:146.112.61.110 + // Check for IP block ::ffff:146.112.61.104 - ::ffff:146.112.61.110 else if(flags & F_IPV6 && addr->addr6.s6_addr32[0] == 0 && addr->addr6.s6_addr32[1] == 0 && @@ -2289,8 +2479,9 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { char answer[ADDRSTRLEN]; answer[0] = '\0'; inet_ntop(AF_INET6, addr, answer, ADDRSTRLEN); - log_debug(DEBUG_QUERIES, "Upstream responded with known blocking page (IPv6), ID %i:\n\t\"%s\" -> \"%s\"", - query->id, getstr(domain->domainpos), answer); + blockingreason = "blocked upstream with known address (IPv6)"; + cacheStatus = QUERY_EXTERNAL_BLOCKED_IP; + log_debug(DEBUG_QUERIES, "%s -> \"%s\"", blockingreason, answer); } // Update status @@ -2304,8 +2495,9 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { if(config.debug.queries.v.b) { - log_debug(DEBUG_QUERIES, "Upstream responded with 0.0.0.0, ID %i:\n\t\"%s\" -> \"0.0.0.0\"", - query->id, getstr(domain->domainpos)); + blockingreason = "blocked upstream with 0.0.0.0"; + cacheStatus = QUERY_EXTERNAL_BLOCKED_NULL; + log_debug(DEBUG_QUERIES, "%s", blockingreason); } // Update status @@ -2319,8 +2511,9 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni { if(config.debug.queries.v.b) { - log_debug(DEBUG_QUERIES, "Upstream responded with ::, ID %i:\n\t\"%s\" -> \"::\"", - query->id, getstr(domain->domainpos)); + blockingreason = "blocked upstream with ::"; + cacheStatus = QUERY_EXTERNAL_BLOCKED_NULL; + log_debug(DEBUG_QUERIES, "%s", blockingreason); } // Update status @@ -2328,10 +2521,10 @@ static enum query_status detect_blocked_IP(const unsigned short flags, const uni } // Nothing happened here - return query->status; + return QUERY_UNKNOWN; } -static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const enum query_status new_status) +static void query_blocked(queriesData *query, domainsData *domain, clientsData *client, const enum query_status new_status) { // Get response time struct timeval response; @@ -2386,7 +2579,7 @@ static void FTL_dnssec(const char *arg, const union all_addr *addr, const int id } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this DNSSEC details @@ -2468,7 +2661,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this query @@ -2569,7 +2762,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl unlock_shm(); } -static void FTL_mark_externally_blocked(const int id, const char* file, const int line) +static void FTL_blocked_upstream_by_header(const enum query_status new_status, const int id, const char* file, const int line) { // Lock shared memory lock_shm(); @@ -2584,7 +2777,7 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this query @@ -2605,10 +2798,16 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in if(config.debug.queries.v.b) { // Get domain name (domain cannot be NULL here) - const char *domainname = getstr(domain->domainpos); - log_debug(DEBUG_QUERIES, "**** %s externally blocked (ID %i, FTL %i, %s:%i)", domainname, id, queryID, file, line); + const char *domainstr = getstr(domain->domainpos); + log_debug(DEBUG_QUERIES, "**** %s externally blocked by header (ID %i, FTL %i, %s:%i)", domainstr, id, queryID, file, line); } + // Set blocking reason + blockingreason = new_status == QUERY_EXTERNAL_BLOCKED_NXRA ? + "blocked upstream with NXDOMAIN + no RA" : + "blocked upstream with EDE15"; + cacheStatus = new_status; + // Get response time struct timeval response; gettimeofday(&response, 0); @@ -2616,7 +2815,7 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in // Store query as externally blocked clientsData *client = getClient(query->clientID, true); if(client != NULL) - query_blocked(query, domain, client, QUERY_EXTERNAL_BLOCKED_NXRA); + query_blocked(query, domain, client, new_status); // Store reply type as replied with NXDOMAIN query_set_reply(F_NEG | F_NXDOMAIN, 0, NULL, query, response); @@ -2628,11 +2827,55 @@ static void FTL_mark_externally_blocked(const int id, const char* file, const in unlock_shm(); } -void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const struct server *server, - const int id, const char* file, const int line) +static void FTL_blocked_upstream_by_addr(const enum query_status new_status, const int id, const char* file, const int line) { - // Analyze DNS header bits + // Lock shared memory + lock_shm(); + // Save status in corresponding query identified by dnsmasq's ID + const int queryID = findQueryID(id); + if(queryID < 0) + { + // This may happen e.g. if the original query was "pi.hole" + log_debug(DEBUG_QUERIES, "FTL_check_reply(): Query %i has not been found", id); + unlock_shm(); + return; + } + + // Get query pointer + queriesData *query = getQuery(queryID, true); + if(query == NULL) + { + // Memory error, skip this query + log_debug(DEBUG_QUERIES, "FTL_check_reply(): Memory error (ID %i)", id); + unlock_shm(); + return; + } + clientsData *client = getClient(query->clientID, true); + domainsData *domain = getDomain(query->domainID, true); + if(client != NULL && domain != NULL) + query_blocked(query, domain, client, new_status); + + // Possible debugging information + if(config.debug.queries.v.b) + { + // Get domain name (domain cannot be NULL here) + const char *domainname = domain ? getstr(domain->domainpos) : ""; + log_debug(DEBUG_QUERIES, "**** %s externally blocked by address (ID %i, FTL %i, %s:%i)", domainname, id, queryID, file, line); + } + + // Mark query for updating in the database + query->flags.database.changed = true; + + // Unlock shared memory + unlock_shm(); +} + +int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, + const union all_addr *addr, + const int id, const char* file, const int line) +{ + ednsData *edns = getEDNS(); // Check if RA bit is unset in DNS header and rcode is NXDOMAIN // If the response code (rcode) is NXDOMAIN, we may be seeing a response from // an externally blocked query. As they are not always accompany a necessary @@ -2640,14 +2883,59 @@ void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, // FTL_reply() is never getting called from within the cache routines. // Hence, we have to store the necessary information about the NXDOMAIN // reply already here. - if(!(header4 & 0x80) && rcode == NXDOMAIN) + // Alternatively, we also consider EDE15 as a blocking reason. + if(addr == NULL) + { // RA bit is not set and rcode is NXDOMAIN - FTL_mark_externally_blocked(id, file, line); + if(!rabit && rcode == NXDOMAIN) + { + FTL_blocked_upstream_by_header(QUERY_EXTERNAL_BLOCKED_NXRA, id, file, line); + + // Query is blocked + return 1; + } + + // EDE 15 + if(edns != NULL && edns->ede == EDE_BLOCKED) + { + FTL_blocked_upstream_by_header(QUERY_EXTERNAL_BLOCKED_EDE15, id, file, line); + + // Query is blocked + return 1; + } + } + // Further checks if this is an IP address + else if(addr != NULL) + { + // Detect if returned IP indicates that this query was blocked + const enum query_status new_qstatus = detect_blocked_IP(flags, addr); + + // Update status of this query if detected as external blocking + if(new_qstatus != QUERY_UNKNOWN) + { + FTL_blocked_upstream_by_addr(new_qstatus, id, file, line); + + // Query is blocked + return 1; + } + } + + return 0; +} + +void _FTL_header_analysis(const unsigned char header4, const struct server *server, + const int id, const char* file, const int line) +{ + // Analyze DNS header bits // Check if AD bit is set in DNS header adbit = header4 & HB4_AD; - // Store server which sent this reply + // Check if RA bit is set in DNS header. We do it here as it is it is + // forced by dnsmasq shortly after calling FTL_header_analysis() + rabit = header4 & HB4_RA; + + // Store server which sent this reply (if applicable) if(server) { memcpy(&last_server, &server->addr, sizeof(last_server)); @@ -2656,13 +2944,15 @@ void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, char ip[ADDRSTRLEN+1] = { 0 }; in_port_t port = 0; mysockaddr_extract_ip_port(&last_server, ip, &port); - log_debug(DEBUG_EXTRA, "Got forward address: %s#%u (%s:%i)", ip, port, short_path(file), line); + log_debug(DEBUG_EXTRA, "Got forward address: %s#%u for ID %i (%s:%i)", + ip, port, id, short_path(file), line); } } else { memset(&last_server, 0, sizeof(last_server)); - log_debug(DEBUG_EXTRA, "Got forward address: NO"); + log_debug(DEBUG_EXTRA, "Got forward address: NO for ID %i (%s:%i)", + id, short_path(file), line); } } @@ -2814,32 +3104,7 @@ static void init_pihole_PTR(void) // Fallback to "" on memory error ptrname = (char*)hostname(); } - } break; - } - - // Obtain PTR record used for Pi-hole PTR injection (if enabled) - if(config.dns.piholePTR.v.ptr_type != PTR_NONE) - { - // Add PTR record for pi.hole, the address will be injected later - pihole_ptr = calloc(1, sizeof(struct ptr_record)); - pihole_ptr->name = strdup("x.x.x.x.in-addr.arpa"); - pihole_ptr->ptr = (char*)""; - pihole_ptr->next = NULL; - // Add our PTR record to the end of the linked list - if(daemon->ptr != NULL) - { - // Iterate to the last PTR entry in dnsmasq's structure - struct ptr_record *ptr; - for(ptr = daemon->ptr; ptr && ptr->next; ptr = ptr->next); - - // Add our record after the last existing ptr-record - ptr->next = pihole_ptr; - } - else - { - // Ours is the only record for daemon->ptr - daemon->ptr = pihole_ptr; } } } @@ -2865,30 +3130,29 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Flush messages stored in the long-term database flush_message_table(); - // Try to import queries from long-term database if available - if(config.database.DBimport.v.b) - { - import_queries_from_disk(); - DB_read_queries(); - } + // Verify checksum of this binary early on to ensure that the binary is + // not corrupted and that the binary is not tampered with. We can only + // do this here as we need the database to be properly initialized + // in case we need to store the verification result + verify_FTL(false); // Initialize in-memory database starting index update_disk_db_idx(); - // Log some information about the imported queries (if any) - log_counter_info(); - // Handle real-time signals in this process (and its children) // Helper processes are already split from the main instance // so they will not listen to real-time signals handle_realtime_signals(); - // We will use the attributes object later to start all threads in - // detached mode - pthread_attr_t attr; // Initialize thread attributes object with default attribute values + // Do NOT detach threads as we want to join them during shutdown with a + // fixed timeout to give them time to clean up and finish their work + pthread_attr_t attr; pthread_attr_init(&attr); + // Start NTP sync thread + ntp_start_sync_thread(&attr); + // Start database thread if database is used if(pthread_create( &threads[DB], &attr, DB_thread, NULL ) != 0) { @@ -2928,18 +3192,42 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) if(getuid() == 0) { // Only print this and change ownership of shmem objects when - // we're actually dropping root (user/group my be set to root) + // we're actually dropping root (user/group may be set to root) if(ent_pw != NULL && ent_pw->pw_uid != 0) { - log_info("FTL is going to drop from root to user %s (UID %u)", - ent_pw->pw_name, ent_pw->pw_uid); - if(chown(config.files.log.ftl.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, config.files.log.ftl.v.s, strerror(errno), errno); - if(chown(config.files.database.v.s, ent_pw->pw_uid, ent_pw->pw_gid) == -1) - log_warn("Setting ownership (%u:%u) of %s failed: %s (%i)", - ent_pw->pw_uid, ent_pw->pw_gid, config.files.database.v.s, strerror(errno), errno); + log_info("FTL is going to drop from root to user pihole"); + + // Change ownership of shared memory objects chown_all_shmem(ent_pw); + + // Configured FTL log file + chown_pihole(config.files.log.ftl.v.s, ent_pw); + + // Configured FTL database file + chown_pihole(config.files.database.v.s, ent_pw); + + // Check if auxiliary files exist and change ownership + char *extrafile = calloc(strlen(config.files.database.v.s) + 5, sizeof(char)); + if(extrafile == NULL) + { + log_err("Memory allocation failed. Skipping some file ownership checks."); + return; + } + + // Check -wal file (write-ahead log) + strcpy(extrafile, config.files.database.v.s); + strcat(extrafile, "-wal"); + if(file_exists(extrafile)) + chown_pihole(extrafile, ent_pw); + + // Check -shm file (mmapped shared memory) + strcpy(extrafile, config.files.database.v.s); + strcat(extrafile, "-shm"); + if(file_exists(extrafile)) + chown_pihole(extrafile, ent_pw); + + // Free allocated memory + free(extrafile); } else { @@ -2962,6 +3250,8 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) // Initialize Pi-hole PTR pointer init_pihole_PTR(); + + forked = true; } static char *get_ptrname(struct in_addr *addr) @@ -3086,7 +3376,7 @@ void FTL_forwarding_retried(const struct server *serv, const int oldID, const in if(queryID >= 0) { // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); // Set retried status if(query != NULL) @@ -3344,7 +3634,7 @@ void FTL_query_in_progress(const int id) } // Get query pointer - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) { // Memory error, skip this DNSSEC details @@ -3413,9 +3703,9 @@ void FTL_multiple_replies(const int id, int *firstID) // Get (read-only) pointer of the query that contains all relevant // information (all others are mere duplicates and were only added to the // list of duplicates rather than havong been forwarded on their own) - const queriesData* source_query = getQuery(*firstID, true); + const queriesData *source_query = getQuery(*firstID, true); // Get query pointer of duplicated reply - queriesData* duplicated_query = getQuery(queryID, true); + queriesData *duplicated_query = getQuery(queryID, true); if(duplicated_query == NULL || source_query == NULL) { @@ -3502,4 +3792,58 @@ void get_dnsmasq_metrics_obj(cJSON *json) { for (unsigned int i = 0; i < __METRIC_MAX; i++) cJSON_AddNumberToObject(json, get_metric_name(i), daemon->metrics[i]); -} \ No newline at end of file +} + +void FTL_connection_error(const char *reason, const union mysockaddr *addr) +{ + // Make a private copy of the error + const int errnum = errno; + const char *error = strerror(errnum); + + // Set log priority + int priority = LOG_ERR; + + // If this is a TCP connection error and errno == 0, this isn't a + // connection error but the remote side closed the connection + if(errnum == 0 && strstr(reason, "TCP(read_write)") != NULL) + { + error = "Connection prematurely closed by remote server"; + priority = LOG_INFO; + } + + // Format the address into a string (if available) + in_port_t port = 0; + char ip[ADDRSTRLEN + 1] = { 0 }; + if(addr != NULL) + mysockaddr_extract_ip_port(addr, ip, &port); + + // Log to FTL.log + const int id = daemon->log_display_id; + log_debug(DEBUG_QUERIES, "Connection error (%s#%u, ID %d): %s (%s)", ip, port, id, reason, error); + + // Log to pihole.log + my_syslog(priority, "%s: %s", reason, error); + + // Add to Pi-hole diagnostics but do not add messages more often than + // once every five seconds to avoid hammering the database with errors + // on continuously failing connections + static time_t last = 0; + if(time(NULL) - last > 5) + { + last = time(NULL); + char *server = NULL; + if(ip[0] != '\0') + { + const size_t len = strlen(ip) + 6; + server = calloc(len, sizeof(char)); + if(server != NULL) + { + snprintf(server, len, "%s#%u", ip, port); + server[len - 1] = '\0'; + } + } + log_connection_error(server, reason, error); + if(server != NULL) + free(server); + } +} diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index f58851dc..4004183f 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -27,13 +27,17 @@ void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_fam #define FTL_new_query(flags, name, addr, arg, qtype, id, proto) _FTL_new_query(flags, name, addr, arg, qtype, id, proto, __FILE__, __LINE__) bool _FTL_new_query(const unsigned int flags, const char *name, union mysockaddr *addr, char *arg, const unsigned short qtype, const int id, enum protocol proto, const char* file, const int line); -#define FTL_header_analysis(header4, rcode, server, id) _FTL_header_analysis(header4, rcode, server, id, __FILE__, __LINE__) -void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const struct server *server, const int id, const char* file, const int line); +#define FTL_header_analysis(header4, server, id) _FTL_header_analysis(header4, server, id, __FILE__, __LINE__) +void _FTL_header_analysis(const unsigned char header4, const struct server *server, const int id, const char* file, const int line); + +#define FTL_check_reply(rcode, flags, addr, id) _FTL_check_reply(rcode, flags, addr, id, __FILE__, __LINE__) +int _FTL_check_reply(const unsigned int rcode, const unsigned short flags, const union all_addr *addr, const int id, const char* file, const int line); void FTL_forwarding_retried(const struct server *server, const int oldID, const int newID, const bool dnssec); -#define FTL_make_answer(header, limit, len, ede) _FTL_make_answer(header, limit, len, ede, __FILE__, __LINE__) -size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, int *ede, const char* file, const int line); +#define MAX_EDE_DATA 128 +#define FTL_make_answer(header, limit, len, ede_data, ede_len) _FTL_make_answer(header, limit, len, ede_data, ede_len, __FILE__, __LINE__) +size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len, unsigned char ede_data[MAX_EDE_DATA], size_t *ede_len, const char* file, const int line); #define FTL_CNAME(dst, src, id) _FTL_CNAME(dst, src, id, __FILE__, __LINE__) bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file, const int line); @@ -48,6 +52,8 @@ void FTL_TCP_worker_terminating(bool finished); bool FTL_unlink_DHCP_lease(const char *ipaddr, const char **hint); +void FTL_connection_error(const char *reason, const union mysockaddr *addr); + // defined in src/dnsmasq/cache.c extern char *querystr(char *desc, unsigned short type); diff --git a/src/edns0.c b/src/edns0.c index a310a3d9..54f7f709 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -400,13 +400,16 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) // this document. The value of the INFO-CODE is encoded // as a two-octet unsigned integer in network byte // order. - // - // The EXTRA-TEXT from the EDE EDNS option is ignored by - // FTL // Debug output log_debug(DEBUG_EDNS0, "EDE: %s (code %d)", edestr(edns.ede), edns.ede); + if(optlen > 2) + { + // Debug output + log_debug(DEBUG_EDNS0, "EDE: EXTRA-TEXT: %.*s", optlen - 2, p + 2); + } + // Advance working pointer p += optlen; } @@ -419,4 +422,4 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) p += optlen; } } -} \ No newline at end of file +} diff --git a/src/enums.h b/src/enums.h index bc80472b..1dd8c929 100644 --- a/src/enums.h +++ b/src/enums.h @@ -49,6 +49,7 @@ enum query_status { QUERY_DBBUSY, QUERY_SPECIAL_DOMAIN, QUERY_CACHE_STALE, + QUERY_EXTERNAL_BLOCKED_EDE15, QUERY_STATUS_MAX } __attribute__ ((packed)); @@ -120,21 +121,9 @@ enum blocking_status { BLOCKING_UNKNOWN } __attribute__ ((packed)); -// Blocking status constants used by the dns_cache->blocking_status vector -// We explicitly force UNKNOWN_BLOCKED to zero on all platforms as this is the -// default value set initially with calloc -enum domain_client_status { - UNKNOWN_BLOCKED = 0, - GRAVITY_BLOCKED, - DENYLIST_BLOCKED, - REGEX_BLOCKED, - ALLOWED, - SPECIAL_DOMAIN, - NOT_BLOCKED -} __attribute__ ((packed)); - enum debug_flag { - DEBUG_DATABASE = 1, + DEBUG_NONE = 0, + DEBUG_DATABASE, DEBUG_NETWORKING, DEBUG_LOCKS, DEBUG_QUERIES, @@ -161,6 +150,7 @@ enum debug_flag { DEBUG_WEBSERVER, DEBUG_EXTRA, DEBUG_RESERVED, + DEBUG_NTP, DEBUG_MAX } __attribute__ ((packed)); @@ -203,8 +193,6 @@ enum gravity_tables { CLIENTS_TABLE, GROUPS_TABLE, ADLISTS_TABLE, - DENIED_DOMAINS_TABLE, - ALLOWED_DOMAINS_TABLE, UNKNOWN_TABLE } __attribute__ ((packed)); @@ -227,8 +215,7 @@ enum refresh_hostnames { enum api_auth_status { API_AUTH_UNAUTHORIZED = -1, - API_AUTH_LOCALHOST = -2, - API_AUTH_EMPTYPASS = -3, + API_AUTH_EMPTYPASS = -2, } __attribute__ ((packed)); enum db_result { @@ -248,8 +235,10 @@ enum thread_types { DB, GC, DNSclient, - CONF_READER, TIMER, + NTP_CLIENT, + NTP_SERVER4, + NTP_SERVER6, THREADS_MAX } __attribute__ ((packed)); @@ -273,6 +262,9 @@ enum message_type { INACCESSIBLE_ADLIST_MESSAGE, DISK_MESSAGE_EXTENDED, CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, + CONNECTION_ERROR_MESSAGE, + NTP_MESSAGE, + VERIFY_MESSAGE, MAX_MESSAGE, } __attribute__ ((packed)); @@ -309,6 +301,12 @@ enum temp_unit { TEMP_UNIT_K } __attribute__ ((packed)); +enum edns_mode { + EDNS_MODE_NONE = 0, + EDNS_MODE_CODE, + EDNS_MODE_TEXT, +} __attribute__ ((packed)); + enum adlist_type { ADLIST_BLOCK = 0, ADLIST_ALLOW @@ -323,4 +321,21 @@ enum cert_check { CERT_OKAY } __attribute__ ((packed)); +enum http_method { + HTTP_UNKNOWN = 0, + HTTP_GET = 1 << 0, + HTTP_POST = 1 << 1, + HTTP_PUT = 1 << 2, + HTTP_PATCH = 1 << 3, + HTTP_DELETE = 1 << 4, + HTTP_OPTIONS = 1 << 5, +}; + +enum api_flags { + API_FLAG_NONE = 0, + API_DOMAINS = 1 << 0, + API_PARSE_JSON = 1 << 1, + API_BATCHDELETE = 1 << 2, +}; + #endif // ENUMS_H diff --git a/src/events.h b/src/events.h index ea5b7f27..3ae0a70d 100644 --- a/src/events.h +++ b/src/events.h @@ -19,4 +19,4 @@ void _set_event(const enum events event, int line, const char *function, const c #define get_and_clear_event(event) _get_and_clear_event(event, __LINE__, __FUNCTION__, __FILE__) bool _get_and_clear_event(const enum events event, int line, const char *function, const char *file); -#endif // EVENTS_H \ No newline at end of file +#endif // EVENTS_H diff --git a/src/files.c b/src/files.c index d34bea09..7a1a02d5 100644 --- a/src/files.c +++ b/src/files.c @@ -13,11 +13,13 @@ #include "config/config.h" #include "config/setupVars.h" #include "log.h" +// sha256_raw_to_hex() +#include "config/password.h" +// log_verify_message() +#include "database/message-table.h" // opendir(), readdir() #include -// getpwuid() -#include // getgrgid() #include // NAME_MAX @@ -29,6 +31,10 @@ // sendfile() #include #include +// PRIu64 +#include +//basename() +#include // chmod_file() changes the file mode bits of a given file (relative // to the directory file descriptor) according to mode. mode is an @@ -246,11 +252,30 @@ unsigned int get_path_usage(const char *path, char buffer[64]) return 0; } - // Explicitly cast the block counts to unsigned long long to avoid - // overflowing with drives larger than 4 GB on 32bit systems - const unsigned long long size = (unsigned long long)f.f_blocks * f.f_frsize; - const unsigned long long free = (unsigned long long)f.f_bavail * f.f_bsize; - const unsigned long long used = size - free; + // Explicitly cast the block counts to uint64_t to avoid overflowing + // with drives larger than 4 GB on 32bit systems. Multiply the block + // count with the fragment size to get the total size in bytes, see + // https://github.com/torvalds/linux/blob/39cd87c4eb2b893354f3b850f916353f2658ae6f/fs/nfs/super.c#L285-L291 + const uint64_t size = (uint64_t)f.f_blocks * f.f_frsize; + const uint64_t free = (uint64_t)f.f_bavail * f.f_frsize; + const uint64_t used = size - free; + + // Print statvfs() results if in debug.gc mode + if(config.debug.gc.v.b) + { + log_debug(DEBUG_GC, "Statvfs() results for %s:", path); + log_debug(DEBUG_GC, " Block size: %lu", f.f_bsize); + log_debug(DEBUG_GC, " Fragment size: %lu", f.f_frsize); + log_debug(DEBUG_GC, " Total blocks: %"PRIu64, f.f_blocks); + log_debug(DEBUG_GC, " Free blocks: %"PRIu64, f.f_bfree); + log_debug(DEBUG_GC, " Available blocks: %"PRIu64, f.f_bavail); + log_debug(DEBUG_GC, " Total inodes: %"PRIu64, f.f_files); + log_debug(DEBUG_GC, " Free inodes: %"PRIu64, f.f_ffree); + log_debug(DEBUG_GC, " Available inodes: %"PRIu64, f.f_favail); + log_debug(DEBUG_GC, " Filesystem ID: %lu", f.f_fsid); + log_debug(DEBUG_GC, " Mount flags: %lu", f.f_flag); + log_debug(DEBUG_GC, " Maximum filename length: %lu", f.f_namemax); + } // Create human-readable total size char prefix_size[2] = { 0 }; @@ -269,37 +294,51 @@ unsigned int get_path_usage(const char *path, char buffer[64]) // If size is 0, we return 0% to avoid division by zero below if(size == 0) return 0; - // If used is larger than size, we return 100% - if(used > size) - return 100; + // Return percentage of used memory at this path (rounded down) - return (used*100)/(size + 1); + // If the used size is larger than the total size, this intentionally + // returns more than 100% so that the caller can handle this case + // (this can happen with docker on macOS) + return (used * 100) / size; } // Get the filesystem where the given path is located struct mntent *get_filesystem_details(const char *path) { - /* stat the file in question */ + // stat the file in question struct stat path_stat; stat(path, &path_stat); - /* iterate through the list of devices */ + // iterate through the list of devices FILE *file = setmntent("/proc/mounts", "r"); struct mntent *ent = NULL; + bool found = false; while(file != NULL && (ent = getmntent(file)) != NULL) { - /* stat the mount point */ + // stat the mount point struct stat dev_stat; - stat(ent->mnt_dir, &dev_stat); + if(stat(ent->mnt_dir, &dev_stat) < 0) + { + if(config.debug.gc.v.b) + { + log_warn("get_filesystem_details(): Failed to get stat for \"%s\": %s", + ent->mnt_dir, strerror(errno)); + } + continue; + } - /* check if our file and the mount point are on the same device */ + // check if our file and the mount point are on the same device if(dev_stat.st_dev == path_stat.st_dev) + { + found = true; break; + } } + // Close mount table file handle endmntent(file); - return ent; + return found ? ent : NULL; } // Credits: https://stackoverflow.com/a/55410469 @@ -395,29 +434,35 @@ static int copy_file(const char *source, const char *destination) } // Change ownership of file to pihole user -static bool chown_pihole(const char *path) +bool chown_pihole(const char *path, struct passwd *pwd) { - // Get pihole user's uid and gid - struct passwd *pwd = getpwnam("pihole"); + // Get pihole user's UID and GID if not provided if(pwd == NULL) { - log_warn("chown_pihole(): Failed to get pihole user's uid: %s", strerror(errno)); - return false; + pwd = getpwnam("pihole"); + if(pwd == NULL) + { + log_warn("chown_pihole(): Failed to get pihole user's UID/GID: %s", strerror(errno)); + return false; + } } - struct group *grp = getgrnam("pihole"); - if(grp == NULL) + + // Get group name + struct group *grp = getgrgid(pwd->pw_gid); + const char *grp_name = grp != NULL ? grp->gr_name : ""; + + // Change ownership of file to pihole user + if(chown(path, pwd->pw_uid, pwd->pw_gid) < 0) { - log_warn("chown_pihole(): Failed to get pihole user's gid: %s", strerror(errno)); + log_warn("Failed to change ownership of \"%s\" to %s:%s (%u:%u): %s", + path, pwd->pw_name, grp_name, pwd->pw_uid, pwd->pw_gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return false; } - // Change ownership of file to pihole user - if(chown(path, pwd->pw_uid, grp->gr_gid) < 0) - { - log_warn("chown_pihole(): Failed to change ownership of \"%s\" to %u:%u: %s", - path, pwd->pw_uid, grp->gr_gid, strerror(errno)); - return false; - } + log_debug(DEBUG_INOTIFY, "Changed ownership of \"%s\" to %s:%s (%u:%u)", + path, pwd->pw_name, grp_name, pwd->pw_uid, pwd->pw_gid); return true; } @@ -494,7 +539,7 @@ void rotate_files(const char *path, char **first_file) } // Change ownership of file to pihole user - chown_pihole(new_path); + chown_pihole(new_path, NULL); } // Free memory @@ -671,7 +716,7 @@ bool files_different(const char *pathA, const char* pathB, unsigned int from) } // Create SHA256 checksum of a file -bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) +bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE], const bool skip_end) { // Open file FILE *fp = fopen(path, "rb"); @@ -685,14 +730,30 @@ bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) struct sha256_ctx ctx; sha256_init(&ctx); - // Read file in chunks of bytes - const size_t pagesize = getpagesize(); - unsigned char *buf = calloc(pagesize, sizeof(char)); + // Get size of file + fseek(fp, 0, SEEK_END); + size_t filesize = ftell(fp); + fseek(fp, 0, SEEK_SET); + + // Determine chunk size + size_t chunksize = getpagesize(); + + // Read file in chunks + unsigned char *buf = calloc(chunksize, sizeof(char)); size_t len; - while((len = fread(buf, sizeof(char), pagesize, fp)) > 0) + while((len = fread(buf, sizeof(char), chunksize, fp)) > 0) { // Update SHA256 context sha256_update(&ctx, len, buf); + + // Reduce filesize by the number of bytes read + filesize -= len; + + // If we want to skip the end of the file, we have to adjust the + // chunk size to the remaining bytes minus the size of the SHA256 + // checksum itself + if(skip_end && filesize <= chunksize + SHA256_DIGEST_SIZE) + chunksize = filesize - SHA256_DIGEST_SIZE; } // Finalize SHA256 context @@ -706,3 +767,78 @@ bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) return true; } + +/** + * @brief Verifies the integrity of the current executable file by comparing its + * SHA256 checksum with a pre-computed hash stored in the last 8 bytes of the + * binary. + * + * @param verbose A boolean value indicating whether verbose output should be + * enabled. + * @return Returns true if the checksum matches the expected value, false + * otherwise. + */ +bool verify_FTL(bool verbose) +{ + // Get the filename of the current executable + char filename[PATH_MAX] = { 0 }; + if(readlink("/proc/self/exe", filename, sizeof(filename)) == -1) + { + log_err("Failed to read self filename: %s", strerror(errno)); + return -1; + } + + // Read the pre-computed hash - it is stored in the last 8 bytes of the + // binary itself + uint8_t self_hash[SHA256_DIGEST_SIZE]; + FILE *f = fopen(filename, "r"); + if(f == NULL) + { + log_err("Failed to open self file \"%s\": %s", filename, strerror(errno)); + return -1; + } + if(fseek(f, -SHA256_DIGEST_SIZE, SEEK_END) != 0) + { + log_err("Failed to seek to hash: %s", strerror(errno)); + fclose(f); + return -1; + } + if(fread(self_hash, SHA256_DIGEST_SIZE, 1, f) != 1) + { + log_err("Failed to read hash: %s", strerror(errno)); + fclose(f); + return -1; + } + fclose(f); + + // Calculate the hash of the binary + // Skip the last 256 bit as it contains the hast itself + uint8_t checksum[SHA256_DIGEST_SIZE]; + if(!sha256sum(filename, checksum, true)) + { + log_err("Failed to calculate SHA256 checksum of %s", filename); + return false; + } + + // Compare the checksums + bool success = memcmp(checksum, self_hash, SHA256_DIGEST_SIZE) == 0; + if(!success) + { + // Convert checksums to human-readable hex strings + char expected_hex[SHA256_DIGEST_SIZE*2+1]; + sha256_raw_to_hex(self_hash, expected_hex); + char actual_hex[SHA256_DIGEST_SIZE*2+1]; + sha256_raw_to_hex(checksum, actual_hex); + + if(!verbose) // during startup + log_verify_message(expected_hex, actual_hex); + else // CLI verification + { + log_err("Checksum verification failed!"); + log_err("Expected: %s", expected_hex); + log_err("Actual: %s", actual_hex); + } + } + + return success; +} diff --git a/src/files.h b/src/files.h index 742e8d9a..b1096e14 100644 --- a/src/files.h +++ b/src/files.h @@ -16,6 +16,8 @@ #include // SHA256_DIGEST_SIZE #include +// getpwuid() +#include #define MAX_ROTATIONS 15 #define BACKUP_DIR "/etc/pihole/config_backups" @@ -31,9 +33,11 @@ void ls_dir(const char* path); unsigned int get_path_usage(const char *path, char buffer[64]); struct mntent *get_filesystem_details(const char *path); bool directory_exists(const char *path); +bool chown_pihole(const char *path, struct passwd *pwd); void rotate_files(const char *path, char **first_file); bool files_different(const char *pathA, const char* pathB, unsigned int from); -bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]); +bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE], const bool skip_end); +bool verify_FTL(bool verbose); int parse_line(char *line, char **key, char **value); diff --git a/src/gc.c b/src/gc.c index af1680b3..1aaf036a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -62,7 +62,7 @@ static void recycle(void) // and recycle them for(int queryID = 0; queryID < counters->queries; queryID++) { - queriesData* query = getQuery(queryID, true); + queriesData *query = getQuery(queryID, true); if(query == NULL) continue; @@ -262,8 +262,8 @@ static void check_load(void) if (getloadavg(load, 3) == -1) return; - // Get number of CPU cores - const int nprocs = get_nprocs(); + // Get total number of CPU cores + const int nprocs = get_nprocs_conf(); // Warn if 15 minute average of load exceeds number of available // processors @@ -298,7 +298,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) if(config.debug.gc.v.b) { timer_start(GC_TIMER); - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; get_timestr(timestring, mintime, false, false); log_debug(DEBUG_GC, "GC starting, mintime: %s (%lu), counters->queries = %d", timestring, (unsigned long)mintime, counters->queries); @@ -308,7 +308,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) unsigned int removed = 0; for(long int i = 0; i < counters->queries; i++) { - queriesData* query = getQuery(i, true); + queriesData *query = getQuery(i, true); if(query == NULL) continue; @@ -354,6 +354,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through) case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through) case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through) + case QUERY_EXTERNAL_BLOCKED_EDE15: // Blocked by upstream provider (fall through) case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through) case QUERY_REGEX_CNAME: // Regex denied domain in CNAME chain (fall through) case QUERY_DENYLIST_CNAME: // Exactly denied domain in CNAME chain (fall through) @@ -481,8 +482,6 @@ static bool check_files_on_same_device(const char *path1, const char *path2) void *GC_thread(void *val) { // Set thread name - thread_names[GC] = "housekeeper"; - thread_running[GC] = true; prctl(PR_SET_NAME, thread_names[GC], 0, 0, 0); // Remember when we last ran the actions @@ -568,6 +567,5 @@ void *GC_thread(void *val) watch_config(false); log_info("Terminating GC thread"); - thread_running[GC] = false; return NULL; } diff --git a/src/log.c b/src/log.c index 8eac8c70..8b885ca0 100644 --- a/src/log.c +++ b/src/log.c @@ -53,8 +53,9 @@ void init_FTL_log(const char *name) FILE *logfile = NULL; if((logfile = fopen(config.files.log.ftl.v.s, "a+")) == NULL) { + printf("ERROR: Opening of FTL log (%s) failed: %s\nUsing syslog instead!\n", + config.files.log.ftl.v.s, strerror(errno)); syslog(LOG_ERR, "Opening of FTL\'s log file failed, using syslog instead!"); - printf("ERROR: Opening of FTL log (%s) failed!\n",config.files.log.ftl.v.s); config.files.log.ftl.v.s = NULL; } @@ -85,8 +86,7 @@ double double_time(void) return tp.tv_sec + 1e-9*tp.tv_nsec; } -// The size of 84 bytes has been carefully selected for all possible timestamps -// to always fit into the available space without buffer overflows +// Get a human-readable time string void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible) { struct tm tm; @@ -105,16 +105,19 @@ void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool gettimeofday(&tv, NULL); const int millisec = tv.tv_usec/1000; - sprintf(timestring,"%d-%02d-%02d%c%02d%c%02d%c%02d.%03i", + snprintf(timestring, TIMESTR_SIZE, "%d-%02d-%02d%c%02d%c%02d%c%02d.%03i%c%s", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, space, - tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec, millisec); + tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec, millisec, space, tm.tm_zone); } else { - sprintf(timestring,"%d-%02d-%02d%c%02d%c%02d%c%02d", + snprintf(timestring, TIMESTR_SIZE, "%d-%02d-%02d%c%02d%c%02d%c%02d%c%s", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, space, - tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec); + tm.tm_hour, colon, tm.tm_min, colon, tm.tm_sec, space, tm.tm_zone); } + + // Ensure that the string is zero-terminated + timestring[TIMESTR_SIZE - 1] = '\0'; } // Return the current year @@ -217,16 +220,19 @@ const char *debugstr(const enum debug_flag flag) return "DEBUG_WEBSERVER"; case DEBUG_RESERVED: return "DEBUG_RESERVED"; + case DEBUG_NTP: + return "DEBUG_NTP"; case DEBUG_MAX: return "DEBUG_MAX"; + case DEBUG_NONE: // fall through default: return "DEBUG_ANY"; } } -void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) +void __attribute__ ((format (printf, 3, 4))) _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) { - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; va_list args; // We have been explicitly asked to not print anything to the log @@ -283,6 +289,7 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co va_end(args); add_to_fifo_buffer(FIFO_FTL, buffer, prio, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); + bool logged = false; if(config.files.log.ftl.v.s != NULL) { // Open log file @@ -304,6 +311,8 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co // Close file after writing fclose(logfile); + + logged = true; } else if(!daemonmode) { @@ -311,7 +320,7 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co syslog(LOG_ERR, "Writing to FTL\'s log file failed!"); } } - else + if(!logged) { // Syslog logging va_start(args, format); @@ -321,9 +330,9 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co } } -void __attribute__ ((format (gnu_printf, 1, 2))) log_web(const char *format, ...) +void __attribute__ ((format (printf, 1, 2))) log_web(const char *format, ...) { - char timestring[TIMESTR_SIZE] = ""; + char timestring[TIMESTR_SIZE]; const time_t now = time(NULL); va_list args; @@ -362,7 +371,7 @@ void __attribute__ ((format (gnu_printf, 1, 2))) log_web(const char *format, ... } // Log helper activity (may be script or lua) -void FTL_log_helper(const unsigned char n, ...) +void FTL_log_helper(const unsigned int n, ...) { // Only log helper debug messages if enabled if(!(config.debug.helper.v.b)) @@ -372,7 +381,7 @@ void FTL_log_helper(const unsigned char n, ...) va_list args; char **arg = calloc(n, sizeof(char*)); va_start(args, n); - for(unsigned char i = 0; i < n; i++) + for(unsigned int i = 0; i < n; i++) { const char *argin = va_arg(args, char*); if(argin == NULL) @@ -401,25 +410,24 @@ void FTL_log_helper(const unsigned char n, ...) } // Free allocated memory - for(unsigned char i = 0; i < n; i++) + for(unsigned int i = 0; i < n; i++) if(arg[i] != NULL) free(arg[i]); free(arg); } -void format_memory_size(char prefix[2], const unsigned long long int bytes, - double * const formatted) +void format_memory_size(char prefix[2], const uint64_t bytes, double * const formatted) { unsigned int i; *formatted = bytes; // Determine exponent for human-readable display - for(i = 0; i < 7; i++) + const char prefixes[] = { '\0', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y', 'R', '?' }; + for(i = 0; i < sizeof(prefixes)/sizeof(*prefixes) - 1; i++) { if(*formatted <= 1e3) break; *formatted /= 1e3; } - const char prefixes[8] = { '\0', 'K', 'M', 'G', 'T', 'P', 'E', '?' }; // Chose matching SI prefix prefix[0] = prefixes[i]; prefix[1] = '\0'; diff --git a/src/log.h b/src/log.h index 59495921..6e550ad9 100644 --- a/src/log.h +++ b/src/log.h @@ -15,9 +15,11 @@ // enums #include "enums.h" #include +// uint64_t +#include #define DEBUG_ANY 0 -#define TIMESTR_SIZE 84 +#define TIMESTR_SIZE 128 // Credit: https://stackoverflow.com/a/75116514 #define LEFT(str, w) \ @@ -43,8 +45,7 @@ extern bool only_testing; void clear_debug_flags(void); void init_FTL_log(const char *name); void log_counter_info(void); -void format_memory_size(char prefix[2], unsigned long long int bytes, - double * const formatted); +void format_memory_size(char prefix[2], const uint64_t bytes, double * const formatted); void format_time(char buffer[42], unsigned long seconds, double milliseconds); unsigned int get_year(const time_t timein); const char *get_FTL_version(void); @@ -52,7 +53,7 @@ void log_FTL_version(bool crashreport); double double_time(void); void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible); const char *debugstr(const enum debug_flag flag) __attribute__((const)); -void log_web(const char *format, ...) __attribute__ ((format (gnu_printf, 1, 2))); +void log_web(const char *format, ...) __attribute__ ((format (printf, 1, 2))); const char *get_ordinal_suffix(unsigned int number) __attribute__ ((const)); void print_FTL_version(void); unsigned int countchar(const char *str, const char c) __attribute__ ((pure)); @@ -65,14 +66,13 @@ void dnsmasq_diagnosis_warning(char *message); #define log_warn(format, ...) _FTL_log(LOG_WARNING, 0, format, ## __VA_ARGS__) #define log_notice(format, ...) _FTL_log(LOG_NOTICE, 0, format, ## __VA_ARGS__) #define log_info(format, ...) _FTL_log(LOG_INFO, 0, format, ## __VA_ARGS__) -#define log_debug(flag, format, ...)({ \ +#define log_debug(flag, format, ...) \ if(flag > -1 && flag < DEBUG_MAX && debug_flags[flag]) \ - _FTL_log(LOG_DEBUG, flag, format, ## __VA_ARGS__); \ -}) -void _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) __attribute__ ((format (gnu_printf, 3, 4))); -void FTL_log_dnsmasq_fatal(const char *format, ...) __attribute__ ((format (gnu_printf, 1, 2))); + _FTL_log(LOG_DEBUG, flag, format, ## __VA_ARGS__) +void _FTL_log(const int priority, const enum debug_flag flag, const char *format, ...) __attribute__ ((format (printf, 3, 4))); +void FTL_log_dnsmasq_fatal(const char *format, ...) __attribute__ ((format (printf, 1, 2))); void log_ctrl(bool vlog, bool vstdout); -void FTL_log_helper(const unsigned char n, ...); +void FTL_log_helper(const unsigned int n, ...); int binbuf_to_escaped_C_literal(const char *src_buf, size_t src_sz, char *dst_str, size_t dst_sz); @@ -83,7 +83,7 @@ int blocked_queries(void) __attribute__ ((pure)); const char *short_path(const char *full_path) __attribute__ ((pure)); // How long is each line in the FIFO buffer allowed to be? -#define MAX_MSG_FIFO 256u +#define MAX_MSG_FIFO 260u // How many messages do we keep in memory (FIFO message buffer)? // This number multiplied by MAX_MSG_FIFO (see above) gives the total buffer size @@ -96,9 +96,9 @@ bool flush_dnsmasq_log(void); typedef struct { struct { + char message[LOG_SIZE][MAX_MSG_FIFO]; unsigned int next_id; double timestamp[LOG_SIZE]; - char message[LOG_SIZE][MAX_MSG_FIFO]; const char *prio[LOG_SIZE]; } logs[FIFO_MAX]; } fifologData; diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 908b5a7e..90ddda91 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -1,6 +1,11 @@ -set(sources +set(ftl_sources ftl_lua.c ftl_lua.h +) + +add_library(ftl_lua OBJECT ${ftl_sources}) + +set(sources lapi.c lapi.h lauxlib.c @@ -65,7 +70,10 @@ set(sources ) add_library(lua OBJECT ${sources}) -target_compile_options(lua PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-value) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(lua PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-value) + target_compile_options(ftl_lua PRIVATE -Wno-unused-value ${EXTRAWARN}) +endif() # LUA_USE_POSIX: ensures recommended POSIX functions are used instead of # (partially obsoleted) standard C functions @@ -78,11 +86,5 @@ if(LUA_DL STREQUAL "true") target_compile_definitions(lua PRIVATE LUA_USE_DLOPEN) endif() -if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP) - message(STATUS "Embedded LUA will use readline for history: YES") - target_compile_definitions(lua PRIVATE LUA_USE_READLINE) -else() - message(STATUS "Embedded LUA will use readline for history: NO") -endif() - target_include_directories(lua PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/lua) +target_include_directories(ftl_lua PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/lua) diff --git a/src/lua/ftl_lua.c b/src/lua/ftl_lua.c index 5f448406..5da8202d 100644 --- a/src/lua/ftl_lua.c +++ b/src/lua/ftl_lua.c @@ -8,23 +8,33 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" #include "ftl_lua.h" + +#include "FTL.h" // struct luaL_Reg #include "lauxlib.h" // get_FTL_version() -#include "../log.h" +#include "log.h" // config struct -#include "../config/config.h" +#include "config/config.h" // file_exists -#include "../files.h" +#include "files.h" // get_web_theme_str -#include "../datastructure.h" -#include -#include +#include "datastructure.h" +#include "api/api.h" #include "scripts/scripts.h" -#include "api/api.h" +// prototype for luaopen_pihole() +#include "lualib.h" + +#if defined(LUA_USE_READLINE) +# include +#endif +#include + +// hostname() +#include "daemon.h" + int run_lua_interpreter(const int argc, char **argv, bool dnsmasq_debug) { @@ -98,11 +108,8 @@ static int pihole_ftl_version(lua_State *L) { // pihole.hostname() static int pihole_hostname(lua_State *L) { - // Get host name - char name[256]; - if(gethostname(name, sizeof(name)) != 0) - strcpy(name, "N/A"); - lua_pushstring(L, name); + // Get and immediately push host name + lua_pushstring(L, hostname()); return 1; // number of results } @@ -223,32 +230,13 @@ static int pihole_boxedlayout(lua_State *L) { return 1; // number of results } -// pihole.needLogin(remote_addr:str) +// pihole.needLogin() static int pihole_needLogin(lua_State *L) { - // Get remote_addr (first argument to LUA function) - const char *remote_addr = luaL_checkstring(L, 1); - // Check if password is set const bool has_password = config.webserver.api.pwhash.v.s != NULL && config.webserver.api.pwhash.v.s[0] != '\0'; - // Check if address is loopback - const bool is_loopback = strcmp(remote_addr, LOCALHOSTv4) == 0 || - strcmp(remote_addr, LOCALHOSTv6) == 0; - - // Check if local API authentication is enabled - const bool localAPIauth = config.webserver.api.localAPIauth.v.b; - - // Check if login is required - const bool need_login = has_password || (is_loopback && !localAPIauth); - - lua_pushboolean(L, need_login); - return 1; // number of results -} - -// pihole.rev_proxy() -static int pihole_rev_proxy(lua_State *L) { - lua_pushboolean(L, config.webserver.tls.rev_proxy.v.b); + lua_pushboolean(L, has_password); return 1; // number of results } @@ -261,7 +249,6 @@ static const luaL_Reg piholelib[] = { {"include", pihole_include}, {"boxedlayout", pihole_boxedlayout}, {"needLogin", pihole_needLogin}, - {"rev_proxy", pihole_rev_proxy}, {NULL, NULL} }; diff --git a/src/lua/ftl_lua.h b/src/lua/ftl_lua.h index 29c4cb5f..30bad1f9 100644 --- a/src/lua/ftl_lua.h +++ b/src/lua/ftl_lua.h @@ -21,9 +21,7 @@ int run_luac(const int argc, char **argv); int lua_main (int argc, char **argv); int luac_main (int argc, char **argv); -extern int dolibrary (lua_State *L, char *name); - void print_embedded_scripts(void); void ftl_lua_init(lua_State *L); -#endif //FTL_LUA_H \ No newline at end of file +#endif //FTL_LUA_H diff --git a/src/lua/lapi.c b/src/lua/lapi.c index 34e64af1..332e97d1 100644 --- a/src/lua/lapi.c +++ b/src/lua/lapi.c @@ -417,9 +417,9 @@ LUA_API const char *lua_tolstring (lua_State *L, int idx, size_t *len) { o = index2value(L, idx); /* previous call may reallocate the stack */ } if (len != NULL) - *len = vslen(o); + *len = tsslen(tsvalue(o)); lua_unlock(L); - return svalue(o); + return getstr(tsvalue(o)); } diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c index 4ca6c654..923105ed 100644 --- a/src/lua/lauxlib.c +++ b/src/lua/lauxlib.c @@ -80,6 +80,7 @@ static int pushglobalfuncname (lua_State *L, lua_Debug *ar) { int top = lua_gettop(L); lua_getinfo(L, "f", ar); /* push function */ lua_getfield(L, LUA_REGISTRYINDEX, LUA_LOADED_TABLE); + luaL_checkstack(L, 6, "not enough stack"); /* slots for 'findfield' */ if (findfield(L, top + 1, 2)) { const char *name = lua_tostring(L, -1); if (strncmp(name, LUA_GNAME ".", 3) == 0) { /* name start with '_G.'? */ @@ -249,11 +250,13 @@ LUALIB_API int luaL_fileresult (lua_State *L, int stat, const char *fname) { return 1; } else { + const char *msg; luaL_pushfail(L); + msg = (en != 0) ? strerror(en) : "(no extra info)"; if (fname) - lua_pushfstring(L, "%s: %s", fname, strerror(en)); + lua_pushfstring(L, "%s: %s", fname, msg); else - lua_pushstring(L, strerror(en)); + lua_pushstring(L, msg); lua_pushinteger(L, en); return 3; } @@ -732,9 +735,12 @@ static const char *getF (lua_State *L, void *ud, size_t *size) { static int errfile (lua_State *L, const char *what, int fnameindex) { - const char *serr = strerror(errno); + int err = errno; const char *filename = lua_tostring(L, fnameindex) + 1; - lua_pushfstring(L, "cannot %s %s: %s", what, filename, serr); + if (err != 0) + lua_pushfstring(L, "cannot %s %s: %s", what, filename, strerror(err)); + else + lua_pushfstring(L, "cannot %s %s", what, filename); lua_remove(L, fnameindex); return LUA_ERRFILE; } @@ -787,6 +793,7 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, } else { lua_pushfstring(L, "@%s", filename); + errno = 0; lf.f = fopen(filename, "r"); if (lf.f == NULL) return errfile(L, "open", fnameindex); } @@ -796,6 +803,7 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, if (c == LUA_SIGNATURE[0]) { /* binary file? */ lf.n = 0; /* remove possible newline */ if (filename) { /* "real" file? */ + errno = 0; lf.f = freopen(filename, "rb", lf.f); /* reopen in binary mode */ if (lf.f == NULL) return errfile(L, "reopen", fnameindex); skipcomment(lf.f, &c); /* re-read initial portion */ @@ -803,6 +811,7 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, } if (c != EOF) lf.buff[lf.n++] = c; /* 'c' is the first character of the stream */ + errno = 0; status = lua_load(L, getF, &lf, lua_tostring(L, -1), mode); readstatus = ferror(lf.f); if (filename) fclose(lf.f); /* close file (even in case of errors) */ @@ -933,7 +942,7 @@ LUALIB_API const char *luaL_tolstring (lua_State *L, int idx, size_t *len) { LUALIB_API void luaL_setfuncs (lua_State *L, const luaL_Reg *l, int nup) { luaL_checkstack(L, nup, "too many upvalues"); for (; l->name != NULL; l++) { /* fill the table with given functions */ - if (l->func == NULL) /* place holder? */ + if (l->func == NULL) /* placeholder? */ lua_pushboolean(L, 0); else { int i; @@ -1025,9 +1034,14 @@ static void *l_alloc (void *ud, void *ptr, size_t osize, size_t nsize) { } +/* +** Standard panic funcion just prints an error message. The test +** with 'lua_type' avoids possible memory errors in 'lua_tostring'. +*/ static int panic (lua_State *L) { - const char *msg = lua_tostring(L, -1); - if (msg == NULL) msg = "error object is not a string"; + const char *msg = (lua_type(L, -1) == LUA_TSTRING) + ? lua_tostring(L, -1) + : "error object is not a string"; lua_writestringerror("PANIC: unprotected error in call to Lua API (%s)\n", msg); return 0; /* return to Lua to abort */ diff --git a/src/lua/lcode.c b/src/lua/lcode.c index 1a371ca9..87616140 100644 --- a/src/lua/lcode.c +++ b/src/lua/lcode.c @@ -415,7 +415,7 @@ int luaK_codeABx (FuncState *fs, OpCode o, int a, unsigned int bc) { /* ** Format and emit an 'iAsBx' instruction. */ -int luaK_codeAsBx (FuncState *fs, OpCode o, int a, int bc) { +static int codeAsBx (FuncState *fs, OpCode o, int a, int bc) { unsigned int b = bc + OFFSET_sBx; lua_assert(getOpMode(o) == iAsBx); lua_assert(a <= MAXARG_A && b <= MAXARG_Bx); @@ -671,7 +671,7 @@ static int fitsBx (lua_Integer i) { void luaK_int (FuncState *fs, int reg, lua_Integer i) { if (fitsBx(i)) - luaK_codeAsBx(fs, OP_LOADI, reg, cast_int(i)); + codeAsBx(fs, OP_LOADI, reg, cast_int(i)); else luaK_codek(fs, reg, luaK_intK(fs, i)); } @@ -680,7 +680,7 @@ void luaK_int (FuncState *fs, int reg, lua_Integer i) { static void luaK_float (FuncState *fs, int reg, lua_Number f) { lua_Integer fi; if (luaV_flttointeger(f, &fi, F2Ieq) && fitsBx(fi)) - luaK_codeAsBx(fs, OP_LOADF, reg, cast_int(fi)); + codeAsBx(fs, OP_LOADF, reg, cast_int(fi)); else luaK_codek(fs, reg, luaK_numberK(fs, f)); } @@ -776,7 +776,8 @@ void luaK_dischargevars (FuncState *fs, expdesc *e) { break; } case VLOCAL: { /* already in a register */ - e->u.info = e->u.var.ridx; + int temp = e->u.var.ridx; + e->u.info = temp; /* (can't do a direct assignment; values overlap) */ e->k = VNONRELOC; /* becomes a non-relocatable value */ break; } @@ -1025,7 +1026,7 @@ static int luaK_exp2K (FuncState *fs, expdesc *e) { ** in the range of R/K indices). ** Returns 1 iff expression is K. */ -int luaK_exp2RK (FuncState *fs, expdesc *e) { +static int exp2RK (FuncState *fs, expdesc *e) { if (luaK_exp2K(fs, e)) return 1; else { /* not a constant in the right range: put it in a register */ @@ -1037,7 +1038,7 @@ int luaK_exp2RK (FuncState *fs, expdesc *e) { static void codeABRK (FuncState *fs, OpCode o, int a, int b, expdesc *ec) { - int k = luaK_exp2RK(fs, ec); + int k = exp2RK(fs, ec); luaK_codeABCk(fs, o, a, b, ec->u.info, k); } @@ -1215,7 +1216,7 @@ static void codenot (FuncState *fs, expdesc *e) { /* -** Check whether expression 'e' is a small literal string +** Check whether expression 'e' is a short literal string */ static int isKstr (FuncState *fs, expdesc *e) { return (e->k == VK && !hasjumps(e) && e->u.info <= MAXARG_B && @@ -1225,7 +1226,7 @@ static int isKstr (FuncState *fs, expdesc *e) { /* ** Check whether expression 'e' is a literal integer. */ -int luaK_isKint (expdesc *e) { +static int isKint (expdesc *e) { return (e->k == VKINT && !hasjumps(e)); } @@ -1235,7 +1236,7 @@ int luaK_isKint (expdesc *e) { ** proper range to fit in register C */ static int isCint (expdesc *e) { - return luaK_isKint(e) && (l_castS2U(e->u.ival) <= l_castS2U(MAXARG_C)); + return isKint(e) && (l_castS2U(e->u.ival) <= l_castS2U(MAXARG_C)); } @@ -1244,7 +1245,7 @@ static int isCint (expdesc *e) { ** proper range to fit in register sC */ static int isSCint (expdesc *e) { - return luaK_isKint(e) && fitsC(e->u.ival); + return isKint(e) && fitsC(e->u.ival); } @@ -1283,15 +1284,17 @@ void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k) { if (t->k == VUPVAL && !isKstr(fs, k)) /* upvalue indexed by non 'Kstr'? */ luaK_exp2anyreg(fs, t); /* put it in a register */ if (t->k == VUPVAL) { - t->u.ind.t = t->u.info; /* upvalue index */ - t->u.ind.idx = k->u.info; /* literal string */ + int temp = t->u.info; /* upvalue index */ + lua_assert(isKstr(fs, k)); + t->u.ind.t = temp; /* (can't do a direct assignment; values overlap) */ + t->u.ind.idx = k->u.info; /* literal short string */ t->k = VINDEXUP; } else { /* register index of the table */ t->u.ind.t = (t->k == VLOCAL) ? t->u.var.ridx: t->u.info; if (isKstr(fs, k)) { - t->u.ind.idx = k->u.info; /* literal string */ + t->u.ind.idx = k->u.info; /* literal short string */ t->k = VINDEXSTR; } else if (isCint(k)) { @@ -1459,7 +1462,7 @@ static void codebinK (FuncState *fs, BinOpr opr, */ static int finishbinexpneg (FuncState *fs, expdesc *e1, expdesc *e2, OpCode op, int line, TMS event) { - if (!luaK_isKint(e2)) + if (!isKint(e2)) return 0; /* not an integer constant */ else { lua_Integer i2 = e2->u.ival; @@ -1592,7 +1595,7 @@ static void codeeq (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2) { op = OP_EQI; r2 = im; /* immediate operand */ } - else if (luaK_exp2RK(fs, e2)) { /* 2nd expression is constant? */ + else if (exp2RK(fs, e2)) { /* 2nd expression is constant? */ op = OP_EQK; r2 = e2->u.info; /* constant index */ } @@ -1658,7 +1661,7 @@ void luaK_infix (FuncState *fs, BinOpr op, expdesc *v) { } case OPR_EQ: case OPR_NE: { if (!tonumeral(v, NULL)) - luaK_exp2RK(fs, v); + exp2RK(fs, v); /* else keep numeral, which may be an immediate operand */ break; } diff --git a/src/lua/lcode.h b/src/lua/lcode.h index 32658244..0b971fc4 100644 --- a/src/lua/lcode.h +++ b/src/lua/lcode.h @@ -61,10 +61,8 @@ typedef enum UnOpr { OPR_MINUS, OPR_BNOT, OPR_NOT, OPR_LEN, OPR_NOUNOPR } UnOpr; LUAI_FUNC int luaK_code (FuncState *fs, Instruction i); LUAI_FUNC int luaK_codeABx (FuncState *fs, OpCode o, int A, unsigned int Bx); -LUAI_FUNC int luaK_codeAsBx (FuncState *fs, OpCode o, int A, int Bx); LUAI_FUNC int luaK_codeABCk (FuncState *fs, OpCode o, int A, int B, int C, int k); -LUAI_FUNC int luaK_isKint (expdesc *e); LUAI_FUNC int luaK_exp2const (FuncState *fs, const expdesc *e, TValue *v); LUAI_FUNC void luaK_fixline (FuncState *fs, int line); LUAI_FUNC void luaK_nil (FuncState *fs, int from, int n); @@ -76,7 +74,6 @@ LUAI_FUNC int luaK_exp2anyreg (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_exp2anyregup (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_exp2nextreg (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_exp2val (FuncState *fs, expdesc *e); -LUAI_FUNC int luaK_exp2RK (FuncState *fs, expdesc *e); LUAI_FUNC void luaK_self (FuncState *fs, expdesc *e, expdesc *key); LUAI_FUNC void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k); LUAI_FUNC void luaK_goiftrue (FuncState *fs, expdesc *e); diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c index 28b1caab..591b3528 100644 --- a/src/lua/ldebug.c +++ b/src/lua/ldebug.c @@ -31,7 +31,7 @@ -#define noLuaClosure(f) ((f) == NULL || (f)->c.tt == LUA_VCCL) +#define LuaClosure(f) ((f) != NULL && (f)->c.tt == LUA_VLCL) static const char *funcnamefromcall (lua_State *L, CallInfo *ci, @@ -254,7 +254,7 @@ LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n) { static void funcinfo (lua_Debug *ar, Closure *cl) { - if (noLuaClosure(cl)) { + if (!LuaClosure(cl)) { ar->source = "=[C]"; ar->srclen = LL("=[C]"); ar->linedefined = -1; @@ -288,29 +288,31 @@ static int nextline (const Proto *p, int currentline, int pc) { static void collectvalidlines (lua_State *L, Closure *f) { - if (noLuaClosure(f)) { + if (!LuaClosure(f)) { setnilvalue(s2v(L->top.p)); api_incr_top(L); } else { - int i; - TValue v; const Proto *p = f->l.p; int currentline = p->linedefined; Table *t = luaH_new(L); /* new table to store active lines */ sethvalue2s(L, L->top.p, t); /* push it on stack */ api_incr_top(L); - setbtvalue(&v); /* boolean 'true' to be the value of all indices */ - if (!p->is_vararg) /* regular function? */ - i = 0; /* consider all instructions */ - else { /* vararg function */ - lua_assert(GET_OPCODE(p->code[0]) == OP_VARARGPREP); - currentline = nextline(p, currentline, 0); - i = 1; /* skip first instruction (OP_VARARGPREP) */ - } - for (; i < p->sizelineinfo; i++) { /* for each instruction */ - currentline = nextline(p, currentline, i); /* get its line */ - luaH_setint(L, t, currentline, &v); /* table[line] = true */ + if (p->lineinfo != NULL) { /* proto with debug information? */ + int i; + TValue v; + setbtvalue(&v); /* boolean 'true' to be the value of all indices */ + if (!p->is_vararg) /* regular function? */ + i = 0; /* consider all instructions */ + else { /* vararg function */ + lua_assert(GET_OPCODE(p->code[0]) == OP_VARARGPREP); + currentline = nextline(p, currentline, 0); + i = 1; /* skip first instruction (OP_VARARGPREP) */ + } + for (; i < p->sizelineinfo; i++) { /* for each instruction */ + currentline = nextline(p, currentline, i); /* get its line */ + luaH_setint(L, t, currentline, &v); /* table[line] = true */ + } } } } @@ -339,7 +341,7 @@ static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar, } case 'u': { ar->nups = (f == NULL) ? 0 : f->c.nupvalues; - if (noLuaClosure(f)) { + if (!LuaClosure(f)) { ar->isvararg = 1; ar->nparams = 0; } @@ -417,40 +419,6 @@ LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar) { ** ======================================================= */ -static const char *getobjname (const Proto *p, int lastpc, int reg, - const char **name); - - -/* -** Find a "name" for the constant 'c'. -*/ -static void kname (const Proto *p, int c, const char **name) { - TValue *kvalue = &p->k[c]; - *name = (ttisstring(kvalue)) ? svalue(kvalue) : "?"; -} - - -/* -** Find a "name" for the register 'c'. -*/ -static void rname (const Proto *p, int pc, int c, const char **name) { - const char *what = getobjname(p, pc, c, name); /* search for 'c' */ - if (!(what && *what == 'c')) /* did not find a constant name? */ - *name = "?"; -} - - -/* -** Find a "name" for a 'C' value in an RK instruction. -*/ -static void rkname (const Proto *p, int pc, Instruction i, const char **name) { - int c = GETARG_C(i); /* key index */ - if (GETARG_k(i)) /* is 'c' a constant? */ - kname(p, c, name); - else /* 'c' is a register */ - rname(p, pc, c, name); -} - static int filterpc (int pc, int jmptarget) { if (pc < jmptarget) /* is code conditional (inside a jump)? */ @@ -509,28 +477,29 @@ static int findsetreg (const Proto *p, int lastpc, int reg) { /* -** Check whether table being indexed by instruction 'i' is the -** environment '_ENV' +** Find a "name" for the constant 'c'. */ -static const char *gxf (const Proto *p, int pc, Instruction i, int isup) { - int t = GETARG_B(i); /* table index */ - const char *name; /* name of indexed variable */ - if (isup) /* is an upvalue? */ - name = upvalname(p, t); - else - getobjname(p, pc, t, &name); - return (name && strcmp(name, LUA_ENV) == 0) ? "global" : "field"; +static const char *kname (const Proto *p, int index, const char **name) { + TValue *kvalue = &p->k[index]; + if (ttisstring(kvalue)) { + *name = getstr(tsvalue(kvalue)); + return "constant"; + } + else { + *name = "?"; + return NULL; + } } -static const char *getobjname (const Proto *p, int lastpc, int reg, - const char **name) { - int pc; - *name = luaF_getlocalname(p, reg + 1, lastpc); +static const char *basicgetobjname (const Proto *p, int *ppc, int reg, + const char **name) { + int pc = *ppc; + *name = luaF_getlocalname(p, reg + 1, pc); if (*name) /* is a local? */ return "local"; /* else try symbolic execution */ - pc = findsetreg(p, lastpc, reg); + *ppc = pc = findsetreg(p, pc, reg); if (pc != -1) { /* could find instruction? */ Instruction i = p->code[pc]; OpCode op = GET_OPCODE(i); @@ -538,18 +507,80 @@ static const char *getobjname (const Proto *p, int lastpc, int reg, case OP_MOVE: { int b = GETARG_B(i); /* move from 'b' to 'a' */ if (b < GETARG_A(i)) - return getobjname(p, pc, b, name); /* get name for 'b' */ + return basicgetobjname(p, ppc, b, name); /* get name for 'b' */ break; } + case OP_GETUPVAL: { + *name = upvalname(p, GETARG_B(i)); + return "upvalue"; + } + case OP_LOADK: return kname(p, GETARG_Bx(i), name); + case OP_LOADKX: return kname(p, GETARG_Ax(p->code[pc + 1]), name); + default: break; + } + } + return NULL; /* could not find reasonable name */ +} + + +/* +** Find a "name" for the register 'c'. +*/ +static void rname (const Proto *p, int pc, int c, const char **name) { + const char *what = basicgetobjname(p, &pc, c, name); /* search for 'c' */ + if (!(what && *what == 'c')) /* did not find a constant name? */ + *name = "?"; +} + + +/* +** Find a "name" for a 'C' value in an RK instruction. +*/ +static void rkname (const Proto *p, int pc, Instruction i, const char **name) { + int c = GETARG_C(i); /* key index */ + if (GETARG_k(i)) /* is 'c' a constant? */ + kname(p, c, name); + else /* 'c' is a register */ + rname(p, pc, c, name); +} + + +/* +** Check whether table being indexed by instruction 'i' is the +** environment '_ENV' +*/ +static const char *isEnv (const Proto *p, int pc, Instruction i, int isup) { + int t = GETARG_B(i); /* table index */ + const char *name; /* name of indexed variable */ + if (isup) /* is 't' an upvalue? */ + name = upvalname(p, t); + else /* 't' is a register */ + basicgetobjname(p, &pc, t, &name); + return (name && strcmp(name, LUA_ENV) == 0) ? "global" : "field"; +} + + +/* +** Extend 'basicgetobjname' to handle table accesses +*/ +static const char *getobjname (const Proto *p, int lastpc, int reg, + const char **name) { + const char *kind = basicgetobjname(p, &lastpc, reg, name); + if (kind != NULL) + return kind; + else if (lastpc != -1) { /* could find instruction? */ + Instruction i = p->code[lastpc]; + OpCode op = GET_OPCODE(i); + switch (op) { case OP_GETTABUP: { int k = GETARG_C(i); /* key index */ kname(p, k, name); - return gxf(p, pc, i, 1); + return isEnv(p, lastpc, i, 1); } case OP_GETTABLE: { int k = GETARG_C(i); /* key index */ - rname(p, pc, k, name); - return gxf(p, pc, i, 0); + rname(p, lastpc, k, name); + return isEnv(p, lastpc, i, 0); } case OP_GETI: { *name = "integer index"; @@ -558,24 +589,10 @@ static const char *getobjname (const Proto *p, int lastpc, int reg, case OP_GETFIELD: { int k = GETARG_C(i); /* key index */ kname(p, k, name); - return gxf(p, pc, i, 0); - } - case OP_GETUPVAL: { - *name = upvalname(p, GETARG_B(i)); - return "upvalue"; - } - case OP_LOADK: - case OP_LOADKX: { - int b = (op == OP_LOADK) ? GETARG_Bx(i) - : GETARG_Ax(p->code[pc + 1]); - if (ttisstring(&p->k[b])) { - *name = svalue(&p->k[b]); - return "constant"; - } - break; + return isEnv(p, lastpc, i, 0); } case OP_SELF: { - rkname(p, pc, i, name); + rkname(p, lastpc, i, name); return "method"; } default: break; /* go through to return NULL */ @@ -627,7 +644,7 @@ static const char *funcnamefromcode (lua_State *L, const Proto *p, default: return NULL; /* cannot find a reasonable name */ } - *name = getstr(G(L)->tmname[tm]) + 2; + *name = getshrstr(G(L)->tmname[tm]) + 2; return "metamethod"; } @@ -865,6 +882,28 @@ static int changedline (const Proto *p, int oldpc, int newpc) { } +/* +** Traces Lua calls. If code is running the first instruction of a function, +** and function is not vararg, and it is not coming from an yield, +** calls 'luaD_hookcall'. (Vararg functions will call 'luaD_hookcall' +** after adjusting its variable arguments; otherwise, they could call +** a line/count hook before the call hook. Functions coming from +** an yield already called 'luaD_hookcall' before yielding.) +*/ +int luaG_tracecall (lua_State *L) { + CallInfo *ci = L->ci; + Proto *p = ci_func(ci)->p; + ci->u.l.trap = 1; /* ensure hooks will be checked */ + if (ci->u.l.savedpc == p->code) { /* first instruction (not resuming)? */ + if (p->is_vararg) + return 0; /* hooks will start at VARARGPREP instruction */ + else if (!(ci->callstatus & CIST_HOOKYIELD)) /* not yieded? */ + luaD_hookcall(L, ci); /* check 'call' hook */ + } + return 1; /* keep 'trap' on */ +} + + /* ** Traces the execution of a Lua function. Called before the execution ** of each opcode, when debug is on. 'L->oldpc' stores the last @@ -888,12 +927,12 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { } pc++; /* reference is always next instruction */ ci->u.l.savedpc = pc; /* save 'pc' */ - counthook = (--L->hookcount == 0 && (mask & LUA_MASKCOUNT)); + counthook = (mask & LUA_MASKCOUNT) && (--L->hookcount == 0); if (counthook) resethookcount(L); /* reset count */ else if (!(mask & LUA_MASKLINE)) return 1; /* no line hook and count != 0; nothing to be done now */ - if (ci->callstatus & CIST_HOOKYIELD) { /* called hook last time? */ + if (ci->callstatus & CIST_HOOKYIELD) { /* hook yielded last time? */ ci->callstatus &= ~CIST_HOOKYIELD; /* erase mark */ return 1; /* do not call hook again (VM yielded, so it did not move) */ } @@ -915,7 +954,6 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { if (L->status == LUA_YIELD) { /* did hook yield? */ if (counthook) L->hookcount = 1; /* undo decrement to zero */ - ci->u.l.savedpc--; /* undo increment (resume will increment it again) */ ci->callstatus |= CIST_HOOKYIELD; /* mark that it yielded */ luaD_throw(L, LUA_YIELD); } diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h index 2c3074c6..2bfce3cb 100644 --- a/src/lua/ldebug.h +++ b/src/lua/ldebug.h @@ -58,6 +58,7 @@ LUAI_FUNC const char *luaG_addinfo (lua_State *L, const char *msg, TString *src, int line); LUAI_FUNC l_noret luaG_errormsg (lua_State *L); LUAI_FUNC int luaG_traceexec (lua_State *L, const Instruction *pc); +LUAI_FUNC int luaG_tracecall (lua_State *L); #endif diff --git a/src/lua/ldo.c b/src/lua/ldo.c index 2a0017ca..ea052950 100644 --- a/src/lua/ldo.c +++ b/src/lua/ldo.c @@ -409,7 +409,7 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { ** stack, below original 'func', so that 'luaD_precall' can call it. Raise ** an error if there is no '__call' metafield. */ -StkId luaD_tryfuncTM (lua_State *L, StkId func) { +static StkId tryfuncTM (lua_State *L, StkId func) { const TValue *tm; StkId p; checkstackGCp(L, 1, func); /* space for metamethod */ @@ -568,7 +568,7 @@ int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, return -1; } default: { /* not a function */ - func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + func = tryfuncTM(L, func); /* try to get '__call' metamethod */ /* return luaD_pretailcall(L, ci, func, narg1 + 1, delta); */ narg1++; goto retry; /* try again */ @@ -609,7 +609,7 @@ CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { return ci; } default: { /* not a function */ - func = luaD_tryfuncTM(L, func); /* try to get '__call' metamethod */ + func = tryfuncTM(L, func); /* try to get '__call' metamethod */ /* return luaD_precall(L, func, nresults); */ goto retry; /* try again with metamethod */ } @@ -792,6 +792,10 @@ static void resume (lua_State *L, void *ud) { lua_assert(L->status == LUA_YIELD); L->status = LUA_OK; /* mark that it is running (again) */ if (isLua(ci)) { /* yielded inside a hook? */ + /* undo increment made by 'luaG_traceexec': instruction was not + executed yet */ + lua_assert(ci->callstatus & CIST_HOOKYIELD); + ci->u.l.savedpc--; L->top.p = firstArg; /* discard arguments */ luaV_execute(L, ci); /* just continue running Lua code */ } diff --git a/src/lua/ldo.h b/src/lua/ldo.h index 1aa446ad..56008ab3 100644 --- a/src/lua/ldo.h +++ b/src/lua/ldo.h @@ -71,7 +71,6 @@ LUAI_FUNC int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, LUAI_FUNC CallInfo *luaD_precall (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_callnoyield (lua_State *L, StkId func, int nResults); -LUAI_FUNC StkId luaD_tryfuncTM (lua_State *L, StkId func); LUAI_FUNC int luaD_closeprotected (lua_State *L, ptrdiff_t level, int status); LUAI_FUNC int luaD_pcall (lua_State *L, Pfunc func, void *u, ptrdiff_t oldtop, ptrdiff_t ef); diff --git a/src/lua/lgc.c b/src/lua/lgc.c index a3094ff5..5817f9ee 100644 --- a/src/lua/lgc.c +++ b/src/lua/lgc.c @@ -542,10 +542,12 @@ static void traversestrongtable (global_State *g, Table *h) { static lu_mem traversetable (global_State *g, Table *h) { const char *weakkey, *weakvalue; const TValue *mode = gfasttm(g, h->metatable, TM_MODE); + TString *smode; markobjectN(g, h->metatable); - if (mode && ttisstring(mode) && /* is there a weak mode? */ - (cast_void(weakkey = strchr(svalue(mode), 'k')), - cast_void(weakvalue = strchr(svalue(mode), 'v')), + if (mode && ttisshrstring(mode) && /* is there a weak mode? */ + (cast_void(smode = tsvalue(mode)), + cast_void(weakkey = strchr(getshrstr(smode), 'k')), + cast_void(weakvalue = strchr(getshrstr(smode), 'v')), (weakkey || weakvalue))) { /* is really weak? */ if (!weakkey) /* strong keys? */ traverseweakvalue(g, h); @@ -638,7 +640,9 @@ static int traversethread (global_State *g, lua_State *th) { for (uv = th->openupval; uv != NULL; uv = uv->u.open.next) markobject(g, uv); /* open upvalues cannot be collected */ if (g->gcstate == GCSatomic) { /* final traversal? */ - for (; o < th->stack_last.p + EXTRA_STACK; o++) + if (!g->gcemergency) + luaD_shrinkstack(th); /* do not change stack in emergency cycle */ + for (o = th->top.p; o < th->stack_last.p + EXTRA_STACK; o++) setnilvalue(s2v(o)); /* clear dead stack slice */ /* 'remarkupvals' may have removed thread from 'twups' list */ if (!isintwups(th) && th->openupval != NULL) { @@ -646,8 +650,6 @@ static int traversethread (global_State *g, lua_State *th) { g->twups = th; } } - else if (!g->gcemergency) - luaD_shrinkstack(th); /* do not change stack in emergency cycle */ return 1 + stacksize(th); } @@ -1409,7 +1411,7 @@ static void stepgenfull (lua_State *L, global_State *g) { setminordebt(g); } else { /* another bad collection; stay in incremental mode */ - g->GCestimate = gettotalbytes(g); /* first estimate */; + g->GCestimate = gettotalbytes(g); /* first estimate */ entersweep(L); luaC_runtilstate(L, bitmask(GCSpause)); /* finish collection */ setpause(g); @@ -1604,7 +1606,7 @@ static lu_mem singlestep (lua_State *L) { case GCSenteratomic: { work = atomic(L); /* work is what was traversed by 'atomic' */ entersweep(L); - g->GCestimate = gettotalbytes(g); /* first estimate */; + g->GCestimate = gettotalbytes(g); /* first estimate */ break; } case GCSswpallgc: { /* sweep "regular" objects */ @@ -1710,6 +1712,8 @@ static void fullinc (lua_State *L, global_State *g) { entersweep(L); /* sweep everything to turn them back to white */ /* finish any pending sweep phase to start a new cycle */ luaC_runtilstate(L, bitmask(GCSpause)); + luaC_runtilstate(L, bitmask(GCSpropagate)); /* start new cycle */ + g->gcstate = GCSenteratomic; /* go straight to atomic phase */ luaC_runtilstate(L, bitmask(GCScallfin)); /* run up to finalizers */ /* estimate must be correct after a full GC cycle */ lua_assert(g->GCestimate == gettotalbytes(g)); diff --git a/src/lua/linit.c b/src/lua/linit.c index 9a5bcfdc..787865c0 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -8,6 +8,10 @@ #define linit_c #define LUA_LIB +/** Pi-hole modification **/ +#include "ftl_lua.h" +/**************************/ + /* ** If you embed Lua in your program and need to open the standard ** libraries, call luaL_openlibs in your program. If you need a @@ -64,5 +68,7 @@ LUALIB_API void luaL_openlibs (lua_State *L) { luaL_requiref(L, lib->name, lib->func, 1); lua_pop(L, 1); /* remove lib */ } + // Load and enable libraries bundled with Pi-hole + ftl_lua_init(L); } diff --git a/src/lua/liolib.c b/src/lua/liolib.c index b08397da..c5075f3e 100644 --- a/src/lua/liolib.c +++ b/src/lua/liolib.c @@ -245,8 +245,8 @@ static int f_gc (lua_State *L) { */ static int io_fclose (lua_State *L) { LStream *p = tolstream(L); - int res = fclose(p->f); - return luaL_fileresult(L, (res == 0), NULL); + errno = 0; + return luaL_fileresult(L, (fclose(p->f) == 0), NULL); } @@ -272,6 +272,7 @@ static int io_open (lua_State *L) { LStream *p = newfile(L); const char *md = mode; /* to traverse/check mode */ luaL_argcheck(L, l_checkmode(md), 2, "invalid mode"); + errno = 0; p->f = fopen(filename, mode); return (p->f == NULL) ? luaL_fileresult(L, 0, filename) : 1; } @@ -292,6 +293,7 @@ static int io_popen (lua_State *L) { const char *mode = luaL_optstring(L, 2, "r"); LStream *p = newprefile(L); luaL_argcheck(L, l_checkmodep(mode), 2, "invalid mode"); + errno = 0; p->f = l_popen(L, filename, mode); p->closef = &io_pclose; return (p->f == NULL) ? luaL_fileresult(L, 0, filename) : 1; @@ -300,6 +302,7 @@ static int io_popen (lua_State *L) { static int io_tmpfile (lua_State *L) { LStream *p = newfile(L); + errno = 0; p->f = tmpfile(); return (p->f == NULL) ? luaL_fileresult(L, 0, NULL) : 1; } @@ -567,6 +570,7 @@ static int g_read (lua_State *L, FILE *f, int first) { int nargs = lua_gettop(L) - 1; int n, success; clearerr(f); + errno = 0; if (nargs == 0) { /* no arguments? */ success = read_line(L, f, 1); n = first + 1; /* to return 1 result */ @@ -660,6 +664,7 @@ static int io_readline (lua_State *L) { static int g_write (lua_State *L, FILE *f, int arg) { int nargs = lua_gettop(L) - arg; int status = 1; + errno = 0; for (; nargs--; arg++) { if (lua_type(L, arg) == LUA_TNUMBER) { /* optimization: could be done exactly as for strings */ @@ -678,7 +683,8 @@ static int g_write (lua_State *L, FILE *f, int arg) { } if (l_likely(status)) return 1; /* file handle already on stack top */ - else return luaL_fileresult(L, status, NULL); + else + return luaL_fileresult(L, status, NULL); } @@ -703,6 +709,7 @@ static int f_seek (lua_State *L) { l_seeknum offset = (l_seeknum)p3; luaL_argcheck(L, (lua_Integer)offset == p3, 3, "not an integer in proper range"); + errno = 0; op = l_fseek(f, offset, mode[op]); if (l_unlikely(op)) return luaL_fileresult(L, 0, NULL); /* error */ @@ -719,19 +726,25 @@ static int f_setvbuf (lua_State *L) { FILE *f = tofile(L); int op = luaL_checkoption(L, 2, NULL, modenames); lua_Integer sz = luaL_optinteger(L, 3, LUAL_BUFFERSIZE); - int res = setvbuf(f, NULL, mode[op], (size_t)sz); + int res; + errno = 0; + res = setvbuf(f, NULL, mode[op], (size_t)sz); return luaL_fileresult(L, res == 0, NULL); } static int io_flush (lua_State *L) { - return luaL_fileresult(L, fflush(getiofile(L, IO_OUTPUT)) == 0, NULL); + FILE *f = getiofile(L, IO_OUTPUT); + errno = 0; + return luaL_fileresult(L, fflush(f) == 0, NULL); } static int f_flush (lua_State *L) { - return luaL_fileresult(L, fflush(tofile(L)) == 0, NULL); + FILE *f = tofile(L); + errno = 0; + return luaL_fileresult(L, fflush(f) == 0, NULL); } @@ -773,7 +786,7 @@ static const luaL_Reg meth[] = { ** metamethods for file handles */ static const luaL_Reg metameth[] = { - {"__index", NULL}, /* place holder */ + {"__index", NULL}, /* placeholder */ {"__gc", f_gc}, {"__close", f_gc}, {"__tostring", f_tostring}, diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c index d0b1e1e5..43810634 100644 --- a/src/lua/lmathlib.c +++ b/src/lua/lmathlib.c @@ -249,6 +249,15 @@ static int math_type (lua_State *L) { ** =================================================================== */ +/* +** This code uses lots of shifts. ANSI C does not allow shifts greater +** than or equal to the width of the type being shifted, so some shifts +** are written in convoluted ways to match that restriction. For +** preprocessor tests, it assumes a width of 32 bits, so the maximum +** shift there is 31 bits. +*/ + + /* number of binary digits in the mantissa of a float */ #define FIGS l_floatatt(MANT_DIG) @@ -271,16 +280,19 @@ static int math_type (lua_State *L) { /* 'long' has at least 64 bits */ #define Rand64 unsigned long +#define SRand64 long #elif !defined(LUA_USE_C89) && defined(LLONG_MAX) /* there is a 'long long' type (which must have at least 64 bits) */ #define Rand64 unsigned long long +#define SRand64 long long #elif ((LUA_MAXUNSIGNED >> 31) >> 31) >= 3 /* 'lua_Unsigned' has at least 64 bits */ #define Rand64 lua_Unsigned +#define SRand64 lua_Integer #endif @@ -319,23 +331,30 @@ static Rand64 nextrand (Rand64 *state) { } -/* must take care to not shift stuff by more than 63 slots */ - - /* ** Convert bits from a random integer into a float in the ** interval [0,1), getting the higher FIG bits from the ** random unsigned integer and converting that to a float. +** Some old Microsoft compilers cannot cast an unsigned long +** to a floating-point number, so we use a signed long as an +** intermediary. When lua_Number is float or double, the shift ensures +** that 'sx' is non negative; in that case, a good compiler will remove +** the correction. */ /* must throw out the extra (64 - FIGS) bits */ #define shift64_FIG (64 - FIGS) -/* to scale to [0, 1), multiply by scaleFIG = 2^(-FIGS) */ +/* 2^(-FIGS) == 2^-1 / 2^(FIGS-1) */ #define scaleFIG (l_mathop(0.5) / ((Rand64)1 << (FIGS - 1))) static lua_Number I2d (Rand64 x) { - return (lua_Number)(trim64(x) >> shift64_FIG) * scaleFIG; + SRand64 sx = (SRand64)(trim64(x) >> shift64_FIG); + lua_Number res = (lua_Number)(sx) * scaleFIG; + if (sx < 0) + res += l_mathop(1.0); /* correct the two's complement if negative */ + lua_assert(0 <= res && res < 1); + return res; } /* convert a 'Rand64' to a 'lua_Unsigned' */ @@ -471,8 +490,6 @@ static lua_Number I2d (Rand64 x) { #else /* 32 < FIGS <= 64 */ -/* must take care to not shift stuff by more than 31 slots */ - /* 2^(-FIGS) = 1.0 / 2^30 / 2^3 / 2^(FIGS-33) */ #define scaleFIG \ (l_mathop(1.0) / (UONE << 30) / l_mathop(8.0) / (UONE << (FIGS - 33))) diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c index d792dffa..6d289fce 100644 --- a/src/lua/loadlib.c +++ b/src/lua/loadlib.c @@ -24,15 +24,6 @@ #include "lualib.h" -/* -** LUA_IGMARK is a mark to ignore all before it when building the -** luaopen_ function name. -*/ -#if !defined (LUA_IGMARK) -#define LUA_IGMARK "-" -#endif - - /* ** LUA_CSUBSEP is the character that replaces dots in submodule names ** when searching for a C loader. diff --git a/src/lua/lobject.c b/src/lua/lobject.c index f73ffc6d..9cfa5227 100644 --- a/src/lua/lobject.c +++ b/src/lua/lobject.c @@ -542,7 +542,7 @@ const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) { addstr2buff(&buff, fmt, strlen(fmt)); /* rest of 'fmt' */ clearbuff(&buff); /* empty buffer into the stack */ lua_assert(buff.pushed == 1); - return svalue(s2v(L->top.p - 1)); + return getstr(tsvalue(s2v(L->top.p - 1))); } diff --git a/src/lua/lobject.h b/src/lua/lobject.h index 556608e4..980e42f8 100644 --- a/src/lua/lobject.h +++ b/src/lua/lobject.h @@ -386,7 +386,7 @@ typedef struct GCObject { typedef struct TString { CommonHeader; lu_byte extra; /* reserved words for short strings; "has hash" for longs */ - lu_byte shrlen; /* length for short strings */ + lu_byte shrlen; /* length for short strings, 0xFF for long strings */ unsigned int hash; union { size_t lnglen; /* length for long strings */ @@ -398,19 +398,17 @@ typedef struct TString { /* -** Get the actual string (array of bytes) from a 'TString'. +** Get the actual string (array of bytes) from a 'TString'. (Generic +** version and specialized versions for long and short strings.) */ -#define getstr(ts) ((ts)->contents) +#define getstr(ts) ((ts)->contents) +#define getlngstr(ts) check_exp((ts)->shrlen == 0xFF, (ts)->contents) +#define getshrstr(ts) check_exp((ts)->shrlen != 0xFF, (ts)->contents) -/* get the actual string (array of bytes) from a Lua value */ -#define svalue(o) getstr(tsvalue(o)) - /* get string length from 'TString *s' */ -#define tsslen(s) ((s)->tt == LUA_VSHRSTR ? (s)->shrlen : (s)->u.lnglen) - -/* get string length from 'TValue *o' */ -#define vslen(o) tsslen(tsvalue(o)) +#define tsslen(s) \ + ((s)->shrlen != 0xFF ? (s)->shrlen : (s)->u.lnglen) /* }================================================================== */ diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h index 4c551453..46911cac 100644 --- a/src/lua/lopcodes.h +++ b/src/lua/lopcodes.h @@ -210,15 +210,15 @@ OP_LOADNIL,/* A B R[A], R[A+1], ..., R[A+B] := nil */ OP_GETUPVAL,/* A B R[A] := UpValue[B] */ OP_SETUPVAL,/* A B UpValue[B] := R[A] */ -OP_GETTABUP,/* A B C R[A] := UpValue[B][K[C]:string] */ +OP_GETTABUP,/* A B C R[A] := UpValue[B][K[C]:shortstring] */ OP_GETTABLE,/* A B C R[A] := R[B][R[C]] */ OP_GETI,/* A B C R[A] := R[B][C] */ -OP_GETFIELD,/* A B C R[A] := R[B][K[C]:string] */ +OP_GETFIELD,/* A B C R[A] := R[B][K[C]:shortstring] */ -OP_SETTABUP,/* A B C UpValue[A][K[B]:string] := RK(C) */ +OP_SETTABUP,/* A B C UpValue[A][K[B]:shortstring] := RK(C) */ OP_SETTABLE,/* A B C R[A][R[B]] := RK(C) */ OP_SETI,/* A B C R[A][B] := RK(C) */ -OP_SETFIELD,/* A B C R[A][K[B]:string] := RK(C) */ +OP_SETFIELD,/* A B C R[A][K[B]:shortstring] := RK(C) */ OP_NEWTABLE,/* A B C k R[A] := {} */ diff --git a/src/lua/loslib.c b/src/lua/loslib.c index ad5a9276..ba80d72c 100644 --- a/src/lua/loslib.c +++ b/src/lua/loslib.c @@ -155,6 +155,7 @@ static int os_execute (lua_State *L) { static int os_remove (lua_State *L) { const char *filename = luaL_checkstring(L, 1); + errno = 0; return luaL_fileresult(L, remove(filename) == 0, filename); } @@ -162,6 +163,7 @@ static int os_remove (lua_State *L) { static int os_rename (lua_State *L) { const char *fromname = luaL_checkstring(L, 1); const char *toname = luaL_checkstring(L, 2); + errno = 0; return luaL_fileresult(L, rename(fromname, toname) == 0, NULL); } diff --git a/src/lua/lparser.c b/src/lua/lparser.c index b745f236..2b888c7c 100644 --- a/src/lua/lparser.c +++ b/src/lua/lparser.c @@ -1022,10 +1022,11 @@ static int explist (LexState *ls, expdesc *v) { } -static void funcargs (LexState *ls, expdesc *f, int line) { +static void funcargs (LexState *ls, expdesc *f) { FuncState *fs = ls->fs; expdesc args; int base, nparams; + int line = ls->linenumber; switch (ls->t.token) { case '(': { /* funcargs -> '(' [ explist ] ')' */ luaX_next(ls); @@ -1063,8 +1064,8 @@ static void funcargs (LexState *ls, expdesc *f, int line) { } init_exp(f, VCALL, luaK_codeABC(fs, OP_CALL, base, nparams+1, 2)); luaK_fixline(fs, line); - fs->freereg = base+1; /* call remove function and arguments and leaves - (unless changed) one result */ + fs->freereg = base+1; /* call removes function and arguments and leaves + one result (unless changed later) */ } @@ -1103,7 +1104,6 @@ static void suffixedexp (LexState *ls, expdesc *v) { /* suffixedexp -> primaryexp { '.' NAME | '[' exp ']' | ':' NAME funcargs | funcargs } */ FuncState *fs = ls->fs; - int line = ls->linenumber; primaryexp(ls, v); for (;;) { switch (ls->t.token) { @@ -1123,12 +1123,12 @@ static void suffixedexp (LexState *ls, expdesc *v) { luaX_next(ls); codename(ls, &key); luaK_self(fs, v, &key); - funcargs(ls, v, line); + funcargs(ls, v); break; } case '(': case TK_STRING: case '{': { /* funcargs */ luaK_exp2nextreg(fs, v); - funcargs(ls, v, line); + funcargs(ls, v); break; } default: return; diff --git a/src/lua/lstate.c b/src/lua/lstate.c index 1e925e5a..7fefacba 100644 --- a/src/lua/lstate.c +++ b/src/lua/lstate.c @@ -119,7 +119,7 @@ CallInfo *luaE_extendCI (lua_State *L) { /* ** free all CallInfo structures not in use by a thread */ -void luaE_freeCI (lua_State *L) { +static void freeCI (lua_State *L) { CallInfo *ci = L->ci; CallInfo *next = ci->next; ci->next = NULL; @@ -204,7 +204,7 @@ static void freestack (lua_State *L) { if (L->stack.p == NULL) return; /* stack not completely built yet */ L->ci = &L->base_ci; /* free the entire 'ci' list */ - luaE_freeCI(L); + freeCI(L); lua_assert(L->nci == 0); luaM_freearray(L, L->stack.p, stacksize(L) + EXTRA_STACK); /* free stack */ } @@ -433,7 +433,7 @@ void luaE_warning (lua_State *L, const char *msg, int tocont) { void luaE_warnerror (lua_State *L, const char *where) { TValue *errobj = s2v(L->top.p - 1); /* error object */ const char *msg = (ttisstring(errobj)) - ? svalue(errobj) + ? getstr(tsvalue(errobj)) : "error object is not a string"; /* produce warning "error in %s (%s)" (where, msg) */ luaE_warning(L, "error in ", 1); diff --git a/src/lua/lstate.h b/src/lua/lstate.h index 8bf6600e..007704c8 100644 --- a/src/lua/lstate.h +++ b/src/lua/lstate.h @@ -181,7 +181,7 @@ struct CallInfo { union { struct { /* only for Lua functions */ const Instruction *savedpc; - volatile l_signalT trap; + volatile l_signalT trap; /* function is tracing lines/counts */ int nextraargs; /* # of extra arguments in vararg functions */ } l; struct { /* only for C functions */ @@ -396,7 +396,6 @@ union GCUnion { LUAI_FUNC void luaE_setdebt (global_State *g, l_mem debt); LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1); LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L); -LUAI_FUNC void luaE_freeCI (lua_State *L); LUAI_FUNC void luaE_shrinkCI (lua_State *L); LUAI_FUNC void luaE_checkcstack (lua_State *L); LUAI_FUNC void luaE_incCstack (lua_State *L); diff --git a/src/lua/lstring.c b/src/lua/lstring.c index 13dcaf42..97757355 100644 --- a/src/lua/lstring.c +++ b/src/lua/lstring.c @@ -36,7 +36,7 @@ int luaS_eqlngstr (TString *a, TString *b) { lua_assert(a->tt == LUA_VLNGSTR && b->tt == LUA_VLNGSTR); return (a == b) || /* same instance or... */ ((len == b->u.lnglen) && /* equal length and ... */ - (memcmp(getstr(a), getstr(b), len) == 0)); /* equal contents */ + (memcmp(getlngstr(a), getlngstr(b), len) == 0)); /* equal contents */ } @@ -52,7 +52,7 @@ unsigned int luaS_hashlongstr (TString *ts) { lua_assert(ts->tt == LUA_VLNGSTR); if (ts->extra == 0) { /* no hash? */ size_t len = ts->u.lnglen; - ts->hash = luaS_hash(getstr(ts), len, ts->hash); + ts->hash = luaS_hash(getlngstr(ts), len, ts->hash); ts->extra = 1; /* now it has its hash */ } return ts->hash; @@ -157,6 +157,7 @@ static TString *createstrobj (lua_State *L, size_t l, int tag, unsigned int h) { TString *luaS_createlngstrobj (lua_State *L, size_t l) { TString *ts = createstrobj(L, l, LUA_VLNGSTR, G(L)->seed); ts->u.lnglen = l; + ts->shrlen = 0xFF; /* signals that it is a long string */ return ts; } @@ -193,7 +194,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { TString **list = &tb->hash[lmod(h, tb->size)]; lua_assert(str != NULL); /* otherwise 'memcmp'/'memcpy' are undefined */ for (ts = *list; ts != NULL; ts = ts->u.hnext) { - if (l == ts->shrlen && (memcmp(str, getstr(ts), l * sizeof(char)) == 0)) { + if (l == ts->shrlen && (memcmp(str, getshrstr(ts), l * sizeof(char)) == 0)) { /* found! */ if (isdead(g, ts)) /* dead (but not collected yet)? */ changewhite(ts); /* resurrect it */ @@ -206,8 +207,8 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { list = &tb->hash[lmod(h, tb->size)]; /* rehash with new size */ } ts = createstrobj(L, l, LUA_VSHRSTR, h); - memcpy(getstr(ts), str, l * sizeof(char)); ts->shrlen = cast_byte(l); + memcpy(getshrstr(ts), str, l * sizeof(char)); ts->u.hnext = *list; *list = ts; tb->nuse++; @@ -223,10 +224,10 @@ TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { return internshrstr(L, str, l); else { TString *ts; - if (l_unlikely(l >= (MAX_SIZE - sizeof(TString))/sizeof(char))) + if (l_unlikely(l * sizeof(char) >= (MAX_SIZE - sizeof(TString)))) luaM_toobig(L); ts = luaS_createlngstrobj(L, l); - memcpy(getstr(ts), str, l * sizeof(char)); + memcpy(getlngstr(ts), str, l * sizeof(char)); return ts; } } diff --git a/src/lua/ltable.c b/src/lua/ltable.c index 3c690c5f..3353c047 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -252,7 +252,7 @@ LUAI_FUNC unsigned int luaH_realasize (const Table *t) { return t->alimit; /* this is the size */ else { unsigned int size = t->alimit; - /* compute the smallest power of 2 not smaller than 'n' */ + /* compute the smallest power of 2 not smaller than 'size' */ size |= (size >> 1); size |= (size >> 2); size |= (size >> 4); @@ -662,7 +662,8 @@ static Node *getfreepos (Table *t) { ** put new key in its main position; otherwise (colliding node is in its main ** position), new key goes to an empty position. */ -void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { +static void luaH_newkey (lua_State *L, Table *t, const TValue *key, + TValue *value) { Node *mp; TValue aux; if (l_unlikely(ttisnil(key))) @@ -721,22 +722,36 @@ void luaH_newkey (lua_State *L, Table *t, const TValue *key, TValue *value) { /* ** Search function for integers. If integer is inside 'alimit', get it -** directly from the array part. Otherwise, if 'alimit' is not equal to -** the real size of the array, key still can be in the array part. In -** this case, try to avoid a call to 'luaH_realasize' when key is just -** one more than the limit (so that it can be incremented without -** changing the real size of the array). +** directly from the array part. Otherwise, if 'alimit' is not +** the real size of the array, the key still can be in the array part. +** In this case, do the "Xmilia trick" to check whether 'key-1' is +** smaller than the real size. +** The trick works as follow: let 'p' be an integer such that +** '2^(p+1) >= alimit > 2^p', or '2^(p+1) > alimit-1 >= 2^p'. +** That is, 2^(p+1) is the real size of the array, and 'p' is the highest +** bit on in 'alimit-1'. What we have to check becomes 'key-1 < 2^(p+1)'. +** We compute '(key-1) & ~(alimit-1)', which we call 'res'; it will +** have the 'p' bit cleared. If the key is outside the array, that is, +** 'key-1 >= 2^(p+1)', then 'res' will have some bit on higher than 'p', +** therefore it will be larger or equal to 'alimit', and the check +** will fail. If 'key-1 < 2^(p+1)', then 'res' has no bit on higher than +** 'p', and as the bit 'p' itself was cleared, 'res' will be smaller +** than 2^p, therefore smaller than 'alimit', and the check succeeds. +** As special cases, when 'alimit' is 0 the condition is trivially false, +** and when 'alimit' is 1 the condition simplifies to 'key-1 < alimit'. +** If key is 0 or negative, 'res' will have its higher bit on, so that +** if cannot be smaller than alimit. */ const TValue *luaH_getint (Table *t, lua_Integer key) { - if (l_castS2U(key) - 1u < t->alimit) /* 'key' in [1, t->alimit]? */ + lua_Unsigned alimit = t->alimit; + if (l_castS2U(key) - 1u < alimit) /* 'key' in [1, t->alimit]? */ return &t->array[key - 1]; - else if (!limitequalsasize(t) && /* key still may be in the array part? */ - (l_castS2U(key) == t->alimit + 1 || - l_castS2U(key) - 1u < luaH_realasize(t))) { + else if (!isrealasize(t) && /* key still may be in the array part? */ + (((l_castS2U(key) - 1u) & ~(alimit - 1u)) < alimit)) { t->alimit = cast_uint(key); /* probably '#t' is here now */ return &t->array[key - 1]; } - else { + else { /* key is not in the array part; check the hash */ Node *n = hashint(t, key); for (;;) { /* check whether 'key' is somewhere in the chain */ if (keyisinteger(n) && keyival(n) == key) diff --git a/src/lua/ltable.h b/src/lua/ltable.h index 75dd9e26..8e689034 100644 --- a/src/lua/ltable.h +++ b/src/lua/ltable.h @@ -41,8 +41,6 @@ LUAI_FUNC void luaH_setint (lua_State *L, Table *t, lua_Integer key, LUAI_FUNC const TValue *luaH_getshortstr (Table *t, TString *key); LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key); LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key); -LUAI_FUNC void luaH_newkey (lua_State *L, Table *t, const TValue *key, - TValue *value); LUAI_FUNC void luaH_set (lua_State *L, Table *t, const TValue *key, TValue *value); LUAI_FUNC void luaH_finishset (lua_State *L, Table *t, const TValue *key, diff --git a/src/lua/ltm.h b/src/lua/ltm.h index c309e2ae..73b833c6 100644 --- a/src/lua/ltm.h +++ b/src/lua/ltm.h @@ -9,7 +9,6 @@ #include "lobject.h" -#include "lstate.h" /* @@ -96,8 +95,8 @@ LUAI_FUNC int luaT_callorderiTM (lua_State *L, const TValue *p1, int v2, int inv, int isfloat, TMS event); LUAI_FUNC void luaT_adjustvarargs (lua_State *L, int nfixparams, - CallInfo *ci, const Proto *p); -LUAI_FUNC void luaT_getvarargs (lua_State *L, CallInfo *ci, + struct CallInfo *ci, const Proto *p); +LUAI_FUNC void luaT_getvarargs (lua_State *L, struct CallInfo *ci, StkId where, int wanted); diff --git a/src/lua/lua.c b/src/lua/lua.c index f269c997..111a1b2b 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,10 +20,6 @@ #include "lauxlib.h" #include "lualib.h" -/** Pi-hole modification **/ -#include "ftl_lua.h" -/**************************/ - #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -119,12 +115,13 @@ static void l_message (const char *pname, const char *msg) { /* ** Check whether 'status' is not OK and, if so, prints the error -** message on the top of the stack. It assumes that the error object -** is a string, as it was either generated by Lua or by 'msghandler'. +** message on the top of the stack. */ static int report (lua_State *L, int status) { if (status != LUA_OK) { const char *msg = lua_tostring(L, -1); + if (msg == NULL) + msg = "(error message not a string)"; l_message(progname, msg); lua_pop(L, 1); /* remove message */ } @@ -214,14 +211,17 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. +** If there is no explicit modname and globname contains a '-', cut +** the suffix after '-' (the "version") to make the global name. */ -/************** Pi-hole modification ***************/ -int dolibrary (lua_State *L, char *globname) { -/***************************************************/ +static int dolibrary (lua_State *L, char *globname) { int status; + char *suffix = NULL; char *modname = strchr(globname, '='); - if (modname == NULL) /* no explicit name? */ + if (modname == NULL) { /* no explicit name? */ modname = globname; /* module name is equal to global name */ + suffix = strchr(modname, *LUA_IGMARK); /* look for a suffix mark */ + } else { *modname = '\0'; /* global name ends here */ modname++; /* module name starts after the '=' */ @@ -229,8 +229,11 @@ int dolibrary (lua_State *L, char *globname) { lua_getglobal(L, "require"); lua_pushstring(L, modname); status = docall(L, 1, 1); /* call 'require(modname)' */ - if (status == LUA_OK) + if (status == LUA_OK) { + if (suffix != NULL) /* is there a suffix mark? */ + *suffix = '\0'; /* remove suffix from global name */ lua_setglobal(L, globname); /* globname = require(modname) */ + } return report(L, status); } @@ -646,11 +649,6 @@ static int pmain (lua_State *L) { return 0; /* error running LUA_INIT */ } - /************** Pi-hole modification ***************/ - // Load and enable libraries bundled with Pi-hole - ftl_lua_init(L); - /***************************************************/ - if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ if (script > 0) { /* execute main script (if there is one) */ diff --git a/src/lua/lua.h b/src/lua/lua.h index fd16cf80..f050dac0 100644 --- a/src/lua/lua.h +++ b/src/lua/lua.h @@ -18,14 +18,14 @@ #define LUA_VERSION_MAJOR "5" #define LUA_VERSION_MINOR "4" -#define LUA_VERSION_RELEASE "6" +#define LUA_VERSION_RELEASE "7" #define LUA_VERSION_NUM 504 -#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 6) +#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 7) #define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR #define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE -#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2023 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2024 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo, W. Celes" @@ -497,7 +497,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2023 Lua.org, PUC-Rio. +* Copyright (C) 1994-2024 Lua.org, PUC-Rio. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h index 137103ed..dacc5221 100644 --- a/src/lua/luaconf.h +++ b/src/lua/luaconf.h @@ -257,6 +257,15 @@ #endif + +/* +** LUA_IGMARK is a mark to ignore all after it when building the +** module name (e.g., used to build the luaopen_ function name). +** Typically, the suffix after the mark is the module version, +** as in "mod-v1.2.so". +*/ +#define LUA_IGMARK "-" + /* }================================================================== */ @@ -756,7 +765,7 @@ ** of a function in debug information. ** CHANGE it if you want a different size. */ -#define LUA_IDSIZE 60 +#define LUA_IDSIZE 256 /* diff --git a/src/lua/lundump.c b/src/lua/lundump.c index 02aed64f..e8d92a85 100644 --- a/src/lua/lundump.c +++ b/src/lua/lundump.c @@ -81,7 +81,7 @@ static size_t loadUnsigned (LoadState *S, size_t limit) { static size_t loadSize (LoadState *S) { - return loadUnsigned(S, ~(size_t)0); + return loadUnsigned(S, MAX_SIZET); } @@ -122,7 +122,7 @@ static TString *loadStringN (LoadState *S, Proto *p) { ts = luaS_createlngstrobj(L, size); /* create string */ setsvalue2s(L, L->top.p, ts); /* anchor it ('loadVector' can GC) */ luaD_inctop(L); - loadVector(S, getstr(ts), size); /* load directly in final place */ + loadVector(S, getlngstr(ts), size); /* load directly in final place */ L->top.p--; /* pop string */ } luaC_objbarrier(L, p, ts); diff --git a/src/lua/lundump.h b/src/lua/lundump.h index f3748a99..a97676ca 100644 --- a/src/lua/lundump.h +++ b/src/lua/lundump.h @@ -21,8 +21,7 @@ /* ** Encode major-minor version in one byte, one nibble for each */ -#define MYINT(s) (s[0]-'0') /* assume one-digit numerals */ -#define LUAC_VERSION (MYINT(LUA_VERSION_MAJOR)*16+MYINT(LUA_VERSION_MINOR)) +#define LUAC_VERSION (((LUA_VERSION_NUM / 100) * 16) + LUA_VERSION_NUM % 100) #define LUAC_FORMAT 0 /* this is the official format */ diff --git a/src/lua/lvm.c b/src/lua/lvm.c index 8493a770..fcd24e11 100644 --- a/src/lua/lvm.c +++ b/src/lua/lvm.c @@ -91,8 +91,10 @@ static int l_strton (const TValue *obj, TValue *result) { lua_assert(obj != result); if (!cvt2num(obj)) /* is object not a string? */ return 0; - else - return (luaO_str2num(svalue(obj), result) == vslen(obj) + 1); + else { + TString *st = tsvalue(obj); + return (luaO_str2num(getstr(st), result) == tsslen(st) + 1); + } } @@ -366,30 +368,32 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key, /* -** Compare two strings 'ls' x 'rs', returning an integer less-equal- -** -greater than zero if 'ls' is less-equal-greater than 'rs'. +** Compare two strings 'ts1' x 'ts2', returning an integer less-equal- +** -greater than zero if 'ts1' is less-equal-greater than 'ts2'. ** The code is a little tricky because it allows '\0' in the strings -** and it uses 'strcoll' (to respect locales) for each segments -** of the strings. +** and it uses 'strcoll' (to respect locales) for each segment +** of the strings. Note that segments can compare equal but still +** have different lengths. */ -static int l_strcmp (const TString *ls, const TString *rs) { - const char *l = getstr(ls); - size_t ll = tsslen(ls); - const char *r = getstr(rs); - size_t lr = tsslen(rs); +static int l_strcmp (const TString *ts1, const TString *ts2) { + const char *s1 = getstr(ts1); + size_t rl1 = tsslen(ts1); /* real length */ + const char *s2 = getstr(ts2); + size_t rl2 = tsslen(ts2); for (;;) { /* for each segment */ - int temp = strcoll(l, r); + int temp = strcoll(s1, s2); if (temp != 0) /* not equal? */ return temp; /* done */ else { /* strings are equal up to a '\0' */ - size_t len = strlen(l); /* index of first '\0' in both strings */ - if (len == lr) /* 'rs' is finished? */ - return (len == ll) ? 0 : 1; /* check 'ls' */ - else if (len == ll) /* 'ls' is finished? */ - return -1; /* 'ls' is less than 'rs' ('rs' is not finished) */ - /* both strings longer than 'len'; go on comparing after the '\0' */ - len++; - l += len; ll -= len; r += len; lr -= len; + size_t zl1 = strlen(s1); /* index of first '\0' in 's1' */ + size_t zl2 = strlen(s2); /* index of first '\0' in 's2' */ + if (zl2 == rl2) /* 's2' is finished? */ + return (zl1 == rl1) ? 0 : 1; /* check 's1' */ + else if (zl1 == rl1) /* 's1' is finished? */ + return -1; /* 's1' is less than 's2' ('s2' is not finished) */ + /* both strings longer than 'zl'; go on comparing after the '\0' */ + zl1++; zl2++; + s1 += zl1; rl1 -= zl1; s2 += zl2; rl2 -= zl2; } } } @@ -624,8 +628,9 @@ int luaV_equalobj (lua_State *L, const TValue *t1, const TValue *t2) { static void copy2buff (StkId top, int n, char *buff) { size_t tl = 0; /* size already copied */ do { - size_t l = vslen(s2v(top - n)); /* length of string being copied */ - memcpy(buff + tl, svalue(s2v(top - n)), l * sizeof(char)); + TString *st = tsvalue(s2v(top - n)); + size_t l = tsslen(st); /* length of string being copied */ + memcpy(buff + tl, getstr(st), l * sizeof(char)); tl += l; } while (--n > 0); } @@ -651,12 +656,12 @@ void luaV_concat (lua_State *L, int total) { } else { /* at least two non-empty string values; get as many as possible */ - size_t tl = vslen(s2v(top - 1)); + size_t tl = tsslen(tsvalue(s2v(top - 1))); TString *ts; /* collect total length and number of strings */ for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) { - size_t l = vslen(s2v(top - n - 1)); - if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) { + size_t l = tsslen(tsvalue(s2v(top - n - 1))); + if (l_unlikely(l >= MAX_SIZE - sizeof(TString) - tl)) { L->top.p = top - total; /* pop strings to avoid wasting stack */ luaG_runerror(L, "string length overflow"); } @@ -669,7 +674,7 @@ void luaV_concat (lua_State *L, int total) { } else { /* long string; copy strings directly to final result */ ts = luaS_createlngstrobj(L, tl); - copy2buff(top, n, getstr(ts)); + copy2buff(top, n, getlngstr(ts)); } setsvalue2s(L, top - n, ts); /* create result */ } @@ -1155,18 +1160,11 @@ void luaV_execute (lua_State *L, CallInfo *ci) { startfunc: trap = L->hookmask; returning: /* trap already set */ - cl = clLvalue(s2v(ci->func.p)); + cl = ci_func(ci); k = cl->p->k; pc = ci->u.l.savedpc; - if (l_unlikely(trap)) { - if (pc == cl->p->code) { /* first instruction (not resuming)? */ - if (cl->p->is_vararg) - trap = 0; /* hooks will start after VARARGPREP instruction */ - else /* check 'call' hook */ - luaD_hookcall(L, ci); - } - ci->u.l.trap = 1; /* assume trap is on, for now */ - } + if (l_unlikely(trap)) + trap = luaG_tracecall(L); base = ci->func.p + 1; /* main loop of interpreter */ for (;;) { @@ -1253,7 +1251,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { const TValue *slot; TValue *upval = cl->upvals[GETARG_B(i)]->v.p; TValue *rc = KC(i); - TString *key = tsvalue(rc); /* key must be a string */ + TString *key = tsvalue(rc); /* key must be a short string */ if (luaV_fastget(L, upval, key, slot, luaH_getshortstr)) { setobj2s(L, ra, slot); } @@ -1296,7 +1294,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { const TValue *slot; TValue *rb = vRB(i); TValue *rc = KC(i); - TString *key = tsvalue(rc); /* key must be a string */ + TString *key = tsvalue(rc); /* key must be a short string */ if (luaV_fastget(L, rb, key, slot, luaH_getshortstr)) { setobj2s(L, ra, slot); } @@ -1309,7 +1307,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { TValue *upval = cl->upvals[GETARG_A(i)]->v.p; TValue *rb = KB(i); TValue *rc = RKC(i); - TString *key = tsvalue(rb); /* key must be a string */ + TString *key = tsvalue(rb); /* key must be a short string */ if (luaV_fastget(L, upval, key, slot, luaH_getshortstr)) { luaV_finishfastset(L, upval, slot, rc); } @@ -1352,7 +1350,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { const TValue *slot; TValue *rb = KB(i); TValue *rc = RKC(i); - TString *key = tsvalue(rb); /* key must be a string */ + TString *key = tsvalue(rb); /* key must be a short string */ if (luaV_fastget(L, s2v(ra), key, slot, luaH_getshortstr)) { luaV_finishfastset(L, s2v(ra), slot, rc); } diff --git a/src/lua/scripts/CMakeLists.txt b/src/lua/scripts/CMakeLists.txt index 9afd1afa..0f7aa19d 100644 --- a/src/lua/scripts/CMakeLists.txt +++ b/src/lua/scripts/CMakeLists.txt @@ -27,8 +27,8 @@ foreach(INPUT_FILE ${COMPILED_RESOURCES}) list(APPEND COMPILED_RESOURCES ${OUTPUT_FILE}) endforeach() -# Ensure target lua_scripts is build before target lua -add_dependencies(lua lua_scripts) +# Ensure target lua_scripts is build before target ftl_lua depending on it +add_dependencies(ftl_lua lua_scripts) add_library(lua_scripts OBJECT ${sources}) target_compile_options(lua_scripts PRIVATE ${EXTRAWARN}) diff --git a/src/main.c b/src/main.c index 3d18c74e..d6909495 100644 --- a/src/main.c +++ b/src/main.c @@ -27,17 +27,14 @@ #include "overTime.h" // export_queries_to_disk() #include "database/query-table.h" - -#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#pragma message "Minimum GLIBC version: " xstr(__GLIBC__) "." xstr(__GLIBC_MINOR__) -#else -#pragma message "Minimum GLIBC version: unknown, assuming this is a MUSL build" -#endif +// verify_FTL() +#include "files.h" char *username; bool needGC = false; bool needDBGC = false; bool startup = true; +bool forked = false; jmp_buf exit_jmp; int main (int argc, char *argv[]) @@ -77,6 +74,10 @@ int main (int argc, char *argv[]) if(readFTLconf(&config, true)) log_info("Parsed config file "GLOBALTOMLPATH" successfully"); + // Check if another FTL process is already running + if(another_FTL()) + return EXIT_FAILURE; + // Set process priority set_nice(); @@ -84,8 +85,6 @@ int main (int argc, char *argv[]) if(!init_shmem()) { log_crit("Initialization of shared memory failed."); - // Check if there is already a running FTL process - check_running_FTL(); return EXIT_FAILURE; } @@ -130,7 +129,7 @@ int main (int argc, char *argv[]) log_debug(DEBUG_ANY, "Jumped back to main() from dnsmasq/die()"); dnsmasq_failed = true; - if(!resolver_ready) + if(!forked) { // If dnsmasq never finished initializing, we need to // launch the threads @@ -138,11 +137,11 @@ int main (int argc, char *argv[]) } // Loop here to keep the webserver running unless requested to restart - while(!FTL_terminate) + while(!killed) sleepms(100); } - log_info("Shutting down... // exit code %d // jmpret %d", exit_code, jmpret); + log_debug(DEBUG_ANY, "Shutting down... // exit code %d // jmpret %d", exit_code, jmpret); // Extra grace time is needed as dnsmasq script-helpers and the API may not // be terminating immediately sleepms(250); diff --git a/src/main.h b/src/main.h index 7a40c894..7918c889 100644 --- a/src/main.h +++ b/src/main.h @@ -20,6 +20,7 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start); extern char *username; extern bool startup; +extern bool forked; extern jmp_buf exit_jmp; #endif //MAIN_H diff --git a/src/ntp/CMakeLists.txt b/src/ntp/CMakeLists.txt new file mode 100644 index 00000000..5cdc5d12 --- /dev/null +++ b/src/ntp/CMakeLists.txt @@ -0,0 +1,20 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2024 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# /src/ntp/CMakeList.txt +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +set(ntp_sources + server.c + client.c + rtc.c + ntp.h + ) + +add_library(ntp OBJECT ${ntp_sources}) +target_compile_options(ntp PRIVATE "${EXTRAWARN}") +target_include_directories(ntp PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/ntp/client.c b/src/ntp/client.c new file mode 100644 index 00000000..340358bf --- /dev/null +++ b/src/ntp/client.c @@ -0,0 +1,678 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* NTP client routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "ntp.h" +// close() +#include +// clock_gettime() +#include +// socket(), connect(), send(), recv(), AF_INET, SOCK_DGRAM, IPPROTO_UDP +#include +// getaddrinfo(), freeaddrinfo(), struct addrinfo +#include +// memcpy() +#include +// pow() +#include +// ctime() +#include +// errno +#include +// PRIi64 +#include +// config struct +#include "config/config.h" +// adjtime() +#include +// threads[] +#include "daemon.h" +// thread_names[] +#include "signals.h" +// adjtimex() +#include +// log_ntp_message() +#include "database/message-table.h" +// check_capability() +#include "capabilities.h" + +struct ntp_sync +{ + bool valid; + uint64_t org; + uint64_t xmt; + double theta; + double delta; + double precision; +}; + +// Create minimal NTP request, see server implementation for details about the +// packet structure +static bool request(int fd, const char *server, struct ntp_sync *ntp) +{ + // NTP Packet buffer + unsigned char buf[48] = {0}; + + // LI = 0, VN = 4 (current version), Mode = 3 (Client) + buf[0] = 0x23; + + // Minimum poll interval (2^6 = 64 seconds) + buf[2] = 0x06; + + // Set Reference Timestamp (ref) to 0 + // This is the time at which the local clock was last set or corrected. + memset(&buf[8], 0, sizeof(uint64_t)); + + // Set Origin Timestamp (org) in NTP format + ntp->org = gettime64(); + const uint64_t norg = hton64(ntp->org); + memcpy(&buf[40], &norg, sizeof(norg)); + + // Send request + if(send(fd, buf, 48, 0) != 48) + { + log_err("Failed to send data to NTP server %s: %s", + server, errno == EAGAIN ? "Timeout" : strerror(errno)); + return false; + } + + return true; +} + +// Display NTP time in human-readable format +// This function is similar to get_timestr() in src/log.c but differs in that it +// includes microseconds whereas get_timestr() only includes milliseconds +static void format_NTP_time(char time_str[TIMESTR_SIZE], const uint64_t ntp_time) +{ + struct timeval client_time; + client_time.tv_sec = NTPtoSEC(ntp_time); + client_time.tv_usec = NTPtoUSEC(ntp_time); + struct tm client_tm = {0}; + localtime_r(&client_time.tv_sec, &client_tm); + snprintf(time_str, TIMESTR_SIZE, "%04i-%02i-%02i %02i:%02i:%02i.%06li %s", + client_tm.tm_year + 1900, client_tm.tm_mon + 1, client_tm.tm_mday, + client_tm.tm_hour, client_tm.tm_min, client_tm.tm_sec, + (long int)client_time.tv_usec, client_tm.tm_zone); + time_str[TIMESTR_SIZE - 1] = '\0'; +} + +// Print NTP timestamp in human-readable form for debugging +void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t ntp_time) +{ + // Get the time from the appropriate buffer + uint64_t timevar; + if(u32p != NULL) + { + memcpy(&timevar, u32p, sizeof(uint64_t)); + // Convert to host byte order + timevar = ntoh64(timevar); + } + else + { + // Use the provided time (already in host byte order) + timevar = ntp_time; + } + + + // Format the time + char time_str[TIMESTR_SIZE]; + format_NTP_time(time_str, timevar); + + // Print the time + log_debug(DEBUG_NTP, "%s: %08"PRIx64".%08"PRIx64" = %s", label, + (timevar >> 32) & 0xFFFFFFFF, timevar & 0xFFFFFFFF, time_str); +} + +static uint64_t get_new_time(struct timeval *unix_time, const double offset) +{ + // Get current time + gettimeofday(unix_time, NULL); + + // Convert from double to native format (signed) and add to the + // current time. Note the addition is done in native format to + // avoid overflow or loss of precision. + const uint64_t ntp_time = U2LFP(*unix_time) + D2LFP(offset); + + // Convert NTP to native format + unix_time->tv_sec = NTPtoSEC(ntp_time); + unix_time->tv_usec = NTPtoUSEC(ntp_time); + + return ntp_time; +} + +static bool settime_step(struct timeval *unix_time, const double offset) +{ + log_debug(DEBUG_NTP, "Stepping system time by %e s", offset); + + // Set time immediately + if(settimeofday(unix_time, NULL) != 0) + { + char errbuf[1024]; + strncpy(errbuf, "Failed to set time during NTP sync: ", sizeof(errbuf)); + strncat(errbuf, errno == EPERM ? "Insufficient permissions" : strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); + return false; + } + + return true; +} + +static bool settime_skew(const double offset) +{ + // This function gradually adjusts the system clock. + // + // Linux uses David L. Mills' clock adjustment algorithm (see RFC 5905). + // If the adjustment in delta is positive, then the system clock is + // speeded up by some small percentage (i.e., by adding a small amount + // of time to the clock value in each second) until the adjustment has + // been completed. If the adjustment in delta is negative, then the + // clock is slowed down in a similar fashion. + // + // If a clock adjustment from an earlier adjtime() call is already in + // progress at the time of a later adjtime() call, and delta is not NULL + // for the later call, then the earlier adjustment is stopped, but any + // already completed part of that adjustment is not undone. + // + // The adjustment that adjtimex() makes to the clock is carried out in + // such a manner that the clock is always monotonically increasing. + // Using adjtimex() to adjust the time prevents the problems that can be + // caused for certain applications (e.g., make(1)) by abrupt positive or + // negative jumps in the system time. + // + // adjtimex() is intended to be used to make small adjustments to the + // system time. The actual time adjustment rate is implementation-specific + // but is typically on the order of 500 ppm, i.e., 0.5 ms/s. + // + // man rtc(4) adds: + // When the kernel's system time is synchronized with an external + // reference using adjtimex() it will update a designated RTC + // periodically every 11 minutes. + + struct timex tx = { 0 }; + tx.offset = 1000000 * offset; + tx.modes = ADJ_OFFSET_SINGLESHOT; + + log_debug(DEBUG_NTP, "Gradually adjusting system time by %ld us", tx.offset); + + if(adjtimex(&tx) < 0) + { + char errbuf[1024]; + strncpy(errbuf, "Failed to adjust time during NTP sync: ", sizeof(errbuf)); + strncat(errbuf, errno == EPERM ? "Insufficient permissions" : strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); + return false; + } + + return true; +} + +static bool reply(int fd, const char *server, struct ntp_sync *ntp, const bool verbose) +{ + // NTP Packet buffer + unsigned char buf[48]; + + // Receive reply + if(recv(fd, buf, 48, 0) < 48) + { + log_err("Failed to receive data from NTP server %s: %s", + server, errno == EAGAIN ? "Timeout" : strerror(errno)); + return false; + } + + // Extract precision of server clock + signed char rho = (signed char)buf[3]; + if(rho < -32 || rho > 0) + { + // Accepted limits are 2^-32 (~ 0.2 nanoseconds) + // to 2^0 (= 1 second) + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), "Received NTP reply has invalid precision: 2^(%i), assuming microsecond accuracy", rho); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(false, false, errbuf); + rho = -19; + } + // Compute precision of server clock in seconds 2^rho + ntp->precision = pow(2, rho); + + // Extract root delay and root dispersion of server clock + uint32_t srv_root_delay, srv_root_dispersion; + memcpy(&srv_root_delay, &buf[4], sizeof(srv_root_delay)); + memcpy(&srv_root_dispersion, &buf[8], sizeof(srv_root_dispersion)); + + // Extract Transmit Timestamp + uint64_t netbuffer; + // ref = Reference Timestamp (Time at which the clock was last set or corrected) + memcpy(&netbuffer, &buf[16], sizeof(netbuffer)); + const uint64_t ref = ntoh64(netbuffer); + // org = Origin Timestamp (Transmit Timestamp @ Client) + memcpy(&netbuffer, &buf[24], sizeof(netbuffer)); + const uint64_t org = ntoh64(netbuffer); + // rec = Receive Timestamp (Receive Timestamp @ Server) + memcpy(&netbuffer, &buf[32], sizeof(netbuffer)); + const uint64_t rec = ntoh64(netbuffer); + // xmt = Transmit Timestamp (Transmit Timestamp @ Server) + memcpy(&netbuffer, &buf[40], sizeof(netbuffer)); + ntp->xmt = ntoh64(netbuffer); + + // dst = Destination Timestamp (Receive Timestamp @ Client) + uint64_t dst = gettime64(); + + // Check org_ and org are identical (otherwise, the reply corresponds to + // a different request and should be ignored), note that the byte order + // of the received packet is already converted while org_ is still in + // network byte order + if(ntp->org != org) + { + log_warn("Received NTP reply does not match request (request %"PRIx64", reply %"PRIx64"), ignoring", + ntp->org, org); + return false; + } + + // Check stratum, mode, version, etc. + if((buf[0] & 0x07) != 4) + { + log_warn("Received NTP reply has invalid version, ignoring"); + return false; + } + + // Calculate delay and offset + const double T1 = ntp->org / FRAC; + const double T2 = rec / FRAC; + const double T3 = ntp->xmt / FRAC; + const double T4 = dst / FRAC; + + // RFC 5905, Section 8: On-wire protocol + // It is recommended to use double precision floating point arithmetic + // for the calculations to allow unambiguous interpretation of the + // results within the maximum adjustment range of 68 years. + + // Compute offset of client clock relative to server clock + ntp->theta = ( ( T2 - T1 ) + ( T3 - T4 ) ) / 2; + // Compute round-trip delay, which represents the delay of the packet + // passing through the network, which can be due switches and network + // technologies are highly variable + ntp->delta = ( T4 - T1 ) - ( T3 - T2 ); + + // This reply is valid + ntp->valid = true; + + // In some scenarios where the initial frequency offset of the client is + // relatively large and the actual propagation time small, it is + // possible for the delay computation to become negative. For instance, + // if the frequency difference is 100 ppm and the interval T4-T1 is 64 + // s, the apparent delay is -6.4 ms. Since negative values are + // misleading in subsequent computations, the value of delta should be + // clamped not less than s.rho, where s.rho is the system precision + // described in Section 11.1, expressed in seconds. + if(ntp->delta < ntp->precision) + ntp->delta = 0; + + // Return early if not verbose + if(!config.debug.ntp.v.b) + return true; + + // Print current time at server + print_debug_time("Server reference time", NULL, ref); + + // Print current time at client + print_debug_time("Current time at client", NULL, dst); + + // Print current time at server + print_debug_time("Current time at server", NULL, ntp->xmt); + + // Print offset and delay + log_debug(DEBUG_NTP, "Time offset: %e s", ntp->theta); + log_debug(DEBUG_NTP, "Round-trip delay: %e s", ntp->delta); + const uint32_t root_delay = ntohl(srv_root_delay); + log_debug(DEBUG_NTP, "Root delay: %e s", FP2D(root_delay)); + const uint32_t root_dispersion = ntohl(srv_root_dispersion); + log_debug(DEBUG_NTP, "Root dispersion: %e s", FP2D(root_dispersion)); + + return true; +} + +static int getsock(const struct addrinfo *saddr) +{ + // Create UDP socket + const int protocol = saddr->ai_addrlen == sizeof(struct sockaddr_in6) ? AF_INET6 : AF_INET; + const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); + if(s == -1) + { + char errbuf[1024]; + strncpy(errbuf, "Cannot create UDP socket: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); + return -1; + } + + // Set socket timeout to 5 seconds + struct timeval tv; + tv.tv_sec = 5; + tv.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) != 0) + { + char errbuf[1024]; + strncpy(errbuf, "Cannot set socket timeout: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); + close(s); + return -1; + } + + // Set address to send to/receive from + if(connect(s, saddr->ai_addr, saddr->ai_addrlen) != 0) + { + char errbuf[1024]; + strncpy(errbuf, "Cannot connect to NTP server: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); + close(s); + return -1; + } + + // Return socket + return s; +} + +bool ntp_client(const char *server, const bool settime, const bool print) +{ + // Resolve server address + int eai; + struct addrinfo *saddr; + // Resolve server address, port 123 is used for NTP + if((eai = getaddrinfo(server, "123", NULL, &saddr)) != 0) + { + char errbuf[1024]; + strncpy(errbuf, "Cannot resolve NTP server address: ", sizeof(errbuf)); + strncat(errbuf, errno == EAI_SYSTEM ? strerror(errno) : gai_strerror(eai), + sizeof(errbuf) - strlen(errbuf) - 1); + if(eai == EAI_NONAME || eai == EAI_NODATA) + { + strncat(errbuf, " \"", sizeof(errbuf) - strlen(errbuf) - 1); + strncat(errbuf, server, sizeof(errbuf) - strlen(errbuf) - 1); + strncat(errbuf, "\"", sizeof(errbuf) - strlen(errbuf) - 1); + } + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, false, errbuf); + return false; + } + + const unsigned int count = config.ntp.sync.count.v.ui; + struct ntp_sync *ntp = calloc(count, sizeof(struct ntp_sync)); + if(ntp == NULL) + { + log_err("Cannot allocate memory for NTP client"); + return false; + } + + // Send and receive NTP packets + for(unsigned int i = 0; i < count; i++) + { + // Create socket + const int s = getsock(saddr); + if(s == -1) + continue; + + // Send request + if(!request(s, server, &ntp[i])) + { + close(s); + free(ntp); + freeaddrinfo(saddr); + return false; + } + // Get reply + if(!reply(s, server, &ntp[i], false)) + { + close(s); + continue; + } + + // Close socket + close(s); + + // Sleep for some time to avoid flooding the server + if(print) + printf("."); + fflush(stdout); + usleep(NTP_DELAY); + } + if(print) + printf("\n"); + + // Free allocated memory + freeaddrinfo(saddr); + + // Compute average and standard deviation + unsigned int valid = 0; + double theta_avg = 0.0, theta_stdev = 0.0; + double delta_avg = 0.0, delta_stdev = 0.0; + for(unsigned int i = 0; i < count; i++) + { + // Skip invalid values + if(fabs(ntp[i].theta) < ntp[i].precision || + fabs(ntp[i].delta) < ntp[i].precision || + !ntp[i].valid) + continue; + + theta_avg += ntp[i].theta; + delta_avg += ntp[i].delta; + valid++; + } + + if(valid == 0) + { + log_ntp_message(false, false, "No valid NTP replies received, check server and network connectivity"); + free(ntp); + return false; + } + log_info("Received %u/%u valid NTP replies from %s", valid, count, server); + + theta_avg /= valid; + delta_avg /= valid; + for(unsigned int i = 0; i < count; i++) + { + // Skip invalid values + if(fabs(ntp[i].theta) < ntp[i].precision || + fabs(ntp[i].delta) < ntp[i].precision || + !ntp[i].valid) + continue; + + theta_stdev += pow(ntp[i].theta - theta_avg, 2); + delta_stdev += pow(ntp[i].delta - delta_avg, 2); + } + theta_stdev = sqrt(theta_stdev / valid); + delta_stdev = sqrt(delta_stdev / valid); + + log_debug(DEBUG_NTP, "Average time offset: (%e +/- %e) s", theta_avg, theta_stdev); + log_debug(DEBUG_NTP, "Average round-trip delay: (%e +/- %e) s", delta_avg, delta_stdev); + + // Reject synchronization if the standard deviation of the time offset + // or round-trip delay is larger than 1 second + if(theta_stdev > 1.0 || delta_stdev > 1.0) + { + log_ntp_message(false, false, "Standard deviation of time offset is too large, rejecting synchronization"); + free(ntp); + return false; + } + + // Compute trimmed mean (average excluding outliers) + double theta_trim = 0.0, delta_trim = 0.0; + unsigned int trim = 0; + for(unsigned int i = 0; i < count; i++) + { + // Skip invalid values + if(fabs(ntp[i].theta) < ntp[i].precision || + fabs(ntp[i].delta) < ntp[i].precision || + !ntp[i].valid) + continue; + + // Skip outliers + // We consider values > 2 standard deviations from the mean as + // outliers + if(fabs(ntp[i].theta - theta_avg) > 2 * theta_stdev || + fabs(ntp[i].delta - delta_avg) > 2 * delta_stdev) + continue; + + theta_trim += ntp[i].theta; + delta_trim += ntp[i].delta; + trim++; + } + + // Free allocated memory + free(ntp); + + if(trim == 0) + { + log_warn("No valid NTP replies after outlier removal, check server and network connectivity"); + return false; + } + theta_trim /= trim; + delta_trim /= trim; + + log_info("Time offset: %e ms (excluded %u outliers)", 1e3*theta_trim, count - trim); + log_info("Round-trip delay: %e ms (excluded %u outliers)", 1e3*delta_trim, count - trim); + + // Set time if requested + if(settime) + { + // Calculate corrected time + struct timeval unix_time; + const uint64_t ntp_time = get_new_time(&unix_time, theta_trim); + + // If the clock deviates more than 0.5 seconds from the NTP server, + // the time is updated immediately. Otherwise, the time is updated + // gradually to avoid sudden jumps in the system clock. + // The threshold of 0.5 seconds is hard-wired into the kernel + // since Linux 2.6.26, see man ntp_adjtime(2) for details. + bool success; + if(fabs(theta_trim) > 0.5) + success = settime_step(&unix_time, theta_trim); + else + success = settime_skew(theta_trim); + + // Return early if time could not be set + if(!success) + return false; + + // Update last NTP sync time + ntp_last_sync = ntp_time; + + // Compute our server's root dispersion and delay + // Both quantities are the maximum error and maximum delay of + // the server's time relative to the reference time. The root + // dispersion is the maximum error of the server's time relative + // to the reference time, while the root delay is the maximum + // delay of the server's time relative to the reference time + ntp_root_delay = D2FP(theta_trim); + ntp_root_dispersion = D2FP(theta_stdev); + + // Finally, adjust RTC if configured + if(config.ntp.sync.rtc.set.v.b) + ntp_sync_rtc(); + } + + // Offset and delay larger than 0.1 seconds are considered as invalid + // during local testing (e.g., when the server is on the same machine) + return theta_avg < 0.1 && delta_avg < 0.1; +} + +static void *ntp_client_thread(void *arg) +{ + // Set thread name + prctl(PR_SET_NAME, thread_names[NTP_CLIENT], 0, 0, 0); + + // Run NTP client + bool ntp_server_started = false; + bool first_run = true; + while(!killed) + { + // Get time before NTP sync + const double before = double_time(); + + // Run NTP client + const bool success = ntp_client(config.ntp.sync.server.v.s, true, false); + + // Get time after NTP sync + const double after = double_time(); + + // If the time was updated by more than a certain amount, + // restart FTL to import recent data. This is relevant when the + // system time was set to an incorrect value (e.g., due to a + // dead CMOS battery or overall missing RTC) and the time was + // off. + double time_delta = fabs(after - before); + if(first_run && time_delta > GCinterval) + { + log_info("System time was updated by %.1f seconds", time_delta); + restart_ftl("System time updated"); + } + + // Set first run to false + first_run = false; + + if(success && !ntp_server_started) + { + // Initialize NTP server only after first NTP + // synchronization to ensure that the time is set + // correctly + ntp_server_started = ntp_server_start(); + } + + // Intermediate cancellation-point + BREAK_IF_KILLED(); + + // Sleep before retrying + thread_sleepms(NTP_CLIENT, 1000 * config.ntp.sync.interval.v.ui); + } + + log_info("Terminating NTP thread"); + + return NULL; +} + +bool ntp_start_sync_thread(pthread_attr_t *attr) +{ + // Return early if NTP client is disabled + if(config.ntp.sync.active.v.b == false || + config.ntp.sync.server.v.s == NULL || + strlen(config.ntp.sync.server.v.s) == 0 || + config.ntp.sync.interval.v.ui == 0) + { + log_info("NTP sync is disabled"); + ntp_server_start(); + return false; + } + + // Check if we have the ambient capabilities to set the system time. + // Without CAP_SYS_TIME, we cannot set the system time and the NTP + // client will not be able to synchronize the time so there is no point + // in starting the thread. + if(!check_capability(CAP_SYS_TIME)) + { + log_warn("Insufficient permissions to set system time (CAP_SYS_TIME required), NTP client not available"); + ntp_server_start(); + return false; + } + + // Create thread + if(pthread_create(&threads[NTP_CLIENT], attr, ntp_client_thread, NULL) != 0) + { + log_err("Cannot create NTP client thread"); + ntp_server_start(); + return false; + } + + return true; +} diff --git a/src/ntp/ntp.h b/src/ntp/ntp.h new file mode 100644 index 00000000..7adbe8ad --- /dev/null +++ b/src/ntp/ntp.h @@ -0,0 +1,78 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* NTP prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#ifndef NTP_H +#define NTP_H + +#include "FTL.h" +// TIMESTR_SIZE +#include "log.h" + +// uint64_t +#include +// bool +#include + +// Get current time in NTP (64bit) format +uint64_t gettime64(void); + +// Print NTP timestamp in human-readable form +void print_debug_time(const char *label, const uint32_t *u32p, const uint64_t ntp_time); + +// Start NTP server +bool ntp_server_start(void); + +// Start NTP client +bool ntp_client(const char *server, const bool settime, const bool print); + +// Start NTP sync thread +bool ntp_start_sync_thread(pthread_attr_t *attr); + +// Sync RTC time +bool ntp_sync_rtc(void); + +// Number of NTP queries to average. The more queries, the more accurate the +// time, but the longer it takes to synchronize. The minimum is 1. +#define NTP_AVERGAGE_COUNT 8 + +// Delay between consecutive NTP queries in microseconds +#define NTP_DELAY 500000 + +// number of seconds between 1900 and 1970 (MSB=1) +#define DIFF_SEC_1900_1970 (2208988800UL) +// number of seconds between 1970 and Feb 7, 2036 (6:28:16 UTC) (MSB=0) +#define DIFF_SEC_1970_2036 (2085978496UL) + +// Timestamp conversion macroni (RFC 5905, Appendix A) +#define FRIC 65536. // 2^16 as a double +#define D2FP(r) ((uint32_t)((r) * FRIC)) // NTP short +#define FP2D(r) ((double)(r) / FRIC) +#define FRAC 4294967296. // 2^32 as double +#define D2LFP(a) ((uint64_t)((a) * FRAC)) // NTP timestamp +#define LFP2D(a) ((double)(a) / FRAC) +#define U2LFP(a) (((uint64_t)((a).tv_sec + DIFF_SEC_1900_1970) << 32) + (uint64_t) ((a).tv_usec / 1e6 * FRAC)) + +// Convert NTP timestamp to seconds and microseconds +//#define NTPtoSEC(x) (((x & 0x80000000) != 0) ? ((x >> 32) - DIFF_SEC_1900_1970) : ((x >> 32) + DIFF_SEC_1970_2036)) +#define NTPtoSEC(x) ((x >> 32) - DIFF_SEC_1900_1970) +#define NTPtoUSEC(x) (suseconds_t)((LFP2D(x & 0xFFFFFFFF) * 1e6)) + +// Convert uint64_t to network byte order and vice versa +#define hton64(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) +#define ntoh64(x) ((((uint64_t)ntohl(x)) << 32) + ntohl((x) >> 32)) + +extern uint64_t ntp_last_sync; +extern uint32_t ntp_root_delay; +extern uint32_t ntp_root_dispersion; + +#endif // NTP_H + + + diff --git a/src/ntp/rtc.c b/src/ntp/rtc.c new file mode 100644 index 00000000..07232711 --- /dev/null +++ b/src/ntp/rtc.c @@ -0,0 +1,300 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Real Time Clock (RTC) functions +* The routines in this file have been inspired by man pages +* and the source of the hwclock which is part of the util-linux +* project (https://github.com/util-linux/util-linux/) +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "ntp/ntp.h" + +// ioctl() +#include +// RTC +#include +// O_WRONLY +#include +// struct config +#include "config/config.h" + +// List of RTC devices from +// https://github.com/util-linux/util-linux/blob/41e7686c9ad1ea7892b9d8941c266869bf6a28dd/sys-utils/hwclock-rtc.c#L85-L93 +static const char * const rtc_devices[] = { +#ifdef __ia64__ + "/dev/efirtc", + "/dev/misc/efirtc", +#endif + "/dev/rtc0", + "/dev/rtc", + "/dev/misc/rtc" +}; + +static void print_tm_time(const char *label, const struct tm *tm) +{ + char timestr[TIMESTR_SIZE] = { 0 }; + strftime(timestr, sizeof(timestr), "%Y-%m-%d %H:%M:%S", tm); + log_info("%s %s", label, timestr); +} + +// Try to find the RTC device and open it +static int open_rtc(void) +{ + int rtc_fd = -1; + + // Get current user's UID and GID + const uid_t uid = getuid(); + const gid_t gid = getgid(); + + // If the user has specified an RTC device, try to open it + if(config.ntp.sync.rtc.device.v.s != NULL && + strlen(config.ntp.sync.rtc.device.v.s) > 0) + { + // Open the RTC device + rtc_fd = open(config.ntp.sync.rtc.device.v.s, O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + config.ntp.sync.rtc.device.v.s); + return rtc_fd; + } + + // If the open failed because of permissions, try to change them + // momentarily. On some embedded systems, the RTC device is owned by + // root exclusively and users do not have permission to even open it. + // Without being able to access the RTC, the capability to set the + // time (CAP_SYS_TIME) is useless. + if(errno == EACCES) + { + // Get current owner of the device + struct stat st = { 0 }; + if(stat(config.ntp.sync.rtc.device.v.s, &st) == -1) + { + log_debug(DEBUG_NTP, "stat(\"%s\") failed: %s", + config.ntp.sync.rtc.device.v.s, strerror(errno)); + return -1; + } + + if(chown(config.ntp.sync.rtc.device.v.s, uid, gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + config.ntp.sync.rtc.device.v.s, uid, gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return -1; + } + + rtc_fd = open(config.ntp.sync.rtc.device.v.s, O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + config.ntp.sync.rtc.device.v.s); + } + + // Chown the device back to the original owner + if(chown(config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + config.ntp.sync.rtc.device.v.s, st.st_uid, st.st_gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return -1; + } + + // Return the RTC file descriptor (can be -1) + return rtc_fd; + } + + log_debug(DEBUG_NTP, "Failed to open RTC at \"%s\": %s", + config.ntp.sync.rtc.device.v.s, strerror(errno)); + + return -1; + } + + // If the user has not specified an RTC device, try to open the default + // ones + for(size_t i = 0; i < ArraySize(rtc_devices); i++) + { + rtc_fd = open(rtc_devices[i], O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + rtc_devices[i]); + break; + } + + // If the open failed because of permissions, try to change them + // momentarily + if(errno == EACCES) + { + // Get current owner of the device + struct stat st = { 0 }; + if(stat(rtc_devices[i], &st) == -1) + { + log_debug(DEBUG_NTP, "stat(\"%s\") failed: %s", + rtc_devices[i], strerror(errno)); + return -1; + } + + if(chown(rtc_devices[i], uid, gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + rtc_devices[i], uid, gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return -1; + } + + rtc_fd = open(rtc_devices[i], O_RDONLY); + if (rtc_fd != -1) + { + log_debug(DEBUG_NTP, "Successfully opened RTC at \"%s\"", + rtc_devices[i]); + } + + // Chown the device back to the original owner + if(chown(rtc_devices[i], st.st_uid, st.st_gid) == -1) + { + log_debug(DEBUG_NTP, "chown(\"%s\", %u, %u) failed: %s", + rtc_devices[i], st.st_uid, st.st_gid, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return -1; + } + + // Return the RTC file descriptor (can be -1) + return rtc_fd; + } + + log_debug(DEBUG_NTP, "Failed to open RTC at \"%s\": %s", + rtc_devices[i], strerror(errno)); + } + + return rtc_fd; +} + +static bool read_rtc(struct tm *tm) +{ + // Open the RTC device + const int rtc_fd = open_rtc(); + if(rtc_fd == -1) + return false; + + // Read the RTC time + struct rtc_time rtc_tm = { 0 }; + const int rc = ioctl(rtc_fd, RTC_RD_TIME, &rtc_tm); + if(rc == -1) + { + log_debug(DEBUG_NTP, "ioctl(RTC_RD_NAME) failed: %s", + strerror(errno)); + close(rtc_fd); + return false; + } + + // Convert the kernel's struct tm to the standard struct tm + tm->tm_sec = rtc_tm.tm_sec; + tm->tm_min = rtc_tm.tm_min; + tm->tm_hour = rtc_tm.tm_hour; + tm->tm_mday = rtc_tm.tm_mday; + tm->tm_mon = rtc_tm.tm_mon; + tm->tm_year = rtc_tm.tm_year; + tm->tm_wday = rtc_tm.tm_wday; + tm->tm_yday = rtc_tm.tm_yday; + tm->tm_isdst = -1; // the RTC does not provide this information + print_tm_time("Current RTC time is", tm); + + // Close the RTC device + close(rtc_fd); + + return true; +} + +// Set the Hardware Clock to the broken down time . +// Use ioctls to "rtc" device to set the time. +static bool set_rtc(const struct tm *new_time) +{ + // Open the RTC device + const int rtc_fd = open_rtc(); + if(rtc_fd == -1) + return false; + + // Set the RTC time from the broken down time + struct rtc_time rtc_tm = { 0 }; + rtc_tm.tm_sec = new_time->tm_sec; + rtc_tm.tm_min = new_time->tm_min; + rtc_tm.tm_hour = new_time->tm_hour; + rtc_tm.tm_mday = new_time->tm_mday; + rtc_tm.tm_mon = new_time->tm_mon; + rtc_tm.tm_year = new_time->tm_year; + rtc_tm.tm_wday = new_time->tm_wday; + rtc_tm.tm_yday = new_time->tm_yday; + rtc_tm.tm_isdst = new_time->tm_isdst; + + // Set the RTC time + const int rc = ioctl(rtc_fd, RTC_SET_TIME, &rtc_tm); + if(rc == -1) + { + log_debug(DEBUG_NTP, "ioctl(RTC_SET_TIME) failed: %s", + strerror(errno)); + close(rtc_fd); + return false; + } + print_tm_time("RTC time set to", new_time); + + // Close the RTC device + close(rtc_fd); + return true; +} + +bool ntp_sync_rtc(void) +{ + // Wait until the beginning of the next second as the RTC only has a + // resolution of one second + struct timespec ts = { 0 }; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec++; + ts.tv_nsec = 0; + clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &ts, NULL); + + // Time to which we will set Hardware Clock, in broken down format + struct tm new_time = { 0 }; + const time_t newtime = time(NULL); + if(config.ntp.sync.rtc.utc.v.b) + // UTC + gmtime_r(&newtime, &new_time); + else + // Local time + localtime_r(&newtime, &new_time); + + // Read the current time from the RTC + struct tm rtc_time = { 0 }; + if(!read_rtc(&rtc_time)) + { + log_debug(DEBUG_NTP, "Failed to read RTC time"); + return false; + } + + // If the RTC time is the same as the current time, we don't need to set + // it. We don't use memcmp() here because the tm struct may contain + // additional fields that are not filled in by the RTC (e.g. tm_isdst). + if(rtc_time.tm_sec == new_time.tm_sec && + rtc_time.tm_min == new_time.tm_min && + rtc_time.tm_hour == new_time.tm_hour && + rtc_time.tm_mday == new_time.tm_mday && + rtc_time.tm_mon == new_time.tm_mon && + rtc_time.tm_year == new_time.tm_year) + { + // The RTC time is already correct, return early + log_debug(DEBUG_NTP, "RTC time is already correct"); + return true; + } + + // Set the RTC time + if(!set_rtc(&new_time)) + { + log_debug(DEBUG_NTP, "Failed to set RTC time"); + return false; + } + + return true; +} diff --git a/src/ntp/server.c b/src/ntp/server.c new file mode 100644 index 00000000..7c89fd07 --- /dev/null +++ b/src/ntp/server.c @@ -0,0 +1,409 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* NTP server routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "ntp/ntp.h" +// exit(0) +#include +// memcpy() +#include +// close() +#include +// fork(), wait() +#include +// clock_gettime() +#include +#include +// wait() +#include +// htonl(), etc. +#include +// errno +#include +// ctime() +#include +// pthread_create +#include +// PR_SET_NAME +#include +// config struct +#include "config/config.h" +// PRIi64 +#include +// log_ntp_message() +#include "database/message-table.h" +// NTP_SERVER_IPV4,6 +#include "enums.h" +// threads +#include "signals.h" + +uint64_t ntp_last_sync = 0u; +uint32_t ntp_root_delay = 0u; +uint32_t ntp_root_dispersion = 0u; + +// RFC 5905 Appendix A.4: Kernel System Clock Interface +uint64_t gettime64(void) +{ + struct timeval unix_time; + gettimeofday(&unix_time, NULL); + return (U2LFP(unix_time)); +} + +// Create and send an NTP reply to the client +static bool ntp_reply(const int socket_fd, const struct sockaddr *saddr_p, const socklen_t saddrlen, + const unsigned char recv_buf[], const uint64_t *recv_time) +{ + // Buffer for the response + unsigned char send_buf[48]; + memset(send_buf, 0, sizeof(send_buf)); + + // DWORD-aligned pointer to the send buffer + uint32_t *u32p = (uint32_t*)((void*)&send_buf[0]); + // DWORD-aligned read-only pointer to the receive buffer + const uint32_t *u32r = (uint32_t*)((void*)&recv_buf[0]); + +// NTP Packet Header Format (RFC 5905), page 18 +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |LI | VN |Mode | Stratum | Poll | Precision | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Check if the first byte is valid: mode is expected to be 3 ("client") + if ((recv_buf[0] & 0x07) != 0x3) { + log_warn("Received invalid NTP request: not from an NTP client, ignoring"); + return false; + } + + // set LI = 0 (no warning about leap seconds), set version-number to + // 4 and set mode = 4 ("server") + send_buf[0] = (0x04 << 3) + 0x04; + + // Set stratum to "secondary server" as we have derived time via + // external NTP as well. May be set to 1 if we want to be a primary + // server (synchronized by a hardware clock with GPS, etc.) + send_buf[1] = 0x02; + + // Copy Poll value from client + send_buf[2] = recv_buf[2]; + + // Precision: the precision of the local clock, in seconds to the + // nearest power of two. + // log2(1 usec = 1e-6 s) = -19.931568569324174 + send_buf[3] = (signed char)(-20); + + // Advance 32 bit pointer to the next field + u32p++; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Root Delay | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Root Dispersion | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Set Root Delay (total roundtrip delay to the primary reference + // source) and Root Dispersion (the nominal error relative to the + // primary reference source) to the values obtained from the upstream + // NTP server. + *u32p++ = htonl(ntp_root_delay); + *u32p++ = htonl(ntp_root_dispersion); + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Reference ID | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Reference ID = 'LOCL" (LOCAL CLOCK) + // A four-octet, left-justified, zero-padded ASCII string assigned to + // the reference clock + memcpy(u32p++, "LOCL", sizeof(uint32_t)); + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Reference Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Time when the system clock was last set or corrected, in NTP + // timestamp format. + const uint64_t last_sync = hton64(ntp_last_sync); + memcpy(u32p, &last_sync, sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Reference Timestamp", u32p, 0); + u32p += 2; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Origin Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Time at the client when the request departed for the server, in NTP + // timestamp format. (this is the client's transmit time) + memcpy(u32p, &u32r[10], sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Origin Timestamp", u32p, 0); + u32p += 2; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Receive Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Time at the server when the request arrived from the client, in NTP + // timestamp format. (this is the server's receive time) + const uint64_t net_recv_time = hton64(*recv_time); + memcpy(u32p, &net_recv_time, sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Receive Timestamp", u32p, 0); + u32p += 2; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// + Transmit Timestamp (64) + +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + // Time at the server when the response left for the client, in NTP + // timestamp format. (this is the server's transmit time) + const uint64_t transmit_time = gettime64(); + const uint64_t net_transmit_time = hton64(transmit_time); + memcpy(u32p, &net_transmit_time, sizeof(uint64_t)); + if(config.debug.ntp.v.b) + print_debug_time("Transmit Timestamp", u32p, 0); + u32p += 2; + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Extension Field 1 (variable) . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// . . +// . Extension Field 2 (variable) . +// . . +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Key Identifier | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | +// | dgst (128) | +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// Figure 8: Packet Header Format + + // Send the response + errno = 0; + if(sendto(socket_fd, send_buf, sizeof(send_buf), 0, saddr_p, saddrlen) < 48) + { + log_err("NTP send error: %s", strerror(errno)); + return false; + } + + return true; +} + +// Process incoming NTP requests +static void request_process_loop(const int fd, const char *ipstr, const int protocol) +{ + log_info("NTP server listening on %s:123 (%s)", ipstr, protocol == AF_INET ? "IPv4" : "IPv6"); + while (true) + { + unsigned char buf[48]; + struct sockaddr src_addr; + socklen_t src_addrlen = sizeof(src_addr); + while(recvfrom(fd, buf, sizeof(buf), 0, &src_addr, &src_addrlen) < 48); // ignore invalid requests + + // Get the current time in NTP format directly after receiving + // the request + const uint64_t recv_time = gettime64(); + + // Print the request + if(config.debug.ntp.v.b) + { + if(protocol == AF_INET6) + { + struct sockaddr_in6 sin6; + memcpy(&sin6, &src_addr, sizeof(sin6)); + + char ip[INET6_ADDRSTRLEN]; + const in_port_t port = ntohs(sin6.sin6_port); + inet_ntop(protocol, &sin6.sin6_addr, ip, sizeof(ip)); + log_debug(DEBUG_NTP, "Received NTP request from [%s]:%u", ip, port); + } + else + { + struct sockaddr_in sin; + memcpy(&sin, &src_addr, sizeof(sin)); + + char ip[INET6_ADDRSTRLEN]; + const in_port_t port = ntohs(sin.sin_port); + inet_ntop(protocol, &sin.sin_addr, ip, sizeof(ip)); + log_debug(DEBUG_NTP, "Received NTP request from %s:%u", ip, port); + } + } + + // Fork a child to handle the request + const pid_t pid = fork(); + if (pid == 0) { + // Child + ntp_reply(fd, &src_addr , src_addrlen, buf, &recv_time); + exit(0); + } else if (pid == -1) { + log_err("fork() error"); + return; + } + // return to parent + } +} + +// Start the NTP server +static void *ntp_bind_and_listen(void *param) +{ + // Set thread name + const unsigned int thread_id = param == 0 ? NTP_SERVER4 : NTP_SERVER6; + prctl(PR_SET_NAME, thread_names[thread_id], 0, 0, 0); + + // Create a socket + const int protocol = param == 0 ? AF_INET : AF_INET6; + errno = 0; + const int s = socket(protocol, SOCK_DGRAM, IPPROTO_UDP); + if(s == -1) + { + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), + "Cannot create NTP socket (%s), IPv%i NTP server not available", + strerror(errno), protocol == AF_INET ? 4 : 6); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); + return NULL; + } + + // Bind the socket to the NTP port + char ipstr[INET6_ADDRSTRLEN + 1]; + memset(ipstr, 0, sizeof(ipstr)); + if(protocol == AF_INET) + { + // IPv4 NTP server + + // Prepare the bind address + struct sockaddr_in bind_addr; + memset(&bind_addr, 0, sizeof(bind_addr)); + bind_addr.sin_family = AF_INET; // IPv4 + bind_addr.sin_port = htons(123); // NTP port + memcpy(&bind_addr.sin_addr, &config.ntp.ipv4.address.v.in_addr, sizeof(bind_addr.sin_addr)); + inet_ntop(AF_INET, &bind_addr.sin_addr, ipstr, sizeof(ipstr) - 1); + + // Bind the socket + errno = 0; + if(bind(s, (struct sockaddr *)&bind_addr, sizeof(bind_addr)) != 0) + { + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), + "Cannot bind to IPv4 address %s:123 (%s), IPv4 NTP server not available", + ipstr, strerror(errno)); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); + return NULL; + } + } + else + { + // IPv6 NTP server + + // Set socket options to allow IPv6 only, otherwise it will bind + // to both IPv4 and IPv6 and show IPv4 addresses as + // v4-mapped-on-v6 addresses + int opt = 1; + if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) != 0) + { + char errbuf[1024]; + strncpy(errbuf, "Cannot set socket option IPV6_V6ONLY, IPv6 NTP server not available: ", sizeof(errbuf)); + strncat(errbuf, strerror(errno), sizeof(errbuf) - strlen(errbuf) - 1); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); + return NULL; + } + + // Prepare the bind address + struct sockaddr_in6 bind_addr; + memset(&bind_addr, 0, sizeof(bind_addr)); + bind_addr.sin6_family = AF_INET6; // IPv6 + bind_addr.sin6_port = htons(123); // NTP port + memcpy(&bind_addr.sin6_addr, &config.ntp.ipv6.address.v.in6_addr, sizeof(bind_addr.sin6_addr)); + inet_ntop(AF_INET6, &bind_addr.sin6_addr, ipstr, sizeof(ipstr) - 1); + + // Bind the socket + errno = 0; + if(bind(s, (struct sockaddr *)&bind_addr, sizeof(bind_addr)) != 0) + { + char errbuf[1024]; + snprintf(errbuf, sizeof(errbuf), + "Cannot bind to IPv6 address %s:123 (%s), IPv6 NTP server not available", + ipstr, strerror(errno)); + errbuf[sizeof(errbuf) - 1] = '\0'; + log_ntp_message(true, true, errbuf); + return NULL; + } + } + + request_process_loop(s, ipstr, protocol); + close(s); + + return NULL; +} + +// Start the NTP server +bool ntp_server_start(void) +{ + // Spawn two pthreads, one for IPv4 and one for IPv6 + + // IPv4 + if(config.ntp.ipv4.active.v.b) + { + // Create a thread for the IPv4 NTP server + pthread_t thread; + if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)0) != 0) + { + log_ntp_message(true, true, "Cannot create NTP server thread for IPv4"); + return false; + } + } + + // IPv6 + if(config.ntp.ipv6.active.v.b) + { + // Create a thread for the IPv6 NTP server + pthread_t thread; + if (pthread_create(&thread, NULL, ntp_bind_and_listen, (void *)1) != 0) + { + log_ntp_message(true, true, "Cannot create NTP server thread for IPv6"); + return false; + } + } + + return true; +} diff --git a/src/overTime.c b/src/overTime.c index 39681311..34b51755 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -30,7 +30,9 @@ static void initSlot(const unsigned int index, const time_t timestamp) if(config.debug.overtime.v.b) { char timestr[20]; - strftime(timestr, 20, "%Y-%m-%d %H:%M:%S", localtime(×tamp)); + struct tm tm = { 0 }; + localtime_r(×tamp, &tm); + strftime(timestr, 20, "%Y-%m-%d %H:%M:%S", &tm); log_debug(DEBUG_OVERTIME, "initSlot(%u, %lu): Zeroing overTime slot at %s", index, (unsigned long)timestamp, timestr); } @@ -73,8 +75,11 @@ void initOverTime(void) if(config.debug.overtime.v.b) { char first[20], last[20]; - strftime(first, 20, "%Y-%m-%d %H:%M:%S", localtime(&oldest)); - strftime(last, 20, "%Y-%m-%d %H:%M:%S", localtime(&newest)); + struct tm tm_o = { 0 }, tm_n = { 0 }; + localtime_r(&oldest, &tm_o); + localtime_r(&newest, &tm_n); + strftime(first, 20, "%Y-%m-%d %H:%M:%S", &tm_o); + strftime(last, 20, "%Y-%m-%d %H:%M:%S", &tm_n); log_debug(DEBUG_OVERTIME, "initOverTime(): Initializing %i slots from %s (%lu) to %s (%lu)", OVERTIME_SLOTS, first, (unsigned long)oldest, last, (unsigned long)newest); } @@ -119,11 +124,11 @@ unsigned int _getOverTimeID(time_t timestamp, const char *file, const int line) // This is definitely wrong. We warn about this (but only once) if(!warned_about_hwclock) { - char timestampStr[TIMESTR_SIZE] = ""; + char timestampStr[TIMESTR_SIZE]; get_timestr(timestampStr, timestamp, false, false); const time_t lastTimestamp = overTime[OVERTIME_SLOTS-1].timestamp; - char lastTimestampStr[TIMESTR_SIZE] = ""; + char lastTimestampStr[TIMESTR_SIZE]; get_timestr(lastTimestampStr, lastTimestamp, false, false); log_warn("Found database entries in the future (%s (%lu), last timestamp for importing: %s (%lu)). " diff --git a/src/procps.c b/src/procps.c index 8e3656a4..d23ce7d7 100644 --- a/src/procps.c +++ b/src/procps.c @@ -114,10 +114,34 @@ static bool get_process_creation_time(const pid_t pid, char timestr[TIMESTR_SIZE return true; } +// This function checks if a given PID is running inside a docker container +static bool is_in_docker(const pid_t pid) +{ + char filename[sizeof("/proc/%u/cgroup") + sizeof(int)*3]; + snprintf(filename, sizeof(filename), "/proc/%d/cgroup", pid); + + FILE *f = fopen(filename, "r"); + if(f == NULL) + return false; + + char buffer[128]; + while(fgets(buffer, sizeof(buffer), f) != NULL) + { + if(strstr(buffer, "/docker") != NULL) + { + fclose(f); + return true; + } + } + fclose(f); + + return false; +} + // This function prints an info message about if another FTL process is already // running. It returns true if another FTL process is already running, false // otherwise. -bool check_running_FTL(void) +bool another_FTL(void) { DIR *dirPos; struct dirent *entry; @@ -144,7 +168,7 @@ bool check_running_FTL(void) { // Note: kill(pid, 0) does not send a // signal, but merely checks if the - // process exists If the process does + // process exists. If the process does // not exist, kill() returns -1 and sets // errno to ESRCH. However, if the // process exists, but security @@ -162,20 +186,22 @@ bool check_running_FTL(void) } else { - log_debug(DEBUG_SHMEM, "Failed to parse PID in PID file"); + log_debug(DEBUG_SHMEM, "Failed to parse PID in PID file: %s", + strerror(errno)); } fclose(pidFile); } else { - log_debug(DEBUG_SHMEM, "Failed to open PID file"); + log_debug(DEBUG_SHMEM, "Failed to open PID file \"%s\": %s", + config.files.pid.v.s, strerror(errno)); } } // If already_running is true, we are done if(already_running) { - log_info("%s is already running (PID %d)!", PROCESS_NAME, pid); + log_crit("%s is already running (PID %d)!", PROCESS_NAME, pid); return true; } @@ -217,7 +243,7 @@ bool check_running_FTL(void) if(pid == ourselves) continue; - // Only process this is this is our own process + // Only process this if this is our own process if(strcasecmp(name, PROCESS_NAME) != 0) continue; @@ -229,6 +255,10 @@ bool check_running_FTL(void) if(!get_process_name(ppid, ppid_name)) continue; + // Skip if this is an instance running inside a docker container + if(is_in_docker(pid)) + continue; + log_debug(DEBUG_SHMEM, " └ PPID: %d -> name: %s", ppid, ppid_name); char timestr[TIMESTR_SIZE] = { 0 }; @@ -238,7 +268,7 @@ bool check_running_FTL(void) if(!already_running) { already_running = true; - log_info("%s is already running!", PROCESS_NAME); + log_crit("%s is already running!", PROCESS_NAME); } if(last_pid != ppid) diff --git a/src/procps.h b/src/procps.h index 986d79bb..45568adf 100644 --- a/src/procps.h +++ b/src/procps.h @@ -10,7 +10,7 @@ #ifndef PROCPS_H #define PROCPS_H -bool check_running_FTL(void); +bool another_FTL(void); struct proc_mem { // Memory currently resident in RAM (in kB) @@ -43,4 +43,4 @@ bool read_self_memory_status(struct statm_t *result); bool getProcessMemory(struct proc_mem *mem, const unsigned long total_memory); bool parse_proc_meminfo(struct proc_meminfo *mem); -#endif // PROCPS_H \ No newline at end of file +#endif // PROCPS_H diff --git a/src/regex.c b/src/regex.c index 0d14d07b..2da9de08 100644 --- a/src/regex.c +++ b/src/regex.c @@ -34,7 +34,7 @@ const char *regextype[REGEX_MAX] = { "deny", "allow", "CLI" }; static regexData *allow_regex = NULL; static regexData *deny_regex = NULL; -static regexData *cli_regex = NULL; +static regexData cli_regex = { 0 }; static unsigned int num_regex[REGEX_MAX] = { 0 }; unsigned int regex_change = 0; static char regex_msg[REGEX_MSG_LEN] = { 0 }; @@ -48,7 +48,7 @@ static inline regexData *get_regex_ptr(const enum regex_type regexid) case REGEX_ALLOW: return allow_regex; case REGEX_CLI: - return cli_regex; + return &cli_regex; case REGEX_MAX: // Fall through default: // This is not possible return NULL; @@ -57,7 +57,7 @@ static inline regexData *get_regex_ptr(const enum regex_type regexid) static inline void free_regex_ptr(const enum regex_type regexid) { - regexData **regex; + regexData **regex = NULL; switch (regexid) { case REGEX_DENY: @@ -67,8 +67,8 @@ static inline void free_regex_ptr(const enum regex_type regexid) regex = &allow_regex; break; case REGEX_CLI: - regex = &cli_regex; - break; + // cannot be freed + return; case REGEX_MAX: // Fall through default: // This is not possible return; @@ -626,8 +626,7 @@ void free_regex(void) { // Return early if we don't use any regex filters if(allow_regex == NULL && - deny_regex == NULL && - cli_regex == NULL) + deny_regex == NULL) { log_debug(DEBUG_DATABASE, "Not using any regex filters, nothing to free or reset"); return; @@ -788,7 +787,7 @@ static void read_regex_table(const enum regex_type regexid) if(!compile_regex(regex_string, ®ex[index], &message) && message != NULL) { logg_regex_warning(regextype[regexid], message, - regex->database_id, regex_string); + rowid, regex_string); free(message); } @@ -895,15 +894,13 @@ int regex_test(const bool debug_mode, const bool quiet, const char *domainin, co { // Compile CLI regex log_info("%s Compiling regex filter...", cli_info()); - regexData regex = { 0 }; - cli_regex = ®ex; num_regex[REGEX_CLI] = 1; // Compile CLI regex timer_start(REGEX_TIMER); log_ctrl(false, true); // Temporarily re-enable terminal output for error logging char *message = NULL; - if(!compile_regex(regexin, ®ex, &message) && message != NULL) + if(!compile_regex(regexin, &cli_regex, &message) && message != NULL) { logg_regex_warning("CLI", message, 0, regexin); free(message); diff --git a/src/resolve.c b/src/resolve.c index b083b0cc..0f717b47 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -33,6 +33,8 @@ #include "regex_r.h" // statis_assert() #include +// TCP_MAX_QUERIES +#include "dnsmasq/config.h" // Function Prototypes static void name_toDNS(unsigned char *dns, const size_t dnslen, const char *host, const size_t hostlen) __attribute__((nonnull(1,3))); @@ -40,7 +42,7 @@ static unsigned char *name_fromDNS(unsigned char *reader, unsigned char *buffer, // Avoid "error: packed attribute causes inefficient alignment for ..." on ARM32 // builds due to the use of __attribute__((packed)) in the following structs -// Their correct size is ensured for each by static_assert() below +// Their correct size is ensured for each by check_struct_sizes() below _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wattributes\"") @@ -66,7 +68,6 @@ struct DNS_HEADER uint16_t auth_count; // number of authority entries uint16_t add_count; // number of resource entries } __attribute__((packed)); -static_assert(sizeof(struct DNS_HEADER) == 12); // Constant sized fields of query structure struct QUESTION @@ -74,7 +75,6 @@ struct QUESTION uint16_t qtype; uint16_t qclass; }; -static_assert(sizeof(struct QUESTION) == 4); // Constant sized fields of the resource record structure struct R_DATA @@ -84,9 +84,18 @@ struct R_DATA uint32_t ttl; // RFC 1035 defines the TTL field as "positive values of a signed 32bit number" uint16_t data_len; } __attribute__((packed)); -static_assert(sizeof(struct R_DATA) == 10); _Pragma("GCC diagnostic pop") +static bool check_struct_sizes(void) +{ + // Check sizes of structs + assert(sizeof(struct DNS_HEADER) == 12); + assert(sizeof(struct QUESTION) == 4); + assert(sizeof(struct R_DATA) == 10); + + return true; +} + // Pointers to resource record contents struct RES_RECORD { @@ -95,6 +104,64 @@ struct RES_RECORD uint8_t *rdata; }; +// see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml +static const char *getDNScode(int code) +{ + switch(code) + { + case 0: + return "NoError"; + case 1: + return "FormErr (Format Error)"; + case 2: + return "ServFail (Server Failure)"; + case 3: + return "NXDomain (Non-Existent Domain)"; + case 4: + return "NotImp (Not Implemented)"; + case 5: + return "Refused (Query Refused)"; + case 6: + return "YXDomain (Name Exists when it should not)"; + case 7: + return "YXRRSet (RR Set Exists when it should not)"; + case 8: + return "NXRRSet (RR Set that should exist does not)"; + case 9: + return "NotAuth (Server Not Authoritative for zone)"; + case 10: + return "NotZone (Name not contained in zone)"; + case 11: + return "DSOTYPENI (DSO-TYPE Not Implemented)"; + case 16: + return "BADVERS (Bad OPT Version) -or- BADSIG (TSIG Signature Failure)"; + case 17: + return "BADKEY (Key not recognized)"; + case 18: + return "BADTIME (Signature out of time window)"; + case 19: + return "BADMODE (Bad TKEY Mode)"; + case 20: + return "BADNAME (Duplicate key name)"; + case 21: + return "BADALG (Algorithm not supported)"; + case 22: + return "BADTRUNC (Bad Truncation)"; + case 23: + return "BADCOOKIE (Bad/missing Server Cookie)"; + default: + ; + } + + if((code >= 24 && code <= 3840) || (code >= 4096 && code <= 65535)) + return "Unassigned"; + else if(code >= 3841 && code <= 4095) + return "Reserved for Private Use"; + + // else: + return "Unknown"; +} + // Validate given hostname static bool valid_hostname(char* name, const char* clientip) { @@ -154,10 +221,50 @@ bool __attribute__((pure)) resolve_this_name(const char *ipaddr) return true; } -// Perform a name lookup by sending a packet to ourselves -static char *__attribute__((malloc)) ngethostbyname(const char *host, const char *ipaddr) +int create_socket(bool tcp, struct sockaddr_in *dest) { - uint8_t buf[1024] = { 0 }; + // Create a UDP (datagram) or TCP (stream) socket + const int sock = socket(AF_INET, tcp ? SOCK_STREAM : SOCK_DGRAM, tcp ? IPPROTO_TCP : IPPROTO_UDP); + if(sock < 0) + { + log_err("Unable to create DNS resolver socket: %s", strerror(errno)); + return -1; + } + + // Set timeout for socket (2 seconds) + struct timeval tv; + tv.tv_sec = 2; + tv.tv_usec = 0; + if(setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) + { + log_err("Unable to set DNS resolver socket timeout: %s", strerror(errno)); + close(sock); + return -1; + } + + // Create socket destination structure + memset(dest, 0, sizeof(*dest)); + dest->sin_family = AF_INET; // IPv4 + dest->sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1 + dest->sin_port = htons(config.dns.port.v.u16); // Configured DNS port + + // Connect to the DNS server (only done for TCP as UDP is + // connectionless) + if(tcp && connect(sock, (struct sockaddr*)dest, sizeof(*dest)) < 0) + { + log_err("Unable to connect to DNS resolver: %s", strerror(errno)); + close(sock); + return -1; + } + + return sock; +} + +// Perform a name lookup by sending a packet to ourselves +static char *__attribute__((malloc)) ngethostbyname(const int sock, const bool tcp, struct sockaddr_in *dest, + const char *host, const char *ipaddr, bool *truncated) +{ + uint8_t buf[4096] = { 0 }; // buffer for DNS query uint8_t *qname = NULL, *reader = NULL; struct RES_RECORD answers[20] = { 0 }; // buffer for DNS replies struct DNS_HEADER *dns = NULL; @@ -192,7 +299,7 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char if(hname == NULL) { log_err("Unable to allocate memory for hname"); - return strdup(""); + return NULL; } strncpy(hname, host, hnamelen); strncat(hname, ".", hnamelen - strlen(hname)); @@ -204,43 +311,91 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char qinfo->qtype = htons(T_PTR); // Type of the query, A, MX, CNAME, NS etc qinfo->qclass = htons(1); // IN + const size_t len = sizeof(struct DNS_HEADER) + (strlen((const char*)qname) + 1) + sizeof(struct QUESTION); - // UDP packet for DNS queries - const int s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - struct sockaddr_in dest = { 0 }; - dest.sin_family = AF_INET; // IPv4 - dest.sin_addr.s_addr = htonl(INADDR_LOOPBACK); // 127.0.0.1 - dest.sin_port = htons(config.dns.port.v.u16); // Configured DNS port + // Log query in debug mode + log_debug(DEBUG_RESOLVER, "Resolving PTR \"%s\" on 127.0.0.1#%u (%s)", + host, config.dns.port.v.u16, tcp ? "TCP" : "UDP"); - const size_t questionlen = sizeof(struct DNS_HEADER) + (strlen((const char*)qname) + 1) + sizeof(struct QUESTION); - if(sendto(s, buf, questionlen, 0, (struct sockaddr*)&dest, sizeof(dest)) < 0) + if(!tcp) { - perror("sendto failed"); - close(s); - return strdup(""); - } + // Send the query + socklen_t addrlen = sizeof(*dest); + if(sendto(sock, buf, len, 0, (struct sockaddr*)dest, addrlen) < 0) + { + log_err("Cannot send UDP DNS query: %s", strerror(errno)); + return NULL; + } - // Receive the answer - socklen_t addrlen = sizeof(dest); - if(recvfrom (s, buf, sizeof(buf), 0, (struct sockaddr*)&dest, &addrlen) < 0) + // Receive the answer + if(recvfrom (sock, buf, sizeof(buf), 0, (struct sockaddr*)dest, &addrlen) < 0) + { + log_err("Cannot receive UDP DNS reply: %s", strerror(errno)); + return NULL; + } + } + else { - perror("recvfrom failed"); - close(s); - return strdup(""); - } + // Send the query + // For TCP streams, we first have to send the length of the data + // we are sending. The reason for this is that with TCP, we are + // not sending messages (datagrams) but a continuous stream of + // bytes. We therefore need a way to tell the receiver about + // this length of the message. + uint16_t prefix = htons(len & 0xffffu); + if(send(sock, &prefix, sizeof(prefix), 0) < 0 || + send(sock, buf, len, 0) < 0) + { + log_err("Cannot send TCP DNS query: %s", strerror(errno)); + return NULL; + } - // Close socket - close(s); + // Receive the answer, first the length of the message ... + prefix = 0; + if(recv(sock, &prefix, sizeof(prefix), 0) < 0) + { + log_err("Cannot receive TCP DNS reply (1): %s", strerror(errno)); + return NULL; + } + prefix = ntohs(prefix); + + // Sanity check the length of the message + if(prefix > sizeof(buf)) + { + log_err("Received TCP DNS reply is too long (%u bytes)", prefix); + return NULL; + } + bzero(buf, prefix + 1); + // ... then the message itself + if(recv(sock, buf, sizeof(buf), 0) < 0) + { + log_err("Cannot receive TCP DNS reply (2): %s", strerror(errno)); + return NULL; + } + } // Parse the reply dns = (struct DNS_HEADER*) buf; // Move ahead of the dns header and the query field - reader = &buf[questionlen]; + reader = &buf[len]; + + // Log the status of the query + log_debug(DEBUG_RESOLVER, "DNS query for PTR \"%s\" returned status %s (%i)", + host, getDNScode(dns->rcode), dns->rcode); + + // Abort if the query was not successful + if(dns->tc != 0) + { + log_debug(DEBUG_RESOLVER, " --> DNS response truncated"); + if(truncated != NULL) + *truncated = true; + return NULL; + } // Start reading answers uint16_t stop = 0; char *name = NULL; - for(uint16_t i = 0; i < ntohs(dns->ans_count); i++) + for(uint16_t i = 0; i < min(ntohs(dns->ans_count), ArraySize(answers)); i++) { answers[i].name = name_fromDNS(reader, buf, &stop); reader = reader + stop; @@ -248,27 +403,38 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char answers[i].resource = (struct R_DATA*)(reader); reader = reader + sizeof(struct R_DATA); - // We only care about PTR answers and ignore all others - if(ntohs(answers[i].resource->type) != T_PTR) - continue; - // Read the answer and convert from network to host representation answers[i].rdata = name_fromDNS(reader, buf, &stop); reader = reader + stop; + // We only care about PTR answers and ignore all others + const uint16_t rtype = ntohs(answers[i].resource->type); + if(rtype != T_PTR) + { + log_debug(DEBUG_RESOLVER, "Answer %u is not of type PTR but %u (skipping)", + i, rtype); + + // Skip this answer + free(answers[i].name); + free(answers[i].rdata); + continue; + } + name = (char *)answers[i].rdata; - log_debug(DEBUG_RESOLVER, "Resolving %s (PTR \"%s\"): %u = \"%s\"", - ipaddr, answers[i].name, i, answers[i].rdata); + log_debug(DEBUG_RESOLVER, "Answer %u is PTR \"%s\" => \"%s\"", + i, answers[i].name, answers[i].rdata); // We break out of the loop if this is a valid hostname if(strlen(name) > 0 && valid_hostname(name, ipaddr)) { + free(answers[i].name); break; } else { // Discard this answer: free memory and set name to NULL - free(name); + free(answers[i].name); + free(answers[i].rdata); name = NULL; } } @@ -291,7 +457,8 @@ static char *__attribute__((malloc)) ngethostbyname(const char *host, const char // 3www6google3com -> www.google.com static u_char * __attribute__((malloc)) __attribute__((nonnull(1,2,3))) name_fromDNS(unsigned char *reader, unsigned char *buffer, uint16_t *count) { - unsigned char *name = calloc(MAXHOSTNAMELEN, sizeof(char)); + const size_t MAXNAMELEN = 256; + unsigned char *name = calloc(MAXNAMELEN, sizeof(char)); unsigned int p = 0, jumped = 0; // Initialize count @@ -304,7 +471,7 @@ static u_char * __attribute__((malloc)) __attribute__((nonnull(1,2,3))) name_fro // Instead, each label is preceded by a byte containing its length, and // the name is terminated by a zero-length label representing the root // zone. - while(*reader != 0) + while(*reader != 0 && p < MAXNAMELEN - 2) { if(*reader >= 0xC0) { @@ -368,7 +535,7 @@ static u_char * __attribute__((malloc)) __attribute__((nonnull(1,2,3))) name_fro } // Strip off the trailing dot - name[i-1] = '\0'; + name[i > 0 ? i-1 : i] = '\0'; return name; } @@ -398,7 +565,8 @@ static void __attribute__((nonnull(1,3))) name_toDNS(unsigned char *dns, const s *dns++='\0'; } -char *__attribute__((malloc)) resolveHostname(const char *addr, const bool force) +char *__attribute__((malloc)) resolveHostname(const int sock, const bool tcp, struct sockaddr_in *dest, + const char *addr, const bool force, bool *truncated) { // Get host name char *hostn = NULL; @@ -456,7 +624,7 @@ char *__attribute__((malloc)) resolveHostname(const char *addr, const bool force if(inaddr == NULL) { log_err("Unable to allocate memory for reverse lookup"); - return strdup(""); + return NULL; } // Convert IPv6 address to reverse lookup format @@ -506,7 +674,7 @@ char *__attribute__((malloc)) resolveHostname(const char *addr, const bool force if(inaddr == NULL) { log_err("Unable to allocate memory for reverse lookup"); - return strdup(""); + return NULL; } // Convert IPv4 address to reverse lookup format @@ -521,11 +689,18 @@ char *__attribute__((malloc)) resolveHostname(const char *addr, const bool force // Get host name by making a reverse lookup to ourselves (server at 127.0.0.1 with port 53) // We implement a minimalistic resolver here as we cannot rely on the system resolver using whatever // nameserver we configured in /etc/resolv.conf - return ngethostbyname(inaddr, addr); + hostn = ngethostbyname(sock, tcp, dest, inaddr, addr, truncated); + + // Free allocated memory + free(inaddr); + + // Return obtained host name + return hostn; } // Resolve upstream destination host names -static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) +static size_t resolveAndAddHostname(const int udp_sock, struct sockaddr_in *dest, + size_t ippos, size_t oldnamepos, bool *success) { // Get IP and host name strings. They are cloned in case shared memory is // resized before the next lock @@ -550,7 +725,28 @@ static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) // Important: Don't hold a lock while resolving as the main thread // (dnsmasq) needs to be operable during the call to resolveHostname() - char *newname = resolveHostname(ipaddr, false); + bool truncated = false; + char *newname = resolveHostname(udp_sock, false, dest, ipaddr, false, &truncated); + if(newname == NULL && truncated) + { + // Retry with TCP if UDP failed due to truncation (RFC 7766) + const int tcp_sock = create_socket(true, dest); + newname = resolveHostname(tcp_sock, true, dest, ipaddr, false, NULL); + close(tcp_sock); + } + + if(newname == NULL) + { + // We could not resolve the hostname, so we keep the old one + // and mark the entry as not new + log_debug(DEBUG_RESOLVER, " ---> \"%s\" (failed to resolve via UDP, too)", oldname); + + // Free allocated memory + *success = false; + free(ipaddr); + free(oldname); + return oldnamepos; + } // If no hostname was found, try to obtain hostname from the network table // This may be disabled due to a user setting @@ -569,6 +765,8 @@ static size_t resolveAndAddHostname(size_t ippos, size_t oldnamepos) { lock_shm(); size_t newnamepos = addstr(newname); + + // Free allocated memory // newname has already been checked against NULL // so we can safely free it free(newname); @@ -601,6 +799,15 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) int clientscount = counters->clients; unlock_shm(); + // Create DNS client socket + struct sockaddr_in dest = { 0 }; + const int udp_sock = create_socket(false, &dest); + if(udp_sock < 0) + { + log_err("Unable to create DNS resolver socket, client host name resolution failed"); + return; + } + int skipped = 0; for(int clientID = 0; clientID < clientscount; clientID++) { @@ -632,7 +839,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) // Limit for a "recently active" client is two hours ago if(!force_refreshing && !onlynew && client->lastQuery < now - 2*60*60) { - log_debug(DEBUG_RESOLVER, "Skipping client %s (%s) because it was inactive for %i seconds", + log_debug(DEBUG_RESOLVER, "Skipping client %s -> \"%s\" because it was inactive for %i seconds", getstr(ippos), getstr(oldnamepos), (int)(now - client->lastQuery)); unlock_shm(); @@ -644,7 +851,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) // If not, we will try to re-resolve all known clients if(!force_refreshing && onlynew && !newflag) { - log_debug(DEBUG_RESOLVER, "Skipping client %s (%s) because it is not new", + log_debug(DEBUG_RESOLVER, "Skipping client %s -> \"%s\" because it is not new", getstr(ippos), getstr(oldnamepos)); unlock_shm(); @@ -652,14 +859,14 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) continue; } - unlock_shm(); - // Check if we want to resolve an IPv6 address bool IPv6 = false; const char *ipaddr = NULL; if((ipaddr = getstr(ippos)) != NULL && strstr(ipaddr,":") != NULL) IPv6 = true; + unlock_shm(); + // If we're in refreshing mode (onlynew == false), we skip clients if // 1. We should not refresh any hostnames // 2. We should only refresh IPv4 client, but this client is IPv6 @@ -681,7 +888,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) reason = "Looking only for unknown hostnames"; lock_shm(); - log_debug(DEBUG_RESOLVER, "Skipping client %s (%s) because it should not be refreshed: %s", + log_debug(DEBUG_RESOLVER, "Skipping client %s -> \"%s\" because it should not be refreshed: %s", getstr(ippos), getstr(oldnamepos), reason); unlock_shm(); } @@ -690,7 +897,8 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) } // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); + bool success = true; + size_t newnamepos = resolveAndAddHostname(udp_sock, &dest, ippos, oldnamepos, &success); lock_shm(); // Get client pointer for the second time (writing data) @@ -706,6 +914,20 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) continue; } + if(!success) + { + // We could not resolve the hostname, so we keep the old one + // and mark the entry as not new - it will be retried later + client->flags.new = false; + + log_debug(DEBUG_RESOLVER, "Client %s -> \"%s\" could not be resolved, retrying later", + getstr(ippos), getstr(oldnamepos)); + + unlock_shm(); + continue; + } + + // else: // Store obtained host name (may be unchanged) client->namepos = newnamepos; // Mark entry as not new @@ -716,6 +938,9 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) unlock_shm(); } + // Close socket + close(udp_sock); + log_debug(DEBUG_RESOLVER, "%i / %i client host names resolved", clientscount-skipped, clientscount); } @@ -729,6 +954,15 @@ static void resolveUpstreams(const bool onlynew) int upstreams = counters->upstreams; unlock_shm(); + // Create socket + struct sockaddr_in dest = { 0 }; + const int udp_sock = create_socket(false, &dest); + if(udp_sock < 0) + { + log_err("Unable to create DNS resolver socket, client host name resolution failed"); + return; + } + int skipped = 0; for(int upstreamID = 0; upstreamID < upstreams; upstreamID++) { @@ -752,7 +986,7 @@ static void resolveUpstreams(const bool onlynew) // Limit for a "recently active" upstream server is two hours ago if(upstream->lastQuery < now - 2*60*60) { - log_debug(DEBUG_RESOLVER, "Skipping upstream %s (%s) because it was inactive for %i seconds", + log_debug(DEBUG_RESOLVER, "Skipping upstream %s -> \"%s\" because it was inactive for %i seconds", getstr(ippos), getstr(oldnamepos), (int)(now - upstream->lastQuery)); unlock_shm(); @@ -775,7 +1009,8 @@ static void resolveUpstreams(const bool onlynew) } // Obtain/update hostname of this client - size_t newnamepos = resolveAndAddHostname(ippos, oldnamepos); + bool success = true; + size_t newnamepos = resolveAndAddHostname(udp_sock, &dest, ippos, oldnamepos, &success); lock_shm(); // Get upstream pointer for the second time (writing data) @@ -791,6 +1026,19 @@ static void resolveUpstreams(const bool onlynew) continue; } + if(!success) + { + // We could not resolve the hostname, so we keep the old one + // and mark the entry as not new - it will be retried later + upstream->flags.new = false; + + log_debug(DEBUG_RESOLVER, "Upstream %s -> \"%s\" could not be resolved, retrying later", + getstr(ippos), getstr(oldnamepos)); + + unlock_shm(); + continue; + } + // Store obtained host name (may be unchanged) upstream->namepos = newnamepos; // Mark entry as not new @@ -801,6 +1049,9 @@ static void resolveUpstreams(const bool onlynew) unlock_shm(); } + // Close socket + close(udp_sock); + log_debug(DEBUG_RESOLVER, "%i / %i upstream server host names resolved", upstreams-skipped, upstreams); } @@ -808,10 +1059,15 @@ static void resolveUpstreams(const bool onlynew) void *DNSclient_thread(void *val) { // Set thread name - thread_names[DNSclient] = "DNS client"; - thread_running[DNSclient] = true; prctl(PR_SET_NAME, thread_names[DNSclient], 0, 0, 0); + // Test struct sizes + if(!check_struct_sizes()) + { + log_err("Struct sizes do not match expected sizes, aborting resolver thread"); + return NULL; + } + // Initial delay until we first try to resolve anything thread_sleepms(DNSclient, 2000); @@ -872,6 +1128,5 @@ void *DNSclient_thread(void *val) } log_info("Terminating resolver thread"); - thread_running[DNSclient] = false; return NULL; } diff --git a/src/resolve.h b/src/resolve.h index 2dfc50e1..d80654b7 100644 --- a/src/resolve.h +++ b/src/resolve.h @@ -11,7 +11,9 @@ #define RESOLVE_H void *DNSclient_thread(void *val); -char *resolveHostname(const char *addr, const bool force) __attribute__((malloc)); +int create_socket(bool tcp, struct sockaddr_in *dest); +char *resolveHostname(const int sock, const bool tcp, struct sockaddr_in *dest, + const char *addr, const bool force, bool *truncated) __attribute__((malloc)); bool resolve_names(void) __attribute__((pure)); bool resolve_this_name(const char *ipaddr) __attribute__((pure)); diff --git a/src/shmem.c b/src/shmem.c index 1d267146..e5569a5d 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -32,8 +32,6 @@ #include "files.h" // log_resource_shortage() #include "database/message-table.h" -// check_running_FTL() -#include "procps.h" /// The version of shared memory used #define SHARED_MEMORY_VERSION 14 @@ -149,58 +147,26 @@ static int get_dev_shm_usage(char buffer[64]) return percentage; } -// Verify the PID stored during shared memory initialization is the same as ours -// (while we initialized the shared memory objects) -static void verify_shmem_pid(void) -{ - // Open shared memory settings object - const int settingsfd = shm_open(SHARED_SETTINGS_NAME, O_RDONLY, S_IRUSR | S_IWUSR); - if(settingsfd == -1) - { - log_crit("verify_shmem_pid(): Failed to open shared memory object \"%s\": %s", - SHARED_SETTINGS_NAME, strerror(errno)); - exit(EXIT_FAILURE); - } - - ShmSettings shms = { 0 }; - if(read(settingsfd, &shms, sizeof(shms)) != sizeof(shms)) - { - log_crit("verify_shmem_pid(): Failed to read %zu bytes from shared memory object \"%s\": %s", - sizeof(shms), SHARED_SETTINGS_NAME, strerror(errno)); - exit(EXIT_FAILURE); - } - - close(settingsfd); - - // Compare the SHM's PID to the one we had when creating the SHM objects - if(shms.pid == shmem_pid) - return; - - // If we reach here, we are in serious trouble. Terminating with error - // code is the most sensible thing we can do at this point - log_crit("Shared memory is owned by a different process (PID %d)", shms.pid); - check_running_FTL(); - log_crit("Exiting now!"); - exit(EXIT_FAILURE); -} - // chown_shmem() changes the file ownership of a given shared memory object static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) { // Open shared memory object const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); log_debug(DEBUG_SHMEM, "Changing %s (%d) to %u:%u", sharedMemory->name, fd, ent_pw->pw_uid, ent_pw->pw_gid); + if(fd == -1) { - log_crit("chown_shmem(): Failed to open shared memory object \"%s\": %s", + log_crit("Failed to open shared memory object \"%s\" for chown: %s", sharedMemory->name, strerror(errno)); exit(EXIT_FAILURE); } + if(fchown(fd, ent_pw->pw_uid, ent_pw->pw_gid) == -1) { - log_warn("chown_shmem(%d, %u, %u): failed for %s: %s (%d)", - fd, ent_pw->pw_uid, ent_pw->pw_gid, sharedMemory->name, - strerror(errno), errno); + log_crit("Failed to change ownership of shared memory object \"%s\": %s", + sharedMemory->name, + errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); + return false; } @@ -286,7 +252,7 @@ const char *_getstr(const size_t pos, const char *func, const int line, const ch // Create a mutex for shared memory static void create_mutex(pthread_mutex_t *lock) { log_debug(DEBUG_SHMEM, "Creating SHM mutex lock"); - pthread_mutexattr_t lock_attr = {}; + pthread_mutexattr_t lock_attr; // Initialize the lock attributes pthread_mutexattr_init(&lock_attr); @@ -621,25 +587,41 @@ static bool create_shm(const char *name, SharedMemory *sharedMemory, const size_ // - O_CREAT: Create the shared memory object if it does not exist. // - O_EXCL: Return an error if a shared memory object with the given name already exists. errno = 0; - const int fd = shm_open(sharedMemory->name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); + sharedMemory->fd = shm_open(sharedMemory->name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); // Check for `shm_open` error - if(fd == -1) + if(sharedMemory->fd == -1) { log_err("create_shm(): Failed to create shared memory object \"%s\": %s", name, strerror(errno)); return sharedMemory; } + // Create exclusive file lock on shared memory object + // The lock will be automatically released when the file descriptor is closed + sharedMemory->lock.l_type = F_WRLCK; // write = exclusive lock + sharedMemory->lock.l_whence = SEEK_SET; + sharedMemory->lock.l_start = 0; // lock everything from the start ... + sharedMemory->lock.l_len = 0; // ... to the end of the file (magic 0 = EOF) + + // Try to lock the shared memory object + if(fcntl(sharedMemory->fd, F_SETLK, &sharedMemory->lock) == -1) + { + log_err("create_shm(): Failed to exclusively lock shared memory object \"%s\": %s", + name, strerror(errno)); + close(sharedMemory->fd); + return sharedMemory; + } + // Allocate shared memory object to specified size // Using f[tl]allocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = ftlallocate(fd, 0U, size); + const int ret = ftlallocate(sharedMemory->fd, 0U, size); if(ret != 0) { log_err("create_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", - sharedMemory->name, fd, size, strerror(errno), ret); + sharedMemory->name, sharedMemory->fd, size, strerror(errno), ret); exit(EXIT_FAILURE); } @@ -648,23 +630,19 @@ static bool create_shm(const char *name, SharedMemory *sharedMemory, const size_ used_shmem += size; // Create shared memory mapping - void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + void *shm = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sharedMemory->fd, 0); // Check for `mmap` error if(shm == MAP_FAILED) { log_err("create_shm(): Failed to map shared memory object \"%s\" (%i): %s", - sharedMemory->name, fd, strerror(errno)); + sharedMemory->name, sharedMemory->fd, strerror(errno)); return sharedMemory; } // Initialize shared memory object to zero memset(shm, 0, size); - // Close shared memory object file descriptor as it is no longer - // needed after having called mmap() - close(fd); - sharedMemory->ptr = shm; return sharedMemory; } @@ -740,11 +718,15 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si // Log output if(resize) + { log_debug(DEBUG_SHMEM, "Resizing \"%s\" from %zu to (%zu * %zu) == %zu (%s)", sharedMemory->name, sharedMemory->size, size1, size2, size, df); + } else + { log_debug(DEBUG_SHMEM, "Remapping \"%s\" from %zu to (%zu * %zu) == %zu", sharedMemory->name, sharedMemory->size, size1, size2, size); + } if(config.misc.check.shmem.v.ui > 0 && percentage > config.misc.check.shmem.v.ui) log_resource_shortage(-1.0, 0, percentage, -1, SHMEM_PATH, df); @@ -755,34 +737,18 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si // TCP requests. if(resize) { - // Verify shared memory ownership - verify_shmem_pid(); - - // Open shared memory object - const int fd = shm_open(sharedMemory->name, O_RDWR, S_IRUSR | S_IWUSR); - if(fd == -1) - { - log_crit("realloc_shm(): Failed to open shared memory object \"%s\": %s", - sharedMemory->name, strerror(errno)); - exit(EXIT_FAILURE); - } - // Allocate shared memory object to specified size // Using f[tl]allocate() will ensure that there's actually space for // this file. Otherwise we end up with a sparse file that can give // SIGBUS if we run out of space while writing to it. - const int ret = ftlallocate(fd, 0U, size); + const int ret = ftlallocate(sharedMemory->fd, 0U, size); if(ret != 0) { log_crit("realloc_shm(): Failed to resize \"%s\" (%i) to %zu: %s (%i)", - sharedMemory->name, fd, size, strerror(ret), ret); + sharedMemory->name, sharedMemory->fd, size, strerror(ret), ret); exit(EXIT_FAILURE); } - // Close shared memory object file descriptor as it is no longer - // needed after having called f[tl]allocate() - close(fd); - // Update shm counters to indicate that at least one shared memory object changed shmSettings->global_shm_counter++; local_shm_counter++; @@ -801,11 +767,15 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si used_shmem += (size - sharedMemory->size); if(sharedMemory->ptr == new_ptr) + { log_debug(DEBUG_SHMEM, "SHMEM pointer not updated: %p (%zu %zu)", sharedMemory->ptr, sharedMemory->size, size); + } else + { log_debug(DEBUG_SHMEM, "SHMEM pointer updated: %p -> %p (%zu %zu)", sharedMemory->ptr, new_ptr, sharedMemory->size, size); + } sharedMemory->ptr = new_ptr; sharedMemory->size = size; @@ -837,6 +807,11 @@ static void delete_shm(SharedMemory *sharedMemory) // Set unmapped pointer to NULL sharedMemory->ptr = NULL; + // Close shared memory file descriptor + if(close(sharedMemory->fd) != 0) + log_warn("delete_shm(): close(%i) failed: %s", sharedMemory->fd, strerror(errno)); + sharedMemory->fd = -1; + // Now you can no longer `shm_open` the memory, and once all others // unlink, it will be destroyed. if(shm_unlink(sharedMemory->name) != 0) @@ -976,7 +951,7 @@ void reset_per_client_regex(const int clientID) void add_per_client_regex(unsigned int clientID) { const unsigned int num_regex_tot = get_num_regex(REGEX_MAX); // total number - const size_t size = get_optimal_object_size(1, counters->clients * num_regex_tot); + const size_t size = get_optimal_object_size(1, (size_t)counters->clients * num_regex_tot); if(size > shm_per_client_regex.size && realloc_shm(&shm_per_client_regex, 1, size, true)) { @@ -1049,7 +1024,7 @@ static inline bool check_magic(int ID, bool checkMagic, unsigned char magic, con return true; } -queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file) +queriesData *_getQuery(int queryID, bool checkMagic, int line, const char *func, const char *file) { // This does not exist, return a NULL pointer if(queryID == -1) @@ -1218,3 +1193,59 @@ DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *f return NULL; } + +// Return 1 if this fd is associated with any shared memory object to avoid +// dnsmasq closing it during initialization +int __attribute__((pure)) is_shm_fd(const int fd) +{ + // Check all shared memory objects + for(unsigned int i = 0; i < ArraySize(sharedMemories); i++) + if(sharedMemories[i]->fd == fd) + return 1; + + // Not found + return 0; +} + +// Update queries per second (qps) value +// This is done in shared memory to allow for both UDP and TCP workers to +// contribute. +void update_qps(const double timestamp) +{ + // Get the timeslot for the current timestamp + const unsigned int slot = (unsigned int)timestamp % QPS_AVGLEN; + + // Check if the timestamp is in the same slot as the last one + if(shmSettings->qps.last != slot) + { + // Reset all the slots in between + // This is relevant if less than one query per second is + // received and the intermediate slots are not updated + for(unsigned int i = (shmSettings->qps.last + 1) % QPS_AVGLEN; i != slot; i = (i + 1) % QPS_AVGLEN) + shmSettings->qps.buf[i] = 0; + + // Reset the current slot + shmSettings->qps.buf[slot] = 0; + + // Update the last slot index + shmSettings->qps.last = slot; + } + + // Add the query + shmSettings->qps.buf[slot]++; +} + +// Compute queries per second (qps) value +double __attribute__((pure)) get_qps(void) +{ + // Compute the arithmetic mean of all slots + // 1 N + // QPS = --- Σ buf[i] + // N i=0 + // + double qps = 0.0; + for(unsigned int i = 0; i < QPS_AVGLEN; i++) + qps += shmSettings->qps.buf[i]; + + return qps / QPS_AVGLEN; +} diff --git a/src/shmem.h b/src/shmem.h index 8a6ed627..f717445e 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -22,6 +22,8 @@ typedef struct { const char *name; size_t size; void *ptr; + int fd; + struct flock lock; } SharedMemory; typedef struct { @@ -29,6 +31,10 @@ typedef struct { pid_t pid; unsigned int global_shm_counter; unsigned int next_str_pos; + struct { + unsigned int last; + unsigned int buf[QPS_AVGLEN]; + } qps; } ShmSettings; typedef struct { @@ -56,8 +62,14 @@ typedef struct { int groups; int lists; struct { - int allowed; - int denied; + struct { + int exact; + int regex; + } allowed; + struct { + int exact; + int regex; + } denied; } domains; } database; int querytype[TYPE_MAX]; @@ -140,4 +152,10 @@ void reset_per_client_regex(const int clientID); bool get_per_client_regex(const int clientID, const int regexID); void set_per_client_regex(const int clientID, const int regexID, const bool value); +// Used in dnsmasq/utils.c +int is_shm_fd(const int fd); + +void update_qps(const double timestamp); +double get_qps(void) __attribute__((pure)); + #endif //SHARED_MEMORY_SERVER_H diff --git a/src/signals.c b/src/signals.c index 9c638445..ba0107df 100644 --- a/src/signals.c +++ b/src/signals.c @@ -29,13 +29,20 @@ #define BINARY_NAME "pihole-FTL" volatile sig_atomic_t killed = 0; -static volatile pid_t mpid = -1; +static volatile pid_t mpid = 0; static time_t FTLstarttime = 0; volatile int exit_code = EXIT_SUCCESS; volatile sig_atomic_t thread_cancellable[THREADS_MAX] = { false }; -volatile sig_atomic_t thread_running[THREADS_MAX] = { false }; -const char *thread_names[THREADS_MAX] = { "" }; +const char * const thread_names[THREADS_MAX] = { + "database", + "housekeeper", + "dns-client", + "timer", + "ntp-client", + "ntp-server4", + "ntp-server6", + }; // Return the (null-terminated) name of the calling thread // The name is stored in the buffer as well as returned for convenience @@ -246,7 +253,7 @@ static void __attribute__((noreturn)) signal_handler(int sig, siginfo_t *si, voi log_info("Thank you for helping us to improve our FTL engine!"); // Terminate main process if crash happened in a TCP worker - if(mpid != getpid()) + if(main_pid() != getpid()) { // This is a forked process log_info("Asking parent pihole-FTL (PID %i) to shut down", (int)mpid); @@ -316,6 +323,8 @@ static void SIGRT_handler(int signum, siginfo_t *si, void *unused) // // Signal internally used to signal dnsmasq it has to stop // } + // SIGRT32: Used internally by valgrind, do not use + // Restore errno before returning back to previous context errno = _errno; } @@ -324,7 +333,11 @@ static void SIGTERM_handler(int signum, siginfo_t *si, void *unused) { // Ignore SIGTERM outside of the main process (TCP forks) if(mpid != getpid()) + { + log_debug(DEBUG_ANY, "Ignoring SIGTERM in TCP worker"); return; + } + log_debug(DEBUG_ANY, "Received SIGTERM"); // Get PID and UID of the process that sent the terminating signal const pid_t kill_pid = si->si_pid; @@ -392,11 +405,19 @@ static void SIGTERM_handler(int signum, siginfo_t *si, void *unused) // Log who sent the signal log_info("Asked to terminate by \"%s\" (PID %ld, user %s UID %ld)", - kill_name, (long int)kill_pid, - kill_user, (long int)kill_uid); + kill_name, (long int)kill_pid, kill_user, (long int)kill_uid); // Terminate dnsmasq to stop DNS service - raise(SIGUSR6); + if(!dnsmasq_failed) + { + log_debug(DEBUG_ANY, "Sending SIGUSR6 to dnsmasq to stop DNS service"); + raise(SIGUSR6); + } + else + { + log_debug(DEBUG_ANY, "Embedded dnsmasq failed, exiting on request"); + killed = true; + } } // Register ordinary signals handler @@ -404,29 +425,21 @@ void handle_signals(void) { struct sigaction old_action; - const int signals[] = { SIGSEGV, SIGBUS, SIGILL, SIGFPE }; + const int signals[] = { SIGSEGV, SIGBUS, SIGILL, SIGFPE, SIGTERM }; for(unsigned int i = 0; i < ArraySize(signals); i++) { // Catch this signal sigaction (signals[i], NULL, &old_action); if(old_action.sa_handler != SIG_IGN) { - struct sigaction SIGaction; - memset(&SIGaction, 0, sizeof(struct sigaction)); + struct sigaction SIGaction = { 0 }; SIGaction.sa_flags = SA_SIGINFO; sigemptyset(&SIGaction.sa_mask); - SIGaction.sa_sigaction = &signal_handler; + SIGaction.sa_sigaction = signals[i] != SIGTERM ? &signal_handler : &SIGTERM_handler; sigaction(signals[i], &SIGaction, NULL); } } - // Also catch SIGTERM - struct sigaction SIGaction = { 0 }; - SIGaction.sa_flags = SA_SIGINFO; - sigemptyset(&SIGaction.sa_mask); - SIGaction.sa_sigaction = &SIGTERM_handler; - sigaction(SIGTERM, &SIGaction, NULL); - // Log start time of FTL FTLstarttime = time(NULL); } @@ -445,6 +458,10 @@ void handle_realtime_signals(void) // Skip SIGUSR6 as it is used internally to signify // dnsmasq to stop continue; + if(signum == SIGUSR32) + // Skip SIGUSR32 as it is used internally by valgrind + // and should not be used + continue; struct sigaction SIGACTION = { 0 }; SIGACTION.sa_flags = SA_SIGINFO; @@ -457,7 +474,7 @@ void handle_realtime_signals(void) // Return PID of the main FTL process pid_t main_pid(void) { - if(mpid > -1) + if(mpid > 0) // Has already been set return mpid; else @@ -474,3 +491,43 @@ void thread_sleepms(const enum thread_types thread, const int milliseconds) sleepms(milliseconds); thread_cancellable[thread] = false; } + +static void print_signal(int signum, siginfo_t *si, void *unused) +{ + printf("Received signal %d: \"%s\"\n", signum, strsignal(signum)); + fflush(stdin); + if(signum == SIGTERM) + exit(EXIT_SUCCESS); +} + +// Register handler that catches *all* signals and displays them +int sigtest(void) +{ + printf("PID: %d\n", getpid()); + // Catch all real-time signals + for(int signum = 0; signum <= SIGRTMAX; signum++) + { + struct sigaction SIGACTION = { 0 }; + SIGACTION.sa_flags = SA_SIGINFO; + sigemptyset(&SIGACTION.sa_mask); + SIGACTION.sa_sigaction = &print_signal; + sigaction(signum, &SIGACTION, NULL); + } + + printf("Waiting (30sec)...\n"); + fflush(stdin); + + // Sleep here for 30 seconds + sleepms(30000); + + // Exit successfully + return EXIT_SUCCESS; +} + +void restart_ftl(const char *reason) +{ + log_info("Restarting FTL: %s", reason); + exit_code = RESTART_FTL_CODE; + // Send SIGTERM to FTL + kill(main_pid(), SIGTERM); +} diff --git a/src/signals.h b/src/signals.h index 4a08e4b9..3c2e8a75 100644 --- a/src/signals.h +++ b/src/signals.h @@ -13,15 +13,15 @@ #include "enums.h" #define SIGUSR6 (SIGRTMIN + 6) - -// defined in dnsmasq/dnsmasq.h -extern volatile char FTL_terminate; +#define SIGUSR32 (SIGRTMIN + 32) void handle_signals(void); void handle_realtime_signals(void); pid_t main_pid(void); void thread_sleepms(const enum thread_types thread, const int milliseconds); void generate_backtrace(void); +int sigtest(void); +void restart_ftl(const char *reason); extern volatile int exit_code; extern volatile sig_atomic_t killed; @@ -29,7 +29,8 @@ extern volatile sig_atomic_t want_to_reimport_aliasclients; extern volatile sig_atomic_t want_to_reload_lists; extern volatile sig_atomic_t thread_cancellable[THREADS_MAX]; -extern volatile sig_atomic_t thread_running[THREADS_MAX]; -extern const char *thread_names[THREADS_MAX]; +extern const char * const thread_names[THREADS_MAX]; + +#define BREAK_IF_KILLED() { if(killed) break; } #endif //SIGNALS_H diff --git a/src/struct_size.h b/src/struct_size.h index 94c692c7..53bfe12b 100644 --- a/src/struct_size.h +++ b/src/struct_size.h @@ -15,4 +15,4 @@ int check_one_struct(const char *struct_name, const size_t found_size, const size_t size64, const size_t size32); -#endif // STRUCT_SIZE_HEADER \ No newline at end of file +#endif // STRUCT_SIZE_HEADER diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 10103094..7ba43aa4 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -36,3 +36,4 @@ set(sources add_library(syscalls OBJECT ${sources}) target_compile_options(syscalls PRIVATE ${EXTRAWARN}) +target_include_directories(syscalls PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/syscalls/accept.c b/src/syscalls/accept.c index 710a7f3d..ef53bad2 100644 --- a/src/syscalls/accept.c +++ b/src/syscalls/accept.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef accept int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char *file, const char *func, const int line) @@ -39,4 +39,4 @@ int FTLaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen, const char errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/asprintf.c b/src/syscalls/asprintf.c index 4da1ea82..67585bb0 100644 --- a/src/syscalls/asprintf.c +++ b/src/syscalls/asprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) { diff --git a/src/syscalls/calloc.c b/src/syscalls/calloc.c index 60c2d2f0..e8de9821 100644 --- a/src/syscalls/calloc.c +++ b/src/syscalls/calloc.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef calloc void* __attribute__((malloc)) __attribute__((alloc_size(1,2))) FTLcalloc(const size_t nmemb, const size_t size, const char *file, const char *func, const int line) diff --git a/src/syscalls/fopen.c b/src/syscalls/fopen.c index 71912a69..9dd603a4 100644 --- a/src/syscalls/fopen.c +++ b/src/syscalls/fopen.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" static uint8_t already_writing = 0; diff --git a/src/syscalls/fprintf.c b/src/syscalls/fprintf.c index be3bee68..d64d85ab 100644 --- a/src/syscalls/fprintf.c +++ b/src/syscalls/fprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLfprintf(FILE *stream, const char *file, const char *func, const int line, const char *format, ...) { diff --git a/src/syscalls/free.c b/src/syscalls/free.c index 1091aa14..73c9c905 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -8,31 +8,25 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef free -void FTLfree(void **ptr, const char *file, const char *func, const int line) +bool FTLfree(void *ptr, const char *file, const char *func, const int line) { // The free() function frees the memory space pointed to by ptr, which // must have been returned by a previous call to malloc(), calloc(), or // realloc(). Otherwise, or if free(ptr) has already been called before, // undefined behavior occurs. If ptr is NULL, no operation is performed. if(ptr == NULL) - { - log_warn("Trying to free NULL memory location in %s() (%s:%i)", func, file, line); - return; - } - if(*ptr == NULL) { log_warn("Trying to free NULL pointer in %s() (%s:%i)", func, file, line); - return; + return false; } // Actually free the memory - free(*ptr); + free(ptr); - // Set the pointer to NULL - *ptr = NULL; + return true; } diff --git a/src/syscalls/ftlallocate.c b/src/syscalls/ftlallocate.c index 8140f1b8..b1330ae7 100644 --- a/src/syscalls/ftlallocate.c +++ b/src/syscalls/ftlallocate.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include // off_t is automatically set as off64_t when this is a 64bit system diff --git a/src/syscalls/pthread_mutex_lock.c b/src/syscalls/pthread_mutex_lock.c index ab4b0112..1fc1821a 100644 --- a/src/syscalls/pthread_mutex_lock.c +++ b/src/syscalls/pthread_mutex_lock.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include diff --git a/src/syscalls/realloc.c b/src/syscalls/realloc.c index 77f83f4a..aa2b12a3 100644 --- a/src/syscalls/realloc.c +++ b/src/syscalls/realloc.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef realloc void __attribute__((alloc_size(2))) *FTLrealloc(void *ptr_in, const size_t size, const char * file, const char * func, const int line) diff --git a/src/syscalls/recv.c b/src/syscalls/recv.c index c0e77ecf..0dce546d 100644 --- a/src/syscalls/recv.c +++ b/src/syscalls/recv.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include @@ -41,4 +41,4 @@ ssize_t FTLrecv(int sockfd, void *buf, size_t len, int flags, const char *file, errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index d40dfadf..b703ad04 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include #include @@ -45,4 +45,4 @@ ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockadd errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/select.c b/src/syscalls/select.c index 5eb07c9a..1907ba51 100644 --- a/src/syscalls/select.c +++ b/src/syscalls/select.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include @@ -41,4 +41,4 @@ int FTLselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, st errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index 4e7b4a8e..c3d0710b 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #include #include @@ -43,4 +43,4 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc errno = _errno; return ret; -} \ No newline at end of file +} diff --git a/src/syscalls/snprintf.c b/src/syscalls/snprintf.c index 699d942c..7384eec4 100644 --- a/src/syscalls/snprintf.c +++ b/src/syscalls/snprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) { diff --git a/src/syscalls/sprintf.c b/src/syscalls/sprintf.c index a6cc4094..c3ef0563 100644 --- a/src/syscalls/sprintf.c +++ b/src/syscalls/sprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) { diff --git a/src/syscalls/strdup.c b/src/syscalls/strdup.c index f83dde47..dc912f5d 100644 --- a/src/syscalls/strdup.c +++ b/src/syscalls/strdup.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" char* __attribute__((malloc)) FTLstrdup(const char *src, const char *file, const char *func, const int line) { @@ -35,4 +35,4 @@ char* __attribute__((malloc)) FTLstrdup(const char *src, const char *file, const dest[len] = '\0'; return dest; -} \ No newline at end of file +} diff --git a/src/syscalls/string.c b/src/syscalls/string.c index 88254e35..1d394252 100644 --- a/src/syscalls/string.c +++ b/src/syscalls/string.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef strlen size_t FTLstrlen(const char *s, const char *file, const char *func, const int line) diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index 3d77ebe7..02024a84 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -14,24 +14,24 @@ char *FTLstrdup(const char *src, const char *file, const char *func, const int line) __attribute__((malloc)); void *FTLcalloc(size_t n, size_t size, const char *file, const char *func, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *func, const int line) __attribute__((alloc_size(2))); -void FTLfree(void **ptr, const char*file, const char *func, const int line); +bool FTLfree(void *ptr, const char*file, const char *func, const int line); int FTLfallocate(const int fd, const off_t offset, const off_t len, const char *file, const char *func, const int line); // Interrupt-safe printing routines // printf() is derived from fprintf(stdout, ...) // vprintf() is derived from vfprintf(stdout, ...) -int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, ...) __attribute__ ((format (printf, 5, 6))); +int FTLvfprintf(FILE *stream, const char*file, const char *func, const int line, const char *format, va_list args) __attribute__ ((format (printf, 5, 0))); -int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, ...) __attribute__ ((format (printf, 5, 6))); +int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) __attribute__ ((format (printf, 5, 0))); -int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6))); -int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) __attribute__ ((format (gnu_printf, 5, 0))); +int FTLasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, ...) __attribute__ ((format (printf, 5, 6))); +int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) __attribute__ ((format (printf, 5, 0))); -int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (gnu_printf, 6, 7))); -int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (gnu_printf, 6, 0))); +int FTLsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, ...) __attribute__ ((format (printf, 6, 7))); +int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) __attribute__ ((format (printf, 6, 0))); // Interrupt-safe socket routines ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line); diff --git a/src/syscalls/vasprintf.c b/src/syscalls/vasprintf.c index 3ac340e6..d1a268e9 100644 --- a/src/syscalls/vasprintf.c +++ b/src/syscalls/vasprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef vasprintf int FTLvasprintf(const char *file, const char *func, const int line, char **buffer, const char *format, va_list args) diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index c7a82de6..516271dc 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" // itoa implementation using only static memory // taken from Kernighan and Ritchie's "The C Programming Language" diff --git a/src/syscalls/vsnprintf.c b/src/syscalls/vsnprintf.c index 4f4badfc..690d90f0 100644 --- a/src/syscalls/vsnprintf.c +++ b/src/syscalls/vsnprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef vsnprintf int FTLvsnprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const size_t maxlen, const char *format, va_list args) diff --git a/src/syscalls/vsprintf.c b/src/syscalls/vsprintf.c index cce0b35f..72aee733 100644 --- a/src/syscalls/vsprintf.c +++ b/src/syscalls/vsprintf.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef vsprintf int FTLvsprintf(const char *file, const char *func, const int line, char *__restrict__ buffer, const char *format, va_list args) diff --git a/src/syscalls/write.c b/src/syscalls/write.c index df62adda..bdb8eafc 100644 --- a/src/syscalls/write.c +++ b/src/syscalls/write.c @@ -8,9 +8,9 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" +#include "FTL.h" //#include "syscalls.h" is implicitly done in FTL.h -#include "../log.h" +#include "log.h" #undef write ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const char *func, const int line) @@ -50,4 +50,4 @@ ssize_t FTLwrite(int fd, const void *buf, size_t total, const char *file, const // Return number of written bytes return written; -} \ No newline at end of file +} diff --git a/src/timers.c b/src/timers.c index fe66d760..ddfddb0b 100644 --- a/src/timers.c +++ b/src/timers.c @@ -16,7 +16,7 @@ // set_blockingmode() #include "config/config.h" -struct timespec t0[NUMTIMERS]; +static struct timespec t0[NUMTIMERS]; void timer_start(const enum timers i) { @@ -66,7 +66,7 @@ void sleepms(const int milliseconds) } static double timer_delay = -1.0; -static bool timer_target_status; +static bool timer_target_status = true; void set_blockingmode_timer(double delay, bool target_status) { @@ -84,7 +84,7 @@ void get_blockingmode_timer(double *delay, bool *target_status) void *timer(void *val) { // Set thread name - prctl(PR_SET_NAME, "int.timer", 0, 0, 0); + prctl(PR_SET_NAME, thread_names[TIMER], 0, 0, 0); // Save timestamp as we do not want to store immediately // to the database @@ -105,9 +105,10 @@ void *timer(void *val) set_blockingstatus(timer_target_status); timer_delay = -1.0; } - sleepms(SLEEPING_TIME * 1000); + thread_sleepms(TIMER, SLEEPING_TIME * 1000); } + log_info("Terminating timer thread"); return NULL; } diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 47b9e5e1..ce5d4244 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -15,6 +15,9 @@ set(tools_sources dhcp-discover.h gravity-parseList.c gravity-parseList.h + netlink_consts.h + netlink.c + netlink.h ) add_library(tools OBJECT ${tools_sources}) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 680a90a3..923625a3 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -381,7 +381,7 @@ static void *arp_scan_iface(void *args) thread_data->dst_cidr = netmask_to_cidr(&thread_data->mask.sin_addr); // Get interface index - const int ifindex = if_nametoindex(iface); + const int ifindex = (int)if_nametoindex(iface); // Scan only interfaces with CIDR >= 24 if(thread_data->dst_cidr < 24 && !thread_data->scan_all) @@ -701,7 +701,7 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) { // Calculate progress (total number of scans / total number of addresses) // We add 1 to total_scans to avoid division by zero - const unsigned int new_progress = 100 * num_scans / (total_scans + 1); + const unsigned int new_progress = 100 * (unsigned int)(num_scans / (total_scans + 1)); if(new_progress > progress) { // Print progress diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 85e994b4..c68a74f9 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -54,15 +54,12 @@ // we scan for DHCP activity. #define MAXTHREADS 32 -// Probe DHCP servers responding to the broadcast address -#define PROBE_BCAST - // Should we generate test data for DHCP option 249? //#define TEST_OPT_249 // Global lock used by all threads static pthread_mutex_t lock; -static void __attribute__((format(gnu_printf, 1, 2))) printf_locked(const char *format, ...) +static void __attribute__((format(printf, 1, 2))) printf_locked(const char *format, ...) { va_list args; va_start(args, format); @@ -179,7 +176,7 @@ struct dhcp_packet_data unsigned char chaddr [MAX_DHCP_CHADDR_LENGTH]; // hardware address of this machine char sname [MAX_DHCP_SNAME_LENGTH]; // name of DHCP server char file [MAX_DHCP_FILE_LENGTH]; // boot file name (used for diskless booting?) - char options[MAX_DHCP_OPTIONS_LENGTH]; // options + unsigned char options[MAX_DHCP_OPTIONS_LENGTH]; // options }; // sends a DHCPDISCOVER message to the specified in an attempt to find DHCP servers @@ -219,7 +216,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i discover_packet.options[6] = 1; // DHCP message type code for DHCPDISCOVER // Place end option at the end of the options - discover_packet.options[7] = 255; + discover_packet.options[7] = (char)255; // Send the DHCPDISCOVER packet to the specified address struct sockaddr_in target = { 0 }; @@ -236,7 +233,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i printf_locked("DHCDISCOVER giaddr: %s\n", inet_ntoa(discover_packet.giaddr)); #endif // send the DHCPDISCOVER packet - const int bytes = sendto(sock, (char *)&discover_packet, sizeof(discover_packet), 0, (struct sockaddr *)&target, sizeof(target)); + const ssize_t bytes = sendto(sock, (char *)&discover_packet, sizeof(discover_packet), 0, (struct sockaddr *)&target, sizeof(target)); if(bytes < 0) { // strerror() returns "Required key not available" for ENOKEY @@ -250,7 +247,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i } #ifdef DEBUG - printf_locked("Sent %d bytes\n", bytes); + printf_locked("Sent %zu bytes\n", (size_t)bytes); #endif return true; } @@ -340,7 +337,7 @@ static void print_dhcp_offer(struct in_addr source, struct dhcp_packet_data *off // possible "(empty)" const size_t bufsiz = 4*optlen + 9; char *buffer = calloc(bufsiz, sizeof(char)); - binbuf_to_escaped_C_literal(&offer_packet->options[x], optlen, buffer, bufsiz); + binbuf_to_escaped_C_literal((char*)&offer_packet->options[x], optlen, buffer, bufsiz); printf("%s: \"%s\"\n", opttab[i].name, buffer); free(buffer); } @@ -428,7 +425,7 @@ static void print_dhcp_offer(struct in_addr source, struct dhcp_packet_data *off // chars per control character plus room for // possible "(empty)" char *buffer = calloc(4*optlen + 9, sizeof(char)); - binbuf_to_escaped_C_literal(&offer_packet->options[x], optlen, buffer, sizeof(buffer)); + binbuf_to_escaped_C_literal((char*)&offer_packet->options[x], optlen, buffer, sizeof(buffer)); printf("wpad-server: \"%s\"\n", buffer); free(buffer); } @@ -730,7 +727,7 @@ int run_dhcp_discover(void) pthread_attr_init(&attr); // Create processing/printfing lock - pthread_mutexattr_t lock_attr = {}; + pthread_mutexattr_t lock_attr; // Initialize the lock attributes pthread_mutexattr_init(&lock_attr); // Initialize the lock diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index de30f7ca..42a7c267 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -94,12 +94,6 @@ inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t if(domain[last_dot + 1] == '-' || domain[len - 1] == '-') return false; - // TLD length check - // The last label must be at least 2 characters long - // (len-1) because we start counting from zero - if((len - 1) - last_dot < 2) - return false; - return true; } diff --git a/src/tools/gravity-parseList.h b/src/tools/gravity-parseList.h index e9e54ac4..34ca169c 100644 --- a/src/tools/gravity-parseList.h +++ b/src/tools/gravity-parseList.h @@ -8,7 +8,12 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#ifndef GRAVITY_PARSELIST_H +#define GRAVITY_PARSELIST_H + #include "FTL.h" int gravity_parseList(const char *infile, const char *outfile, const char *adlistID, const bool checkOnly, const bool antigravity); bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool fqdn_only); + +#endif // GRAVITY_PARSELIST_H diff --git a/src/tools/netlink.c b/src/tools/netlink.c new file mode 100644 index 00000000..a0fe97b3 --- /dev/null +++ b/src/tools/netlink.c @@ -0,0 +1,1151 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Network implementation for netlink +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" +#include "netlink.h" +#include "netlink_consts.h" +#include "log.h" +#include +#include +#include +#include + +// defined in src/dnsmasq/rfc1035.c +extern int private_net(struct in_addr addr, int ban_localhost); + +static bool nlrequest(int fd, struct sockaddr_nl *sa, int nlmsg_type) +{ + char buf[BUFLEN] = { 0 }; + // Assemble the message according to the netlink protocol + struct nlmsghdr *nl; + nl = (struct nlmsghdr*)(void*)buf; + nl->nlmsg_flags = NLM_F_REQUEST | NLM_F_ROOT; + + if(nlmsg_type == RTM_GETADDR) + { + // Request address information + nl->nlmsg_len = NLMSG_LENGTH(sizeof(struct ifaddrmsg)); + + struct ifaddrmsg *ifa; + ifa = (struct ifaddrmsg*)NLMSG_DATA(nl); + ifa->ifa_family = AF_LOCAL; + } + else if(nlmsg_type == RTM_GETROUTE) + { + // Request route information + nl->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + + struct rtmsg *rt; + rt = (struct rtmsg*)NLMSG_DATA(nl); + rt->rtm_family = AF_LOCAL; + } + else if(nlmsg_type == RTM_GETLINK) + { + // Request link information + nl->nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + + struct ifinfomsg *link; + link = (struct ifinfomsg*)NLMSG_DATA(nl); + link->ifi_family = AF_UNSPEC; + } + nl->nlmsg_type = nlmsg_type; + + // Prepare struct msghdr for sending + struct iovec iov = { nl, nl->nlmsg_len }; + struct msghdr msg = { 0 }; + msg.msg_name = sa; + msg.msg_namelen = sizeof(*sa); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + // Send netlink message to kernel + return sendmsg(fd, &msg, 0) >= 0; +} + +static ssize_t nlgetmsg(int fd, struct sockaddr_nl *sa, void *buf, size_t len) +{ + struct iovec iov; + struct msghdr msg; + iov.iov_base = buf; + iov.iov_len = len; + + memset(&msg, 0, sizeof(msg)); + msg.msg_name = sa; + msg.msg_namelen = sizeof(*sa); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + return recvmsg(fd, &msg, 0); +} + +static int nlparsemsg_route(struct rtmsg *rt, void *buf, size_t len, cJSON *routes, const bool detailed) +{ + char ifname[IF_NAMESIZE]; + cJSON *route = cJSON_CreateObject(); + cJSON_AddNumberToObject(route, "table", rt->rtm_table); + cJSON_AddStringReferenceToObject(route, "family", family_name(rt->rtm_family)); + + // Print human-readable protocol + for(unsigned int i = 0; i < sizeof(rtprots)/sizeof(rtprots[0]); i++) + if (rtprots[i].flag == rt->rtm_protocol) + { + cJSON_AddStringReferenceToObject(route, "protocol", rtprots[i].name); + break; + } + // If the protocol is not found, add it as a number + if (cJSON_GetObjectItem(route, "protocol") == NULL) { + cJSON_AddNumberToObject(route, "protocol", rt->rtm_protocol); + } + + // Print human-readable scope + for(unsigned int i = 0; i < sizeof(rtscopes)/sizeof(rtscopes[0]); i++) + if (rtscopes[i].flag == rt->rtm_scope) + { + cJSON_AddStringReferenceToObject(route, "scope", rtscopes[i].name); + break; + } + // If the scope is not found, add it as a number + if (cJSON_GetObjectItem(route, "scope") == NULL) + cJSON_AddNumberToObject(route, "scope", rt->rtm_scope); + + // Print human-readable type + for(unsigned int i = 0; i < sizeof(rttypes)/sizeof(rttypes[0]); i++) + if (rttypes[i].flag == rt->rtm_type) + { + cJSON_AddStringReferenceToObject(route, "type", rttypes[i].name); + break; + } + // If the type is not found, add it as a number + if (cJSON_GetObjectItem(route, "type") == NULL) + cJSON_AddNumberToObject(route, "type", rt->rtm_type); + + // Add array of human-readable flags + cJSON *flags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(rtmflags)/sizeof(rtmflags[0]); i++) + if (rtmflags[i].flag & rt->rtm_flags) + cJSON_AddStringReferenceToArray(flags, rtmflags[i].name); + for(unsigned int i = 0; i < sizeof(rtnhflags)/sizeof(rtnhflags[0]); i++) + if (rtnhflags[i].flag & rt->rtm_flags) + cJSON_AddStringReferenceToArray(flags, rtnhflags[i].name); + cJSON_AddItemToObject(route, "flags", flags); + if(detailed) + cJSON_AddNumberToObject(route, "iflags", rt->rtm_flags); + + // Parse the route attributes + struct rtattr *rta = NULL; + static char ip[INET6_ADDRSTRLEN]; + for_each_rattr(rta, buf, len) + { + switch (rta->rta_type) + { + case RTA_DST: // route destination address + case RTA_SRC: // route source address + case RTA_GATEWAY: // gateway of the route + case RTA_PREFSRC: // preferred source address + case RTA_NEWDST: // change package destination address + inet_ntop(rt->rtm_family, RTA_DATA(rta), ip, INET6_ADDRSTRLEN); + cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ip); + break; + + case RTA_IIF: // incoming interface + case RTA_OIF: // outgoing interface + { + const uint32_t ifidx = *(uint32_t*)RTA_DATA(rta); + if_indextoname(ifidx, ifname); + cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ifname); + break; + } + + case RTA_FLOW: // route realm + case RTA_METRICS: // route metric + case RTA_MARK: // route mark + case RTA_EXPIRES: // route expires (in seconds) + case RTA_UID: // user id + case RTA_TTL_PROPAGATE: // propagate TTL + case RTA_IP_PROTO: // IP protocol + case RTA_SPORT: + case RTA_DPORT: + case RTA_NH_ID: + { + if(!detailed) + break; + const uint32_t number = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), number); + break; + } + + case RTA_TABLE: // routing table id + // Already added above + break; + + case RTA_PRIORITY: // route priority + case RTA_PREF: // route preference + { + const uint32_t num = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, rtaTypeToString(rta->rta_type), num); + break; + } + + case RTA_MULTIPATH: // multipath route + { + if(!detailed) + break; + struct rtnexthop *rtnh = (struct rtnexthop *) RTA_DATA (rta); + cJSON *multipath = cJSON_CreateObject(); + cJSON_AddNumberToObject(multipath, "len", rtnh->rtnh_len); // Length of struct + length of RTAs + + // Add array of human-readable nexthop flags + cJSON *nhflags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(rtnhflags)/sizeof(rtnhflags[0]); i++) + if (rtnhflags[i].flag & rtnh->rtnh_flags) + cJSON_AddStringReferenceToArray(nhflags, rtnhflags[i].name); + cJSON_AddItemToObject(route, "mflags", nhflags); + if(detailed) + cJSON_AddNumberToObject(route, "imflags", rtnh->rtnh_flags); + + cJSON_AddNumberToObject(multipath, "hops", rtnh->rtnh_hops); // Nexthop priority + if_indextoname(rtnh->rtnh_ifindex, ifname); + cJSON_AddStringToObject(multipath, "if", ifname); // Interface for this nexthop + cJSON_AddItemToObject(route, rtaTypeToString(rta->rta_type), multipath); + break; + } + + case RTA_VIA: // next hop address + { + struct rtvia *via = (struct rtvia*)RTA_DATA(rta); + inet_ntop(via->rtvia_family, &via->rtvia_addr, ip, INET6_ADDRSTRLEN); + cJSON_AddStringToObject(route, rtaTypeToString(rta->rta_type), ip); + break; + } + + case RTA_MFC_STATS: // multicast forwarding cache statistics + { + if(!detailed) + break; + struct rta_mfc_stats *mfc = (struct rta_mfc_stats*)RTA_DATA(rta); + cJSON_AddNumberToObject(route, "mfcs_packets", mfc->mfcs_packets); + cJSON_AddNumberToObject(route, "mfcs_bytes", mfc->mfcs_bytes); + cJSON_AddNumberToObject(route, "mfcs_wrong_if", mfc->mfcs_wrong_if); + break; + } + + case RTA_CACHEINFO: + { + if(!detailed) + break; + struct rta_cacheinfo *ci = (struct rta_cacheinfo*)RTA_DATA(rta); + // Get seconds the system is already up ("uptime") + struct timespec wall_clock; + clock_gettime(CLOCK_REALTIME, &wall_clock); + struct timespec boot_clock; + clock_gettime(CLOCK_BOOTTIME, &boot_clock); + const time_t delta_time = wall_clock.tv_sec - boot_clock.tv_sec; + cJSON_AddNumberToObject(route, "cstamp", delta_time + ci->rta_clntref); + cJSON_AddNumberToObject(route, "tstamp", delta_time + ci->rta_lastuse); + cJSON_AddNumberToObject(route, "expires", ci->rta_expires); + cJSON_AddNumberToObject(route, "error", ci->rta_error); + cJSON_AddNumberToObject(route, "used", ci->rta_used); + break; + } + + default: + { + // Unknown rta_type + // Add the rta_type as a number to an array of + // unknown types if in detailed mode + if(!detailed) + break; + + cJSON *unknown = cJSON_GetObjectItem(route, "unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(route, "unknown", unknown); + } + cJSON_AddNumberToArray(unknown, rta->rta_type); + break; + } + } + } + + // The default route is the one which does not have a "dst" attribute + if(cJSON_GetObjectItem(route, "dst") == NULL) + cJSON_AddStringToObject(route, "dst", "default"); + + cJSON_AddItemToArray(routes, route); + return 0; +} + +static int nlparsemsg_address(struct ifaddrmsg *ifa, void *buf, size_t len, cJSON *links, const bool detailed) +{ + cJSON *addr = cJSON_CreateObject(); + + // Add interface ID + if(detailed) + cJSON_AddNumberToObject(addr, "index", ifa->ifa_index); + + // Add family + cJSON_AddStringReferenceToObject(addr, "family", family_name(ifa->ifa_family)); + + // Print human-readable scope + for(unsigned int i = 0; i < sizeof(rtscopes)/sizeof(rtscopes[0]); i++) + if (rtscopes[i].flag == ifa->ifa_scope) + { + cJSON_AddStringReferenceToObject(addr, "scope", rtscopes[i].name); + break; + } + // If the scope is not found, add it as a number + if (cJSON_GetObjectItem(addr, "scope") == NULL) + cJSON_AddNumberToObject(addr, "scope", ifa->ifa_scope); + + // Add array of human-readable flags + cJSON *flags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(ifaf_flags)/sizeof(ifaf_flags[0]); i++) + if (ifaf_flags[i].flag & ifa->ifa_flags) + cJSON_AddStringReferenceToArray(flags, ifaf_flags[i].name); + cJSON_AddItemToObject(addr, "flags", flags); + + // Add prefix length + cJSON_AddNumberToObject(addr, "prefixlen", ifa->ifa_prefixlen); + + // Parse the address attributes + struct rtattr *rta = NULL; + char ifname[IF_NAMESIZE] = { 0 }; + for_each_rattr(rta, buf, len){ + switch(rta->rta_type) + { + case IFA_ADDRESS: + case IFA_LOCAL: + case IFA_BROADCAST: + case IFA_ANYCAST: + { + char ip[INET6_ADDRSTRLEN] = { 0 }; + inet_ntop(ifa->ifa_family, RTA_DATA(rta), ip, INET6_ADDRSTRLEN); + cJSON_AddStringToObject(addr, ifaTypeToString(rta->rta_type), ip); + + // Determine and add address type (GUA, ULA, LL, ...) + const char *type_str = "unknown"; + if(rta->rta_type == IFA_ADDRESS) + type_str = "address_type"; + else if(rta->rta_type == IFA_LOCAL) + type_str = "local_type"; + else if(rta->rta_type == IFA_BROADCAST) + type_str = "broadcast_type"; + else if(rta->rta_type == IFA_ANYCAST) + type_str = "anycast_type"; + + if(ifa->ifa_family == AF_INET6) + { + const struct in6_addr *in6 = (struct in6_addr*)RTA_DATA(rta); + if(IN6_IS_ADDR_UNSPECIFIED(in6)) + cJSON_AddStringToObject(addr, type_str, "unspecified"); + else if(IN6_IS_ADDR_LOOPBACK(in6)) + cJSON_AddStringToObject(addr, type_str, "loopback"); + else if(IN6_IS_ADDR_MULTICAST(in6)) + cJSON_AddStringToObject(addr, type_str, "multicast"); + else if(IN6_IS_ADDR_LINKLOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "link-local (LL)"); + else if(IN6_IS_ADDR_SITELOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "site-local (ULA)"); + else if(IN6_IS_ADDR_V4MAPPED(in6)) + cJSON_AddStringToObject(addr, type_str, "IPv4-mapped"); + else if(IN6_IS_ADDR_V4COMPAT(in6)) + cJSON_AddStringToObject(addr, type_str, "IPv4-compatible"); + else if(IN6_IS_ADDR_MC_NODELOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "node-local"); + else if(IN6_IS_ADDR_MC_LINKLOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "link-local (LL)"); + else if(IN6_IS_ADDR_MC_SITELOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "site-local (ULA)"); + else if(IN6_IS_ADDR_MC_ORGLOCAL(in6)) + cJSON_AddStringToObject(addr, type_str, "organization-local"); + else if(IN6_IS_ADDR_MC_GLOBAL(in6)) + cJSON_AddStringToObject(addr, type_str, "global (GUA)"); + else + { + uint8_t bytes[2]; + memcpy(&bytes, in6, 2); + // Global Unicast Address (2000::/3, RFC 4291) + if((bytes[0] & 0x70) == 0x20) + cJSON_AddStringToObject(addr, type_str, "global (GUA)"); + // Unique Local Address (fc00::/7, RFC 4193) + else if((bytes[0] & 0xfe) == 0xfc) + cJSON_AddStringToObject(addr, type_str, "site-local (ULA)"); + // Link Local Address (fe80::/10, RFC 4291) + else if((bytes[0] & 0xff) == 0xfe && (bytes[1] & 0x30) == 0) + cJSON_AddStringToObject(addr, type_str, "link-local (LL)"); + else + cJSON_AddStringToObject(addr, type_str, "unknown"); + } + } + else if(ifa->ifa_family == AF_INET) + { + const struct in_addr *in = (struct in_addr*)RTA_DATA(rta); + if(in->s_addr == INADDR_ANY) + cJSON_AddStringToObject(addr, type_str, "unspecified"); + else if(in->s_addr == INADDR_LOOPBACK || + (in->s_addr & htonl(0xff000000)) == htonl(0x7f000000)) + cJSON_AddStringToObject(addr, type_str, "loopback"); + else if((in->s_addr & htonl(0xf0000000)) == htonl(0xe0000000)) + cJSON_AddStringToObject(addr, type_str, "multicast"); + else if(private_net(*in, false)) + cJSON_AddStringToObject(addr, type_str, "private"); + else + cJSON_AddStringToObject(addr, type_str, "public"); + } + else + cJSON_AddStringToObject(addr, type_str, "unknown"); + break; + } + + case IFA_LABEL: + strncpy(ifname, (char*)RTA_DATA(rta), IF_NAMESIZE); + cJSON_AddStringToObject(addr, ifaTypeToString(rta->rta_type), (char*)RTA_DATA(rta)); + break; + + case IFA_CACHEINFO: + { + struct ifa_cacheinfo *ci = (struct ifa_cacheinfo*)RTA_DATA(rta); + cJSON_AddNumberToObject(addr, "prefered", ci->ifa_prefered); + cJSON_AddNumberToObject(addr, "valid", ci->ifa_valid); + // Get seconds the system is already up ("uptime") + struct timespec wall_clock; + clock_gettime(CLOCK_REALTIME, &wall_clock); + struct timespec boot_clock; + clock_gettime(CLOCK_BOOTTIME, &boot_clock); + const time_t delta_time = wall_clock.tv_sec - boot_clock.tv_sec; + cJSON_AddNumberToObject(addr, "cstamp", delta_time + 0.01*ci->cstamp); // created timestamp + cJSON_AddNumberToObject(addr, "tstamp", delta_time + 0.01*ci->tstamp); // updated timestamp + break; + } + + case IFA_FLAGS: + // Already added above, ignore this duplicate + break; + + case IFA_RT_PRIORITY: + { + if(!detailed) + break; + const uint32_t prio = *(uint32_t*)RTA_DATA(rta); + cJSON_AddStringToObject(addr, rtaTypeToString(rta->rta_type), rt_priority(prio)); + break; + } + + case IFA_TARGET_NETNSID: + { + if(!detailed) + break; + const uint32_t number = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(addr, ifaTypeToString(rta->rta_type), number); + break; + } + + default: + { + // Unknown rta_type + // Add the rta_type as a number to an array of + // unknown types if in detailed mode + if(!detailed) + break; + + cJSON *unknown = cJSON_GetObjectItem(addr, "unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(addr, "unknown", unknown); + } + cJSON_AddNumberToArray(unknown, rta->rta_type); + break; + } + } + } + + // Get the interface name if it is not already set + if(!ifname[0]) + if_indextoname(ifa->ifa_index, ifname); + + // Return early if the interface is not in the list of known interfaces + cJSON *ifobj = cJSON_GetObjectItem(links, ifname); + if(ifobj == NULL) + { + cJSON_Delete(addr); + return 0; + } + + // Ensure there is an addresses object for the interface + if(cJSON_GetObjectItem(ifobj, "addresses") == NULL) + cJSON_AddItemToObject(ifobj, "addresses", cJSON_CreateArray()); + + // Get the addresses object + cJSON *addrsobj = cJSON_GetObjectItem(ifobj, "addresses"); + + // Add the address to the object + cJSON_AddItemToArray(addrsobj, addr); + return 0; +} + +static int nlparsemsg_link(struct ifinfomsg *ifi, void *buf, size_t len, cJSON *links, const bool detailed) +{ + cJSON *link = cJSON_CreateObject(); + + // Add ifname at the top of the JSON object + char ifname[IF_NAMESIZE] = { 0 }; + if_indextoname(ifi->ifi_index, ifname); + cJSON_AddStringToObject(link, "name", ifname); + + // Add interface ID and family if detailed + if(detailed) + { + cJSON_AddNumberToObject(link, "index", ifi->ifi_index); + cJSON_AddStringReferenceToObject(link, "family", family_name(ifi->ifi_family)); + } + + // Get link speed (not available through netlink) + // (may not be possible, e.g., for WiFi devices with dynamic link speeds) + int speed = -1; + char fname[64]; + snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/speed", ifname); + FILE *f = fopen(fname, "r"); + if(f != NULL) + { + if(fscanf(f, "%i", &(speed)) != 1) + speed = -1; + fclose(f); + } + if(speed > -1) + cJSON_AddNumberToObject(link, "speed", speed); + else + cJSON_AddNullToObject(link, "speed"); + + // Add human-readable type + for(unsigned int i = 0; i < sizeof(iflatypes)/sizeof(iflatypes[0]); i++) + if (iflatypes[i].flag == ifi->ifi_type) + { + cJSON_AddStringReferenceToObject(link, "type", iflatypes[i].name); + break; + } + + // Add interface flags + cJSON *flags = cJSON_CreateArray(); + for(unsigned int i = 0; i < sizeof(iff_flags)/sizeof(iff_flags[0]); i++) + if (iff_flags[i].flag & ifi->ifi_flags) + cJSON_AddStringReferenceToArray(flags, iff_flags[i].name); + cJSON_AddItemToObject(link, "flags", flags); + + // Parse the link attributes + struct rtattr *rta = NULL; + cJSON *jstats = NULL, *jstats64 = NULL; + for_each_rattr(rta, buf, len){ + switch(rta->rta_type) + { + case IFLA_ADDRESS: + case IFLA_BROADCAST: + case IFLA_PERM_ADDRESS: + { + char mac[18]; + const unsigned char *addr = RTA_DATA(rta); + snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", + addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); + + // Addresses may be empty, so only add them if they are not + cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), mac); + break; + } + + case IFLA_IFNAME: + case IFLA_ALT_IFNAME: + case IFLA_PHYS_PORT_NAME: + case IFLA_QDISC: + case IFLA_PARENT_DEV_NAME: + case IFLA_PARENT_DEV_BUS_NAME: + { + if(!detailed) + break; + const char *string = (char*)RTA_DATA(rta); + cJSON_AddStringToObject(link, iflaTypeToString(rta->rta_type), string); + break; + } + + case IFLA_CARRIER: + case IFLA_PROTO_DOWN: + { + const uint8_t carrier = *(uint8_t*)RTA_DATA(rta); + cJSON_AddBoolToObject(link, iflaTypeToString(rta->rta_type), carrier == 0 ? false : true); + break; + } + + case IFLA_OPERSTATE: + for(unsigned int i = 0; i < sizeof(ifstates)/sizeof(ifstates[0]); i++) + if (ifstates[i].flag == *(unsigned int*)RTA_DATA(rta)) + { + cJSON_AddStringReferenceToObject(link, "state", ifstates[i].name); + break; + } + break; + + case IFLA_LINK: // Interface index + case IFLA_PHYS_PORT_ID: + case IFLA_PHYS_SWITCH_ID: + case IFLA_CARRIER_CHANGES: + case IFLA_MTU: + case IFLA_MASTER: + case IFLA_TXQLEN: + case IFLA_MAP: + case IFLA_WEIGHT: + case IFLA_LINKMODE: + case IFLA_COST: + case IFLA_PRIORITY: + case IFLA_GROUP: + case IFLA_NET_NS_PID: + case IFLA_NET_NS_FD: + case IFLA_EXT_MASK: + case IFLA_PROMISCUITY: + case IFLA_NUM_TX_QUEUES: + case IFLA_NUM_RX_QUEUES: + case IFLA_CARRIER_UP_COUNT: + case IFLA_CARRIER_DOWN_COUNT: + case IFLA_GSO_MAX_SEGS: + case IFLA_GSO_MAX_SIZE: + case IFLA_NEW_NETNSID: + case IFLA_MIN_MTU: + case IFLA_MAX_MTU: + case IFLA_LINK_NETNSID: + { + if(!detailed) + break; + const uint32_t number = *(uint32_t*)RTA_DATA(rta); + cJSON_AddNumberToObject(link, iflaTypeToString(rta->rta_type), number); + break; + } + + case IFLA_STATS: + { + // See description of the individual statistics + // below in the IFLA_STATS64 case + jstats = JSON_NEW_OBJECT(); + struct rtnl_link_stats *stats = (struct rtnl_link_stats*)RTA_DATA(rta); + { + // Warning: May be overflown if the interface has been up for a long time + // and has transferred a lot of data as 32 bits are used for the counters + // resulting in a maximum of 4 GiB. It is recommended to use the 64 bit + // counters if available. + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats->rx_bytes, &formatted_size); + cJSON *rx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(rx_bytes, "value", formatted_size); + cJSON_AddStringToObject(rx_bytes, "unit", prefix); + cJSON_AddItemToObject(jstats, "rx_bytes", rx_bytes); + } + { + // Warning: May be overflown if the interface has been up for a long time + // and has transferred a lot of data as 32 bits are used for the counters + // resulting in a maximum of 4 GiB. It is recommended to use the 64 bit + // counters if available. + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats->tx_bytes, &formatted_size); + cJSON *tx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(tx_bytes, "value", formatted_size); + cJSON_AddStringToObject(tx_bytes, "unit", prefix); + cJSON_AddItemToObject(jstats, "tx_bytes", tx_bytes); + } + cJSON_AddNumberToObject(jstats, "bits", 32); + if(!detailed) + break; + cJSON_AddNumberToObject(jstats, "rx_packets", stats->rx_packets); + cJSON_AddNumberToObject(jstats, "tx_packets", stats->tx_packets); + cJSON_AddNumberToObject(jstats, "rx_errors", stats->rx_errors); + cJSON_AddNumberToObject(jstats, "tx_errors", stats->tx_errors); + cJSON_AddNumberToObject(jstats, "rx_dropped", stats->rx_dropped); + cJSON_AddNumberToObject(jstats, "tx_dropped", stats->tx_dropped); + cJSON_AddNumberToObject(jstats, "multicast", stats->multicast); + cJSON_AddNumberToObject(jstats, "collisions", stats->collisions); + cJSON_AddNumberToObject(jstats, "rx_length_errors", stats->rx_length_errors); + cJSON_AddNumberToObject(jstats, "rx_over_errors", stats->rx_over_errors); + cJSON_AddNumberToObject(jstats, "rx_crc_errors", stats->rx_crc_errors); + cJSON_AddNumberToObject(jstats, "rx_frame_errors", stats->rx_frame_errors); + cJSON_AddNumberToObject(jstats, "rx_fifo_errors", stats->rx_fifo_errors); + cJSON_AddNumberToObject(jstats, "rx_missed_errors", stats->rx_missed_errors); + cJSON_AddNumberToObject(jstats, "tx_aborted_errors", stats->tx_aborted_errors); + cJSON_AddNumberToObject(jstats, "tx_carrier_errors", stats->tx_carrier_errors); + cJSON_AddNumberToObject(jstats, "tx_fifo_errors", stats->tx_fifo_errors); + cJSON_AddNumberToObject(jstats, "tx_heartbeat_errors", stats->tx_heartbeat_errors); + cJSON_AddNumberToObject(jstats, "tx_window_errors", stats->tx_window_errors); + cJSON_AddNumberToObject(jstats, "rx_compressed", stats->rx_compressed); + cJSON_AddNumberToObject(jstats, "tx_compressed", stats->tx_compressed); + cJSON_AddNumberToObject(jstats, "rx_nohandler", stats->rx_nohandler); + break; + } + + case IFLA_STATS64: + { + jstats64 = JSON_NEW_OBJECT(); + struct rtnl_link_stats64 *stats64 = (struct rtnl_link_stats64*)RTA_DATA(rta); + { + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats64->rx_bytes, &formatted_size); + cJSON *rx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(rx_bytes, "value", formatted_size); + cJSON_AddStringToObject(rx_bytes, "unit", prefix); + // @rx_bytes: Number of good received + // bytes, corresponding to @rx_packets. + cJSON_AddItemToObject(jstats64, "rx_bytes", rx_bytes); + } + { + char prefix[2] = { 0 }; + double formatted_size; + format_memory_size(prefix, stats64->tx_bytes, &formatted_size); + cJSON *tx_bytes = cJSON_CreateObject(); + cJSON_AddNumberToObject(tx_bytes, "value", formatted_size); + cJSON_AddStringToObject(tx_bytes, "unit", prefix); + // @tx_bytes: Number of transmitted bytes, + // corresponding to @tx_packets. + cJSON_AddItemToObject(jstats64, "tx_bytes", tx_bytes); + } + cJSON_AddNumberToObject(jstats64, "bits", 64); + if(!detailed) + break; + // @rx_packets: Number of good packets received + // by the interface. For hardware interfaces + // counts all good packets received from the + // device by the host, including packets which + // host had to drop at various stages of + // processing (even in the driver). + cJSON_AddNumberToObject(jstats64, "rx_packets", stats64->rx_packets); + // @tx_packets: Number of packets successfully + // transmitted. For hardware interfaces counts + // packets which host was able to successfully + // hand over to the device, which does not + // necessarily mean that packets had been + // successfully transmitted out of the device, + // only that device acknowledged it copied them + // out of host memory. + cJSON_AddNumberToObject(jstats64, "tx_packets", stats64->tx_packets); + // @rx_errors: Total number of bad packets + // received on this network device. This counter + // must include events counted by + // @rx_length_errors, @rx_crc_errors, + // @rx_frame_errors and other errors not + // otherwise counted. + cJSON_AddNumberToObject(jstats64, "rx_errors", stats64->rx_errors); + // @tx_errors: Total number of transmit + // problems. This counter must include events + // counter by @tx_aborted_errors, + // @tx_carrier_errors, @tx_fifo_errors, + // @tx_heartbeat_errors, + // @tx_window_errors and other errors not + // otherwise counted. + cJSON_AddNumberToObject(jstats64, "tx_errors", stats64->tx_errors); + // @rx_dropped: Number of packets received but + // not processed, e.g. due to lack of resources + // or unsupported protocol. For hardware + // interfaces this counter may include packets + // discarded due to L2 address filtering but + // should not include packets dropped by the + // device due to buffer exhaustion which are + // counted separately in + // @rx_missed_errors (since procfs folds those + // two counters together). + cJSON_AddNumberToObject(jstats64, "rx_dropped", stats64->rx_dropped); + // @tx_dropped: Number of packets dropped on + // their way to transmission, e.g. due to lack + // of resources. + cJSON_AddNumberToObject(jstats64, "tx_dropped", stats64->tx_dropped); + // @multicast: Multicast packets received. For + // hardware interfaces this statistic is + // commonly calculated at the device level + // (unlike @rx_packets) and therefore may + // include packets which did not reach the host. + cJSON_AddNumberToObject(jstats64, "multicast", stats64->multicast); + // @collisions: Number of collisions during + // packet transmissions. + cJSON_AddNumberToObject(jstats64, "collisions", stats64->collisions); + // @rx_length_errors: Number of packets dropped + // due to invalid length. Part of aggregate + // "frame" errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "rx_length_errors", stats64->rx_length_errors); + // @rx_over_errors: Receiver FIFO overflow event + // counter. Historically the count of overflow + // events. Such events may be reported in the + // receive descriptors or via interrupts, and + // may not correspond one-to-one with dropped + // packets. + // + // The recommended interpretation for high speed + // interfaces is - number of packets dropped + // because they did not fit into buffers + // provided by the host, e.g. packets larger + // than MTU or next buffer in the ring was not + // available for a scatter transfer. + // + // Part of aggregate "frame" errors in `/proc/net/dev`. + // + // This statistics was historically used + // interchangeably with @rx_fifo_errors. + // + // This statistic corresponds to hardware events + // and is not commonly used on software devices. + cJSON_AddNumberToObject(jstats64, "rx_over_errors", stats64->rx_over_errors); + // @rx_crc_errors: Number of packets received + // with a CRC error. Part of aggregate "frame" + // errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "rx_crc_errors", stats64->rx_crc_errors); + // @rx_frame_errors: Receiver frame alignment + // errors. Part of aggregate "frame" errors in + // `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "rx_frame_errors", stats64->rx_frame_errors); + // @rx_fifo_errors: Receiver FIFO error counter. + // + // Historically the count of overflow events. + // Those events may be reported in the receive + // descriptors or via interrupts, and may not + // correspond one-to-one with dropped packets. + // + // This statistics was used interchangeably with + // @rx_over_errors. Not recommended for use in + // drivers for high speed interfaces. + // + // This statistic is used on software devices, + // e.g. to count software packet queue overflow + // (can) or sequencing errors (GRE). + cJSON_AddNumberToObject(jstats64, "rx_fifo_errors", stats64->rx_fifo_errors); + // @rx_missed_errors: Count of packets missed by + // the host. Folded into the "drop" counter in + // `/proc/net/dev`. + // + // Counts number of packets dropped by the device due to lack + // of buffer space. This usually indicates that the host interface + // is slower than the network interface, or host is not keeping up + // with the receive packet rate. + // + // This statistic corresponds to hardware events and is not used + // on software devices. + cJSON_AddNumberToObject(jstats64, "rx_missed_errors", stats64->rx_missed_errors); + // @tx_aborted_errors: Part of aggregate + // "carrier" errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_aborted_errors", stats64->tx_aborted_errors); + // @tx_carrier_errors: Number of frame + // transmission errors due to loss of carrier + // during transmission. Part of aggregate + // "carrier" errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_carrier_errors", stats64->tx_carrier_errors); + // @tx_fifo_errors: Number of frame transmission + // errors due to device FIFO underrun / + // underflow. This condition occurs when the + // device begins transmission of a frame but is + // unable to deliver the entire frame to the + // transmitter in time for transmission. Part of + // aggregate "carrier" errors in + // `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_fifo_errors", stats64->tx_fifo_errors); + // @tx_heartbeat_errors: Number of Heartbeat / + // SQE Test errors for old half-duplex Ethernet. + // Part of aggregate "carrier" errors in + // `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_heartbeat_errors", stats64->tx_heartbeat_errors); + // @tx_window_errors: Number of frame + // transmission errors due to late collisions + // (for Ethernet - after the first 64B of + // transmission). Part of aggregate "carrier" + // errors in `/proc/net/dev`. + cJSON_AddNumberToObject(jstats64, "tx_window_errors", stats64->tx_window_errors); + // @rx_compressed: Number of received compressed + // packets. This counters is only meaningful for + // interfaces which support packet compression + // (e.g. CSLIP, PPP). + cJSON_AddNumberToObject(jstats64, "rx_compressed", stats64->rx_compressed); + // @tx_compressed: Number of transmitted + // compressed packets. This counters is only + // meaningful for interfaces which support + // packet compression (e.g. CSLIP, PPP). + cJSON_AddNumberToObject(jstats64, "tx_compressed", stats64->tx_compressed); + // @rx_nohandler: Number of packets received on + // the interface but dropped by the networking + // stack because the device is not designated to + // receive packets (e.g. backup link in a bond). + cJSON_AddNumberToObject(jstats64, "rx_nohandler", stats64->rx_nohandler); + break; + } + + case IFLA_LINKINFO: + { + if(!detailed) + break; + struct rtattr *nlinkinfo = NULL; + size_t nlen = RTA_PAYLOAD(rta); + void *ndata = RTA_DATA(rta); + for_each_rattr(nlinkinfo, ndata, nlen){ + switch(nlinkinfo->rta_type) + { + case IFLA_INFO_KIND: + cJSON_AddStringToObject(link, "link_kind", (char*)RTA_DATA(nlinkinfo)); + break; + case IFLA_INFO_SLAVE_KIND: + cJSON_AddStringToObject(link, "slave_kind", (char*)RTA_DATA(nlinkinfo)); + break; + case IFLA_INFO_DATA: + case IFLA_INFO_SLAVE_DATA: + // Needs a very complex + // disassembler, out of + // scope here + break; + default: + { + // Unknown rta_type + cJSON *unknown = cJSON_GetObjectItem(link, "linkinfo_unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(link, "linkinfo_unknown", unknown); + } + cJSON_AddNumberToArray(unknown, nlinkinfo->rta_type); + break; + } + } + } + break; + } + + case IFLA_VFINFO_LIST: + { + if(!detailed) + break; + struct rtattr *vfinfo = RTA_DATA(rta); + if (vfinfo->rta_type != IFLA_VF_INFO) + break; + + struct ifla_vf_mac *vf_mac; + struct ifla_vf_broadcast *vf_broadcast; + struct ifla_vf_tx_rate *vf_tx_rate; + struct rtattr *vf[IFLA_VF_MAX + 1] = {}; + + parse_rtattr_nested(vf, IFLA_VF_MAX, vfinfo); + + vf_mac = RTA_DATA(vf[IFLA_VF_MAC]); + vf_broadcast = RTA_DATA(vf[IFLA_VF_BROADCAST]); + vf_tx_rate = RTA_DATA(vf[IFLA_VF_TX_RATE]); + + if (vf[IFLA_VF_BROADCAST]) + { + char mac[18]; + snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", + vf_broadcast->broadcast[0], vf_broadcast->broadcast[1], + vf_broadcast->broadcast[2], vf_broadcast->broadcast[3], + vf_broadcast->broadcast[4], vf_broadcast->broadcast[5]); + cJSON_AddStringToObject(link, "vf_broadcast", mac); + } + if(vf[IFLA_VF_MAC]) + { + char mac[18]; + snprintf(mac, sizeof(mac), "%02x:%02x:%02x:%02x:%02x:%02x", + vf_mac->mac[0], vf_mac->mac[1], vf_mac->mac[2], + vf_mac->mac[3], vf_mac->mac[4], vf_mac->mac[5]); + cJSON_AddStringToObject(link, "vf_mac", mac); + } + if(vf[IFLA_VF_TX_RATE]) + { + cJSON_AddNumberToObject(link, "vf_tx_rate", vf_tx_rate->rate); + } + if(vf[IFLA_VF_LINK_STATE]) + { + const uint32_t link_state = *(uint32_t*)RTA_DATA(vf[IFLA_VF_LINK_STATE]); + cJSON_AddNumberToObject(link, "vf_link_state", link_state); + } + + break; + } + + case IFLA_EVENT: + { + if(!detailed) + break; + const uint32_t event = *(uint32_t*)RTA_DATA(rta); + for(unsigned int i = 0; i < sizeof(link_events)/sizeof(link_events[0]); i++) + if (link_events[i].flag == event) + { + cJSON_AddStringReferenceToObject(link, "event", link_events[i].name); + break; + } + if(cJSON_GetObjectItem(link, "event") == NULL) + cJSON_AddNumberToObject(link, "event", event); + break; + } + + case IFLA_AF_SPEC: + { + if(!detailed) + break; + struct rtattr *af_spec = RTA_DATA(rta); + struct rtattr *inet6_attr = parse_rtattr_one_nested(AF_INET6, af_spec); + if(!inet6_attr) + break; + + struct rtattr *tb[IFLA_INET6_MAX + 1]; + parse_rtattr_nested(tb, IFLA_INET6_MAX, inet6_attr); + + if(tb[IFLA_INET6_ADDR_GEN_MODE]) + { + const uint8_t mode = *(uint8_t*)RTA_DATA(tb[IFLA_INET6_ADDR_GEN_MODE]); + for(unsigned int i = 0; i < sizeof(addr_gen_modes)/sizeof(addr_gen_modes[0]); i++) + if (addr_gen_modes[i].flag == mode) + { + cJSON_AddStringReferenceToObject(link, "addr_gen_mode", addr_gen_modes[i].name); + break; + } + if(cJSON_GetObjectItem(link, "addr_gen_mode") == NULL) + cJSON_AddNumberToObject(link, "addr_gen_mode", mode); + } + cJSON *af_specs = cJSON_CreateArray(); + for(unsigned int i = 0; i < __IFLA_INET6_MAX; i++) + if(tb[i]) + { + cJSON *jaf_spec = cJSON_CreateObject(); + cJSON_AddNumberToObject(jaf_spec, "type", i); + cJSON_AddNumberToObject(jaf_spec, "len", RTA_PAYLOAD(tb[i])); + cJSON_AddItemToArray(af_specs, jaf_spec); + } + cJSON_AddItemToObject(link, "af_specs", af_specs); + break; + } + + case IFLA_XDP: + // Parsing XDP needs a full BPF program + // disassembler which is clearly out of scope + // here + break; + + default: + { + // Unknown rta_type + // Add the rta_type as a number to an array of + // unknown types if in detailed mode + if(!detailed) + break; + + cJSON *unknown = cJSON_GetObjectItem(link, "unknown"); + if(unknown == NULL) + { + unknown = cJSON_CreateArray(); + cJSON_AddItemToObject(link, "unknown", unknown); + } + cJSON_AddNumberToArray(unknown, rta->rta_type); + break; + } + } + } + + // Add 64 bit statistics if available and delete the 32 bit statistics + if(jstats64) + { + cJSON_AddItemToObject(link, "stats", jstats64); + if(jstats) + { + cJSON_Delete(jstats); + jstats = NULL; + } + } + // otherwise add the 32 bit statistics (64 has never been allocated) + else if(jstats) + cJSON_AddItemToObject(link, "stats", jstats); + + // Add the link to the object + cJSON_AddItemToObject(links, ifname, link); + + return 0; +} + +static uint32_t parse_nl_msg(void *buf, size_t len, cJSON *json, const bool detailed) +{ + struct nlmsghdr *nl = NULL; + for_each_nlmsg(nl, buf, len) + { + if (nl->nlmsg_type == NLMSG_ERROR) + { + log_info("error"); + return -1; + } + else if (nl->nlmsg_type == RTM_NEWROUTE) + { + struct rtmsg *rt; + rt = (struct rtmsg*)NLMSG_DATA(nl); + nlparsemsg_route(rt, RTM_RTA(rt), RTM_PAYLOAD(nl), json, detailed); + continue; + } + else if (nl->nlmsg_type == RTM_NEWADDR) + { + struct ifaddrmsg *ifa; + ifa = (struct ifaddrmsg*)NLMSG_DATA(nl); + nlparsemsg_address(ifa, IFA_RTA(ifa), IFA_PAYLOAD(nl), json, detailed); + continue; + } + else if (nl->nlmsg_type == RTM_NEWLINK) + { + struct ifinfomsg *ifi; + ifi = (struct ifinfomsg*)NLMSG_DATA(nl); + nlparsemsg_link(ifi, IFLA_RTA(ifi), IFLA_PAYLOAD(nl), json, detailed); + continue; + } + else + { + log_err("unknown nlmsg_type: %d", nl->nlmsg_type); + } + + } + return nl->nlmsg_type; +} + +static int nlquery(const int type, cJSON *json, const bool detailed) +{ + // First of all, we need to create a socket with the AF_NETLINK domain + const int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if(fd < 0) + { + log_info("socket error: %s", strerror(errno)); + return -1; + } + + struct sockaddr_nl sa; + memset(&sa, 0, sizeof(sa)); + sa.nl_family = AF_NETLINK; + + if(!nlrequest(fd, &sa, type)) + { + log_info("nlrequest error: %s", strerror(errno)); + return -1; + } + + uint32_t nl_msg_type; + do { + char buf[BUFLEN]; + ssize_t len = nlgetmsg(fd, &sa, buf, BUFLEN); + nl_msg_type = parse_nl_msg(buf, len, json, detailed); + } while (nl_msg_type != NLMSG_DONE && nl_msg_type != NLMSG_ERROR); + + return 0; + +} + +bool nlroutes(cJSON *routes, const bool detailed) +{ + return nlquery(RTM_GETROUTE, routes, detailed); +} + +bool nladdrs(cJSON *interfaces, const bool detailed) +{ + return nlquery(RTM_GETADDR, interfaces, detailed); +} + +bool nllinks(cJSON *interfaces, const bool detailed) +{ + return nlquery(RTM_GETLINK, interfaces, detailed); +} diff --git a/src/tools/netlink.h b/src/tools/netlink.h new file mode 100644 index 00000000..1d94f241 --- /dev/null +++ b/src/tools/netlink.h @@ -0,0 +1,53 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Netlink prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef NETLINK_H +#define NETLINK_H + +#include +#include "webserver/cJSON/cJSON.h" +#include "webserver/json_macros.h" + +// ICMPV6_PREF_LOW, etc. +#include +#include +// IFF_UP, etc. +#include +#include +#include +#include + +bool nlroutes(cJSON *routes, const bool detailed); +bool nladdrs(cJSON *interfaces, const bool detailed); +bool nllinks(cJSON *interfaces, const bool detailed); + + +#define BUFLEN 4096 + +#define for_each_nlmsg(n, buf, len) \ + for (n = (struct nlmsghdr*)buf; \ + NLMSG_OK(n, (uint32_t)len) && n->nlmsg_type != NLMSG_DONE; \ + n = NLMSG_NEXT(n, len)) + +#define for_each_rattr(n, buf, len) \ + for (n = (struct rtattr*)buf; RTA_OK(n, len); n = RTA_NEXT(n, len)) + +struct flag_names { + uint32_t flag; + const char *name; +}; + +// Manually taken from kernel source code in include/net/ipv6.h +#define IFA_GLOBAL 0x0000U +#define IFA_HOST 0x0010U +#define IFA_LINK 0x0020U +#define IFA_SITE 0x0040U +#define IFA_COMPATv4 0x0080U + +#endif // NETLINK_H diff --git a/src/tools/netlink_consts.h b/src/tools/netlink_consts.h new file mode 100644 index 00000000..f6a5e95f --- /dev/null +++ b/src/tools/netlink_consts.h @@ -0,0 +1,601 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2024 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Netlink constants +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "netlink.h" + +static struct flag_names iflatypes[] = +{ + { ARPHRD_NETROM, "netrom" }, + { ARPHRD_ETHER, "ether" }, + { ARPHRD_EETHER, "eether" }, + { ARPHRD_AX25, "ax25" }, + { ARPHRD_PRONET, "pronet" }, + { ARPHRD_CHAOS, "chaos" }, + { ARPHRD_IEEE802, "ieee802" }, + { ARPHRD_ARCNET, "arcnet" }, + { ARPHRD_APPLETLK, "appletlk" }, + { ARPHRD_DLCI, "dlci" }, + { ARPHRD_ATM, "atm" }, + { ARPHRD_METRICOM, "metricom" }, + { ARPHRD_IEEE1394, "ieee1394" }, + { ARPHRD_EUI64, "eui64" }, + { ARPHRD_INFINIBAND, "infiniband" }, + { ARPHRD_SLIP, "slip" }, + { ARPHRD_CSLIP, "cslip" }, + { ARPHRD_SLIP6, "slip6" }, + { ARPHRD_CSLIP6, "cslip6" }, + { ARPHRD_RSRVD, "rsrvd" }, + { ARPHRD_ADAPT, "adapt" }, + { ARPHRD_ROSE, "rose" }, + { ARPHRD_X25, "x25" }, + { ARPHRD_HWX25, "hwx25" }, + { ARPHRD_CAN, "can" }, + { ARPHRD_MCTP, "mctp" }, + { ARPHRD_PPP, "ppp" }, + { ARPHRD_CISCO, "cisco" }, + { ARPHRD_HDLC, "hdlc" }, + { ARPHRD_CISCO, "cisco" }, + { ARPHRD_LAPB, "lapb" }, + { ARPHRD_DDCMP, "ddcmp" }, + { ARPHRD_RAWHDLC, "rawhdlc" }, + { ARPHRD_RAWIP, "rawip" }, + { ARPHRD_TUNNEL, "tunnel" }, + { ARPHRD_TUNNEL6, "tunnel6" }, + { ARPHRD_FRAD, "frad" }, + { ARPHRD_SKIP, "skip" }, + { ARPHRD_LOOPBACK, "loopback" }, + { ARPHRD_LOCALTLK, "localtlk" }, + { ARPHRD_FDDI, "fddi" }, + { ARPHRD_BIF, "bif" }, + { ARPHRD_SIT, "sit" }, + { ARPHRD_IPDDP, "ipddp" }, + { ARPHRD_IPGRE, "ipgre" }, + { ARPHRD_PIMREG, "pimreg" }, + { ARPHRD_HIPPI, "hippi" }, + { ARPHRD_ASH, "ash" }, + { ARPHRD_ECONET, "econet" }, + { ARPHRD_IRDA, "irda" }, + { ARPHRD_FCPP, "fcpp" }, + { ARPHRD_FCAL, "fcal" }, + { ARPHRD_FCPL, "fcpl" }, + { ARPHRD_FCFABRIC, "fcfabric" }, + { ARPHRD_IEEE802_TR, "ieee802_tr" }, + { ARPHRD_IEEE80211, "ieee80211" }, + { ARPHRD_IEEE80211_PRISM, "ieee80211_prism" }, + { ARPHRD_IEEE80211_RADIOTAP, "ieee80211_radiotap" }, + { ARPHRD_IEEE802154, "ieee802154" }, + { ARPHRD_IEEE802154_MONITOR, "ieee802154_monitor" }, + { ARPHRD_PHONET, "phonet" }, + { ARPHRD_PHONET_PIPE, "phonet_pipe" }, + { ARPHRD_CAIF, "caif" }, + { ARPHRD_IP6GRE, "ip6gre" }, + { ARPHRD_NETLINK, "netlink" }, + { ARPHRD_6LOWPAN, "6lowpan" }, + { ARPHRD_VSOCKMON, "vsockmon" }, + { ARPHRD_VOID, "void" }, + { ARPHRD_NONE, "none" }, +}; + +static struct flag_names ifaf_flags[] = { + { IFA_F_SECONDARY, "secondary" }, + { IFA_F_TEMPORARY, "temporary" }, + { IFA_F_NODAD, "nodad" }, + { IFA_F_OPTIMISTIC, "optimistic" }, + { IFA_F_DADFAILED, "dadfailed" }, + { IFA_F_HOMEADDRESS, "homeaddress" }, + { IFA_F_DEPRECATED, "deprecated" }, + { IFA_F_TENTATIVE, "tentative" }, + { IFA_F_PERMANENT, "permanent" }, + { IFA_F_MANAGETEMPADDR, "managetempaddr" }, + { IFA_F_NOPREFIXROUTE, "noprefixroute" }, + { IFA_F_MCAUTOJOIN, "mcautojoin" }, + { IFA_F_STABLE_PRIVACY, "stable_privacy" }, +}; + +static struct flag_names iff_flags[] = { + { IFF_UP, "up" }, + { IFF_BROADCAST, "broadcast" }, + { IFF_DEBUG, "debug" }, + { IFF_LOOPBACK, "loopback" }, + { IFF_POINTOPOINT, "pointopoint" }, + { IFF_NOTRAILERS, "notrailers" }, + { IFF_RUNNING, "running" }, + { IFF_NOARP, "noarp" }, + { IFF_PROMISC, "promisc" }, + { IFF_ALLMULTI, "allmulti" }, + { IFF_MASTER, "master" }, + { IFF_SLAVE, "slave" }, + { IFF_MULTICAST, "multicast" }, + { IFF_PORTSEL, "portsel" }, + { IFF_AUTOMEDIA, "automedia" }, + { IFF_DYNAMIC, "dynamic" }, +#ifdef IFF_LOWER_UP + { IFF_LOWER_UP, "lower_up" }, +#endif +#ifdef IFF_DORMANT + { IFF_DORMANT, "dormant" }, +#endif +#ifdef IFF_ECHO + { IFF_ECHO, "echo" }, +#endif +}; + +static struct flag_names rtprots[] = { + { RTPROT_UNSPEC, "unspec" }, + { RTPROT_REDIRECT, "redirect" }, + { RTPROT_KERNEL, "kernel" }, + { RTPROT_BOOT, "boot" }, + { RTPROT_STATIC, "static" }, + { RTPROT_GATED, "gated" }, + { RTPROT_RA, "ra" }, + { RTPROT_MRT, "mrt" }, + { RTPROT_ZEBRA, "zebra" }, + { RTPROT_BIRD, "bird" }, + { RTPROT_DNROUTED, "dnrouted" }, + { RTPROT_XORP, "xorp" }, + { RTPROT_NTK, "ntk" }, + { RTPROT_DHCP, "dhcp" }, + { RTPROT_MROUTED, "mrouted" }, + { RTPROT_KEEPALIVED, "keepalived" }, + { RTPROT_BABEL, "babel" }, + { RTPROT_OPENR, "openr" }, + { RTPROT_BGP, "bgp" }, + { RTPROT_ISIS, "isis" }, + { RTPROT_OSPF, "ospf" }, + { RTPROT_RIP, "rip" }, + { RTPROT_EIGRP, "eigrp" }, +}; + +static struct flag_names rtscopes[] = { + { RT_SCOPE_UNIVERSE, "universe" }, + { RT_SCOPE_SITE, "site" }, + { RT_SCOPE_LINK, "link" }, + { RT_SCOPE_HOST, "host" }, + { RT_SCOPE_NOWHERE, "nowhere" }, +}; + +static struct flag_names rttypes[] = { + { RTN_UNSPEC, "unspec" }, + { RTN_UNICAST, "unicast" }, + { RTN_LOCAL, "local" }, + { RTN_BROADCAST, "broadcast" }, + { RTN_ANYCAST, "anycast" }, + { RTN_MULTICAST, "multicast" }, + { RTN_BLACKHOLE, "blackhole" }, + { RTN_UNREACHABLE, "unreachable" }, + { RTN_PROHIBIT, "prohibit" }, + { RTN_THROW, "throw" }, + { RTN_NAT, "nat" }, + { RTN_XRESOLVE, "xresolve" }, +}; + +static struct flag_names rtmflags[] = { + { RTM_F_NOTIFY, "notify" }, + { RTM_F_CLONED, "cloned" }, + { RTM_F_EQUALIZE, "equalize" }, + { RTM_F_PREFIX, "prefix" }, + { RTM_F_LOOKUP_TABLE, "lookup_table" }, + { RTM_F_FIB_MATCH, "fib_match" }, + { RTM_F_OFFLOAD, "offload" }, + { RTM_F_TRAP, "trap" }, + { RTM_F_OFFLOAD_FAILED, "offload_failed" }, +}; + +static struct flag_names rtnhflags[] = { + { RTNH_F_DEAD, "dead" }, + { RTNH_F_PERVASIVE, "pervasive" }, + { RTNH_F_ONLINK, "onlink" }, + { RTNH_F_OFFLOAD, "offload" }, + { RTNH_F_LINKDOWN, "linkdown" }, + { RTNH_F_UNRESOLVED, "unresolved" }, + { RTNH_F_TRAP, "trap" }, +}; + +static struct flag_names ifstates[] = { + { IF_OPER_UNKNOWN, "unknown" }, + { IF_OPER_NOTPRESENT, "notpresent" }, + { IF_OPER_DOWN, "down" }, + { IF_OPER_LOWERLAYERDOWN, "lower_layer_down" }, + { IF_OPER_TESTING, "testing" }, + { IF_OPER_DORMANT, "dormant" }, + { IF_OPER_UP, "up" }, +}; + +static struct flag_names link_events[] = { + { IFLA_EVENT_NONE, "none" }, + { IFLA_EVENT_REBOOT, "reboot" }, + { IFLA_EVENT_FEATURES, "feature change" }, + { IFLA_EVENT_BONDING_FAILOVER, "bonding failover" }, + { IFLA_EVENT_NOTIFY_PEERS, "notify peers" }, + { IFLA_EVENT_IGMP_RESEND, "resend igmp" }, + { IFLA_EVENT_BONDING_OPTIONS, "bonding option" }, +}; + +static struct flag_names addr_gen_modes[] = { + { IN6_ADDR_GEN_MODE_EUI64, "eui64" }, + { IN6_ADDR_GEN_MODE_NONE, "none" }, + { IN6_ADDR_GEN_MODE_STABLE_PRIVACY, "stable_secret" }, + { IN6_ADDR_GEN_MODE_RANDOM, "random" }, +}; + +static const char *__attribute__ ((const)) rtaTypeToString(const int rta_type) +{ + switch (rta_type) { + case RTA_UNSPEC: + return "unspec"; + case RTA_DST: + return "dst"; + case RTA_SRC: + return "src"; + case RTA_IIF: + return "iif"; + case RTA_OIF: + return "oif"; + case RTA_GATEWAY: + return "gateway"; + case RTA_PRIORITY: + return "priority"; + case RTA_PREFSRC: + return "prefsrc"; + case RTA_METRICS: + return "metrics"; + case RTA_MULTIPATH: + return "multipath"; + case RTA_PROTOINFO: + return "protoinfo"; + case RTA_FLOW: + return "flow"; + case RTA_CACHEINFO: + return "cacheinfo"; + case RTA_SESSION: + return "session"; + case RTA_MP_ALGO: + return "mp_algo"; + case RTA_TABLE: + return "table"; + case RTA_MARK: + return "mark"; + case RTA_MFC_STATS: + return "mfc_stats"; + case RTA_VIA: + return "via"; + case RTA_NEWDST: + return "newdst"; + case RTA_PREF: + return "pref"; + case RTA_ENCAP_TYPE: + return "encap_type"; + case RTA_ENCAP: + return "encap"; + case RTA_EXPIRES: + return "expires"; + case RTA_PAD: + return "pad"; + case RTA_UID: + return "uid"; + case RTA_TTL_PROPAGATE: + return "ttl_propagate"; + case RTA_IP_PROTO: + return "ip_proto"; + case RTA_SPORT: + return "sport"; + case RTA_DPORT: + return "dport"; + case RTA_NH_ID: + return "nh_id"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) ifaTypeToString(const int ifa_type) +{ + switch (ifa_type) { + case IFA_ADDRESS: + return "address"; + case IFA_LOCAL: + return "local"; + case IFA_LABEL: + return "label"; + case IFA_BROADCAST: + return "broadcast"; + case IFA_ANYCAST: + return "anycast"; + case IFA_CACHEINFO: + return "cacheinfo"; + case IFA_MULTICAST: + return "multicast"; + case IFA_FLAGS: + return "flags"; + case IFA_RT_PRIORITY: + return "rt_priority"; + case IFA_TARGET_NETNSID: + return "target_netnsid"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) iflaTypeToString(const int ifla_type) +{ + switch (ifla_type) + { + case IFLA_UNSPEC: + return "unspec"; + case IFLA_ADDRESS: + return "address"; + case IFLA_BROADCAST: + return "broadcast"; + case IFLA_IFNAME: + return "ifname"; + case IFLA_MTU: + return "mtu"; + case IFLA_LINK: + return "link"; + case IFLA_QDISC: + return "qdisc"; + case IFLA_STATS: + return "stats"; + case IFLA_COST: + return "cost"; + case IFLA_PRIORITY: + return "priority"; + case IFLA_MASTER: + return "master"; + case IFLA_WIRELESS: + return "wireless"; + case IFLA_PROTINFO: + return "protinfo"; + case IFLA_TXQLEN: + return "txqlen"; + case IFLA_MAP: + return "map"; + case IFLA_WEIGHT: + return "weight"; + case IFLA_OPERSTATE: + return "operstate"; + case IFLA_LINKMODE: + return "linkmode"; + case IFLA_LINKINFO: + return "linkinfo"; + case IFLA_NET_NS_FD: + return "net_ns_fd"; + case IFLA_IFALIAS: + return "ifalias"; + case IFLA_NUM_VF: + return "num_vf"; + case IFLA_VFINFO_LIST: + return "vfinfo_list"; + case IFLA_STATS64: + return "stats64"; + case IFLA_VF_PORTS: + return "vf_ports"; + case IFLA_PORT_SELF: + return "port_self"; + case IFLA_AF_SPEC: + return "af_spec"; + case IFLA_GROUP: + return "group"; + case IFLA_NET_NS_PID: + return "net_ns_pid"; + case IFLA_EXT_MASK: + return "ext_mask"; + case IFLA_PROMISCUITY: + return "promiscuity"; + case IFLA_NUM_TX_QUEUES: + return "num_tx_queues"; + case IFLA_NUM_RX_QUEUES: + return "num_rx_queues"; + case IFLA_CARRIER: + return "carrier"; + case IFLA_PHYS_PORT_ID: + return "phys_port_id"; + case IFLA_CARRIER_CHANGES: + return "carrier_changes"; + case IFLA_PHYS_SWITCH_ID: + return "phys_switch_id"; + case IFLA_LINK_NETNSID: + return "link_netnsid"; + case IFLA_PHYS_PORT_NAME: + return "phys_port_name"; + case IFLA_PROTO_DOWN: + return "proto_down"; + case IFLA_GSO_MAX_SEGS: + return "gso_max_segs"; + case IFLA_GSO_MAX_SIZE: + return "gso_max_size"; + case IFLA_PAD: + return "pad"; + case IFLA_XDP: + return "xdp"; + case IFLA_EVENT: + return "event"; + case IFLA_NEW_NETNSID: + return "new_netnsid"; + case IFLA_IF_NETNSID: + return "if_netnsid"; + case IFLA_CARRIER_UP_COUNT: + return "carrier_up_count"; + case IFLA_CARRIER_DOWN_COUNT: + return "carrier_down_count"; + case IFLA_NEW_IFINDEX: + return "new_ifindex"; + case IFLA_MIN_MTU: + return "min_mtu"; + case IFLA_MAX_MTU: + return "max_mtu"; + case IFLA_PROP_LIST: + return "prop_list"; + case IFLA_ALT_IFNAME: + return "alt_ifname"; + case IFLA_PERM_ADDRESS: + return "perm_address"; + case IFLA_PROTO_DOWN_REASON: + return "proto_down_reason"; + case IFLA_PARENT_DEV_NAME: + return "parent_dev_name"; + case IFLA_PARENT_DEV_BUS_NAME: + return "parent_dev_bus_name"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) rt_priority(const uint32_t pref) +{ + switch (pref) { + case ICMPV6_ROUTER_PREF_HIGH: + return "high"; + case ICMPV6_ROUTER_PREF_MEDIUM: + return "medium"; + case ICMPV6_ROUTER_PREF_LOW: + return "low"; + case ICMPV6_ROUTER_PREF_INVALID: + return "invalid"; + default: + return "unknown"; + } +} + +static const char *__attribute__ ((const)) family_name(int family) +{ + switch(family) + { + case PF_UNSPEC: + return "unspec"; + case PF_LOCAL: + return "local"; + case PF_INET: + return "inet"; + case PF_AX25: + return "ax25"; + case PF_IPX: + return "ipx"; + case PF_APPLETALK: + return "appletalk"; + case PF_NETROM: + return "netrom"; + case PF_BRIDGE: + return "bridge"; + case PF_ATMPVC: + return "atmpvc"; + case PF_X25: + return "x25"; + case PF_INET6: + return "inet6"; + case PF_ROSE: + return "rose"; + case PF_DECnet: + return "decnet"; + case PF_NETBEUI: + return "netbeui"; + case PF_SECURITY: + return "security"; + case PF_KEY: + return "key"; + case PF_NETLINK: + return "netlink"; + case PF_PACKET: + return "packet"; + case PF_ASH: + return "ash"; + case PF_ECONET: + return "econet"; + case PF_ATMSVC: + return "atmsvc"; + case PF_RDS: + return "rds"; + case PF_SNA: + return "sna"; + case PF_IRDA: + return "irda"; + case PF_PPPOX: + return "pppox"; + case PF_WANPIPE: + return "wanpipe"; + case PF_LLC: + return "llc"; + case PF_IB: + return "ib"; + case PF_MPLS: + return "mpls"; + case PF_CAN: + return "can"; + case PF_TIPC: + return "tipc"; + case PF_BLUETOOTH: + return "bluetooth"; + case PF_IUCV: + return "iucv"; + case PF_RXRPC: + return "rxrpc"; + case PF_ISDN: + return "isdn"; + case PF_PHONET: + return "phonet"; + case PF_IEEE802154: + return "ieee802154"; + case PF_CAIF: + return "caif"; + case PF_ALG: + return "alg"; + case PF_NFC: + return "nfc"; + case PF_VSOCK: + return "vsock"; + case PF_KCM: + return "kcm"; + case PF_QIPCRTR: + return "qipcrtr"; + case PF_SMC: + return "smc"; + case PF_XDP: + return "xdp"; +#ifdef PF_MCTP + // 2024-July: defined by glibc but not musl + case PF_MCTP: + return "mctp"; +#endif + default: + return "unknown"; + } +} + +// Taken from https://github.com/Gandi/packet-journey/blob/master/lib/libnetlink/netlink.c +#define parse_rtattr_nested(tb, max, rta) \ + (parse_rtattr_flags((tb), (max), RTA_DATA(rta), RTA_PAYLOAD(rta), 0)) +#define parse_rtattr_one_nested(type, rta) \ + (parse_rtattr_one(type, RTA_DATA(rta), RTA_PAYLOAD(rta))) + +static int parse_rtattr_flags(struct rtattr *tb[], int max, + struct rtattr *rta, int len, + unsigned short flags) +{ + unsigned short type; + + memset(tb, 0, sizeof(struct rtattr *) * (max + 1)); + while (RTA_OK(rta, len)) { + type = rta->rta_type & ~flags; + if ((type <= max) && (!tb[type])) + tb[type] = rta; + rta = RTA_NEXT(rta, len); + } + return 0; +} + +static struct rtattr * __attribute__((pure)) parse_rtattr_one(int type, struct rtattr *rta, int len) +{ + while (RTA_OK(rta, len)) { + if (rta->rta_type == type) + return rta; + rta = RTA_NEXT(rta, len); + } + return NULL; +} diff --git a/src/tre-regex/CMakeLists.txt b/src/tre-regex/CMakeLists.txt index f4a8ba96..cdf8ec2e 100644 --- a/src/tre-regex/CMakeLists.txt +++ b/src/tre-regex/CMakeLists.txt @@ -27,4 +27,6 @@ set(sources ) add_library(tre-regex OBJECT ${sources}) -target_compile_options(tre-regex PRIVATE -Wno-maybe-uninitialized -Wno-unused-value -Wno-empty-body) +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_compile_options(tre-regex PRIVATE -Wno-maybe-uninitialized -Wno-unused-value -Wno-empty-body) +endif() diff --git a/src/tre-regex/tre-config.h b/src/tre-regex/tre-config.h index c93e539c..fdcac795 100644 --- a/src/tre-regex/tre-config.h +++ b/src/tre-regex/tre-config.h @@ -10,17 +10,17 @@ /* #undef C_ALLOCA */ /* Define to 1 if you have `alloca', as a function or macro. */ -#define HAVE_ALLOCA 1 +#define HAVE_ALLOCA 0 /* Define to 1 if you have and it should be used (not on Ultrix). */ -#define HAVE_ALLOCA_H 1 +#define HAVE_ALLOCA_H 0 /* Define if the GNU gettext() function is already present or preinstalled. */ /* #define HAVE_GETTEXT 1 */ /* Define to 1 if you have the `isascii' function. */ -#define HAVE_ISASCII 1 +//#define HAVE_ISASCII 1 /* Define to 1 if you have the `isblank' function. */ #define HAVE_ISBLANK 1 @@ -72,7 +72,7 @@ /* Define if you want TRE to use alloca() instead of malloc() when allocating memory needed for regexec operations. */ -#define TRE_USE_ALLOCA 1 +// #define TRE_USE_ALLOCA 1 /* Define to include the system regex.h from TRE regex.h */ /* #undef TRE_USE_SYSTEM_REGEX_H */ diff --git a/src/tre-regex/xmalloc.c b/src/tre-regex/xmalloc.c index 3459d2d9..afe1bd14 100644 --- a/src/tre-regex/xmalloc.c +++ b/src/tre-regex/xmalloc.c @@ -340,6 +340,7 @@ xrealloc_impl(void *ptr, size_t new_size, const char *file, int line, new_ptr = realloc(ptr, new_size); if (new_ptr != NULL) { + ptr = NULL; hash_table_del(xmalloc_table, ptr); hash_table_add(xmalloc_table, new_ptr, (int)new_size, file, line, func); } diff --git a/src/vector.c b/src/vector.c index 2abb617f..431498ef 100644 --- a/src/vector.c +++ b/src/vector.c @@ -118,6 +118,16 @@ void free_sqlite3_stmt_vec(sqlite3_stmt_vec **v) if(v == NULL || *v == NULL || (*v)->items == NULL) return; + // Run sqlite3_finalize on all statements in the vector + for(unsigned int i = 0; i < (*v)->capacity; i++) + { + if((*v)->items[i] != NULL) + { + log_debug(DEBUG_VECTORS, "Finalizing sqlite3_stmt** %p[%u] --> %p", *v, i, (*v)->items[i]); + sqlite3_finalize((*v)->items[i]); + } + } + // Free elements of the vector... free((*v)->items); // ...and then the vector itself diff --git a/src/webserver/cJSON/cJSON.c b/src/webserver/cJSON/cJSON.c index 4e4979e9..61483d90 100644 --- a/src/webserver/cJSON/cJSON.c +++ b/src/webserver/cJSON/cJSON.c @@ -117,7 +117,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) } /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 17) +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 18) #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. #endif @@ -263,10 +263,12 @@ CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) { global_hooks.deallocate(item->valuestring); + item->valuestring = NULL; } if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) { global_hooks.deallocate(item->string); + item->string = NULL; } global_hooks.deallocate(item); item = next; @@ -397,6 +399,7 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) return object->valuedouble = number; } +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as an error and return NULL */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { char *copy = NULL; @@ -405,8 +408,8 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { return NULL; } - /* return NULL if the object is corrupted */ - if (object->valuestring == NULL) + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) { return NULL; } @@ -893,6 +896,7 @@ fail: if (output != NULL) { input_buffer->hooks.deallocate(output); + output = NULL; } if (input_pointer != NULL) @@ -1235,6 +1239,7 @@ static unsigned char *print(const cJSON * const item, cJSON_bool format, const i /* free the buffer */ hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; } return printed; @@ -1243,11 +1248,13 @@ fail: if (buffer->buffer != NULL) { hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; } if (printed != NULL) { hooks->deallocate(printed); + printed = NULL; } return NULL; @@ -1288,6 +1295,7 @@ CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON if (!print_value(item, &p)) { global_hooks.deallocate(p.buffer); + p.buffer = NULL; return NULL; } @@ -1659,6 +1667,11 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu current_item = new_item; } + if (cannot_access_at_index(input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + /* parse the name of the child */ input_buffer->offset++; buffer_skip_whitespace(input_buffer); @@ -3126,4 +3139,5 @@ CJSON_PUBLIC(void *) cJSON_malloc(size_t size) CJSON_PUBLIC(void) cJSON_free(void *object) { global_hooks.deallocate(object); + object = NULL; } diff --git a/src/webserver/cJSON/cJSON.h b/src/webserver/cJSON/cJSON.h index 218cc9ea..88cf0bcf 100644 --- a/src/webserver/cJSON/cJSON.h +++ b/src/webserver/cJSON/cJSON.h @@ -81,7 +81,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ /* project version */ #define CJSON_VERSION_MAJOR 1 #define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 17 +#define CJSON_VERSION_PATCH 18 #include diff --git a/src/webserver/civetweb/CMakeLists.txt b/src/webserver/civetweb/CMakeLists.txt index 6a6cf922..943619d5 100644 --- a/src/webserver/civetweb/CMakeLists.txt +++ b/src/webserver/civetweb/CMakeLists.txt @@ -32,16 +32,5 @@ target_compile_definitions(civetweb PRIVATE NO_CGI USE_LUA TIMER_RESOLUTION=1000) -if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS) - # Enable TLS support in civetweb if mbedTLS is available - message(STATUS "Building FTL with TLS support: YES") - target_compile_definitions(civetweb PRIVATE USE_MBEDTLS) - target_compile_definitions(webserver PRIVATE HAVE_TLS) -else() - # Disable TLS support in civetweb if mbedTLS is not available - message(STATUS "Building FTL with TLS support: NO") - target_compile_definitions(civetweb PRIVATE NO_SSL) -endif() - include_directories(${PROJECT_SOURCE_DIR}/src/lua /usr/local/include) target_include_directories(civetweb PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c index 367e19ae..6cef4dc4 100644 --- a/src/webserver/civetweb/civetweb.c +++ b/src/webserver/civetweb/civetweb.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2021 the Civetweb developers +/* Copyright (c) 2013-2024 the Civetweb developers * Copyright (c) 2004-2013 Sergey Lyubka * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -51,8 +51,8 @@ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS /* Disable deprecation warning in VS2005 */ #endif -#if !defined(_WIN32_WINNT) /* defined for tdm-gcc so we can use getnameinfo */ -#define _WIN32_WINNT 0x0502 +#if !defined(_WIN32_WINNT) /* Minimum API version */ +#define _WIN32_WINNT 0x0601 #endif #else #if !defined(_GNU_SOURCE) @@ -1126,7 +1126,15 @@ mg_atomic_inc(volatile ptrdiff_t *addr) #if defined(_WIN64) && !defined(NO_ATOMICS) ret = InterlockedIncrement64(addr); #elif defined(_WIN32) && !defined(NO_ATOMICS) +#ifdef __cplusplus + /* For C++ the Microsoft Visual Studio compiler can not decide what + * overloaded function prototpye in the SDC corresponds to "ptrdiff_t". */ + static_assert(sizeof(ptrdiff_t) == sizeof(LONG), "Size mismatch"); + static_assert(sizeof(ptrdiff_t) == sizeof(int32_t), "Size mismatch"); + ret = InterlockedIncrement((LONG *)addr); +#else ret = InterlockedIncrement(addr); +#endif #elif defined(__GNUC__) \ && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 0))) \ && !defined(NO_ATOMICS) @@ -1149,7 +1157,14 @@ mg_atomic_dec(volatile ptrdiff_t *addr) #if defined(_WIN64) && !defined(NO_ATOMICS) ret = InterlockedDecrement64(addr); #elif defined(_WIN32) && !defined(NO_ATOMICS) +#ifdef __cplusplus + /* see mg_atomic_inc */ + static_assert(sizeof(ptrdiff_t) == sizeof(LONG), "Size mismatch"); + static_assert(sizeof(ptrdiff_t) == sizeof(int32_t), "Size mismatch"); + ret = InterlockedDecrement((LONG *)addr); +#else ret = InterlockedDecrement(addr); +#endif #elif defined(__GNUC__) \ && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 0))) \ && !defined(NO_ATOMICS) @@ -1308,13 +1323,13 @@ mg_malloc_ex(size_t size, #endif if (data) { + uintptr_t *tmp = (uintptr_t *)data; ptrdiff_t mmem = mg_atomic_add(&mstat->totalMemUsed, (ptrdiff_t)size); mg_atomic_max(&mstat->maxMemUsed, mmem); - mg_atomic_inc(&mstat->blockCount); - ((uintptr_t *)data)[0] = size; - ((uintptr_t *)data)[1] = (uintptr_t)mstat; - memory = (void *)(((char *)data) + 2 * sizeof(uintptr_t)); + tmp[0] = size; + tmp[1] = (uintptr_t)mstat; + memory = (void *)&tmp[2]; } #if defined(MEMORY_DEBUGGING) @@ -1537,11 +1552,13 @@ static void mg_snprintf(const struct mg_connection *conn, #if defined(vsnprintf) #undef vsnprintf #endif +#if !defined(NDEBUG) #define malloc DO_NOT_USE_THIS_FUNCTION__USE_mg_malloc #define calloc DO_NOT_USE_THIS_FUNCTION__USE_mg_calloc #define realloc DO_NOT_USE_THIS_FUNCTION__USE_mg_realloc #define free DO_NOT_USE_THIS_FUNCTION__USE_mg_free #define snprintf DO_NOT_USE_THIS_FUNCTION__USE_mg_snprintf +#endif #if defined(_WIN32) /* vsnprintf must not be used in any system, * but this define only works well for Windows. */ @@ -1907,7 +1924,9 @@ struct socket { unsigned char is_ssl; /* Is port SSL-ed */ unsigned char ssl_redir; /* Is port supposed to redirect everything to SSL * port */ - unsigned char in_use; /* 0: invalid, 1: valid, 2: free */ + unsigned char + is_optional; /* Shouldn't cause us to exit if we can't bind to it */ + unsigned char in_use; /* 0: invalid, 1: valid, 2: free */ }; @@ -1920,6 +1939,7 @@ enum { /* Once for each server */ LISTENING_PORTS, NUM_THREADS, + PRESPAWN_THREADS, RUN_AS_USER, CONFIG_TCP_NODELAY, /* Prepended CONFIG_ to avoid conflict with the * socket option typedef TCP_NODELAY. */ @@ -1953,6 +1973,7 @@ enum { /* Once for each domain */ DOCUMENT_ROOT, + FALLBACK_DOCUMENT_ROOT, ACCESS_LOG_FILE, ERROR_LOG_FILE, @@ -2034,6 +2055,7 @@ enum { #if defined(USE_WEBSOCKET) WEBSOCKET_ROOT, + FALLBACK_WEBSOCKET_ROOT, #endif #if defined(USE_LUA) && defined(USE_WEBSOCKET) LUA_WEBSOCKET_EXTENSIONS, @@ -2042,6 +2064,8 @@ enum { ACCESS_CONTROL_ALLOW_ORIGIN, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_HEADERS, + ACCESS_CONTROL_EXPOSE_HEADERS, + ACCESS_CONTROL_ALLOW_CREDENTIALS, ERROR_PAGES, #if !defined(NO_CACHING) STATIC_FILE_MAX_AGE, @@ -2065,6 +2089,7 @@ static const struct mg_option config_options[] = { /* Once for each server */ {"listening_ports", MG_CONFIG_TYPE_STRING_LIST, "8080"}, {"num_threads", MG_CONFIG_TYPE_NUMBER, "50"}, + {"prespawn_threads", MG_CONFIG_TYPE_NUMBER, "0"}, {"run_as_user", MG_CONFIG_TYPE_STRING, NULL}, {"tcp_nodelay", MG_CONFIG_TYPE_NUMBER, "0"}, {"max_request_size", MG_CONFIG_TYPE_NUMBER, "16384"}, @@ -2097,6 +2122,7 @@ static const struct mg_option config_options[] = { /* Once for each domain */ {"document_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, + {"fallback_document_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, {"access_log_file", MG_CONFIG_TYPE_FILE, NULL}, {"error_log_file", MG_CONFIG_TYPE_FILE, NULL}, @@ -2195,6 +2221,7 @@ static const struct mg_option config_options[] = { #if defined(USE_WEBSOCKET) {"websocket_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, + {"fallback_websocket_root", MG_CONFIG_TYPE_DIRECTORY, NULL}, #endif #if defined(USE_LUA) && defined(USE_WEBSOCKET) {"lua_websocket_pattern", MG_CONFIG_TYPE_EXT_PATTERN, "**.lua$"}, @@ -2202,6 +2229,8 @@ static const struct mg_option config_options[] = { {"access_control_allow_origin", MG_CONFIG_TYPE_STRING, "*"}, {"access_control_allow_methods", MG_CONFIG_TYPE_STRING, "*"}, {"access_control_allow_headers", MG_CONFIG_TYPE_STRING, "*"}, + {"access_control_expose_headers", MG_CONFIG_TYPE_STRING, ""}, + {"access_control_allow_credentials", MG_CONFIG_TYPE_STRING, ""}, {"error_pages", MG_CONFIG_TYPE_DIRECTORY, NULL}, #if !defined(NO_CACHING) {"static_file_max_age", MG_CONFIG_TYPE_NUMBER, "3600"}, @@ -2311,7 +2340,7 @@ STOP_FLAG_IS_TWO(stop_flag_t *f) static void STOP_FLAG_ASSIGN(stop_flag_t *f, stop_flag_t v) { - stop_flag_t sf; + stop_flag_t sf = 0; do { sf = mg_atomic_compare_and_swap(f, *f, v); } while (sf != v); @@ -2374,10 +2403,18 @@ struct mg_context { stop_flag_t stop_flag; /* Should we stop event loop */ pthread_mutex_t thread_mutex; /* Protects client_socks or queue */ - pthread_t masterthreadid; /* The master thread ID */ + pthread_t masterthreadid; /* The master thread ID */ + unsigned int cfg_max_worker_threads; /* How many worker-threads we are + allowed to create, total */ + + unsigned int spawned_worker_threads; /* How many worker-threads currently + exist (modified by master thread) */ unsigned int - cfg_worker_threads; /* The number of configured worker threads. */ - pthread_t *worker_threadids; /* The worker thread IDs */ + idle_worker_thread_count; /* How many worker-threads are currently + sitting around with nothing to do */ + /* Access to this value MUST be synchronized by thread_mutex */ + + pthread_t *worker_threadids; /* The worker thread IDs */ unsigned long starter_thread_idx; /* thread index which called mg_start */ /* Connection to thread dispatching */ @@ -2424,6 +2461,11 @@ struct mg_context { int lua_bg_log_available; /* Use Lua background state for access log */ #endif + int user_shutdown_notification_socket; /* mg_stop() will close this + socket... */ + int thread_shutdown_notification_socket; /* to cause poll() in all threads + to return immediately */ + /* Server nonce */ pthread_mutex_t nonce_mutex; /* Protects ssl_ctx, handlers, * ssl_cert_last_mtime, nonce_count, and @@ -4140,6 +4182,8 @@ send_additional_header(struct mg_connection *conn) } #endif + // Content-Security-Policy + if (header && header[0]) { mg_response_header_add_lines(conn, header); } @@ -4160,6 +4204,14 @@ send_cors_header(struct mg_connection *conn) const char *origin_hdr = mg_get_header(conn, "Origin"); const char *cors_orig_cfg = conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_ORIGIN]; + const char *cors_cred_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_CREDENTIALS]; + const char *cors_hdr_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_HEADERS]; + const char *cors_exphdr_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_EXPOSE_HEADERS]; + const char *cors_meth_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_METHODS]; if (cors_orig_cfg && *cors_orig_cfg && origin_hdr && *origin_hdr) { /* Cross-origin resource sharing (CORS), see @@ -4171,6 +4223,37 @@ send_cors_header(struct mg_connection *conn) cors_orig_cfg, -1); } + + if (cors_cred_cfg && *cors_cred_cfg && origin_hdr && *origin_hdr) { + /* Cross-origin resource sharing (CORS), see + * https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials + */ + mg_response_header_add(conn, + "Access-Control-Allow-Credentials", + cors_cred_cfg, + -1); + } + + if (cors_hdr_cfg && *cors_hdr_cfg) { + mg_response_header_add(conn, + "Access-Control-Allow-Headers", + cors_hdr_cfg, + -1); + } + + if (cors_exphdr_cfg && *cors_exphdr_cfg) { + mg_response_header_add(conn, + "Access-Control-Expose-Headers", + cors_exphdr_cfg, + -1); + } + + if (cors_meth_cfg && *cors_meth_cfg) { + mg_response_header_add(conn, + "Access-Control-Allow-Methods", + cors_meth_cfg, + -1); + } } @@ -6179,12 +6262,20 @@ push_inner(struct mg_context *ctx, mg_sleep(5); } else { /* For sockets, wait for the socket using poll */ - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; + unsigned int num_sock = 1; pfd[0].fd = sock; pfd[0].events = POLLOUT; - pollres = mg_poll(pfd, 1, (int)(ms_wait), &(ctx->stop_flag)); + + if (ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + + pollres = mg_poll(pfd, num_sock, (int)(ms_wait), &(ctx->stop_flag)); if (!STOP_FLAG_IS_ZERO(&ctx->stop_flag)) { return -2; } @@ -6292,9 +6383,10 @@ pull_inner(FILE *fp, #if defined(USE_MBEDTLS) } else if (conn->ssl != NULL) { - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int to_read; int pollres; + unsigned int num_sock = 1; to_read = mbedtls_ssl_get_bytes_avail(conn->ssl); @@ -6310,10 +6402,17 @@ pull_inner(FILE *fp, pfd[0].fd = conn->client.sock; pfd[0].events = POLLIN; + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + to_read = len; pollres = mg_poll(pfd, - 1, + num_sock, (int)(timeout * 1000.0), &(conn->phys_ctx->stop_flag)); @@ -6348,8 +6447,9 @@ pull_inner(FILE *fp, #elif !defined(NO_SSL) } else if (conn->ssl != NULL) { int ssl_pending; - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; + unsigned int num_sock = 1; if ((ssl_pending = SSL_pending(conn->ssl)) > 0) { /* We already know there is no more data buffered in conn->buf @@ -6362,8 +6462,16 @@ pull_inner(FILE *fp, } else { pfd[0].fd = conn->client.sock; pfd[0].events = POLLIN; + + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + pollres = mg_poll(pfd, - 1, + num_sock, (int)(timeout * 1000.0), &(conn->phys_ctx->stop_flag)); if (!STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { @@ -6401,13 +6509,22 @@ pull_inner(FILE *fp, #endif } else { - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; + unsigned int num_sock = 1; pfd[0].fd = conn->client.sock; pfd[0].events = POLLIN; + + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + pollres = mg_poll(pfd, - 1, + num_sock, (int)(timeout * 1000.0), &(conn->phys_ctx->stop_flag)); if (!STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { @@ -6429,7 +6546,7 @@ pull_inner(FILE *fp, } } - if (!STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { + if (conn != NULL && !STOP_FLAG_IS_ZERO(&conn->phys_ctx->stop_flag)) { return -2; } @@ -6453,7 +6570,7 @@ pull_inner(FILE *fp, /* See https://www.chilkatsoft.com/p/p_299.asp */ return -2; } else { - DEBUG_TRACE("recv() failed, error %d", err); + DEBUG_TRACE("read()/recv() failed, error %d", err); return -2; } #else @@ -6475,7 +6592,7 @@ pull_inner(FILE *fp, * (see signal(7)). * => stay in the while loop */ } else { - DEBUG_TRACE("recv() failed, error %d", err); + DEBUG_TRACE("read()/recv() failed, error %d", err); return -2; } #endif @@ -7622,10 +7739,10 @@ extention_matches_template_text( * Return 1 if index file has been found, 0 if not found. * If the file is found, it's stats is returned in stp. */ static int -substitute_index_file(struct mg_connection *conn, - char *path, - size_t path_len, - struct mg_file_stat *filestat) +substitute_index_file_aux(struct mg_connection *conn, + char *path, + size_t path_len, + struct mg_file_stat *filestat) { const char *list = conn->dom_ctx->config[INDEX_FILES]; struct vec filename_vec; @@ -7666,6 +7783,61 @@ substitute_index_file(struct mg_connection *conn, return found; } + +/* Same as above, except if the first try fails and a fallback-root is + * configured, we'll try there also */ +static int +substitute_index_file(struct mg_connection *conn, + char *path, + size_t path_len, + struct mg_file_stat *filestat) +{ + int ret = substitute_index_file_aux(conn, path, path_len, filestat); + if (ret == 0) { + const char *root_prefix = conn->dom_ctx->config[DOCUMENT_ROOT]; + const char *fallback_root_prefix = + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]; + if ((root_prefix) && (fallback_root_prefix)) { + const size_t root_prefix_len = strlen(root_prefix); + if ((strncmp(path, root_prefix, root_prefix_len) == 0)) { + char scratch_path[UTF8_PATH_MAX]; /* separate storage, to avoid + side effects if we fail */ + size_t sub_path_len; + + const size_t fallback_root_prefix_len = + strlen(fallback_root_prefix); + const char *sub_path = path + root_prefix_len; + while (*sub_path == '/') { + sub_path++; + } + sub_path_len = strlen(sub_path); + + if (((fallback_root_prefix_len + 1 + sub_path_len + 1) + < sizeof(scratch_path))) { + /* The concatenations below are all safe because we + * pre-verified string lengths above */ + char *nul; + strcpy(scratch_path, fallback_root_prefix); + nul = strchr(scratch_path, '\0'); + if ((nul > scratch_path) && (*(nul - 1) != '/')) { + *nul++ = '/'; + *nul = '\0'; + } + strcat(scratch_path, sub_path); + if (substitute_index_file_aux(conn, + scratch_path, + sizeof(scratch_path), + filestat)) { + mg_strlcpy(path, scratch_path, path_len); + return 1; + } + } + } + } + } + return ret; +} + #endif @@ -7686,12 +7858,16 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ #if !defined(NO_FILES) const char *uri = conn->request_info.local_uri; - const char *root = conn->dom_ctx->config[DOCUMENT_ROOT]; + const char *roots[] = {conn->dom_ctx->config[DOCUMENT_ROOT], + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT], + NULL}; + int fileExists = 0; const char *rewrite; struct vec a, b; ptrdiff_t match_len; char gz_path[UTF8_PATH_MAX]; int truncated; + int i; #if !defined(NO_CGI) || defined(USE_LUA) || defined(USE_DUKTAPE) char *tmp_str; size_t tmp_str_len, sep_pos; @@ -7721,7 +7897,8 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ *is_websocket_request = (conn->protocol_type == PROTOCOL_TYPE_WEBSOCKET); #if !defined(NO_FILES) if ((*is_websocket_request) && conn->dom_ctx->config[WEBSOCKET_ROOT]) { - root = conn->dom_ctx->config[WEBSOCKET_ROOT]; + roots[0] = conn->dom_ctx->config[WEBSOCKET_ROOT]; + roots[1] = conn->dom_ctx->config[FALLBACK_WEBSOCKET_ROOT]; } #endif /* !NO_FILES */ #else /* USE_WEBSOCKET */ @@ -7738,53 +7915,65 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */ #if !defined(NO_FILES) /* Step 5: If there is no root directory, don't look for files. */ - /* Note that root == NULL is a regular use case here. This occurs, + /* Note that roots[0] == NULL is a regular use case here. This occurs, * if all requests are handled by callbacks, so the WEBSOCKET_ROOT * config is not required. */ - if (root == NULL) { + if (roots[0] == NULL) { /* all file related outputs have already been set to 0, just return */ return; } - /* Step 6: Determine the local file path from the root path and the - * request uri. */ - /* Using filename_buf_len - 1 because memmove() for PATH_INFO may shift - * part of the path one byte on the right. */ - truncated = 0; - mg_snprintf( - conn, &truncated, filename, filename_buf_len - 1, "%s%s", root, uri); + for (i = 0; roots[i] != NULL; i++) { + /* Step 6: Determine the local file path from the root path and the + * request uri. */ + /* Using filename_buf_len - 1 because memmove() for PATH_INFO may shift + * part of the path one byte on the right. */ + truncated = 0; + mg_snprintf(conn, + &truncated, + filename, + filename_buf_len - 1, + "%s%s", + roots[i], + uri); - FTL_rewrite_pattern(filename, filename_buf_len - 1); + FTL_rewrite_pattern(filename, filename_buf_len - 1); - if (truncated) { - goto interpret_cleanup; - } + if (truncated) { + goto interpret_cleanup; + } - /* Step 7: URI rewriting */ - rewrite = conn->dom_ctx->config[URL_REWRITE_PATTERN]; - while ((rewrite = next_option(rewrite, &a, &b)) != NULL) { - if ((match_len = match_prefix(a.ptr, a.len, uri)) > 0) { - mg_snprintf(conn, - &truncated, - filename, - filename_buf_len - 1, - "%.*s%s", - (int)b.len, - b.ptr, - uri + match_len); + /* Step 7: URI rewriting */ + rewrite = conn->dom_ctx->config[URL_REWRITE_PATTERN]; + while ((rewrite = next_option(rewrite, &a, &b)) != NULL) { + if ((match_len = match_prefix(a.ptr, a.len, uri)) > 0) { + mg_snprintf(conn, + &truncated, + filename, + filename_buf_len - 1, + "%.*s%s", + (int)b.len, + b.ptr, + uri + match_len); + break; + } + } + + if (truncated) { + goto interpret_cleanup; + } + + /* Step 8: Check if the file exists at the server */ + /* Local file path and name, corresponding to requested URI + * is now stored in "filename" variable. */ + if (mg_stat(conn, filename, filestat)) { + fileExists = 1; break; } } - if (truncated) { - goto interpret_cleanup; - } - - /* Step 8: Check if the file exists at the server */ - /* Local file path and name, corresponding to requested URI - * is now stored in "filename" variable. */ - if (mg_stat(conn, filename, filestat)) { + if (fileExists) { int uri_len = (int)strlen(uri); int is_uri_end_slash = (uri_len > 0) && (uri[uri_len - 1] == '/'); @@ -8289,10 +8478,12 @@ static const struct { {".iso", 4, "application/octet-stream"}, {".js", 3, "application/javascript"}, {".json", 5, "application/json"}, + {".mjs", 4, "application/javascript"}, {".msi", 4, "application/octet-stream"}, {".pdf", 4, "application/pdf"}, {".ps", 3, "application/postscript"}, {".rtf", 4, "application/rtf"}, + {".wasm", 5, "application/wasm"}, {".xhtml", 6, "application/xhtml+xml"}, {".xsl", 4, "application/xml"}, {".xslt", 5, "application/xml"}, @@ -8595,7 +8786,7 @@ open_auth_file(struct mg_connection *conn, /* Parsed Authorization header */ -struct ah { +struct auth_header { char *user; int type; /* 1 = basic, 2 = digest */ char *plain_password; /* Basic only */ @@ -8603,32 +8794,32 @@ struct ah { }; -/* Return 1 on success. Always initializes the ah structure. */ +/* Return 1 on success. Always initializes the auth_header structure. */ static int parse_auth_header(struct mg_connection *conn, char *buf, size_t buf_size, - struct ah *ah) + struct auth_header *auth_header) { char *name, *value, *s; - const char *auth_header; + const char *ah; uint64_t nonce; - if (!ah || !conn) { + if (!auth_header || !conn) { return 0; } - (void)memset(ah, 0, sizeof(*ah)); - auth_header = mg_get_header(conn, "Authorization"); + (void)memset(auth_header, 0, sizeof(*auth_header)); + ah = mg_get_header(conn, "Authorization"); - if (auth_header == NULL) { + if (ah == NULL) { /* No Authorization header at all */ return 0; } - if (0 == mg_strncasecmp(auth_header, "Basic ", 6)) { + if (0 == mg_strncasecmp(ah, "Basic ", 6)) { /* Basic Auth (we never asked for this, but some client may send it) */ char *split; - const char *userpw_b64 = auth_header + 6; + const char *userpw_b64 = ah + 6; size_t userpw_b64_len = strlen(userpw_b64); size_t buf_len_r = buf_size; if (mg_base64_decode( @@ -8645,15 +8836,15 @@ parse_auth_header(struct mg_connection *conn, *split = 0; /* User name is before ':', Password is after ':' */ - ah->user = buf; - ah->type = 1; - ah->plain_password = split + 1; + auth_header->user = buf; + auth_header->type = 1; + auth_header->plain_password = split + 1; return 1; - } else if (0 == mg_strncasecmp(auth_header, "Digest ", 7)) { + } else if (0 == mg_strncasecmp(ah, "Digest ", 7)) { /* Digest Auth ... implemented below */ - ah->type = 2; + auth_header->type = 2; } else { /* Unknown or invalid Auth method */ @@ -8661,7 +8852,7 @@ parse_auth_header(struct mg_connection *conn, } /* Make modifiable copy of the auth header */ - (void)mg_strlcpy(buf, auth_header + 7, buf_size); + (void)mg_strlcpy(buf, ah + 7, buf_size); s = buf; /* Parse authorization header */ @@ -8688,29 +8879,29 @@ parse_auth_header(struct mg_connection *conn, } if (!strcmp(name, "username")) { - ah->user = value; + auth_header->user = value; } else if (!strcmp(name, "cnonce")) { - ah->cnonce = value; + auth_header->cnonce = value; } else if (!strcmp(name, "response")) { - ah->response = value; + auth_header->response = value; } else if (!strcmp(name, "uri")) { - ah->uri = value; + auth_header->uri = value; } else if (!strcmp(name, "qop")) { - ah->qop = value; + auth_header->qop = value; } else if (!strcmp(name, "nc")) { - ah->nc = value; + auth_header->nc = value; } else if (!strcmp(name, "nonce")) { - ah->nonce = value; + auth_header->nonce = value; } } #if !defined(NO_NONCE_CHECK) /* Read the nonce from the response. */ - if (ah->nonce == NULL) { + if (auth_header->nonce == NULL) { return 0; } s = NULL; - nonce = strtoull(ah->nonce, &s, 10); + nonce = strtoull(auth_header->nonce, &s, 10); if ((s == NULL) || (*s != 0)) { return 0; } @@ -8741,7 +8932,7 @@ parse_auth_header(struct mg_connection *conn, (void)nonce; #endif - return (ah->user != NULL); + return (auth_header->user != NULL); } @@ -8773,7 +8964,7 @@ mg_fgets(char *buf, size_t size, struct mg_file *filep) #if !defined(NO_FILESYSTEMS) struct read_auth_file_struct { struct mg_connection *conn; - struct ah ah; + struct auth_header auth_header; const char *domain; char buf[256 + 256 + 40]; const char *f_user; @@ -8874,9 +9065,9 @@ read_auth_file(struct mg_file *filep, *(char *)(workdata->f_ha1) = 0; (workdata->f_ha1)++; - if (!strcmp(workdata->ah.user, workdata->f_user) + if (!strcmp(workdata->auth_header.user, workdata->f_user) && !strcmp(workdata->domain, workdata->f_domain)) { - switch (workdata->ah.type) { + switch (workdata->auth_header.type) { case 1: /* Basic */ { char md5[33]; @@ -8885,7 +9076,7 @@ read_auth_file(struct mg_file *filep, ":", workdata->domain, ":", - workdata->ah.plain_password, + workdata->auth_header.plain_password, NULL); return 0 == memcmp(workdata->f_ha1, md5, 33); } @@ -8893,12 +9084,12 @@ read_auth_file(struct mg_file *filep, return check_password_digest( workdata->conn->request_info.request_method, workdata->f_ha1, - workdata->ah.uri, - workdata->ah.nonce, - workdata->ah.nc, - workdata->ah.cnonce, - workdata->ah.qop, - workdata->ah.response); + workdata->auth_header.uri, + workdata->auth_header.nonce, + workdata->auth_header.nc, + workdata->auth_header.cnonce, + workdata->auth_header.qop, + workdata->auth_header.response); default: /* None/Other/Unknown */ return 0; } @@ -8923,13 +9114,13 @@ authorize(struct mg_connection *conn, struct mg_file *filep, const char *realm) memset(&workdata, 0, sizeof(workdata)); workdata.conn = conn; - if (!parse_auth_header(conn, buf, sizeof(buf), &workdata.ah)) { + if (!parse_auth_header(conn, buf, sizeof(buf), &workdata.auth_header)) { return 0; } /* CGI needs it as REMOTE_USER */ conn->request_info.remote_user = - mg_strdup_ctx(workdata.ah.user, conn->phys_ctx); + mg_strdup_ctx(workdata.auth_header.user, conn->phys_ctx); if (realm) { workdata.domain = realm; @@ -9564,11 +9755,11 @@ connect_socket( #endif /* Data for poll */ - struct mg_pollfd pfd[1]; + struct mg_pollfd pfd[2]; int pollres; - int ms_wait = 10000; /* 10 second timeout */ - stop_flag_t nonstop; - STOP_FLAG_ASSIGN(&nonstop, 0); + int ms_wait = 10000; /* 10 second timeout */ + stop_flag_t nonstop = 0; /* STOP_FLAG_ASSIGN(&nonstop, 0); */ + unsigned int num_sock = 1; /* use one or two sockets */ /* For a non-blocking socket, the connect sequence is: * 1) call connect (will not block) @@ -9577,7 +9768,15 @@ connect_socket( */ pfd[0].fd = *sock; pfd[0].events = POLLOUT; - pollres = mg_poll(pfd, 1, ms_wait, ctx ? &(ctx->stop_flag) : &nonstop); + + if (ctx && (ctx->context_type == CONTEXT_SERVER)) { + pfd[num_sock].fd = ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + + pollres = + mg_poll(pfd, num_sock, ms_wait, ctx ? &(ctx->stop_flag) : &nonstop); if (pollres != 1) { /* Not connected */ @@ -10198,8 +10397,11 @@ send_file_data(struct mg_connection *conn, } /* Read from file, exit the loop on error */ - if ((num_read = - (int)fread(buf, 1, (size_t)to_read, filep->access.fp)) + if ((num_read = pull_inner(filep->access.fp, + NULL, + buf, + to_read, + /* unused */ 0.0)) <= 0) { break; } @@ -10733,6 +10935,7 @@ static int skip_to_end_of_word_and_terminate(char **ppw, int eol) { /* Forward until a space is found - use isgraph here */ + /* Extended ASCII characters are also treated as word characters. */ /* See http://www.cplusplus.com/reference/cctype/ */ while ((unsigned char)**ppw > 127 || isgraph((unsigned char)**ppw)) { (*ppw)++; @@ -10831,7 +11034,7 @@ parse_http_headers(char **buf, struct mg_header hdr[MG_MAX_HEADERS]) } /* here *dp is either 0 or '\n' */ - /* in any case, we have a new header */ + /* in any case, we have found a complete header */ num_headers = i + 1; if (*dp) { @@ -10840,9 +11043,11 @@ parse_http_headers(char **buf, struct mg_header hdr[MG_MAX_HEADERS]) *buf = dp; if ((dp[0] == '\r') || (dp[0] == '\n')) { - /* This is the end of the header */ + /* We've had CRLF twice in a row + * This is the end of the headers */ break; } + /* continue within the loop, find the next header */ } else { *buf = dp; break; @@ -11219,11 +11424,11 @@ read_message(FILE *fp, request_len = get_http_header_len(buf, *nread); } - if ((request_len == 0) && (request_timeout >= 0)) { + if ((n <= 0) && (request_timeout >= 0)) { if (mg_difftimespec(&last_action_time, &(conn->req_time)) > request_timeout) { /* Timeout */ - return -1; + return -3; } } } @@ -11443,6 +11648,11 @@ prepare_cgi_environment(struct mg_connection *conn, addenv(env, "SERVER_NAME=%s", conn->dom_ctx->config[AUTHENTICATION_DOMAIN]); addenv(env, "SERVER_ROOT=%s", conn->dom_ctx->config[DOCUMENT_ROOT]); addenv(env, "DOCUMENT_ROOT=%s", conn->dom_ctx->config[DOCUMENT_ROOT]); + if (conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]) { + addenv(env, + "FALLBACK_DOCUMENT_ROOT=%s", + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]); + } addenv(env, "SERVER_SOFTWARE=CivetWeb/%s", mg_version()); /* Prepare the environment block */ @@ -12861,11 +13071,14 @@ dav_lock_file(struct mg_connection *conn, const char *path) int i; uint64_t LOCK_DURATION_NS = (uint64_t)(LOCK_DURATION_S) * (uint64_t)1000000000; - struct twebdav_lock *dav_lock = conn->phys_ctx->webdav_lock; + struct twebdav_lock *dav_lock = NULL; - if (!path || !conn->dom_ctx || !conn->request_info.remote_user) { + if (!path || !conn || !conn->dom_ctx || !conn->request_info.remote_user + || !conn->phys_ctx) { return; } + + dav_lock = conn->phys_ctx->webdav_lock; mg_get_request_link(conn, link_buf, sizeof(link_buf)); /* const char *refresh = mg_get_header(conn, "If"); */ @@ -14904,6 +15117,10 @@ handle_request(struct mg_connection *conn) get_header(ri->http_headers, ri->num_headers, "Access-Control-Request-Headers"); + const char *cors_cred_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_CREDENTIALS]; + const char *cors_exphdr_cfg = + conn->dom_ctx->config[ACCESS_CONTROL_EXPOSE_HEADERS]; gmt_time_string(date, sizeof(date), &curtime); mg_printf(conn, @@ -14918,7 +15135,19 @@ handle_request(struct mg_connection *conn) ((cors_meth_cfg[0] == '*') ? cors_acrm : cors_meth_cfg), suggest_connection_header(conn)); - if (cors_acrh != NULL) { + if (cors_cred_cfg && *cors_cred_cfg) { + mg_printf(conn, + "Access-Control-Allow-Credentials: %s\r\n", + cors_cred_cfg); + } + + if (cors_exphdr_cfg && *cors_exphdr_cfg) { + mg_printf(conn, + "Access-Control-Expose-Headers: %s\r\n", + cors_exphdr_cfg); + } + + if (cors_acrh || (cors_cred_cfg && *cors_cred_cfg)) { /* CORS request is asking for additional headers */ const char *cors_hdr_cfg = conn->dom_ctx->config[ACCESS_CONTROL_ALLOW_HEADERS]; @@ -15556,7 +15785,7 @@ parse_port_string(const struct vec *vec, struct socket *so, int *ip_version) unsigned int a, b, c, d; unsigned port; unsigned long portUL; - int ch, len; + int len; const char *cb; char *endptr; #if defined(USE_IPV6) @@ -15709,14 +15938,38 @@ parse_port_string(const struct vec *vec, struct socket *so, int *ip_version) } /* sscanf and the option splitting code ensure the following condition - * Make sure the port is valid and vector ends with the port, 's' or 'r' */ - if ((len > 0) && is_valid_port(port) - && (((size_t)len == vec->len) || (((size_t)len + 1) == vec->len))) { - /* Next character after the port number */ - ch = ((size_t)len < vec->len) ? vec->ptr[len] : '\0'; - so->is_ssl = (ch == 's'); - so->ssl_redir = (ch == 'r'); - if ((ch == '\0') || (ch == 's') || (ch == 'r')) { + * Make sure the port is valid and vector ends with the port, 'o', 's', or + * 'r' */ + if ((len > 0) && (is_valid_port(port))) { + int bad_suffix = 0; + size_t i; + + /* Parse any suffix character(s) after the port number */ + for (i = len; i < vec->len; i++) { + unsigned char *opt = NULL; + switch (vec->ptr[i]) { + case 'o': + opt = &so->is_optional; + break; + case 'r': + opt = &so->ssl_redir; + break; + case 's': + opt = &so->is_ssl; + break; + default: /* empty */ + break; + } + + if ((opt) && (*opt == 0)) + *opt = 1; + else { + bad_suffix = 1; + break; + } + } + + if ((bad_suffix == 0) && ((so->is_ssl == 0) || (so->ssl_redir == 0))) { return 1; } } @@ -15771,8 +16024,14 @@ is_ssl_port_used(const char *ports) char prevIsNumber = 0; for (i = 0; i < portslen; i++) { - if (prevIsNumber && (ports[i] == 's' || ports[i] == 'r')) { - return 1; + if (prevIsNumber) { + int suffixCharIdx = (ports[i] == 'o') + ? (i + 1) + : i; /* allow "os" and "or" suffixes */ + if (ports[suffixCharIdx] == 's' + || ports[suffixCharIdx] == 'r') { + return 1; + } } if (ports[i] >= '0' && ports[i] <= '9') { prevIsNumber = 1; @@ -15955,6 +16214,10 @@ set_ports_option(struct mg_context *phys_ctx) strerror(errno)); closesocket(so.sock); so.sock = INVALID_SOCKET; + if (so.is_optional) { + portsOk++; /* it's okay if we couldn't bind, this port is + optional anyway */ + } continue; } } @@ -15971,6 +16234,10 @@ set_ports_option(struct mg_context *phys_ctx) strerror(errno)); closesocket(so.sock); so.sock = INVALID_SOCKET; + if (so.is_optional) { + portsOk++; /* it's okay if we couldn't bind, this port is + optional anyway */ + } continue; } } @@ -15987,6 +16254,10 @@ set_ports_option(struct mg_context *phys_ctx) strerror(errno)); closesocket(so.sock); so.sock = INVALID_SOCKET; + if (so.is_optional) { + portsOk++; /* it's okay if we couldn't bind, this port is + optional anyway */ + } continue; } } @@ -16064,9 +16335,14 @@ set_ports_option(struct mg_context *phys_ctx) continue; } + /* The +2 below includes the original +1 (for the socket we're about to + * add), plus another +1 for the thread_shutdown_notification_socket + * that we'll also want to poll() on so that mg_stop() can return + * quickly + */ if ((pfd = (struct mg_pollfd *) mg_realloc_ctx(phys_ctx->listening_socket_fds, - (phys_ctx->num_listening_sockets + 1) + (phys_ctx->num_listening_sockets + 2) * sizeof(phys_ctx->listening_socket_fds[0]), phys_ctx)) == NULL) { @@ -16588,15 +16864,26 @@ sslize(struct mg_connection *conn, /* Need to retry the function call "later". * See https://linux.die.net/man/3/ssl_get_error * This is typical for non-blocking sockets. */ - struct mg_pollfd pfd; + struct mg_pollfd pfd[2]; int pollres; - pfd.fd = conn->client.sock; - pfd.events = ((err == SSL_ERROR_WANT_CONNECT) - || (err == SSL_ERROR_WANT_WRITE)) - ? POLLOUT - : POLLIN; - pollres = - mg_poll(&pfd, 1, 50, &(conn->phys_ctx->stop_flag)); + unsigned int num_sock = 1; + pfd[0].fd = conn->client.sock; + pfd[0].events = ((err == SSL_ERROR_WANT_CONNECT) + || (err == SSL_ERROR_WANT_WRITE)) + ? POLLOUT + : POLLIN; + + if (conn->phys_ctx->context_type == CONTEXT_SERVER) { + pfd[num_sock].fd = + conn->phys_ctx->thread_shutdown_notification_socket; + pfd[num_sock].events = POLLIN; + num_sock++; + } + + pollres = mg_poll(pfd, + num_sock, + 50, + &(conn->phys_ctx->stop_flag)); if (pollres < 0) { /* Break if error occurred (-1) * or server shutdown (-2) */ @@ -18019,7 +18306,7 @@ mg_close_connection(struct mg_connection *conn) * timeouts, we will just wait a few seconds in mg_join_thread. */ /* join worker thread */ - for (i = 0; i < conn->phys_ctx->cfg_worker_threads; i++) { + for (i = 0; i < conn->phys_ctx->spawned_worker_threads; i++) { mg_join_thread(conn->phys_ctx->worker_threadids[i]); } } @@ -18678,7 +18965,8 @@ get_message(struct mg_connection *conn, char *ebuf, size_t ebuf_len, int *err) ebuf, ebuf_len, "%s", - "Malformed message"); + conn->request_len == -3 ? "Request timeout" + : "Malformed message"); *err = 400; } else { /* Server did not recv anything -> just close the connection */ @@ -19049,6 +19337,24 @@ websocket_client_thread(void *data) #endif +#if defined(USE_WEBSOCKET) +static void +generate_websocket_magic(char *magic25) +{ + uint64_t rnd; + unsigned char buffer[2 * sizeof(rnd)]; + + rnd = get_random(); + memcpy(buffer, &rnd, sizeof(rnd)); + rnd = get_random(); + memcpy(buffer + sizeof(rnd), &rnd, sizeof(rnd)); + + size_t dst_len = 24 + 1; + mg_base64_encode(buffer, sizeof(buffer), magic25, &dst_len); +} +#endif + + static struct mg_connection * mg_connect_websocket_client_impl(const struct mg_client_options *client_options, int use_ssl, @@ -19065,7 +19371,8 @@ mg_connect_websocket_client_impl(const struct mg_client_options *client_options, #if defined(USE_WEBSOCKET) struct websocket_client_thread_data *thread_data; - static const char *magic = "x3JJHMbDL1EzLkh9GBhXDw=="; + char magic[32]; + generate_websocket_magic(magic); const char *host = client_options->host; int i; @@ -19229,7 +19536,8 @@ mg_connect_websocket_client_impl(const struct mg_client_options *client_options, /* Now upgrade to ws/wss client context */ conn->phys_ctx->user_data = user_data; conn->phys_ctx->context_type = CONTEXT_WS_CLIENT; - conn->phys_ctx->cfg_worker_threads = 1; /* one worker thread */ + conn->phys_ctx->cfg_max_worker_threads = 1; /* one worker thread */ + conn->phys_ctx->spawned_worker_threads = 1; /* one worker thread */ /* Start a thread to read the websocket client connection * This thread will automatically stop when mg_disconnect is @@ -19238,7 +19546,7 @@ mg_connect_websocket_client_impl(const struct mg_client_options *client_options, thread_data, conn->phys_ctx->worker_threadids) != 0) { - conn->phys_ctx->cfg_worker_threads = 0; + conn->phys_ctx->spawned_worker_threads = 0; mg_free(thread_data); mg_close_connection(conn); conn = NULL; @@ -19609,6 +19917,9 @@ process_new_connection(struct mg_connection *conn) #endif } +static int +mg_start_worker_thread(struct mg_context *ctx, + int only_if_no_idle_threads); /* forward declaration */ #if defined(ALTERNATIVE_QUEUE) @@ -19617,8 +19928,12 @@ produce_socket(struct mg_context *ctx, const struct socket *sp) { unsigned int i; + (void)mg_start_worker_thread( + ctx, 1); /* will start a worker-thread only if there aren't currently + any idle worker-threads */ + while (!ctx->stop_flag) { - for (i = 0; i < ctx->cfg_worker_threads; i++) { + for (i = 0; i < ctx->spawned_worker_threads; i++) { /* find a free worker slot and signal it */ if (ctx->client_socks[i].in_use == 2) { (void)pthread_mutex_lock(&ctx->thread_mutex); @@ -19643,10 +19958,18 @@ produce_socket(struct mg_context *ctx, const struct socket *sp) static int -consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) +consume_socket(struct mg_context *ctx, + struct socket *sp, + int thread_index, + int counter_was_preincremented) { DEBUG_TRACE("%s", "going idle"); (void)pthread_mutex_lock(&ctx->thread_mutex); + if (counter_was_preincremented + == 0) { /* first call only: the master-thread pre-incremented this + before he spawned us */ + ctx->idle_worker_thread_count++; + } ctx->client_socks[thread_index].in_use = 2; (void)pthread_mutex_unlock(&ctx->thread_mutex); @@ -19663,6 +19986,7 @@ consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) } return 0; } + ctx->idle_worker_thread_count--; (void)pthread_mutex_unlock(&ctx->thread_mutex); if (sp->in_use == 1) { DEBUG_TRACE("grabbed socket %d, going busy", sp->sock); @@ -19677,12 +20001,20 @@ consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) /* Worker threads take accepted socket from the queue */ static int -consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) +consume_socket(struct mg_context *ctx, + struct socket *sp, + int thread_index, + int counter_was_preincremented) { (void)thread_index; - (void)pthread_mutex_lock(&ctx->thread_mutex); DEBUG_TRACE("%s", "going idle"); + (void)pthread_mutex_lock(&ctx->thread_mutex); + if (counter_was_preincremented + == 0) { /* first call only: the master-thread pre-incremented this + before he spawned us */ + ctx->idle_worker_thread_count++; + } /* If the queue is empty, wait. We're idle at this point. */ while ((ctx->sq_head == ctx->sq_tail) @@ -19706,6 +20038,8 @@ consume_socket(struct mg_context *ctx, struct socket *sp, int thread_index) } (void)pthread_cond_signal(&ctx->sq_empty); + + ctx->idle_worker_thread_count--; (void)pthread_mutex_unlock(&ctx->thread_mutex); return STOP_FLAG_IS_ZERO(&ctx->stop_flag); @@ -19752,6 +20086,10 @@ produce_socket(struct mg_context *ctx, const struct socket *sp) (void)pthread_cond_signal(&ctx->sq_full); (void)pthread_mutex_unlock(&ctx->thread_mutex); + + (void)mg_start_worker_thread( + ctx, 1); /* will start a worker-thread only if there aren't currently + any idle worker-threads */ } #endif /* ALTERNATIVE_QUEUE */ @@ -19762,6 +20100,7 @@ worker_thread_run(struct mg_connection *conn) struct mg_context *ctx = conn->phys_ctx; int thread_index; struct mg_workerTLS tls; + int first_call_to_consume_socket = 1; mg_set_thread_name("worker"); @@ -19787,7 +20126,7 @@ worker_thread_run(struct mg_connection *conn) /* Connection structure has been pre-allocated */ thread_index = (int)(conn - ctx->worker_connections); if ((thread_index < 0) - || ((unsigned)thread_index >= (unsigned)ctx->cfg_worker_threads)) { + || ((unsigned)thread_index >= (unsigned)ctx->cfg_max_worker_threads)) { mg_cry_ctx_internal(ctx, "Internal error: Invalid worker index %i", thread_index); @@ -19828,7 +20167,9 @@ worker_thread_run(struct mg_connection *conn) /* Call consume_socket() even when ctx->stop_flag > 0, to let it * signal sq_empty condvar to wake up the master waiting in * produce_socket() */ - while (consume_socket(ctx, &conn->client, thread_index)) { + while (consume_socket( + ctx, &conn->client, thread_index, first_call_to_consume_socket)) { + first_call_to_consume_socket = 0; /* New connections must start with new protocol negotiation */ tls.alpn_proto = NULL; @@ -20035,6 +20376,7 @@ accept_new_connection(const struct socket *listener, struct mg_context *ctx) set_close_on_exec(so.sock, NULL, ctx); so.is_ssl = listener->is_ssl; so.ssl_redir = listener->ssl_redir; + so.is_optional = listener->is_optional; if (getsockname(so.sock, &so.lsa.sa, &len) != 0) { mg_cry_ctx_internal(ctx, "%s: getsockname() failed: %s", @@ -20183,8 +20525,17 @@ master_thread_run(struct mg_context *ctx) pfd[i].events = POLLIN; } + /* We listen on this socket just so that mg_stop() can cause mg_poll() + * to return ASAP. Don't worry, we did allocate an extra slot at the end + * of listening_socket_fds[] just to hold this + */ + pfd[ctx->num_listening_sockets].fd = + ctx->thread_shutdown_notification_socket; + pfd[ctx->num_listening_sockets].events = POLLIN; + if (mg_poll(pfd, - ctx->num_listening_sockets, + ctx->num_listening_sockets + + 1, // +1 for the thread_shutdown_notification_socket SOCKET_TIMEOUT_QUANTUM, &(ctx->stop_flag)) > 0) { @@ -20210,7 +20561,7 @@ master_thread_run(struct mg_context *ctx) /* Wakeup workers that are waiting for connections to handle. */ #if defined(ALTERNATIVE_QUEUE) - for (i = 0; i < ctx->cfg_worker_threads; i++) { + for (i = 0; i < ctx->spawned_worker_threads; i++) { event_signal(ctx->client_wait_events[i]); } #else @@ -20220,7 +20571,7 @@ master_thread_run(struct mg_context *ctx) #endif /* Join all worker threads to avoid leaking threads. */ - workerthreadcount = ctx->cfg_worker_threads; + workerthreadcount = ctx->spawned_worker_threads; for (i = 0; i < workerthreadcount; i++) { if (ctx->worker_threadids[i] != 0) { mg_join_thread(ctx->worker_threadids[i]); @@ -20324,7 +20675,7 @@ free_context(struct mg_context *ctx) #if defined(ALTERNATIVE_QUEUE) mg_free(ctx->client_socks); if (ctx->client_wait_events != NULL) { - for (i = 0; (unsigned)i < ctx->cfg_worker_threads; i++) { + for (i = 0; (unsigned)i < ctx->spawned_worker_threads; i++) { event_destroy(ctx->client_wait_events[i]); } mg_free(ctx->client_wait_events); @@ -20342,6 +20693,14 @@ free_context(struct mg_context *ctx) (void)pthread_mutex_destroy(&ctx->lua_bg_mutex); #endif + /* Deallocate shutdown-triggering socket-pair */ + if (ctx->user_shutdown_notification_socket >= 0) { + closesocket(ctx->user_shutdown_notification_socket); + } + if (ctx->thread_shutdown_notification_socket >= 0) { + closesocket(ctx->thread_shutdown_notification_socket); + } + /* Deallocate config parameters */ for (i = 0; i < NUM_OPTIONS; i++) { if (ctx->dd.config[i] != NULL) { @@ -20418,6 +20777,12 @@ mg_stop(struct mg_context *ctx) /* Set stop flag, so all threads know they have to exit. */ STOP_FLAG_ASSIGN(&ctx->stop_flag, 1); + /* Closing this socket will cause mg_poll() in all the I/O threads to return + * immediately */ + closesocket(ctx->user_shutdown_notification_socket); + ctx->user_shutdown_notification_socket = + -1; /* to avoid calling closesocket() again in free_context() */ + /* Join timer thread */ #if defined(USE_TIMERS) timers_exit(ctx); @@ -20515,13 +20880,122 @@ legacy_init(const char **options) } } +/* we'll assume it's only Windows that doesn't have socketpair() available */ +#if !defined(HAVE_SOCKETPAIR) && !defined(_WIN32) +#define HAVE_SOCKETPAIR 1 +#endif + +static int +mg_socketpair(int *sockA, int *sockB) +{ + int temp[2] = {-1, -1}; + int asock = -1; + + /** Default to unallocated */ + *sockA = -1; + *sockB = -1; + +#if defined(HAVE_SOCKETPAIR) + int ret = socketpair(AF_UNIX, SOCK_STREAM, 0, temp); + if (ret == 0) { + *sockA = temp[0]; + *sockB = temp[1]; + set_close_on_exec(*sockA, NULL, NULL); + set_close_on_exec(*sockB, NULL, NULL); + } + (void)asock; /* not used */ + return ret; +#else + /** No socketpair() call is available, so we'll have to roll our own + * implementation */ + asock = socket(PF_INET, SOCK_STREAM, 0); + if (asock >= 0) { + struct sockaddr_in addr; + struct sockaddr *pa = (struct sockaddr *)&addr; + socklen_t addrLen = sizeof(addr); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; + + if ((bind(asock, pa, sizeof(addr)) == 0) + && (getsockname(asock, pa, &addrLen) == 0) + && (listen(asock, 1) == 0)) { + temp[0] = socket(PF_INET, SOCK_STREAM, 0); + if ((temp[0] >= 0) && (connect(temp[0], pa, sizeof(addr)) == 0)) { + temp[1] = accept(asock, pa, &addrLen); + if (temp[1] >= 0) { + closesocket(asock); + *sockA = temp[0]; + *sockB = temp[1]; + set_close_on_exec(*sockA, NULL, NULL); + set_close_on_exec(*sockB, NULL, NULL); + return 0; /* success! */ + } + } + } + } + + /* Cleanup */ + if (asock >= 0) + closesocket(asock); + if (temp[0] >= 0) + closesocket(temp[0]); + if (temp[1] >= 0) + closesocket(temp[1]); + return -1; /* fail! */ +#endif +} + +static int +mg_start_worker_thread(struct mg_context *ctx, int only_if_no_idle_threads) +{ + const unsigned int i = ctx->spawned_worker_threads; + if (i >= ctx->cfg_max_worker_threads) { + return -1; /* Oops, we hit our worker-thread limit! No more worker + threads, ever! */ + } + + (void)pthread_mutex_lock(&ctx->thread_mutex); +#if defined(ALTERNATIVE_QUEUE) + if ((only_if_no_idle_threads) && (ctx->idle_worker_thread_count > 0)) { +#else + if ((only_if_no_idle_threads) + && (ctx->idle_worker_thread_count + > (unsigned)(ctx->sq_head - ctx->sq_tail))) { +#endif + (void)pthread_mutex_unlock(&ctx->thread_mutex); + return -2; /* There are idle threads available, so no need to spawn a + new worker thread now */ + } + ctx->idle_worker_thread_count++; /* we do this here to avoid a race + condition while the thread is starting + up */ + (void)pthread_mutex_unlock(&ctx->thread_mutex); + + ctx->worker_connections[i].phys_ctx = ctx; + int ret = mg_start_thread_with_id(worker_thread, + &ctx->worker_connections[i], + &ctx->worker_threadids[i]); + if (ret == 0) { + ctx->spawned_worker_threads++; /* note that we've filled another slot in + the table */ + DEBUG_TRACE("Started worker_thread #%i", ctx->spawned_worker_threads); + } else { + (void)pthread_mutex_lock(&ctx->thread_mutex); + ctx->idle_worker_thread_count--; /* whoops, roll-back on error */ + (void)pthread_mutex_unlock(&ctx->thread_mutex); + } + return ret; +} CIVETWEB_API struct mg_context * mg_start2(struct mg_init_data *init, struct mg_error_data *error) { struct mg_context *ctx; const char *name, *value, *default_value; - int idx, ok, workerthreadcount; + int idx, ok, prespawnthreadcount, workerthreadcount; unsigned int i; int itmp; void (*exit_callback)(const struct mg_context *ctx) = 0; @@ -20598,6 +21072,15 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) #if defined(USE_LUA) ok &= (0 == pthread_mutex_init(&ctx->lua_bg_mutex, &pthread_mutex_attr)); #endif + + /** mg_stop() will close the user_shutdown_notification_socket, and that + * will cause poll() to return immediately in the master-thread, so that + * mg_stop() can also return immediately. + */ + ok &= (0 + == mg_socketpair(&ctx->user_shutdown_notification_socket, + &ctx->thread_shutdown_notification_socket)); + if (!ok) { unsigned error_id = (unsigned)ERRNO; const char *err_msg = @@ -20765,6 +21248,13 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) /* Worker thread count option */ workerthreadcount = atoi(ctx->dd.config[NUM_THREADS]); + prespawnthreadcount = atoi(ctx->dd.config[PRESPAWN_THREADS]); + + if ((prespawnthreadcount < 0) + || (prespawnthreadcount > workerthreadcount)) { + prespawnthreadcount = + workerthreadcount; /* can't prespawn more than all of them! */ + } if ((workerthreadcount > MAX_WORKER_THREADS) || (workerthreadcount <= 0)) { if (workerthreadcount <= 0) { @@ -21029,10 +21519,11 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) return NULL; } - ctx->cfg_worker_threads = ((unsigned int)(workerthreadcount)); - ctx->worker_threadids = (pthread_t *)mg_calloc_ctx(ctx->cfg_worker_threads, - sizeof(pthread_t), - ctx); + ctx->cfg_max_worker_threads = ((unsigned int)(workerthreadcount)); + ctx->worker_threadids = + (pthread_t *)mg_calloc_ctx(ctx->cfg_max_worker_threads, + sizeof(pthread_t), + ctx); if (ctx->worker_threadids == NULL) { const char *err_msg = "Not enough memory for worker thread ID array"; @@ -21040,8 +21531,8 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = - (unsigned)ctx->cfg_worker_threads * (unsigned)sizeof(pthread_t); + error->code_sub = (unsigned)ctx->cfg_max_worker_threads + * (unsigned)sizeof(pthread_t); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ error->text, @@ -21055,7 +21546,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) return NULL; } ctx->worker_connections = - (struct mg_connection *)mg_calloc_ctx(ctx->cfg_worker_threads, + (struct mg_connection *)mg_calloc_ctx(ctx->cfg_max_worker_threads, sizeof(struct mg_connection), ctx); if (ctx->worker_connections == NULL) { @@ -21065,7 +21556,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = (unsigned)ctx->cfg_worker_threads + error->code_sub = (unsigned)ctx->cfg_max_worker_threads * (unsigned)sizeof(struct mg_connection); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ @@ -21082,7 +21573,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) #if defined(ALTERNATIVE_QUEUE) ctx->client_wait_events = - (void **)mg_calloc_ctx(ctx->cfg_worker_threads, + (void **)mg_calloc_ctx(ctx->cfg_max_worker_threads, sizeof(ctx->client_wait_events[0]), ctx); if (ctx->client_wait_events == NULL) { @@ -21092,7 +21583,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = (unsigned)ctx->cfg_worker_threads + error->code_sub = (unsigned)ctx->cfg_max_worker_threads * (unsigned)sizeof(ctx->client_wait_events[0]); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ @@ -21108,7 +21599,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) } ctx->client_socks = - (struct socket *)mg_calloc_ctx(ctx->cfg_worker_threads, + (struct socket *)mg_calloc_ctx(ctx->cfg_max_worker_threads, sizeof(ctx->client_socks[0]), ctx); if (ctx->client_socks == NULL) { @@ -21119,7 +21610,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) if (error != NULL) { error->code = MG_ERROR_DATA_CODE_OUT_OF_MEMORY; - error->code_sub = (unsigned)ctx->cfg_worker_threads + error->code_sub = (unsigned)ctx->cfg_max_worker_threads * (unsigned)sizeof(ctx->client_socks[0]); mg_snprintf(NULL, NULL, /* No truncation check for error buffers */ @@ -21134,7 +21625,7 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) return NULL; } - for (i = 0; (unsigned)i < ctx->cfg_worker_threads; i++) { + for (i = 0; (unsigned)i < ctx->cfg_max_worker_threads; i++) { ctx->client_wait_events[i] = event_create(); if (ctx->client_wait_events[i] == 0) { const char *err_msg = "Error creating worker event %i"; @@ -21198,23 +21689,18 @@ mg_start2(struct mg_init_data *init, struct mg_error_data *error) ctx->context_type = CONTEXT_SERVER; /* server context */ /* Start worker threads */ - for (i = 0; i < ctx->cfg_worker_threads; i++) { + for (i = 0; (int)i < prespawnthreadcount; i++) { /* worker_thread sets up the other fields */ - ctx->worker_connections[i].phys_ctx = ctx; - if (mg_start_thread_with_id(worker_thread, - &ctx->worker_connections[i], - &ctx->worker_threadids[i]) - != 0) { - + if (mg_start_worker_thread(ctx, 0) != 0) { long error_no = (long)ERRNO; /* thread was not created */ - if (i > 0) { + if (ctx->spawned_worker_threads > 0) { /* If the second, third, ... thread cannot be created, set a * warning, but keep running. */ mg_cry_ctx_internal(ctx, "Cannot start worker thread %i: error %ld", - i + 1, + ctx->spawned_worker_threads + 1, error_no); /* If the server initialization should stop here, all @@ -22144,7 +22630,7 @@ mg_get_connection_info(const struct mg_context *ctx, return 0; } - if ((unsigned)idx >= ctx->cfg_worker_threads) { + if ((unsigned)idx >= ctx->cfg_max_worker_threads) { /* Out of range */ return 0; } @@ -22354,44 +22840,44 @@ mg_get_connection_info(const struct mg_context *ctx, return (int)connection_info_length; } + #if 0 -/* Get handler information. It can be printed or stored by the caller. - * Return the size of available information. */ +/* Get handler information. Not fully implemented. Is it required? */ CIVETWEB_API int mg_get_handler_info(struct mg_context *ctx, - char *buffer, - int buflen) + char *buffer, + int buflen) { - int handler_info_len = 0; - struct mg_handler_info *tmp_rh; - mg_lock_context(ctx); + int handler_info_len = 0; + struct mg_handler_info *tmp_rh; + mg_lock_context(ctx); - for (tmp_rh = ctx->dd.handlers; tmp_rh != NULL; tmp_rh = tmp_rh->next) { + for (tmp_rh = ctx->dd.handlers; tmp_rh != NULL; tmp_rh = tmp_rh->next) { - if (buflen > handler_info_len+ tmp_rh->uri_len) { - memcpy(buffer+handler_info_len, tmp_rh->uri, tmp_rh->uri_len); - } - handler_info_len += tmp_rh->uri_len; + if (buflen > handler_info_len + tmp_rh->uri_len) { + memcpy(buffer + handler_info_len, tmp_rh->uri, tmp_rh->uri_len); + } + handler_info_len += tmp_rh->uri_len; - switch (tmp_rh->handler_type) { - case REQUEST_HANDLER: - (void)tmp_rh->handler; - break; - case WEBSOCKET_HANDLER: - (void)tmp_rh->connect_handler; - (void) tmp_rh->ready_handler; - (void) tmp_rh->data_handler; - (void) tmp_rh->close_handler; - break; - case AUTH_HANDLER: - (void) tmp_rh->auth_handler; - break; - } - (void)cbdata; - } + switch (tmp_rh->handler_type) { + case REQUEST_HANDLER: + (void)tmp_rh->handler; + break; + case WEBSOCKET_HANDLER: + (void)tmp_rh->connect_handler; + (void)tmp_rh->ready_handler; + (void)tmp_rh->data_handler; + (void)tmp_rh->close_handler; + break; + case AUTH_HANDLER: + (void)tmp_rh->auth_handler; + break; + } + (void)cbdata; + } - mg_unlock_context(ctx); - return handler_info_len; + mg_unlock_context(ctx); + return handler_info_len; } #endif #endif diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h index a50be337..e0ef3443 100644 --- a/src/webserver/civetweb/civetweb.h +++ b/src/webserver/civetweb/civetweb.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2021 the Civetweb developers +/* Copyright (c) 2013-2024 the Civetweb developers * Copyright (c) 2004-2013 Sergey Lyubka * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -23,9 +23,9 @@ #ifndef CIVETWEB_HEADER_INCLUDED #define CIVETWEB_HEADER_INCLUDED -#define CIVETWEB_VERSION "1.16" +#define CIVETWEB_VERSION "1.17" #define CIVETWEB_VERSION_MAJOR (1) -#define CIVETWEB_VERSION_MINOR (16) +#define CIVETWEB_VERSION_MINOR (17) #define CIVETWEB_VERSION_PATCH (0) #ifndef CIVETWEB_API @@ -936,7 +936,8 @@ int my_send_http_error_headers(struct mg_connection *conn, int status, const char* mime_type, long long content_length); -void FTL_rewrite_pattern(char *filename, size_t filename_buf_len); +void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len); + #define MG_CONFIG_MBEDTLS_DEBUG 3 void FTL_mbed_debug(void *user_param, int level, const char *file, diff --git a/src/webserver/civetweb/handle_form.inl b/src/webserver/civetweb/handle_form.inl index be477a05..a7b7fc10 100644 --- a/src/webserver/civetweb/handle_form.inl +++ b/src/webserver/civetweb/handle_form.inl @@ -162,14 +162,17 @@ search_boundary(const char *buf, const char *boundary, size_t boundary_len) { - /* We must do a binary search here, not a string search, since the buffer - * may contain '\x00' bytes, if binary data is transferred. */ - int clen = (int)buf_len - (int)boundary_len - 4; + char *boundary_start = "\r\n--"; + size_t boundary_start_len = strlen(boundary_start); + + /* We must do a binary search here, not a string search, since the + * buffer may contain '\x00' bytes, if binary data is transferred. */ + int clen = (int)buf_len - (int)boundary_len - boundary_start_len; int i; for (i = 0; i <= clen; i++) { - if (!memcmp(buf + i, "\r\n--", 4)) { - if (!memcmp(buf + i + 4, boundary, boundary_len)) { + if (!memcmp(buf + i, boundary_start, boundary_start_len)) { + if (!memcmp(buf + i + boundary_start_len, boundary, boundary_len)) { return buf + i; } } @@ -185,7 +188,7 @@ mg_handle_form_request(struct mg_connection *conn, char path[512]; char buf[MG_BUF_LEN]; /* Must not be smaller than ~900 */ int field_storage; - int buf_fill = 0; + size_t buf_fill = 0; int r; int field_count = 0; struct mg_file fstore = STRUCT_FILE_INITIALIZER; @@ -394,10 +397,10 @@ mg_handle_form_request(struct mg_connection *conn, int end_of_key_value_pair_found = 0; int get_block; - if ((size_t)buf_fill < (sizeof(buf) - 1)) { + if (buf_fill < (sizeof(buf) - 1)) { - size_t to_read = sizeof(buf) - 1 - (size_t)buf_fill; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + size_t to_read = sizeof(buf) - 1 - buf_fill; + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { /* read error */ return -1; @@ -526,11 +529,11 @@ mg_handle_form_request(struct mg_connection *conn, buf + (size_t)used, sizeof(buf) - (size_t)used); next = buf; - buf_fill -= (int)used; - if ((size_t)buf_fill < (sizeof(buf) - 1)) { + buf_fill -= used; + if (buf_fill < (sizeof(buf) - 1)) { - size_t to_read = sizeof(buf) - 1 - (size_t)buf_fill; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + size_t to_read = sizeof(buf) - 1 - buf_fill; + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { #if !defined(NO_FILESYSTEMS) /* read error */ @@ -589,7 +592,7 @@ mg_handle_form_request(struct mg_connection *conn, /* Proceed to next entry */ used = next - buf; memmove(buf, buf + (size_t)used, sizeof(buf) - (size_t)used); - buf_fill -= (int)used; + buf_fill -= used; } return field_count; @@ -624,6 +627,7 @@ mg_handle_form_request(struct mg_connection *conn, } /* Copy boundary string to variable "boundary" */ + /* fbeg is pointer to start of value of boundary */ fbeg = content_type + bl + 9; bl = strlen(fbeg); boundary = (char *)mg_malloc(bl + 1); @@ -678,12 +682,12 @@ mg_handle_form_request(struct mg_connection *conn, for (part_no = 0;; part_no++) { size_t towrite, fnlen, n; int get_block; - size_t to_read = sizeof(buf) - 1 - (size_t)buf_fill; + size_t to_read = sizeof(buf) - 1 - buf_fill; /* Unused without filesystems */ (void)n; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { /* read error */ mg_free(boundary); @@ -701,43 +705,75 @@ mg_handle_form_request(struct mg_connection *conn, return -1; } + /* @see https://www.rfc-editor.org/rfc/rfc2046.html#section-5.1.1 + * + * multipart-body := [preamble CRLF] + * dash-boundary transport-padding CRLF + * body-part *encapsulation + * close-delimiter transport-padding + * [CRLF epilogue] + */ + if (part_no == 0) { - int d = 0; - while ((d < buf_fill) && (buf[d] != '-')) { - d++; + size_t preamble_length = 0; + /* skip over the preamble until we find a complete boundary + * limit the preamble length to prevent abuse */ + /* +2 for the -- preceding the boundary */ + while (preamble_length < 1024 + && (preamble_length < buf_fill - bl) + && strncmp(buf + preamble_length + 2, boundary, bl)) { + preamble_length++; } - if ((d > 0) && (buf[d] == '-')) { - memmove(buf, buf + d, (unsigned)buf_fill - (unsigned)d); - buf_fill -= d; + /* reset the start of buf to remove the preamble */ + if (0 == strncmp(buf + preamble_length + 2, boundary, bl)) { + memmove(buf, + buf + preamble_length, + (unsigned)buf_fill - (unsigned)preamble_length); + buf_fill -= preamble_length; buf[buf_fill] = 0; } } - if (buf[0] != '-' || buf[1] != '-') { + /* either it starts with a boundary and it's fine, or it's malformed + * because: + * - the preamble was longer than accepted + * - couldn't find a boundary at all in the body + * - didn't have a terminating boundary */ + if (buf_fill < (bl + 2) || strncmp(buf, "--", 2) + || strncmp(buf + 2, boundary, bl)) { /* Malformed request */ mg_free(boundary); return -1; } - if (0 != strncmp(buf + 2, boundary, bl)) { - /* Malformed request */ - mg_free(boundary); - return -1; + + /* skip the -- */ + char *boundary_start = buf + 2; + size_t transport_padding = 0; + while (boundary_start[bl + transport_padding] == ' ' + || boundary_start[bl + transport_padding] == '\t') { + transport_padding++; } - if (buf[bl + 2] != '\r' || buf[bl + 3] != '\n') { - /* Every part must end with \r\n, if there is another part. - * The end of the request has an extra -- */ - if (((size_t)buf_fill != (size_t)(bl + 6)) - || (strncmp(buf + bl + 2, "--\r\n", 4))) { + char *boundary_end = boundary_start + bl + transport_padding; + + /* after the transport padding, if the boundary isn't + * immediately followed by a \r\n then it is either... */ + if (strncmp(boundary_end, "\r\n", 2)) + { + /* ...the final boundary, and it is followed by --, (in which + * case it's the end of the request) or it's a malformed + * request */ + if (strncmp(boundary_end, "--", 2)) { /* Malformed request */ mg_free(boundary); return -1; } - /* End of the request */ + /* Ingore any epilogue here */ break; } + /* skip the \r\n */ + hbuf = boundary_end + 2; /* Next, we need to get the part header: Read until \r\n\r\n */ - hbuf = buf + bl + 4; hend = strstr(hbuf, "\r\n\r\n"); if (!hend) { /* Malformed request */ @@ -965,12 +1001,12 @@ mg_handle_form_request(struct mg_connection *conn, #endif /* NO_FILESYSTEMS */ memmove(buf, hend + towrite, bl + 4); - buf_fill = (int)(bl + 4); + buf_fill = bl + 4; hend = buf; /* Read new data */ - to_read = sizeof(buf) - 1 - (size_t)buf_fill; - r = mg_read(conn, buf + (size_t)buf_fill, to_read); + to_read = sizeof(buf) - 1 - buf_fill; + r = mg_read(conn, buf + buf_fill, to_read); if ((r < 0) || ((r == 0) && all_data_read)) { #if !defined(NO_FILESYSTEMS) /* read error */ @@ -989,7 +1025,7 @@ mg_handle_form_request(struct mg_connection *conn, /* buf_fill is at least 8 here */ /* Find boundary */ - next = search_boundary(buf, (size_t)buf_fill, boundary, bl); + next = search_boundary(buf, buf_fill, boundary, bl); if (!next && (r == 0)) { /* incomplete request */ @@ -1064,7 +1100,7 @@ mg_handle_form_request(struct mg_connection *conn, if (next) { used = next - buf + 2; memmove(buf, buf + (size_t)used, sizeof(buf) - (size_t)used); - buf_fill -= (int)used; + buf_fill -= used; } else { buf_fill = 0; } diff --git a/src/webserver/civetweb/match.inl b/src/webserver/civetweb/match.inl index a5011f57..34ee00ef 100644 --- a/src/webserver/civetweb/match.inl +++ b/src/webserver/civetweb/match.inl @@ -47,8 +47,8 @@ mg_match_impl(const char *pat, /* Advance as long as there are ? */ i_pat++; i_str++; - } while ((pat[i_pat] == '?') && (str[i_str] != '\0') - && (str[i_str] != '/') && (i_pat < pat_len)); + } while ((i_pat < pat_len) && (pat[i_pat] == '?') + && (str[i_str] != '\0') && (str[i_str] != '/')); /* If we have a match context, add the substring we just found */ if (mcx) { @@ -72,7 +72,7 @@ mg_match_impl(const char *pat, ptrdiff_t ret; i_pat++; - if ((pat[i_pat] == '*') && (i_pat < pat_len)) { + if ((i_pat < pat_len) && (pat[i_pat] == '*')) { /* Pattern ** matches all */ i_pat++; len = strlen(str + i_str); diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index e9d90ca5..49129e09 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -10,6 +10,8 @@ #include "civetweb_lua.h" #include "civetweb_private_lua.h" +static int +lua_error_handler(lua_State *L); #if defined(_WIN32) static void * @@ -641,14 +643,22 @@ run_lsp_kepler(struct mg_connection *conn, /* Only send a HTML header, if this is the top level page. * If this page is included by some mg.include calls, do not add a * header. */ - mg_printf(conn, "HTTP/1.1 200 OK\r\n"); + + /* Initialize a new HTTP response, either with some-predefined + * status code (e.g. 404 if this is called from an error + * handler) or with 200 OK */ + mg_response_header_start(conn, conn->status_code > 0 ? conn->status_code : 200); + + /* Add additional headers */ send_no_cache_header(conn); send_additional_header(conn); - mg_printf(conn, - "Date: %s\r\n" - "Connection: close\r\n" - "Content-Type: text/html; charset=utf-8\r\n\r\n", - date); + send_cors_header(conn); + + /* Add content type */ + mg_response_header_add(conn, "Content-Type", "text/html; charset=utf-8", -1); + + /* Send the HTTP response (status and all headers) */ + mg_response_header_send(conn); } data.begin = p; @@ -662,11 +672,19 @@ run_lsp_kepler(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); - lua_cry(conn, lua_ok, L, "LSP", "execute"); /* XXX TODO: everywhere ! */ + lua_cry(conn, lua_ok, L, "LSP Kepler", "execute"); + lua_error_handler(L); + return 1; } else { /* Success loading chunk. Call it. */ - lua_pcall(L, 0, 0, 1); + lua_ok = lua_pcall(L, 0, 0, 0); + if(lua_ok != LUA_OK) + { + lua_cry(conn, lua_ok, L, "LSP Kepler", "call"); + lua_error_handler(L); + return 1; + } } return 0; } @@ -780,9 +798,18 @@ run_lsp_civetweb(struct mg_connection *conn, /* Syntax error or OOM. * Error message is pushed on stack. */ lua_pcall(L, 1, 0, 0); + lua_cry(conn, lua_ok, L, "LSP", "call"); + lua_error_handler(L); + return 1; } else { /* Success loading chunk. Call it. */ - lua_pcall(L, 0, 0, 1); + lua_ok = lua_pcall(L, 0, 0, 0); + if(lua_ok != LUA_OK) + { + lua_cry(conn, lua_ok, L, "LSP", "execute"); + lua_error_handler(L); + return 1; + } } /* Progress until after the Lua closing tag. */ @@ -2773,18 +2800,39 @@ lua_error_handler(lua_State *L) lua_getglobal(L, "mg"); if (!lua_isnil(L, -1)) { - lua_getfield(L, -1, "write"); /* call mg.write() */ + /* Write the error message to the error log */ + lua_getfield(L, -1, "write"); lua_pushstring(L, error_msg); lua_pushliteral(L, "\n"); - lua_call(L, 2, 0); - IGNORE_UNUSED_RESULT( - luaL_dostring(L, "mg.write(debug.traceback(), '\\n')")); + lua_call(L, 2, 0); /* call mg.write(error_msg + \n) */ + lua_pop(L, 1); /* pop mg */ + + /* Get Lua traceback */ + lua_getglobal(L, "debug"); + lua_getfield(L, -1, "traceback"); + lua_call(L, 0, 1); /* call debug.traceback() */ + lua_remove(L, -2); /* remove debug */ + + /* Write the Lua traceback to the error log */ + lua_getglobal(L, "mg"); + lua_getfield(L, -1, "write"); + lua_pushvalue(L, -3); /* push the traceback */ + + /* Only print the traceback if it is not empty */ + if (strcmp(lua_tostring(L, -1), "stack traceback:") != 0) { + lua_pushliteral(L, "\n"); /* append a newline */ + lua_call(L, 2, 0); /* call mg.write(traceback + \n) */ + lua_pop(L, 2); /* pop mg and traceback */ + } else { + lua_pop(L, 3); /* pop mg, traceback and write */ + } + } else { printf("Lua error: [%s]\n", error_msg); IGNORE_UNUSED_RESULT( luaL_dostring(L, "print(debug.traceback(), '\\n')")); } - /* TODO(lsm, low): leave the stack balanced */ + lua_pop(L, 1); /* pop error message */ return 0; } @@ -2793,11 +2841,7 @@ lua_error_handler(lua_State *L) static void prepare_lua_environment(struct mg_context *ctx, struct mg_connection *conn, -#if defined(USE_WEBSOCKET) struct lua_websock_data *ws_conn_list, -#else - void *ws_conn_list, -#endif lua_State *L, const char *script_name, int lua_env_type) @@ -2943,6 +2987,11 @@ prepare_lua_environment(struct mg_context *ctx, if ((conn != NULL) && (conn->dom_ctx != NULL)) { reg_string(L, "document_root", conn->dom_ctx->config[DOCUMENT_ROOT]); + if (conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]) { + reg_string(L, + "fallback_document_root", + conn->dom_ctx->config[FALLBACK_DOCUMENT_ROOT]); + } reg_string(L, "auth_domain", conn->dom_ctx->config[AUTHENTICATION_DOMAIN]); @@ -2951,6 +3000,11 @@ prepare_lua_environment(struct mg_context *ctx, reg_string(L, "websocket_root", conn->dom_ctx->config[WEBSOCKET_ROOT]); + if (conn->dom_ctx->config[FALLBACK_WEBSOCKET_ROOT]) { + reg_string(L, + "fallback_websocket_root", + conn->dom_ctx->config[FALLBACK_WEBSOCKET_ROOT]); + } } else { reg_string(L, "websocket_root", @@ -3066,9 +3120,14 @@ mg_exec_lua_script(struct mg_connection *conn, } if (luaL_loadfile(L, path) != 0) { + mg_send_http_error(conn, 500, "Lua error:\r\n"); lua_error_handler(L); } else { - lua_pcall(L, 0, 0, -2); + int call_status = lua_pcall(L, 0, 0, 0); + if (call_status != 0) { + mg_send_http_error(conn, 500, "Lua error:\r\n"); + lua_error_handler(L); + } } DEBUG_TRACE("Close Lua environment %p", L); lua_close(L); diff --git a/src/webserver/civetweb/mod_mbedtls.inl b/src/webserver/civetweb/mod_mbedtls.inl index d9675580..5dd1f438 100644 --- a/src/webserver/civetweb/mod_mbedtls.inl +++ b/src/webserver/civetweb/mod_mbedtls.inl @@ -87,19 +87,23 @@ mbed_sslctx_init(SSL_CTX *ctx, const char *crt) mbedtls_ssl_conf_dbg(conf, FTL_mbed_debug, NULL); /****************************************************/ -#ifdef MBEDTLS_SSL_PROTO_TLS1_3 - psa_status_t status = psa_crypto_init(); + /* Initialize TLS key and cert */ + mbedtls_pk_init(&ctx->pkey); + mbedtls_ctr_drbg_init(&ctx->ctr); + mbedtls_x509_crt_init(&ctx->cert); + +#ifdef MBEDTLS_PSA_CRYPTO_C + /* Initialize PSA crypto (mandatory with TLS 1.3) + * This must be done before calling any other PSA Crypto + * functions or they will fail with PSA_ERROR_BAD_STATE + */ + const psa_status_t status = psa_crypto_init(); if (status != PSA_SUCCESS) { DEBUG_TRACE("Failed to initialize PSA crypto, returned %d\n", (int) status); return -1; } #endif - /* Initialize TLS key and cert */ - mbedtls_pk_init(&ctx->pkey); - mbedtls_ctr_drbg_init(&ctx->ctr); - mbedtls_x509_crt_init(&ctx->cert); - rc = mbedtls_ctr_drbg_seed(&ctx->ctr, mbedtls_entropy_func, &ctx->entropy, @@ -153,19 +157,6 @@ mbed_sslctx_init(SSL_CTX *ctx, const char *crt) DEBUG_TRACE("TLS cannot set certificate and private key (%i)", rc); return -1; } - -// /* Set ciphersuites */ -// static const int tls_cipher_suites[] = { -// MBEDTLS_CIPHER_CHACHA20_POLY1305, -// MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, -// MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, -// 0 -// }; -// mbedtls_ssl_conf_ciphersuites(conf, tls_cipher_suites); -// -// /* Set protocol version */ -// mbedtls_ssl_conf_min_version(conf, MBEDTLS_SSL_MAJOR_VERSION_3, MBEDTLS_SSL_MINOR_VERSION_3); - return 0; } @@ -213,7 +204,13 @@ mbed_ssl_accept(mbedtls_ssl_context **ssl, return -1; } - DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->MBEDTLS_PRIVATE(state)); +#if MBEDTLS_VERSION_NUMBER >= 0x03000000 + DEBUG_TRACE("TLS connection %p accepted, state: %d", + ssl, + (*ssl)->MBEDTLS_PRIVATE(state)); +#else + DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->state); +#endif return 0; } @@ -239,7 +236,13 @@ mbed_ssl_handshake(mbedtls_ssl_context *ssl) } } - DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->MBEDTLS_PRIVATE(state)); +#if MBEDTLS_VERSION_NUMBER >= 0x03000000 + DEBUG_TRACE("TLS handshake rc: %d, state: %d", + rc, + ssl->MBEDTLS_PRIVATE(state)); +#else + DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->state); +#endif return rc; } @@ -249,13 +252,6 @@ mbed_ssl_read(mbedtls_ssl_context *ssl, unsigned char *buf, int len) { int rc = mbedtls_ssl_read(ssl, buf, len); /* DEBUG_TRACE("mbedtls_ssl_read: %d", rc); */ - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && defined(MBEDTLS_CLIENT_SSL_SESSION_TICKETS) - if (ret == MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET) { - DEBUG_TRACE("got session ticket in TLS 1.3 connection, retrying read"); - rc = mbedtls_ssl_read(ssl, buf, len); - } -#endif return rc; } diff --git a/src/webserver/civetweb/timer.inl b/src/webserver/civetweb/timer.inl index 9b8d5539..39d68dfc 100644 --- a/src/webserver/civetweb/timer.inl +++ b/src/webserver/civetweb/timer.inl @@ -39,13 +39,16 @@ TIMER_API double timer_getcurrenttime(struct mg_context *ctx) { #if defined(_WIN32) + uint64_t now_tick64 = 0; +#if defined(_WIN64) + now_tick64 = GetTickCount64(); +#else /* GetTickCount returns milliseconds since system start as * unsigned 32 bit value. It will wrap around every 49.7 days. * We need to use a 64 bit counter (will wrap in 500 mio. years), * by adding the 32 bit difference since the last call to a * 64 bit counter. This algorithm will only work, if this * function is called at least once every 7 weeks. */ - uint64_t now_tick64 = 0; DWORD now_tick = GetTickCount(); if (ctx->timers) { @@ -55,6 +58,7 @@ timer_getcurrenttime(struct mg_context *ctx) ctx->timers->last_tick = now_tick; pthread_mutex_unlock(&ctx->timers->mutex); } +#endif return (double)now_tick64 * 1.0E-3; #else struct timespec now_ts; diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index da9cca3e..75e687ba 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -8,11 +8,11 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" -#include "http-common.h" -#include "../config/config.h" -#include "../log.h" -#include "json_macros.h" +#include "FTL.h" +#include "webserver/http-common.h" +#include "config/config.h" +#include "log.h" +#include "webserver/json_macros.h" // UINT_MAX #include // HUGE_VAL @@ -68,27 +68,33 @@ int send_http_code(struct ftl_conn *api, const char *mime_type, int send_json_unauthorized(struct ftl_conn *api) { - return send_json_error(api, 401, - "unauthorized", - "Unauthorized", - NULL); + // Log API warnings only if debug.api is true + return send_json_error_free(api, 401, + "unauthorized", + "Unauthorized", + NULL, false, + config.debug.api.v.b); } int send_json_error(struct ftl_conn *api, const int code, const char *key, const char* message, const char *hint) { - return send_json_error_free(api, code, key, message, (char*)hint, false); + return send_json_error_free(api, code, key, message, + (char*)hint, false, true); } int send_json_error_free(struct ftl_conn *api, const int code, const char *key, const char* message, - char *hint, bool free_hint) + char *hint, const bool free_hint, const bool log) { - if(hint != NULL) - log_warn("API: %s (%s)", message, hint); - else - log_warn("API: %s", message); + if(log) + { + if(hint != NULL) + log_warn("API: %s (%s)", message, hint); + else + log_warn("API: %s", message); + } cJSON *error = JSON_NEW_OBJECT(); JSON_REF_STR_IN_OBJECT(error, "key", key); @@ -515,8 +521,7 @@ void read_and_parse_payload(struct ftl_conn *api) api->payload.avail = true; // Try to parse possibly existing JSON payload - api->payload.json = cJSON_Parse(api->payload.raw); - api->payload.json_error = cJSON_GetErrorPtr(); + api->payload.json = cJSON_ParseWithOpts(api->payload.raw, &api->payload.json_error, 0); } // Escape a string to mask HTML special characters, the resulting string is @@ -570,6 +575,69 @@ char *__attribute__((malloc)) escape_html(const char *string) return escaped; } +// Check if the payload is valid JSON, if not send an error response with the +// appropriate status code. If the payload is NULL, send a 400 Bad Request +// response with a hint that no payload was received. If the payload is not +// valid JSON, send a 400 Bad Request response with a hint that the payload is +// invalid JSON. +int check_json_payload(struct ftl_conn *api) +{ + if (api->payload.json == NULL) + { + if (api->payload.json_error == NULL) + return send_json_error(api, 400, + "bad_request", + "No request body data", + NULL); + else + return send_json_error(api, 400, + "bad_request", + "Invalid request body data (no valid JSON), error at hint", + api->payload.json_error); + } + + // All okay + return 0; +} + +// Black magic at work here: We build a JSON array from the group_concat result +// delivered from the database, parse it as valid array and append it as row to +// the data +int parse_groupIDs(struct ftl_conn *api, tablerow *table, cJSON *row) +{ + const size_t buflen = strlen(table->group_ids) + 3u; + char *group_ids_str = calloc(buflen, sizeof(char)); + if(group_ids_str == NULL) + { + return send_json_error(api, 500, // 500 Internal Server Error + "out_of_memory", + "Out of memory", + NULL); + } + group_ids_str[0] = '['; + strcpy(group_ids_str+1u , table->group_ids); + group_ids_str[buflen-2u] = ']'; + group_ids_str[buflen-1u] = '\0'; + const char *json_error = NULL; + cJSON *group_ids = cJSON_ParseWithOpts(group_ids_str, &json_error, false); + free(group_ids_str); + if(group_ids == NULL) + { + // Error parsing group_ids, substitute empty array + // Note: This should never happen as the database's aggregate + // function should always return a valid JSON array + log_err("Error parsing group_ids, error at: %.20s", json_error); + JSON_ADD_ITEM_TO_OBJECT(row, "groups", JSON_NEW_ARRAY()); + } + else + { + JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids); + } + + // Success + return 0; +} + // Escape a string to mask JSON special characters, the resulting string is // always allocated and must be freed (unless NULL is returned) // See https://tools.ietf.org/html/rfc8259#section-7 @@ -594,3 +662,46 @@ char *__attribute__((malloc)) escape_json(const char *string) // Return the JSON escaped string return namep; } + +// Remove duplicates from a cJSON array +// This function uses the less efficient cJSON_GetArraySize() function compared +// to cJSON_ArrayForEach() as we are going to modify the array in-place while +// iterating over it +void cJSON_unique_array(cJSON *array) +{ + // Check if the array is an array + if(!cJSON_IsArray(array)) + return; + + for(int oi = 0; oi < cJSON_GetArraySize(array); oi++) + { + // Get the outer item + cJSON *outer_item = cJSON_GetArrayItem(array, oi); + // Check if the item is a string + if (!cJSON_IsString(outer_item)) + continue; + + // Check for duplicates in the remainder of the array + for(int ii = oi + 1; ii < cJSON_GetArraySize(array); ii++) + { + // Get the inner item + cJSON *inner_item = cJSON_GetArrayItem(array, ii); + // Check if the inner item is a string + if (!cJSON_IsString(inner_item)) + continue; + + // Compare the two strings + if(strcmp(outer_item->valuestring, inner_item->valuestring) == 0) + { + // Remove the duplicate item, this is safe as we are + // at least one item ahead of the outer item + cJSON_DeleteItemFromArray(array, ii); + // Compensate for removed item (the for loop + // will increment ii for the next step, thus + // we need to decrement it here) + ii--; + continue; + } + } + } +} diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index 7fba7888..ad85e11b 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -15,30 +15,19 @@ #include "webserver/cJSON/cJSON.h" // enum fifo_logs #include "enums.h" +// tablerow +#include "database/gravity-db.h" // strlen() #include +// struct session +#include "api/auth.h" + // API-internal definitions // Maximum size of received and processed payload: 64 KB #define MAX_PAYLOAD_BYTES 64*1024 -enum http_method { - HTTP_UNKNOWN = 0, - HTTP_GET = 1 << 0, - HTTP_POST = 1 << 1, - HTTP_PUT = 1 << 2, - HTTP_PATCH = 1 << 3, - HTTP_DELETE = 1 << 4, - HTTP_OPTIONS = 1 << 5, -}; - -enum api_flags { - API_FLAG_NONE = 0, - API_DOMAINS = 1 << 0, - API_PARSE_JSON = 1 << 1, - API_BATCHDELETE = 1 << 2, -}; struct api_options { enum api_flags flags; @@ -51,6 +40,7 @@ struct ftl_conn { const enum http_method method; char *action_path; const char *item; + const char *message; int user_id; double now; struct { @@ -61,8 +51,10 @@ struct ftl_conn { long unsigned int size; } payload; struct { - bool restart; + bool restart :1; + const char *restart_reason; } ftl; + struct session *session; struct api_options opts; }; @@ -79,7 +71,7 @@ int send_json_error(struct ftl_conn *api, const int code, const char *hint); int send_json_error_free(struct ftl_conn *api, const int code, const char *key, const char* message, - char *hint, bool free_hint); + char *hint, bool free_hint, const bool log); int send_json_success(struct ftl_conn *api); const char *get_http_method_str(const enum http_method method) __attribute__((const)); @@ -109,6 +101,9 @@ enum http_method __attribute__((pure)) http_method(struct mg_connection *conn); const char* __attribute__((pure)) startsWith(const char *path, struct ftl_conn *api); void read_and_parse_payload(struct ftl_conn *api); char * __attribute__((malloc)) escape_html(const char *string); +int check_json_payload(struct ftl_conn *api); +int parse_groupIDs(struct ftl_conn *api, tablerow *table, cJSON *row); char * __attribute__((malloc)) escape_json(const char *string); +void cJSON_unique_array(cJSON *array); #endif // HTTP_H diff --git a/src/webserver/json_macros.h b/src/webserver/json_macros.h index f3966f08..6c12c650 100644 --- a/src/webserver/json_macros.h +++ b/src/webserver/json_macros.h @@ -12,10 +12,10 @@ // logging routines #include "log.h" -#define JSON_NEW_OBJECT() cJSON_CreateObject(); -#define JSON_NEW_ARRAY() cJSON_CreateArray(); +#define JSON_NEW_OBJECT() cJSON_CreateObject() +#define JSON_NEW_ARRAY() cJSON_CreateArray() -#define JSON_ADD_ITEM_TO_ARRAY(array, item) cJSON_AddItemToArray(array, item); +#define JSON_ADD_ITEM_TO_ARRAY(array, item) cJSON_AddItemToArray(array, item) #define JSON_COPY_STR_TO_OBJECT(object, key, string)({ \ cJSON *string_item = NULL; \ @@ -57,6 +57,29 @@ cJSON_AddItemToObject(object, key, string_item); \ }) +// Hand over allocated string to cJSON - it will thereafter take care of freeing +// it when the cJSON object is deleted +#define JSON_GIVE_STR_TO_OBJECT(object, key, string)({ \ + cJSON *string_item = NULL; \ + if(string != NULL) \ + { \ + string_item = cJSON_CreateStringReference((const char*)(string)); \ + string_item->type &= ~cJSON_IsReference; \ + } \ + else \ + { \ + string_item = cJSON_CreateNull(); \ + } \ + if(string_item == NULL) \ + { \ + cJSON_Delete(object); \ + send_http_internal_error(api); \ + log_err("JSON_GIVE_STR_TO_OBJECT FAILED (key: \"%s\", string: \"%s\")!", key, string); \ + return 500; \ + } \ + cJSON_AddItemToObject(object, key, string_item); \ +}) + #define JSON_ADD_NUMBER_TO_OBJECT(object, key, num)({ \ const double number = num; \ if(cJSON_AddNumberToObject(object, key, number) == NULL) \ @@ -254,3 +277,21 @@ #define JSON_INCREMENT_NUMBER(number_obj, inc)({ \ cJSON_SetNumberHelper(number_obj, number_obj->valuedouble + inc); \ }) + +// Returns true if the key exists and is true, otherwise false +#define JSON_KEY_TRUE(obj, key)({ \ + cJSON *elem = cJSON_GetObjectItemCaseSensitive(obj, key); \ + elem != NULL ? cJSON_IsTrue(elem) : false; \ +}) + +#define cJSON_AddStringReferenceToObject(object, key, string) \ + cJSON_AddItemToObject(object, key, cJSON_CreateStringReference((const char*)(string))) + +#define cJSON_AddStringReferenceToArray(array, string) \ + cJSON_AddItemToArray(array, cJSON_CreateStringReference((const char*)(string))) + +#define cJSON_AddNumberToArray(array, num) \ + cJSON_AddItemToArray(array, cJSON_CreateNumber(num)) + +#define cJSON_AddStringToArray(array, string) \ + cJSON_AddItemToArray(array, cJSON_CreateString(string)) diff --git a/src/webserver/lua_web.c b/src/webserver/lua_web.c index 72b26a28..e1087035 100644 --- a/src/webserver/lua_web.c +++ b/src/webserver/lua_web.c @@ -57,12 +57,7 @@ void free_lua(void) void init_lua(const struct mg_connection *conn, void *L, unsigned context_flags) { - // Set onerror handler to print errors to the log - if(luaL_dostring(L, "mg.onerror = function(e) mg.cry('Error at ' .. e) end") != LUA_OK) - { - log_err("Error setting Lua onerror handler: %s", lua_tostring(L, -1)); - lua_pop(L, 1); - } + return; } int request_handler(struct mg_connection *conn, void *cbdata) @@ -74,7 +69,6 @@ int request_handler(struct mg_connection *conn, void *cbdata) /* Handler may access the request info using mg_get_request_info */ const struct mg_request_info *req_info = mg_get_request_info(conn); - const size_t uri_raw_len = strlen(req_info->local_uri_raw); // Build minimal api struct to check authentication struct ftl_conn api = { 0 }; @@ -95,8 +89,7 @@ int request_handler(struct mg_connection *conn, void *cbdata) return send_json_error_free(&api, 400, "bad_request", "Bad request", - hint, - true); + hint, true, true); } // Check if last part of the URI contains a dot (is a file) @@ -123,48 +116,11 @@ int request_handler(struct mg_connection *conn, void *cbdata) // Check if the user is authenticated if(!authorized) { - // Append query string to target - char *target = NULL; - if(req_info->query_string != NULL) - { - target = calloc(uri_raw_len + strlen(req_info->query_string) + 2u, sizeof(char)); - strcpy(target, req_info->local_uri_raw); - strcat(target, "?"); - strcat(target, req_info->query_string); - } - else - { - target = strdup(req_info->local_uri_raw); - } - if(target == NULL) - { - log_err("Error allocating memory for redirection target"); - return send_json_error(&api, 500, - "internal_error", - "Internal server error", - "Cannot allocate memory for redirection target"); - } - - // Encode target string - const size_t encoded_target_len = strlen(target) * 3u + 1u; - char *encoded_target = calloc(encoded_target_len, sizeof(char)); - if(encoded_target == NULL) - { - log_err("Error allocating memory for encoded redirection target"); - return send_json_error(&api, 500, - "internal_error", - "Internal server error", - "Cannot allocate memory for encoded redirection target"); - } - - // Encode target string - mg_url_encode(target, encoded_target, encoded_target_len); - free(target); - // User is not authenticated, redirect to login page - log_web("Authentication required, redirecting to %slogin?target=%s", config.webserver.paths.webhome.v.s, encoded_target); - mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: %slogin?target=%s\r\n\r\n", config.webserver.paths.webhome.v.s, encoded_target); - free(encoded_target); + log_web("Authentication required, redirecting to %slogin", + config.webserver.paths.webhome.v.s); + mg_printf(conn, "HTTP/1.1 302 Found\r\nLocation: %slogin\r\n\r\n", + config.webserver.paths.webhome.v.s); return 302; } } diff --git a/src/webserver/lua_web.h b/src/webserver/lua_web.h index 478a28a0..5c0b2fe6 100644 --- a/src/webserver/lua_web.h +++ b/src/webserver/lua_web.h @@ -18,4 +18,4 @@ void free_lua(void); void init_lua(const struct mg_connection *conn, void *L, unsigned context_flags); int request_handler(struct mg_connection *conn, void *cbdata); -#endif // LUA_WEB_H \ No newline at end of file +#endif // LUA_WEB_H diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 5485046f..cc4fe929 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -28,6 +28,8 @@ #include "webserver/lua_web.h" // log_certificate_domain_mismatch() #include "database/message-table.h" +// create_cli_password() +#include "config/password.h" // Server context handle static struct mg_context *ctx = NULL; @@ -301,6 +303,7 @@ unsigned short get_api_string(char **buf, const bool domain) if(this_len < 0) { log_err("Failed to append API URL to buffer: %s", strerror(errno)); + free(api_str); return 0; } @@ -309,6 +312,7 @@ unsigned short get_api_string(char **buf, const bool domain) if((size_t)this_len >= bufsz - len - 1) { log_err("API URL buffer too small!"); + free(api_str); return 0; } @@ -316,8 +320,8 @@ unsigned short get_api_string(char **buf, const bool domain) if(memmem(*buf, len, api_str, this_len) != NULL) { // This string is already present, so skip it - free(api_str); log_debug(DEBUG_API, "Skipping duplicate API URL: %s", api_str); + free(api_str); continue; } @@ -353,7 +357,7 @@ void http_init(void) MG_FEATURES_IPV6 | MG_FEATURES_CACHE; -#ifdef HAVE_TLS +#ifdef HAVE_MBEDTLS features |= MG_FEATURES_TLS; #endif @@ -394,6 +398,13 @@ void http_init(void) // send no referrer information. // The latter four headers are set as expected by https://securityheaders.io char num_threads[3] = { 0 }; + // Use 16 threads if more than 8 cores are available, otherwise use + // 2*cores. This is to prevent overloading the system with too many + // threads. + // We use the number of available (= online) cores which may be less + // than the total number of cores in the system, e.g., if a + // virtualization environment is used and fewer cores are assigned to + // the VM than are available on the host. sprintf(num_threads, "%d", get_nprocs() > 8 ? 16 : 2*get_nprocs()); const char *options[] = { "document_root", config.webserver.paths.webroot.v.s, @@ -419,9 +430,19 @@ void http_init(void) // from the end of the array. unsigned int next_option = ArraySize(options) - 6; -#ifdef HAVE_TLS +#ifdef HAVE_MBEDTLS // Add TLS options if configured - if(config.webserver.tls.cert.v.s != NULL && + + // TLS is used when webserver.port contains "s" (e.g. "443s") + const bool tls_used = config.webserver.port.v.s != NULL && + strchr(config.webserver.port.v.s, 's') != NULL; + + // Check certificate domain if + // - TLS is used + // - A certificate is configured + // - The certificate is readable + if(tls_used && + config.webserver.tls.cert.v.s != NULL && strlen(config.webserver.tls.cert.v.s) > 0) { // Try to generate certificate if not present @@ -439,6 +460,8 @@ void http_init(void) } } + // Check if the certificate is readable (we may have just + // created it) if(file_readable(config.webserver.tls.cert.v.s)) { if(read_certificate(config.webserver.tls.cert.v.s, config.webserver.domain.v.s, false) != CERT_DOMAIN_MATCH) @@ -469,7 +492,8 @@ void http_init(void) } // Configure logging handlers - struct mg_callbacks callbacks = { NULL }; + struct mg_callbacks callbacks; + memset(&callbacks, 0, sizeof(callbacks)); callbacks.log_message = log_http_message; callbacks.log_access = log_http_access; callbacks.init_lua = init_lua; @@ -529,6 +553,9 @@ void http_init(void) // Restore sessions from database init_api(); + + // Create CLI password (if enabled) + create_cli_password(); } static char *append_to_path(char *path, const char *append) @@ -547,8 +574,9 @@ static char *append_to_path(char *path, const char *append) return new_path; } -void FTL_rewrite_pattern(char *filename, size_t filename_buf_len) +void FTL_rewrite_pattern(char *filename, unsigned long filename_buf_len) { + log_debug(DEBUG_API, "Rewriting filename: %s", filename); const bool trailing_slash = filename[strlen(filename) - 1] == '/'; char *filename_lp = NULL; @@ -578,7 +606,7 @@ void FTL_rewrite_pattern(char *filename, size_t filename_buf_len) filename_lp = append_to_path(filename, ".lp"); if(filename_lp == NULL) { - //Failed to allocate memory for filename!"); + // Failed to allocate memory for filename return; } @@ -622,6 +650,9 @@ void http_terminate(void) // Free Lua-related resources free_lua(); + // Remove CLI password + remove_cli_password(); + // Free error_pages path if(error_pages != NULL) { diff --git a/src/webserver/webserver.h b/src/webserver/webserver.h index d87001f4..4fff052a 100644 --- a/src/webserver/webserver.h +++ b/src/webserver/webserver.h @@ -18,4 +18,4 @@ void http_terminate(void); in_port_t get_https_port(void) __attribute__((pure)); unsigned short get_api_string(char **buf, const bool domain); -#endif // WEBSERVER_H \ No newline at end of file +#endif // WEBSERVER_H diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 7c2a3d82..d53f65e7 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -11,11 +11,17 @@ #include "FTL.h" #include "log.h" #include "x509.h" -#include -#include -#include -#include -#include + +#ifdef HAVE_MBEDTLS +# include +# include +# include + +// We enforce at least mbedTLS v3.5.0 if we use it +#if MBEDTLS_VERSION_NUMBER < 0x03050000 +# error "mbedTLS version 3.5.0 or later is required" +#endif + #define RSA_KEY_SIZE 4096 #define BUFFER_SIZE 16000 @@ -104,6 +110,10 @@ static bool write_to_file(const char *filename, const char *type, const char *su return false; } + // Restrict permissions to owner read/write only + if(fchmod(fileno(f), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", targetname, strerror(errno)); + // Write key (if provided) if(key != NULL) { @@ -228,6 +238,9 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) char not_after[16] = { 0 }; strftime(not_before, sizeof(not_before), "%Y%m%d%H%M%S", tm); tm->tm_year += 30; // 30 years from now + // Check for leap year, and adjust the date accordingly + const bool isLeapYear = tm->tm_year % 4 == 0 && (tm->tm_year % 100 != 0 || tm->tm_year % 400 == 0); + tm->tm_mday = tm->tm_mon == 2 && tm->tm_mday == 29 && !isLeapYear ? 28 : tm->tm_mday; strftime(not_after, sizeof(not_after), "%Y%m%d%H%M%S", tm); // 1. Create CA certificate @@ -621,3 +634,19 @@ end: return CERT_OKAY; } + +#else + +bool generate_certificate(const char* certfile, bool rsa, const char *domain) +{ + log_err("FTL was not compiled with mbedtls support"); + return false; +} + +enum cert_check read_certificate(const char* certfile, const char *domain, const bool private_key) +{ + log_err("FTL was not compiled with mbedtls support"); + return CERT_FILE_NOT_FOUND; +} + +#endif diff --git a/src/webserver/x509.h b/src/webserver/x509.h index e59ee1a7..1c6f4af6 100644 --- a/src/webserver/x509.h +++ b/src/webserver/x509.h @@ -10,8 +10,10 @@ #ifndef X509_H #define X509_H -#include -#include +#ifdef HAVE_MBEDTLS +# include +# include +#endif #include "enums.h" diff --git a/src/zip/gzip.c b/src/zip/gzip.c index 74aed94c..d930656a 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -8,14 +8,19 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ +#include "FTL.h" +#include "gzip.h" +#include "log.h" + #include #include #include #include // le32toh and friends +#ifndef __USE_MISC +#define __USE_MISC +#endif #include -#include "gzip.h" -#include "log.h" static int mz_uncompress2_raw(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong *pSource_len); @@ -92,7 +97,7 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul // ITU-T V.42.) // isize: This contains the size of the original (uncompressed) input // data modulo 2^32 (little endian). - const uint32_t crc = mz_crc32(MZ_CRC32_INIT, buffer_uncompressed, size_uncompressed); + const uint32_t crc = (uint32_t)mz_crc32(MZ_CRC32_INIT, buffer_uncompressed, size_uncompressed); memcpy(*buffer_compressed + *size_compressed, &crc, sizeof(crc)); *size_compressed += sizeof(crc); const uint32_t isize = htole32(size_uncompressed); @@ -311,9 +316,21 @@ bool inflate_file(const char *infilename, const char *outfilename, bool verbose) return false; } + // Restrict permissions to owner read/write only + if(fchmod(fileno(outfile), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", outfilename, strerror(errno)); + // Get file size fseek(infile, 0, SEEK_END); - const mz_ulong size_compressed = ftell(infile); + const long sc = ftell(infile); + if(sc < 0) + { + log_warn("Failed to get file size of %s", infilename); + fclose(infile); + fclose(outfile); + return false; + } + const mz_ulong size_compressed = (mz_ulong)sc; fseek(infile, 0, SEEK_SET); // Read file into memory @@ -396,9 +413,13 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) return false; } + // Restrict permissions to owner read/write only + if(fchmod(fileno(outfile), S_IRUSR | S_IWUSR) != 0) + log_warn("Unable to set permissions on file \"%s\": %s", outfilename, strerror(errno)); + // Get file size fseek(infile, 0, SEEK_END); - const mz_ulong size_uncompressed = ftell(infile); + const long size_uncompressed = ftell(infile); fseek(infile, 0, SEEK_SET); // Read file into memory @@ -410,7 +431,7 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) fclose(outfile); return false; } - if(fread(buffer_uncompressed, 1, size_uncompressed, infile) != size_uncompressed) + if(fread(buffer_uncompressed, 1, size_uncompressed, infile) != (size_t)size_uncompressed) { log_warn("Failed to read %lu bytes from %s", (unsigned long)size_uncompressed, infilename); fclose(infile); diff --git a/src/zip/miniz/CMakeLists.txt b/src/zip/miniz/CMakeLists.txt index c6e7de96..5047aea3 100644 --- a/src/zip/miniz/CMakeLists.txt +++ b/src/zip/miniz/CMakeLists.txt @@ -14,5 +14,5 @@ set(sources ) add_library(miniz OBJECT ${sources}) -target_compile_options(miniz PRIVATE) +target_compile_options(miniz PRIVATE -Wno-padded -Wno-type-limits) target_include_directories(miniz PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/zip/miniz/miniz.h b/src/zip/miniz/miniz.h index d6a354bf..35c740c7 100644 --- a/src/zip/miniz/miniz.h +++ b/src/zip/miniz/miniz.h @@ -1419,4 +1419,4 @@ MINIZ_EXPORT void *mz_zip_extract_archive_file_to_heap_v2(const char *pZip_filen } #endif -#endif /* MINIZ_NO_ARCHIVE_APIS */ \ No newline at end of file +#endif /* MINIZ_NO_ARCHIVE_APIS */ diff --git a/src/zip/tar.c b/src/zip/tar.c index 5e497622..fa65cd6d 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -125,4 +125,4 @@ cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, c } while (p + newOffset + TAR_BLOCK_SIZE <= tarSize); return files; -} \ No newline at end of file +} diff --git a/src/zip/tar.h b/src/zip/tar.h index 11f5e200..a8f30769 100644 --- a/src/zip/tar.h +++ b/src/zip/tar.h @@ -16,4 +16,4 @@ const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize) __attribute__((nonnull (1,3,4))); cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) __attribute__((nonnull (1))); -#endif // TAR_H \ No newline at end of file +#endif // TAR_H diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index b9970e22..0185edb8 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -37,11 +37,13 @@ #include "webserver/cJSON/cJSON.h" // set_event() #include "events.h" - +// JSON_KEY_TRUE +#include "webserver/json_macros.h" +// exit_code +#include "signals.h" // Tables to copy from the gravity database to the Teleporter database static const char *gravity_tables[] = { - "info", "group", "adlist", "adlist_by_group", @@ -181,7 +183,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], voi if(file_exists(file_path) && !mz_zip_writer_add_file(zip, file_path+1, file_path, file_comment, (uint16_t)strlen(file_comment), MZ_BEST_COMPRESSION)) { mz_zip_writer_end(zip); - return "Failed to add /etc/hosts to heap ZIP archive!"; + return "Failed to add /etc/pihole/dhcp.leases to heap ZIP archive!"; } const char *directory = "/etc/dnsmasq.d"; @@ -277,7 +279,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], voi // Generate filename for ZIP archive (it has both the hostname and the // current datetime) - char timestr[TIMESTR_SIZE] = ""; + char timestr[TIMESTR_SIZE]; get_timestr(timestr, time(NULL), false, true); snprintf(filename, 128, "pi-hole_%s_teleporter_%s.zip", hostname(), timestr); @@ -365,7 +367,7 @@ static const char *import_dhcp_leases(void *ptr, size_t size, char * const hint) } static const char *test_and_import_database(void *ptr, size_t size, const char *destination, - const char **tables, const unsigned int num_tables, + const char **tables, const size_t num_tables, char * const hint) { // Check if the file is empty @@ -523,7 +525,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char * return NULL; } -const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files) +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *import, cJSON *imported_files) { // Initialize ZIP archive mz_zip_archive zip = { 0 }; @@ -584,9 +586,19 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con file_stat.m_comment, (unsigned long)file_stat.m_time); // Process file + const char *import_tables[ArraySize(gravity_tables)] = { NULL }; + size_t num_tables = 0u; // Is this "etc/pihole/pihole.toml" ? - if(strcmp(file_stat.m_filename, "etc/pihole/pihole.toml") == 0) + if(strcmp(file_stat.m_filename, extract_files[0]) == 0) { + // Check whether we should import this file + if(import != NULL && !JSON_KEY_TRUE(import, "config")) + { + log_info("Ignoring file %s in Teleporter archive (not in import list)", file_stat.m_filename); + free(ptr); + continue; + } + // Import Pi-hole configuration memset(hint, 0, ERRBUF_SIZE); const char *err = test_and_import_pihole_toml(ptr, file_stat.m_uncomp_size, hint); @@ -598,8 +610,16 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con log_debug(DEBUG_CONFIG, "Imported Pi-hole configuration: %s", file_stat.m_filename); } // Is this "etc/pihole/dhcp.leases"? - else if(strcmp(file_stat.m_filename, "etc/pihole/dhcp.leases") == 0) + else if(strcmp(file_stat.m_filename, extract_files[1]) == 0) { + // Check whether we should import this file + if(import != NULL && !JSON_KEY_TRUE(import, "dhcp_leases")) + { + log_info("Ignoring file %s in Teleporter archive (not in import list)", file_stat.m_filename); + free(ptr); + continue; + } + // Import DHCP leases memset(hint, 0, ERRBUF_SIZE); const char *err = import_dhcp_leases(ptr, file_stat.m_uncomp_size, hint); @@ -610,22 +630,88 @@ const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * con } log_debug(DEBUG_CONFIG, "Imported DHCP leases: %s", file_stat.m_filename); } - else if(strcmp(file_stat.m_filename, "etc/pihole/gravity.db") == 0) + // Is this "etc/pihole/gravity.db"? + else if(strcmp(file_stat.m_filename, extract_files[2]) == 0) { + // Check whether we should import this file + if(import != NULL && !cJSON_HasObjectItem(import, "gravity")) + { + log_info("Ignoring file %s in Teleporter archive (not in import list)", file_stat.m_filename); + free(ptr); + continue; + } + + if(import == NULL) + { + // Import all tables + num_tables = ArraySize(gravity_tables); + memcpy(import_tables, gravity_tables, sizeof(gravity_tables)); + } + else + { + // Get object at import.gravity + cJSON *import_gravity = cJSON_GetObjectItem(import, "gravity"); + + // Check if import.gravity is a JSON object + if(import_gravity == NULL || !cJSON_IsObject(import_gravity)) + { + log_warn("Ignoring file %s in Teleporter archive (import.gravity is not a JSON object)", file_stat.m_filename); + free(ptr); + continue; + } + + // Import selected tables from import.gravity object + for(size_t j = 0; j < ArraySize(gravity_tables); j++) + { + if(JSON_KEY_TRUE(import_gravity, gravity_tables[j])) + import_tables[num_tables++] = gravity_tables[j]; + else + log_info("Ignoring table %s in %s (not in import list)", gravity_tables[j], file_stat.m_filename); + } + } + // Import gravity database memset(hint, 0, ERRBUF_SIZE); const char *err = test_and_import_database(ptr, file_stat.m_uncomp_size, config.files.gravity.v.s, - gravity_tables, ArraySize(gravity_tables), hint); + import_tables, num_tables, hint); if(err != NULL) { free(ptr); return err; } log_debug(DEBUG_CONFIG, "Imported database: %s", file_stat.m_filename); + + // Add filename of processed files to JSON array + for(unsigned j = 0; j < num_tables; j++) + { + const size_t len = strlen(file_stat.m_filename) + 3 + strlen(import_tables[j]); + char *tablename = calloc(len, sizeof(char)); + if(tablename == NULL) + { + log_err("Failed to allocate memory for table name"); + free(ptr); + continue; + } + + // Create imported pseudo file name in the + // format "filename->table" and add it to the + // JSON array + snprintf(tablename, len, "%s->%s", file_stat.m_filename, import_tables[j]); + if(imported_files != NULL && !cJSON_AddItemToArray(imported_files, cJSON_CreateString(tablename))) + log_warn("Failed to add table %s to JSON array", tablename); + free(tablename); + } + + // Free allocated memory and skip to next file without + // adding it to the JSON array again below + free(ptr); + continue; } else { log_warn("Ignoring file %s in Teleporter archive", file_stat.m_filename); + + // Free allocated memory and skip to next file free(ptr); continue; } @@ -730,7 +816,13 @@ bool read_teleporter_zip_from_disk(const char *filename) // Process ZIP archive char hint[ERRBUF_SIZE] = ""; cJSON *imported_files = cJSON_CreateArray(); - const char *error = read_teleporter_zip(ptr, size, hint, imported_files); + if(imported_files == NULL) + { + log_err("Failed to create JSON array for imported files"); + free(ptr); + return false; + } + const char *error = read_teleporter_zip(ptr, size, hint, NULL, imported_files); if(error != NULL) { diff --git a/src/zip/teleporter.h b/src/zip/teleporter.h index a5743028..0394648f 100644 --- a/src/zip/teleporter.h +++ b/src/zip/teleporter.h @@ -15,7 +15,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size); bool free_teleporter_zip(mz_zip_archive *zip); -const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files); +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *import, cJSON *json_files); bool write_teleporter_zip_to_disk(void); bool read_teleporter_zip_from_disk(const char *filename); diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index ab6878b2..27558907 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -11,6 +11,7 @@ import io import ipaddress +import json import random import zipfile from libs.openAPI import openApi @@ -23,7 +24,7 @@ class ResponseVerifyer(): # Translate between OpenAPI and Python types YAML_TYPES = { "string": [str], "integer": [int], "number": [int, float], "boolean": [bool], "array": [list] } TELEPORTER_FILES_EXPORT = ["etc/pihole/gravity.db", "etc/pihole/pihole.toml", "etc/pihole/pihole-FTL.db", "etc/hosts"] - TELEPORTER_FILES_IMPORT = ['etc/pihole/pihole.toml', 'etc/pihole/dhcp.leases', 'etc/pihole/gravity.db'] + TELEPORTER_FILES_IMPORT = ['etc/pihole/pihole.toml', 'etc/pihole/dhcp.leases', 'etc/pihole/gravity.db->group', 'etc/pihole/gravity.db->adlist', 'etc/pihole/gravity.db->adlist_by_group', 'etc/pihole/gravity.db->domainlist', 'etc/pihole/gravity.db->domainlist_by_group', 'etc/pihole/gravity.db->client', 'etc/pihole/gravity.db->client_by_group' ] auth_method = "?" teleporter_archive = None @@ -229,6 +230,7 @@ class ResponseVerifyer(): for expected_file in self.TELEPORTER_FILES_IMPORT: if expected_file not in FTLresponse['files']: self.errors.append("File " + expected_file + " is missing in FTL response") + self.errors.append(json.dumps(FTLresponse['files'], indent=4)) return self.errors @@ -290,7 +292,6 @@ class ResponseVerifyer(): # Check if the property is defined in the API specs (unless we know there are "any-key" items here) if props[-1] not in YAMLprops: self.errors.append("Property '" + flat_path + "' missing in the API specs (2)") - print(YAMLprop) return False YAMLprop = YAMLprops[props[-1]] diff --git a/test/arch_test.sh b/test/arch_test.sh index 01ca0f4e..ff4bc003 100644 --- a/test/arch_test.sh +++ b/test/arch_test.sh @@ -95,10 +95,16 @@ check_minimum_glibc_version() { if [[ "${CI_ARCH}" == "linux/amd64" ]]; then - check_machine "ELF64" "Advanced Micro Devices X86-64" - check_static # Binary should not rely on any dynamic interpreter - check_libs "" # No dependency on any shared library is intended - check_file "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, with debug_info, not stripped" + if [[ "${STATIC}" == "true" ]]; then + check_machine "ELF64" "Advanced Micro Devices X86-64" + check_static # Binary should not rely on any dynamic interpreter + check_libs "" # No dependency on any shared library is intended + check_file "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, with debug_info, not stripped" +else + check_machine "ELF64" "Advanced Micro Devices X86-64" + check_libs "[libgmp.so.10] [libidn2.so.0] [libc.musl-x86_64.so.1]" + check_file "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-musl-x86_64.so.1, with debug_info, not stripped" + fi elif [[ "${CI_ARCH}" == "linux/386" ]]; then diff --git a/test/broken_lua.lp b/test/broken_lua.lp new file mode 100644 index 00000000..bf8a5fef --- /dev/null +++ b/test/broken_lua.lp @@ -0,0 +1,4 @@ + diff --git a/test/broken_lua_2.lp b/test/broken_lua_2.lp new file mode 100644 index 00000000..e0ada85c --- /dev/null +++ b/test/broken_lua_2.lp @@ -0,0 +1,4 @@ + diff --git a/test/pdns/luadns.lua b/test/pdns/luadns.lua new file mode 100644 index 00000000..dd88dbe0 --- /dev/null +++ b/test/pdns/luadns.lua @@ -0,0 +1,41 @@ +refused_ede15 = newDN("refused.ede15.ftl") +nxdomain_ede15 = newDN("nxdomain.ede15.ftl") +null_ede15 = newDN("null.ede15.ftl") + + +-- this hook is called before doing any resolving +function preresolve(dq) + pdnslog("Got question for "..dq.qname:toString().." from "..dq.remoteaddr:toString().." to "..dq.localaddr:toString()) + + if dq.qname == refused_ede15 then + pdnslog("Blocking REFUSED + EDE 15 for "..dq.qname:toString()) + -- Set EDE 15 in response + dq.extendedErrorCode = 15 + -- Set REFUSED in response + dq.rcode = pdns.REFUSED + return true + end + + if dq.qname == nxdomain_ede15 then + pdnslog("Blocking NXDOMAIN + EDE 15 for "..dq.qname:toString()) + -- Set EDE 15 in response + dq.extendedErrorCode = 15 + -- Set NXDOMAIN in response + dq.rcode = pdns.NXDOMAIN + return true + end + + if dq.qname == null_ede15 then + pdnslog("Blocking NULL + EDE 15 for "..dq.qname:toString()) + -- Set EDE 15 in response + dq.extendedErrorCode = 15 + -- Add a NULL RR to the response + dq:addAnswer(pdns.A, "0.0.0.0") + dq:addAnswer(pdns.AAAA, "::") + return true + end + + -- as we do not set dq.variable, our decision here will be cached + + return false +end diff --git a/test/pdns/pdns.conf b/test/pdns/pdns.conf index 2449edf5..8cf7854c 100644 --- a/test/pdns/pdns.conf +++ b/test/pdns/pdns.conf @@ -19,5 +19,8 @@ launch=gsqlite3 # Database location gsqlite3-database=/var/lib/powerdns/pdns.sqlite3 +# Enable DNSSEC in the backend +gsqlite3-dnssec=yes + # Used when creating a new zone default-soa-content=ns1.@ hostmaster.@ 0 10800 3600 604800 3600 diff --git a/test/pdns/recursor.conf b/test/pdns/recursor.conf index 06f40946..3021a700 100644 --- a/test/pdns/recursor.conf +++ b/test/pdns/recursor.conf @@ -10,8 +10,8 @@ # Local DNS address and port local-address=127.0.0.1:5555 -# Use authoritative server for ftl. and arpa. zones -forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554 +# Use authoritative server for ftl., dnssec. and arpa. zones +forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=127.0.0.1:5554,dnssec=127.0.0.1:5554,bogus=127.0.0.1:5554 # In this mode the Recursor acts as a “security aware, non-validating” # nameserver, meaning it will set the DO-bit on outgoing queries and will @@ -21,3 +21,6 @@ forward-zones=ftl=127.0.0.1:5554,168.192.in-addr.arpa=127.0.0.1:5554,ip6.arpa=12 # requested by the client. # The default mode until PowerDNS Recursor 4.5.0. dnssec=process-no-validate + +# Enable LUA support +lua-dns-script=/etc/pdns/luadns.lua diff --git a/test/pdns/setup.sh b/test/pdns/setup.sh index 715d5571..9aaaf28d 100644 --- a/test/pdns/setup.sh +++ b/test/pdns/setup.sh @@ -25,6 +25,7 @@ else exit 1 fi +cp test/pdns/luadns.lua /etc/pdns/luadns.lua cp test/pdns/recursor.conf $RECURSOR_CONF # Create zone database @@ -117,6 +118,44 @@ pdnsutil add-record ftl. regex-notMultiple AAAA fe80::3f41 # TXT pdnsutil add-record ftl. any TXT "\"Some example text\"" +# NOERROR +pdnsutil add-record ftl. noerror A + +# Blocked Cisco Umbrella IP (https://support.opendns.com/hc/en-us/articles/227986927-What-are-the-Cisco-Umbrella-Block-Page-IP-Addresses) +pdnsutil add-record ftl. umbrella A 146.112.61.104 +pdnsutil add-record ftl. umbrella AAAA ::ffff:146.112.61.104 + +# Special record which consists of both blocked and non-blocked IP +pdnsutil add-record ftl. umbrella-multi A 1.2.3.4 +pdnsutil add-record ftl. umbrella-multi A 146.112.61.104 +pdnsutil add-record ftl. umbrella-multi A 8.8.8.8 + +# Null address +pdnsutil add-record ftl. null A 0.0.0.0 +pdnsutil add-record ftl. null AAAA :: + +# Create valid internal DNSSEC zone +pdnsutil create-zone dnssec ns1.ftl +pdnsutil add-record dnssec. a A 192.168.4.1 +pdnsutil add-record dnssec. aaaa AAAA fe80::4c01 +pdnsutil secure-zone dnssec +# Export zone DS records and convert to dnsmasq trust-anchor format +# Example: +# dnssec. IN DS 42206 8 2 6d2007e292483fa061db37011676d9592649d1600e5b2ece1326f792ebedd412 ; ( SHA256 digest ) +# ---> +# trust-anchor=dnssec.,42206,8,2,6d2007e292483fa061db37011676d9592649d1600e5b2ece1326f792ebedd412 +pdnsutil export-zone-ds dnssec. | head -n1 | awk '{FS=" "; OFS=""; print "trust-anchor=",$1,",",$4,",",$5,",",$6,",",$7}' > /etc/dnsmasq.d/02-trust-anchor.conf + +# Create intentionally broken DNSSEC (BOGUS) zone +# The only difference to above is that this zone is signed with a key that is +# not in the trust chain +# It will cause the DNSSEC validation to fail with error message: +# unsupported DS digest +pdnsutil create-zone bogus ns1.ftl +pdnsutil add-record bogus. a A 192.168.5.1 +pdnsutil add-record bogus. aaaa AAAA fe80::5c01 +pdnsutil secure-zone bogus + # Create reverse lookup zone pdnsutil create-zone arpa ns1.ftl pdnsutil add-record arpa. 1.1.168.192.in-addr PTR ftl. diff --git a/test/pihole.toml b/test/pihole.toml index 75f8244c..e30dedb6 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1,14 +1,11 @@ +# Pi-hole configuration file (v5.25.2-1921-gd3948088-dirty) +# Encoding: UTF-8 # This file is managed by pihole-FTL -# -# Do not edit the file while FTL is -# running or your changes may be overwritten -# -# Last updated on 2023-01-23 14:51:44 -# by FTL v5.20.1-552-g5184ed28 +# Last updated on 2024-06-15 09:10:13 UTC [dns] # Array of upstream DNS servers used by Pi-hole - # Example: [ "8.8.8.8", "127.0.0.1#5353", "docker-resolver" ] + # Example: [ "8.8.8.8", "127.0.0.1#5335", "docker-resolver" ] # # Possible values are: # array of IP addresses and/or hostnames, optionally with a port (#...) @@ -50,23 +47,25 @@ analyzeOnlyAandAAAA = false # Controls whether and how FTL will reply with for address for which a local interface - # exists. + # exists. Changing this setting causes FTL to restart. # # Possible values are: # - "NONE" # Pi-hole will not respond automatically on PTR requests to local interface # addresses. Ensure pi.hole and/or hostname records exist elsewhere. # - "HOSTNAME" - # Pi-hole will not respond automatically on PTR requests to local interface - # addresses. Ensure pi.hole and/or hostname records exist elsewhere. + # Serve the machine's hostname. The hostname is queried from the kernel through + # uname(2)->nodename. If the machine has multiple network interfaces, it can + # also have multiple nodenames. In this case, it is unspecified and up to the + # kernel which one will be returned. On Linux, the returned string is what has + # been set using sethostname(2) which is typically what has been set in + # /etc/hostname. # - "HOSTNAMEFQDN" - # Serve the machine's global hostname as fully qualified domain by adding the - # local suffix. If no local suffix has been defined, FTL appends the local - # domain .no_fqdn_available. In this case you should either add - # domain=whatever.com to a custom config file inside /etc/dnsmasq.d/ (to set - # whatever.com as local domain) or use domain=# which will try to derive the - # local domain from /etc/resolv.conf (or whatever is set with resolv-file, when - # multiple search directives exist, the first one is used). + # Serve the machine's hostname (see limitations above) as fully qualified domain + # by adding the local domain. If no local domain has been defined (config option + # dns.domain), FTL tries to query the domain name from the kernel using + # getdomainname(2). If this fails, FTL appends ".no_fqdn_available" to the + # hostname. # - "PI.HOLE" # Respond with "pi.hole". piholePTR = "PI.HOLE" @@ -140,7 +139,7 @@ bogusPriv = true # Validate DNS replies using DNSSEC? - dnssec = true + dnssec = true ### CHANGED, default = false # Interface to use for DNS (see also dnsmasq.listening.mode) and DHCP (if enabled) # @@ -197,11 +196,10 @@ # given, it overwrites the value of local-ttl # # Possible values are: - # Array of static leases each on in one of the following forms: - # ",[,]" + # Array of CNAMEs each on in one of the following forms: ",[,]" cnameRecords = [ - "brücke.com,äste.com,2", - ] + "brücke.com,äste.com,2" + ] ### CHANGED, default = [] # Port used by the DNS server port = 53 @@ -225,7 +223,10 @@ # : Domain used for the reverse server feature (e.g., "fritz.box") # Example: "fritz.box" # - # A valid line could look like this: "true,192.168.0.0/24,192.168.0.1,fritz.box" + # Possible values are: + # array of reverse servers each one in one of the following forms: + # ",[/],[#],", e.g., + # "true,192.168.0.0/24,192.168.0.1,fritz.box" revServers = [] [dns.cache] @@ -239,13 +240,23 @@ # expired only recently, the data will be used anyway (a refreshing from upstream is # triggered). This can improve DNS query delays especially over unreliable Internet # connections. This feature comes at the expense of possibly sometimes returning - # out-of-date data and less efficient cache utilisation, since old data cannot be + # out-of-date data and less efficient cache utilization, since old data cannot be # flushed when its TTL expires, so the cache becomes mostly least-recently-used. To # mitigate issues caused by massively outdated DNS replies, the maximum overaging of # cached records is limited. We strongly recommend staying below 86400 (1 day) with # this option. + # Setting the TTL excess time to zero will serve stale cache data regardless how long + # it has expired. This is not recommended as it may lead to stale data being served + # for a long time. Setting this option to any negative value will disable this feature + # altogether. optimizer = 3600 + # This setting allows you to specify the TTL used for queries blocked upstream. Once + # the TTL expires, the query will be forwarded to the upstream server again to check + # if the block is still valid. Defaults to caching for one day (86400 seconds). + # Setting this value to zero disables caching of queries blocked upstream. + upstreamBlockedTTL = 86400 + [dns.blocking] # Should FTL block queries? active = true @@ -259,7 +270,7 @@ # (0.0.0.0 or ::). The "unspecified address" is a reserved IP address specified # by RFC 3513 - Internet Protocol Version 6 (IPv6) Addressing Architecture, # section 2.5.2. - # - "IP-NODATA-AAAA" + # - "IP_NODATA_AAAA" # In IP-NODATA-AAAA mode, blocked queries will be answered with the local IPv4 # addresses of your Pi-hole. Blocked AAAA queries will be answered with # NODATA-IPV6 and clients will only try to reach your Pi-hole over its static @@ -267,7 +278,7 @@ # - "IP" # In IP mode, blocked queries will be answered with the local IP addresses of # your Pi-hole. - # - "NXDOMAIN" + # - "NX" # In NXDOMAIN mode, blocked queries will be answered with an empty response # (i.e., there won't be an answer section) and status NXDOMAIN. A NXDOMAIN # response should indicate that there is no such domain to the client making the @@ -278,6 +289,18 @@ # exists, but there is no record for the requested query type. mode = "NULL" + # Should FTL enrich blocked replies with EDNS0 information? + # + # Possible values are: + # - "NONE" + # In NONE mode, no additional EDNS information is added to blocked queries + # - "CODE" + # In CODE mode, blocked queries will be enriched with EDNS info-code BLOCKED (15) + # - "TEXT" + # In TEXT mode, blocked queries will be enriched with EDNS info-code BLOCKED (15) + # and a text message describing the reason for the block + edns = "TEXT" + [dns.specialDomains] # Should Pi-hole always replies with NXDOMAIN to A and AAAA queries of # use-application-dns.net to disable Firefox automatic DNS-over-HTTP? This is @@ -300,21 +323,21 @@ # "pi.hole.", "." ] force4 = true ### CHANGED, default = false - # Use a specific IPv6 address for the Pi-hole host? See description for the IPv4 - # variant above for further details. - force6 = true ### CHANGED, default = false - # Custom IPv4 address for the Pi-hole host # # Possible values are: # or empty string ("") - IPv4 = "10.100.0.10" ### CHANGED, default = "0.0.0.0" + IPv4 = "10.100.0.10" ### CHANGED, default = "" + + # Use a specific IPv6 address for the Pi-hole host? See description for the IPv4 + # variant above for further details. + force6 = true ### CHANGED, default = false # Custom IPv6 address for the Pi-hole host # # Possible values are: # or empty string ("") - IPv6 = "fe80::10" ### CHANGED, default = "::" + IPv6 = "fe80::10" ### CHANGED, default = "" [dns.reply.blocking] # Use a specific IPv4 address in IP blocking mode? By default, FTL determines the @@ -325,26 +348,26 @@ # blocked, regular expressions with the ;reply=IP regex extension. force4 = true ### CHANGED, default = false - # Use a specific IPv6 address in IP blocking mode? See description for the IPv4 variant - # above for further details. - force6 = true ### CHANGED, default = false - # Custom IPv4 address for IP blocking mode # # Possible values are: # or empty string ("") - IPv4 = "10.100.0.11" ### CHANGED, default = "0.0.0.0" + IPv4 = "10.100.0.11" ### CHANGED, default = "" + + # Use a specific IPv6 address in IP blocking mode? See description for the IPv4 variant + # above for further details. + force6 = true ### CHANGED, default = false # Custom IPv6 address for IP blocking mode # # Possible values are: # or empty string ("") - IPv6 = "fe80::11" ### CHANGED, default = "::" + IPv6 = "fe80::11" ### CHANGED, default = "" [dns.rateLimit] # Rate-limited queries are answered with a REFUSED reply and not further processed by # FTL. - #The default settings for FTL's rate-limiting are to permit no more than 1000 queries + # The default settings for FTL's rate-limiting are to permit no more than 1000 queries # in 60 seconds. Both numbers can be customized independently. It is important to note # that rate-limiting is happening on a per-client basis. Other clients can continue to # use FTL while rate-limited clients are short-circuited at the same time. @@ -377,32 +400,33 @@ # Start address of the DHCP address pool # # Possible values are: - # , e.g., "192.168.0.10" + # or empty string (""), e.g., "192.168.0.10" start = "" # End address of the DHCP address pool # # Possible values are: - # , e.g., "192.168.0.250" + # or empty string (""), e.g., "192.168.0.250" end = "" # Address of the gateway to be used (typically the address of your router in a home # installation) # # Possible values are: - # , e.g., "192.168.0.1" + # or empty string (""), e.g., "192.168.0.1" router = "" # The netmask used by your Pi-hole. For directly connected networks (i.e., networks on # which the machine running Pi-hole has an interface) the netmask is optional and may - # be set to "0.0.0.0": it will then be determined from the interface configuration - # itself. For networks which receive DHCP service via a relay agent, we cannot - # determine the netmask itself, so it should explicitly be specified, otherwise + # be set to an empty string (""): it will then be determined from the interface + # configuration itself. For networks which receive DHCP service via a relay agent, we + # cannot determine the netmask itself, so it should explicitly be specified, otherwise # Pi-hole guesses based on the class (A, B or C) of the network address. # # Possible values are: - # , e.g., "255.255.255.0" or "0.0.0.0" for auto-discovery - netmask = "0.0.0.0" + # (e.g., "255.255.255.0") or empty string ("") for + # auto-discovery + netmask = "" # If the lease time is given, then leases will be given for that length of time. If not # given, the default lease time is one hour for IPv4 and one day for IPv6. @@ -433,6 +457,18 @@ # the file specified by files.log.dnsmasq below. logging = false + # Ignore unknown DHCP clients. + # If this option is set, Pi-hole ignores all clients which are not explicitly + # configured through dhcp.hosts. This can be useful to prevent unauthorized clients + # from getting an IP address from the DHCP server. + # It should be noted that this option is not a security feature, as clients can still + # assign themselves an IP address and use the network. It is merely a convenience + # feature to prevent unknown clients from getting a valid IP configuration assigned + # automatically. + # Note that you will need to configure new clients manually in dhcp.hosts before they + # can use the network when this feature is enabled. + ignoreUnknownClients = false + # Per host parameters for the DHCP server. This allows a machine with a particular # hardware address to be always allocated the same hostname, IP address and lease time # or to specify static DHCP leases @@ -442,6 +478,57 @@ # "[][,id:|*][,set:][,tag:][,][,][,][,ignore]" hosts = [] + [ntp.ipv4] + # Should FTL act as network time protocol (NTP) server (IPv4)? + active = true + + # IPv4 address to listen on for NTP requests + # + # Possible values are: + # or empty string ("") for wildcard (0.0.0.0) + address = "" + + [ntp.ipv6] + # Should FTL act as network time protocol (NTP) server (IPv6)? + active = true + + # IPv6 address to listen on for NTP requests + # + # Possible values are: + # or empty string ("") for wildcard (::) + address = "" + + [ntp.sync] + # Should FTL try to synchronize the system time with an upstream NTP server? + active = true + + # NTP upstream server to sync with, e.g., "pool.ntp.org". Note that the NTP server + # should be located as close as possible to you in order to minimize the time offset + # possibly introduced by different routing paths. + # + # Possible values are: + # valid NTP upstream server + server = "pool.ntp.org" + + # Interval in seconds between successive synchronization attempts with the NTP server + interval = 3600 + + # Number of NTP syncs to perform and average before updating the system time + count = 8 + + [ntp.sync.rtc] + # Should FTL update a real-time clock (RTC) if available? + set = true + + # Path to the RTC device to update. Leave empty for auto-discovery + # + # Possible values are: + # Path to the RTC device, e.g., "/dev/rtc0" + device = "" + + # Should the RTC be set to UTC? + utc = true + [resolver] # Should FTL try to resolve IPv4 addresses to hostnames? resolveIPv4 = false ### CHANGED, default = true @@ -451,10 +538,11 @@ # Control whether FTL should use the fallback option to try to obtain client names from # checking the network table. This behavior can be disabled with this option. - #Assume an IPv6 client without a host names. However, the network table knows - though - # the client's MAC address - that this is the same device where we have a host name - # for another IP address (e.g., a DHCP server managed IPv4 address). In this case, we - # use the host name associated to the other address as this is the same device. + # Assume an IPv6 client without a host names. However, the network table knows - + # though the client's MAC address - that this is the same device where we have a host + # name for another IP address (e.g., a DHCP server managed IPv4 address). In this + # case, we use the host name associated to the other address as this is the same + # device. networkNames = false ### CHANGED, default = true # With this option, you can change how (and if) hourly PTR requests are made to check @@ -485,8 +573,7 @@ DBimport = true # How long should queries be stored in the database [days]? - # Setting this to 0 disables exporting queries to the database. - maxDBdays = 365 + maxDBdays = 91 # How often do we store queries in FTL's database [seconds]? DBinterval = 60 @@ -508,7 +595,7 @@ # How long should IP addresses be kept in the network_addresses table [days]? IP # addresses (and associated host names) older than the specified number of days are # removed to avoid dead entries in the network overview table. - expire = 365 + expire = 91 [webserver] # On which domain is the web interface served? @@ -573,21 +660,13 @@ # the total number of concurrent sessions is limited so setting this value too high # may result in users being rejected and unable to log in if there are already too # many sessions active. - timeout = 300 + timeout = 300 ### CHANGED, default = 1800 # Should Pi-hole backup and restore sessions from the database? This is useful if you # want to keep your sessions after a restart of the web interface. restore = true [webserver.tls] - # Is Pi-hole running behind a reverse proxy? If yes, Pi-hole will not consider - # HTTP-only connections being insecure. This is useful if you are running Pi-hole in a - # trusted environment, for example, in a local network, and you are using a reverse - # proxy to provide TLS encryption, e.g., by using Traefik (docker). If you are using a - # reverse proxy, you can alternatively set webserver.tls.cert to the path of the TLS - # certificate file and let Pi-hole handle true end-to-end encryption. - rev_proxy = false - # Path to the TLS (SSL) certificate file. This option is only required when at least # one of webserver.port is TLS. The file must be in PEM format, and it must have both, # private key and certificate (the *.pem file created must contain a 'CERTIFICATE' @@ -599,7 +678,7 @@ # # Possible values are: # - cert = "/etc/pihole/test.pem" + cert = "/etc/pihole/test.pem" ### CHANGED, default = "/etc/pihole/tls.pem" [webserver.paths] # Server root on the host @@ -622,30 +701,22 @@ # # Possible values are: # - "default-auto" - # Pi-hole auto theme (light/dark, default) + # Pi-hole auto # - "default-light" - # Pi-hole day theme (light) + # Pi-hole day # - "default-dark" - # Pi-hole midnight theme (dark) + # Pi-hole midnight # - "default-darker" - # Pi-hole deep-midnight theme (dark) + # Pi-hole deep-midnight # - "high-contrast" - # High-contrast theme (light) + # High-contrast light # - "high-contrast-dark" - # High-contrast theme (dark) + # High-contrast dark # - "lcars" - # Star Trek LCARS theme (dark) + # Star Trek LCARS theme = "default-auto" [webserver.api] - # Does local clients need to authenticate to access the API? - localAPIauth = true - - # Do local clients need to authenticate to access the search API? This settings allows - # local clients to use pihole -q ... without authentication. Note that "local" in the - # sense of the option means only 127.0.0.1 and [::1] - searchAPIauth = false - # Number of concurrent sessions allowed for the API. If the number of sessions exceeds # this value, no new sessions will be allowed until the number of sessions drops due # to session expiration or logout. Note that the number of concurrent sessions is @@ -683,6 +754,21 @@ # app_pwhash = "" + # Should application password API sessions be allowed to modify config settings? + # Setting this to true allows third-party applications using the application password + # to modify settings, e.g., the upstream DNS servers, DHCP server settings, or + # changing passwords. This setting should only be enabled if really needed and only if + # you trust the applications using the application password. + app_sudo = false + + # Should FTL create a temporary CLI password? This password is stored in clear in + # /etc/pihole and can be used by the CLI (pihole ... commands) to authenticate + # against the API. Note that the password is only valid for the current session and + # regenerated on each FTL restart. Sessions initiated with this password cannot modify + # the Pi-hole configuration (change passwords, etc.) for security reasons but can + # still use the API to query data and manage lists. + cli_pw = true + # Array of clients to be excluded from certain API responses (regex): # - Query Log (/api/queries) # - Top Clients (/api/stats/top_clients) @@ -708,13 +794,15 @@ # array of regular expressions describing domains excludeDomains = [] - # How much history should be imported from the database [seconds]? (max 24*60*60 = - # 86400) + # How much history should be imported from the database and returned by the API + # [seconds]? (max 24*60*60 = 86400) maxHistory = 86400 # Up to how many clients should be returned in the activity graph endpoint # (/api/history/clients)? - # This setting can be overwritten at run-time using the parameter N + # This setting can be overwritten at run-time using the parameter N. Setting this to 0 + # will always send all clients. Be aware that this may be challenging for the GUI if + # you have many (think > 1.000 clients) in your network maxClients = 10 # How should the API compute the most active clients? If set to true, the API will @@ -765,7 +853,7 @@ # directory must be writable by the user running gravity (typically pihole). # # Possible values are: - # + # gravity_tmp = "/tmp" # The database containing MAC -> Vendor information for the network table @@ -774,7 +862,7 @@ # macvendor = "/etc/pihole/macvendor.db" - # The config file of Pi-hole + # The old config file of Pi-hole used before v6.0 # # Possible values are: # @@ -812,11 +900,11 @@ [misc] # Using privacy levels you can specify which level of detail you want to see in your - # Pi-hole statistics. + # Pi-hole statistics. Changing this setting will trigger a restart of FTL # # Possible values are: # - 0 - # Doesn't hide anything, all statistics are available. + # Don't hide anything, all statistics are available. # - 1 # Hide domains. This setting disables Top Domains and Top Ads # - 2 @@ -841,7 +929,7 @@ # CPU scheduler to favor or disfavor a process in scheduling decisions. The range of # the nice value varies across UNIX systems. On modern Linux, the range is -20 (high # priority = not very nice to other processes) to +19 (low priority). - nice = -999 ### CHANGED, default = -10 + nice = -11 ### CHANGED (env), default = -10 # Should FTL translate its own stack addresses into code lines during the bug # backtrace? This improves the analysis of crashed significantly. It is recommended to @@ -853,11 +941,15 @@ # Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/? etc_dnsmasq_d = true ### CHANGED, default = false - # Additional lines to inject into the generated dnsmasq configuration. Warning: This is - # an advanced setting and should only be used with care. Incorrectly formatted or - # duplicated lines as well as lines conflicting with the automatic configuration of - # Pi-hole can break the embedded dnsmasq and will stop DNS resolution from working. + # Additional lines to inject into the generated dnsmasq configuration. + # Warning: This is an advanced setting and should only be used with care. Incorrectly + # formatted or duplicated lines as well as lines conflicting with the automatic + # configuration of Pi-hole can break the embedded dnsmasq and will stop DNS resolution + # from working. # Use this option with extra care. + # + # Possible values are: + # array of valid dnsmasq config line options dnsmasq_lines = [] # Log additional information about queries and replies to pihole.log @@ -868,6 +960,12 @@ # debugging and is not recommended for normal use. extraLogging = false + # Put configuration into read-only mode. This will prevent any changes to the + # configuration file via the API or CLI. This setting useful when a configuration is + # to be forced/modified by some third-party application (like infrastructure-as-code + # providers) and should not be changed by any means. + readOnly = false + [misc.check] # Pi-hole is very lightweight on resources. Nevertheless, this does not mean that you # should run Pi-hole on a server that is otherwise extremely busy as queuing on the @@ -937,7 +1035,7 @@ # when debugging specific API issues and can be helpful, e.g., when a client cannot # connect due to an obscure API error. Furthermore, this setting enables logging of # all API requests (auth log) and details about user authentication attempts. - api = true ### CHANGED, default = false + api = true ### CHANGED (env), default = false # Print extra debugging information about TLS connections. This includes the TLS # version, the cipher suite, the certificate chain and much more. This very verbose @@ -997,7 +1095,7 @@ # Debug monitoring of /etc/pihole filesystem events inotify = true ### CHANGED, default = false - # Logging of webserver (CivetWeb) debug messages + # Debug monitoring of the webserver (CivetWeb) events webserver = true ### CHANGED, default = false # Temporary flag that may print additional information. This debug flag is meant to be @@ -1008,8 +1106,17 @@ # Reserved debug flag reserved = true ### CHANGED, default = false + # Print information about NTP synchronization + ntp = true ### CHANGED, default = false + # Set all debug flags at once. This is a convenience option to enable all debug flags # at once. Note that this option is not persistent, setting it to true will enable all # *remaining* debug flags but unsetting it will disable *all* debug flags. all = true ### CHANGED, default = false +# Configuration statistics: +# 150 total entries out of which 95 entries are default +# --> 55 entries are modified +# 2 entries are forced through environment: +# - misc.nice +# - debug.api diff --git a/test/run.sh b/test/run.sh index 5f34e9d6..598ecff3 100755 --- a/test/run.sh +++ b/test/run.sh @@ -23,13 +23,14 @@ done rm -rf /etc/pihole /var/log/pihole /dev/shm/FTL-* # Create necessary directories and files -mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole +mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole /etc/pihole/config_backups /var/www/html echo "" > /var/log/pihole/FTL.log echo "" > /var/log/pihole/pihole.log -touch /run/pihole-FTL.pid /run/pihole-FTL.port dig.log ptr.log -touch /var/log/pihole/HTTP_info.log /var/log/pihole/PH7.log /etc/pihole/dhcp.leases -chown pihole:pihole /etc/pihole /run/pihole /var/log/pihole/pihole.log /var/log/pihole/FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port -chown pihole:pihole /var/log/pihole/HTTP_info.log /var/log/pihole/PH7.log /etc/pihole/dhcp.leases +echo "" > /var/log/pihole/webserver.log +touch /run/pihole-FTL.pid dig.log ptr.log +touch /etc/pihole/dhcp.leases +chown -R pihole:pihole /etc/pihole /run/pihole /var/log/pihole +chown pihole:pihole /run/pihole-FTL.pid # Copy binary into a location the new user pihole can access cp ./pihole-FTL /home/pihole/pihole-FTL @@ -61,6 +62,10 @@ cp test/01-pihole-tests.conf /etc/dnsmasq.d/01-pihole-tests.conf # Prepare versions file (read by /api/version) cp test/versions /etc/pihole/versions +# Prepare Lua test script +cp test/broken_lua.lp /var/www/html/broken_lua.lp +cp test/broken_lua_2.lp /var/www/html/broken_lua_2.lp + # Prepare local powerDNS resolver bash test/pdns/setup.sh @@ -73,6 +78,9 @@ export FTLCONF_misc_nice="-11" export FTLCONF_dns_upstrrr="-11" export FTLCONF_debug_api="not_a_bool" +# Prepare gdb session +echo "handle SIGHUP nostop SIGPIPE nostop SIGTERM nostop SIG32 nostop SIG33 nostop SIG34 nostop SIG35 nostop SIG41 nostop" > /root/.gdbinit + # Start FTL if ! su pihole -s /bin/sh -c /home/pihole/pihole-FTL; then echo "pihole-FTL failed to start" @@ -89,6 +97,10 @@ fi # Give FTL some time for startup preparations sleep 2 +# Attach debugger and immediately continue running the binary +# In case a non-ignored signal occurs (a crash), create a full backtrace +gdb -p $(cat /run/pihole-FTL.pid) --ex continue --ex "bt full" & + # Print versions of pihole-FTL echo -n "FTL version (DNS): " dig TXT CHAOS version.FTL @127.0.0.1 +short @@ -98,7 +110,7 @@ echo -n "Contained dnsmasq version (DNS): " dig TXT CHAOS version.bind @127.0.0.1 +short # Run tests -$BATS "test/test_suite.bats" +$BATS -p "test/test_suite.bats" RET=$? curl_to_tricorder() { @@ -118,11 +130,8 @@ if [[ $RET != 0 ]]; then echo -n "ptr.log: " curl_to_tricorder ./ptr.log echo "" - echo -n "HTTP_info.log: " - curl_to_tricorder /var/log/pihole/HTTP_info.log - echo "" - echo -n "PH7.log: " - curl_to_tricorder /var/log/pihole/PH7.log + echo -n "webserver.log: " + curl_to_tricorder /var/log/pihole/webserver.log echo "" echo -n "pihole.toml: " curl_to_tricorder /etc/pihole/pihole.toml @@ -130,6 +139,7 @@ if [[ $RET != 0 ]]; then fi # Kill pihole-FTL after having completed tests +# This will also shut down the debugger kill "$(pidof pihole-FTL)" # Restore umask diff --git a/test/test_suite.bats b/test/test_suite.bats index 64a4a9ee..133e3905 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1,35 +1,23 @@ #!./test/libs/bats/bin/bats -#@test "Version, Tag, Branch, Hash, Date is reported" { -# run bash -c 'echo ">version >quit" | nc -v 127.0.0.1 4711' -# printf "%s\n" "${lines[@]}" -# [[ ${lines[1]} == "version "* ]] -# [[ ${lines[2]} == "tag "* ]] -# [[ ${lines[3]} == "branch "* ]] -# [[ ${lines[4]} == "hash "* ]] -# [[ ${lines[5]} == "date "* ]] -# [[ ${lines[6]} == "" ]] -#} -# -#@test "DNS server port is reported over Telnet API" { -# run bash -c 'echo ">dns-port >quit" | nc -v 127.0.0.1 4711' -# printf "%s\n" "${lines[@]}" -# [[ ${lines[1]} == "53" ]] -# [[ ${lines[2]} == "" ]] -#} -# -#@test "Maxlogage value is reported over Telnet API" { -# run bash -c 'echo ">maxlogage >quit" | nc -v 127.0.0.1 4711' -# printf "%s\n" "${lines[@]}" -# [[ ${lines[1]} == "86400" ]] -# [[ ${lines[2]} == "" ]] -#} -# -@test "Running a second instance is detected and prevented" { - run bash -c 'su pihole -s /bin/sh -c "/home/pihole/pihole-FTL -f"' +@test "Compare template and test TOML config files" { + # We skip the first 5 lines of the files as they contain the version and + # timestamp of the file creation/modification + run bash -c 'diff <(tail -n +6 test/pihole.toml) <(tail -n +6 /etc/pihole/pihole.toml)' printf "%s\n" "${lines[@]}" - [[ "${lines[@]}" == *"CRIT: Initialization of shared memory failed."* ]] - [[ "${lines[@]}" == *"INFO: pihole-FTL is already running"* ]] + [[ "${lines[@]}" == "" ]] +} + +@test "Check FTL binary integrity" { + run bash -c './pihole-FTL verify' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == *"Binary integrity check: OK" ]] +} + +@test "Running a second instance is detected and prevented" { + run bash -c 'su pihole -s /bin/sh -c "./pihole-FTL -f"' + printf "%s\n" "${lines[@]}" + [[ "${lines[@]}" == *"CRIT: pihole-FTL is already running"* ]] } @test "dnsmasq options as expected" { @@ -39,12 +27,6 @@ [[ ${lines[1]} == "" ]] } -@test "Starting tests without prior history" { - run bash -c 'grep -c "Total DNS queries: 0" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "1" ]] -} - @test "Initial blocking status is enabled" { run bash -c 'grep -c "Blocking status is enabled" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" @@ -54,14 +36,18 @@ @test "Number of compiled regex filters as expected" { run bash -c 'grep "Compiled [0-9]* allow" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == *"Compiled 2 allow and 11 deny regex for 1 client in "* ]] + [[ ${lines[0]} == *"Compiled 2 allow and 11 deny regex"* ]] } -@test "denied domain is blocked" { +@test "Denied domain is blocked" { run bash -c "dig denied.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig denied.ftl @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (denylist)" ]] + [[ ${lines[1]} == "" ]] } @test "Gravity domain is blocked" { @@ -69,6 +55,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig gravity.ftl @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (gravity)" ]] + [[ ${lines[1]} == "" ]] } @test "Gravity domain is blocked (TCP)" { @@ -76,6 +66,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig gravity.ftl @127.0.0.1 +tcp | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (gravity)" ]] + [[ ${lines[1]} == "" ]] } @test "Gravity domain + allowed exact match is not blocked" { @@ -101,6 +95,10 @@ printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "0.0.0.0" ]] [[ ${lines[1]} == "" ]] + run bash -c "dig regex5.ftl @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 15 (Blocked): (regex)" ]] + [[ ${lines[1]} == "" ]] } @test "Regex denylist mismatch is not blocked" { @@ -419,13 +417,13 @@ } @test "DNSSEC: SECURE domain is resolved" { - run bash -c "dig A dnssec.works @127.0.0.1" + run bash -c "dig A a.dnssec @127.0.0.1" printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: NOERROR"* ]] } @test "DNSSEC: BOGUS domain is rejected" { - run bash -c "dig A fail01.dnssec.works @127.0.0.1" + run bash -c "dig A a.bogus @127.0.0.1" printf "%s\n" "${lines[@]}" [[ ${lines[@]} == *"status: SERVFAIL"* ]] } @@ -442,6 +440,244 @@ [[ ${lines[@]} == *"status: NOERROR"* ]] } +# NXRA + RA unset cannot be tested with PowerDNS as upstream provider + +@test "Upstream blocked domain: NULL is recognized" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A null.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] + [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"A"*"0.0.0.0"* ]] + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream NULL)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/null.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with 0.0.0.0"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: NULL is recognized (cached)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A null.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] + [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"A"*"0.0.0.0"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: null.ftl is known as blocked upstream with NULL address (expires in"* ]] + [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: NULL is recognized (IPv6)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig AAAA null.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] + [[ ${lines[@]} == *"null.ftl."*"2"*"IN"*"AAAA"*"::"* ]] + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream NULL)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/null.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded null.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with ::"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"null.ftl AAAA ::\""* ]] +} + +@test "Upstream blocked domain: IP is recognized" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A umbrella.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream IP)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with known address (IPv4)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella.ftl -> EXTERNAL_BLOCKED_IP"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: IP is recognized (cached)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A umbrella.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream IP)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: umbrella.ftl is known as blocked upstream with known address (expires in"* ]] + [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: IP is recognized (IPv6)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig AAAA umbrella.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: blocked upstream with known address (IPv6)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: AAAA/127.0.0.1/umbrella.ftl -> EXTERNAL_BLOCKED_IP"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella.ftl AAAA ::\""* ]] +} + +@test "Upstream blocked domain: IP is recognized (multi)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A umbrella-multi.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded umbrella-multi.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/umbrella-multi.ftl -> EXTERNAL_BLOCKED_IP"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"umbrella-multi.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: EDE 15 is recognized" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A nxdomain.ede15.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream EDE 15)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl is not blocked (domainlist ID: -1)"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: **** forwarded nxdomain.ede15.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: DNS cache: A/127.0.0.1/nxdomain.ede15.ftl -> EXTERNAL_BLOCKED_EDE15"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"nxdomain.ede15.ftl A 0.0.0.0\""* ]] +} + +@test "Upstream blocked domain: EDE 15 is recognized (cached)" { + # Get number of lines in the log before the test + before="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Run test + run bash -c "dig A nxdomain.ede15.ftl @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"EDE: 15 (Blocked): (upstream EDE 15)"* ]] + + # Get number of lines in the log after the test + after="$(grep -c ^ /var/log/pihole/FTL.log)" + + # Extract relevant log lines + log="$(sed -n "${before},${after}p" /var/log/pihole/FTL.log)" + # Split log into array by newline + lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "${log}" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"DEBUG_QUERIES: nxdomain.ede15.ftl is known as blocked upstream with EDE15 (expires in"* ]] + [[ ${lines[@]} != *"DEBUG_QUERIES: **** forwarded umbrella.ftl to 127.0.0.1#5555"* ]] + [[ ${lines[@]} == *"DEBUG_QUERIES: Adding RR: \"nxdomain.ede15.ftl A 0.0.0.0\""* ]] +} + @test "ABP-style matching working as expected" { run bash -c "dig A special.gravity.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" @@ -463,7 +699,7 @@ [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network\" (id INTEGER PRIMARY KEY NOT NULL, hwaddr TEXT UNIQUE NOT NULL, interface TEXT NOT NULL, firstSeen INTEGER NOT NULL, lastQuery INTEGER NOT NULL, numQueries INTEGER NOT NULL, macVendor TEXT, aliasclient_id INTEGER);"* ]] [[ "${lines[@]}" == *"CREATE TABLE IF NOT EXISTS \"network_addresses\" (network_id INTEGER NOT NULL, ip TEXT UNIQUE NOT NULL, lastSeen INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), name TEXT, nameUpdated INTEGER, FOREIGN KEY(network_id) REFERENCES network(id));"* ]] [[ "${lines[@]}" == *"CREATE TABLE aliasclient (id INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL, comment TEXT);"* ]] - [[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,17,'Database version');"* ]] + [[ "${lines[@]}" == *"INSERT INTO ftl VALUES(0,19,'Database version');"* ]] # vvv This has been added in version 10 vvv [[ "${lines[@]}" == *"CREATE VIEW queries AS SELECT id, timestamp, type, status, CASE typeof(domain) WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain,CASE typeof(client) WHEN 'integer' THEN (SELECT ip FROM client_by_id c WHERE c.id = q.client) ELSE client END client,CASE typeof(forward) WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward,CASE typeof(additional_info) WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, reply_type, reply_time, dnssec, list_id FROM query_storage q;"* ]] [[ "${lines[@]}" == *"CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);"* ]] @@ -475,7 +711,7 @@ [[ "${lines[@]}" == *"CREATE TABLE addinfo_by_id (id INTEGER PRIMARY KEY, type INTEGER NOT NULL, content NOT NULL);"* ]] [[ "${lines[@]}" == *"CREATE UNIQUE INDEX addinfo_by_id_idx ON addinfo_by_id(type,content);"* ]] # vvv This has been added in version 15 vvv - [[ "${lines[@]}" == *"CREATE TABLE session (id INTEGER PRIMARY KEY, login_at TIMESTAMP NOT NULL, valid_until TIMESTAMP NOT NULL, remote_addr TEXT NOT NULL, user_agent TEXT, sid TEXT NOT NULL, csrf TEXT NOT NULL, tls_login BOOL, tls_mixed BOOL, app BOOL);"* ]] + [[ "${lines[@]}" == *"CREATE TABLE session (id INTEGER PRIMARY KEY, login_at TIMESTAMP NOT NULL, valid_until TIMESTAMP NOT NULL, remote_addr TEXT NOT NULL, user_agent TEXT, sid TEXT NOT NULL, csrf TEXT NOT NULL, tls_login BOOL, tls_mixed BOOL, app BOOL, cli BOOL, x_forwarded_for TEXT);"* ]] } @test "Ownership, permissions and type of pihole-FTL.db correct" { @@ -483,34 +719,42 @@ printf "%s\n" "${lines[@]}" # Depending on the shell (x86_64-musl is built on busybox) there can be one or multiple spaces between user and group [[ ${lines[0]} == *"pihole"?*"pihole"* ]] - [[ ${lines[0]} == "-rw-rw-r--"* ]] + [[ ${lines[0]} == "-rw-r-----"* ]] run bash -c 'file /etc/pihole/pihole-FTL.db' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "/etc/pihole/pihole-FTL.db: SQLite 3.x database"* ]] } @test "Test fail on invalid CLI argument" { - run bash -c '/home/pihole/pihole-FTL abc' + run bash -c './pihole-FTL abc' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "pihole-FTL: invalid option -- 'abc'" ]] - [[ ${lines[1]} == "Command: '/home/pihole/pihole-FTL abc'" ]] - [[ ${lines[2]} == "Try '/home/pihole/pihole-FTL --help' for more information" ]] + [[ ${lines[1]} == "Command: './pihole-FTL abc'" ]] + [[ ${lines[2]} == "Try './pihole-FTL --help' for more information" ]] } @test "Help CLI argument return help text" { - run bash -c '/home/pihole/pihole-FTL help' + run bash -c './pihole-FTL help' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "The Pi-hole FTL engine - "* ]] } @test "No WARNING messages in FTL.log (besides known warnings)" { - run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|(Cannot set process priority)|FTLCONF_"' + run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|CAP_SYS_TIME|FTLCONF_"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == "" ]] } +@test "No ERROR messages in FTL.log (besides known/intended error)" { + run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)|(Failed to set|adjust time during NTP sync: Insufficient permissions)"' + printf "count: %s\n" "${lines[@]}" + [[ ${lines[0]} == "0" ]] +} + @test "No CRIT messages in FTL.log (besides error due to starting FTL more than once)" { - run bash -c 'grep "CRIT:" /var/log/pihole/FTL.log | grep -v "CRIT: Initialization of shared memory failed"' + run bash -c 'grep "CRIT:" /var/log/pihole/FTL.log | grep -v "CRIT: pihole-FTL is already running"' printf "%s\n" "${lines[@]}" [[ "${lines[@]}" == "" ]] } @@ -938,17 +1182,6 @@ [[ "${api}" == "${domain_api}" ]] } -# x86_64-musl is built on busybox which has a slightly different -# variant of ls displaying three, instead of one, spaces between the -# user and group names. - -@test "Ownership and permissions of pihole-FTL.db correct" { - run bash -c 'ls -l /etc/pihole/pihole-FTL.db' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == *"pihole pihole"* || ${lines[0]} == *"pihole pihole"* ]] - [[ ${lines[0]} == "-rw-rw-r--"* ]] -} - # "ldd" prints library dependencies and the used interpreter for a given program # # Dependencies on shared libraries are displayed like @@ -983,19 +1216,6 @@ [[ "${STATIC}" == "true" && "${lines[@]}" != *"interpreter"* ]] } -@test "Architecture is correctly reported on startup" { - run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' - printf "Output: %s\n\$CI_ARCH: %s\nuname -m: %s\n" "${lines[@]:-not set}" "${CI_ARCH:-not set}" "$(uname -m)" - [[ ${lines[0]} == *"Compiled for ${CI_ARCH:-$(uname -m)}"* ]] -} - -@test "Building machine (CI) is reported on startup" { - [[ ${CI_ARCH} != "" ]] && compiled_str="on CI" || compiled_str="locally" && export compiled_str - run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' - printf "Output: %s\n\$CI_ARCH: %s\n" "${lines[@]:-not set}" "${CI_ARCH:-not set}" - [[ ${lines[0]} == *"(compiled ${compiled_str})"* ]] -} - @test "Compiler version is correctly reported on startup" { compiler_version="$(${CC} --version | head -n1)" && export compiler_version run bash -c 'grep "Compiled for" /var/log/pihole/FTL.log' @@ -1012,7 +1232,7 @@ @test "Blocking status is correctly logged in pihole.log" { run bash -c 'grep -c "gravity blocked gravity.ftl is 0.0.0.0" /var/log/pihole/pihole.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "2" ]] + [[ ${lines[0]} == "4" ]] } @test "HTTP server responds with JSON error 404 to unknown API path" { @@ -1021,10 +1241,10 @@ [[ ${lines[0]} == '{"error":{"key":"not_found","message":"Not found","hint":"/api/undefined"},"took":'*'}' ]] } -@test "HTTP server responds with normal error 404 to path outside /admin" { - run bash -c 'curl -s 127.0.0.1/undefined' +@test "HTTP server responds with error 404 to path outside /admin" { + run bash -c 'curl -sI 127.0.0.1/undefined' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "Error 404: Not Found" ]] + [[ ${lines[@]} == *"HTTP/1.1 404 Not Found"* ]] } @test "LUA: Interpreter returns FTL version" { @@ -1209,22 +1429,6 @@ [[ "${lines[@]}" != *"ERROR"* ]] } -@test "No ERROR messages in FTL.log (besides known/intended error)" { - run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - run bash -c 'grep "ERROR: " /var/log/pihole/FTL.log | grep -c -v -E "(index\.html)|(Failed to create shared memory object)|(FTLCONF_debug_api is invalid)"' - printf "count: %s\n" "${lines[@]}" - [[ ${lines[0]} == "0" ]] -} - -@test "No CRIT messages in FTL.log (besides error due to testing to start FTL more than once)" { - run bash -c 'grep "CRIT: " /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - run bash -c 'grep "CRIT: " /var/log/pihole/FTL.log | grep -c -v "Initialization of shared memory failed."' - printf "count: %s\n" "${lines[@]}" - [[ ${lines[0]} == "0" ]] -} - @test "No missing config items in pihole.toml" { run bash -c 'grep "DEBUG_CONFIG: " /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" @@ -1246,6 +1450,15 @@ run bash -c "dig AAAA pi.hole +short @127.0.0.1" printf "AAAA: %s\n" "${lines[@]}" [[ "${lines[0]}" == "fe80::10" ]] + + run bash -c "dig A pi.hole @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig AAAA pi.hole @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] } @test "Pi-hole uses dns.reply.host.IPv4/6 for hostname" { @@ -1255,6 +1468,15 @@ run bash -c "dig AAAA $(hostname) +short @127.0.0.1" printf "AAAA: %s\n" "${lines[@]}" [[ "${lines[0]}" == "fe80::10" ]] + + run bash -c "dig A $(hostname) @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] + run bash -c "dig AAAA $(hostname) @127.0.0.1 | grep 'EDE: '" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == *"EDE: 29: (synthesized)" ]] + [[ ${lines[1]} == "" ]] } @test "Pi-hole uses dns.reply.blocking.IPv4/6 for blocked domain" { @@ -1337,10 +1559,9 @@ @test "Environmental variable is favored over config file" { # The config file has -10 but we set FTLCONF_misc_nice="-11" - run bash -c 'grep -B1 "nice = -11" /etc/pihole/pihole.toml' + run bash -c 'grep "nice = -11" /etc/pihole/pihole.toml' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == " # >>> This config is overwritten by an environmental variable <<<" ]] - [[ ${lines[1]} == " nice = -11 ### CHANGED, default = -10" ]] + [[ ${lines[0]} == " nice = -11 ### CHANGED (env), default = -10" ]] } @test "Correct number of environmental variables is logged" { @@ -1368,6 +1589,15 @@ [[ ${lines[1]} == *"WARNING: - FTLCONF_dns_upstreams" ]] } +@test "cJSON_GetErrorPtr and cJSON_InitHooks are never used (for thread-safety reasons)" { + # cJSON_GetErrorPtr() is not thread-safe but can be replaces by cJSON_ParseWithOpts() + # cJSON_InitHooks() is only thread-safe if used before any other cJSON function in a thread + # We grep for the two functions recursively and exclude cJSON.{c,h} where they are defined + run bash -c 'grep -rE "(cJSON_GetErrorPtr)|(cJSON_InitHooks)" src/ | grep -vE "^src/webserver/cJSON/cJSON."' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "" ]] +} + @test "CLI complains about unknown config key and offers a suggestion" { run bash -c './pihole-FTL --config dbg.all' [[ ${lines[0]} == "Unknown config option dbg.all, did you mean:" ]] @@ -1392,6 +1622,8 @@ [[ ${lines[0]} == '{"error":{"key":"bad_request","message":"Config items set via environment variables cannot be changed via the API","hint":"misc.nice"},"took":'*'}' ]] } +# We cannot easily test IPv6 as it may not be available in docker (CI) + @test "API domain search: Non-existing domain returns expected JSON" { run bash -c 'curl -s 127.0.0.1/api/search/non.existent' printf "%s\n" "${lines[@]}" @@ -1458,10 +1690,29 @@ [[ ${lines[0]} == "0" ]] } +# This test should run before a password it set +@test "Lua server page is generating proper backtrace" { + # Run a page with a syntax error + run bash -c 'curl -s 127.0.0.1/broken_lua' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == 'Hello, world 1!' ]] + [[ ${lines[1]} == 'Hello, world 2!' ]] + [[ ${lines[2]} == '[string "/var/www/html/broken_lua_2.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' ]] + [[ ${lines[3]} == 'stack traceback:' ]] + [[ ${lines[4]} == " [C]: in field 'include'" ]] + [[ ${lines[5]} == ' [string "/var/www/html/broken_lua.lp"]:4: in main chunk' ]] + [[ ${lines[6]} == 'aborting' ]] + [[ ${lines[7]} == '' ]] + + # Check if the error is logged (-F = fixed string (no regex), -q = quiet) + run grep -qF 'LSP Kepler: call failed: runtime error: [string "/var/www/html/broken_lua_2.lp"]:4: Cannot include [/var/www/html/does_not_exist.lp]: not found' /var/log/pihole/webserver.log + [[ $status == 0 ]] +} + @test "API authorization (without password): No login required" { run bash -c 'curl -s 127.0.0.1/api/auth' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == '{"session":{"valid":true,"totp":false,"sid":null,"validity":-1},"took":'*'}' ]] + [[ ${lines[0]} == '{"session":{"valid":true,"totp":false,"sid":null,"validity":-1,"message":"no password set"},"took":'*'}' ]] } @test "Config validation working on the CLI (type-based checking)" { @@ -1472,7 +1723,7 @@ run bash -c './pihole-FTL --config dns.revServers "abc"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == 'Config setting dns.revServers is invalid: not valid JSON, error before: abc' ]] + [[ ${lines[0]} == 'Config setting dns.revServers is invalid: not valid JSON, error at: abc' ]] [[ $status == 2 ]] } @@ -1524,7 +1775,7 @@ run bash -c './pihole-FTL --config dns.revServers "[\"true,1.1.1.1,def,ghi\"]"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == 'New dnsmasq configuration is not valid ('*'Name does not resolve at line '*' of /etc/pihole/dnsmasq.conf.temp: "rev-server=1.1.1.1,def"), config remains unchanged' ]] + [[ ${lines[0]} == 'New dnsmasq configuration is not valid ('*'resolve at line '*' of /etc/pihole/dnsmasq.conf.temp: "rev-server=1.1.1.1,def"), config remains unchanged' ]] [[ $status == 3 ]] run bash -c './pihole-FTL --config webserver.api.excludeClients "[\".*\",\"$$$\",\"[[[\"]"' @@ -1574,6 +1825,28 @@ [[ ${lines[0]} == "true" ]] } +@test "CLI password file is as expected" { + # Check the file is non-empty + run bash -c 'cat /etc/pihole/cli_pw' + printf "%s\n" "${lines[@]}" + [[ ${#lines[0]} -gt 0 ]] + + # Check if file has exactly one line + [[ ${#lines[@]} -eq 1 ]] + + # Check if this line does NOT have a newline character at the end + [[ ${lines[0]} != *$'\n' ]] + + # Check the file content is valid base64 + run bash -c 'echo ${0} | base64 -d > /dev/null' "${lines[0]}" + [[ $status == 0 ]] + + # Check permission set on the file is 640 + run bash -c 'stat -c "%a" /etc/pihole/cli_pw' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "640" ]] +} + @test "API authorization: Setting password" { # Password: ABC run bash -c 'curl -s -X PATCH http://127.0.0.1/api/config/webserver/api/password -d "{\"config\":{\"webserver\":{\"api\":{\"password\":\"ABC\"}}}}"' @@ -1583,17 +1856,16 @@ @test "API authorization (with password): Incorrect password is rejected if password auth is enabled" { # Password: ABC - run bash -c 'curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"XXX\"}" | jq .session.valid' + run bash -c 'curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"XXX\"}"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "false" ]] + [[ ${lines[0]} == "{\"session\":{\"valid\":false,\"totp\":false,\"sid\":null,\"validity\":-1,\"message\":\"password incorrect\"},\"took\":"*"}" ]] } @test "API authorization (with password): Correct password is accepted" { - session="$(curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"ABC\"}")" - printf "Session: %s\n" "${session}" - run jq .session.valid <<< "${session}" + # Password: ABC + run bash -c 'curl -s -X POST 127.0.0.1/api/auth -d "{\"password\":\"ABC\"}"' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "true" ]] + [[ ${lines[0]} == "{\"session\":{\"valid\":true,\"totp\":false,\"sid\":\""*"\",\"csrf\":\""*"\",\"validity\":300,\"message\":\"password correct\"},\"took\":"*"}" ]] } @test "Test TLS/SSL server using self-signed certificate" { @@ -1732,15 +2004,30 @@ [[ ${lines[0]} == "ce4c01340ef46bf3bc26831f7c53763d57c863528826aa795f1da5e16d6e7b2d test/test.pem" ]] } -@test "Internal IP -> name resolution works" { +@test "Internal IP -> name resolution works (UDP IPv4)" { run bash -c "./pihole-FTL ptr 127.0.0.1 | tail -n1" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "localhost" ]] +} + +@test "Internal IP -> name resolution works (UDP IPv6)" { run bash -c "./pihole-FTL ptr ::1 | tail -n1" printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "localhost" ]] } +@test "Internal IP -> name resolution works (TCP IPv4)" { + run bash -c "./pihole-FTL ptr 127.0.0.1 tcp | tail -n1" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "localhost" ]] +} + +@test "Internal IP -> name resolution works (TCP IPv6)" { + run bash -c "./pihole-FTL ptr ::1 tcp | tail -n1" + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "localhost" ]] +} + @test "API validation" { run python3 test/api/checkAPI.py printf "%s\n" "${lines[@]}" @@ -1779,9 +2066,15 @@ # [[ $status == 0 ]] run bash -c "./pihole-FTL --teleporter ${filename}" printf "%s\n" "${lines[@]}" - [[ "${lines[-3]}" == "Imported etc/pihole/pihole.toml" ]] - [[ "${lines[-2]}" == "Imported etc/pihole/dhcp.leases" ]] - [[ "${lines[-1]}" == "Imported etc/pihole/gravity.db" ]] + [[ "${lines[-9]}" == "Imported etc/pihole/pihole.toml" ]] + [[ "${lines[-8]}" == "Imported etc/pihole/dhcp.leases" ]] + [[ "${lines[-7]}" == "Imported etc/pihole/gravity.db->group" ]] + [[ "${lines[-6]}" == "Imported etc/pihole/gravity.db->adlist" ]] + [[ "${lines[-5]}" == "Imported etc/pihole/gravity.db->adlist_by_group" ]] + [[ "${lines[-4]}" == "Imported etc/pihole/gravity.db->domainlist" ]] + [[ "${lines[-3]}" == "Imported etc/pihole/gravity.db->domainlist_by_group" ]] + [[ "${lines[-2]}" == "Imported etc/pihole/gravity.db->client" ]] + [[ "${lines[-1]}" == "Imported etc/pihole/gravity.db->client_by_group" ]] [[ $status == 0 ]] run bash -c "rm ${filename}" } @@ -1789,20 +2082,19 @@ @test "Expected number of config file rotations" { run bash -c 'grep -c "INFO: Config file written to /etc/pihole/pihole.toml" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] - run bash -c 'grep -c "DEBUG_CONFIG: pihole.toml unchanged" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] + [[ ${lines[0]} == "2" ]] run bash -c 'grep -c "DEBUG_CONFIG: Config file written to /etc/pihole/dnsmasq.conf" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "DEBUG_CONFIG: dnsmasq.conf unchanged" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "2" ]] run bash -c 'grep -c "DEBUG_CONFIG: HOSTS file written to /etc/pihole/hosts/custom.list" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] - run bash -c 'grep -c "DEBUG_CONFIG: custom.list unchanged" /var/log/pihole/FTL.log' - printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] +} + +@test "Check NTP server is broadcasting correct time" { + # Run this test at the very end of the test suite + # to ensure the NTP server has been started + run bash -c './pihole-FTL ntp 127.0.0.1' + printf "%s\n" "${lines[@]}" + [[ $status == 0 ]] } diff --git a/tools/macvendor.py b/tools/macvendor.py index 003f4e7a..d0cabeed 100644 --- a/tools/macvendor.py +++ b/tools/macvendor.py @@ -12,21 +12,19 @@ import os import re -import urllib.request +import requests import sqlite3 # Download raw data from Wireshark's website # We use the official URL recommended in the header of this file -# Thanks to mibere for the update +URL = "https://www.wireshark.org/download/automated/data/manuf" +# User-Agent string to use for the request +USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" print("Downloading...") -opener = urllib.request.build_opener() -opener.addheaders = [('User-agent', 'Mozilla/5.0')] -urllib.request.install_opener(opener) -urllib.request.urlretrieve("https://gitlab.com/wireshark/wireshark/-/raw/master/manuf", "manuf.data") +manuf = requests.get(URL, headers={"User-Agent": USER_AGENT}).text.splitlines() print("...done") # Read file into memory and process lines -manuf = open("manuf.data", "r") data = [] print("Processing...") for line in manuf: @@ -63,7 +61,6 @@ for line in manuf: else: data.append([mac, desc_short]) print("...done") -manuf.close() # Create database database = "macvendor.db"