From dfcfafd6d9ca69ea8393eb8aadec15f82ad951f1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 21:15:50 +0200 Subject: [PATCH 01/34] Implement whitelist regex support to web interface. Signed-off-by: DL6ER --- list.php | 8 -------- scripts/pi-hole/js/list.js | 27 +++++++++++---------------- scripts/pi-hole/php/add.php | 10 ++++++++-- scripts/pi-hole/php/auth.php | 1 - scripts/pi-hole/php/get.php | 10 ++++++---- scripts/pi-hole/php/sub.php | 13 ++++++++----- 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/list.php b/list.php index 5cbff9c7..8d4fc1ba 100644 --- a/list.php +++ b/list.php @@ -35,13 +35,9 @@ function getFullName() {
- - - -
@@ -66,14 +62,10 @@ function getFullName() { -

Exact blocking

- -

Regex & Wildcard blocking

- diff --git a/scripts/pi-hole/js/list.js b/scripts/pi-hole/js/list.js index 53d6b7e5..9a6644ca 100644 --- a/scripts/pi-hole/js/list.js +++ b/scripts/pi-hole/js/list.js @@ -43,12 +43,8 @@ function addListEntry(entry, index, list, button, type) } function refresh(fade) { - var listw; var list = $("#list"); - if(listType === "black") - { - listw = $("#list-regex"); - } + var listw = $("#list-regex"); if(fade) { list.fadeOut(100); if(listw) @@ -69,9 +65,10 @@ function refresh(fade) { if((listType === "black" && response.blacklist.length === 0 && - response.regex.length === 0) || + response.regex_blacklist.length === 0) || (listType === "white" && - response.whitelist.length === 0)) + response.whitelist.length === 0 && + response.regex_whitelist.length === 0)) { $("h3").hide(); list.html("
Your " + fullName + " is empty!
"); @@ -82,12 +79,12 @@ function refresh(fade) { if(listType === "white") { data = response.whitelist.sort(); - data2 = []; // No regex data, use empty array + data2 = response.regex_whitelist.sort(); } else if(listType === "black") { data = response.blacklist.sort(); - data2 = response.regex.sort(); + data2 = response.regex_blacklist.sort(); } data.forEach(function (entry, index) { @@ -97,7 +94,7 @@ function refresh(fade) { // Add regex domains if present in returned list data data2.forEach(function (entry, index) { - addListEntry(entry, index, listw, "#list-regex", "regex"); + addListEntry(entry, index, listw, "#list-regex", listType+"_regex"); }); } list.fadeIn(100); @@ -117,9 +114,9 @@ window.onload = refresh(false); function sub(index, entry, arg) { var domain = $("#list #"+index); var locallistType = listType; - if(arg === "regex") + if(arg === "black_regex" || arg === "white_regex") { - locallistType = "regex"; + locallistType = arg; domain = $("#list-regex #"+index); } domain.hide("highlight"); @@ -143,14 +140,12 @@ function sub(index, entry, arg) { function add(arg) { var locallistType = listType; var domain = $("#domain"); - var wild = false; if(domain.val().length === 0){ return; } - if(arg === "wild" || arg === "regex") + if(arg === "wild") { - locallistType = arg; - wild = true; + locallistType = listType+"_wild"; } var alInfo = $("#alInfo"); diff --git a/scripts/pi-hole/php/add.php b/scripts/pi-hole/php/add.php index 68a7496d..9f3c1b0a 100644 --- a/scripts/pi-hole/php/add.php +++ b/scripts/pi-hole/php/add.php @@ -44,12 +44,18 @@ switch($type) { echo shell_exec("sudo pihole -a audit ".$domains); } break; - case "regex": + case "black_regex": echo shell_exec("sudo pihole --regex --web ".$domains); break; - case "wild": + case "white_regex": + echo shell_exec("sudo pihole --whiteregex --web ".$domains); + break; + case "black_wild": echo shell_exec("sudo pihole --wild --web ".$domains); break; + case "white_wild": + echo shell_exec("sudo pihole --whitewild --web ".$domains); + break; case "audit": echo shell_exec("sudo pihole -a audit ".$domains); break; diff --git a/scripts/pi-hole/php/auth.php b/scripts/pi-hole/php/auth.php index 69fe6b28..1ea6014d 100644 --- a/scripts/pi-hole/php/auth.php +++ b/scripts/pi-hole/php/auth.php @@ -11,7 +11,6 @@ $ERRORLOG = getenv('PHP_ERROR_LOG'); if (empty($ERRORLOG)) { $ERRORLOG = '/var/log/lighttpd/error.log'; } -$regexfile = "/etc/pihole/regex.list"; function pi_log($message) { error_log(date('Y-m-d H:i:s') . ': ' . $message . "\n", 3, $GLOBALS['ERRORLOG']); diff --git a/scripts/pi-hole/php/get.php b/scripts/pi-hole/php/get.php index c84b991a..b38446bc 100644 --- a/scripts/pi-hole/php/get.php +++ b/scripts/pi-hole/php/get.php @@ -21,7 +21,7 @@ function getTableContent($listname) { global $db; $entries = array(); $querystr = implode(" ",array("SELECT ${listname}.*,\"group\".enabled as group_enabled", - "FROM $listname", + "FROM ${listname}", "LEFT JOIN ${listname}_by_group ON ${listname}_by_group.${listname}_id = ${listname}.id", "LEFT JOIN \"group\" ON \"group\".id = ${listname}_by_group.group_id", "GROUP BY domain;")); @@ -54,13 +54,15 @@ function filterArray(&$inArray) { switch ($listtype) { case "white": - $list = getTableContent("whitelist"); + $exact = getTableContent("whitelist"); + $regex = getTableContent("regex_whitelist"); + $list = array_merge($exact, $regex); break; case "black": $exact = getTableContent("blacklist"); - $regex = getTableContent("regex"); - $list = array_merge($exact, $regex); + $regex = getTableContent("regex_blacklist"); + $list = array_merge($exact, $regex); break; default: diff --git a/scripts/pi-hole/php/sub.php b/scripts/pi-hole/php/sub.php index 97c8f84b..4eae79c7 100644 --- a/scripts/pi-hole/php/sub.php +++ b/scripts/pi-hole/php/sub.php @@ -18,7 +18,7 @@ if (empty($api)) { // Don't check if the added item is a valid domain for regex expressions. // Regex filters are validated by FTL on import and skipped if invalid -if($type !== "regex") { +if($type !== "black_regex" && $type !== "white_regex") { check_domain(); } @@ -27,13 +27,16 @@ $domain = escapeshellcmd($_POST['domain']); switch($type) { case "white": - exec("sudo pihole -w -q -d ".$domain); + echo shell_exec("sudo pihole -w -q -d ".$domain); break; case "black": - exec("sudo pihole -b -q -d ".$domain); + echo shell_exec("sudo pihole -b -q -d ".$domain); break; - case "regex": - exec("sudo pihole --regex -q -d ".$domain); + case "black_regex": + echo shell_exec("sudo pihole --regex -q -d ".$domain); + break; + case "white_regex": + echo shell_exec("sudo pihole --whiteregex -q -d ".$domain); break; } From 31d0b32103df6b2445a2ce8cf75c7c672b5036e9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 21:35:42 +0200 Subject: [PATCH 02/34] Only show headings when we have domains for the respective list (regex / exact). Signed-off-by: DL6ER --- list.php | 4 ++-- scripts/pi-hole/js/list.js | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/list.php b/list.php index 8d4fc1ba..d6ea04b9 100644 --- a/list.php +++ b/list.php @@ -62,9 +62,9 @@ function getFullName() { -

Exact blocking

+
    -

    Regex & Wildcard blocking

    +
      diff --git a/scripts/pi-hole/js/list.js b/scripts/pi-hole/js/list.js index 9a6644ca..09e67736 100644 --- a/scripts/pi-hole/js/list.js +++ b/scripts/pi-hole/js/list.js @@ -75,7 +75,6 @@ function refresh(fade) { } else { - $("h3").show(); if(listType === "white") { data = response.whitelist.sort(); @@ -86,12 +85,20 @@ function refresh(fade) { data = response.blacklist.sort(); data2 = response.regex_blacklist.sort(); } + + if(data.length > 0) + { + $("#h3-exact").show(); + } + if(data2.length > 0) + { + $("#h3-regex").show(); + } + data.forEach(function (entry, index) { addListEntry(entry, index, list, "#list", "exact"); }); - - // Add regex domains if present in returned list data data2.forEach(function (entry, index) { addListEntry(entry, index, listw, "#list-regex", listType+"_regex"); From b7fe51815983f2e93d3ce912ab9d1ce206bd4372 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 21:38:58 +0200 Subject: [PATCH 03/34] Remove heading when list goes empty. Signed-off-by: DL6ER --- scripts/pi-hole/js/list.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/pi-hole/js/list.js b/scripts/pi-hole/js/list.js index 09e67736..8545de5b 100644 --- a/scripts/pi-hole/js/list.js +++ b/scripts/pi-hole/js/list.js @@ -119,13 +119,16 @@ function refresh(fade) { window.onload = refresh(false); function sub(index, entry, arg) { - var domain = $("#list #"+index); + var list = "#list"; + var heading = "#h3-exact"; var locallistType = listType; if(arg === "black_regex" || arg === "white_regex") { + list = "#list-regex"; + heading = "#h3-regex"; locallistType = arg; - domain = $("#list-regex #"+index); } + var domain = $(list+" #"+index); domain.hide("highlight"); $.ajax({ url: "scripts/pi-hole/php/sub.php", @@ -136,6 +139,10 @@ function sub(index, entry, arg) { return; } domain.remove(); + if($(list+" li").length < 1) + { + $(heading).hide(); + } }, error: function(jqXHR, exception) { alert("Failed to remove the domain!"); From 3ebf766240f91ef2a44298504449bf8c1d22a76a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 21:43:12 +0200 Subject: [PATCH 04/34] Simplify add() in list.js. Signed-off-by: DL6ER --- scripts/pi-hole/js/list.js | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/scripts/pi-hole/js/list.js b/scripts/pi-hole/js/list.js index 8545de5b..3feb24fc 100644 --- a/scripts/pi-hole/js/list.js +++ b/scripts/pi-hole/js/list.js @@ -151,16 +151,11 @@ function sub(index, entry, arg) { }); } -function add(arg) { - var locallistType = listType; +function add(type) { var domain = $("#domain"); if(domain.val().length === 0){ return; } - if(arg === "wild") - { - locallistType = listType+"_wild"; - } var alInfo = $("#alInfo"); var alSuccess = $("#alSuccess"); @@ -175,7 +170,7 @@ function add(arg) { $.ajax({ url: "scripts/pi-hole/php/add.php", method: "post", - data: {"domain":domain.val().trim(), "list":locallistType, "token":token}, + data: {"domain":domain.val().trim(), "list":type, "token":token}, success: function(response) { if (response.indexOf(" already exists in ") !== -1) { alWarning.show(); @@ -232,15 +227,15 @@ $(document).keypress(function(e) { // Handle buttons $("#btnAdd").on("click", function() { - add("exact"); + add(listType); }); $("#btnAddWildcard").on("click", function() { - add("wild"); + add(listType+"_wild"); }); $("#btnAddRegex").on("click", function() { - add("regex"); + add(listType+"_regex"); }); $("#btnRefresh").on("click", function() { From cd2daa323990a49029a2fcd4608b9763a8cb6655 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Jul 2019 19:41:04 +0200 Subject: [PATCH 05/34] Review comments Signed-off-by: DL6ER --- scripts/pi-hole/js/list.js | 15 +++------------ scripts/pi-hole/php/func.php | 17 ----------------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/scripts/pi-hole/js/list.js b/scripts/pi-hole/js/list.js index 3feb24fc..a5bcafd9 100644 --- a/scripts/pi-hole/js/list.js +++ b/scripts/pi-hole/js/list.js @@ -47,10 +47,7 @@ function refresh(fade) { var listw = $("#list-regex"); if(fade) { list.fadeOut(100); - if(listw) - { - listw.fadeOut(100); - } + listw.fadeOut(100); } $.ajax({ url: "scripts/pi-hole/php/get.php", @@ -58,10 +55,7 @@ function refresh(fade) { data: {"list":listType}, success: function(response) { list.html(""); - if(listw) - { - listw.html(""); - } + listw.html(""); if((listType === "black" && response.blacklist.length === 0 && @@ -105,10 +99,7 @@ function refresh(fade) { }); } list.fadeIn(100); - if(listw) - { - listw.fadeIn(100); - } + listw.fadeIn(100); }, error: function(jqXHR, exception) { $("#alFailure").show(); diff --git a/scripts/pi-hole/php/func.php b/scripts/pi-hole/php/func.php index 7c1d3d65..bc353218 100644 --- a/scripts/pi-hole/php/func.php +++ b/scripts/pi-hole/php/func.php @@ -45,21 +45,4 @@ if(!function_exists('hash_equals')) { } } -function add_regex($regex, $mode=FILE_APPEND, $append="\n") -{ - global $regexfile; - if(file_put_contents($regexfile, $regex.$append, $mode) === FALSE) - { - $err = error_get_last()["message"]; - echo "Unable to add regex \"".htmlspecialchars($regex)."\" to ${regexfile}
      Error message: $err"; - } - else - { - // Send SIGHUP to pihole-FTL using a frontend command - // to force reloading of the regex domains - // This will also wipe the resolver's cache - echo exec("sudo pihole restartdns reload"); - } -} - ?> From 1ddf36a752260d06614d10c5807455d7477ce13c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Jul 2019 21:10:19 +0200 Subject: [PATCH 06/34] Archive and restore full tables instead of only domains. Use JSON as backup file format. Signed-off-by: DL6ER --- scripts/pi-hole/php/teleporter.php | 161 ++++++++++++++++++++++++++--- settings.php | 14 ++- 2 files changed, 160 insertions(+), 15 deletions(-) diff --git a/scripts/pi-hole/php/teleporter.php b/scripts/pi-hole/php/teleporter.php index 9b03a788..54254466 100644 --- a/scripts/pi-hole/php/teleporter.php +++ b/scripts/pi-hole/php/teleporter.php @@ -15,7 +15,7 @@ if (php_sapi_name() !== "cli") { check_csrf(isset($_POST["token"]) ? $_POST["token"] : ""); } -$db = SQLite3_connect(getGravityDBFilename()); +$db = SQLite3_connect(getGravityDBFilename(), SQLITE3_OPEN_READWRITE); function archive_add_file($path,$name,$subdir="") { @@ -29,20 +29,112 @@ function archive_add_file($path,$name,$subdir="") * * @param $name string The name of the file in the archive to save the table to * @param $table string The table to export - * @param $column string The column on the table to export */ -function archive_add_table($name, $table, $column) +function archive_add_table($name, $table) { global $archive, $db; - $results = $db->query("SELECT $column FROM $table"); - $content = ""; + $results = $db->query("SELECT * FROM $table"); - while($row = $results->fetchArray()) { - $content .= $row[0]."\n"; + // Return early without creating a file if th + // requested table cannot be accessed + if(is_null($results)) + return; + + $content = array(); + while ($row = $results->fetchArray(SQLITE3_ASSOC)) + { + array_push($content, $row); } - $archive[$name] = $content; + $archive[$name] = json_encode($content); +} + +/** + * Restore the contents of a table from an uploaded archive + * + * @param $file object The file of the file in the archive to restore the table from + * @param $table string The table to import + * @param $flush boolean Whether to flush the table before importing the archived data + */ +function archive_restore_table($file, $table, $flush=false) +{ + global $archive, $db; + + $json_string = file_get_contents($file); + // Return early if we cannot extract the JSON string + if(is_null($json_string)) + return 0; + + $contents = json_decode($json_string, true); + // Return early if we cannot decode the JSON string + if(is_null($contents)) + return 0; + + // Flush table if requested + if($flush) + $db->exec("DELETE FROM ".$table); + + // Prepare field name for domain/address depending on the table we restore to + if($table === "adlist") + { + $sql = "INSERT OR IGNORE INTO adlist"; + $sql .= " (id,address,enabled,date_added,comment)"; + $sql .= " VALUES (:id,:address,:enabled,:date_added,:comment);"; + } + elseif($table === "domain_audit") + { + $sql = "INSERT OR IGNORE INTO domain_audit"; + $sql .= " (id,domain,date_added)"; + $sql .= " VALUES (:id,:domain,:date_added);"; + } + else + { + $sql = "INSERT OR IGNORE INTO ".$table; + $sql .= " (id,domain,enabled,date_added,comment)"; + $sql .= " VALUES (:id,:domain,:enabled,:date_added,:comment);"; + } + + // Prepare SQLite statememt + $stmt = $db->prepare($sql); + + // Return early if we prepare the SQLite statement + if(!$stmt) + { + echo "Failed to prepare statement for ".$table." table."; + echo $sql; + return 0; + } + + // Loop over rows and inject the lines into the database + $num = 0; + foreach($contents as $row) + { + $stmt->bindValue(":id", $row["id"], SQLITE3_INTEGER); + $stmt->bindValue(":date_added", $row["date_added"], SQLITE3_INTEGER); + + if($table === "adlist") + $stmt->bindValue(":address", $row["address"], SQLITE3_TEXT); + else + $stmt->bindValue(":domain", $row["domain"], SQLITE3_TEXT); + + if($table !== "domain_audit") + { + $stmt->bindValue(":enabled", $row["enabled"], SQLITE3_INTEGER); + if(is_null($row["comment"])) + $type = SQLITE3_NULL; + else + $type = SQLITE3_TEXT; + $stmt->bindValue(":comment", $row["comment"], $type); + } + + $stmt->execute(); + $stmt->reset(); + $stmt->clear(); + $num++; + } + + return $num; } function archive_add_directory($path,$subdir="") @@ -176,6 +268,48 @@ if(isset($_POST["action"])) exec("sudo pihole -a audit ".implode(" ",$auditlog)); } + if(isset($_POST["blacklist"]) && $file->getFilename() === "blacklist.exact.json") + { + $num = archive_restore_table($file, "blacklist"); + echo "Processed blacklist (exact) (".$num." entries)
      \n"; + $importedsomething = true; + } + + if(isset($_POST["regexlist"]) && $file->getFilename() === "blacklist.regex.json") + { + $num = archive_restore_table($file, "regex_blacklist"); + echo "Processed blacklist (regex) (".$num." entries)
      \n"; + $importedsomething = true; + } + + if(isset($_POST["whitelist"]) && $file->getFilename() === "whitelist.exact.json") + { + $num = archive_restore_table($file, "whitelist"); + echo "Processed whitelist (exact) (".$num." entries)
      \n"; + $importedsomething = true; + } + + if(isset($_POST["regex_whitelist"]) && $file->getFilename() === "whitelist.regex.json") + { + $num = archive_restore_table($file, "regex_whitelist"); + echo "Processed whitelist (regex) (".$num." entries)
      \n"; + $importedsomething = true; + } + + if(isset($_POST["adlist"]) && $file->getFilename() === "adlist.json") + { + $num = archive_restore_table($file, "adlist"); + echo "Processed adlist (".$num." entries)
      \n"; + $importedsomething = true; + } + + if(isset($_POST["auditlog"]) && $file->getFilename() === "domain_audit.json") + { + $num = archive_restore_table($file, "domain_audit"); + echo "Processed domain_audit (".$num." entries)
      \n"; + $importedsomething = true; + } + if($importedsomething) { exec("sudo pihole restartdns"); @@ -201,12 +335,13 @@ else exit("cannot open/create ".htmlentities($archive_file_name)."
      \nPHP user: ".exec('whoami')."\n"); } - archive_add_table("whitelist.txt", "whitelist", "domain"); - archive_add_table("blacklist.txt", "blacklist", "domain"); - archive_add_table("regex.list", "regex", "domain"); - archive_add_table("adlists.list", "adlist", "address"); + archive_add_table("whitelist.exact.json", "whitelist"); + archive_add_table("whitelist.regex.json", "regex_whitelist"); + archive_add_table("blacklist.exact.json", "blacklist"); + archive_add_table("blacklist.regex.json", "regex_blacklist"); + archive_add_table("adlists.json", "adlist"); + archive_add_table("domain_audit.json", "domain_audit"); archive_add_file("/etc/pihole/","setupVars.conf"); - archive_add_file("/etc/pihole/","auditlog.list"); archive_add_directory("/etc/dnsmasq.d/","dnsmasq.d/"); $archive->compress(Phar::GZ); // Creates a gziped copy diff --git a/settings.php b/settings.php index e9423ecb..f0c5a187 100644 --- a/settings.php +++ b/settings.php @@ -1145,7 +1145,12 @@ if (isset($_GET['tab']) && in_array($_GET['tab'], array("sysadmin", "blocklists"
      + Whitelist (exact) +
      +
      +
      +
      +
      +
      From 72057cfa26353f2260e4318fe5edb51b9c13ee7b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Jul 2019 22:14:07 +0200 Subject: [PATCH 08/34] Close database connection at the end of a table recovery. Signed-off-by: DL6ER --- scripts/pi-hole/php/teleporter.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/pi-hole/php/teleporter.php b/scripts/pi-hole/php/teleporter.php index 65e7f490..2d377c83 100644 --- a/scripts/pi-hole/php/teleporter.php +++ b/scripts/pi-hole/php/teleporter.php @@ -59,7 +59,7 @@ function archive_add_table($name, $table) */ function archive_restore_table($file, $table, $flush=false) { - global $archive, $db; + global $db; $json_string = file_get_contents($file); // Return early if we cannot extract the JSON string @@ -128,12 +128,17 @@ function archive_restore_table($file, $table, $flush=false) $stmt->bindValue(":comment", $row["comment"], $type); } - $stmt->execute(); - $stmt->reset(); - $stmt->clear(); - $num++; + if($stmt->execute() && $stmt->reset() && $stmt->clear()) + $num++; + else + { + $stmt->close(); + return $num; + } } + // Close database connection and return number or processed rows + $stmt->close(); return $num; } From 3cfb2267571f83f9f0b8525673e29cb36d628a40 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Jul 2019 22:14:48 +0200 Subject: [PATCH 09/34] Reload only ONCE not after EACH imported file. Signed-off-by: DL6ER --- scripts/pi-hole/php/teleporter.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/pi-hole/php/teleporter.php b/scripts/pi-hole/php/teleporter.php index 2d377c83..ddc4e7c4 100644 --- a/scripts/pi-hole/php/teleporter.php +++ b/scripts/pi-hole/php/teleporter.php @@ -316,11 +316,11 @@ if(isset($_POST["action"])) echo "Processed domain_audit (".$num." entries)
      \n"; $importedsomething = true; } + } - if($importedsomething) - { - exec("sudo pihole restartdns"); - } + if($importedsomething) + { + exec("sudo pihole restartdns"); } unlink($fullfilename); From eac87f139f480641c2e96755f64bea7e007f52de Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Jul 2019 22:19:15 +0200 Subject: [PATCH 10/34] pihole-FTL only needs to be reloaded not restarted when the lists have been updated. This is much faster than a hard restart. Signed-off-by: DL6ER --- scripts/pi-hole/php/teleporter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pi-hole/php/teleporter.php b/scripts/pi-hole/php/teleporter.php index ddc4e7c4..db5688df 100644 --- a/scripts/pi-hole/php/teleporter.php +++ b/scripts/pi-hole/php/teleporter.php @@ -320,7 +320,7 @@ if(isset($_POST["action"])) if($importedsomething) { - exec("sudo pihole restartdns"); + exec("sudo pihole restartdns reload"); } unlink($fullfilename); From 13f07f9d87c681eccd9ee884a11ffba795cd2a89 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Jul 2019 23:15:19 +0200 Subject: [PATCH 11/34] Switch the teleporter to only interact with gravity.db directly. This avoids any need for escaping regexes and just seems to be the right thing to do. Ensure that we flush a table only once (in case multiple files add to the same table). Signed-off-by: DL6ER --- scripts/pi-hole/js/list.js | 6 +- scripts/pi-hole/php/teleporter.php | 152 ++++++++++++++++++----------- 2 files changed, 96 insertions(+), 62 deletions(-) diff --git a/scripts/pi-hole/js/list.js b/scripts/pi-hole/js/list.js index a5bcafd9..adb7c642 100644 --- a/scripts/pi-hole/js/list.js +++ b/scripts/pi-hole/js/list.js @@ -82,11 +82,11 @@ function refresh(fade) { if(data.length > 0) { - $("#h3-exact").show(); + $("#h3-exact").fadeIn(100); } if(data2.length > 0) { - $("#h3-regex").show(); + $("#h3-regex").fadeIn(100); } data.forEach(function (entry, index) @@ -132,7 +132,7 @@ function sub(index, entry, arg) { domain.remove(); if($(list+" li").length < 1) { - $(heading).hide(); + $(heading).fadeOut(100); } }, error: function(jqXHR, exception) { diff --git a/scripts/pi-hole/php/teleporter.php b/scripts/pi-hole/php/teleporter.php index db5688df..6d4083f8 100644 --- a/scripts/pi-hole/php/teleporter.php +++ b/scripts/pi-hole/php/teleporter.php @@ -17,6 +17,8 @@ if (php_sapi_name() !== "cli") { $db = SQLite3_connect(getGravityDBFilename(), SQLITE3_OPEN_READWRITE); +$flushed_tables = array(); + function archive_add_file($path,$name,$subdir="") { global $archive; @@ -59,7 +61,7 @@ function archive_add_table($name, $table) */ function archive_restore_table($file, $table, $flush=false) { - global $db; + global $db, $flushed_tables; $json_string = file_get_contents($file); // Return early if we cannot extract the JSON string @@ -71,9 +73,12 @@ function archive_restore_table($file, $table, $flush=false) if(is_null($contents)) return 0; - // Flush table if requested - if($flush) + // Flush table if requested, only flush each table once + if($flush && !in_array($table, $flushed_tables)) + { $db->exec("DELETE FROM ".$table); + array_push($flushed_tables, $table); + } // Prepare field name for domain/address depending on the table we restore to if($table === "adlist") @@ -142,6 +147,80 @@ function archive_restore_table($file, $table, $flush=false) return $num; } +/** + * Create table rows from an uploaded archive file + * + * @param $file object The file of the file in the archive to import + * @param $table string The target table + * @param $flush boolean Whether to flush the table before importing the archived data + */ +function archive_insert_into_table($file, $table, $flush=false, $wildcardstyle=false) +{ + global $db, $flushed_tables; + + $rows = array_filter(explode("\n",file_get_contents($file))); + // Return early if we cannot extract the lines in the file + if(is_null($rows)) + return 0; + + // Flush table if requested, only flush each table once + if($flush && !in_array($table, $flushed_tables)) + { + $db->exec("DELETE FROM ".$table); + array_push($flushed_tables, $table); + } + + // Prepare field name for domain/address depending on the table we restore to + if($table === "adlist") + { + $sql = "INSERT OR IGNORE INTO adlist"; + $sql .= " (address) VALUES (:address);"; + } + else + { + $sql = "INSERT OR IGNORE INTO ".$table; + $sql .= " (domain) VALUES (:domain);"; + } + + // Prepare SQLite statememt + $stmt = $db->prepare($sql); + + // Return early if we prepare the SQLite statement + if(!$stmt) + { + echo "Failed to prepare statement for ".$table." table."; + echo $sql; + return 0; + } + + // Loop over rows and inject the lines into the database + $num = 0; + foreach($rows as $row) + { + if($wildcardstyle) + $line = "(\\.|^)".str_replace(".","\\.",$row)."$"; + else + $line = $row; + + if($table === "adlist") + $stmt->bindValue(":address", $line, SQLITE3_TEXT); + else + $stmt->bindValue(":domain", $line, SQLITE3_TEXT); + + if($stmt->execute() && $stmt->reset() && $stmt->clear()) + $num++; + else + { + $stmt->close(); + return $num; + } + } + + // Close database connection and return number or processed rows + $stmt->close(); + return $num; +} + function archive_add_directory($path,$subdir="") { if($dir = opendir($path)) @@ -157,40 +236,6 @@ function archive_add_directory($path,$subdir="") } } -function limit_length(&$item, $key) -{ - // limit max length for a domain entry to 253 chars - // return only a part of the string if it is longer - $item = substr($item, 0, 253); -} - -function process_file($contents,$check=True) -{ - $domains = array_filter(explode("\n",$contents)); - - // Walk array and apply a max string length - // function to every member of the array of domains - array_walk($domains, "limit_length"); - - // Check validity of domains (don't do it for regex filters) - if($check) - { - check_domains($domains); - } - - return $domains; -} - -function check_domains($domains) -{ - foreach($domains as $domain) - { - if(!is_valid_domain_name($domain)){ - die(htmlspecialchars($domain).' is not a valid domain'); - } - } -} - if(isset($_POST["action"])) { if($_FILES["zip_file"]["name"] && $_POST["action"] == "in") @@ -230,49 +275,38 @@ if(isset($_POST["action"])) { if(isset($_POST["blacklist"]) && $file->getFilename() === "blacklist.txt") { - $blacklist = process_file(file_get_contents($file)); - echo "Processing blacklist.txt (".count($blacklist)." entries)
      \n"; - exec("sudo pihole -b -nr --nuke"); - exec("sudo pihole -b -q -nr ".implode(" ", $blacklist)); + $num = archive_insert_into_table($file, "blacklist", $flushtables); + echo "Processed blacklist (exact) (".$num." entries)
      \n"; $importedsomething = true; } if(isset($_POST["whitelist"]) && $file->getFilename() === "whitelist.txt") { - $whitelist = process_file(file_get_contents($file)); - echo "Processing whitelist.txt (".count($whitelist)." entries)
      \n"; - exec("sudo pihole -w -nr --nuke"); - exec("sudo pihole -w -q -nr ".implode(" ", $whitelist)); + $num = archive_insert_into_table($file, "whitelist", $flushtables); + echo "Processed whitelist (exact) (".$num." entries)
      \n"; $importedsomething = true; } if(isset($_POST["regexlist"]) && $file->getFilename() === "regex.list") { - $regexlist = process_file(file_get_contents($file),false); - echo "Processing regex.list (".count($regexlist)." entries)
      \n"; - - $escapedRegexlist = array_map("escapeshellcmd", $regexlist); - exec("sudo pihole --regex -nr --nuke"); - exec("sudo pihole --regex -q -nr ".implode(" ", $escapedRegexlist)); + $num = archive_insert_into_table($file, "regex_blacklist", $flushtables); + echo "Processed blacklist (regex) (".$num." entries)
      \n"; $importedsomething = true; } // Also try to import legacy wildcard list if found if(isset($_POST["regexlist"]) && $file->getFilename() === "wildcardblocking.txt") { - $wildlist = process_file(file_get_contents($file)); - echo "Processing wildcardblocking.txt (".count($wildlist)." entries)
      \n"; - exec("sudo pihole --wild -nr --nuke"); - exec("sudo pihole --wild -q -nr ".implode(" ", $wildlist)); + $num = archive_insert_into_table($file, "regex_blacklist", $flushtables, true); + echo "Processed blacklist (regex, wildcard style) (".$num." entries)
      \n"; $importedsomething = true; } if(isset($_POST["auditlog"]) && $file->getFilename() === "auditlog.list") { - $auditlog = process_file(file_get_contents($file)); - echo "Processing auditlog.list (".count($auditlog)." entries)
      \n"; - exec("sudo pihole -a clearaudit"); - exec("sudo pihole -a audit ".implode(" ",$auditlog)); + $num = archive_insert_into_table($file, "domain_audit", $flushtables); + echo "Processed blacklist (regex) (".$num." entries)
      \n"; + $importedsomething = true; } if(isset($_POST["blacklist"]) && $file->getFilename() === "blacklist.exact.json") From bc7aef744cdd4f557c8b6bc65dbb06ac31d1c417 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:13:53 +0200 Subject: [PATCH 12/34] "pihole --whiteregex" has been renamed to "pihole --white-regex". Similar for "--whitewild" -> "--white-wild". Signed-off-by: DL6ER --- scripts/pi-hole/php/add.php | 4 ++-- scripts/pi-hole/php/sub.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pi-hole/php/add.php b/scripts/pi-hole/php/add.php index 9f3c1b0a..663320f2 100644 --- a/scripts/pi-hole/php/add.php +++ b/scripts/pi-hole/php/add.php @@ -48,13 +48,13 @@ switch($type) { echo shell_exec("sudo pihole --regex --web ".$domains); break; case "white_regex": - echo shell_exec("sudo pihole --whiteregex --web ".$domains); + echo shell_exec("sudo pihole --white-regex --web ".$domains); break; case "black_wild": echo shell_exec("sudo pihole --wild --web ".$domains); break; case "white_wild": - echo shell_exec("sudo pihole --whitewild --web ".$domains); + echo shell_exec("sudo pihole --white-wild --web ".$domains); break; case "audit": echo shell_exec("sudo pihole -a audit ".$domains); diff --git a/scripts/pi-hole/php/sub.php b/scripts/pi-hole/php/sub.php index 4eae79c7..570520a8 100644 --- a/scripts/pi-hole/php/sub.php +++ b/scripts/pi-hole/php/sub.php @@ -36,7 +36,7 @@ switch($type) { echo shell_exec("sudo pihole --regex -q -d ".$domain); break; case "white_regex": - echo shell_exec("sudo pihole --whiteregex -q -d ".$domain); + echo shell_exec("sudo pihole --white-regex -q -d ".$domain); break; } From c9f5c9500a30e7cc6cd6faef4c4550338600a8ee Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:48:31 +0200 Subject: [PATCH 13/34] Directly modify gravity tables from the lists pages. Signed-off-by: DL6ER --- list.php | 2 +- scripts/pi-hole/js/list.js | 37 ++++++++---- scripts/pi-hole/php/add.php | 115 +++++++++++++++++++++++++----------- scripts/pi-hole/php/sub.php | 76 +++++++++++++++++------- 4 files changed, 160 insertions(+), 70 deletions(-) diff --git a/list.php b/list.php index d6ea04b9..21db22cc 100644 --- a/list.php +++ b/list.php @@ -49,7 +49,7 @@ function getFullName() { From d101465c62eb18d49b952e61607fce0183f203ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 27 Aug 2019 18:53:10 +0200 Subject: [PATCH 31/34] Pass through errors to user interface Signed-off-by: DL6ER --- scripts/pi-hole/php/database.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/pi-hole/php/database.php b/scripts/pi-hole/php/database.php index b2a7bf82..bbdbec72 100644 --- a/scripts/pi-hole/php/database.php +++ b/scripts/pi-hole/php/database.php @@ -105,7 +105,7 @@ function add_to_table($db, $table, $domains, $wildcardstyle=false, $returnnum=fa if($returnnum) return $num; else - return "Error, added: ".$num; + return "Error: ".$db->lastErrorMsg().", added: ".$num; } } @@ -134,11 +134,10 @@ function remove_from_table($db, $table, $domains, $returnnum=false) // Return early if we failed to prepare the SQLite statement if(!$stmt) { - echo "Failed to prepare statement for ".$table." table."; if($returnnum) return 0; else - return "Error, added: 0"; + return "Error: Failed to prepare statement for ".$table." table."; } // Loop over domains and remove the lines from the database @@ -155,7 +154,7 @@ function remove_from_table($db, $table, $domains, $returnnum=false) if($returnnum) return $num; else - return "Error, removed: ".$num; + return "Error: ".$db->lastErrorMsg().", removed: ".$num; } } From fcc49e392870b270cd258e9ebd1cc4f238c51a81 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 27 Aug 2019 19:26:51 +0200 Subject: [PATCH 32/34] Wrap addition and removal of domains in a transaction + deliver a more detailed success message to the user specifying how many domains are actually added (in case some already existed). Signed-off-by: DL6ER --- list.php | 2 +- scripts/pi-hole/js/list.js | 7 ++++ scripts/pi-hole/php/database.php | 58 +++++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/list.php b/list.php index 21db22cc..7e9d5f62 100644 --- a/list.php +++ b/list.php @@ -49,7 +49,7 @@ function getFullName() {