Merge pull request #933 from pi-hole/new/gravityDB

Store domains in a database

api_db.php

@@ -8,6 +8,7 @@
 
 $api = true;
 header('Content-type: application/json');
+require("scripts/pi-hole/php/database.php");
 require("scripts/pi-hole/php/password.php");
 require("scripts/pi-hole/php/auth.php");
 check_cors();
@@ -48,7 +49,7 @@ function resolveHostname($clientip, $printIP)
 	return $clientname;
 }
 
-// Get posible non-standard location of FTL's database
+// Get possible non-standard location of FTL's database
 $FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf");
 if(isset($FTLsettings["DBFILE"]))
 {
@@ -62,37 +63,7 @@ else
 // Needs package php5-sqlite, e.g.
 //    sudo apt-get install php5-sqlite
 
-function SQLite3_connect($trytoreconnect)
-{
-	global $DBFILE;
-	try
-	{
-		// connect to database
-		return new SQLite3($DBFILE, SQLITE3_OPEN_READONLY);
-	}
-	catch (Exception $exception)
-	{
-		// sqlite3 throws an exception when it is unable to connect, try to reconnect after 3 seconds
-		if($trytoreconnect)
-		{
-			sleep(3);
-			$db = SQLite3_connect(false);
-		}
-	}
-}
-
-if(strlen($DBFILE) > 0)
-{
-	$db = SQLite3_connect(true);
-}
-else
-{
-	die("No database available");
-}
-if(!$db)
-{
-	die("Error connecting to database");
-}
+$db = SQLite3_connect($DBFILE);
 
 if(isset($_GET["network"]) && $auth)
 {

scripts/pi-hole/js/list.js

@@ -12,29 +12,24 @@ var token = $("#token").html();
 var listType = $("#list-type").html();
 var fullName = listType === "white" ? "Whitelist" : "Blacklist";
 
-function sub(index, entry, arg) {
-    var domain = $("#list #"+index);
-    var locallistType = listType;
-    if(arg === "regex")
-    {
-        locallistType = "regex";
-        domain = $("#list-regex #"+index);
-    }
-    domain.hide("highlight");
-    $.ajax({
-        url: "scripts/pi-hole/php/sub.php",
-        method: "post",
-        data: {"domain":entry, "list":locallistType, "token":token},
-        success: function(response) {
-            if(response.length !== 0){
-                return;
-            }
-            domain.remove();
-        },
-        error: function(jqXHR, exception) {
-            alert("Failed to remove the domain!");
-            domain.show({queue:true});
-        }
+function addListEntry(entry, index, list, button, type)
+{
+    var used = entry.enabled === "1" ? "used" : "not-used";
+    var comment = entry.comment.length > 0 ? " - " + entry.comment : "";
+    var date_added = new Date(parseInt(entry.date_added)*1000);
+    var date_modified = new Date(parseInt(entry.date_modified)*1000);
+    var tooltip = "Added: " + date_added.toLocaleString() +
+                  "\nModified: " + date_modified.toLocaleString();
+    list.append(
+        "<li id=\"" + index + "\" class=\"list-group-item " + used + " clearfix\">" +
+        "<span title=\"" + tooltip + "\" data-toggle=\"tooltip\" data-placement=\"right\">" +
+        entry.domain + comment + "</span>" +
+        "<button class=\"btn btn-danger btn-xs pull-right\" type=\"button\">" +
+        "<span class=\"glyphicon glyphicon-trash\"></span></button></li>"
+    );
+    // Handle button
+    $(button+" #"+index).on("click", "button", function() {
+        sub(index, entry.domain, type);
     });
 }
 
@@ -62,53 +57,39 @@ function refresh(fade) {
             {
                 listw.html("");
             }
-            var data = JSON.parse(response);
 
-            if(data.length === 0) {
+            if((listType === "black" &&
+               response.blacklist.length === 0 &&
+               response.regex.length === 0) ||
+               (listType === "white" &&
+               response.whitelist.length === 0))
+            {
                 $("h3").hide();
-                if(listw)
-                {
-                    listw.html("<div class=\"alert alert-info\" role=\"alert\">Your " + fullName + " is empty!</div>");
-                }
-                else
-                {
-                    list.html("<div class=\"alert alert-info\" role=\"alert\">Your " + fullName + " is empty!</div>");
-                }
+                list.html("<div class=\"alert alert-info\" role=\"alert\">Your " + fullName + " is empty!</div>");
             }
             else
             {
                 $("h3").show();
-                data[0] = data[0].sort();
-                data[0].forEach(function (entry, index) {
-                    // Whitelist entry or Blacklist (exact entry) are in the zero-th
-                    // array returned by get.php
-                    list.append(
-                    "<li id=\"" + index + "\" class=\"list-group-item clearfix\">" + entry +
-                    "<button class=\"btn btn-danger btn-xs pull-right\" type=\"button\">" +
-                    "<span class=\"glyphicon glyphicon-trash\"></span></button></li>");
-                    // Handle button
-                    $("#list #"+index+"").on("click", "button", function() {
-                        sub(index, entry, "exact");
-                    });
+                if(listType === "white")
+                {
+                    data = response.whitelist.sort();
+                    data2 = []; // No regex data, use empty array
+                }
+                else if(listType === "black")
+                {
+                    data = response.blacklist.sort();
+                    data2 = response.regex.sort();
+                }
+                data.forEach(function (entry, index)
+                {
+                    addListEntry(entry, index, list, "#list", "exact");
                 });
 
                 // Add regex domains if present in returned list data
-                if(data.length === 2)
+                data2.forEach(function (entry, index)
                 {
-                    data[1] = data[1].sort();
-                    data[1].forEach(function (entry, index) {
-                        // Whitelist entry or Blacklist (exact entry) are in the zero-th
-                        // array returned by get.php
-                        listw.append(
-                        "<li id=\"" + index + "\" class=\"list-group-item clearfix\">" + entry +
-                        "<button class=\"btn btn-danger btn-xs pull-right\" type=\"button\">" +
-                        "<span class=\"glyphicon glyphicon-trash\"></span></button></li>");
-                        // Handle button
-                        $("#list-regex #"+index+"").on("click", "button", function() {
-                            sub(index, entry, "regex");
-                        });
-                    });
-                }
+                    addListEntry(entry, index, listw, "#list-regex", "regex");
+                });
             }
             list.fadeIn(100);
             if(listw)
@@ -124,6 +105,32 @@ function refresh(fade) {
 
 window.onload = refresh(false);
 
+function sub(index, entry, arg) {
+    var domain = $("#list #"+index);
+    var locallistType = listType;
+    if(arg === "regex")
+    {
+        locallistType = "regex";
+        domain = $("#list-regex #"+index);
+    }
+    domain.hide("highlight");
+    $.ajax({
+        url: "scripts/pi-hole/php/sub.php",
+        method: "post",
+        data: {"domain":entry, "list":locallistType, "token":token},
+        success: function(response) {
+            if(response.length !== 0){
+                return;
+            }
+            domain.remove();
+        },
+        error: function(jqXHR, exception) {
+            alert("Failed to remove the domain!");
+            domain.show({queue:true});
+        }
+    });
+}
+
 function add(arg) {
     var locallistType = listType;
     var domain = $("#domain");
@@ -152,7 +159,7 @@ function add(arg) {
         method: "post",
         data: {"domain":domain.val().trim(), "list":locallistType, "token":token},
         success: function(response) {
-          if (!wild && response.indexOf(" already exists in ") !== -1) {
+          if (response.indexOf(" already exists in ") !== -1) {
             alWarning.show();
             warn.html(response);
             alWarning.delay(8000).fadeOut(2000, function() {
@@ -161,8 +168,7 @@ function add(arg) {
             alInfo.delay(8000).fadeOut(2000, function() {
                 alInfo.hide();
             });
-          } else if (!wild && response.indexOf("] Pi-hole blocking is ") === -1 ||
-               wild && response.length > 1) {
+          } else if (response.indexOf("DONE") === -1) {
             alFailure.show();
             err.html(response);
             alFailure.delay(8000).fadeOut(2000, function() {

scripts/pi-hole/php/add.php

@@ -22,36 +22,36 @@ if($type !== "regex") {
     check_domain();
 }
 
+// Escape shell metacharacters
+$domains = escapeshellcmd($_POST['domain']);
+
 switch($type) {
     case "white":
         if(!isset($_POST["auditlog"]))
-            echo shell_exec("sudo pihole -w ${_POST['domain']}");
+            echo shell_exec("sudo pihole -w --web ".$domains);
         else
         {
-            echo shell_exec("sudo pihole -w -n ${_POST['domain']}");
-            echo shell_exec("sudo pihole -a audit ${_POST['domain']}");
+            echo shell_exec("sudo pihole -w --web ".$domains);
+            echo shell_exec("sudo pihole -a audit ".$domains);
         }
         break;
     case "black":
         if(!isset($_POST["auditlog"]))
-            echo shell_exec("sudo pihole -b ${_POST['domain']}");
+            echo shell_exec("sudo pihole -b --web ".$domains);
         else
         {
-            echo shell_exec("sudo pihole -b -n ${_POST['domain']}");
-            echo shell_exec("sudo pihole -a audit ${_POST['domain']}");
+            echo shell_exec("sudo pihole -b --web ".$domains);
+            echo shell_exec("sudo pihole -a audit ".$domains);
         }
         break;
-    case "wild":
-        // Escape "." so it won't be interpreted as the wildcard character
-        $domain = str_replace(".","\.",$_POST['domain']);
-        // Add regex filter for legacy wildcard behavior
-        add_regex("(^|\.)".$domain."$");
-        break;
     case "regex":
-        add_regex($_POST['domain']);
+        echo shell_exec("sudo pihole --regex --web ".$domains);
+        break;
+    case "wild":
+        echo shell_exec("sudo pihole --wild --web ".$domains);
         break;
     case "audit":
-        echo exec("sudo pihole -a audit ${_POST['domain']}");
+        echo exec("sudo pihole -a audit ".$domain);
         break;
 }
 

scripts/pi-hole/php/database.php

@@ -0,0 +1,57 @@
+<?php
+/*   Pi-hole: A black hole for Internet advertisements
+*    (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*    Network-wide ad blocking via your own hardware.
+*
+*    This file is copyright under the latest version of the EUPL.
+*    Please see LICENSE file for your rights under this license */
+
+function getGravityDBFilename()
+{
+	// Get possible non-standard location of FTL's database
+	$FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf");
+	if(isset($FTLsettings["GRAVITYDB"]))
+	{
+		return $FTLsettings["GRAVITYDB"];
+	}
+	else
+	{
+		return "/etc/pihole/gravity.db";
+	}
+}
+
+function SQLite3_connect_try($filename, $mode, $trytoreconnect)
+{
+	try
+	{
+		// connect to database
+		return new SQLite3($filename, $mode);
+	}
+	catch (Exception $exception)
+	{
+		// sqlite3 throws an exception when it is unable to connect, try to reconnect after 3 seconds
+		if($trytoreconnect)
+		{
+			sleep(3);
+			$db = SQLite3_connect_try($filename, $mode, false);
+		}
+	}
+}
+
+function SQLite3_connect($filename, $mode=SQLITE3_OPEN_READONLY)
+{
+	if(strlen($filename) > 0)
+	{
+		$db = SQLite3_connect_try($filename, $mode, true);
+	}
+	else
+	{
+		die("No database available");
+	}
+	if(!$db)
+	{
+		die("Error connecting to database");
+	}
+	return $db;
+}
+?>

scripts/pi-hole/php/get.php

@@ -11,57 +11,60 @@ if(!isset($_GET['list']))
 
 $listtype = $_GET['list'];
 
-$basedir = "/etc/pihole/";
+require_once("func.php");
 
-require_once "func.php";
+require("database.php");
+$GRAVITYDB = getGravityDBFilename();
+$db = SQLite3_connect($GRAVITYDB);
 
-switch ($listtype) {
-    case "white":
-        $list = array(getListContent("whitelist.txt"));
-        break;
+function getTableContent($listname) {
+	global $db;
+	$entries = array();
+	$results = $db->query("SELECT * FROM $listname");
 
-    case "black":
-        $exact = getListContent("blacklist.txt");
-        $regex = getListContent("regex.list");
-        $list = array($exact, $regex);
-        break;
-
-    default:
-        die("Invalid list parameter");
-        break;
-}
-
-
-function getListContent($listname) {
-    global $basedir;
-    $rawList = file_get_contents(checkfile($basedir.$listname));
-    $list = explode("\n", $rawList);
-
-    // Get rid of empty lines and comments
-    for($i = sizeof($list)-1; $i >= 0; $i--) {
-        if(strlen($list[$i]) < 1 || $list[$i][0] === '#')
-            unset($list[$i]);
-    }
-
-    // Re-index list after possible unset() activity
-    $newlist = array_values($list);
-
-    return $newlist;
+	while($results !== false && $res = $results->fetchArray(SQLITE3_ASSOC))
+	{
+		array_push($entries, $res);
+	}
 
+	return array($listname => $entries);
 }
 
 function filterArray(&$inArray) {
-    $outArray = array();
-    foreach ($inArray as $key=>$value) {
-        if (is_array($value)) {
-            $outArray[htmlspecialchars($key)] = filterArray($value);
-        } else {
-            $outArray[htmlspecialchars($key)] = htmlspecialchars($value);
-        }
-    }
-    return $outArray;
+	$outArray = array();
+	foreach ($inArray as $key => $value)
+	{
+		if (is_array($value))
+		{
+			$outArray[htmlspecialchars($key)] = filterArray($value);
+		}
+		else
+		{
+			$outArray[htmlspecialchars($key)] = htmlspecialchars($value);
+		}
+	}
+	return $outArray;
 }
 
+switch ($listtype)
+{
+	case "white":
+		$list = getTableContent("whitelist");
+		break;
+
+	case "black":
+		$exact = getTableContent("blacklist");
+		$regex = getTableContent("regex");
+		$list = array_merge($exact, $regex);
+		break;
+
+	default:
+		die("Invalid list parameter");
+		break;
+}
 // Protect against XSS attacks
-$list = filterArray($list);
-echo json_encode(array_values($list));
+$output = filterArray($list);
+
+// Return results
+header('Content-type: application/json');
+echo json_encode($output);

scripts/pi-hole/php/savesettings.php

@@ -155,37 +155,22 @@ function readDNSserversList()
 	return $list;
 }
 
+require_once("database.php");
 $adlist = [];
 function readAdlists()
 {
 	// Reset list
 	$list = [];
-	$handle = @fopen("/etc/pihole/adlists.list", "r");
-	if ($handle)
+	$db = SQLite3_connect(getGravityDBFilename());
+	if ($db)
 	{
-		while (($line = fgets($handle)) !== false)
+		$results = $db->query("SELECT * FROM adlists");
+
+		while($results !== false && $res = $results->fetchArray(SQLITE3_ASSOC))
 		{
-			if(strlen($line) < 3)
-			{
-				continue;
-			}
-			elseif($line[0] === "#")
-			{
-				// Comments start either with "##" or "# "
-				if($line[1] !== "#" &&
-				   $line[1] !== " ")
-				{
-					// Commented list
-					array_push($list, [false,rtrim(substr($line, 1))]);
-				}
-			}
-			else
-			{
-				// Active list
-				array_push($list, [true,rtrim($line)]);
-			}
+			array_push($list, $res);
 		}
-		fclose($handle);
+		$db->close();
 	}
 	return $list;
 }
@@ -689,18 +674,18 @@ function readAdlists()
 					if(isset($_POST["adlist-del-".$key]))
 					{
 						// Delete list
-						exec("sudo pihole -a adlist del ".escapeshellcmd($value[1]));
+						exec("sudo pihole -a adlist del ".escapeshellcmd($value["address"]));
 					}
-					elseif(isset($_POST["adlist-enable-".$key]) && !$value[0])
+					elseif(isset($_POST["adlist-enable-".$key]) && $value["enabled"] !== 1)
 					{
 						// Is not enabled, but should be
-						exec("sudo pihole -a adlist enable ".escapeshellcmd($value[1]));
+						exec("sudo pihole -a adlist enable ".escapeshellcmd($value["address"]));
 
 					}
-					elseif(!isset($_POST["adlist-enable-".$key]) && $value[0])
+					elseif(!isset($_POST["adlist-enable-".$key]) && $value["enabled"] === 1)
 					{
 						// Is enabled, but shouldn't be
-						exec("sudo pihole -a adlist disable ".escapeshellcmd($value[1]));
+						exec("sudo pihole -a adlist disable ".escapeshellcmd($value["address"]));
 					}
 				}
 

scripts/pi-hole/php/sub.php

@@ -16,43 +16,24 @@ if (empty($api)) {
     list_verify($type);
 }
 
-// Don't check if the added item is a valid domain for regex expressions. Regex
-// filters are validated by FTL on import and skipped if invalid
+// Don't check if the added item is a valid domain for regex expressions.
+// Regex filters are validated by FTL on import and skipped if invalid
 if($type !== "regex") {
     check_domain();
 }
 
+// Escape shell metacharacters
+$domain = escapeshellcmd($_POST['domain']);
+
 switch($type) {
     case "white":
-        exec("sudo pihole -w -q -d ${_POST['domain']}");
+        exec("sudo pihole -w -q -d ".$domain);
         break;
     case "black":
-        exec("sudo pihole -b -q -d ${_POST['domain']}");
+        exec("sudo pihole -b -q -d ".$domain);
         break;
     case "regex":
-        if(($list = file_get_contents($regexfile)) === FALSE)
-        {
-            $err = error_get_last()["message"];
-            echo "Unable to read ${regexfile}<br>Error message: $err";
-        }
-
-        // Remove the regex and any empty lines from the list
-        $list = explode("\n", $list);
-        $list = array_diff($list, array($_POST['domain'], ""));
-        $list = implode("\n", $list);
-
-        if(file_put_contents($regexfile, $list."\n") === FALSE)
-        {
-            $err = error_get_last()["message"];
-            echo "Unable to remove regex \"".htmlspecialchars($_POST['domain'])."\" from ${regexfile}<br>Error message: $err";
-        }
-        else
-        {
-            // Send SIGHUP to pihole-FTL using a frontend command
-            // to force reloading of the regex domains
-            // This will also wipe the resolver's cache
-            echo exec("sudo pihole restartdns reload");
-        }
+        exec("sudo pihole --regex -q -d ".$domain);
         break;
 }
 

settings.php

@@ -257,8 +257,11 @@ if (isset($_GET['tab']) && in_array($_GET['tab'], array("sysadmin", "blocklists"
                                             <table class="table table-striped table-bordered dt-responsive nowrap">
                                                 <thead>
                                                 <tr>
-                                                    <th>Enabled</th>
+                                                    <th style="width:1%">Enabled</th>
                                                     <th>List</th>
+                                                    <th>Added</th>
+                                                    <th>Last modified</th>
+                                                    <th>Comment</th>
                                                     <th style="width:1%">Delete</th>
                                                 </tr>
                                                 </thead>
@@ -266,10 +269,19 @@ if (isset($_GET['tab']) && in_array($_GET['tab'], array("sysadmin", "blocklists"
                                                     <?php foreach ($adlist as $key => $value) { ?>
                                                         <tr>
                                                             <td>
-                                                                <input type="checkbox" name="adlist-enable-<?php echo $key; ?>" <?php if ($value[0]){ ?>checked<?php } ?>>
+                                                                <input type="checkbox" name="adlist-enable-<?php echo $key; ?>" <?php if ($value["enabled"] === 1){ ?>checked<?php } ?>>
                                                             </td>
                                                             <td>
-                                                                <a href="<?php echo htmlentities($value[1]); ?>" target="_new" id="adlist-text-<?php echo $key; ?>"><?php echo htmlentities($value[1]); ?></a>
+                                                                <a href="<?php echo htmlentities($value["address"]); ?>" target="_new" id="adlist-text-<?php echo $key; ?>"><?php echo htmlentities($value["address"]); ?></a>
+                                                            </td>
+                                                            <td>
+                                                                <?php echo date(DateTime::RFC2822, intval($value["date_added"])); ?>
+                                                            </td>
+                                                            <td>
+                                                                <?php echo date(DateTime::RFC2822, intval($value["date_modified"])); ?>
+                                                            </td>
+                                                            <td>
+                                                                <?php echo htmlentities($value["comment"]); ?>
                                                             </td>
                                                             <td class="text-center">
                                                                 <button class="btn btn-danger btn-xs" id="adlist-btn-<?php echo $key; ?>">
@@ -283,7 +295,7 @@ if (isset($_GET['tab']) && in_array($_GET['tab'], array("sysadmin", "blocklists"
                                             </table>
                                         </div>
                                         <div class="form-group">
-                                            <textarea name="newuserlists" class="form-control" rows="1" placeholder="Enter one URL per line to add new blocklists"></textarea>
+                                            <input name="newuserlists" class="form-control" placeholder="Enter a URL to add a new blocklist">
                                         </div>
                                         <input type="hidden" name="field" value="adlists">
                                         <input type="hidden" name="token" value="<?php echo $token ?>">

style/pi-hole.css

@@ -24,6 +24,23 @@
 .skin-blue .list-group-item:hover {
 	background: #ddd;
 }
+
+.skin-blue .not-used {
+	background: #eee;
+}
+
+.skin-blue .not-used:hover {
+	background: #c5c5c95;
+}
+
+.skin-blue .used {
+	background: #fff;
+}
+
+.skin-blue .used:hover {
+	background: #ddd;
+}
+
 @-webkit-keyframes Pulse{
 	from {color:#630030;-webkit-text-shadow:0 0 2px transparent;}
 	50% {color:#e33100;-webkit-text-shadow:0 0 5px #e33100;}