Merge branch 'devel' into mobile-query-log2

This commit is contained in:
Mark Drobnak
2018-01-12 11:03:48 -05:00
committed by GitHub
14 changed files with 222 additions and 64 deletions
+127 -42
View File
@@ -12,18 +12,35 @@ require("scripts/pi-hole/php/password.php");
require("scripts/pi-hole/php/auth.php");
check_cors();
// Set maximum execution time to 10 minutes
ini_set("max_execution_time","600");
$data = array();
// Get posible non-standard location of FTL's database
$FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf");
if(isset($FTLsettings["DBFILE"]))
{
$DBFILE = $FTLsettings["DBFILE"];
}
else
{
$DBFILE = "/etc/pihole/pihole-FTL.db";
}
// Needs package php5-sqlite, e.g.
// sudo apt-get install php5-sqlite
function SQLite3_connect($trytoreconnect)
{
try {
global $DBFILE;
try
{
// connect to database
return new SQLite3('/etc/pihole/pihole-FTL.db', SQLITE3_OPEN_READONLY);
return new SQLite3($DBFILE, SQLITE3_OPEN_READONLY);
}
catch (Exception $exception) {
catch (Exception $exception)
{
// sqlite3 throws an exception when it is unable to connect, try to reconnect after 3 seconds
if($trytoreconnect)
{
@@ -33,8 +50,14 @@ function SQLite3_connect($trytoreconnect)
}
}
$db = SQLite3_connect(true);
if(strlen($DBFILE) > 0)
{
$db = SQLite3_connect(true);
}
else
{
die("No database available");
}
if(!$db)
{
die("Error connecting to database");
@@ -42,20 +65,17 @@ if(!$db)
if (isset($_GET['getAllQueries']) && $auth)
{
if($_GET['getAllQueries'] === "empty")
{
$allQueries = array();
}
else
$allQueries = array();
if($_GET['getAllQueries'] !== "empty")
{
$from = intval($_GET["from"]);
$until = intval($_GET["until"]);
$results = $db->query('SELECT timestamp,type,domain,client,status FROM queries WHERE timestamp >= '.$from.' AND timestamp <= '.$until.' ORDER BY timestamp ASC');
$allQueries = array();
while ($row = $results->fetchArray())
{
$allQueries[] = [$row[0],$row[1] == 1 ? "IPv4" : "IPv6",$row[2],$row[3],$row[4]];
}
if(!is_bool($results))
while ($row = $results->fetchArray())
{
$allQueries[] = [$row[0],$row[1] == 1 ? "IPv4" : "IPv6",$row[2],$row[3],$row[4]];
}
}
$result = array('data' => $allQueries);
$data = array_merge($data, $result);
@@ -77,13 +97,33 @@ if (isset($_GET['topClients']) && $auth)
{
$limit = "WHERE timestamp <= ".$_GET["until"];
}
$results = $db->query('SELECT client,count(client) FROM queries '.$limit.' GROUP by client order by count(client) desc limit 10');
$results = $db->query('SELECT client,count(client) FROM queries '.$limit.' GROUP by client order by count(client) desc limit 20');
$clients = array();
while ($row = $results->fetchArray())
{
$clients[$row[0]] = intval($row[1]);
// var_dump($row);
}
if(!is_bool($results))
while ($row = $results->fetchArray())
{
// Convert client to lower case
$c = strtolower($row[0]);
if(array_key_exists($c, $clients))
{
// Entry already exists, add to it (might appear multiple times due to mixed capitalization in the database)
$clients[$c] += intval($row[1]);
}
else
{
// Entry does not yet exist
$clients[$c] = intval($row[1]);
}
}
// Sort by number of hits
arsort($clients);
// Extract only the first ten entries
$clients = array_slice($clients, 0, 10);
$result = array('top_sources' => $clients);
$data = array_merge($data, $result);
}
@@ -104,12 +144,33 @@ if (isset($_GET['topDomains']) && $auth)
{
$limit = " AND timestamp <= ".$_GET["until"];
}
$results = $db->query('SELECT domain,count(domain) FROM queries WHERE (STATUS == 2 OR STATUS == 3)'.$limit.' GROUP by domain order by count(domain) desc limit 10');
$results = $db->query('SELECT domain,count(domain) FROM queries WHERE (STATUS == 2 OR STATUS == 3)'.$limit.' GROUP by domain order by count(domain) desc limit 20');
$domains = array();
while ($row = $results->fetchArray())
{
$domains[$row[0]] = intval($row[1]);
}
if(!is_bool($results))
while ($row = $results->fetchArray())
{
// Convert client to lower case
$c = strtolower($row[0]);
if(array_key_exists($c, $domains))
{
// Entry already exists, add to it (might appear multiple times due to mixed capitalization in the database)
$domains[$c] += intval($row[1]);
}
else
{
// Entry does not yet exist
$domains[$c] = intval($row[1]);
}
}
// Sort by number of hits
arsort($domains);
// Extract only the first ten entries
$domains = array_slice($domains, 0, 10);
$result = array('top_domains' => $domains);
$data = array_merge($data, $result);
}
@@ -131,11 +192,14 @@ if (isset($_GET['topAds']) && $auth)
$limit = " AND timestamp <= ".$_GET["until"];
}
$results = $db->query('SELECT domain,count(domain) FROM queries WHERE (STATUS == 1 OR STATUS == 4)'.$limit.' GROUP by domain order by count(domain) desc limit 10');
$addomains = array();
while ($row = $results->fetchArray())
{
$addomains[$row[0]] = intval($row[1]);
}
if(!is_bool($results))
while ($row = $results->fetchArray())
{
$addomains[$row[0]] = intval($row[1]);
}
$result = array('top_ads' => $addomains);
$data = array_merge($data, $result);
}
@@ -143,21 +207,36 @@ if (isset($_GET['topAds']) && $auth)
if (isset($_GET['getMinTimestamp']) && $auth)
{
$results = $db->query('SELECT MIN(timestamp) FROM queries');
$result = array('mintimestamp' => $results->fetchArray()[0]);
if(!is_bool($results))
$result = array('mintimestamp' => $results->fetchArray()[0]);
else
$result = array();
$data = array_merge($data, $result);
}
if (isset($_GET['getMaxTimestamp']) && $auth)
{
$results = $db->query('SELECT MAX(timestamp) FROM queries');
$result = array('maxtimestamp' => $results->fetchArray()[0]);
if(!is_bool($results))
$result = array('maxtimestamp' => $results->fetchArray()[0]);
else
$result = array();
$data = array_merge($data, $result);
}
if (isset($_GET['getQueriesCount']) && $auth)
{
$results = $db->query('SELECT COUNT(timestamp) FROM queries');
$result = array('count' => $results->fetchArray()[0]);
if(!is_bool($results))
$result = array('count' => $results->fetchArray()[0]);
else
$result = array();
$data = array_merge($data, $result);
}
@@ -195,22 +274,28 @@ if (isset($_GET['getGraphData']) && $auth)
}
// Count permitted queries in intervals
$results = $db->query('SELECT (timestamp/'.$interval.')*'.$interval.' interval, COUNT(*) FROM queries WHERE (status == 2 OR status == 3)'.$limit.' GROUP by interval ORDER by interval');
$results = $db->query('SELECT (timestamp/'.$interval.')*'.$interval.' interval, COUNT(*) FROM queries WHERE (status != 0 )'.$limit.' GROUP by interval ORDER by interval');
$domains = array();
while ($row = $results->fetchArray())
{
$domains[$row[0]] = intval($row[1]);
}
if(!is_bool($results))
while ($row = $results->fetchArray())
{
$domains[$row[0]] = intval($row[1]);
}
$result = array('domains_over_time' => $domains);
$data = array_merge($data, $result);
// Count blocked queries in intervals
$results = $db->query('SELECT (timestamp/'.$interval.')*'.$interval.' interval, COUNT(*) FROM queries WHERE (status == 1 OR status == 4 OR status == 5)'.$limit.' GROUP by interval ORDER by interval');
$addomains = array();
while ($row = $results->fetchArray())
{
$addomains[$row[0]] = intval($row[1]);
}
if(!is_bool($results))
while ($row = $results->fetchArray())
{
$addomains[$row[0]] = intval($row[1]);
}
$result = array('ads_over_time' => $addomains);
$data = array_merge($data, $result);
}
+5 -1
View File
@@ -22,7 +22,6 @@ $token = $_SESSION['token'];
<h1>Compute graphical statistics from the Pi-hole query database</h1>
</div>
<div class="row">
<div class="col-md-12">
<!-- Date Input -->
@@ -39,6 +38,11 @@ $token = $_SESSION['token'];
</div>
</div>
</div>
<div id="timeoutWarning" class="alert alert-warning alert-dismissible fade in" role="alert" hidden="true">
Depending on how large of a range you specified, the request may time out while Pi-hole tries to retrieve all the data.<br/><span id="err"></span>
</div>
<div class="row">
<div class="col-md-12">
<div class="box" id="queries-over-time">
+5
View File
@@ -39,6 +39,11 @@ $token = $_SESSION['token'];
</div>
</div>
</div>
<div id="timeoutWarning" class="alert alert-warning alert-dismissible fade in" role="alert" hidden="true">
Depending on how large of a range you specified, the request may time out while Pi-hole tries to retrieve all the data.<br/><span id="err"></span>
</div>
<?php
if($boxedlayout)
{
+5
View File
@@ -39,6 +39,11 @@ $token = $_SESSION['token'];
</div>
</div>
</div>
<div id="timeoutWarning" class="alert alert-warning alert-dismissible fade in" role="alert" hidden="true">
Depending on how large of a range you specified, the request may time out while Pi-hole tries to retrieve all the data.<br/><span id="err"></span>
</div>
<!-- Small boxes (Stat box) -->
<div class="row">
<div class="col-lg-3 col-xs-12">
+2
View File
@@ -118,6 +118,7 @@ if(strlen($showing) > 0)
<th>Domain</th>
<th>Client</th>
<th>Status</th>
<th>DNSSEC</th>
<th>Action</th>
</tr>
</thead>
@@ -128,6 +129,7 @@ if(strlen($showing) > 0)
<th>Domain</th>
<th>Client</th>
<th>Status</th>
<th>DNSSEC</th>
<th>Action</th>
</tr>
</tfoot>
+4
View File
@@ -14,6 +14,8 @@ var from = moment(start__).utc().valueOf()/1000;
var end__ = moment();
var until = moment(end__).utc().valueOf()/1000;
var timeoutWarning = $("#timeoutWarning");
$(function () {
$("#querytime").daterangepicker(
{
@@ -80,6 +82,7 @@ function compareNumbers(a, b) {
function updateQueriesOverTime() {
$("#queries-over-time .overlay").show();
timeoutWarning.show();
$.getJSON("api_db.php?getGraphData&from="+from+"&until="+until, function(data) {
// convert received objects to arrays
@@ -135,6 +138,7 @@ function updateQueriesOverTime() {
timeLineChart.options.scales.xAxes[0].display=true;
$("#queries-over-time .overlay").hide();
timeoutWarning.hide();
timeLineChart.update();
});
}
+17
View File
@@ -14,6 +14,9 @@ var from = moment(start__).utc().valueOf()/1000;
var end__ = moment();
var until = moment(end__).utc().valueOf()/1000;
var timeoutWarning = $("#timeoutWarning");
var listsStillLoading = 0;
$(function () {
$("#querytime").daterangepicker(
{
@@ -97,6 +100,10 @@ function updateTopClientsChart() {
}
$("#client-frequency .overlay").hide();
listsStillLoading--;
if(listsStillLoading === 0)
timeoutWarning.hide();
});
}
@@ -135,6 +142,10 @@ function updateTopDomainsChart() {
}
$("#domain-frequency .overlay").hide();
listsStillLoading--;
if(listsStillLoading === 0)
timeoutWarning.hide();
});
}
@@ -171,10 +182,16 @@ function updateTopAdsChart() {
}
$("#ad-frequency .overlay").hide();
listsStillLoading--;
if(listsStillLoading === 0)
timeoutWarning.hide();
});
}
$("#querytime").on("apply.daterangepicker", function(ev, picker) {
timeoutWarning.show();
listsStillLoading = 3;
updateTopClientsChart();
updateTopDomainsChart();
updateTopAdsChart();
+4
View File
@@ -16,6 +16,8 @@ var until = moment(end__).utc().valueOf()/1000;
var instantquery = false;
var daterange;
var timeoutWarning = $("#timeoutWarning");
// Do we want to filter queries?
var GETDict = {};
location.search.substr(1).split("&").forEach(function(item) {GETDict[item.split("=")[0]] = item.split("=")[1];});
@@ -141,6 +143,7 @@ function handleAjaxError( xhr, textStatus, error ) {
var reloadCallback = function()
{
timeoutWarning.hide();
statistics = [0,0,0,0];
var data = tableApi.rows().data();
for (var i = 0; i < data.length; i++) {
@@ -171,6 +174,7 @@ var reloadCallback = function()
};
function refreshTableData() {
timeoutWarning.show();
var APIstring = "api_db.php?getAllQueries&from="+from+"&until="+until;
statistics = [0,0,0];
tableApi.ajax.url(APIstring).load(reloadCallback);
+45 -15
View File
@@ -135,37 +135,66 @@ $(document).ready(function() {
{
$(row).css("color","red");
$("td:eq(4)", row).html( "Pi-holed" );
$("td:eq(5)", row).html( "<button style=\"color:green; white-space: nowrap;\"><i class=\"fa fa-pencil-square-o\"></i> Whitelist</button>" );
$("td:eq(6)", row).html( "<button style=\"color:green; white-space: nowrap;\"><i class=\"fa fa-pencil-square-o\"></i> Whitelist</button>" );
}
else if (data[4] === "2")
{
$(row).css("color","green");
$("td:eq(4)", row).html( "OK <br class='hidden-lg'>(forwarded)" );
$("td:eq(5)", row).html( "<button style=\"color:red; white-space: nowrap;\"><i class=\"fa fa-ban\"></i> Blacklist</button>" );
$("td:eq(6)", row).html( "<button style=\"color:red; white-space: nowrap;\"><i class=\"fa fa-ban\"></i> Blacklist</button>" );
}
else if (data[4] === "3")
{
$(row).css("color","green");
$("td:eq(4)", row).html( "OK <br class='hidden-lg'>(cached)" );
$("td:eq(5)", row).html( "<button style=\"color:red; white-space: nowrap;\"><i class=\"fa fa-ban\"></i> Blacklist</button>" );
$("td:eq(6)", row).html( "<button style=\"color:red; white-space: nowrap;\"><i class=\"fa fa-ban\"></i> Blacklist</button>" );
}
else if (data[4] === "4")
{
$(row).css("color","red");
$("td:eq(4)", row).html( "Pi-holed <br class='hidden-lg'>(wildcard)" );
$("td:eq(5)", row).html( "" );
$("td:eq(6)", row).html( "" );
}
else if (data[4] === "5")
{
$(row).css("color","red");
$("td:eq(4)", row).html( "Pi-holed <br class='hidden-lg'>(blacklist)" );
$("td:eq(5)", row).html( "<button style=\"color:green; white-space: nowrap;\"><i class=\"fa fa-pencil-square-o\"></i> Whitelist</button>" );
$("td:eq(6)", row).html( "<button style=\"color:green; white-space: nowrap;\"><i class=\"fa fa-pencil-square-o\"></i> Whitelist</button>" );
}
else
{
$("td:eq(4)", row).html( "Unknown" );
$("td:eq(5)", row).html( "" );
$("td:eq(6)", row).html( "" );
}
if (data[5] === "1")
{
$("td:eq(5)", row).css("color","green");
$("td:eq(5)", row).html( "SECURE" );
}
else if (data[5] === "2")
{
$("td:eq(5)", row).css("color","orange");
$("td:eq(5)", row).html( "INSECURE" );
}
else if (data[5] === "3")
{
$("td:eq(5)", row).css("color","red");
$("td:eq(5)", row).html( "BOGUS" );
}
else if (data[5] === "4")
{
$("td:eq(5)", row).css("color","red");
$("td:eq(5)", row).html( "ABANDONED" );
}
else if (data[5] === "5")
{
$("td:eq(5)", row).css("color","red");
$("td:eq(5)", row).html( "?" );
}
else
{
$("td:eq(5)", row).css("color","black");
$("td:eq(5)", row).html( "-" );
}
},
dom: "<'row'<'col-sm-12'f>>" +
@@ -179,10 +208,11 @@ $(document).ready(function() {
"columns": [
{ "width" : "15%", "render": function (data, type, full, meta) { if(type === "display"){return moment.unix(data).format("Y-MM-DD [<br class='hidden-lg'>]HH:mm:ss z");}else{return data;} }},
{ "width" : "10%" },
{ "width" : "40%", "render": $.fn.dataTable.render.text() },
{ "width" : "20%", "render": $.fn.dataTable.render.text() },
{ "width" : "37%", "render": $.fn.dataTable.render.text() },
{ "width" : "8%", "render": $.fn.dataTable.render.text() },
{ "width" : "10%" },
{ "width" : "5%" },
{ "width" : "10%" }
],
"lengthMenu": [[10, 25, 50, 100, -1], [10, 25, 50, 100, "All"]],
"columnDefs": [ {
@@ -195,22 +225,22 @@ $(document).ready(function() {
// Query type IPv4 / IPv6
api.$("td:eq(1)").click( function () { if(autofilter()){ api.search( this.innerHTML ).draw(); $("#resetButton").show(); }});
api.$("td:eq(1)").hover(
function () { this.title="Click to show only "+this.innerHTML+" queries"; this.style.color="#72afd2" },
function () { this.style.color="" }
function () { this.title="Click to show only "+this.innerHTML+" queries"; this.style.color="#72afd2"; },
function () { this.style.color=""; }
);
api.$("td:eq(1)").css("cursor","pointer");
// Domain
api.$("td:eq(2)").click( function () { if(autofilter()){ api.search( this.innerHTML ).draw(); $("#resetButton").show(); }});
api.$("td:eq(2)").hover(
function () { this.title="Click to show only queries with domain "+this.innerHTML; this.style.color="#72afd2" },
function () { this.style.color="" }
function () { this.title="Click to show only queries with domain "+this.innerHTML; this.style.color="#72afd2"; },
function () { this.style.color=""; }
);
api.$("td:eq(2)").css("cursor","pointer");
// Client
api.$("td:eq(3)").click( function () { if(autofilter()){ api.search( this.innerHTML ).draw(); $("#resetButton").show(); }});
api.$("td:eq(3)").hover(
function () { this.title="Click to show only queries made by "+this.innerHTML; this.style.color="#72afd2" },
function () { this.style.color="" }
function () { this.title="Click to show only queries made by "+this.innerHTML; this.style.color="#72afd2"; },
function () { this.style.color=""; }
);
api.$("td:eq(3)").css("cursor","pointer");
}
+1 -1
View File
@@ -119,7 +119,7 @@ function check_csrf($token) {
function check_domain() {
if(isset($_POST['domain'])){
$domains = preg_split('\s+', $_POST['domain']);
$domains = preg_split('/\s+/', $_POST['domain']);
foreach($domains as $domain)
{
$validDomain = is_valid_domain_name($domain);
+3 -3
View File
@@ -123,7 +123,7 @@ function isinserverlist($addr) {
"Norton" => ["v4_1" => "199.85.126.10", "v4_2" => "199.85.127.10"],
"Comodo" => ["v4_1" => "8.26.56.26", "v4_2" => "8.20.247.20"],
"DNS.WATCH" => ["v4_1" => "84.200.69.80", "v4_2" => "84.200.70.40", "v6_1" => "2001:1608:10:25:0:0:1c04:b12f", "v6_2" => "2001:1608:10:25:0:0:9249:d69b"],
"Quad9" => ["v4_1" => "9.9.9.9", "v6_1" => "2620:fe::fe"]
"Quad9" => ["v4_1" => "9.9.9.9", "v4_2" => "149.112.112.112", "v6_1" => "2620:fe::fe"]
];
$adlist = [];
@@ -320,9 +320,9 @@ function readAdlists()
$first = true;
foreach($clients as $client)
{
if(!validDomainWildcard($client))
if(!validDomainWildcard($client) && !validIP($client))
{
$error .= "Top Clients entry ".htmlspecialchars($client)." is invalid (use only IP addresses)!<br>";
$error .= "Top Clients entry ".htmlspecialchars($client)." is invalid (use only host names and IP addresses)!<br>";
}
if(!$first)
{
+1
View File
@@ -176,6 +176,7 @@ else
archive_add_file("/etc/pihole/","blacklist.txt");
archive_add_file("/etc/pihole/","adlists.list");
archive_add_file("/etc/pihole/","setupVars.conf");
archive_add_file("/etc/pihole/","auditlog.list");
archive_add_directory("/etc/dnsmasq.d/");
$archive["wildcardblocking.txt"] = getWildcardListContent();
+2 -2
View File
@@ -44,12 +44,12 @@ else
/********** Get Pi-hole FTL (not a git repository) **********/
$FTL_branch = $branches[2];
if($FTL_branch !== "master") {
if(substr($versions[2], 0, 4) === "vDev") {
$FTL_current = "vDev";
$FTL_commit = $versions[2];
}
else {
$FTL_current = explode("-",$versions[2])[0];
$FTL_current = $versions[2];
}
// Get data from GitHub
+1
View File
@@ -288,6 +288,7 @@ if (isset($_GET['tab']) && in_array($_GET['tab'], array("sysadmin", "blocklists"
</div>
<div class="box-footer clearfix">
<button type="submit" class="btn btn-primary" name="submit" value="save" id="blockinglistsave">Save</button>
<span><strong>Important: </strong>Save and Update when you're done!</span>
<button type="submit" class="btn btn-primary pull-right" name="submit" id="blockinglistsaveupdate" value="saveupdate">Save and Update</button>
</div>
</div>