mirror of
https://github.com/pi-hole/web.git
synced 2024-12-06 19:36:21 +01:00
Long-term statistics implementation (using FTL's database) (#521)
* - Add DB Top Lists - Add DB Query Log - Add DB API * Allow to select only queries that are equal/newer than the oldest entry in the database
This commit is contained in:
DL6ER
GitHub
@@ -0,0 +1,190 @@
|
||||
/* Pi-hole: A black hole for Internet advertisements
|
||||
* (c) 2017 Pi-hole, LLC (https://pi-hole.net)
|
||||
* Network-wide ad blocking via your own hardware.
|
||||
*
|
||||
* This file is copyright under the latest version of the EUPL.
|
||||
* Please see LICENSE file for your rights under this license. */
|
||||
|
||||
/*global
|
||||
moment
|
||||
*/
|
||||
|
||||
var start__ = moment().subtract(6, "days");
|
||||
var from = moment(start__).utc().valueOf()/1000;
|
||||
var end__ = moment();
|
||||
var until = moment(end__).utc().valueOf()/1000;
|
||||
|
||||
$(function () {
|
||||
// Get first time stamp we have valid data for to limit selectable date/time range
|
||||
$.getJSON("api_db.php?getMinTimestamp", function(data) {
|
||||
var minDate = parseInt(data.mintimestamp);
|
||||
if(!isNaN(minDate))
|
||||
{
|
||||
$("#querytime").data("daterangepicker").minDate = moment.unix(minDate);
|
||||
}
|
||||
});
|
||||
|
||||
$("#querytime").daterangepicker(
|
||||
{
|
||||
timePicker: true, timePickerIncrement: 15,
|
||||
locale: { format: "MMMM Do YYYY, h:mm A" },
|
||||
ranges: {
|
||||
"Today": [moment(), moment()],
|
||||
"Yesterday": [moment().subtract(1, "days"), moment().subtract(1, "days")],
|
||||
"Last 7 Days": [moment().subtract(6, "days"), moment()],
|
||||
"Last 30 Days": [moment().subtract(29, "days"), moment()],
|
||||
"This Month": [moment().startOf("month"), moment().endOf("month")],
|
||||
"Last Month": [moment().subtract(1, "month").startOf("month"), moment().subtract(1, "month").endOf("month")]
|
||||
},
|
||||
startDate: start__, endDate: end__,
|
||||
"opens": "center", "showDropdowns": true
|
||||
},
|
||||
function (startt, endt) {
|
||||
from = moment(startt).utc().valueOf()/1000;
|
||||
until = moment(endt).utc().valueOf()/1000;
|
||||
});
|
||||
});
|
||||
|
||||
// Credit: http://stackoverflow.com/questions/1787322/htmlspecialchars-equivalent-in-javascript/4835406#4835406
|
||||
function escapeHtml(text) {
|
||||
var map = {
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
"\"": """,
|
||||
"\'": "'"
|
||||
};
|
||||
|
||||
return text.replace(/[&<>"']/g, function(m) { return map[m]; });
|
||||
}
|
||||
|
||||
function updateTopClientsChart() {
|
||||
$("#client-frequency .overlay").show();
|
||||
$.getJSON("api_db.php?topClients&from="+from+"&until="+until, function(data) {
|
||||
|
||||
// Clear tables before filling them with data
|
||||
$("#client-frequency td").parent().remove();
|
||||
var clienttable = $("#client-frequency").find("tbody:last");
|
||||
var client, percentage, clientname, clientip;
|
||||
var sum = 0;
|
||||
for (client in data.top_sources) {
|
||||
if ({}.hasOwnProperty.call(data.top_sources, client)){
|
||||
sum += data.top_sources[client];
|
||||
}
|
||||
}
|
||||
|
||||
for (client in data.top_sources) {
|
||||
|
||||
if ({}.hasOwnProperty.call(data.top_sources, client)){
|
||||
// Sanitize client
|
||||
client = escapeHtml(client);
|
||||
if(escapeHtml(client) !== client)
|
||||
{
|
||||
// Make a copy with the escaped index if necessary
|
||||
data.top_sources[escapeHtml(client)] = data.top_sources[client];
|
||||
}
|
||||
if(client.indexOf("|") > -1)
|
||||
{
|
||||
var idx = client.indexOf("|");
|
||||
clientname = client.substr(0, idx);
|
||||
clientip = client.substr(idx+1, client.length-idx);
|
||||
}
|
||||
else
|
||||
{
|
||||
clientname = client;
|
||||
clientip = client;
|
||||
}
|
||||
|
||||
var url = clientname;
|
||||
percentage = data.top_sources[client] / sum * 100.0;
|
||||
clienttable.append("<tr> <td>" + url +
|
||||
"</td> <td>" + data.top_sources[client] + "</td> <td> <div class=\"progress progress-sm\" title=\""+percentage.toFixed(1)+"% of " + data.dns_queries_today + "\"> <div class=\"progress-bar progress-bar-blue\" style=\"width: " +
|
||||
percentage + "%\"></div> </div> </td> </tr> ");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
$("#client-frequency .overlay").hide();
|
||||
});
|
||||
}
|
||||
|
||||
function updateTopDomainsChart() {
|
||||
$("#domain-frequency .overlay").show();
|
||||
$.getJSON("api_db.php?topDomains&from="+from+"&until="+until, function(data) {
|
||||
|
||||
// Clear tables before filling them with data
|
||||
$("#domain-frequency td").parent().remove();
|
||||
var domaintable = $("#domain-frequency").find("tbody:last");
|
||||
var domain, percentage;
|
||||
var sum = 0;
|
||||
for (domain in data.top_domains) {
|
||||
if ({}.hasOwnProperty.call(data.top_domains, domain)){
|
||||
sum += data.top_domains[domain];
|
||||
}
|
||||
}
|
||||
|
||||
for (domain in data.top_domains) {
|
||||
|
||||
if ({}.hasOwnProperty.call(data.top_domains, domain)){
|
||||
// Sanitize domain
|
||||
domain = escapeHtml(domain);
|
||||
if(escapeHtml(domain) !== domain)
|
||||
{
|
||||
// Make a copy with the escaped index if necessary
|
||||
data.top_domains[escapeHtml(domain)] = data.top_domains[domain];
|
||||
}
|
||||
|
||||
percentage = data.top_domains[domain] / sum * 100.0;
|
||||
domaintable.append("<tr> <td>" + domain +
|
||||
"</td> <td>" + data.top_domains[domain] + "</td> <td> <div class=\"progress progress-sm\" title=\""+percentage.toFixed(1)+"% of " + data.dns_queries_today + "\"> <div class=\"progress-bar progress-bar-blue\" style=\"width: " +
|
||||
percentage + "%\"></div> </div> </td> </tr> ");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
$("#domain-frequency .overlay").hide();
|
||||
});
|
||||
}
|
||||
|
||||
function updateTopAdsChart() {
|
||||
$("#ad-frequency .overlay").show();
|
||||
$.getJSON("api_db.php?topAds&from="+from+"&until="+until, function(data) {
|
||||
|
||||
// Clear tables before filling them with data
|
||||
$("#ad-frequency td").parent().remove();
|
||||
var adtable = $("#ad-frequency").find("tbody:last");
|
||||
var ad, percentage;
|
||||
var sum = 0;
|
||||
for (ad in data.top_ads) {
|
||||
if ({}.hasOwnProperty.call(data.top_ads, ad)){
|
||||
sum += data.top_ads[ad];
|
||||
}
|
||||
}
|
||||
|
||||
for (ad in data.top_ads) {
|
||||
|
||||
if ({}.hasOwnProperty.call(data.top_ads, ad)){
|
||||
// Sanitize ad
|
||||
ad = escapeHtml(ad);
|
||||
if(escapeHtml(ad) !== ad)
|
||||
{
|
||||
// Make a copy with the escaped index if necessary
|
||||
data.top_ads[escapeHtml(ad)] = data.top_ads[ad];
|
||||
}
|
||||
|
||||
percentage = data.top_ads[ad] / sum * 100.0;
|
||||
adtable.append("<tr> <td>" + ad + "</td> <td>" + data.top_ads[ad] + "</td> <td> <div class=\"progress progress-sm\" title=\""+percentage.toFixed(1)+"% of " + data.dns_queries_today + "\"> <div class=\"progress-bar progress-bar-blue\" style=\"width: " + percentage + "%\"></div> </div> </td> </tr> ");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
$("#ad-frequency .overlay").hide();
|
||||
});
|
||||
}
|
||||
|
||||
// Handle "Go" Button
|
||||
$("#btnGo").on("click", function() {
|
||||
updateTopClientsChart();
|
||||
updateTopDomainsChart();
|
||||
updateTopAdsChart();
|
||||
});
|
||||
@@ -0,0 +1,252 @@
|
||||
/* Pi-hole: A black hole for Internet advertisements
|
||||
* (c) 2017 Pi-hole, LLC (https://pi-hole.net)
|
||||
* Network-wide ad blocking via your own hardware.
|
||||
*
|
||||
* This file is copyright under the latest version of the EUPL.
|
||||
* Please see LICENSE file for your rights under this license. */
|
||||
|
||||
/*global
|
||||
moment
|
||||
*/
|
||||
|
||||
var start__ = moment().subtract(6, "days");
|
||||
var from = moment(start__).utc().valueOf()/1000;
|
||||
var end__ = moment();
|
||||
var until = moment(end__).utc().valueOf()/1000;
|
||||
|
||||
$(function () {
|
||||
// Get first time stamp we have valid data for to limit selectable date/time range
|
||||
$.getJSON("api_db.php?getMinTimestamp", function(data) {
|
||||
var minDate = parseInt(data.mintimestamp);
|
||||
if(!isNaN(minDate))
|
||||
{
|
||||
$("#querytime").data("daterangepicker").minDate = moment.unix(minDate);
|
||||
}
|
||||
});
|
||||
|
||||
$("#querytime").daterangepicker(
|
||||
{
|
||||
timePicker: true, timePickerIncrement: 15,
|
||||
locale: { format: "MMMM Do YYYY, h:mm A" },
|
||||
ranges: {
|
||||
"Today": [moment(), moment()],
|
||||
"Yesterday": [moment().subtract(1, "days"), moment().subtract(1, "days")],
|
||||
"Last 7 Days": [moment().subtract(6, "days"), moment()],
|
||||
"Last 30 Days": [moment().subtract(29, "days"), moment()],
|
||||
"This Month": [moment().startOf("month"), moment().endOf("month")],
|
||||
"Last Month": [moment().subtract(1, "month").startOf("month"), moment().subtract(1, "month").endOf("month")]
|
||||
},
|
||||
startDate: start__, endDate: end__,
|
||||
"opens": "center", "showDropdowns": true
|
||||
},
|
||||
function (startt, endt) {
|
||||
from = moment(startt).utc().valueOf()/1000;
|
||||
until = moment(endt).utc().valueOf()/1000;
|
||||
});
|
||||
});
|
||||
|
||||
var tableApi, statistics;
|
||||
|
||||
function escapeRegex(text) {
|
||||
var map = {
|
||||
"(": "\\(",
|
||||
")": "\\)",
|
||||
".": "\\.",
|
||||
};
|
||||
return text.replace(/[().]/g, function(m) { return map[m]; });
|
||||
}
|
||||
|
||||
function add(domain,list) {
|
||||
var token = $("#token").html();
|
||||
var alInfo = $("#alInfo");
|
||||
var alList = $("#alList");
|
||||
var alDomain = $("#alDomain");
|
||||
alDomain.html(domain);
|
||||
var alSuccess = $("#alSuccess");
|
||||
var alFailure = $("#alFailure");
|
||||
var err = $("#err");
|
||||
|
||||
if(list === "white")
|
||||
{
|
||||
alList.html("Whitelist");
|
||||
}
|
||||
else
|
||||
{
|
||||
alList.html("Blacklist");
|
||||
}
|
||||
|
||||
alInfo.show();
|
||||
alSuccess.hide();
|
||||
alFailure.hide();
|
||||
$.ajax({
|
||||
url: "scripts/pi-hole/php/add.php",
|
||||
method: "post",
|
||||
data: {"domain":domain, "list":list, "token":token},
|
||||
success: function(response) {
|
||||
if (response.indexOf("not a valid argument") >= 0 || response.indexOf("is not a valid domain") >= 0)
|
||||
{
|
||||
alFailure.show();
|
||||
err.html(response);
|
||||
alFailure.delay(4000).fadeOut(2000, function() { alFailure.hide(); });
|
||||
}
|
||||
else
|
||||
{
|
||||
alSuccess.show();
|
||||
alSuccess.delay(1000).fadeOut(2000, function() { alSuccess.hide(); });
|
||||
}
|
||||
alInfo.delay(1000).fadeOut(2000, function() {
|
||||
alInfo.hide();
|
||||
alList.html("");
|
||||
alDomain.html("");
|
||||
});
|
||||
},
|
||||
error: function(jqXHR, exception) {
|
||||
alFailure.show();
|
||||
err.html("");
|
||||
alFailure.delay(1000).fadeOut(2000, function() {
|
||||
alFailure.hide();
|
||||
});
|
||||
alInfo.delay(1000).fadeOut(2000, function() {
|
||||
alInfo.hide();
|
||||
alList.html("");
|
||||
alDomain.html("");
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
function handleAjaxError( xhr, textStatus, error ) {
|
||||
if ( textStatus === "timeout" )
|
||||
{
|
||||
alert( "The server took too long to send the data." );
|
||||
}
|
||||
else if(xhr.responseText.indexOf("Connection refused") >= 0)
|
||||
{
|
||||
alert( "An error occured while loading the data: Connection refused. Is FTL running?" );
|
||||
}
|
||||
else
|
||||
{
|
||||
alert( "An unknown error occured while loading the data.\n"+xhr.responseText );
|
||||
}
|
||||
$("#all-queries_processing").hide();
|
||||
tableApi.clear();
|
||||
tableApi.draw();
|
||||
}
|
||||
|
||||
var reloadCallback = function()
|
||||
{
|
||||
statistics = [0,0,0,0];
|
||||
var data = tableApi.rows().data();
|
||||
for (var i = 0; i < data.length; i++) {
|
||||
statistics[0]++;
|
||||
if(data[i][4] === 1)
|
||||
{
|
||||
statistics[2]++;
|
||||
}
|
||||
else if(data[i][4] === 3)
|
||||
{
|
||||
statistics[1]++;
|
||||
}
|
||||
else if(data[i][4] === 4)
|
||||
{
|
||||
statistics[3]++;
|
||||
}
|
||||
}
|
||||
$("h3#dns_queries").text(statistics[0]);
|
||||
$("h3#ads_blocked_exact").text(statistics[2]);
|
||||
$("h3#ads_wildcard_blocked").text(statistics[3]);
|
||||
|
||||
var percent = 0.0;
|
||||
if(statistics[2] + statistics[3] > 0)
|
||||
{
|
||||
percent = 100.0*(statistics[2] + statistics[3]) / statistics[0];
|
||||
}
|
||||
$("h3#ads_percentage_today").text(parseFloat(percent).toFixed(1)+" %");
|
||||
};
|
||||
|
||||
function refreshTableData() {
|
||||
var APIstring = "api_db.php?getAllQueries&from="+from+"&until="+until;
|
||||
statistics = [0,0,0];
|
||||
tableApi.ajax.url(APIstring).load(reloadCallback);
|
||||
}
|
||||
|
||||
$(document).ready(function() {
|
||||
var status;
|
||||
|
||||
tableApi = $("#all-queries").DataTable( {
|
||||
"rowCallback": function( row, data, index ){
|
||||
if (data[4] === 1)
|
||||
{
|
||||
$(row).css("color","red");
|
||||
$("td:eq(4)", row).html( "Pi-holed" );
|
||||
$("td:eq(5)", row).html( "<button style=\"color:green; white-space: nowrap;\"><i class=\"fa fa-pencil-square-o\"></i> Whitelist</button>" );
|
||||
// statistics[2]++;
|
||||
}
|
||||
else if (data[4] === 2)
|
||||
{
|
||||
$(row).css("color","green");
|
||||
$("td:eq(4)", row).html( "OK (forwarded)" );
|
||||
$("td:eq(5)", row).html( "<button style=\"color:red; white-space: nowrap;\"><i class=\"fa fa-ban\"></i> Blacklist</button>" );
|
||||
}
|
||||
else if (data[4] === 3)
|
||||
{
|
||||
$(row).css("color","green");
|
||||
$("td:eq(4)", row).html( "OK (cached)" );
|
||||
$("td:eq(5)", row).html( "<button style=\"color:red; white-space: nowrap;\"><i class=\"fa fa-ban\"></i> Blacklist</button>" );
|
||||
// statistics[1]++;
|
||||
|
||||
}
|
||||
else if (data[4] === 4)
|
||||
{
|
||||
$(row).css("color","red");
|
||||
$("td:eq(4)", row).html( "Pi-holed (wildcard)" );
|
||||
$("td:eq(5)", row).html( "" );
|
||||
// statistics[3]++;
|
||||
}
|
||||
else
|
||||
{
|
||||
$("td:eq(4)", row).html( "Unknown ("+data[4]+")" );
|
||||
$("td:eq(5)", row).html( "" );
|
||||
}
|
||||
// statistics[0]++;
|
||||
},
|
||||
dom: "<'row'<'col-sm-12'f>>" +
|
||||
"<'row'<'col-sm-4'l><'col-sm-8'p>>" +
|
||||
"<'row'<'col-sm-12'tr>>" +
|
||||
"<'row'<'col-sm-5'i><'col-sm-7'p>>",
|
||||
"ajax": {"url": "api_db.php?getAllQueries=empty", "error": handleAjaxError },
|
||||
"autoWidth" : false,
|
||||
"processing": true,
|
||||
"deferRender": true,
|
||||
"order" : [[0, "desc"]],
|
||||
"columns": [
|
||||
{ "width" : "20%", "render": function (data, type, full, meta) { if(type === "display"){return moment.unix(data).format("Y-MM-DD HH:mm:ss z");}else{return data;} }},
|
||||
{ "width" : "10%" },
|
||||
{ "width" : "40%" },
|
||||
{ "width" : "10%" },
|
||||
{ "width" : "10%" },
|
||||
{ "width" : "10%" },
|
||||
],
|
||||
"lengthMenu": [[10, 25, 50, 100, -1], [10, 25, 50, 100, "All"]],
|
||||
"columnDefs": [ {
|
||||
"targets": -1,
|
||||
"data": null,
|
||||
"defaultContent": ""
|
||||
} ]
|
||||
});
|
||||
$("#all-queries tbody").on( "click", "button", function () {
|
||||
var data = tableApi.row( $(this).parents("tr") ).data();
|
||||
if (data[4] === "1")
|
||||
{
|
||||
add(data[2],"white");
|
||||
}
|
||||
else
|
||||
{
|
||||
add(data[2],"black");
|
||||
}
|
||||
} );
|
||||
} );
|
||||
|
||||
// Handle "Go" Button
|
||||
$("#btnGo").on("click", function() {
|
||||
refreshTableData();
|
||||
});
|
||||
@@ -184,6 +184,7 @@
|
||||
<link href="style/vendor/font-awesome-4.5.0/css/font-awesome.min.css" rel="stylesheet" type="text/css" />
|
||||
<link href="style/vendor/ionicons-2.0.1/css/ionicons.min.css" rel="stylesheet" type="text/css" />
|
||||
<link href="style/vendor/dataTables.bootstrap.min.css" rel="stylesheet" type="text/css" />
|
||||
<link href="style/vendor/daterangepicker.css" rel="stylesheet" type="text/css" />
|
||||
|
||||
<link href="style/vendor/AdminLTE.min.css" rel="stylesheet" type="text/css" />
|
||||
<link href="style/vendor/skin-blue.min.css" rel="stylesheet" type="text/css" />
|
||||
@@ -416,6 +417,26 @@ if($auth) {
|
||||
<i class="fa fa-file-text-o"></i> <span>Query Log</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="treeview<?php if($scriptname === "db_queries.php" || $scriptname === "db_lists.php"){ ?> active<?php } ?>">
|
||||
<a href="#">
|
||||
<i class="fa fa-clock-o"></i> <span>Long term data</span>
|
||||
<span class="pull-right-container">
|
||||
<i class="fa fa-angle-down pull-right" style="padding-right: 5px;"></i>
|
||||
</span>
|
||||
</a>
|
||||
<ul class="treeview-menu">
|
||||
<li<?php if($scriptname === "db_queries.php"){ ?> class="active"<?php } ?>>
|
||||
<a href="db_queries.php">
|
||||
<i class="fa fa-file-text-o"></i> <span>Query Log</span>
|
||||
</a>
|
||||
</li>
|
||||
<li<?php if($scriptname === "db_lists.php"){ ?> class="active"<?php } ?>>
|
||||
<a href="db_lists.php">
|
||||
<i class="fa fa-file-text-o"></i> <span>Top Lists</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<!-- Whitelist -->
|
||||
<li<?php if($scriptname === "whitelist"){ ?> class="active"<?php } ?>>
|
||||
<a href="list.php?l=white">
|
||||
|
||||
Vendored
+1542
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user