From d168dbf2165ed54d2725c94d073fd1b5fab0725c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 25 Dec 2016 19:56:50 +0100 Subject: [PATCH 1/6] New routines for Top Items generation (use same criteria as Query Log) --- data.php | 95 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/data.php b/data.php index 2e7ec397..9e04f34b 100644 --- a/data.php +++ b/data.php @@ -66,16 +66,50 @@ ); } - function getTopItems() { - global $log; - $dns_queries = getDnsQueries($log); - $ads_blocked = getBlockedQueries($log); + function getTopItems($qty=10) { + global $log,$setupVars; - $topAds = topItems($ads_blocked); - $topQueries = topItems($dns_queries, $topAds); + // Process log file + $dns_domains = getDnsQueryDomains($log); + // Get list of ad domains + $gravity_domains = getGravity(); + + // Exclude domains the user doesn't want to see + if(isset($setupVars["API_EXCLUDE_DOMAINS"])) + { + excludeFromList($dns_domains, "API_EXCLUDE_DOMAINS"); + } + + // Sort array in descending order + arsort($dns_domains); + + // Prepare arrays for Top Items + $topDomains = []; $domaincounter = 0; + $topAds = []; $adcounter = 0; + + foreach ($dns_domains as $key => $value) { + if(isset($gravity_domains[$key]) && $adcounter < $qty) + { + // New entry for Top Ads + $topAds[$key] = $value; + $adcounter++; + } + else if($domaincounter < $qty) + { + // New entry for Top Domains + $topDomains[$key] = $value; + $domaincounter++; + } + else + { + // Already collected enough entries for both lists + // Exit loop early + break; + } + } return Array( - 'top_queries' => $topQueries, + 'top_queries' => $topDomains, 'top_ads' => $topAds, ); } @@ -186,7 +220,7 @@ global $setupVars; if(isset($setupVars["API_EXCLUDE_CLIENTS"])) { - $sources = excludeFromList($sources, "API_EXCLUDE_CLIENTS"); + excludeFromList($sources, "API_EXCLUDE_CLIENTS"); } arsort($sources); @@ -339,6 +373,24 @@ return $lines; } + function getDnsQueryDomains(\SplFileObject $log) { + $log->rewind(); + $domains = []; + foreach ($log as $line) { + if(strpos($line, ": query[A") !== false) { + $exploded = explode(" ", $line); + $domain = trim($exploded[count($exploded) - 3]); + if (isset($domains[$domain])) { + $domains[$domain]++; + } + else { + $domains[$domain] = 1; + } + } + } + return $domains; + } + function countDnsQueries() { global $logListName; return exec("grep -c \": query\\[A\" $logListName"); @@ -451,32 +503,7 @@ return $lines; } - function topItems($queries, $exclude = array(), $qty=10) { - $splitQueries = array(); - foreach ($queries as $query) { - $exploded = explode(" ", $query); - $domain = trim($exploded[count($exploded) - 3]); - if (!isset($exclude[$domain])) { - if (isset($splitQueries[$domain])) { - $splitQueries[$domain]++; - } - else { - $splitQueries[$domain] = 1; - } - } - } - - global $setupVars; - if(isset($setupVars["API_EXCLUDE_DOMAINS"])) - { - $splitQueries = excludeFromList($splitQueries, "API_EXCLUDE_DOMAINS"); - } - - arsort($splitQueries); - return array_slice($splitQueries, 0, $qty); - } - - function excludeFromList($array,$key) + function excludeFromList(&$array,$key) { global $setupVars; $domains = explode(",",$setupVars[$key]); From dfa173754288d9226b3a7825a621380c40847f1f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 25 Dec 2016 20:00:24 +0100 Subject: [PATCH 2/6] Be ablw to specify how many Top Items a user wants to see through the API call --- api.php | 2 +- data.php | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/api.php b/api.php index 70dbc110..d29f381f 100644 --- a/api.php +++ b/api.php @@ -36,7 +36,7 @@ // Auth Required if (isset($_GET['topItems']) && $auth) { - $data = array_merge($data, getTopItems()); + $data = array_merge($data, getTopItems($_GET['topItems'])); } if (isset($_GET['recentItems']) && $auth) { diff --git a/data.php b/data.php index 9e04f34b..caae9a6f 100644 --- a/data.php +++ b/data.php @@ -66,7 +66,7 @@ ); } - function getTopItems($qty=10) { + function getTopItems($argument) { global $log,$setupVars; // Process log file @@ -87,6 +87,14 @@ $topDomains = []; $domaincounter = 0; $topAds = []; $adcounter = 0; + // Default number of Top Items to show is 10 + $qty = 10; + + if(is_numeric($argument)) + { + $qty = intval($argument); + } + foreach ($dns_domains as $key => $value) { if(isset($gravity_domains[$key]) && $adcounter < $qty) { From e83dc5636f8f4480fc9e3a4af9f815390642a227 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 25 Dec 2016 20:14:59 +0100 Subject: [PATCH 3/6] Improved comments --- data.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/data.php b/data.php index caae9a6f..4fba7ca5 100644 --- a/data.php +++ b/data.php @@ -83,18 +83,21 @@ // Sort array in descending order arsort($dns_domains); - // Prepare arrays for Top Items + // Prepare arrays and counters for Top Items $topDomains = []; $domaincounter = 0; $topAds = []; $adcounter = 0; // Default number of Top Items to show is 10 $qty = 10; + // If argument is numeric, the user may want to + // see a different number of entries if(is_numeric($argument)) { $qty = intval($argument); } + // Process sorted domain names foreach ($dns_domains as $key => $value) { if(isset($gravity_domains[$key]) && $adcounter < $qty) { From b38a6c809432282d9388be6fd2732e7cf305772f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 25 Dec 2016 20:30:10 +0100 Subject: [PATCH 4/6] New algorithm also for main graph (function getOverTimeData() and function getOverTimeData10mins()) --- data.php | 88 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/data.php b/data.php index 4fba7ca5..bebfc266 100644 --- a/data.php +++ b/data.php @@ -40,11 +40,17 @@ function getOverTimeData() { global $log; - $dns_queries = getDnsQueries($log); - $ads_blocked = getBlockedQueries($log); - $domains_over_time = overTime($dns_queries); - $ads_over_time = overTime($ads_blocked); + // Get log lines + $dns_queries = getDnsQueries($log); + + // Get list of ad domains + $gravity_domains = getGravity(); + + // Bin log entries separated into Domains and Ads in 1 hour intervals + list($domains_over_time, $ads_over_time) = overTime($dns_queries, $gravity_domains); + + // Align arrays alignTimeArrays($ads_over_time, $domains_over_time); return Array( 'domains_over_time' => $domains_over_time, @@ -54,11 +60,17 @@ function getOverTimeData10mins() { global $log; - $dns_queries = getDnsQueries($log); - $ads_blocked = getBlockedQueries($log); - $domains_over_time = overTime10mins($dns_queries); - $ads_over_time = overTime10mins($ads_blocked); + // Get log lines + $dns_queries = getDnsQueries($log); + + // Get list of ad domains + $gravity_domains = getGravity(); + + // Bin log entries separated into Domains and Ads in 10 minute intervals + list($domains_over_time, $ads_over_time) = overTime10mins($dns_queries, $gravity_domains); + + // Align arrays (in case there have been hours without ad queries) alignTimeArrays($ads_over_time, $domains_over_time); return Array( 'domains_over_time' => $domains_over_time, @@ -527,24 +539,41 @@ return $array; } - function overTime($entries) { - $byTime = array(); + function overTime($entries, $gravity_domains) { + $byTimeDomains = []; + $byTimeAds = []; foreach ($entries as $entry) { $time = date_create(substr($entry, 0, 16)); $hour = $time->format('G'); - if (isset($byTime[$hour])) { - $byTime[$hour]++; + $exploded = explode(" ", $entry); + $domain = trim($exploded[count($exploded) - 3]); + + if(isset($gravity_domains[$domain])) + { + if (isset($byTimeAds[$time])) { + $byTimeAds[$time]++; + } + else { + $byTimeAds[$time] = 1; + } } - else { - $byTime[$hour] = 1; + else + { + if (isset($byTimeDomains[$time])) { + $byTimeDomains[$time]++; + } + else { + $byTimeDomains[$time] = 1; + } } } - return $byTime; + return [$byTimeDomains,$byTimeAds]; } - function overTime10mins($entries) { - $byTime = array(); + function overTime10mins($entries, $gravity_domains) { + $byTimeDomains = []; + $byTimeAds = []; foreach ($entries as $entry) { $time = date_create(substr($entry, 0, 16)); $hour = $time->format('G'); @@ -559,14 +588,29 @@ // etc. $time = ($minute-$minute%10)/10 + 6*$hour; - if (isset($byTime[$time])) { - $byTime[$time]++; + $exploded = explode(" ", $entry); + $domain = trim($exploded[count($exploded) - 3]); + + if(isset($gravity_domains[$domain])) + { + if (isset($byTimeAds[$time])) { + $byTimeAds[$time]++; + } + else { + $byTimeAds[$time] = 1; + } } - else { - $byTime[$time] = 1; + else + { + if (isset($byTimeDomains[$time])) { + $byTimeDomains[$time]++; + } + else { + $byTimeDomains[$time] = 1; + } } } - return $byTime; + return [$byTimeDomains,$byTimeAds]; } function alignTimeArrays(&$times1, &$times2) { From 2d72ae75b2cc90a9b4746beb69f34b42780a2812 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Jan 2017 11:23:57 +0100 Subject: [PATCH 5/6] Some fixes to some warnings and errors --- scripts/pi-hole/php/data.php | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/scripts/pi-hole/php/data.php b/scripts/pi-hole/php/data.php index 541e756e..1c311a88 100644 --- a/scripts/pi-hole/php/data.php +++ b/scripts/pi-hole/php/data.php @@ -60,17 +60,15 @@ list($domains_over_time, $ads_over_time) = overTime($dns_queries, $gravity_domains); // Align arrays - $domains_over_time = overTime($dns_queries); - $ads_over_time = overTime($ads_blocked); + alignTimeArrays($ads_over_time, $domains_over_time); // Provide a minimal valid array if there have are no blocked // queries at all. Otherwise the output of the API is inconsistent. - if(count($ads_blocked) == 0) + if(count($ads_over_time) == 0) { $ads_over_time = [1 => 0]; } - alignTimeArrays($ads_over_time, $domains_over_time); return Array( 'domains_over_time' => $domains_over_time, 'ads_over_time' => $ads_over_time, @@ -90,18 +88,15 @@ list($domains_over_time, $ads_over_time) = overTime10mins($dns_queries, $gravity_domains); // Align arrays (in case there have been hours without ad queries) - - $domains_over_time = overTime10mins($dns_queries); - $ads_over_time = overTime10mins($ads_blocked); + alignTimeArrays($ads_over_time, $domains_over_time); // Provide a minimal valid array if there have are no blocked // queries at all. Otherwise the output of the API is inconsistent. - if(count($ads_blocked) == 0) + if(count($ads_over_time) == 0) { $ads_over_time = [1 => 0]; } - alignTimeArrays($ads_over_time, $domains_over_time); return Array( 'domains_over_time' => $domains_over_time, 'ads_over_time' => $ads_over_time, @@ -613,7 +608,7 @@ return [$byTimeDomains,$byTimeAds]; } - function overTime10mins($entries, $gravity_domains) { + function overTime10mins($entries, $gravity_domains=[]) { $byTimeDomains = []; $byTimeAds = []; foreach ($entries as $entry) { From 33d6f6da8fc9e8e1cafe63bf285ed2b41e0aca28 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Jan 2017 11:29:30 +0100 Subject: [PATCH 6/6] Exit TopLists loop early, but not too early --- scripts/pi-hole/php/data.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pi-hole/php/data.php b/scripts/pi-hole/php/data.php index 8d55d9f1..73176ca0 100644 --- a/scripts/pi-hole/php/data.php +++ b/scripts/pi-hole/php/data.php @@ -148,7 +148,7 @@ $topDomains[$key] = $value; $domaincounter++; } - else + elseif($domaincounter >= $qty && $adcounter >= $qty) { // Already collected enough entries for both lists // Exit loop early