Merge pull request #75 from pi-hole/new/API

Use MessagePack for the Unix socket API

FTL.h

@@ -72,6 +72,9 @@
 // Default -60 (one minute before a full hour)
 #define GCdelay (-60)
 
+// How many client connection do we accept at once?
+#define MAXCONNS 20
+
 // Static structs
 typedef struct {
 	const char* conf;
@@ -207,7 +210,6 @@ typedef struct {
 enum { DATABASE_WRITE_TIMER, EXIT_TIMER };
 
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
-enum { SOCKET };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
 
 // Used to check memory integrity in various structs
@@ -257,3 +259,4 @@ bool DBdeleteoldqueries;
 bool rereadgravity;
 long int lastDBimportedtimestamp;
 bool ipv4telnet, ipv6telnet;
+bool istelnet[MAXCONNS];

Makefile

@@ -8,8 +8,8 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-DEPS = FTL.h routines.h version.h
-OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o database.o
+DEPS = FTL.h routines.h api.h version.h
+OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o database.o api.o msgpack.o
 
 # Get git commit version and date
 GIT_BRANCH := $(shell git branch | sed -n 's/^\* //p')

api.h

@@ -0,0 +1,42 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API commands and MessagePack helpers
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+// Statistic methods
+void getStats(int *sock);
+void getOverTime(int *sock);
+void getTopDomains(char *client_message, int *sock);
+void getTopClients(char *client_message, int *sock);
+void getForwardDestinations(char *client_message, int *sock);
+void getQueryTypes(int *sock);
+void getAllQueries(char *client_message, int *sock);
+void getRecentBlocked(char *client_message, int *sock);
+void getForwardDestinationsOverTime(int *sock);
+void getQueryTypesOverTime(int *sock);
+void getClientsOverTime(int *sock);
+void getClientNames(int *sock);
+
+// FTL methods
+void getMemoryUsage(int *sock);
+void getClientID(int *sock);
+void getVersion(int *sock);
+void getDBstats(int *sock);
+void getUnknownQueries(int *sock);
+
+// MessagePack serialization helpers
+void pack_eom(int sock);
+void pack_bool(int sock, bool value);
+void pack_uint8(int sock, uint8_t value);
+void pack_uint64(int sock, uint64_t value);
+void pack_int32(int sock, int32_t value);
+void pack_int64(int sock, int64_t value);
+void pack_float(int sock, float value);
+bool pack_fixstr(int sock, char *string);
+bool pack_str32(int sock, char *string);
+void pack_map16_start(int sock, uint16_t length);

grep.c

@@ -109,8 +109,12 @@ void readWildcardsList()
 
 			return;
 		}
+
+		// Trim off the newline (could even be CR-LF)
+		linebuffer[strcspn(linebuffer, "\r\n")] = 0;
+
 		// Try to read up to 511 characters
-		if(sscanf(linebuffer, "address=/%511[^/]/%*[^\n]\n", buffer) > 0)
+		if(sscanf(linebuffer, "address=/%511[^/]/", buffer) > 0)
 		{
 			unsigned long int addrbuffer = 0;
 			// Skip leading '.' by incrementing memory location step by step until the first

main.c

@@ -67,6 +67,7 @@ int main (int argc, char* argv[]) {
 	// the system without the need for another thread to join with the terminated thread
 	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
+	// Start log analyzing thread
 	pthread_t piholelogthread;
 	if(pthread_create( &piholelogthread, &attr, pihole_log_thread, NULL ) != 0)
 	{
@@ -77,6 +78,7 @@ int main (int argc, char* argv[]) {
 	// Bind to sockets after initial log parsing
 	bind_sockets();
 
+	// Start TELNET IPv4 thread
 	pthread_t telnet_listenthreadv4;
 	if(ipv4telnet && pthread_create( &telnet_listenthreadv4, &attr, telnet_listening_thread_IPv4, NULL ) != 0)
 	{
@@ -84,6 +86,7 @@ int main (int argc, char* argv[]) {
 		killed = 1;
 	}
 
+	// Start TELNET IPv6 thread
 	pthread_t telnet_listenthreadv6;
 	if(ipv6telnet &&  pthread_create( &telnet_listenthreadv6, &attr, telnet_listening_thread_IPv6, NULL ) != 0)
 	{
@@ -91,6 +94,7 @@ int main (int argc, char* argv[]) {
 		killed = 1;
 	}
 
+	// Start SOCKET thread
 	pthread_t socket_listenthread;
 	if(pthread_create( &socket_listenthread, &attr, socket_listening_thread, NULL ) != 0)
 	{

msgpack.c

@@ -0,0 +1,118 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  MessagePack serialization
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "api.h"
+
+void pack_eom(int sock) {
+	// This byte is explicitly never used in the MessagePack spec, so it is perfect to use as an EOM for this API.
+	uint8_t eom = 0xc1;
+	swrite(sock, &eom, sizeof(eom));
+}
+
+void pack_basic(int sock, uint8_t format, void *value, size_t size) {
+	swrite(sock, &format, sizeof(format));
+	swrite(sock, value, size);
+}
+
+uint64_t leToBe64(uint64_t value) {
+	char *ptr = (char *) &value;
+	uint32_t part1, part2;
+
+	// Copy the two halves of the 64 bit input into uint32_t's so we can use htonl
+	memcpy(&part1, ptr, 4);
+	memcpy(&part2, ptr + 4, 4);
+
+	// Flip each half around
+	part1 = htonl(part1);
+	part2 = htonl(part2);
+
+	// Arrange them to form the big-endian version of the original input
+	return (uint64_t) part1 << 32 | part2;
+}
+
+void pack_bool(int sock, bool value) {
+	uint8_t packed = (uint8_t) (value ? 0xc3 : 0xc2);
+	swrite(sock, &packed, sizeof(packed));
+}
+
+void pack_uint8(int sock, uint8_t value) {
+	pack_basic(sock, 0xcc, &value, sizeof(value));
+}
+
+void pack_uint64(int sock, uint64_t value) {
+	uint64_t bigEValue = leToBe64(value);
+	pack_basic(sock, 0xcf, &bigEValue, sizeof(bigEValue));
+}
+
+void pack_int32(int sock, int32_t value) {
+	uint32_t bigEValue = htonl((uint32_t) value);
+	pack_basic(sock, 0xd2, &bigEValue, sizeof(bigEValue));
+}
+
+void pack_int64(int sock, int64_t value) {
+	// Need to use memcpy to do a direct copy without reinterpreting the bytes (making negatives into positives).
+	// It should get optimized away.
+	uint64_t bigEValue;
+	memcpy(&bigEValue, &value, sizeof(bigEValue));
+	bigEValue = leToBe64(bigEValue);
+	pack_basic(sock, 0xd3, &bigEValue, sizeof(bigEValue));
+}
+
+void pack_float(int sock, float value) {
+	// Need to use memcpy to do a direct copy without reinterpreting the bytes. It should get optimized away.
+	uint32_t bigEValue;
+	memcpy(&bigEValue, &value, sizeof(bigEValue));
+	bigEValue = htonl(bigEValue);
+	pack_basic(sock, 0xca, &bigEValue, sizeof(bigEValue));
+}
+
+// Return true if successful
+bool pack_fixstr(int sock, char *string) {
+	// Make sure that the length is less than 32
+	size_t length = strlen(string);
+
+	if(length >= 32) {
+		logg("Tried to send a fixstr longer than 31 bytes!");
+		return false;
+	}
+
+	uint8_t format = (uint8_t) (0xA0 | length);
+	swrite(sock, &format, sizeof(format));
+	swrite(sock, string, length);
+
+	return true;
+}
+
+// Return true if successful
+bool pack_str32(int sock, char *string) {
+	// Make sure that the length is less than 4294967296
+	size_t length = strlen(string);
+
+	if(length >= 2147483648) {
+		logg("Tried to send a str32 longer than 2147483647 bytes!");
+		return false;
+	}
+
+	uint8_t format = 0xdb;
+	swrite(sock, &format, sizeof(format));
+	uint32_t bigELength = htonl((uint32_t) length);
+	swrite(sock, &bigELength, sizeof(bigELength));
+	swrite(sock, string, length);
+
+	return true;
+}
+
+void pack_map16_start(int sock, uint16_t length) {
+	uint8_t format = 0xde;
+	swrite(sock, &format, sizeof(format));
+	uint16_t bigELength = htons(length);
+	swrite(sock, &bigELength, sizeof(bigELength));
+}

routines.h

@@ -44,11 +44,11 @@ void memory_check(int which);
 
 void close_telnet_socket(void);
 void close_unix_socket(void);
-void swrite(char server_message[], int sock);
-void *telnet_listening_thread_IPv4(void *args);
-void *telnet_listening_thread_IPv6(void *args);
 void seom(int sock);
 void ssend(int sock, const char *format, ...);
+void swrite(int sock, void *value, size_t size);
+void *telnet_listening_thread_IPv4(void *args);
+void *telnet_listening_thread_IPv6(void *args);
 
 void *socket_listening_thread(void *args);
 bool ipv6_available(void);
@@ -56,7 +56,7 @@ void bind_sockets(void);
 
 void process_request(char *client_message, int *sock);
 bool command(char *client_message, const char* cmd);
-void formatNumber(bool raw, int n, char* buffer);
+bool matchesEndpoint(char *client_message, const char *cmd);
 
 void read_gravity_files(void);
 int countlines(const char* fname);

socket.c

@@ -9,6 +9,7 @@
 *  Please see LICENSE file for your rights under this license. */
 
 #include "FTL.h"
+#include "api.h"
 
 // The backlog argument defines the maximum length
 // to which the queue of pending connections for
@@ -24,6 +25,7 @@
 int socketfd, telnetfd4 = 0, telnetfd6 = 0;
 bool dualstack = false;
 bool ipv4telnet = false, ipv6telnet = false;
+bool istelnet[MAXCONNS];
 
 void saveport(void)
 {
@@ -40,7 +42,7 @@ void saveport(void)
 	}
 }
 
-bool bind_to_telnet_port_IPv4(char type, int *socketdescriptor)
+bool bind_to_telnet_port_IPv4(int *socketdescriptor)
 {
 	// IPv4 socket
 	*socketdescriptor = socket(AF_INET, SOCK_STREAM, 0);
@@ -65,7 +67,7 @@ bool bind_to_telnet_port_IPv4(char type, int *socketdescriptor)
 	memset(&serv_addr4, 0, sizeof(serv_addr4));
 	serv_addr4.sin_family = AF_INET;
 
-	if(config.socket_listenlocal && type == SOCKET)
+	if(config.socket_listenlocal)
 		serv_addr4.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 	else
 		serv_addr4.sin_addr.s_addr = INADDR_ANY;
@@ -85,11 +87,11 @@ bool bind_to_telnet_port_IPv4(char type, int *socketdescriptor)
 		return false;
 	}
 
-	logg("Listening on port %i for incoming IPv4 connections", config.port);
+	logg("Listening on port %i for incoming IPv4 telnet connections", config.port);
 	return true;
 }
 
-bool bind_to_telnet_port_IPv6(char type, int *socketdescriptor)
+bool bind_to_telnet_port_IPv6(int *socketdescriptor)
 {
 	// IPv6 socket
 	*socketdescriptor = socket(AF_INET6, SOCK_STREAM, 0);
@@ -120,7 +122,7 @@ bool bind_to_telnet_port_IPv6(char type, int *socketdescriptor)
 	memset(&serv_addr, 0, sizeof(serv_addr));
 	serv_addr.sin6_family = AF_INET6;
 
-	if(config.socket_listenlocal && type == SOCKET)
+	if(config.socket_listenlocal)
 		serv_addr.sin6_addr = in6addr_loopback;
 	else
 		serv_addr.sin6_addr = in6addr_any;
@@ -147,7 +149,7 @@ bool bind_to_telnet_port_IPv6(char type, int *socketdescriptor)
 		return false;
 	}
 
-	logg("Listening on port %i for incoming IPv6 connections", config.port);
+	logg("Listening on port %i for incoming IPv6 telnet connections", config.port);
 	return true;
 }
 
@@ -200,7 +202,10 @@ void removeport(void)
 
 void seom(int sock)
 {
-	ssend(sock, "---EOM---\n\n");
+	if(istelnet[sock])
+		ssend(sock, "---EOM---\n\n");
+	else
+		pack_eom(sock);
 }
 
 void ssend(int sock, const char *format, ...)
@@ -218,33 +223,58 @@ void ssend(int sock, const char *format, ...)
 	}
 }
 
+void swrite(int sock, void *value, size_t size) {
+	if(write(sock, value, size) == -1)
+		logg("WARNING: Socket write returned error code %i", errno);
+}
+
+int checkClientLimit(int socket) {
+	if(socket < MAXCONNS)
+	{
+		if(debugclients)
+			logg("Client connected: %i", socket);
+		return socket;
+	}
+	else
+	{
+		if(debugclients)
+			logg("Client denied (at max capacity of %i): %i", MAXCONNS, socket);
+
+		close(socket);
+		return -1;
+	}
+}
+
 int listener(int sockfd, char type)
 {
 	struct sockaddr_un un_addr;
 	struct sockaddr_in in4_addr;
 	struct sockaddr_in6 in6_addr;
 	socklen_t socklen = 0;
+	int socket;
 
 	switch(type)
 	{
 		case 0: // Unix socket
-		memset(&un_addr, 0, sizeof(un_addr));
-		socklen = sizeof(un_addr);
-		return accept(sockfd, (struct sockaddr *) &un_addr, &socklen);
+			memset(&un_addr, 0, sizeof(un_addr));
+			socklen = sizeof(un_addr);
+			return accept(sockfd, (struct sockaddr *) &un_addr, &socklen);
 
 		case 4: // Internet socket (IPv4)
-		memset(&in4_addr, 0, sizeof(in4_addr));
-		socklen = sizeof(un_addr);
-		return accept(sockfd, (struct sockaddr *) &in4_addr, &socklen);
+			memset(&in4_addr, 0, sizeof(in4_addr));
+			socklen = sizeof(un_addr);
+			socket = accept(sockfd, (struct sockaddr *) &in4_addr, &socklen);
+			return checkClientLimit(socket);
 
 		case 6: // Internet socket (IPv6)
-		memset(&in6_addr, 0, sizeof(in6_addr));
-		socklen = sizeof(un_addr);
-		return accept(sockfd, (struct sockaddr *) &in6_addr, &socklen);
+			memset(&in6_addr, 0, sizeof(in6_addr));
+			socklen = sizeof(un_addr);
+			socket = accept(sockfd, (struct sockaddr *) &in6_addr, &socklen);
+			return checkClientLimit(socket);
 
 		default: // Should not happen
-		logg("Cannot listen on type %i connection, code error!", type);
-		exit(EXIT_FAILURE);
+			logg("Cannot listen on type %i connection, code error!", type);
+			exit(EXIT_FAILURE);
 	}
 }
 
@@ -270,6 +300,9 @@ void *telnet_connection_handler_thread(void *socket_desc)
 {
 	//Get the socket descriptor
 	int sock = *(int*)socket_desc;
+	// Set connection type to telnet
+	istelnet[sock] = true;
+
 	// Store copy only for displaying the debug messages
 	int sockID = sock;
 	char client_message[SOCKETBUFFERLEN] = "";
@@ -328,6 +361,8 @@ void *socket_connection_handler_thread(void *socket_desc)
 {
 	//Get the socket descriptor
 	int sock = *(int*)socket_desc;
+	// Set connection type to not telnet
+	istelnet[sock] = false;
 	// Store copy only for displaying the debug messages
 	int sockID = sock;
 	char client_message[SOCKETBUFFERLEN] = "";
@@ -383,13 +418,13 @@ void *socket_connection_handler_thread(void *socket_desc)
 void bind_sockets(void)
 {
 	// Initialize IPv4 telnet socket
-	if(bind_to_telnet_port_IPv4(SOCKET, &telnetfd4))
+	if(bind_to_telnet_port_IPv4(&telnetfd4))
 		ipv4telnet = true;
 
 	// Initialize IPv6 telnet socket
 	// only if IPv6 interfaces are available
 	if(ipv6_available())
-		if(bind_to_telnet_port_IPv6(SOCKET, &telnetfd6))
+		if(bind_to_telnet_port_IPv6(&telnetfd6))
 			ipv6telnet = true;
 
 	saveport();
@@ -496,6 +531,7 @@ void *socket_listening_thread(void *args)
 	{
 		// Look for new clients that want to connect
 		int csck = listener(socketfd, 0);
+		if(csck < 0) continue;
 
 		// Allocate memory used to transport client socket ID to client listening thread
 		int *newsock;

socket_client.c

@@ -16,13 +16,16 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <errno.h>
+
 #define BUF 1024
 
 int main (int argc, char **argv) {
 	int socketfd;
 	char *buffer = malloc (BUF);
 	struct sockaddr_un address;
-	int size, ret;
+	ssize_t size;
+	int ret;
 
 	// Create socket
 	socketfd = socket(PF_LOCAL, SOCK_STREAM, 0);
@@ -36,41 +39,50 @@ int main (int argc, char **argv) {
 	// Set socket family to local socket (not an Internet socket)
 	address.sun_family = AF_LOCAL;
 
-	// Set socket file location (respect special location on the CI system Travis)
-	if(argc == 2 && strcmp(argv[1], "travis") == 0)
-		strcpy(address.sun_path,"pihole-FTL.sock");
-	else
-		strcpy(address.sun_path,"/var/run/pihole/FTL.sock");
+	char *command = ">stats";
+	strcpy(address.sun_path,"/var/run/pihole/FTL.sock");
+
+	int i;
+	for(i = 1; i < argc; i++) {
+		// Get command
+		if(strstr(argv[i], ">") == argv[i]) {
+			command = argv[i];
+			continue;
+		}
+
+		// Set socket file location (respect special location on the CI system Travis)
+		if(strcmp(argv[i], "travis") == 0)
+			strcpy(address.sun_path,"pihole-FTL.sock");
+	}
 
 	// Connect to the socket provided by pihole-FTL
 	ret = connect(socketfd, (struct sockaddr *) &address, sizeof (address));
 	if (ret != 0)
 	{
-		printf("Error establishing connection!\n");
+		printf("Error establishing connection! %s\n", strerror(errno));
 		exit(EXIT_FAILURE);
 	}
 	printf("Connection established\n");
 
 	// As an example, we query the current statistics from FTL through the socket here
-	sprintf(buffer, ">stats");
+	sprintf(buffer, command);
 	send(socketfd, buffer, strlen (buffer), 0);
 
 	// Try to receive data until either recv() fails or we see "--EOM--"
 	while((size = recv(socketfd, buffer, BUF-1, 0)) > -1)
 	{
-		// Zero-terminate incoming message
-		if(size > 0)
-			buffer[size] = '\0';
-
 		// Print received data to stdout
-		printf("%s", buffer);
+		for(i = 0; i < size; ++i) {
+			printf("%02x ", (unsigned char) buffer[i]);
+		}
 
 		// Exit on End Of Message
-		if(strstr(buffer, "--EOM--") != NULL)
+		if((unsigned char) buffer[size-1] == 0xc1)
 			break;
-
 	}
 
+	printf("\n");
+
 	// Close Unix socket connection
 	close(socketfd);
 	return EXIT_SUCCESS;

test/run.sh

@@ -62,6 +62,9 @@ n=0
 until [ $n -ge 45 ]; do
   nc -vv -z -w 30 127.0.0.1 4711 && break
   n=$[$n+1]
+  echo "..."
+  tail -n2 pihole-FTL.log
+  echo "..."
   sleep 1
 done
 

test/test_suite.sh

@@ -235,17 +235,7 @@ load 'libs/bats-support/load'
   echo "output: ${lines[@]}"
   [[ ${lines[0]} == "Socket created" ]]
   [[ ${lines[1]} == "Connection established" ]]
-  [[ ${lines[2]} == "domains_being_blocked -1" ]]
-  [[ ${lines[3]} == "dns_queries_today 7" ]]
-  [[ ${lines[4]} == "ads_blocked_today 2" ]]
-  [[ ${lines[5]} == "ads_percentage_today 28.571428" ]]
-  [[ ${lines[6]} == "unique_domains 6" ]]
-  [[ ${lines[7]} == "queries_forwarded 3" ]]
-  [[ ${lines[8]} == "queries_cached 2" ]]
-  [[ ${lines[9]} == "clients_ever_seen 3" ]]
-  [[ ${lines[10]} == "unique_clients 3" ]]
-  [[ ${lines[11]} == "status unknown" ]]
-  [[ ${lines[12]} == "---EOM---" ]]
+  [[ ${lines[2]} == "d2 ff ff ff ff d2 00 00 00 07 d2 00 00 00 02 ca 41 e4 92 49 d2 00 00 00 06 d2 00 00 00 03 d2 00 00 00 02 d2 00 00 00 03 d2 00 00 00 03 cc 02 c1 " ]]
 }
 
 @test "Final part of the tests: Killing pihole-FTL process" {