Add TAR routines for efficient parsing of a tar archive in memory

Signed-off-by: DL6ER <dl6er@dl6er.de>

src/api/teleporter.c

@@ -15,6 +15,10 @@
 #include "api/api.h"
 // ERRBUF_SIZE
 #include "config/dnsmasq_config.h"
+// inflate_buffer()
+#include "zip/gzip.h"
+// find_file_in_tar()
+#include "zip/tar.h"
 
 #define MAXFILESIZE (50u*1024*1024)
 
@@ -58,7 +62,7 @@ static int api_teleporter_GET(struct ftl_conn *api)
 struct upload_data {
 	bool too_large;
 	char *sid;
-	char *data;
+	uint8_t *data;
 	char *filename;
 	size_t filesize;
 };
@@ -163,6 +167,7 @@ static int free_upload_data(struct upload_data *data)
 
 // Private function prototypes
 static int process_received_zip(struct ftl_conn *api, struct upload_data *data);
+static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data);
 
 static int api_teleporter_POST(struct ftl_conn *api)
 {
@@ -231,14 +236,31 @@ static int api_teleporter_POST(struct ftl_conn *api)
 	{
 		return process_received_zip(api, &data);
 	}
-	else
+	// Check if we received something that claims to be a TAR.GZ archive
+	// - filename
+	//   - shoud be at least 12 characters long,
+	//   - should start in "pi-hole-",
+	//   - have "-teleporter_" in the middle, and
+	//   - end in ".tar.gz"
+	// - the data itself
+	//   - should be at least 40 bytes long
+	//   - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt)
+	else if(strlen(data.filename) >= 12 &&
+	        strncmp(data.filename, "pi-hole-", 8) == 0 &&
+	        strstr(data.filename, "-teleporter_") != NULL &&
+	        strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 &&
+	        data.filesize >= 40 &&
+	        memcmp(data.data, "\x1f\x8b", 2) == 0)
 	{
-		free_upload_data(&data);
-		return send_json_error(api, 400,
-		                       "bad_request",
-		                       "Invalid file",
-		                       "The uploaded file does not appear to be a valid Pi-hole Teleporter archive");
+		return process_received_tar_gz(api, &data);
 	}
+
+	// else: invalid file
+	free_upload_data(&data);
+	return send_json_error(api, 400,
+	                       "bad_request",
+	                       "Invalid file",
+	                       "The uploaded file does not appear to be a valid Pi-hole Teleporter archive");
 }
 
 static int process_received_zip(struct ftl_conn *api, struct upload_data *data)
@@ -274,6 +296,42 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data)
 	JSON_SEND_OBJECT(json);
 }
 
+static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data)
+{
+	// Try to decompress the received data
+	uint8_t *archive = NULL;
+	mz_ulong archive_size = 0u;
+	if(!inflate_buffer(data->data, data->filesize, &archive, &archive_size))
+	{
+		free_upload_data(data);
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "Invalid GZIP archive",
+		                       "The uploaded file does not appear to be a valid gzip archive - decompression failed");
+	}
+
+	// Check if the decompressed data is a valid TAR archive
+	cJSON *json_files = list_files_in_tar(archive, archive_size);
+
+	// Print all files in the TAR archive
+	cJSON *file = NULL;
+	cJSON_ArrayForEach(file, json_files)
+	{
+		cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name");
+		cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size");
+		log_info("Found file in TAR archive: \"%s\" (%d bytes)",
+		          name->valuestring, size->valueint);
+	}
+
+	// Free allocated memory
+	free_upload_data(data);
+
+	// Send response
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files);
+	JSON_SEND_OBJECT(json);
+}
+
 int api_teleporter(struct ftl_conn *api)
 {
 	if(api->method == HTTP_GET)

src/zip/CMakeLists.txt

@@ -11,6 +11,8 @@
 set(sources
         gzip.c
         gzip.h
+        tar.c
+        tar.h
         teleporter.c
         teleporter.h
         )

src/zip/gzip.c

@@ -14,7 +14,6 @@
 #include <string.h>
 // le32toh and friends
 #include <endian.h>
-#include "miniz/miniz.h"
 #include "gzip.h"
 #include "log.h"
 
@@ -103,8 +102,8 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul
 	return true;
 }
 
-static bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
-                           unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed)
+bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
+                    unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed)
 {
 	// Check GZIP header (magic byte 1F 8B and compression algorithm deflate 08)
 	if(buffer_compressed[0] != 0x1F || buffer_compressed[1] != 0x8B)

src/zip/gzip.h

@@ -11,6 +11,10 @@
 #define GZIP_H
 
 #include <stdbool.h>
+#include "miniz/miniz.h"
+
+bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
+                    unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed);
 
 bool deflate_file(const char *in, const char *out, bool verbose);
 bool inflate_file(const char *infile, const char *outfile, bool verbose);

src/zip/tar.c

@@ -0,0 +1,129 @@
+/* Pi-hole: A black hole for Internet advertisements
+ *  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+ *  Network-wide ad blocking via your own hardware.
+ *
+ *  FTL Engine
+ *  In-memory tar reading routines
+ *
+ *  This file is copyright under the latest version of the EUPL.
+ *  Please see LICENSE file for your rights under this license. */
+
+#include "zip/tar.h"
+#include "log.h"
+
+// TAR offsets
+#define TAR_NAME_OFFSET 0
+#define TAR_SIZE_OFFSET 124
+#define TAR_MAGIC_OFFSET 257
+
+// TAR constants
+#define TAR_BLOCK_SIZE 512
+#define TAR_NAME_SIZE 100
+#define TAR_SIZE_SIZE 12
+#define TAR_MAGIC_SIZE 5
+
+static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */
+
+/**
+ * Find a file in a TAR archive
+ * @param tarData Pointer to the TAR archive in memory
+ * @param tarSize Size of the TAR archive in memory in bytes
+ * @param fileName Name of the file to find
+ * @param fileSize Pointer to a size_t variable to store the file size in
+ * @return Pointer to the file data or NULL if not found
+ */
+const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize,
+                                const char *fileName, size_t *fileSize)
+{
+	bool found = false;
+	size_t size, p = 0, newOffset = 0;
+
+	// Convert to char * to be able to do pointer arithmetic more easily
+	const char *tar = (const char *)tarData;
+
+	// Initialize fileSize to 0
+	*fileSize = 0;
+
+	// Loop through TAR file
+	do
+	{
+		// "Load" data from tar - just point to passed memory
+		const char *name = tar + TAR_NAME_OFFSET + p + newOffset;
+		const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str
+		p += newOffset; // pointer to current file's data in TAR
+
+		// Check for supported TAR version or end of TAR
+		for (size_t i = 0; i < TAR_MAGIC_SIZE; i++)
+			if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i])
+				return NULL;
+
+		// Convert file size from string into integer
+		size = 0;
+		for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int
+			if ((sz[i] >= '1') && (sz[i] <= '9'))
+				size += (sz[i] - '0') * mul;
+
+		//Offset size in bytes. Depends on file size and TAR block size
+		newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block
+		if ((size % TAR_BLOCK_SIZE) > 0)
+			newOffset += TAR_BLOCK_SIZE;
+
+		found = strncmp(name, fileName, TAR_NAME_SIZE) == 0;
+	} while (!found && (p + newOffset + TAR_BLOCK_SIZE <= tarSize));
+
+	if (!found)
+		return NULL; // No file found in TAR - return NULL
+
+	// File found in TAR - return pointer to file data and set fileSize
+	*fileSize = size;
+	return tarData + p + TAR_BLOCK_SIZE;
+}
+
+/**
+ * List all files in a TAR archive
+ * @param tarData Pointer to the TAR archive in memory
+ * @param tarSize Size of the TAR archive in memory in bytes
+ * @return Pointer to a cJSON array containing all file names with file size
+ */
+cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize)
+{
+	cJSON *files = cJSON_CreateArray();
+	size_t size, p = 0, newOffset = 0;
+
+	// Convert to char * to be able to do pointer arithmetic more easily
+	const char *tar = (const char *)tarData;
+
+	// Loop through TAR file
+	do
+	{
+		// "Load" data from tar - just point to passed memory
+		const char *name = tar + TAR_NAME_OFFSET + p + newOffset;
+		const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str
+		p += newOffset; // pointer to current file's data in TAR
+
+		// Check for supported TAR version or end of TAR
+		for (size_t i = 0; i < TAR_MAGIC_SIZE; i++)
+			if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i])
+				return files;
+
+		// Convert file size from string into integer
+		size = 0;
+		for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int
+			if ((sz[i] >= '1') && (sz[i] <= '9'))
+				size += (sz[i] - '0') * mul;
+
+		//Offset size in bytes. Depends on file size and TAR block size
+		newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block
+		if ((size % TAR_BLOCK_SIZE) > 0)
+			newOffset += TAR_BLOCK_SIZE;
+
+		// Add file name to cJSON array
+		log_info("Found file '%s' with size %zu", name, size);
+		cJSON *file = cJSON_CreateObject();
+		cJSON_AddItemToObject(file, "name", cJSON_CreateString(name));
+		cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size));
+		cJSON_AddItemToArray(files, file);
+	} while (p + newOffset + TAR_BLOCK_SIZE <= tarSize);
+
+	return files;
+}

src/zip/tar.h

@@ -0,0 +1,19 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  TAR reading routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#ifndef TAR_H
+#define TAR_H
+
+#include "FTL.h"
+#include "webserver/cJSON/cJSON.h"
+
+const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize);
+cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize);
+
+#endif // TAR_H

src/zip/teleporter.c

@@ -524,7 +524,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char *
 	return NULL;
 }
 
-const char *read_teleporter_zip(char *buffer, const size_t buflen, char * const hint, cJSON *imported_files)
+const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files)
 {
 	// Initialize ZIP archive
 	mz_zip_archive zip = { 0 };

src/zip/teleporter.h

@@ -15,7 +15,7 @@
 
 const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size);
 bool free_teleporter_zip(mz_zip_archive *zip);
-const char *read_teleporter_zip(char *buffer, const size_t buflen, char *hint, cJSON *json_files);
+const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files);
 
 bool write_teleporter_zip_to_disk(void);
 bool read_teleporter_zip_from_disk(const char *filename);