Add TAR routines for efficient parsing of a tar archive in memory

Signed-off-by: DL6ER <dl6er@dl6er.de>

src/zip/CMakeLists.txt

@@ -11,6 +11,8 @@
 set(sources
         gzip.c
         gzip.h
+        tar.c
+        tar.h
         teleporter.c
         teleporter.h
         )

src/zip/gzip.c

@@ -14,7 +14,6 @@
 #include <string.h>
 // le32toh and friends
 #include <endian.h>
-#include "miniz/miniz.h"
 #include "gzip.h"
 #include "log.h"
 
@@ -103,8 +102,8 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul
 	return true;
 }
 
-static bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
-                           unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed)
+bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
+                    unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed)
 {
 	// Check GZIP header (magic byte 1F 8B and compression algorithm deflate 08)
 	if(buffer_compressed[0] != 0x1F || buffer_compressed[1] != 0x8B)

src/zip/gzip.h

@@ -11,6 +11,10 @@
 #define GZIP_H
 
 #include <stdbool.h>
+#include "miniz/miniz.h"
+
+bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed,
+                    unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed);
 
 bool deflate_file(const char *in, const char *out, bool verbose);
 bool inflate_file(const char *infile, const char *outfile, bool verbose);

src/zip/tar.c

@@ -0,0 +1,129 @@
+/* Pi-hole: A black hole for Internet advertisements
+ *  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+ *  Network-wide ad blocking via your own hardware.
+ *
+ *  FTL Engine
+ *  In-memory tar reading routines
+ *
+ *  This file is copyright under the latest version of the EUPL.
+ *  Please see LICENSE file for your rights under this license. */
+
+#include "zip/tar.h"
+#include "log.h"
+
+// TAR offsets
+#define TAR_NAME_OFFSET 0
+#define TAR_SIZE_OFFSET 124
+#define TAR_MAGIC_OFFSET 257
+
+// TAR constants
+#define TAR_BLOCK_SIZE 512
+#define TAR_NAME_SIZE 100
+#define TAR_SIZE_SIZE 12
+#define TAR_MAGIC_SIZE 5
+
+static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */
+
+/**
+ * Find a file in a TAR archive
+ * @param tarData Pointer to the TAR archive in memory
+ * @param tarSize Size of the TAR archive in memory in bytes
+ * @param fileName Name of the file to find
+ * @param fileSize Pointer to a size_t variable to store the file size in
+ * @return Pointer to the file data or NULL if not found
+ */
+const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize,
+                                const char *fileName, size_t *fileSize)
+{
+	bool found = false;
+	size_t size, p = 0, newOffset = 0;
+
+	// Convert to char * to be able to do pointer arithmetic more easily
+	const char *tar = (const char *)tarData;
+
+	// Initialize fileSize to 0
+	*fileSize = 0;
+
+	// Loop through TAR file
+	do
+	{
+		// "Load" data from tar - just point to passed memory
+		const char *name = tar + TAR_NAME_OFFSET + p + newOffset;
+		const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str
+		p += newOffset; // pointer to current file's data in TAR
+
+		// Check for supported TAR version or end of TAR
+		for (size_t i = 0; i < TAR_MAGIC_SIZE; i++)
+			if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i])
+				return NULL;
+
+		// Convert file size from string into integer
+		size = 0;
+		for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int
+			if ((sz[i] >= '1') && (sz[i] <= '9'))
+				size += (sz[i] - '0') * mul;
+
+		//Offset size in bytes. Depends on file size and TAR block size
+		newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block
+		if ((size % TAR_BLOCK_SIZE) > 0)
+			newOffset += TAR_BLOCK_SIZE;
+
+		found = strncmp(name, fileName, TAR_NAME_SIZE) == 0;
+	} while (!found && (p + newOffset + TAR_BLOCK_SIZE <= tarSize));
+
+	if (!found)
+		return NULL; // No file found in TAR - return NULL
+
+	// File found in TAR - return pointer to file data and set fileSize
+	*fileSize = size;
+	return tarData + p + TAR_BLOCK_SIZE;
+}
+
+/**
+ * List all files in a TAR archive
+ * @param tarData Pointer to the TAR archive in memory
+ * @param tarSize Size of the TAR archive in memory in bytes
+ * @return Pointer to a cJSON array containing all file names with file size
+ */
+cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize)
+{
+	cJSON *files = cJSON_CreateArray();
+	size_t size, p = 0, newOffset = 0;
+
+	// Convert to char * to be able to do pointer arithmetic more easily
+	const char *tar = (const char *)tarData;
+
+	// Loop through TAR file
+	do
+	{
+		// "Load" data from tar - just point to passed memory
+		const char *name = tar + TAR_NAME_OFFSET + p + newOffset;
+		const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str
+		p += newOffset; // pointer to current file's data in TAR
+
+		// Check for supported TAR version or end of TAR
+		for (size_t i = 0; i < TAR_MAGIC_SIZE; i++)
+			if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i])
+				return files;
+
+		// Convert file size from string into integer
+		size = 0;
+		for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int
+			if ((sz[i] >= '1') && (sz[i] <= '9'))
+				size += (sz[i] - '0') * mul;
+
+		//Offset size in bytes. Depends on file size and TAR block size
+		newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block
+		if ((size % TAR_BLOCK_SIZE) > 0)
+			newOffset += TAR_BLOCK_SIZE;
+
+		// Add file name to cJSON array
+		log_info("Found file '%s' with size %zu", name, size);
+		cJSON *file = cJSON_CreateObject();
+		cJSON_AddItemToObject(file, "name", cJSON_CreateString(name));
+		cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size));
+		cJSON_AddItemToArray(files, file);
+	} while (p + newOffset + TAR_BLOCK_SIZE <= tarSize);
+
+	return files;
+}

src/zip/tar.h

@@ -0,0 +1,19 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  TAR reading routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#ifndef TAR_H
+#define TAR_H
+
+#include "FTL.h"
+#include "webserver/cJSON/cJSON.h"
+
+const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize);
+cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize);
+
+#endif // TAR_H

src/zip/teleporter.c

@@ -524,7 +524,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char *
 	return NULL;
 }
 
-const char *read_teleporter_zip(char *buffer, const size_t buflen, char * const hint, cJSON *imported_files)
+const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files)
 {
 	// Initialize ZIP archive
 	mz_zip_archive zip = { 0 };

src/zip/teleporter.h

@@ -15,7 +15,7 @@
 
 const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size);
 bool free_teleporter_zip(mz_zip_archive *zip);
-const char *read_teleporter_zip(char *buffer, const size_t buflen, char *hint, cJSON *json_files);
+const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files);
 
 bool write_teleporter_zip_to_disk(void);
 bool read_teleporter_zip_from_disk(const char *filename);