From 50a72afcef99d980dc9fbb4e39a4c82f61ba564b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 22:21:48 +0100 Subject: [PATCH] Add TAR routines for efficient parsing of a tar archive in memory Signed-off-by: DL6ER --- src/api/teleporter.c | 72 ++++++++++++++++++++--- src/zip/CMakeLists.txt | 2 + src/zip/gzip.c | 5 +- src/zip/gzip.h | 4 ++ src/zip/tar.c | 129 +++++++++++++++++++++++++++++++++++++++++ src/zip/tar.h | 19 ++++++ src/zip/teleporter.c | 2 +- src/zip/teleporter.h | 2 +- 8 files changed, 223 insertions(+), 12 deletions(-) create mode 100644 src/zip/tar.c create mode 100644 src/zip/tar.h diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 856da89c..b82cd535 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -15,6 +15,10 @@ #include "api/api.h" // ERRBUF_SIZE #include "config/dnsmasq_config.h" +// inflate_buffer() +#include "zip/gzip.h" +// find_file_in_tar() +#include "zip/tar.h" #define MAXFILESIZE (50u*1024*1024) @@ -58,7 +62,7 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; - char *data; + uint8_t *data; char *filename; size_t filesize; }; @@ -163,6 +167,7 @@ static int free_upload_data(struct upload_data *data) // Private function prototypes static int process_received_zip(struct ftl_conn *api, struct upload_data *data); +static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data); static int api_teleporter_POST(struct ftl_conn *api) { @@ -231,14 +236,31 @@ static int api_teleporter_POST(struct ftl_conn *api) { return process_received_zip(api, &data); } - else + // Check if we received something that claims to be a TAR.GZ archive + // - filename + // - shoud be at least 12 characters long, + // - should start in "pi-hole-", + // - have "-teleporter_" in the middle, and + // - end in ".tar.gz" + // - the data itself + // - should be at least 40 bytes long + // - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt) + else if(strlen(data.filename) >= 12 && + strncmp(data.filename, "pi-hole-", 8) == 0 && + strstr(data.filename, "-teleporter_") != NULL && + strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 && + data.filesize >= 40 && + memcmp(data.data, "\x1f\x8b", 2) == 0) { - free_upload_data(&data); - return send_json_error(api, 400, - "bad_request", - "Invalid file", - "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); + return process_received_tar_gz(api, &data); } + + // else: invalid file + free_upload_data(&data); + return send_json_error(api, 400, + "bad_request", + "Invalid file", + "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); } static int process_received_zip(struct ftl_conn *api, struct upload_data *data) @@ -274,6 +296,42 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) JSON_SEND_OBJECT(json); } +static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data) +{ + // Try to decompress the received data + uint8_t *archive = NULL; + mz_ulong archive_size = 0u; + if(!inflate_buffer(data->data, data->filesize, &archive, &archive_size)) + { + free_upload_data(data); + return send_json_error(api, 400, + "bad_request", + "Invalid GZIP archive", + "The uploaded file does not appear to be a valid gzip archive - decompression failed"); + } + + // Check if the decompressed data is a valid TAR archive + cJSON *json_files = list_files_in_tar(archive, archive_size); + + // Print all files in the TAR archive + cJSON *file = NULL; + cJSON_ArrayForEach(file, json_files) + { + cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); + cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); + log_info("Found file in TAR archive: \"%s\" (%d bytes)", + name->valuestring, size->valueint); + } + + // Free allocated memory + free_upload_data(data); + + // Send response + cJSON *json = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); + JSON_SEND_OBJECT(json); +} + int api_teleporter(struct ftl_conn *api) { if(api->method == HTTP_GET) diff --git a/src/zip/CMakeLists.txt b/src/zip/CMakeLists.txt index 8042ed5e..3abeb7ad 100644 --- a/src/zip/CMakeLists.txt +++ b/src/zip/CMakeLists.txt @@ -11,6 +11,8 @@ set(sources gzip.c gzip.h + tar.c + tar.h teleporter.c teleporter.h ) diff --git a/src/zip/gzip.c b/src/zip/gzip.c index bf15ea86..f7fbc571 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -14,7 +14,6 @@ #include // le32toh and friends #include -#include "miniz/miniz.h" #include "gzip.h" #include "log.h" @@ -103,8 +102,8 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul return true; } -static bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, - unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed) +bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, + unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed) { // Check GZIP header (magic byte 1F 8B and compression algorithm deflate 08) if(buffer_compressed[0] != 0x1F || buffer_compressed[1] != 0x8B) diff --git a/src/zip/gzip.h b/src/zip/gzip.h index 7839602c..b0c916b6 100644 --- a/src/zip/gzip.h +++ b/src/zip/gzip.h @@ -11,6 +11,10 @@ #define GZIP_H #include +#include "miniz/miniz.h" + +bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, + unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed); bool deflate_file(const char *in, const char *out, bool verbose); bool inflate_file(const char *infile, const char *outfile, bool verbose); diff --git a/src/zip/tar.c b/src/zip/tar.c new file mode 100644 index 00000000..639af231 --- /dev/null +++ b/src/zip/tar.c @@ -0,0 +1,129 @@ +/* Pi-hole: A black hole for Internet advertisements + * (c) 2023 Pi-hole, LLC (https://pi-hole.net) + * Network-wide ad blocking via your own hardware. + * + * FTL Engine + * In-memory tar reading routines + * + * This file is copyright under the latest version of the EUPL. + * Please see LICENSE file for your rights under this license. */ + +#include "zip/tar.h" +#include "log.h" + +// TAR offsets +#define TAR_NAME_OFFSET 0 +#define TAR_SIZE_OFFSET 124 +#define TAR_MAGIC_OFFSET 257 + +// TAR constants +#define TAR_BLOCK_SIZE 512 +#define TAR_NAME_SIZE 100 +#define TAR_SIZE_SIZE 12 +#define TAR_MAGIC_SIZE 5 + +static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ + +/** + * Find a file in a TAR archive + * @param tarData Pointer to the TAR archive in memory + * @param tarSize Size of the TAR archive in memory in bytes + * @param fileName Name of the file to find + * @param fileSize Pointer to a size_t variable to store the file size in + * @return Pointer to the file data or NULL if not found + */ +const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, + const char *fileName, size_t *fileSize) +{ + bool found = false; + size_t size, p = 0, newOffset = 0; + + // Convert to char * to be able to do pointer arithmetic more easily + const char *tar = (const char *)tarData; + + // Initialize fileSize to 0 + *fileSize = 0; + + // Loop through TAR file + do + { + // "Load" data from tar - just point to passed memory + const char *name = tar + TAR_NAME_OFFSET + p + newOffset; + const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str + p += newOffset; // pointer to current file's data in TAR + + // Check for supported TAR version or end of TAR + for (size_t i = 0; i < TAR_MAGIC_SIZE; i++) + if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i]) + return NULL; + + // Convert file size from string into integer + size = 0; + for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int + if ((sz[i] >= '1') && (sz[i] <= '9')) + size += (sz[i] - '0') * mul; + + //Offset size in bytes. Depends on file size and TAR block size + newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block + if ((size % TAR_BLOCK_SIZE) > 0) + newOffset += TAR_BLOCK_SIZE; + + found = strncmp(name, fileName, TAR_NAME_SIZE) == 0; + } while (!found && (p + newOffset + TAR_BLOCK_SIZE <= tarSize)); + + if (!found) + return NULL; // No file found in TAR - return NULL + + // File found in TAR - return pointer to file data and set fileSize + *fileSize = size; + return tarData + p + TAR_BLOCK_SIZE; +} + +/** + * List all files in a TAR archive + * @param tarData Pointer to the TAR archive in memory + * @param tarSize Size of the TAR archive in memory in bytes + * @return Pointer to a cJSON array containing all file names with file size + */ +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) +{ + cJSON *files = cJSON_CreateArray(); + size_t size, p = 0, newOffset = 0; + + // Convert to char * to be able to do pointer arithmetic more easily + const char *tar = (const char *)tarData; + + // Loop through TAR file + do + { + // "Load" data from tar - just point to passed memory + const char *name = tar + TAR_NAME_OFFSET + p + newOffset; + const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str + p += newOffset; // pointer to current file's data in TAR + + // Check for supported TAR version or end of TAR + for (size_t i = 0; i < TAR_MAGIC_SIZE; i++) + if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i]) + return files; + + // Convert file size from string into integer + size = 0; + for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int + if ((sz[i] >= '1') && (sz[i] <= '9')) + size += (sz[i] - '0') * mul; + + //Offset size in bytes. Depends on file size and TAR block size + newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block + if ((size % TAR_BLOCK_SIZE) > 0) + newOffset += TAR_BLOCK_SIZE; + + // Add file name to cJSON array + log_info("Found file '%s' with size %zu", name, size); + cJSON *file = cJSON_CreateObject(); + cJSON_AddItemToObject(file, "name", cJSON_CreateString(name)); + cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size)); + cJSON_AddItemToArray(files, file); + } while (p + newOffset + TAR_BLOCK_SIZE <= tarSize); + + return files; +} \ No newline at end of file diff --git a/src/zip/tar.h b/src/zip/tar.h new file mode 100644 index 00000000..426a4037 --- /dev/null +++ b/src/zip/tar.h @@ -0,0 +1,19 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* TAR reading routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef TAR_H +#define TAR_H + +#include "FTL.h" +#include "webserver/cJSON/cJSON.h" + +const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); + +#endif // TAR_H \ No newline at end of file diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 62281d89..5935b6de 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -524,7 +524,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char * return NULL; } -const char *read_teleporter_zip(char *buffer, const size_t buflen, char * const hint, cJSON *imported_files) +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files) { // Initialize ZIP archive mz_zip_archive zip = { 0 }; diff --git a/src/zip/teleporter.h b/src/zip/teleporter.h index b20567b9..a5743028 100644 --- a/src/zip/teleporter.h +++ b/src/zip/teleporter.h @@ -15,7 +15,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size); bool free_teleporter_zip(mz_zip_archive *zip); -const char *read_teleporter_zip(char *buffer, const size_t buflen, char *hint, cJSON *json_files); +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files); bool write_teleporter_zip_to_disk(void); bool read_teleporter_zip_from_disk(const char *filename);