From 9cac0a85b4f62caf458bd44d5030ce6edfd12365 Mon Sep 17 00:00:00 2001 From: Micah Elizabeth Scott Date: Fri, 25 Aug 2023 10:08:41 -0700 Subject: [PATCH 1/5] equix: Add NetBSD to "huge pages not supported" path As suggested by @wiz on #40843, let's add an explicit check to hashx_vm_alloc_huge() that avoids using a Linux-style default on NetBSD targets. This doesn't change the questionable Linux-style default, but a future patch will disable this code by default so it's not a portability liability. (This code is in hashx's VM layer but it's actually only relevant to equix.) This addresses bug #40843. Another patch will disable huge pages by default entirely, but this patch is sufficient to fix the NetBSD build. --- src/ext/equix/hashx/src/virtual_memory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c index e9df825c9f..40540a2660 100644 --- a/src/ext/equix/hashx/src/virtual_memory.c +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -106,8 +106,8 @@ void* hashx_vm_alloc_huge(size_t bytes) { #elif defined(__FreeBSD__) mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER, -1, 0); -#elif defined(__OpenBSD__) - mem = MAP_FAILED; // OpenBSD does not support huge pages +#elif defined(__OpenBSD__) || defined(__NetBSD__) + mem = MAP_FAILED; // OpenBSD and NetBSD do not support huge pages #else mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, -1, 0); From ee4e9f7506d87be71fb85e0bce0a671f541ff059 Mon Sep 17 00:00:00 2001 From: Micah Elizabeth Scott Date: Fri, 25 Aug 2023 10:18:44 -0700 Subject: [PATCH 2/5] hashx: Avoid unused arg warning on OpenBSD and NetBSD This path in hashx_vm_alloc_huge() for OpenBSD and NetBSD always fails without checking its parameter. Fix the warning. --- src/ext/equix/hashx/src/virtual_memory.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c index 40540a2660..a3a68f44b9 100644 --- a/src/ext/equix/hashx/src/virtual_memory.c +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -107,6 +107,7 @@ void* hashx_vm_alloc_huge(size_t bytes) { mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER, -1, 0); #elif defined(__OpenBSD__) || defined(__NetBSD__) + (void)bytes; mem = MAP_FAILED; // OpenBSD and NetBSD do not support huge pages #else mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_PRIVATE | MAP_ANONYMOUS From 95e8ffa97e413c19c7257c9e6dc9511e98347b68 Mon Sep 17 00:00:00 2001 From: Micah Elizabeth Scott Date: Fri, 25 Aug 2023 10:31:33 -0700 Subject: [PATCH 3/5] hashx: Fix compiled hash function on NetBSD NetBSD includes the idea of a 'maximum protection' per-region, and an mprotect which exceeds the max protection will be denied. If we explicitly ask for a maximum which includes execute permission, we can successfully swap our code buffer's permissions between read-write and read-execute when each hash program is compiled. With this patch, the crypto/hashx tests pass on NetBSD 9. This addresses bug #40844 --- src/ext/equix/hashx/src/virtual_memory.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c index a3a68f44b9..564325b641 100644 --- a/src/ext/equix/hashx/src/virtual_memory.c +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -18,6 +18,11 @@ #define PAGE_READWRITE (PROT_READ | PROT_WRITE) #define PAGE_EXECUTE_READ (PROT_READ | PROT_EXEC) #define PAGE_EXECUTE_READWRITE (PROT_READ | PROT_WRITE | PROT_EXEC) +#if defined(__NetBSD__) && defined(PROT_MPROTECT) +#define PAGE_MMAP_PROT (PAGE_READWRITE | PROT_MPROTECT(PROT_EXEC)) +#else +#define PAGE_MMAP_PROT PAGE_READWRITE +#endif #endif #ifdef HASHX_WIN @@ -57,7 +62,7 @@ void* hashx_vm_alloc(size_t bytes) { #ifdef HASHX_WIN mem = VirtualAlloc(NULL, bytes, MEM_COMMIT, PAGE_READWRITE); #else - mem = mmap(NULL, bytes, PAGE_READWRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + mem = mmap(NULL, bytes, PAGE_MMAP_PROT, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) return NULL; #endif From a3e7e9bda24ff7eff55c87a2b49b3c7442151a76 Mon Sep 17 00:00:00 2001 From: Micah Elizabeth Scott Date: Fri, 25 Aug 2023 10:51:40 -0700 Subject: [PATCH 4/5] equix: Disable huge page support by default Equi-X supports optionally allocating its solver memory using huge pages, to reduce the virtual memory subsystem overhead required to make the entire solver buffer live. Tor doesn't use this feature, since it seems to have no noticeable performance benefit at this time, but we still included code for it at compile time. To improve portability, this patch disables huge page support by default and enables it only in the cmake build system used for equix benchmarks. With this patch equix-bench still supports huge pages. Verified using strace that we're making the hugepage allocation. There's no fallback for huge pages, so Equi-X initialization will fail if they are requested and we don't support them for any runtime or compile-time reason. Addresses #40843 (NetBSD) but also prevents future porting issues related to huge pages. --- src/ext/equix/CMakeLists.txt | 1 + src/ext/equix/hashx/src/virtual_memory.c | 2 ++ src/ext/equix/hashx/src/virtual_memory.h | 5 ++++- src/ext/equix/src/context.c | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ext/equix/CMakeLists.txt b/src/ext/equix/CMakeLists.txt index 3c4606566f..a9d0665408 100644 --- a/src/ext/equix/CMakeLists.txt +++ b/src/ext/equix/CMakeLists.txt @@ -11,6 +11,7 @@ set(EQUIX_VERSION_STR "${EQUIX_VERSION}.${EQUIX_VERSION_MINOR}.${EQUIX_VERSION_P project(equix) add_definitions(-DHASHX_SIZE=8) +add_definitions(-DEQUIX_SUPPORT_HUGEPAGES) add_subdirectory("hashx") diff --git a/src/ext/equix/hashx/src/virtual_memory.c b/src/ext/equix/hashx/src/virtual_memory.c index 564325b641..2dff6f7552 100644 --- a/src/ext/equix/hashx/src/virtual_memory.c +++ b/src/ext/equix/hashx/src/virtual_memory.c @@ -90,6 +90,7 @@ bool hashx_vm_rx(void* ptr, size_t bytes) { return page_protect(ptr, bytes, PAGE_EXECUTE_READ); } +#ifdef EQUIX_SUPPORT_HUGEPAGES void* hashx_vm_alloc_huge(size_t bytes) { void* mem; #ifdef HASHX_WIN @@ -124,6 +125,7 @@ void* hashx_vm_alloc_huge(size_t bytes) { #endif return mem; } +#endif /* EQUIX_SUPPORT_HUGEPAGES */ void hashx_vm_free(void* ptr, size_t bytes) { if (!ptr) { diff --git a/src/ext/equix/hashx/src/virtual_memory.h b/src/ext/equix/hashx/src/virtual_memory.h index 9780d218c2..63633bed3f 100644 --- a/src/ext/equix/hashx/src/virtual_memory.h +++ b/src/ext/equix/hashx/src/virtual_memory.h @@ -14,7 +14,10 @@ HASHX_PRIVATE void* hashx_vm_alloc(size_t size); HASHX_PRIVATE bool hashx_vm_rw(void* ptr, size_t size); HASHX_PRIVATE bool hashx_vm_rx(void* ptr, size_t size); -HASHX_PRIVATE void* hashx_vm_alloc_huge(size_t size); HASHX_PRIVATE void hashx_vm_free(void* ptr, size_t size); +#ifdef EQUIX_SUPPORT_HUGEPAGES +HASHX_PRIVATE void* hashx_vm_alloc_huge(size_t size); +#endif + #endif diff --git a/src/ext/equix/src/context.c b/src/ext/equix/src/context.c index 28edf5e104..3ae4fdf9cf 100644 --- a/src/ext/equix/src/context.c +++ b/src/ext/equix/src/context.c @@ -27,7 +27,11 @@ equix_ctx* equix_alloc(equix_ctx_flags flags) { if (flags & EQUIX_CTX_SOLVE) { if (flags & EQUIX_CTX_HUGEPAGES) { +#ifdef EQUIX_SUPPORT_HUGEPAGES ctx->heap = hashx_vm_alloc_huge(sizeof(solver_heap)); +#else + ctx->heap = NULL; +#endif } else { ctx->heap = malloc(sizeof(solver_heap)); From 532543e84a96fbc94e9b33b36100ef0046a50429 Mon Sep 17 00:00:00 2001 From: Micah Elizabeth Scott Date: Fri, 25 Aug 2023 11:15:33 -0700 Subject: [PATCH 5/5] Changes files for 40843 and 40844 (NetBSD portability) --- changes/ticket40843 | 3 +++ changes/ticket40844 | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 changes/ticket40843 create mode 100644 changes/ticket40844 diff --git a/changes/ticket40843 b/changes/ticket40843 new file mode 100644 index 0000000000..3af63a9164 --- /dev/null +++ b/changes/ticket40843 @@ -0,0 +1,3 @@ + o Minor bugfix (NetBSD, compilation): + - Fix compilation issue on NetBSD by avoiding an unnecessary dependency on + "huge" page mappings in Equi-X. Fixes bug 40843; bugfix on 0.4.8.1-alpha. \ No newline at end of file diff --git a/changes/ticket40844 b/changes/ticket40844 new file mode 100644 index 0000000000..73d3bb2b76 --- /dev/null +++ b/changes/ticket40844 @@ -0,0 +1,4 @@ + o Minor bugfix (NetBSD, testing): + - Fix test failures in "crypto/hashx" and "slow/crypto/equix" on x86_64 + and aarch64 NetBSD hosts, by adding support for PROT_MPROTECT() flags. + Fixes bug 40844; bugfix on 0.4.8.1-alpha. \ No newline at end of file