Add mremap to std.posix, and use it in std.heap.PageAllocator when available.

lib/std/c.zig

@@ -7640,6 +7640,14 @@ pub const MAP = switch (native_os) {
     else => void,
 };
 
+pub const REMAP = switch (native_os.isGnuLibC(native_abi)) {
+    true => packed struct(u32) {
+        MAYMOVE: bool = false,
+        _: u31 = 0,
+    },
+    false => void,
+};
+
 /// Used by libc to communicate failure. Not actually part of the underlying syscall.
 pub const MAP_FAILED: *anyopaque = @ptrFromInt(maxInt(usize));
 
@@ -9273,6 +9281,7 @@ pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize;
 pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize;
 pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque;
 pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int;
+pub extern "c" fn mremap(addr: *align(page_size) const anyopaque, old_len: usize, new_len: usize, flags: REMAP) *anyopaque;
 pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int;
 pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int;
 pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int;

lib/std/heap.zig

@@ -29,9 +29,6 @@ pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned;
 pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra;
 pub const MemoryPoolOptions = memory_pool.Options;
 
-/// TODO Utilize this on Windows.
-pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null;
-
 const CAllocator = struct {
     comptime {
         if (!builtin.link_libc) {

lib/std/heap/PageAllocator.zig

@@ -8,12 +8,18 @@ const native_os = builtin.os.tag;
 const windows = std.os.windows;
 const posix = std.posix;
 
+/// TODO: utilize this on windows
+pub var next_mmap_addr_hint = std.atomic.Value(?[*]align(mem.page_size) u8).init(null);
+
 pub const vtable = Allocator.VTable{
     .alloc = alloc,
     .resize = resize,
     .free = free,
 };
 
+/// Whether `posix.mremap` may be used
+const use_mremap = @hasDecl(posix.system, "REMAP") and posix.system.REMAP != void;
+
 fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {
     _ = ra;
     _ = log2_align;
@@ -35,18 +41,24 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 {
     }
 
     const aligned_len = mem.alignForward(usize, n, mem.page_size);
-    const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered);
+    const hint = next_mmap_addr_hint.load(.unordered);
+
     const slice = posix.mmap(
         hint,
         aligned_len,
         posix.PROT.READ | posix.PROT.WRITE,
         .{ .TYPE = .PRIVATE, .ANONYMOUS = true },
         -1,
         0,
-    ) catch return null;
+    ) catch {
+        _ = next_mmap_addr_hint.cmpxchgStrong(hint, null, .monotonic, .monotonic);
+        return null;
+    };
+
     assert(mem.isAligned(@intFromPtr(slice.ptr), mem.page_size));
     const new_hint: [*]align(mem.page_size) u8 = @alignCast(slice.ptr + aligned_len);
-    _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic);
+    _ = next_mmap_addr_hint.cmpxchgStrong(hint, new_hint, .monotonic, .monotonic);
+
     return slice.ptr;
 }
 
@@ -59,7 +71,9 @@ fn resize(
 ) bool {
     _ = log2_buf_align;
     _ = return_address;
+
     const new_size_aligned = mem.alignForward(usize, new_size, mem.page_size);
+    const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size);
 
     if (native_os == .windows) {
         if (new_size <= buf_unaligned.len) {
@@ -77,27 +91,36 @@ fn resize(
             }
             return true;
         }
-        const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size);
-        if (new_size_aligned <= old_size_aligned) {
-            return true;
-        }
-        return false;
+        return new_size_aligned <= old_size_aligned;
     }
 
-    const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, mem.page_size);
-    if (new_size_aligned == buf_aligned_len)
-        return true;
-
-    if (new_size_aligned < buf_aligned_len) {
-        const ptr = buf_unaligned.ptr + new_size_aligned;
-        // TODO: if the next_mmap_addr_hint is within the unmapped range, update it
-        posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned]));
-        return true;
+    const buf: []align(mem.page_size) u8 = @alignCast(buf_unaligned.ptr[0..old_size_aligned]);
+    const result = switch (std.math.order(old_size_aligned, new_size_aligned)) {
+        .lt => grow: {
+            if (use_mremap) {
+                const slice = posix.mremap(
+                    buf.ptr[0..old_size_aligned],
+                    new_size_aligned,
+                    false,
+                ) catch break :grow false;
+                assert(slice.ptr == buf.ptr);
+                break :grow true;
+            } else {
+                break :grow false;
+            }
+        },
+        .eq => return true, // return now and don't set the hint
+        .gt => shrink: {
+            posix.munmap(@alignCast(buf.ptr[new_size_aligned..old_size_aligned]));
+            break :shrink true;
+        },
+    };
+    if (result) {
+        const old_end: [*]align(mem.page_size) u8 = @alignCast(buf.ptr + old_size_aligned);
+        const new_end: [*]align(mem.page_size) u8 = @alignCast(buf.ptr + new_size_aligned);
+        _ = next_mmap_addr_hint.cmpxchgStrong(old_end, new_end, .monotonic, .monotonic);
     }
-
-    // TODO: call mremap
-    // TODO: if the next_mmap_addr_hint is within the remapped range, update it
-    return false;
+    return result;
 }
 
 fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void {
@@ -108,6 +131,9 @@ fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) v
         windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE);
     } else {
         const buf_aligned_len = mem.alignForward(usize, slice.len, mem.page_size);
-        posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len]));
+        const head: []align(mem.page_size) u8 = @alignCast(slice.ptr[0..buf_aligned_len]);
+        posix.munmap(head);
+        const tail: [*]align(mem.page_size) u8 = @alignCast(head.ptr + head.len);
+        _ = next_mmap_addr_hint.cmpxchgStrong(tail, head.ptr, .monotonic, .monotonic);
     }
 }

lib/std/os/linux.zig

@@ -305,6 +305,38 @@ pub const MAP = switch (native_arch) {
     else => @compileError("missing std.os.linux.MAP constants for this architecture"),
 };
 
+pub const REMAP = switch (native_arch) {
+    .x86_64,
+    .x86,
+    .aarch64,
+    .aarch64_be,
+    .arm,
+    .armeb,
+    .thumb,
+    .thumbeb,
+    .riscv32,
+    .riscv64,
+    .loongarch64,
+    .sparc64,
+    .mips,
+    .mipsel,
+    .mips64,
+    .mips64el,
+    .powerpc,
+    .powerpcle,
+    .powerpc64,
+    .powerpc64le,
+    .hexagon,
+    .s390x,
+    => packed struct(u32) {
+        MAYMOVE: bool = false,
+        FIXED: bool = false,
+        DONTUNMAP: bool = false,
+        _: u29 = 0,
+    },
+    else => @compileError("missing std.os.linux.REMAP constants for this architecture"),
+};
+
 pub const O = switch (native_arch) {
     .x86_64 => packed struct(u32) {
         ACCMODE: ACCMODE = .RDONLY,
@@ -930,6 +962,17 @@ pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: MAP, fd: i32, of
     }
 }
 
+pub fn mremap(address: ?[*]u8, old_length: usize, new_length: usize, flags: REMAP, new_addr: [*]u8) usize {
+    return syscall5(
+        .mremap,
+        @intFromPtr(address),
+        old_length,
+        new_length,
+        @as(u32, @bitCast(flags)),
+        @intFromPtr(new_addr),
+    );
+}
+
 pub fn mprotect(address: [*]const u8, length: usize, protection: usize) usize {
     return syscall3(.mprotect, @intFromPtr(address), length, protection);
 }

lib/std/posix.zig

@@ -24,6 +24,7 @@ const maxInt = std.math.maxInt;
 const cast = std.math.cast;
 const assert = std.debug.assert;
 const native_os = builtin.os.tag;
+const native_abi = builtin.abi;
 
 test {
     _ = @import("posix/test.zig");
@@ -4767,6 +4768,70 @@ pub fn munmap(memory: []align(mem.page_size) const u8) void {
     }
 }
 
+pub const MRemapError = error{
+    /// "Segmanetation Fault".
+    /// The passed in slice is not a valid virtual address
+    /// for the process.
+    PageFault,
+
+    LockedMemoryLimitExceeded,
+    OutOfMemory,
+} || UnexpectedError;
+
+/// Change the size of memory which was mapped via `mmap`.
+/// Neither `memory.len` nor `new_len` does not need to be aligned.
+/// Only available on linux or glibc targets.
+pub fn mremap(
+    memory: []align(mem.page_size) u8,
+    new_len: usize,
+    may_move: bool,
+) MRemapError![]align(mem.page_size) u8 {
+    // TODO: better support for REMAP_FIXED
+    const err: E = blk: {
+        if (use_libc) {
+            if (native_os.isGnuLibC(native_abi)) {
+                const rc = system.mremap(
+                    memory.ptr,
+                    memory.len,
+                    new_len,
+                    .{ .MAYMOVE = may_move },
+                );
+                if (rc != std.c.MAP_FAILED) {
+                    const head: [*]align(mem.page_size) u8 = @ptrCast(@alignCast(rc));
+                    return head[0..new_len];
+                } else {
+                    break :blk @enumFromInt(system._errno().*);
+                }
+            }
+        } else if (native_os == .linux) {
+            const rc = system.mremap(
+                memory.ptr,
+                memory.len,
+                new_len,
+                .{ .MAYMOVE = may_move },
+                undefined,
+            );
+            switch (errno(rc)) {
+                .SUCCESS => {
+                    const head: [*]align(mem.page_size) u8 = @ptrFromInt(rc);
+                    return head[0..new_len];
+                },
+                else => |err| break :blk err,
+            }
+        }
+        @compileError("mremap is not available on this target");
+    };
+
+    return switch (err) {
+        .SUCCESS => unreachable, // Handled above
+        .INVAL => unreachable, // Invalid parameters
+        .AGAIN => error.LockedMemoryLimitExceeded,
+        .FAULT => error.PageFault,
+        .NOMEM => error.OutOfMemory,
+        else => unexpectedErrno(err),
+    };
+}
+
 pub const MSyncError = error{
     UnmappedMemory,
     PermissionDenied,