diff --git a/lib/std/c.zig b/lib/std/c.zig index c6a5a8552110..818670e94786 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -7640,6 +7640,14 @@ pub const MAP = switch (native_os) { else => void, }; +pub const REMAP = switch (native_os.isGnuLibC(native_abi)) { + true => packed struct(u32) { + MAYMOVE: bool = false, + _: u31 = 0, + }, + false => void, +}; + /// Used by libc to communicate failure. Not actually part of the underlying syscall. pub const MAP_FAILED: *anyopaque = @ptrFromInt(maxInt(usize)); @@ -9273,6 +9281,7 @@ pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize; pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize; pub extern "c" fn mmap(addr: ?*align(page_size) anyopaque, len: usize, prot: c_uint, flags: MAP, fd: fd_t, offset: off_t) *anyopaque; pub extern "c" fn munmap(addr: *align(page_size) const anyopaque, len: usize) c_int; +pub extern "c" fn mremap(addr: *align(page_size) const anyopaque, old_len: usize, new_len: usize, flags: REMAP) *anyopaque; pub extern "c" fn mprotect(addr: *align(page_size) anyopaque, len: usize, prot: c_uint) c_int; pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8) c_int; pub extern "c" fn linkat(oldfd: fd_t, oldpath: [*:0]const u8, newfd: fd_t, newpath: [*:0]const u8, flags: c_int) c_int; diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 3d19d8daa6b2..f691786ba095 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -29,9 +29,6 @@ pub const MemoryPoolAligned = memory_pool.MemoryPoolAligned; pub const MemoryPoolExtra = memory_pool.MemoryPoolExtra; pub const MemoryPoolOptions = memory_pool.Options; -/// TODO Utilize this on Windows. -pub var next_mmap_addr_hint: ?[*]align(mem.page_size) u8 = null; - const CAllocator = struct { comptime { if (!builtin.link_libc) { diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 4188c255285c..d7d44b631b67 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -8,12 +8,18 @@ const native_os = builtin.os.tag; const windows = std.os.windows; const posix = std.posix; +/// TODO: utilize this on windows +pub var next_mmap_addr_hint = std.atomic.Value(?[*]align(mem.page_size) u8).init(null); + pub const vtable = Allocator.VTable{ .alloc = alloc, .resize = resize, .free = free, }; +/// Whether `posix.mremap` may be used +const use_mremap = @hasDecl(posix.system, "REMAP") and posix.system.REMAP != void; + fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { _ = ra; _ = log2_align; @@ -35,7 +41,8 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { } const aligned_len = mem.alignForward(usize, n, mem.page_size); - const hint = @atomicLoad(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, .unordered); + const hint = next_mmap_addr_hint.load(.unordered); + const slice = posix.mmap( hint, aligned_len, @@ -43,10 +50,15 @@ fn alloc(_: *anyopaque, n: usize, log2_align: u8, ra: usize) ?[*]u8 { .{ .TYPE = .PRIVATE, .ANONYMOUS = true }, -1, 0, - ) catch return null; + ) catch { + _ = next_mmap_addr_hint.cmpxchgStrong(hint, null, .monotonic, .monotonic); + return null; + }; + assert(mem.isAligned(@intFromPtr(slice.ptr), mem.page_size)); const new_hint: [*]align(mem.page_size) u8 = @alignCast(slice.ptr + aligned_len); - _ = @cmpxchgStrong(@TypeOf(std.heap.next_mmap_addr_hint), &std.heap.next_mmap_addr_hint, hint, new_hint, .monotonic, .monotonic); + _ = next_mmap_addr_hint.cmpxchgStrong(hint, new_hint, .monotonic, .monotonic); + return slice.ptr; } @@ -59,7 +71,9 @@ fn resize( ) bool { _ = log2_buf_align; _ = return_address; + const new_size_aligned = mem.alignForward(usize, new_size, mem.page_size); + const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size); if (native_os == .windows) { if (new_size <= buf_unaligned.len) { @@ -77,27 +91,36 @@ fn resize( } return true; } - const old_size_aligned = mem.alignForward(usize, buf_unaligned.len, mem.page_size); - if (new_size_aligned <= old_size_aligned) { - return true; - } - return false; + return new_size_aligned <= old_size_aligned; } - const buf_aligned_len = mem.alignForward(usize, buf_unaligned.len, mem.page_size); - if (new_size_aligned == buf_aligned_len) - return true; - - if (new_size_aligned < buf_aligned_len) { - const ptr = buf_unaligned.ptr + new_size_aligned; - // TODO: if the next_mmap_addr_hint is within the unmapped range, update it - posix.munmap(@alignCast(ptr[0 .. buf_aligned_len - new_size_aligned])); - return true; + const buf: []align(mem.page_size) u8 = @alignCast(buf_unaligned.ptr[0..old_size_aligned]); + const result = switch (std.math.order(old_size_aligned, new_size_aligned)) { + .lt => grow: { + if (use_mremap) { + const slice = posix.mremap( + buf.ptr[0..old_size_aligned], + new_size_aligned, + false, + ) catch break :grow false; + assert(slice.ptr == buf.ptr); + break :grow true; + } else { + break :grow false; + } + }, + .eq => return true, // return now and don't set the hint + .gt => shrink: { + posix.munmap(@alignCast(buf.ptr[new_size_aligned..old_size_aligned])); + break :shrink true; + }, + }; + if (result) { + const old_end: [*]align(mem.page_size) u8 = @alignCast(buf.ptr + old_size_aligned); + const new_end: [*]align(mem.page_size) u8 = @alignCast(buf.ptr + new_size_aligned); + _ = next_mmap_addr_hint.cmpxchgStrong(old_end, new_end, .monotonic, .monotonic); } - - // TODO: call mremap - // TODO: if the next_mmap_addr_hint is within the remapped range, update it - return false; + return result; } fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) void { @@ -108,6 +131,9 @@ fn free(_: *anyopaque, slice: []u8, log2_buf_align: u8, return_address: usize) v windows.VirtualFree(slice.ptr, 0, windows.MEM_RELEASE); } else { const buf_aligned_len = mem.alignForward(usize, slice.len, mem.page_size); - posix.munmap(@alignCast(slice.ptr[0..buf_aligned_len])); + const head: []align(mem.page_size) u8 = @alignCast(slice.ptr[0..buf_aligned_len]); + posix.munmap(head); + const tail: [*]align(mem.page_size) u8 = @alignCast(head.ptr + head.len); + _ = next_mmap_addr_hint.cmpxchgStrong(tail, head.ptr, .monotonic, .monotonic); } } diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index e9a84d806256..dfec85728a9f 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -305,6 +305,38 @@ pub const MAP = switch (native_arch) { else => @compileError("missing std.os.linux.MAP constants for this architecture"), }; +pub const REMAP = switch (native_arch) { + .x86_64, + .x86, + .aarch64, + .aarch64_be, + .arm, + .armeb, + .thumb, + .thumbeb, + .riscv32, + .riscv64, + .loongarch64, + .sparc64, + .mips, + .mipsel, + .mips64, + .mips64el, + .powerpc, + .powerpcle, + .powerpc64, + .powerpc64le, + .hexagon, + .s390x, + => packed struct(u32) { + MAYMOVE: bool = false, + FIXED: bool = false, + DONTUNMAP: bool = false, + _: u29 = 0, + }, + else => @compileError("missing std.os.linux.REMAP constants for this architecture"), +}; + pub const O = switch (native_arch) { .x86_64 => packed struct(u32) { ACCMODE: ACCMODE = .RDONLY, @@ -930,6 +962,17 @@ pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: MAP, fd: i32, of } } +pub fn mremap(address: ?[*]u8, old_length: usize, new_length: usize, flags: REMAP, new_addr: [*]u8) usize { + return syscall5( + .mremap, + @intFromPtr(address), + old_length, + new_length, + @as(u32, @bitCast(flags)), + @intFromPtr(new_addr), + ); +} + pub fn mprotect(address: [*]const u8, length: usize, protection: usize) usize { return syscall3(.mprotect, @intFromPtr(address), length, protection); } diff --git a/lib/std/posix.zig b/lib/std/posix.zig index 39b29ea76f02..c933652f5b03 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -24,6 +24,7 @@ const maxInt = std.math.maxInt; const cast = std.math.cast; const assert = std.debug.assert; const native_os = builtin.os.tag; +const native_abi = builtin.abi; test { _ = @import("posix/test.zig"); @@ -4767,6 +4768,70 @@ pub fn munmap(memory: []align(mem.page_size) const u8) void { } } +pub const MRemapError = error{ + /// "Segmanetation Fault". + /// The passed in slice is not a valid virtual address + /// for the process. + PageFault, + + LockedMemoryLimitExceeded, + OutOfMemory, +} || UnexpectedError; + +/// Change the size of memory which was mapped via `mmap`. +/// Neither `memory.len` nor `new_len` does not need to be aligned. +/// Only available on linux or glibc targets. +pub fn mremap( + memory: []align(mem.page_size) u8, + new_len: usize, + may_move: bool, +) MRemapError![]align(mem.page_size) u8 { + // TODO: better support for REMAP_FIXED + const err: E = blk: { + if (use_libc) { + if (native_os.isGnuLibC(native_abi)) { + const rc = system.mremap( + memory.ptr, + memory.len, + new_len, + .{ .MAYMOVE = may_move }, + ); + if (rc != std.c.MAP_FAILED) { + const head: [*]align(mem.page_size) u8 = @ptrCast(@alignCast(rc)); + return head[0..new_len]; + } else { + break :blk @enumFromInt(system._errno().*); + } + } + } else if (native_os == .linux) { + const rc = system.mremap( + memory.ptr, + memory.len, + new_len, + .{ .MAYMOVE = may_move }, + undefined, + ); + switch (errno(rc)) { + .SUCCESS => { + const head: [*]align(mem.page_size) u8 = @ptrFromInt(rc); + return head[0..new_len]; + }, + else => |err| break :blk err, + } + } + @compileError("mremap is not available on this target"); + }; + + return switch (err) { + .SUCCESS => unreachable, // Handled above + .INVAL => unreachable, // Invalid parameters + .AGAIN => error.LockedMemoryLimitExceeded, + .FAULT => error.PageFault, + .NOMEM => error.OutOfMemory, + else => unexpectedErrno(err), + }; +} + pub const MSyncError = error{ UnmappedMemory, PermissionDenied,