kvm: avoid mmio exits when Sentry faults on unmapped memory

Currently, we generate page tables for the entire sentry address space.
Consequently, when the Sentry faults on unmapped memory - meaning a memory
region not yet mapped into the VM - an MMIO exit is triggered. The issue is
that because the current instruction is emulated (instead of executed
natively), it becomes impossible to trigger a "normal" memory fault.

To solve this, we must set up page tables only for the regions that are
explicitly mapped into the VM. This, however, is more challenging than it
sounds for several reasons:

We map memory regions into the VM from a signal handler, where memory
allocation is prohibited. This means all necessary page table entries must be
allocated during platform initialization.

Our memory regions are not aligned to huge page boundaries. Therefore, when
mapping a memory slot, we often need to split huge pages and allocate new page
table entries.

We run into the nosplit stack limit, requiring us to introduce a PTE.Get method
to safely avoid accessing splice entries via indices, which could
trigger a panic and so requires a lot of extra stack.

PiperOrigin-RevId: 827726897

Author: avagin

pkg/ring0/pagetables/BUILD

@@ -49,6 +49,7 @@ go_library(
         "pagetables_aarch64.go",
         "pagetables_amd64.go",
         "pagetables_arm64.go",
+        "pagetables_unsafe.go",
         "pagetables_x86.go",
         "pcids.go",
         "pcids_aarch64.go",

pkg/ring0/pagetables/pagetables_aarch64.go

@@ -91,6 +91,9 @@ type MapOpts struct {
 	// User indicates the page is a user page.
 	User bool
 
+	// Static indicates the entries should not be cleared/freed.
+	Static bool
+
 	// MemoryType is the memory type.
 	MemoryType hostarch.MemoryType
 }

pkg/ring0/pagetables/pagetables_unsafe.go

@@ -0,0 +1,26 @@
+// Copyright 2025 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagetables
+
+import (
+	"unsafe"
+)
+
+// Get returns the entry with the specified index.
+//
+//go:nosplit
+func (p *PTEs) Get(idx uint16) *PTE {
+	return (*PTE)(unsafe.Pointer(uintptr(unsafe.Pointer(&p[0])) + 8*uintptr(idx)))
+}

pkg/ring0/pagetables/pagetables_x86.go

@@ -73,6 +73,9 @@ type MapOpts struct {
 	// User indicates the page is a user page.
 	User bool
 
+	// Static indicates the entries should not be cleared/freed.
+	Static bool
+
 	// MemoryType is the memory type.
 	MemoryType hostarch.MemoryType
 }
@@ -91,7 +94,7 @@ func (p *PTE) Clear() {
 //
 //go:nosplit
 func (p *PTE) Valid() bool {
-	return atomic.LoadUintptr((*uintptr)(p))&present != 0
+	return atomic.LoadUintptr((*uintptr)(p)) != 0
 }
 
 // Opts returns the PTE options.
@@ -139,7 +142,7 @@ func (p *PTE) IsSuper() bool {
 //
 //go:nosplit
 func (p *PTE) Set(addr uintptr, opts MapOpts) {
-	if !opts.AccessType.Any() {
+	if false && !opts.AccessType.Any() && !opts.Static {
 		p.Clear()
 		return
 	}

pkg/ring0/pagetables/walker_amd64.go

@@ -43,7 +43,7 @@ func (w *Walker) walkPTEs(entries *PTEs, start, end uintptr) (bool, uint16) {
 	var clearEntries uint16
 	for start < end {
 		pteIndex := uint16((start & pteMask) >> pteShift)
-		entry := &entries[pteIndex]
+		entry := entries.Get(pteIndex)
 		if !entry.Valid() && !w.visitor.requiresAlloc() {
 			clearEntries++
 			start += pteSize
@@ -81,7 +81,7 @@ func (w *Walker) walkPMDs(pmdEntries *PTEs, start, end uintptr) (bool, uint16) {
 		var pteEntries *PTEs
 		nextBoundary := addrEnd(start, end, pmdSize)
 		pmdIndex := uint16((start & pmdMask) >> pmdShift)
-		pmdEntry := &pmdEntries[pmdIndex]
+		pmdEntry := pmdEntries.Get(pmdIndex)
 		if !pmdEntry.Valid() {
 			if !w.visitor.requiresAlloc() {
 				// Skip over this entry.
@@ -173,7 +173,7 @@ func (w *Walker) walkPUDs(pudEntries *PTEs, start, end uintptr) (bool, uint16) {
 		var pmdEntries *PTEs
 		nextBoundary := addrEnd(start, end, pudSize)
 		pudIndex := uint16((start & pudMask) >> pudShift)
-		pudEntry := &pudEntries[pudIndex]
+		pudEntry := pudEntries.Get(pudIndex)
 		if !pudEntry.Valid() {
 			if !w.visitor.requiresAlloc() {
 				// Skip over this entry.
@@ -261,7 +261,7 @@ func (w *Walker) iterateRangeCanonical(start, end uintptr) bool {
 		var pudEntries *PTEs
 		nextBoundary := addrEnd(start, end, pgdSize)
 		pgdIndex := uint16((start & pgdMask) >> pgdShift)
-		pgdEntry := &w.pageTables.root[pgdIndex]
+		pgdEntry := w.pageTables.root.Get(pgdIndex)
 		if !w.pageTables.largeAddressesEnabled {
 			if !pgdEntry.Valid() {
 				if !w.visitor.requiresAlloc() {

pkg/sentry/platform/kvm/bluepill_fault.go

@@ -25,11 +25,14 @@ import (
 var (
 	// faultBlockSize is the size used for servicing memory faults.
 	//
-	// This should be large enough to avoid frequent faults and avoid using
-	// all available KVM slots (~512), but small enough that KVM does not
-	// complain about slot sizes (~4GB). See handleBluepillFault for how
-	// this block is used.
-	faultBlockSize = uintptr(2 << 30)
+	// This should be large enough so that the total number of slots
+	// required to cover the 47-bit virtual address space does not exceed
+	// the KVM slot limit (e.g. 32764). Linux doesn't allocate virtual
+	// address space above 47-bit by default.
+	// It must be small enough to limit the memory overhead associated with
+	// KVM slot allocation. For example, using a 46-bit address space
+	// results in an overhead of ~250 MB.
+	faultBlockSize = uintptr(8 << 30)
 
 	// faultBlockMask is the mask for the fault blocks.
 	//
@@ -56,13 +59,17 @@ func calculateBluepillFault(physical uintptr) (virtualStart, physicalStart, leng
 		}
 
 		// Adjust the block to match our size.
-		physicalStart = pr.physical + (alignedPhysical-pr.physical)&faultBlockMask
-		virtualStart = pr.virtual + (physicalStart - pr.physical)
+		physicalStart = pr.physical / faultBlockSize * faultBlockSize
+		physicalStart = physicalStart + (alignedPhysical-physicalStart)&faultBlockMask
 		physicalEnd := physicalStart + faultBlockSize
+		if physicalStart < pr.physical {
+			physicalStart = pr.physical
+		}
 		if physicalEnd > end {
 			physicalEnd = end
 		}
 		length = physicalEnd - physicalStart
+		virtualStart = pr.virtual + (physicalStart - pr.physical)
 		return virtualStart, physicalStart, length, &physicalRegions[i]
 	}
 

pkg/sentry/platform/kvm/machine.go

@@ -332,26 +332,33 @@ func newMachine(vm int, config *Config) (*machine, error) {
 		// faultBlockSize has to equal or less than KVM_MEM_MAX_NR_PAGES.
 		faultBlockSize = uintptr(1) << 42
 		faultBlockMask = ^uintptr(faultBlockSize - 1)
+		for _, r := range physicalRegions {
+			m.mapPhysical(r.physical, r.length)
+		}
 	} else {
+		// Apply the physical mappings. Note that these mappings may point to
+		// guest physical addresses that are not actually available. These
+		// physical pages are mapped on demand, see kernel_unsafe.go.
+		applyPhysicalRegions(func(pr physicalRegion) bool {
+			physical := pr.physical
+			for physical < pr.physical+pr.length {
+				virtualStart, physicalStart, length, _ := calculateBluepillFault(physical)
+				// Pre-allocate page tables in the lower half.
+				m.kernel.PageTables.Map(
+					hostarch.Addr(virtualStart),
+					length,
+					pagetables.MapOpts{Static: true},
+					physicalStart)
+				physical += length
+			}
+
+			return true // Keep iterating.
+		})
 		// Install seccomp rules to trap runtime mmap system calls. They will
 		// be handled by seccompMmapHandler.
 		seccompMmapRules(m)
 	}
 
-	// Apply the physical mappings. Note that these mappings may point to
-	// guest physical addresses that are not actually available. These
-	// physical pages are mapped on demand, see kernel_unsafe.go.
-	applyPhysicalRegions(func(pr physicalRegion) bool {
-		// Map everything in the lower half.
-		m.kernel.PageTables.Map(
-			hostarch.Addr(pr.virtual),
-			pr.length,
-			pagetables.MapOpts{AccessType: hostarch.ReadWrite},
-			pr.physical)
-
-		return true // Keep iterating.
-	})
-
 	// Ensure that the currently mapped virtual regions are actually
 	// available in the VM. Note that this doesn't guarantee no future
 	// faults, however it should guarantee that everything is available to
@@ -368,6 +375,9 @@ func newMachine(vm int, config *Config) (*machine, error) {
 				// Cap the length to the end of the area.
 				length = vr.virtual + vr.length - virtual
 			}
+			// Ensure the physical range is mapped.
+			m.mapPhysical(physical, length)
+
 			// Update page tables for executable mappings.
 			if vr.accessType.Execute {
 				if vr.accessType.Write {
@@ -380,8 +390,6 @@ func newMachine(vm int, config *Config) (*machine, error) {
 					physical)
 			}
 
-			// Ensure the physical range is mapped.
-			m.mapPhysical(physical, length)
 			virtual += length
 		}
 	}
@@ -404,11 +412,6 @@ func newMachine(vm int, config *Config) (*machine, error) {
 		mapRegion(vr, 0)
 
 	})
-	if mapEntireAddressSpace {
-		for _, r := range physicalRegions {
-			m.mapPhysical(r.physical, r.length)
-		}
-	}
 	enableAsyncPreemption()
 	// Initialize architecture state.
 	if err := m.initArchState(); err != nil {
@@ -458,8 +461,15 @@ func (m *machine) mapPhysical(physical, length uintptr) {
 		}
 
 		// Is this already mapped? Check the usedSlots.
-		if !pr.mmio && !m.hasSlot(physicalStart) {
-			m.mapMemorySlot(virtualStart, physicalStart, length, pr.readOnly)
+		if !m.hasSlot(physicalStart) {
+			m.kernel.PageTables.Map(
+				hostarch.Addr(virtualStart),
+				length,
+				pagetables.MapOpts{AccessType: hostarch.ReadWrite},
+				physicalStart)
+			if !pr.mmio {
+				m.mapMemorySlot(virtualStart, physicalStart, length, pr.readOnly)
+			}
 		}
 
 		// Move to the next chunk.

pkg/sentry/platform/kvm/physical_map.go

@@ -16,6 +16,7 @@ package kvm
 
 import (
 	"fmt"
+	"runtime"
 	"sort"
 
 	"golang.org/x/sys/unix"
@@ -66,6 +67,7 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 	pSize := uintptr(1) << ring0.PhysicalAddressBits
 	pSize -= reservedMemory
 
+	maxUserAddr := uintptr(0)
 	// Add specifically excluded regions; see excludeVirtualRegion.
 	if err := applyVirtualRegions(func(vr virtualRegion) {
 		if excludeVirtualRegion(vr) {
@@ -81,9 +83,29 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 			})
 			log.Infof("mmio: virtual [%x,%x)", vr.virtual, vr.virtual+vr.length)
 		}
+		if vr.filename != "[vsyscall]" {
+			maxUserAddr = vr.region.virtual + vr.region.length
+		}
 	}); err != nil {
 		panic(fmt.Sprintf("error parsing /proc/self/maps: %v", err))
 	}
+	if runtime.GOARCH == "amd64" {
+		vSize47 := uintptr(1) << 47
+		// This is a workaround for the kernel bug when vdso can be
+		// mapped above the 47-bit address space boundary.
+		if vSize47 > maxUserAddr {
+			maxUserAddr = vSize47
+		}
+		r := region{
+			virtual: maxUserAddr,
+			length:  vSize - vSize47,
+		}
+		specialRegions = append(specialRegions, specialVirtualRegion{
+			region: r,
+		})
+		vSize -= r.length
+		log.Infof("excluded: virtual [%x,%x)", r.virtual, r.virtual+r.length)
+	}
 
 	// Do we need any more work?
 	if vSize < pSize {
@@ -109,7 +131,7 @@ func fillAddressSpace() (specialRegions []specialVirtualRegion) {
 	current := required // Attempted mmap size.
 	filled := uintptr(0)
 	suggestedAddr := uintptr(0)
-	if ring0.VirtualAddressBits > 48 {
+	if false && ring0.VirtualAddressBits > 48 {
 		// Pass a hint address above 47 bits to indicate to the kernel that
 		// we can handle, and want, mappings above 47 bits:
 		// https://docs.kernel.org/arch/x86/x86_64/5level-paging.html#user-space-and-large-virtual-address-space.