Page faults: add support for custom fault handlers

This allows custom mmap() implementations for specific file
descriptors to handle page faults on mmap()ed memory. This feature
is used by the Nvidia GPU klib.
The new_pending_fault_locked() function is no longer static so that
it can be called from an arbitrary fault handler when a fault
cannot be resolved synchronously.

src/unix/mmap.c

@@ -92,7 +92,7 @@ closure_func_basic(thunk, void, pending_fault_complete)
     }
 }
 
-static pending_fault new_pending_fault_locked(process p, context ctx, u64 addr)
+pending_fault new_pending_fault_locked(process p, context ctx, u64 addr)
 {
     pending_fault pf;
     list l;
@@ -362,7 +362,7 @@ static status demand_page_internal(process p, context ctx, u64 vaddr, vmap vm, p
             anonymous = false;
             break;
         default:
-            halt("%s: invalid vmap type %d, flags 0x%lx\n", func_ss, mmap_type, vm->flags);
+            return vm->fault(p, ctx, vaddr, vm, pf);
         }
     } else if (vm->flags & VMAP_FLAG_PROG) {
         pf_debug("   file-backed program page fault\n");
@@ -1392,7 +1392,9 @@ static sysreturn mmap(void *addr, u64 length, int prot, int flags, int fd, u64 o
     if (fixed)
         process_remove_range_locked(p, q, vmap_mmap_type != VMAP_MMAP_TYPE_CUSTOM);
     k.node.r = q;
-    vmap_assert(allocate_vmap_locked(p->vmaps, &k) != INVALID_ADDRESS);
+    vmap vm = allocate_vmap_locked(p->vmaps, &k);
+    vmap_assert(vm != INVALID_ADDRESS);
+    vm->fault = k.fault;
     vmap_unlock(p);
 
     if (vmap_mmap_type == VMAP_MMAP_TYPE_FILEBACKED && (vmflags & VMAP_FLAG_SHARED))

src/unix/unix_internal.h

@@ -267,6 +267,7 @@ typedef struct pending_fault {
     enum {
         PENDING_FAULT_ANONYMOUS,
         PENDING_FAULT_FILEBACKED,
+        PENDING_FAULT_CUSTOM,
     } type;
     union {
         struct {
@@ -277,12 +278,15 @@ typedef struct pending_fault {
             closure_struct(pagecache_page_handler, demand_file_page);
             void *page_kvirt;
         } filebacked;
+        void *custom;
     };
     struct list l_free;
     closure_struct(thunk, async_handler);
     closure_struct(thunk, complete);
 } *pending_fault;
 
+pending_fault new_pending_fault_locked(process p, context ctx, u64 addr);
+
 /* XXX probably should bite bullet and allocate these... */
 #define FRAME_MAX_PADDED ((FRAME_MAX + 15) & ~15)
 
@@ -433,6 +437,7 @@ typedef struct vmap {
         fdesc fd;
         u64 bss_offset;
     };
+    status (*fault)(process p, context ctx, u64 vaddr, struct vmap *vm, pending_fault *pf);
 } *vmap;
 
 #define ivmap(__f, __af, __o, __c, __fd) (struct vmap) {    \