blob storage for large atoms

build.zig

@@ -748,6 +748,11 @@ fn buildBinary(
                 .file = "pkg/vere/newt_tests.c",
                 .deps = vere_test_deps,
             },
+            .{
+                .name = "blob-test",
+                .file = "pkg/vere/blob_tests.c",
+                .deps = vere_test_deps,
+            },
             .{
                 .name = "vere-noun-test",
                 .file = "pkg/vere/noun_tests.c",

pkg/c3/platform/windows/mman.h

@@ -23,4 +23,16 @@ int mprotect(void *addr, size_t len, int prot);
 #define MS_SYNC         0      /* Synchronous memory sync.  */
 #define MS_INVALIDATE   2      /* Invalidate the caches.  */
 
+/*  madvise: no-op on Windows.  Advisory only — safe to skip.
+*/
+#define MADV_NORMAL     0
+#define MADV_SEQUENTIAL 2
+#define MADV_DONTNEED   4
+
+static inline int madvise(void* addr, size_t len, int advice)
+{
+  (void)addr; (void)len; (void)advice;
+  return 0;
+}
+
 #endif//_SYS_MMAN_H

pkg/noun/allocate.c

@@ -502,7 +502,10 @@ _me_gain_use(u3_noun dog)
 static inline u3_atom
 _ca_take_atom(u3a_atom* old_u)
 {
-  c3_w*     new_w = u3a_walloc(old_u->len_w + c3_wiseof(u3a_atom));
+  //  use masked length; bob atoms carry u3a_blob_flag in len_w
+  //
+  c3_w      dat_w = old_u->len_w & u3a_blob_mask;
+  c3_w*     new_w = u3a_walloc(dat_w + c3_wiseof(u3a_atom));
   u3a_atom* new_u = (u3a_atom*)(void *)new_w;
   u3_noun     new = u3a_to_pug(u3a_outa(new_u));
 
@@ -523,7 +526,7 @@ _ca_take_atom(u3a_atom* old_u)
   {
     c3_w i_w;
 
-    for ( i_w=0; i_w < old_u->len_w; i_w++ ) {
+    for ( i_w=0; i_w < dat_w; i_w++ ) {
       new_u->buf_w[i_w] = old_u->buf_w[i_w];
     }
   }
@@ -842,6 +845,82 @@ _me_gain_south(u3_noun dog)
   }
 }
 
+/* u3a_blob_get(): look up blb_p entry for (mug_h, seq_h).  RETAINS.
+*/
+u3a_blob*
+u3a_blob_get(c3_h mug_h, c3_h seq_h)
+{
+  u3_noun bid = u3i_chub(((c3_d)mug_h << 32) | (c3_d)seq_h);
+  u3_weak bv  = u3h_get(u3H->blb_p, bid);
+  u3z(bid);
+
+  if ( u3_none == bv ) return 0;
+
+  c3_d off_d = 0;
+  u3r_safe_chub(bv, &off_d);
+  u3z(bv);
+
+  return (u3a_blob*)u3a_into((u3_post)off_d);
+}
+
+/* u3a_blob_new(): allocate fresh u3a_blob and install it under (mug_h, seq_h).
+*/
+u3a_blob*
+u3a_blob_new(c3_h mug_h, c3_h seq_h)
+{
+  u3a_blob* blb_u = u3a_walloc(c3_wiseof(u3a_blob));
+  blb_u->use_w = 0;
+  blb_u->eve_w = 0;
+  blb_u->les_h = 0;
+  blb_u->mug_h = mug_h;
+  blb_u->seq_h = seq_h;
+
+  u3_post off_p = u3a_outa(blb_u);
+  u3_noun bid   = u3i_chub(((c3_d)mug_h << 32) | (c3_d)seq_h);
+  u3h_put(u3H->blb_p, bid, u3i_chub((c3_d)off_p));
+  u3z(bid);
+
+  return blb_u;
+}
+
+/* u3a_blob_drop(): remove blb_p entry and free underlying u3a_blob.
+*/
+void
+u3a_blob_drop(c3_h mug_h, c3_h seq_h)
+{
+  u3a_blob* blb_u = u3a_blob_get(mug_h, seq_h);
+  if ( !blb_u ) return;
+
+  u3a_wfree(blb_u);
+
+  u3_noun bid = u3i_chub(((c3_d)mug_h << 32) | (c3_d)seq_h);
+  u3h_del(u3H->blb_p, bid);
+  u3z(bid);
+}
+
+/* _me_bob_dead(): bob atom's loom refcount just hit zero.
+**
+**   Decrements u3a_blob.use_w (atom cardinality contribution).  If
+**   use_w hits zero, calls blob_del_f to wipe the file and drop the
+**   blb_p entry.
+*/
+static void
+_me_bob_dead(u3a_atom* atm_u)
+{
+  if ( !u3C.blob_del_f ) return;
+
+  u3a_blob* blb_u = (u3a_blob*)u3a_into((u3_post)atm_u->buf_w[0]);
+  if ( !blb_u ) return;
+
+  if ( blb_u->use_w > 0 ) {
+    blb_u->use_w -= 1;
+  }
+
+  if ( 0 == blb_u->use_w ) {
+    u3C.blob_del_f(blb_u->mug_h, blb_u->seq_h);
+  }
+}
+
 /* _me_lose_north(): lose on a north road.
 */
 static void
@@ -874,6 +953,10 @@ _me_lose_north(u3_noun dog)
           }
         }
         else {
+          u3a_atom* atm_u = (u3a_atom*)box_u;
+          if ( atm_u->len_w & u3a_blob_flag ) {
+            _me_bob_dead(atm_u);
+          }
           u3a_wfree(box_u);
         }
       }
@@ -913,6 +996,10 @@ _me_lose_south(u3_noun dog)
           }
         }
         else {
+          u3a_atom* atm_u = (u3a_atom*)box_u;
+          if ( atm_u->len_w & u3a_blob_flag ) {
+            _me_bob_dead(atm_u);
+          }
           u3a_wfree(box_u);
         }
       }
@@ -1152,8 +1239,18 @@ u3a_relocate_noun(u3_noun *som)
     old_p = u3a_to_off(old);
 
     if ( c3n == u3a_is_cell(old) ) {
-      new_p = _pack_relocate(old_p);
-      *som = u3a_to_pug(new_p);
+      //  indirect atom: mark-tracked relocate so bob atoms can rewrite
+      //  their u3a_blob pointer at old_p exactly once.
+      //
+      new_p = _pack_relocate_mark(old_p, &fir_t);
+      *som  = u3a_to_pug(new_p);
+
+      if ( fir_t ) {
+        u3a_atom* atm_u = u3to(u3a_atom, old_p);
+        if ( atm_u->len_w & u3a_blob_flag ) {
+          u3a_relocate_post((u3_post*)&atm_u->buf_w[0]);
+        }
+      }
       return;
     }
 

pkg/noun/allocate.h

@@ -224,6 +224,31 @@
       typedef u3a_64_cell u3a_cell;
 #endif
 
+    /* u3a_blob: loom-resident metadata for a blob file.
+    **
+    **   Stored in u3H->blb_p keyed by bid = (mug_h << 32) | seq_h, with one
+    **   entry per (mug_h, seq_h).  Each bob atom's buf_w[0] is the loom
+    **   offset of its u3a_blob.
+    **
+    **   Single-counter design: the blob file is deleted iff use_w == 0.
+    **   use_w is the sum of three component sources:
+    **     - eve_w: event-log refcount (rebuilt on chop)
+    **     - les_h: active king-held lease count (transient; zeroed on boot)
+    **     - implicit atom cardinality: number of live bob atoms whose
+    **       buf_w[0] points here.  Updated only on atom alloc/free; not
+    **       affected by normal noun-refcount transitions.
+    **
+    **   On boot we zero les_h (and subtract from use_w); eve_w and atom
+    **   cardinality survive the snapshot.
+    */
+      typedef struct __attribute__((aligned(4))) {
+        c3_w  use_w;   //  total refs: eve_w + les_h + atom cardinality
+        c3_w  eve_w;   //  event-log refcount (rebuildable from LMDB)
+        c3_h  les_h;   //  active king-held leases (transient; zeroed on boot)
+        c3_h  mug_h;   //  blob mug — identifies file in .urb/bob
+        c3_h  seq_h;   //  blob seq — identifies file in .urb/bob
+      } u3a_blob;
+
 STATIC_ASSERT( (((c3_w)1) << u3a_min_log) == u3a_minimum,
                "log2 minimum allocation" );
 STATIC_ASSERT( u3a_vits <= u3a_min_log,
@@ -475,6 +500,19 @@ STATIC_ASSERT( u3a_vits <= u3a_min_log,
     /* u3a_is_cell: yes if noun [som] is cell.
     */
 #     define u3a_is_cell(som)    u3a_is_pom(som)
+
+    /* u3a_blob_flag: MSB of u3a_atom.len_w marks an indirect atom as a bob
+    **   (blob reference backed by an on-disk file rather than loom data).
+    **   The remaining bits hold the actual data word count.
+    **   In VERE64, len_w is uint64_t so we use bit 63; in 32-bit we use bit 31.
+    */
+#     ifdef VERE64
+#       define u3a_blob_flag  ((c3_w)0x8000000000000000ULL)
+#       define u3a_blob_mask  ((c3_w)0x7FFFFFFFFFFFFFFFULL)
+#     else
+#       define u3a_blob_flag  ((c3_w)0x80000000U)
+#       define u3a_blob_mask  ((c3_w)0x7FFFFFFFU)
+#     endif
 #     define u3du(som)           u3a_is_cell(som)
 
     /* u3a_h(): get head of cell [som]. Bail if [som] is not cell.
@@ -777,6 +815,49 @@ typedef struct {
             return (pil_u->top_p == u3R->cap_p) ? c3y : c3n;
           }
 
+    /* u3a_is_bob(): yes if [som] is an indirect atom flagged as a bob (blob ref).
+    **   Follows naming convention: u3a_is_cat, u3a_is_pug, u3a_is_pom, u3a_is_bob.
+    */
+    static inline c3_o
+    u3a_is_bob(u3_atom som) {
+      if ( c3n == u3a_is_pug(som) ) return c3n;
+      u3a_atom* atm_u = u3a_to_ptr(som);
+      return (atm_u->len_w & u3a_blob_flag) ? c3y : c3n;
+    }
+
+    /* u3a_bob_blob(): u3a_blob* referenced by a bob atom (via buf_w[0]).
+    */
+    static inline u3a_blob*
+    u3a_bob_blob(u3_atom som) {
+      u3a_atom* atm_u = u3a_to_ptr(som);
+      return (u3a_blob*)u3a_into((u3_post)atm_u->buf_w[0]);
+    }
+
+    /* u3a_bob_mug(): content mug of a bob atom (= blob directory name).
+    **   Stored redundantly on the atom (mug_w) for fast u3r_mug.
+    */
+    static inline c3_h
+    u3a_bob_mug(u3_atom som) {
+      return (c3_h)((u3a_atom*)u3a_to_ptr(som))->mug_w;
+    }
+
+    /* u3a_bob_seq(): sequence number within mug bucket.
+    */
+    static inline c3_h
+    u3a_bob_seq(u3_atom som) {
+      return u3a_bob_blob(som)->seq_h;
+    }
+
+    /* u3a_bob_bid(): blob ID = (mug << 32) | seq.
+    **   On VERE64 this is a direct atom (63 bits).
+    **   On 32-bit this is a c3_d that must go through u3i_chub().
+    */
+    static inline c3_d
+    u3a_bob_bid(u3_atom som) {
+      u3a_blob* blb_u = u3a_bob_blob(som);
+      return ((c3_d)blb_u->mug_h << 32) | (c3_d)blb_u->seq_h;
+    }
+
   /**  Functions.
   **/
 
@@ -983,6 +1064,25 @@ u3a_post_info(u3_post);
           c3_w
           u3a_idle(u3a_road* rod_u);
 
+        /* u3a_blob_get(): look up blb_p entry by (mug_h, seq_h).
+        **   Returns NULL if no entry exists.  RETAINS.
+        */
+          u3a_blob*
+          u3a_blob_get(c3_h mug_h, c3_h seq_h);
+
+        /* u3a_blob_new(): allocate a fresh u3a_blob and install it in
+        **   blb_p under (mug_h, seq_h).  Caller is responsible for
+        **   ensuring no entry exists; struct is zero-initialized.
+        */
+          u3a_blob*
+          u3a_blob_new(c3_h mug_h, c3_h seq_h);
+
+        /* u3a_blob_drop(): remove blb_p entry for (mug_h, seq_h) and
+        **   free the underlying u3a_blob.  No-op if no entry.
+        */
+          void
+          u3a_blob_drop(c3_h mug_h, c3_h seq_h);
+
         /* u3a_ream(): ream free-lists.
         */
           void

pkg/noun/build.zig

@@ -336,6 +336,7 @@ const c_source_files = [_][]const u8{
     "jets/e/fynd_ob.c",
     "jets/e/hmac.c",
     "jets/e/jam.c",
+    "jets/e/jam_shax.c",
     "jets/e/json_de.c",
     "jets/e/json_en.c",
     "jets/e/keccak.c",

pkg/noun/events.h

@@ -128,25 +128,20 @@
       c3_i
       u3e_image_open_any(c3_c* nam_c, c3_c* dir_c, c3_z* len_z, c3_i mod_i);
 
-    /* u3_{32,64}_load(): locate u3v_{32,64}_home in the mapped 32-bit /
-    ** 64-bit image.
+    /* u3_{32,64}_load(): locate u3v_{32,64}_home in the mapped image.
     */
       void
       u3_32_load(c3_z wor_i);
 
       void
       u3_64_load(c3_z wor_i);
 
-    /* u3_migrate_32(): migrate the loaded 64-bit snapshot into the native
-    ** 32-bit loom.  Called from disk.c on a 32-bit vere reading a 64-bit
-    ** snapshot.
+    /* u3_migrate_32(): migrate 64 -> 32.
     */
       void
       u3_migrate_32(c3_d eve_d);
 
-    /* u3_migrate_64(): migrate the loaded 32-bit snapshot into the native
-    ** 64-bit loom.  Called from disk.c on a 64-bit vere reading a 32-bit
-    ** snapshot.
+    /* u3_migrate_64(): migrate 32 -> 64.
     */
       void
       u3_migrate_64(c3_d eve_d);