Fix T2C race conditions with immutable cache

The tier-2 JIT compiler (T2C) had critical race conditions, and multiple
threads were accessing and modifying shared block structures without
proper synchronization.

Root causes identified:
- T2C compilation thread held pointers to blocks that could be evicted
- Block reuse from memory pool caused stale function pointer access
- Complex synchronization with hot2 flag and reference counting was
  insufficient

Solution implemented:
1. Created separate immutable T2C cache (t2c_cache) for compiled blocks
   - T2C entries are never modified once created
   - Eliminates all race conditions on compiled function access
2. Deep copy block IR chains when queuing for compilation
   - Compilation thread works on copies, not live blocks
   - Blocks can be safely evicted during compilation
3. Simplified synchronization logic
   - Removed hot2 flag and complex reference counting
   - Use simple compiled flag only to prevent re-queuing
   - Clear flag when T2C code becomes available
4. Fixed eviction logic to properly check compilation status

Author: jserv

src/emulate.c

@@ -4,6 +4,8 @@
  */
 
 #include <assert.h>
+#include <sched.h>
+#include <stdatomic.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -278,12 +280,13 @@ static block_t *block_alloc(riscv_t *rv)
 #if RV32_HAS(JIT)
     block->translatable = true;
     block->hot = false;
-    block->hot2 = false;
     block->has_loops = false;
     block->n_invoke = 0;
     INIT_LIST_HEAD(&block->list);
 #if RV32_HAS(T2C)
-    block->compiled = false;
+    /* Initialize T2C compilation flag */
+    atomic_store_explicit(&block->compiled, false, memory_order_relaxed);
+    atomic_store_explicit(&block->func, NULL, memory_order_relaxed);
 #endif
 #endif
     return block;
@@ -918,6 +921,29 @@ static block_t *block_find_or_translate(riscv_t *rv)
         return next_blk;
     }
 
+#if RV32_HAS(T2C)
+    /* Avoid evicting blocks being compiled */
+    if (atomic_load_explicit(&replaced_blk->compiled, memory_order_acquire)) {
+        /* This block is being compiled - do not evict it.
+         * Return NULL to signal cache is full.
+         */
+        pthread_mutex_unlock(&rv->cache_lock);
+
+        /* Free the newly translated block */
+        for (rv_insn_t *ir = next_blk->ir_head, *next_ir; ir; ir = next_ir) {
+            next_ir = ir->next;
+            free(ir->fuse);
+            mpool_free(rv->block_ir_mp, ir);
+        }
+        list_del_init(&next_blk->list);
+        atomic_store_explicit(&next_blk->func, NULL, memory_order_relaxed);
+        mpool_free(rv->block_mp, next_blk);
+
+        return NULL;
+    }
+    /* Allow evicting blocks that are not yet compiled */
+#endif
+
     if (prev == replaced_blk)
         prev = NULL;
 
@@ -930,43 +956,60 @@ static block_t *block_find_or_translate(riscv_t *rv)
         rv_insn_t *taken = entry->ir_tail->branch_taken,
                   *untaken = entry->ir_tail->branch_untaken;
 
-        if (taken == replaced_blk_entry) {
+        if (taken == replaced_blk_entry)
             entry->ir_tail->branch_taken = NULL;
-        }
-        if (untaken == replaced_blk_entry) {
+        if (untaken == replaced_blk_entry)
             entry->ir_tail->branch_untaken = NULL;
-        }
 
         /* upadte JALR LUT */
-        if (!entry->ir_tail->branch_table) {
+        if (!entry->ir_tail->branch_table)
             continue;
-        }
 
-        /**
-         * TODO: upadate all JALR instructions which references to this
-         * basic block as the destination.
+        /* TODO: upadate all JALR instructions which references to this basic
+         * block as the destination.
          */
     }
 
-    /* free IRs in replaced block */
-    for (rv_insn_t *ir = replaced_blk->ir_head, *next_ir; ir != NULL;
-         ir = next_ir) {
-        next_ir = ir->next;
+#if RV32_HAS(T2C)
+    /* Double-check - do not evict if being compiled */
+    if (atomic_load_explicit(&replaced_blk->compiled, memory_order_acquire)) {
+        /* Block is being compiled - cannot evict */
+        pthread_mutex_unlock(&rv->cache_lock);
 
-        if (ir->fuse)
+        /* Free the newly translated block */
+        for (rv_insn_t *ir = next_blk->ir_head, *next_ir; ir; ir = next_ir) {
+            next_ir = ir->next;
             free(ir->fuse);
+            mpool_free(rv->block_ir_mp, ir);
+        }
+        list_del_init(&next_blk->list);
+        atomic_store_explicit(&next_blk->func, NULL, memory_order_relaxed);
+        mpool_free(rv->block_mp, next_blk);
 
+        return NULL;
+    }
+#endif
+    /* At this point, block is not compiled - safe to evict */
+    /* Free the replaced block
+     */
+    for (rv_insn_t *ir = replaced_blk->ir_head, *next_ir; ir; ir = next_ir) {
+        next_ir = ir->next;
+        free(ir->fuse);
         mpool_free(rv->block_ir_mp, ir);
     }
 
     list_del_init(&replaced_blk->list);
+#if RV32_HAS(T2C)
+    /* Clear atomic fields before returning to pool */
+    atomic_store_explicit(&replaced_blk->func, NULL, memory_order_relaxed);
+    atomic_store_explicit(&replaced_blk->compiled, false, memory_order_relaxed);
+#endif
     mpool_free(rv->block_mp, replaced_blk);
 #if RV32_HAS(T2C)
     pthread_mutex_unlock(&rv->cache_lock);
 #endif
 #endif
 
-    assert(next_blk);
     return next_blk;
 }
 
@@ -1077,6 +1120,15 @@ void rv_step(void *arg)
          * and move onto the next block.
          */
         block_t *block = block_find_or_translate(rv);
+#if RV32_HAS(T2C)
+        if (!block) {
+            /* Cache is full of compiled blocks.
+             * Try again after yielding to allow T2C thread to complete.
+             */
+            sched_yield();
+            continue;
+        }
+#endif
         /* by now, a block should be available */
         assert(block);
 
@@ -1120,20 +1172,88 @@ void rv_step(void *arg)
         last_pc = rv->PC;
 #if RV32_HAS(JIT)
 #if RV32_HAS(T2C)
-        /* executed through the tier-2 JIT compiler */
-        if (block->hot2) {
-            ((exec_t2c_func_t) block->func)(rv);
+        /* Check if T2C compiled code exists for this block */
+        t2c_entry_t *t2c_entry =
+            cache_get(rv->t2c_cache, block->pc_start, false);
+        if (t2c_entry && t2c_entry->func) {
+            /* Clear compiled flag now that T2C code is available
+             * This allows the block to be evicted if needed */
+            atomic_store_explicit(&block->compiled, false,
+                                  memory_order_relaxed);
+
+            /* Execute the compiled function - no synchronization needed
+             * as T2C entries are immutable
+             */
+            exec_t2c_func_t func = (exec_t2c_func_t) t2c_entry->func;
+            func(rv);
             prev = NULL;
             continue;
-        } /* check if invoking times of t1 generated code exceed threshold */
-        else if (!block->compiled && block->n_invoke >= THRESHOLD) {
-            block->compiled = true;
+        }
+        /* check if invoking times of t1 generated code exceed threshold */
+        if (!atomic_load_explicit(&block->compiled, memory_order_acquire) &&
+            block->n_invoke >= THRESHOLD) {
+            /* Mark block as queued for compilation to avoid re-queueing */
+            atomic_store_explicit(&block->compiled, true, memory_order_release);
+
             queue_entry_t *entry = malloc(sizeof(queue_entry_t));
-            entry->block = block;
-            pthread_mutex_lock(&rv->wait_queue_lock);
-            list_add(&entry->list, &rv->wait_queue);
-            pthread_mutex_unlock(&rv->wait_queue_lock);
+            if (entry) {
+                /* Copy block metadata */
+                entry->pc_start = block->pc_start;
+                entry->pc_end = block->pc_end;
+                entry->n_insn = block->n_insn;
+#if RV32_HAS(SYSTEM)
+                entry->satp = block->satp;
+#endif
+                /* Deep copy the IR chain */
+                entry->ir_head_copy = NULL;
+                rv_insn_t **copy_ptr = &entry->ir_head_copy;
+                for (rv_insn_t *ir = block->ir_head; ir; ir = ir->next) {
+                    rv_insn_t *ir_copy = malloc(sizeof(rv_insn_t));
+                    if (!ir_copy) {
+                        /* Clean up on failure */
+                        for (rv_insn_t *tmp = entry->ir_head_copy, *next; tmp;
+                             tmp = next) {
+                            next = tmp->next;
+                            free(tmp->fuse);
+                            free(tmp);
+                        }
+                        free(entry);
+                        atomic_store_explicit(&block->compiled, false,
+                                              memory_order_release);
+                        goto skip_t2c;
+                    }
+                    memcpy(ir_copy, ir, sizeof(rv_insn_t));
+                    /* Copy fuse data if present */
+                    if (ir->fuse) {
+                        size_t fuse_size = ir->imm2 * sizeof(opcode_fuse_t);
+                        ir_copy->fuse = malloc(fuse_size);
+                        if (ir_copy->fuse)
+                            memcpy(ir_copy->fuse, ir->fuse, fuse_size);
+                    }
+                    /* Clear branch pointers as they are not needed for
+                     * compilation.
+                     */
+                    ir_copy->branch_taken = NULL;
+                    ir_copy->branch_untaken = NULL;
+                    ir_copy->branch_table = NULL;
+                    /* Link the copy */
+                    ir_copy->next = NULL;
+                    *copy_ptr = ir_copy;
+                    copy_ptr = &ir_copy->next;
+                }
+
+                pthread_mutex_lock(&rv->wait_queue_lock);
+                list_add(&entry->list, &rv->wait_queue);
+                pthread_cond_signal(&rv->wait_queue_cond);
+                pthread_mutex_unlock(&rv->wait_queue_lock);
+                /* Keep compiled flag set to prevent re-queueing */
+            } else {
+                /* Failed to allocate - clear compiled flag */
+                atomic_store_explicit(&block->compiled, false,
+                                      memory_order_release);
+            }
         }
+    skip_t2c:; /* Empty statement for label */
 #endif
         /* executed through the tier-1 JIT compiler */
         struct jit_state *state = rv->jit_state;
@@ -1165,6 +1285,12 @@ void rv_step(void *arg)
 #endif
         /* execute the block by interpreter */
         const rv_insn_t *ir = block->ir_head;
+        /* should never happen */
+        if (unlikely(!ir || !ir->impl)) {
+            /* Fatal error: block is corrupted. Break to avoid infinite loop */
+            prev = NULL;
+            break;
+        }
         if (unlikely(!ir->impl(rv, ir, rv->csr_cycle, rv->PC))) {
             /* block should not be extended if execption handler invoked */
             prev = NULL;

src/jit.h

@@ -58,6 +58,7 @@ typedef void (*exec_block_func_t)(riscv_t *rv, uintptr_t);
 
 #if RV32_HAS(T2C)
 void t2c_compile(riscv_t *, block_t *);
+void t2c_compile_from_entry(riscv_t *, queue_entry_t *);
 typedef void (*exec_t2c_func_t)(riscv_t *);
 
 /* The jit-cache records the program counters and the entries of executable

src/riscv.c

@@ -6,6 +6,7 @@
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdatomic.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -206,19 +207,36 @@ static pthread_t t2c_thread;
 static void *t2c_runloop(void *arg)
 {
     riscv_t *rv = (riscv_t *) arg;
+    pthread_mutex_lock(&rv->wait_queue_lock);
     while (!rv->quit) {
-        if (!list_empty(&rv->wait_queue)) {
-            queue_entry_t *entry =
-                list_last_entry(&rv->wait_queue, queue_entry_t, list);
-            pthread_mutex_lock(&rv->wait_queue_lock);
-            list_del_init(&entry->list);
-            pthread_mutex_unlock(&rv->wait_queue_lock);
-            pthread_mutex_lock(&rv->cache_lock);
-            t2c_compile(rv, entry->block);
-            pthread_mutex_unlock(&rv->cache_lock);
-            free(entry);
+        /* Wait for work or quit signal */
+        while (list_empty(&rv->wait_queue) && !rv->quit)
+            pthread_cond_wait(&rv->wait_queue_cond, &rv->wait_queue_lock);
+
+        /* Check if we should quit */
+        if (rv->quit)
+            break;
+
+        /* Process the queue entry */
+        queue_entry_t *entry =
+            list_last_entry(&rv->wait_queue, queue_entry_t, list);
+        list_del_init(&entry->list);
+        pthread_mutex_unlock(&rv->wait_queue_lock);
+
+        /* Compile using the copied block data - thread safe */
+        t2c_compile_from_entry(rv, entry);
+
+        /* Free the copied IR chain */
+        for (rv_insn_t *ir = entry->ir_head_copy, *next; ir; ir = next) {
+            next = ir->next;
+            free(ir->fuse);
+            free(ir);
         }
+        free(entry);
+
+        pthread_mutex_lock(&rv->wait_queue_lock);
     }
+    pthread_mutex_unlock(&rv->wait_queue_lock);
     return NULL;
 }
 #endif
@@ -732,7 +750,11 @@ riscv_t *rv_create(riscv_user_t rv_attr)
     /* prepare wait queue. */
     pthread_mutex_init(&rv->wait_queue_lock, NULL);
     pthread_mutex_init(&rv->cache_lock, NULL);
+    pthread_cond_init(&rv->wait_queue_cond, NULL);
     INIT_LIST_HEAD(&rv->wait_queue);
+    /* Create separate T2C cache for compiled blocks */
+    rv->t2c_cache = cache_create(BLOCK_MAP_CAPACITY_BITS);
+    assert(rv->t2c_cache);
     /* activate the background compilation thread. */
     pthread_create(&t2c_thread, NULL, t2c_runloop, rv);
 #endif
@@ -836,11 +858,16 @@ void rv_delete(riscv_t *rv)
     block_map_destroy(rv);
 #else
 #if RV32_HAS(T2C)
+    pthread_mutex_lock(&rv->wait_queue_lock);
     rv->quit = true;
+    pthread_cond_signal(&rv->wait_queue_cond);
+    pthread_mutex_unlock(&rv->wait_queue_lock);
     pthread_join(t2c_thread, NULL);
     pthread_mutex_destroy(&rv->wait_queue_lock);
     pthread_mutex_destroy(&rv->cache_lock);
+    pthread_cond_destroy(&rv->wait_queue_cond);
     jit_cache_exit(rv->jit_cache);
+    cache_free(rv->t2c_cache);
 #endif
     jit_state_exit(rv->jit_state);
     cache_free(rv->block_cache);