RAMList: Upstream dirty tracking implementation.

This modifies the implementation of dirty RAM tracking to follow
the upstream implementation, which uses 3 parallel bitmap arrays
instead of a single one merging all components.

Change-Id: I75c418444310f159973840aa765db65294859702

Author: digit-android

Makefile.target

@@ -311,6 +311,8 @@ common_LOCAL_SRC_FILES := \
     android/opengles.c \
     android/user-events-qemu.c \
     hw/core/loader.c \
+    util/bitmap.c \
+    util/bitops.c \
     ui/keymaps.c \
     util/qemu-timer-common.c \
     util/iov.c \

arch_init.c

@@ -126,13 +126,14 @@ static int ram_save_block(QEMUFile *f)
     current_addr = block->offset + offset;
 
     do {
-        if (cpu_physical_memory_get_dirty(current_addr, MIGRATION_DIRTY_FLAG)) {
+        if (cpu_physical_memory_get_dirty(current_addr, TARGET_PAGE_SIZE,
+                                          DIRTY_MEMORY_MIGRATION)) {
             uint8_t *p;
             int cont = (block == last_block) ? RAM_SAVE_FLAG_CONTINUE : 0;
 
             cpu_physical_memory_reset_dirty(current_addr,
-                                            current_addr + TARGET_PAGE_SIZE,
-                                            MIGRATION_DIRTY_FLAG);
+                                            TARGET_PAGE_SIZE,
+                                            DIRTY_MEMORY_MIGRATION);
 
             p = block->host + offset;
 
@@ -188,7 +189,8 @@ static ram_addr_t ram_save_remaining(void)
         ram_addr_t addr;
         for (addr = block->offset; addr < block->offset + block->length;
              addr += TARGET_PAGE_SIZE) {
-            if (cpu_physical_memory_get_dirty(addr, MIGRATION_DIRTY_FLAG)) {
+            if (cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
+                                              DIRTY_MEMORY_MIGRATION)) {
                 count++;
             }
         }
@@ -279,9 +281,10 @@ int ram_save_live(QEMUFile *f, int stage, void *opaque)
         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
             for (addr = block->offset; addr < block->offset + block->length;
                  addr += TARGET_PAGE_SIZE) {
-                if (!cpu_physical_memory_get_dirty(addr,
-                                                   MIGRATION_DIRTY_FLAG)) {
-                    cpu_physical_memory_set_dirty(addr);
+                if (!cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
+                                                   DIRTY_MEMORY_MIGRATION)) {
+                    cpu_physical_memory_set_dirty_flag(
+                            addr, DIRTY_MEMORY_MIGRATION);
                 }
             }
         }

cputlb.c

@@ -118,17 +118,16 @@ void tlb_flush_page(CPUArchState *env, target_ulong addr)
    can be detected */
 void tlb_protect_code(ram_addr_t ram_addr)
 {
-    cpu_physical_memory_reset_dirty(ram_addr,
-                                    ram_addr + TARGET_PAGE_SIZE,
-                                    CODE_DIRTY_FLAG);
+    cpu_physical_memory_reset_dirty(ram_addr, TARGET_PAGE_SIZE,
+                                    DIRTY_MEMORY_CODE);
 }
 
 /* update the TLB so that writes in physical page 'phys_addr' are no longer
    tested for self modifying code */
 void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr,
                              target_ulong vaddr)
 {
-    cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
 }
 
 static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe)
@@ -150,6 +149,26 @@ void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
     }
 }
 
+void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
+{
+    CPUState *cpu;
+    CPUArchState *env;
+
+    CPU_FOREACH(cpu) {
+        int mmu_idx;
+
+        env = cpu->env_ptr;
+        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+            unsigned int i;
+
+            for (i = 0; i < CPU_TLB_SIZE; i++) {
+                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
+                                      start1, length);
+            }
+        }
+    }
+}
+
 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 {
     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
@@ -288,7 +307,7 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr,
             /* Write access calls the I/O callback.  */
             te->addr_write = address | TLB_MMIO;
         } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
-                   !cpu_physical_memory_is_dirty(pd)) {
+                   cpu_physical_memory_is_clean(pd)) {
             te->addr_write = address | TLB_NOTDIRTY;
         } else {
             te->addr_write = address;

exec.c

@@ -39,6 +39,7 @@
 #include "hw/hw.h"
 #include "hw/qdev.h"
 #include "hw/xen/xen.h"
+#include "qemu/bitmap.h"
 #include "qemu/osdep.h"
 #include "qemu/tls.h"
 #include "sysemu/kvm.h"
@@ -49,6 +50,7 @@
 #if defined(CONFIG_USER_ONLY)
 #include <qemu.h>
 #endif
+#include "translate-all.h"
 
 //#define DEBUG_SUBPAGE
 
@@ -532,41 +534,28 @@ static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
     return block;
 }
 
-/* Note: start and end must be within the same ram block.  */
-void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
-                                     int dirty_flags)
+static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
 {
-    unsigned long length, start1;
-    int i;
+    ram_addr_t end = TARGET_PAGE_ALIGN(start + length);
 
     start &= TARGET_PAGE_MASK;
-    end = TARGET_PAGE_ALIGN(end);
 
-    length = end - start;
+    RAMBlock* block = qemu_get_ram_block(start);
+    assert(block == qemu_get_ram_block(end - 1));
+    uintptr_t start1 = (uintptr_t)block->host + (start - block->offset);
+    cpu_tlb_reset_dirty_all(start1, length);
+}
+
+/* Note: start and end must be within the same ram block.  */
+void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
+                                     unsigned client)
+{
     if (length == 0)
         return;
-    cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
-
-    /* we modify the TLB cache so that the dirty bit will be set again
-       when accessing the range */
-    start1 = (unsigned long)qemu_safe_ram_ptr(start);
-    /* Chek that we don't span multiple blocks - this breaks the
-       address comparisons below.  */
-    if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
-            != (end - 1) - start) {
-        abort();
-    }
+    cpu_physical_memory_clear_dirty_range(start, length, client);
 
-    CPUState *cpu;
-    CPU_FOREACH(cpu) {
-        int mmu_idx;
-        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-            for(i = 0; i < CPU_TLB_SIZE; i++) {
-                CPUArchState* env = cpu->env_ptr;
-                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
-                                      start1, length);
-            }
-        }
+    if (tcg_enabled()) {
+        tlb_reset_dirty_range_all(start, length);
     }
 }
 
@@ -603,7 +592,7 @@ static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
         p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
             + tlb_entry->addend);
         ram_addr = qemu_ram_addr_from_host_nofail(p);
-        if (!cpu_physical_memory_is_dirty(ram_addr)) {
+        if (cpu_physical_memory_is_clean(ram_addr)) {
             tlb_entry->addr_write |= TLB_NOTDIRTY;
         }
     }
@@ -1079,6 +1068,9 @@ ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
                                    ram_addr_t size, void *host)
 {
     RAMBlock *block, *new_block;
+    ram_addr_t old_ram_size, new_ram_size;
+
+    old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
 
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
@@ -1166,11 +1158,17 @@ ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
     ram_list.version++;
     qemu_mutex_unlock_ramlist();
 
-    ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
-                                       last_ram_offset() >> TARGET_PAGE_BITS);
-    memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
-           0xff, size >> TARGET_PAGE_BITS);
-    //cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
+    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    if (new_ram_size > old_ram_size) {
+        int i;
+        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+            ram_list.dirty_memory[i] =
+                bitmap_zero_extend(ram_list.dirty_memory[i],
+                                   old_ram_size, new_ram_size);
+       }
+    }
+    cpu_physical_memory_set_dirty_range(new_block->offset, size);
 
     qemu_ram_setup_dump(new_block->host, size);
     //qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
@@ -1463,61 +1461,52 @@ static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
 static void notdirty_mem_writeb(void *opaque, hwaddr ram_addr,
                                 uint32_t val)
 {
-    int dirty_flags;
-    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
-#if !defined(CONFIG_USER_ONLY)
+    if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
         tb_invalidate_phys_page_fast0(ram_addr, 1);
-        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-#endif
     }
     stb_p(qemu_get_ram_ptr(ram_addr), val);
-    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
-    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
     /* we remove the notdirty callback only if the code has been
        flushed */
-    if (dirty_flags == 0xff)
-        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
+    if (!cpu_physical_memory_is_clean(ram_addr)) {
+        CPUArchState *env = current_cpu->env_ptr;
+        tlb_set_dirty(env, env->mem_io_vaddr);
+    }
 }
 
 static void notdirty_mem_writew(void *opaque, hwaddr ram_addr,
                                 uint32_t val)
 {
-    int dirty_flags;
-    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
-#if !defined(CONFIG_USER_ONLY)
-        tb_invalidate_phys_page_fast0(ram_addr, 2);
-        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-#endif
+    if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
+        tb_invalidate_phys_page_fast0(ram_addr, 1);
     }
     stw_p(qemu_get_ram_ptr(ram_addr), val);
-    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
-    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
     /* we remove the notdirty callback only if the code has been
        flushed */
-    if (dirty_flags == 0xff)
-        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
+    if (!cpu_physical_memory_is_clean(ram_addr)) {
+        CPUArchState *env = current_cpu->env_ptr;
+        tlb_set_dirty(env, env->mem_io_vaddr);
+    }
 }
 
 static void notdirty_mem_writel(void *opaque, hwaddr ram_addr,
                                 uint32_t val)
 {
-    int dirty_flags;
-    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
-#if !defined(CONFIG_USER_ONLY)
-        tb_invalidate_phys_page_fast0(ram_addr, 4);
-        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-#endif
+    if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
+        tb_invalidate_phys_page_fast0(ram_addr, 1);
     }
     stl_p(qemu_get_ram_ptr(ram_addr), val);
-    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
-    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
     /* we remove the notdirty callback only if the code has been
        flushed */
-    if (dirty_flags == 0xff)
-        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
+    if (!cpu_physical_memory_is_clean(ram_addr)) {
+        CPUArchState *env = current_cpu->env_ptr;
+        tlb_set_dirty(env, env->mem_io_vaddr);
+    }
 }
 
 static CPUReadMemoryFunc * const error_mem_read[3] = {
@@ -1532,16 +1521,6 @@ static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
     notdirty_mem_writel,
 };
 
-static void tb_check_watchpoint(CPUArchState* env)
-{
-    TranslationBlock *tb = tb_find_pc(env->mem_io_pc);
-    if (!tb) {
-        cpu_abort(env, "check_watchpoint: could not find TB for "
-                  "pc=%p", (void *)env->mem_io_pc);
-    }
-    cpu_restore_state(env, env->mem_io_pc);
-    tb_phys_invalidate(tb, -1);
-}
 
 /* Generate a debug exception if a watchpoint has been hit.  */
 static void check_watchpoint(int offset, int len_mask, int flags)
@@ -1919,11 +1898,12 @@ void cpu_physical_memory_rw(hwaddr addr, void *buf,
 static void invalidate_and_set_dirty(hwaddr addr,
                                      hwaddr length)
 {
-    if (!cpu_physical_memory_is_dirty(addr)) {
+    if (cpu_physical_memory_is_clean(addr)) {
         /* invalidate code */
         tb_invalidate_phys_page_range(addr, addr + length, 0);
         /* set dirty bit */
-        cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
+        cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
+        cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
     }
 }
 
@@ -2435,12 +2415,13 @@ void stl_phys_notdirty(hwaddr addr, uint32_t val)
         stl_p(ptr, val);
 
         if (unlikely(in_migration)) {
-            if (!cpu_physical_memory_is_dirty(addr1)) {
+            if (cpu_physical_memory_is_clean(addr1)) {
                 /* invalidate code */
                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                 /* set dirty bit */
-                cpu_physical_memory_set_dirty_flags(
-                    addr1, (0xff & ~CODE_DIRTY_FLAG));
+                cpu_physical_memory_set_dirty_flag(addr1,
+                                                   DIRTY_MEMORY_MIGRATION);
+                cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
             }
         }
     }
@@ -2596,12 +2577,12 @@ static inline void stw_phys_internal(hwaddr addr, uint32_t val,
             stw_p(ptr, val);
             break;
         }
-        if (!cpu_physical_memory_is_dirty(addr1)) {
+        if (cpu_physical_memory_is_clean(addr1)) {
             /* invalidate code */
             tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
             /* set dirty bit */
-            cpu_physical_memory_set_dirty_flags(addr1,
-                (0xff & ~CODE_DIRTY_FLAG));
+            cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_MIGRATION);
+            cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
         }
     }
 }