Skip to content

Commit

Permalink
tcg: define CF_PARALLEL and use it for TB hashing along with CF_COUNT…
Browse files Browse the repository at this point in the history 
…_MASK

This will enable us to decouple code translation from the value
of parallel_cpus at any given time. It will also help us minimize
TB flushes when generating code via EXCP_ATOMIC.

Note that the declaration of parallel_cpus is brought to exec-all.h
to be able to define there the "curr_cflags" inline.

Signed-off-by: Emilio G. Cota <[email protected]>
Signed-off-by: Richard Henderson <[email protected]>
  • Loading branch information
@cota @rth7680
cota authored and rth7680 committed Oct 24, 2017
1 parent e89b28a commit 4e2ca83
Show file tree
Hide file tree
Showing 10 changed files with 65 additions and 39 deletions.
45 changes: 23 additions & 22 deletions accel/tcg/cpu-exec.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,8 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
tb_lock();
tb = tb_gen_code(cpu, orig_tb->pc, orig_tb->cs_base, orig_tb->flags,
max_cycles | CF_NOCACHE
| (ignore_icount ? CF_IGNORE_ICOUNT : 0));
| (ignore_icount ? CF_IGNORE_ICOUNT : 0)
| curr_cflags());
tb->orig_tb = orig_tb;
tb_unlock();

Expand All @@ -225,31 +226,27 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
static void cpu_exec_step(CPUState *cpu)
{
CPUClass *cc = CPU_GET_CLASS(cpu);
CPUArchState *env = (CPUArchState *)cpu->env_ptr;
TranslationBlock *tb;
target_ulong cs_base, pc;
uint32_t flags;
uint32_t cflags = 1 | CF_IGNORE_ICOUNT;

cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
if (sigsetjmp(cpu->jmp_env, 0) == 0) {
mmap_lock();
tb_lock();
tb = tb_gen_code(cpu, pc, cs_base, flags,
1 | CF_NOCACHE | CF_IGNORE_ICOUNT);
tb->orig_tb = NULL;
tb_unlock();
mmap_unlock();
tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags,
cflags & CF_HASH_MASK);
if (tb == NULL) {
mmap_lock();
tb_lock();
tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
tb_unlock();
mmap_unlock();
}

cc->cpu_exec_enter(cpu);
/* execute the generated code */
trace_exec_tb_nocache(tb, pc);
trace_exec_tb(tb, pc);
cpu_tb_exec(cpu, tb);
cc->cpu_exec_exit(cpu);

tb_lock();
tb_phys_invalidate(tb, -1);
tb_free(tb);
tb_unlock();
} else {
/* We may have exited due to another problem here, so we need
* to reset any tb_locks we may have taken but didn't release.
Expand Down Expand Up @@ -281,6 +278,7 @@ struct tb_desc {
CPUArchState *env;
tb_page_addr_t phys_page1;
uint32_t flags;
uint32_t cf_mask;
uint32_t trace_vcpu_dstate;
};

Expand All @@ -294,7 +292,7 @@ static bool tb_cmp(const void *p, const void *d)
tb->cs_base == desc->cs_base &&
tb->flags == desc->flags &&
tb->trace_vcpu_dstate == desc->trace_vcpu_dstate &&
!(atomic_read(&tb->cflags) & CF_INVALID)) {
(tb_cflags(tb) & (CF_HASH_MASK | CF_INVALID)) == desc->cf_mask) {
/* check next page if needed */
if (tb->page_addr[1] == -1) {
return true;
Expand All @@ -313,7 +311,8 @@ static bool tb_cmp(const void *p, const void *d)
}

TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc,
target_ulong cs_base, uint32_t flags)
target_ulong cs_base, uint32_t flags,
uint32_t cf_mask)
{
tb_page_addr_t phys_pc;
struct tb_desc desc;
Expand All @@ -322,11 +321,12 @@ TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc,
desc.env = (CPUArchState *)cpu->env_ptr;
desc.cs_base = cs_base;
desc.flags = flags;
desc.cf_mask = cf_mask;
desc.trace_vcpu_dstate = *cpu->trace_dstate;
desc.pc = pc;
phys_pc = get_page_addr_code(desc.env, pc);
desc.phys_page1 = phys_pc & TARGET_PAGE_MASK;
h = tb_hash_func(phys_pc, pc, flags, *cpu->trace_dstate);
h = tb_hash_func(phys_pc, pc, flags, cf_mask, *cpu->trace_dstate);
return qht_lookup(&tcg_ctx.tb_ctx.htable, tb_cmp, &desc, h);
}

Expand Down Expand Up @@ -373,8 +373,9 @@ static inline TranslationBlock *tb_find(CPUState *cpu,
target_ulong cs_base, pc;
uint32_t flags;
bool acquired_tb_lock = false;
uint32_t cf_mask = curr_cflags();

tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags);
tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, cf_mask);
if (tb == NULL) {
/* mmap_lock is needed by tb_gen_code, and mmap_lock must be
* taken outside tb_lock. As system emulation is currently
Expand All @@ -387,10 +388,10 @@ static inline TranslationBlock *tb_find(CPUState *cpu,
/* There's a chance that our desired tb has been translated while
* taking the locks so we check again inside the lock.
*/
tb = tb_htable_lookup(cpu, pc, cs_base, flags);
tb = tb_htable_lookup(cpu, pc, cs_base, flags, cf_mask);
if (likely(tb == NULL)) {
/* if no translated code available, then translate it now */
tb = tb_gen_code(cpu, pc, cs_base, flags, 0);
tb = tb_gen_code(cpu, pc, cs_base, flags, cf_mask);
}

mmap_unlock();
Expand Down
2 changes: 1 addition & 1 deletion accel/tcg/tcg-runtime.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env)
target_ulong cs_base, pc;
uint32_t flags;

tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags);
tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, curr_cflags());
if (tb == NULL) {
return tcg_ctx.code_gen_epilogue;
}
Expand Down
13 changes: 9 additions & 4 deletions accel/tcg/translate-all.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1101,7 +1101,8 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)

/* remove the TB from the hash list */
phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->trace_vcpu_dstate);
h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK,
tb->trace_vcpu_dstate);
qht_remove(&tcg_ctx.tb_ctx.htable, tb, h);

/* remove the TB from the page list */
Expand Down Expand Up @@ -1245,7 +1246,8 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
}

/* add in the hash table */
h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->trace_vcpu_dstate);
h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK,
tb->trace_vcpu_dstate);
qht_insert(&tcg_ctx.tb_ctx.htable, tb, h);

#ifdef CONFIG_USER_ONLY
Expand Down Expand Up @@ -1548,7 +1550,8 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
/* we generate a block containing just the instruction
modifying the memory. It will ensure that it cannot modify
itself */
tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
tb_gen_code(cpu, current_pc, current_cs_base, current_flags,
1 | curr_cflags());
cpu_loop_exit_noexc(cpu);
}
#endif
Expand Down Expand Up @@ -1666,7 +1669,8 @@ static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
/* we generate a block containing just the instruction
modifying the memory. It will ensure that it cannot modify
itself */
tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
tb_gen_code(cpu, current_pc, current_cs_base, current_flags,
1 | curr_cflags());
/* tb_lock will be reset after cpu_loop_exit_noexc longjmps
* back into the cpu_exec loop. */
return true;
Expand Down Expand Up @@ -1810,6 +1814,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
}

cflags = n | CF_LAST_IO;
cflags |= curr_cflags();
pc = tb->pc;
cs_base = tb->cs_base;
flags = tb->flags;
Expand Down
2 changes: 1 addition & 1 deletion exec.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2476,7 +2476,7 @@ static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
cpu_loop_exit(cpu);
} else {
cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
tb_gen_code(cpu, pc, cs_base, cpu_flags, 1 | curr_cflags());
cpu_loop_exit_noexc(cpu);
}
}
Expand Down
20 changes: 19 additions & 1 deletion include/exec/exec-all.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,9 @@ struct TranslationBlock {
#define CF_USE_ICOUNT 0x20000
#define CF_IGNORE_ICOUNT 0x40000 /* Do not generate icount code */
#define CF_INVALID 0x80000 /* TB is stale. Setters must acquire tb_lock */
#define CF_PARALLEL 0x100000 /* Generate code for a parallel context */
/* cflags' mask for hashing/comparison */
#define CF_HASH_MASK (CF_PARALLEL)

/* Per-vCPU dynamic tracing state used to generate this TB */
uint32_t trace_vcpu_dstate;
Expand Down Expand Up @@ -365,11 +368,26 @@ struct TranslationBlock {
uintptr_t jmp_list_first;
};

extern bool parallel_cpus;

/* Hide the atomic_read to make code a little easier on the eyes */
static inline uint32_t tb_cflags(const TranslationBlock *tb)
{
return atomic_read(&tb->cflags);
}

/* current cflags for hashing/comparison */
static inline uint32_t curr_cflags(void)
{
return parallel_cpus ? CF_PARALLEL : 0;
}

void tb_free(TranslationBlock *tb);
void tb_flush(CPUState *cpu);
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc,
target_ulong cs_base, uint32_t flags);
target_ulong cs_base, uint32_t flags,
uint32_t cf_mask);
void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);

/* GETPC is the true target of the return instruction that we'll execute. */
Expand Down
9 changes: 6 additions & 3 deletions include/exec/tb-hash-xx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@
* xxhash32, customized for input variables that are not guaranteed to be
* contiguous in memory.
*/
static inline
uint32_t tb_hash_func6(uint64_t a0, uint64_t b0, uint32_t e, uint32_t f)
static inline uint32_t
tb_hash_func7(uint64_t a0, uint64_t b0, uint32_t e, uint32_t f, uint32_t g)
{
uint32_t v1 = TB_HASH_XX_SEED + PRIME32_1 + PRIME32_2;
uint32_t v2 = TB_HASH_XX_SEED + PRIME32_2;
Expand Down Expand Up @@ -78,14 +78,17 @@ uint32_t tb_hash_func6(uint64_t a0, uint64_t b0, uint32_t e, uint32_t f)
v4 *= PRIME32_1;

h32 = rol32(v1, 1) + rol32(v2, 7) + rol32(v3, 12) + rol32(v4, 18);
h32 += 24;
h32 += 28;

h32 += e * PRIME32_3;
h32 = rol32(h32, 17) * PRIME32_4;

h32 += f * PRIME32_3;
h32 = rol32(h32, 17) * PRIME32_4;

h32 += g * PRIME32_3;
h32 = rol32(h32, 17) * PRIME32_4;

h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
Expand Down
4 changes: 2 additions & 2 deletions include/exec/tb-hash.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)

static inline
uint32_t tb_hash_func(tb_page_addr_t phys_pc, target_ulong pc, uint32_t flags,
uint32_t trace_vcpu_dstate)
uint32_t cf_mask, uint32_t trace_vcpu_dstate)
{
return tb_hash_func6(phys_pc, pc, flags, trace_vcpu_dstate);
return tb_hash_func7(phys_pc, pc, flags, cf_mask, trace_vcpu_dstate);
}

#endif
6 changes: 3 additions & 3 deletions include/exec/tb-lookup.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
/* Might cause an exception, so have a longjmp destination ready */
static inline TranslationBlock *
tb_lookup__cpu_state(CPUState *cpu, target_ulong *pc, target_ulong *cs_base,
uint32_t *flags)
uint32_t *flags, uint32_t cf_mask)
{
CPUArchState *env = (CPUArchState *)cpu->env_ptr;
TranslationBlock *tb;
Expand All @@ -35,10 +35,10 @@ tb_lookup__cpu_state(CPUState *cpu, target_ulong *pc, target_ulong *cs_base,
tb->cs_base == *cs_base &&
tb->flags == *flags &&
tb->trace_vcpu_dstate == *cpu->trace_dstate &&
!(atomic_read(&tb->cflags) & CF_INVALID))) {
(tb_cflags(tb) & (CF_HASH_MASK | CF_INVALID)) == cf_mask)) {
return tb;
}
tb = tb_htable_lookup(cpu, *pc, *cs_base, *flags);
tb = tb_htable_lookup(cpu, *pc, *cs_base, *flags, cf_mask);
if (tb == NULL) {
return NULL;
}
Expand Down
1 change: 0 additions & 1 deletion tcg/tcg.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,6 @@ struct TCGContext {
};

extern TCGContext tcg_ctx;
extern bool parallel_cpus;

static inline size_t temp_idx(TCGTemp *ts)
{
Expand Down
2 changes: 1 addition & 1 deletion tests/qht-bench.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ static bool is_equal(const void *obj, const void *userp)

static inline uint32_t h(unsigned long v)
{
return tb_hash_func6(v, 0, 0, 0);
return tb_hash_func7(v, 0, 0, 0, 0);
}

/*
Expand Down

0 comments on commit 4e2ca83

Please sign in to comment.