izzy/xemu
0
0
Fork 0
mirror of https://github.com/izzy2lost/xemu.git synced 2026-07-06 00:20:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'pull-tcg-20230117' of https://gitlab.com/rth7680/qemu into staging

Browse Source 
tcg: Fix race conditions in (most) goto_tb implementations

# -----BEGIN PGP SIGNATURE-----
#
# iQFRBAABCgA7FiEEekgeeIaLTbaoWgXAZN846K9+IV8FAmPHKmEdHHJpY2hhcmQu
# aGVuZGVyc29uQGxpbmFyby5vcmcACgkQZN846K9+IV+W+gf/cPFM+cY5QrC/ziVI
# Dbqc9Z+B4QVNoQjA+Qv9uL+ErjJ1zXuaS512NPHaL5nkjqzzbU4rYiIk1UET/vDJ
# 4RDw7o4pia3umt68zf9BfxPhkCqB+Zfdo/iolszL9OBpufPGD9NRjJn1qfM6Mpbd
# 38s6InRrmCWlGT3fsJq2cGzmT64dGDIovbHbJ5Y0EbtWBIcP99w2YfucclMB5AFI
# j6gX8jOKCfuN1Hru8waH77X2B0Jv36xlfRV9GLrwo/FQ/aiCyJLl5Hxuwm0adSql
# PBhDSdTXOTiNFmtnBUhaV1orgYKGGwWWfrGmzd0cGg/UcgHHDuJaoeuZHGNJlYai
# PFLW/Q==
# =fMV8
# -----END PGP SIGNATURE-----
# gpg: Signature made Tue 17 Jan 2023 23:08:17 GMT
# gpg:                using RSA key 7A481E78868B4DB6A85A05C064DF38E8AF7E215F
# gpg:                issuer "richard.henderson@linaro.org"
# gpg: Good signature from "Richard Henderson <richard.henderson@linaro.org>" [full]
# Primary key fingerprint: 7A48 1E78 868B 4DB6 A85A  05C0 64DF 38E8 AF7E 215F

* tag 'pull-tcg-20230117' of https://gitlab.com/rth7680/qemu: (22 commits)
  tcg/riscv: Implement direct branch for goto_tb
  tcg/riscv: Introduce OPC_NOP
  tcg/arm: Implement direct branch for goto_tb
  tcg/sparc64: Reorg goto_tb implementation
  tcg/sparc64: Remove USE_REG_TB
  tcg/ppc: Reorg goto_tb implementation
  tcg/aarch64: Reorg goto_tb implementation
  tcg: Remove TCG_TARGET_HAS_direct_jump
  tcg: Always define tb_target_set_jmp_target
  tcg: Move tb_target_set_jmp_target declaration to tcg.h
  tcg: Change tb_target_set_jmp_target arguments
  tcg: Add TranslationBlock.jmp_insn_offset
  tcg: Add gen_tb to TCGContext
  tcg: Rename TB_JMP_RESET_OFFSET_INVALID to TB_JMP_OFFSET_INVALID
  tcg: Split out tcg_out_goto_tb
  tcg: Introduce get_jmp_target_addr
  tcg: Introduce set_jmp_insn_offset
  tcg: Replace asserts on tcg_jmp_insn_offset
  tcg/sparc64: Remove unused goto_tb code for indirect jump
  tcg/ppc: Remove unused goto_tb code for indirect jump
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell
2023-01-19 11:56:51 +00:00
parent 701ed34833 493c9b19a7
commit ef4f031fab
26 changed files with 528 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
accel/tcg/cpu-exec.c
+12 -9
View File
@@ -572,15 +572,18 @@ void cpu_exec_step_atomic(CPUState *cpu)
void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
{
if (TCG_TARGET_HAS_direct_jump) {
uintptr_t offset = tb->jmp_target_arg[n];
uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
uintptr_t jmp_rx = tc_ptr + offset;
uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
} else {
tb->jmp_target_arg[n] = addr;
}
/*
* Get the rx view of the structure, from which we find the
* executable code address, and tb_target_set_jmp_target can
* produce a pc-relative displacement to jmp_target_addr[n].
*/
const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
uintptr_t offset = tb->jmp_insn_offset[n];
uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
tb->jmp_target_addr[n] = addr;
tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
}
static inline void tb_add_jump(TranslationBlock *tb, int n,
accel/tcg/translate-all.c
+5 -5
View File
@@ -350,7 +350,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
tb->trace_vcpu_dstate = *cpu->trace_dstate;
tb_set_page_addr0(tb, phys_pc);
tb_set_page_addr1(tb, -1);
tcg_ctx->tb_cflags = cflags;
tcg_ctx->gen_tb = tb;
tb_overflow:
#ifdef CONFIG_PROFILER
@@ -508,10 +508,10 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
tb->jmp_dest[1] = (uintptr_t)NULL;
/* init original jump addresses which have been set during tcg_gen_code() */
if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
tb_reset_jump(tb, 0);
}
if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
tb_reset_jump(tb, 1);
}
@@ -693,9 +693,9 @@ static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
if (tb_page_addr1(tb) != -1) {
tst->cross_page++;
}
if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
tst->direct_jmp_count++;
if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
tst->direct_jmp2_count++;
}
}
include/exec/exec-all.h
+3 -2
View File
@@ -585,9 +585,10 @@ struct TranslationBlock {
* setting one of the jump targets (or patching the jump instruction). Only
* two of such jumps are supported.
*/
#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */
uint16_t jmp_reset_offset[2]; /* offset of original jump target */
#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */
uintptr_t jmp_target_arg[2]; /* target address or offset */
uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */
uintptr_t jmp_target_addr[2]; /* target address */
/*
* Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps.
include/tcg/tcg.h
+6 -8
View File
@@ -552,20 +552,15 @@ struct TCGContext {
int nb_indirects;
int nb_ops;
/* goto_tb support */
tcg_insn_unit *code_buf;
uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */
uintptr_t *tb_jmp_insn_offset; /* tb->jmp_target_arg if direct_jump */
uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_arg if !direct_jump */
TCGRegSet reserved_regs;
uint32_t tb_cflags; /* cflags of the current TB */
intptr_t current_frame_offset;
intptr_t frame_start;
intptr_t frame_end;
TCGTemp *frame_temp;
tcg_insn_unit *code_ptr;
TranslationBlock *gen_tb; /* tb for which code is being generated */
tcg_insn_unit *code_buf; /* pointer for start of tb */
tcg_insn_unit *code_ptr; /* pointer for running end of tb */
#ifdef CONFIG_PROFILER
TCGProfile prof;
@@ -838,6 +833,9 @@ void tcg_func_start(TCGContext *s);
int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start);
void tb_target_set_jmp_target(const TranslationBlock *, int,
uintptr_t, uintptr_t);
void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr,
tcg/aarch64/tcg-target.c.inc
+50 -56
View File
@@ -1353,32 +1353,6 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
tcg_out_call_int(s, target);
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
uintptr_t jmp_rw, uintptr_t addr)
{
tcg_insn_unit i1, i2;
TCGType rt = TCG_TYPE_I64;
TCGReg rd = TCG_REG_TMP;
uint64_t pair;
ptrdiff_t offset = addr - jmp_rx;
if (offset == sextract64(offset, 0, 26)) {
i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
i2 = NOP;
} else {
offset = (addr >> 12) - (jmp_rx >> 12);
/* patch ADRP */
i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
/* patch ADDI */
i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
}
pair = (uint64_t)i2 << 32 | i1;
qatomic_set((uint64_t *)jmp_rw, pair);
flush_idcache_range(jmp_rx, jmp_rw, 8);
}
static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
{
if (!l->has_value) {
@@ -1887,6 +1861,54 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
{
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_goto_long(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
tcg_out_goto_long(s, tb_ret_addr);
}
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
/*
* Direct branch, or indirect address load, will be patched
* by tb_target_set_jmp_target. Assert indirect load offset
* in range early, regardless of direct branch distance.
*/
intptr_t i_off = tcg_pcrel_diff(s, (void *)get_jmp_target_addr(s, which));
tcg_debug_assert(i_off == sextract64(i_off, 0, 21));
set_jmp_insn_offset(s, which);
tcg_out32(s, I3206_B);
tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
set_jmp_reset_offset(s, which);
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
uintptr_t d_addr = tb->jmp_target_addr[n];
ptrdiff_t d_offset = d_addr - jmp_rx;
tcg_insn_unit insn;
/* Either directly branch, or indirect branch load. */
if (d_offset == sextract64(d_offset, 0, 28)) {
insn = deposit32(I3206_B, 0, 26, d_offset >> 2);
} else {
uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n];
ptrdiff_t i_offset = i_addr - jmp_rx;
/* Note that we asserted this in range in tcg_out_goto_tb. */
insn = deposit32(I3305_LDR | TCG_REG_TMP, 0, 5, i_offset >> 2);
}
qatomic_set((uint32_t *)jmp_rw, insn);
flush_idcache_range(jmp_rx, jmp_rw, 4);
}
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -1906,36 +1928,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
#define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
switch (opc) {
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_goto_long(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
tcg_out_goto_long(s, tb_ret_addr);
}
break;
case INDEX_op_goto_tb:
tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
/*
* Ensure that ADRP+ADD are 8-byte aligned so that an atomic
* write can be used to patch the target address.
*/
if ((uintptr_t)s->code_ptr & 7) {
tcg_out32(s, NOP);
}
s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
/*
* actual branch destination will be patched by
* tb_target_set_jmp_target later
*/
tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0);
tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0);
tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
tcg_out_insn(s, 3207, BR, a0);
break;
@@ -2305,6 +2297,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
g_assert_not_reached();
}
tcg/aarch64/tcg-target.h
+1 -5
View File
@@ -15,7 +15,7 @@
#define TCG_TARGET_INSN_UNIT_SIZE 4
#define TCG_TARGET_TLB_DISPLACEMENT_BITS 24
#define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
typedef enum {
TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
@@ -123,7 +123,6 @@ typedef enum {
#define TCG_TARGET_HAS_muls2_i64 0
#define TCG_TARGET_HAS_muluh_i64 1
#define TCG_TARGET_HAS_mulsh_i64 1
#define TCG_TARGET_HAS_direct_jump 1
#define TCG_TARGET_HAS_v64 1
#define TCG_TARGET_HAS_v128 1
@@ -151,9 +150,6 @@ typedef enum {
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 0
void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_NEED_LDST_LABELS
#define TCG_TARGET_NEED_POOL_LABELS
tcg/arm/tcg-target.c.inc
+61 -28
View File
@@ -135,6 +135,8 @@ typedef enum {
ARITH_BIC = 0xe << 21,
ARITH_MVN = 0xf << 21,
INSN_B = 0x0a000000,
INSN_CLZ = 0x016f0f10,
INSN_RBIT = 0x06ff0f30,
@@ -546,7 +548,7 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
static void tcg_out_b_imm(TCGContext *s, ARMCond cond, int32_t offset)
{
tcg_out32(s, (cond << 28) | 0x0a000000 |
tcg_out32(s, (cond << 28) | INSN_B |
(((offset - 8) >> 2) & 0x00ffffff));
}
@@ -1933,6 +1935,62 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
static void tcg_out_epilogue(TCGContext *s);
static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
{
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, arg);
tcg_out_epilogue(s);
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
uintptr_t i_addr;
intptr_t i_disp;
/* Direct branch will be patched by tb_target_set_jmp_target. */
set_jmp_insn_offset(s, which);
tcg_out32(s, INSN_NOP);
/* When branch is out of range, fall through to indirect. */
i_addr = get_jmp_target_addr(s, which);
i_disp = tcg_pcrel_diff(s, (void *)i_addr) - 8;
tcg_debug_assert(i_disp < 0);
if (i_disp >= -0xfff) {
tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, i_disp);
} else {
/*
* The TB is close, but outside the 12 bits addressable by
* the load. We can extend this to 20 bits with a sub of a
* shifted immediate from pc.
*/
int h = -i_disp;
int l = h & 0xfff;
h = encode_imm_nofail(h - l);
tcg_out_dat_imm(s, COND_AL, ARITH_SUB, TCG_REG_R0, TCG_REG_PC, h);
tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, l);
}
set_jmp_reset_offset(s, which);
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
uintptr_t addr = tb->jmp_target_addr[n];
ptrdiff_t offset = addr - (jmp_rx + 8);
tcg_insn_unit insn;
/* Either directly branch, or fall through to indirect branch. */
if (offset == sextract64(offset, 0, 26)) {
/* B <addr> */
insn = deposit32((COND_AL << 28) | INSN_B, 0, 24, offset >> 2);
} else {
insn = INSN_NOP;
}
qatomic_set((uint32_t *)jmp_rw, insn);
flush_idcache_range(jmp_rx, jmp_rw, 4);
}
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -1941,33 +1999,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
int c;
switch (opc) {
case INDEX_op_exit_tb:
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, args[0]);
tcg_out_epilogue(s);
break;
case INDEX_op_goto_tb:
{
/* Indirect jump method */
intptr_t ptr, dif, dil;
TCGReg base = TCG_REG_PC;
tcg_debug_assert(s->tb_jmp_insn_offset == 0);
ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]);
dif = tcg_pcrel_diff(s, (void *)ptr) - 8;
dil = sextract32(dif, 0, 12);
if (dif != dil) {
/* The TB is close, but outside the 12 bits addressable by
the load. We can extend this to 20 bits with a sub of a
shifted immediate from pc. In the vastly unlikely event
the code requires more than 1MB, we'll use 2 insns and
be no worse off. */
base = TCG_REG_R0;
tcg_out_movi32(s, COND_AL, base, ptr - dil);
}
tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil);
set_jmp_reset_offset(s, args[0]);
}
break;
case INDEX_op_goto_ptr:
tcg_out_b_reg(s, COND_AL, args[0]);
break;
@@ -2256,6 +2287,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
tcg_abort();
}
tcg/arm/tcg-target.h
-5
View File
@@ -121,7 +121,6 @@ extern bool use_neon_instructions;
#define TCG_TARGET_HAS_mulsh_i32 0
#define TCG_TARGET_HAS_div_i32 use_idiv_instructions
#define TCG_TARGET_HAS_rem_i32 0
#define TCG_TARGET_HAS_direct_jump 0
#define TCG_TARGET_HAS_qemu_st8_i32 0
#define TCG_TARGET_HAS_v64 use_neon_instructions
@@ -150,10 +149,6 @@ extern bool use_neon_instructions;
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 0
/* not defined -- call should be eliminated at compile time */
void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_NEED_LDST_LABELS
#define TCG_TARGET_NEED_POOL_LABELS
tcg/i386/tcg-target.c.inc
+38 -30
View File
@@ -2347,6 +2347,42 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
#endif
}
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
{
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_jmp(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
tcg_out_jmp(s, tb_ret_addr);
}
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
/*
* Jump displacement must be aligned for atomic patching;
* see if we need to add extra nops before jump
*/
int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
if (gap != 1) {
tcg_out_nopn(s, gap - 1);
}
tcg_out8(s, OPC_JMP_long); /* jmp im */
set_jmp_insn_offset(s, which);
tcg_out32(s, 0);
set_jmp_reset_offset(s, which);
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
/* patch the branch destination */
uintptr_t addr = tb->jmp_target_addr[n];
qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
/* no need to flush icache explicitly */
}
static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -2371,36 +2407,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
const_a2 = const_args[2];
switch (opc) {
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_jmp(s, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
tcg_out_jmp(s, tb_ret_addr);
}
break;
case INDEX_op_goto_tb:
if (s->tb_jmp_insn_offset) {
/* direct jump method */
int gap;
/* jump displacement must be aligned for atomic patching;
* see if we need to add extra nops before jump
*/
gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
if (gap != 1) {
tcg_out_nopn(s, gap - 1);
}
tcg_out8(s, OPC_JMP_long); /* jmp im */
s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
tcg_out32(s, 0);
} else {
/* indirect jump method */
tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
(intptr_t)(s->tb_jmp_target_addr + a0));
}
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
/* jmp to the given host address (could be epilogue) */
tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
@@ -2794,6 +2800,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
tcg_abort();
}
tcg/i386/tcg-target.h
-9
View File
@@ -141,7 +141,6 @@ extern bool have_movbe;
#define TCG_TARGET_HAS_muls2_i32 1
#define TCG_TARGET_HAS_muluh_i32 0
#define TCG_TARGET_HAS_mulsh_i32 0
#define TCG_TARGET_HAS_direct_jump 1
#if TCG_TARGET_REG_BITS == 64
/* Keep target addresses zero-extended in a register. */
@@ -220,14 +219,6 @@ extern bool have_movbe;
#define TCG_TARGET_extract_i64_valid(ofs, len) \
(((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
uintptr_t jmp_rw, uintptr_t addr)
{
/* patch the branch destination */
qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4));
/* no need to flush icache explicitly */
}
/* This defines the natural memory order supported by this
* architecture before guarantees made by various barrier
* instructions.
tcg/loongarch64/tcg-target.c.inc
+35 -31
View File
@@ -1039,11 +1039,12 @@ static void tcg_out_nop(TCGContext *s)
tcg_out32(s, NOP);
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
uintptr_t jmp_rw, uintptr_t addr)
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
tcg_insn_unit i1, i2;
ptrdiff_t upper, lower;
uintptr_t addr = tb->jmp_target_addr[n];
ptrdiff_t offset = (ptrdiff_t)(addr - jmp_rx) >> 2;
if (offset == sextreg(offset, 0, 26)) {
@@ -1068,6 +1069,36 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
{
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_call_int(s, tcg_code_gen_epilogue, true);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
tcg_out_call_int(s, tb_ret_addr, true);
}
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
/*
* Ensure that patch area is 8-byte aligned so that an
* atomic write can be used to patch the target address.
*/
if ((uintptr_t)s->code_ptr & 7) {
tcg_out_nop(s);
}
set_jmp_insn_offset(s, which);
/*
* actual branch destination will be patched by
* tb_target_set_jmp_target later
*/
tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0);
tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
set_jmp_reset_offset(s, which);
}
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -1078,35 +1109,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
int c2 = const_args[2];
switch (opc) {
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_call_int(s, tcg_code_gen_epilogue, true);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
tcg_out_call_int(s, tb_ret_addr, true);
}
break;
case INDEX_op_goto_tb:
tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
/*
* Ensure that patch area is 8-byte aligned so that an
* atomic write can be used to patch the target address.
*/
if ((uintptr_t)s->code_ptr & 7) {
tcg_out_nop(s);
}
s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
/*
* actual branch destination will be patched by
* tb_target_set_jmp_target later
*/
tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0);
tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_mb:
tcg_out_mb(s, a0);
break;
@@ -1500,6 +1502,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
g_assert_not_reached();
}
tcg/loongarch64/tcg-target.h
-3
View File
@@ -128,7 +128,6 @@ typedef enum {
#define TCG_TARGET_HAS_clz_i32 1
#define TCG_TARGET_HAS_ctz_i32 1
#define TCG_TARGET_HAS_ctpop_i32 0
#define TCG_TARGET_HAS_direct_jump 1
#define TCG_TARGET_HAS_brcond2 0
#define TCG_TARGET_HAS_setcond2 0
#define TCG_TARGET_HAS_qemu_st8_i32 0
@@ -171,8 +170,6 @@ typedef enum {
#define TCG_TARGET_HAS_muluh_i64 1
#define TCG_TARGET_HAS_mulsh_i64 1
void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_NEED_LDST_LABELS
tcg/mips/tcg-target.c.inc
+33 -26
View File
@@ -1951,6 +1951,37 @@ static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6,
}
}
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
{
TCGReg b0 = TCG_REG_ZERO;
if (a0 & ~0xffff) {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
b0 = TCG_REG_V0;
}
if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)tb_ret_addr);
tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
}
tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
/* indirect jump method */
tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
get_jmp_target_addr(s, which));
tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
tcg_out_nop(s);
set_jmp_reset_offset(s, which);
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
/* Always indirect, nothing to do */
}
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -1970,32 +2001,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
c2 = const_args[2];
switch (opc) {
case INDEX_op_exit_tb:
{
TCGReg b0 = TCG_REG_ZERO;
a0 = (intptr_t)a0;
if (a0 & ~0xffff) {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
b0 = TCG_REG_V0;
}
if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0,
(uintptr_t)tb_ret_addr);
tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
}
tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
}
break;
case INDEX_op_goto_tb:
/* indirect jump method */
tcg_debug_assert(s->tb_jmp_insn_offset == 0);
tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
(uintptr_t)(s->tb_jmp_target_addr + a0));
tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
tcg_out_nop(s);
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
/* jmp to the given host address (could be epilogue) */
tcg_out_opc_reg(s, OPC_JR, 0, a0, 0);
@@ -2403,6 +2408,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
tcg_abort();
}
tcg/mips/tcg-target.h
-5
View File
@@ -134,7 +134,6 @@ extern bool use_mips32r2_instructions;
#define TCG_TARGET_HAS_muluh_i32 1
#define TCG_TARGET_HAS_mulsh_i32 1
#define TCG_TARGET_HAS_bswap32_i32 1
#define TCG_TARGET_HAS_direct_jump 0
#if TCG_TARGET_REG_BITS == 64
#define TCG_TARGET_HAS_add2_i32 0
@@ -205,10 +204,6 @@ extern bool use_mips32r2_instructions;
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
/* not defined -- call should be eliminated at compile time */
void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t)
QEMU_ERROR("code path is reachable");
#define TCG_TARGET_NEED_LDST_LABELS
#endif
tcg/ppc/tcg-target.c.inc
+60 -133
View File
@@ -1854,103 +1854,6 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
tcg_out32(s, insn);
}
static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2)
{
if (HOST_BIG_ENDIAN) {
return (uint64_t)i1 << 32 | i2;
}
return (uint64_t)i2 << 32 | i1;
}
static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw,
tcg_insn_unit i0, tcg_insn_unit i1)
{
#if TCG_TARGET_REG_BITS == 64
qatomic_set((uint64_t *)rw, make_pair(i0, i1));
flush_idcache_range(rx, rw, 8);
#else
qemu_build_not_reached();
#endif
}
static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw,
tcg_insn_unit i0, tcg_insn_unit i1,
tcg_insn_unit i2, tcg_insn_unit i3)
{
uint64_t p[2];
p[!HOST_BIG_ENDIAN] = make_pair(i0, i1);
p[HOST_BIG_ENDIAN] = make_pair(i2, i3);
/*
* There's no convenient way to get the compiler to allocate a pair
* of registers at an even index, so copy into r6/r7 and clobber.
*/
asm("mr %%r6, %1\n\t"
"mr %%r7, %2\n\t"
"stq %%r6, %0"
: "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7");
flush_idcache_range(rx, rw, 16);
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
uintptr_t jmp_rw, uintptr_t addr)
{
tcg_insn_unit i0, i1, i2, i3;
intptr_t tb_diff = addr - tc_ptr;
intptr_t br_diff = addr - (jmp_rx + 4);
intptr_t lo, hi;
if (TCG_TARGET_REG_BITS == 32) {
intptr_t diff = addr - jmp_rx;
tcg_debug_assert(in_range_b(diff));
qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
flush_idcache_range(jmp_rx, jmp_rw, 4);
return;
}
/*
* For 16-bit displacements, we can use a single add + branch.
* This happens quite often.
*/
if (tb_diff == (int16_t)tb_diff) {
i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff);
i1 = B | (br_diff & 0x3fffffc);
ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
return;
}
lo = (int16_t)tb_diff;
hi = (int32_t)(tb_diff - lo);
assert(tb_diff == hi + lo);
i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16);
i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo);
/*
* Without stq from 2.07, we can only update two insns,
* and those must be the ones that load the target address.
*/
if (!have_isa_2_07) {
ppc64_replace2(jmp_rx, jmp_rw, i0, i1);
return;
}
/*
* For 26-bit displacements, we can use a direct branch.
* Otherwise we still need the indirect branch, which we
* must restore after a potential direct branch write.
*/
br_diff -= 4;
if (in_range_b(br_diff)) {
i2 = B | (br_diff & 0x3fffffc);
i3 = NOP;
} else {
i2 = MTSPR | RS(TCG_REG_TB) | CTR;
i3 = BCCTR | BO_ALWAYS;
}
ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3);
}
static void tcg_out_call_int(TCGContext *s, int lk,
const tcg_insn_unit *target)
{
@@ -2616,6 +2519,64 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out32(s, BCLR | BO_ALWAYS);
}
static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
{
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg);
tcg_out_b(s, 0, tcg_code_gen_epilogue);
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
uintptr_t ptr = get_jmp_target_addr(s, which);
if (USE_REG_TB) {
ptrdiff_t offset = tcg_tbrel_diff(s, (void *)ptr);
tcg_out_mem_long(s, LD, LDX, TCG_REG_TB, TCG_REG_TB, offset);
/* Direct branch will be patched by tb_target_set_jmp_target. */
set_jmp_insn_offset(s, which);
tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
/* When branch is out of range, fall through to indirect. */
tcg_out32(s, BCCTR | BO_ALWAYS);
/* For the unlinked case, need to reset TCG_REG_TB. */
set_jmp_reset_offset(s, which);
tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
-tcg_current_code_size(s));
} else {
/* Direct branch will be patched by tb_target_set_jmp_target. */
set_jmp_insn_offset(s, which);
tcg_out32(s, NOP);
/* When branch is out of range, fall through to indirect. */
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, ptr - (int16_t)ptr);
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, (int16_t)ptr);
tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
tcg_out32(s, BCCTR | BO_ALWAYS);
set_jmp_reset_offset(s, which);
}
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
uintptr_t addr = tb->jmp_target_addr[n];
intptr_t diff = addr - jmp_rx;
tcg_insn_unit insn;
if (in_range_b(diff)) {
insn = B | (diff & 0x3fffffc);
} else if (USE_REG_TB) {
insn = MTSPR | RS(TCG_REG_TB) | CTR;
} else {
insn = NOP;
}
qatomic_set((uint32_t *)jmp_rw, insn);
flush_idcache_range(jmp_rx, jmp_rw, 4);
}
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -2623,42 +2584,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
TCGArg a0, a1, a2;
switch (opc) {
case INDEX_op_exit_tb:
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
tcg_out_b(s, 0, tcg_code_gen_epilogue);
break;
case INDEX_op_goto_tb:
if (s->tb_jmp_insn_offset) {
/* Direct jump. */
if (TCG_TARGET_REG_BITS == 64) {
/* Ensure the next insns are 8 or 16-byte aligned. */
while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
tcg_out32(s, NOP);
}
s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0));
tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0));
} else {
s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
tcg_out32(s, B);
s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
break;
}
} else {
/* Indirect jump. */
tcg_debug_assert(s->tb_jmp_insn_offset == NULL);
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0,
(intptr_t)(s->tb_jmp_insn_offset + args[0]));
}
tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR);
tcg_out32(s, BCCTR | BO_ALWAYS);
set_jmp_reset_offset(s, args[0]);
if (USE_REG_TB) {
/* For the unlinked case, need to reset TCG_REG_TB. */
tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
-tcg_current_code_size(s));
}
break;
case INDEX_op_goto_ptr:
tcg_out32(s, MTSPR | RS(args[0]) | CTR);
if (USE_REG_TB) {
@@ -3185,6 +3110,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
tcg_abort();
}
tcg/ppc/tcg-target.h
+1 -6
View File
@@ -27,11 +27,10 @@
#ifdef _ARCH_PPC64
# define TCG_TARGET_REG_BITS 64
# define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
#else
# define TCG_TARGET_REG_BITS 32
# define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB)
#endif
#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
#define TCG_TARGET_NB_REGS 64
#define TCG_TARGET_INSN_UNIT_SIZE 4
@@ -108,7 +107,6 @@ extern bool have_vsx;
#define TCG_TARGET_HAS_muls2_i32 0
#define TCG_TARGET_HAS_muluh_i32 1
#define TCG_TARGET_HAS_mulsh_i32 1
#define TCG_TARGET_HAS_direct_jump 1
#define TCG_TARGET_HAS_qemu_st8_i32 0
#if TCG_TARGET_REG_BITS == 64
@@ -180,11 +178,8 @@ extern bool have_vsx;
#define TCG_TARGET_HAS_bitsel_vec have_vsx
#define TCG_TARGET_HAS_cmpsel_vec 0
void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
#define TCG_TARGET_NEED_LDST_LABELS
#define TCG_TARGET_NEED_POOL_LABELS
tcg/riscv/tcg-target.c.inc
+45 -20
View File
@@ -267,6 +267,7 @@ typedef enum {
#endif
OPC_FENCE = 0x0000000f,
OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */
} RISCVInsn;
/*
@@ -403,7 +404,7 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
{
int i;
for (i = 0; i < count; ++i) {
p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0);
p[i] = OPC_NOP;
}
}
@@ -1289,6 +1290,47 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
{
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_call_int(s, tcg_code_gen_epilogue, true);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
tcg_out_call_int(s, tb_ret_addr, true);
}
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
/* Direct branch will be patched by tb_target_set_jmp_target. */
set_jmp_insn_offset(s, which);
tcg_out32(s, OPC_JAL);
/* When branch is out of range, fall through to indirect. */
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
get_jmp_target_addr(s, which));
tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
set_jmp_reset_offset(s, which);
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
uintptr_t addr = tb->jmp_target_addr[n];
ptrdiff_t offset = addr - jmp_rx;
tcg_insn_unit insn;
/* Either directly branch, or fall through to indirect branch. */
if (offset == sextreg(offset, 0, 20)) {
insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset);
} else {
insn = OPC_NOP;
}
qatomic_set((uint32_t *)jmp_rw, insn);
flush_idcache_range(jmp_rx, jmp_rw, 4);
}
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
const int const_args[TCG_MAX_OP_ARGS])
@@ -1299,25 +1341,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
int c2 = const_args[2];
switch (opc) {
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tcg_out_call_int(s, tcg_code_gen_epilogue, true);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
tcg_out_call_int(s, tb_ret_addr, true);
}
break;
case INDEX_op_goto_tb:
assert(s->tb_jmp_insn_offset == 0);
/* indirect jump method */
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO,
(uintptr_t)(s->tb_jmp_target_addr + a0));
tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0);
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0);
break;
@@ -1617,6 +1640,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
g_assert_not_reached();
}
tcg/riscv/tcg-target.h
-4
View File
@@ -121,7 +121,6 @@ typedef enum {
#define TCG_TARGET_HAS_clz_i32 0
#define TCG_TARGET_HAS_ctz_i32 0
#define TCG_TARGET_HAS_ctpop_i32 0
#define TCG_TARGET_HAS_direct_jump 0
#define TCG_TARGET_HAS_brcond2 1
#define TCG_TARGET_HAS_setcond2 1
#define TCG_TARGET_HAS_qemu_st8_i32 0
@@ -165,9 +164,6 @@ typedef enum {
#define TCG_TARGET_HAS_mulsh_i64 1
#endif
/* not defined -- call should be eliminated at compile time */
void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_NEED_LDST_LABELS
tcg/s390x/tcg-target.c.inc
+41 -26
View File
@@ -1944,6 +1944,45 @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
#endif
}
static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
{
/* Reuse the zeroing that exists for goto_ptr. */
if (a0 == 0) {
tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
}
}
static void tcg_out_goto_tb(TCGContext *s, int which)
{
/*
* Branch displacement must be aligned for atomic patching;
* see if we need to add extra nop before branch
*/
if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
tcg_out16(s, NOP);
}
tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
set_jmp_insn_offset(s, which);
s->code_ptr += 2;
set_jmp_reset_offset(s, which);
}
void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
uintptr_t jmp_rx, uintptr_t jmp_rw)
{
if (!HAVE_FACILITY(GEN_INST_EXT)) {
return;
}
/* patch the branch destination */
uintptr_t addr = tb->jmp_target_addr[n];
intptr_t disp = addr - (jmp_rx - 2);
qatomic_set((int32_t *)jmp_rw, disp / 2);
/* no need to flush icache explicitly */
}
# define OP_32_64(x) \
case glue(glue(INDEX_op_,x),_i32): \
case glue(glue(INDEX_op_,x),_i64)
@@ -1956,32 +1995,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
TCGArg a0, a1, a2;
switch (opc) {
case INDEX_op_exit_tb:
/* Reuse the zeroing that exists for goto_ptr. */
a0 = args[0];
if (a0 == 0) {
tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
} else {
tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
}
break;
case INDEX_op_goto_tb:
a0 = args[0];
/*
* branch displacement must be aligned for atomic patching;
* see if we need to add extra nop before branch
*/
if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
tcg_out16(s, NOP);
}
tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
s->code_ptr += 2;
set_jmp_reset_offset(s, a0);
break;
case INDEX_op_goto_ptr:
a0 = args[0];
tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
@@ -2619,6 +2632,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */
case INDEX_op_mov_i64:
case INDEX_op_call: /* Always emitted via tcg_out_call. */
case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */
case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */
default:
tcg_abort();
}
tcg/s390x/tcg-target.h
-11
View File
@@ -105,7 +105,6 @@ extern uint64_t s390_facilities[3];
#define TCG_TARGET_HAS_mulsh_i32 0
#define TCG_TARGET_HAS_extrl_i64_i32 0
#define TCG_TARGET_HAS_extrh_i64_i32 0
#define TCG_TARGET_HAS_direct_jump 1
#define TCG_TARGET_HAS_qemu_st8_i32 0
#define TCG_TARGET_HAS_div2_i64 1
@@ -174,16 +173,6 @@ extern uint64_t s390_facilities[3];
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
#define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD)
static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
uintptr_t jmp_rw, uintptr_t addr)
{
/* patch the branch destination */
intptr_t disp = addr - (jmp_rx - 2);
qatomic_set((int32_t *)jmp_rw, disp / 2);
/* no need to flush icache explicitly */
}
#define TCG_TARGET_NEED_LDST_LABELS
#define TCG_TARGET_NEED_POOL_LABELS

Some files were not shown because too many files have changed in this diff Show More