diff --git a/lightening/aarch64.c b/lightening/aarch64.c index 1964cb9bd..66fb27b61 100644 --- a/lightening/aarch64.c +++ b/lightening/aarch64.c @@ -223,3 +223,9 @@ static void jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc, jit_pointer_t addr) { } + +static void* +bless_function_pointer(void *ptr) +{ + return ptr; +} diff --git a/lightening/arm-cpu.c b/lightening/arm-cpu.c index 012b73ae8..e758bfd28 100644 --- a/lightening/arm-cpu.c +++ b/lightening/arm-cpu.c @@ -126,11 +126,14 @@ #define THUMB_TST 0x4200 #define THUMB2_TST 0xea100000 #define THUMB2_TSTI 0xf0100000 +#define THUMB_BLX 0x4780 +#define THUMB_BX 0x4700 #define THUMB_CC_B 0xd000 #define THUMB_B 0xe000 #define THUMB2_CC_B 0xf0008000 #define THUMB2_B 0xf0009000 #define THUMB2_BLI 0xf000d000 +#define THUMB2_BLXI 0xf000c000 #define THUMB2_P 0x00000400 #define THUMB2_U 0x00000200 #define THUMB_LDRSB 0x5600 @@ -245,8 +248,8 @@ encode_thumb_word_immediate(unsigned int v) static uint32_t read_wide_thumb(uint32_t *loc) { - uint16_t *sloc = (uint16_t*)sloc; - return (((uint32_t)sloc[0]) << 16) | sloc[1]; + uint16_t *sloc = (uint16_t*)loc; + return (sloc[0] << 16) | sloc[1]; } static void @@ -344,15 +347,13 @@ decode_thumb_cc_jump(uint32_t v) uint32_t s = (v >> 26) & 1; uint32_t j1 = (v >> 13) & 1; uint32_t j2 = (v >> 11) & 1; - uint32_t i1 = s ? j1 : !j1; - uint32_t i2 = s ? j2 : !j2; uint32_t hi = (v >> 16) & 0x3f; uint32_t lo = v & 0x7ff; int32_t ret = s << 31; ret >>= 12; - ret |= i1 << 18; - ret |= i2 << 17; + ret |= j2 << 18; + ret |= j1 << 17; ret |= hi << 11; ret |= lo; return ret; @@ -365,9 +366,11 @@ encode_thumb_cc_jump(int32_t v) { ASSERT(offset_in_jcc_range(v)); uint32_t s = !!(v & 0x80000); - uint32_t j1 = !!(v & 0x40000); - uint32_t j2 = !!(v & 0x20000); - uint32_t ret = (s<<26)|((v&0x1f800)<<5)|(j1<<13)|(j2<<11)|(v&0x7ff); + uint32_t j2 = !!(v & 0x40000); + uint32_t j1 = !!(v & 0x20000); + uint32_t hi = (v >> 11) & 0x3f; + uint32_t lo = v & 0x7ff; + uint32_t ret = (s<<26)|(hi << 16)|(j1<<13)|(j2<<11)|lo; ASSERT(decode_thumb_cc_jump(ret) == v); ASSERT((ret & thumb_cc_jump_mask) == 0); return ret; @@ -408,68 +411,68 @@ emit_thumb_cc_jump(jit_state_t *_jit, uint32_t inst) static void torrr(jit_state_t *_jit, int o, int rn, int rd, int rm) { - assert(!(o & 0xf0f0f)); + ASSERT(!(o & 0xf0f0f)); emit_wide_thumb(_jit, o|(_u4(rn)<<16)|(_u4(rd)<<8)|_u4(rm)); } static void torxr(jit_state_t *_jit, int o, int rn, int rt, int rm) { - assert(!(o & 0xf0f0f)); + ASSERT(!(o & 0xf0f0f)); emit_wide_thumb(_jit, o|(_u4(rn)<<16)|(_u4(rt)<<12)|_u4(rm)); } static void torrrr(jit_state_t *_jit, int o, int rn, int rl, int rh, int rm) { - assert(!(o & 0x000fff0f)); + ASSERT(!(o & 0x000fff0f)); emit_wide_thumb(_jit, o|(_u4(rn)<<16)|(_u4(rl)<<12)|(_u4(rh)<<8)|_u4(rm)); } static void torri(jit_state_t *_jit, int o, int rn, int rd, int im) { - assert(!(o & 0x0c0f7fff)); - assert(!(im & 0xfbff8f00)); + ASSERT(!(o & 0x0c0f7fff)); + ASSERT(!(im & 0xfbff8f00)); emit_wide_thumb(_jit, o|(_u4(rn)<<16)|(_u4(rd)<<8)|im); } static void torri8(jit_state_t *_jit, int o, int rn, int rt, int im) { - assert(!(o & 0x000ff0ff)); - assert(!(im & 0xffffff00)); + ASSERT(!(o & 0x000ff0ff)); + ASSERT(!(im & 0xffffff00)); emit_wide_thumb(_jit, o|(_u4(rn)<<16)|(_u4(rt)<<12)|im); } static void torri12(jit_state_t *_jit, int o, int rn, int rt, int im) { - assert(!(o & 0x000fffff)); - assert(!(im & 0xfffff000)); + ASSERT(!(o & 0x000fffff)); + ASSERT(!(im & 0xfffff000)); emit_wide_thumb(_jit, o|(_u4(rn)<<16)|(_u4(rt)<<12)|im); } static void tshift(jit_state_t *_jit, int o, int rd, int rm, int im) { - assert(!(o & 0x7fcf)); - assert(im >= 0 && im < 32); + ASSERT(!(o & 0x7fcf)); + ASSERT(im >= 0 && im < 32); emit_wide_thumb(_jit, o|((im&0x1c)<<10)|(_u4(rd)<<8)|((im&3)<<6)|_u4(rm)); } static void toriw(jit_state_t *_jit, int o, int rd, int im) { - assert(!(im & 0xffff0000)); + ASSERT(!(im & 0xffff0000)); emit_wide_thumb(_jit, o|((im&0xf000)<<4)|((im&0x800)<<15)|((im&0x700)<<4)|(_u4(rd)<<8)|(im&0xff)); } static jit_reloc_t tcb(jit_state_t *_jit, int cc) { - assert(!(cc & 0xfffffff)); - assert(cc != ARM_CC_AL && cc != ARM_CC_NV); + ASSERT(!(cc & 0xfffffff)); + ASSERT(cc != ARM_CC_AL && cc != ARM_CC_NV); cc = ((uint32_t)cc) >> 6; return emit_thumb_cc_jump(_jit, THUMB2_CC_B|cc); } @@ -477,7 +480,7 @@ tcb(jit_state_t *_jit, int cc) static jit_reloc_t tb(jit_state_t *_jit, int o) { - assert(!(o & 0x07ff2fff)); + ASSERT(!(o & 0x07ff2fff)); return emit_thumb_jump(_jit, o); } @@ -772,7 +775,7 @@ T2_UDIV(jit_state_t *_jit, int32_t rd, int32_t rn, int32_t rm) static void T1_MLS(jit_state_t *_jit, int32_t rd, int32_t rn, int32_t rm, int32_t ra) { - return torrrr(_jit, THUMB_MLS, ra, rn, rd, rm); + return torrrr(_jit, THUMB_MLS, rn, ra, rd, rm); } static void @@ -985,6 +988,18 @@ T2_TSTI(jit_state_t *_jit, int32_t rn, int32_t im) return torri(_jit, THUMB2_TSTI,rn,_NOREG,im); } +static void +T1_BLX(jit_state_t *_jit, int32_t r0) +{ + emit_u16(_jit, THUMB_BLX|(_u4(r0)<<3)); +} + +static void +T1_BX(jit_state_t *_jit, int32_t r0) +{ + emit_u16(_jit, THUMB_BX|(_u4(r0)<<3)); +} + static jit_reloc_t T2_CC_B(jit_state_t *_jit, uint32_t cc) { @@ -1003,6 +1018,12 @@ T2_BLI(jit_state_t *_jit) return tb(_jit, THUMB2_BLI); } +static jit_reloc_t +T2_BLXI(jit_state_t *_jit) +{ + return tb(_jit, THUMB2_BLXI); +} + static void T1_LDRSB(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t rm) { @@ -1297,7 +1318,7 @@ nop(jit_state_t *_jit, int32_t i0) for (; i0 > 0; i0 -= 2) T1_NOP(_jit); - assert(i0 == 0); + ASSERT(i0 == 0); } static void @@ -1353,12 +1374,8 @@ static uint32_t encode_load_from_pool_offset(int32_t off) { ASSERT(offset_in_load_from_pool_range(off)); - uint32_t u; - if (off >= 0) - u = 1; - else - u = 0, off = -off; - uint32_t ret = (off & 0xfff) | (u << 23); + uint32_t u = off >= 0; + uint32_t ret = ((u ? off : -off) & 0xfff) | (u << 23); ASSERT(decode_load_from_pool_offset(ret) == off); return ret; } @@ -1766,7 +1783,7 @@ iqdivr(jit_state_t *_jit, int32_t r0, int32_t r1, divr(_jit, r0, r2, r3); else divr_u(_jit, r0, r2, r3); - T1_MLS(_jit, r1, r0, r3, r2); + T1_MLS(_jit, r1, r3, r0, r2); if (need_tmp) unget_temp_gpr(_jit); } @@ -1808,7 +1825,7 @@ qdivi_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t i0) static void iremr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_bool_t sign) { - return iqdivr(_jit, r0, r0, r1, r2, 1); + return iqdivr(_jit, r0, r0, r1, r2, sign); } static void @@ -1936,7 +1953,7 @@ lshr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2) static void lshi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0) { - assert(i0 >= 0 && i0 <= 31); + ASSERT(i0 >= 0 && i0 <= 31); if (i0 == 0) movr(_jit, r0, r1); else { @@ -1959,7 +1976,7 @@ rshr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2) static void rshi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0) { - assert(i0 >= 0 && i0 <= 31); + ASSERT(i0 >= 0 && i0 <= 31); if (i0 == 0) movr(_jit, r0, r1); else { @@ -1982,7 +1999,7 @@ rshr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2) static void rshi_u(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0) { - assert(i0 >= 0 && i0 <= 31); + ASSERT(i0 >= 0 && i0 <= 31); if (i0 == 0) movr(_jit, r0, r1); else { @@ -2008,6 +2025,8 @@ jmp(jit_state_t *_jit) static void jmpi(jit_state_t *_jit, jit_word_t i0) { + /* Strip thumb bit, if any. */ + i0 &= ~1; return jit_patch_there(_jit, jmp(_jit), (void*)i0); } @@ -2841,21 +2860,22 @@ extr_us(jit_state_t *_jit, int32_t r0, int32_t r1) static void callr(jit_state_t *_jit, int32_t r0) { - T1_MOV(_jit, jit_gpr_regno(_LR), jit_gpr_regno(_PC)); - T1_MOV(_jit, jit_gpr_regno(_PC), r0); - // LR will point here: 4 bytes after the MOV LR,PC instruction. + T1_BLX(_jit, r0); } static void calli(jit_state_t *_jit, jit_word_t i0) { - jit_patch_there(_jit, T2_BLI(_jit), (void*)i0); + if (i0 & 1) + jit_patch_there(_jit, T2_BLI(_jit), (void*)(i0 & ~1)); + else + jit_patch_there(_jit, T2_BLXI(_jit), (void*)i0); } static void ret(jit_state_t *_jit) { - movr(_jit, jit_gpr_regno(_PC), jit_gpr_regno(_LR)); + T1_BX(_jit, jit_gpr_regno(_LR)); } static void diff --git a/lightening/arm.c b/lightening/arm.c index be0e629e8..d587e7158 100644 --- a/lightening/arm.c +++ b/lightening/arm.c @@ -109,7 +109,7 @@ next_abi_arg(struct abi_arg_iterator *iter, jit_operand_t *arg) } } *arg = jit_operand_mem (abi, JIT_SP, iter->stack_size); - iter->stack_size += 8; + iter->stack_size += 4; } static void @@ -130,3 +130,10 @@ static void jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc, jit_pointer_t addr) { } + +static void* +bless_function_pointer(void *ptr) +{ + // Set low bit to mark as thumb mode. + return (void*) (((uintptr_t)ptr) | 1); +} diff --git a/lightening/arm.h b/lightening/arm.h index 8db672ea0..47bd2c257 100644 --- a/lightening/arm.h +++ b/lightening/arm.h @@ -93,8 +93,8 @@ #define JIT_R0 _R0 #define JIT_R1 _R1 #define JIT_R2 _R2 -#define JIT_R3 _R12 -#define JIT_TMP0 _R3 +#define JIT_R3 _R3 +#define JIT_TMP0 _R12 #define JIT_V0 _R4 #define JIT_V1 _R5 diff --git a/lightening/lightening.c b/lightening/lightening.c index 2a1d282a6..572976f40 100644 --- a/lightening/lightening.c +++ b/lightening/lightening.c @@ -88,6 +88,7 @@ static jit_bool_t jit_init(jit_state_t *); static void jit_flush(void *fptr, void *tptr); static void jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc, jit_pointer_t addr); +static void* bless_function_pointer(void *ptr); struct abi_arg_iterator; @@ -239,7 +240,7 @@ jit_end(jit_state_t *_jit, size_t *length) clear_literal_pool(_jit->pool); #endif - return start; + return bless_function_pointer(start); } static int @@ -387,7 +388,8 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) #ifdef JIT_NEEDS_LITERAL_POOL case JIT_RELOC_JMP_WITH_VENEER: { int32_t voff = read_jmp_offset(loc.ui); - if (voff == 0) { + uint8_t *target = pc_base + (voff << reloc.rsh); + if (target == loc.uc) { // PC still in range to reify direct branch. if (offset_in_jmp_range(diff)) { // Target also in range: reify direct branch. @@ -400,14 +402,14 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) } else { // Already emitted a veneer. In this case, patch the veneer // directly. - uint8_t *target = pc_base + (voff << reloc.rsh); patch_veneer((uint32_t *) target, addr); } return; } case JIT_RELOC_JCC_WITH_VENEER: { - uint32_t voff = read_jcc_offset(loc.ui); - if (voff == 0) { + int32_t voff = read_jcc_offset(loc.ui); + uint8_t *target = pc_base + (voff << reloc.rsh); + if (target == loc.uc) { if (offset_in_jcc_range(diff)) { patch_jcc_offset(loc.ui, diff); remove_pending_literal(_jit, reloc); @@ -415,17 +417,16 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) patch_pending_literal(_jit, reloc, (uintptr_t) addr); } } else { - uint8_t *target = pc_base + (voff << reloc.rsh); patch_veneer((uint32_t *) target, addr); } return; } case JIT_RELOC_LOAD_FROM_POOL: { - uint32_t voff = read_load_from_pool_offset(loc.ui); - if (voff == 0) { + int32_t voff = read_load_from_pool_offset(loc.ui); + uint8_t *target = pc_base + (voff << reloc.rsh); + if (target == loc.uc) { patch_pending_literal(_jit, reloc, (uintptr_t) addr); } else { - uint8_t *target = pc_base + (voff << reloc.rsh); *(uintptr_t *) target = (uintptr_t) addr; } return; @@ -782,7 +783,11 @@ move_operand(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) src.loc.mem.offset); case MOVE_FPR_TO_FPR: - return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr); + ASSERT(src.abi == dst.abi); + if (src.abi == JIT_OPERAND_ABI_DOUBLE) + return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr); + else + return jit_movr_f(_jit, dst.loc.fpr, src.loc.fpr); case MOVE_MEM_TO_FPR: return abi_mem_to_fpr(_jit, src.abi, dst.loc.fpr, src.loc.mem.base, @@ -1145,7 +1150,7 @@ prepare_call_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) for (size_t i = 0; i < argc; i++) { switch(args[i].kind) { case JIT_OPERAND_KIND_GPR: - if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) + if (jit_same_gprs (args[i].loc.gpr.gpr, JIT_SP)) args[i].loc.gpr.addend += stack_size; break; case JIT_OPERAND_KIND_MEM: diff --git a/lightening/x86.c b/lightening/x86.c index 4d0f7b9a2..965191a4c 100644 --- a/lightening/x86.c +++ b/lightening/x86.c @@ -399,3 +399,9 @@ jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc, jit_pointer_t addr) abort (); } } + +static void* +bless_function_pointer(void *ptr) +{ + return ptr; +}