diff --git a/lightening/lightening.c b/lightening/lightening.c index 5a98eb43c..272c35bdd 100644 --- a/lightening/lightening.c +++ b/lightening/lightening.c @@ -467,3 +467,511 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) #define IMPL_INSTRUCTION(kind, stem) JIT_IMPL_##kind(stem) FOR_EACH_INSTRUCTION(IMPL_INSTRUCTION) #undef IMPL_INSTRUCTION + +static void +abi_imm_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t dst, + intptr_t imm) +{ + switch (abi) { + case JIT_OPERAND_ABI_UINT8: + ASSERT(0 <= imm); + ASSERT(imm <= UINT8_MAX); + break; + case JIT_OPERAND_ABI_INT8: + ASSERT(INT8_MIN <= imm); + ASSERT(imm <= INT8_MAX); + break; + case JIT_OPERAND_ABI_UINT16: + ASSERT(0 <= imm); + ASSERT(imm <= UINT16_MAX); + break; + case JIT_OPERAND_ABI_INT16: + ASSERT(INT16_MIN <= imm); + ASSERT(imm <= INT16_MAX); + break; + case JIT_OPERAND_ABI_UINT32: + ASSERT(0 <= imm); + ASSERT(imm <= UINT32_MAX); + break; + case JIT_OPERAND_ABI_INT32: + ASSERT(INT32_MIN <= imm); + ASSERT(imm <= INT32_MAX); + break; +#if __WORDSIZE > 32 + case JIT_OPERAND_ABI_UINT64: + case JIT_OPERAND_ABI_INT64: + break; +#endif + case JIT_OPERAND_ABI_POINTER: + break; + default: + abort(); + } + jit_movi (_jit, dst, imm); +} + +static void +abi_gpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, + jit_gpr_t base, ptrdiff_t offset, jit_gpr_t src) +{ + switch (abi) { + case JIT_OPERAND_ABI_UINT8: + case JIT_OPERAND_ABI_INT8: + jit_stxi_c(_jit, offset, base, src); + break; + case JIT_OPERAND_ABI_UINT16: + case JIT_OPERAND_ABI_INT16: + jit_stxi_s(_jit, offset, base, src); + break; + case JIT_OPERAND_ABI_UINT32: + case JIT_OPERAND_ABI_INT32: +#if __WORDSIZE == 32 + case JIT_OPERAND_ABI_POINTER: +#endif + jit_stxi_i(_jit, offset, base, src); + break; +#if __WORDSIZE == 64 + case JIT_OPERAND_ABI_UINT64: + case JIT_OPERAND_ABI_INT64: + case JIT_OPERAND_ABI_POINTER: + jit_stxi_l(_jit, offset, base, src); + break; +#endif + default: + abort(); + } +} + +static void +abi_fpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, + jit_gpr_t base, ptrdiff_t offset, jit_fpr_t src) +{ + switch (abi) { + case JIT_OPERAND_ABI_FLOAT: + jit_stxi_f(_jit, offset, base, src); + break; + case JIT_OPERAND_ABI_DOUBLE: + jit_stxi_d(_jit, offset, base, src); + break; + default: + abort(); + } +} + +static void +abi_mem_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, + jit_gpr_t dst, jit_gpr_t base, ptrdiff_t offset) +{ + switch (abi) { + case JIT_OPERAND_ABI_UINT8: + jit_ldxi_uc(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT8: + jit_ldxi_c(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_UINT16: + jit_ldxi_us(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT16: + jit_ldxi_s(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_UINT32: + jit_ldxi_ui(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT32: + jit_ldxi_i(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_UINT64: + jit_ldxi_l(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT64: + jit_ldxi_l(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_POINTER: + jit_ldxi_l(_jit, dst, base, offset); + break; + default: + abort(); + } +} + +static void +abi_mem_to_fpr(jit_state_t *_jit, enum jit_operand_abi abi, + jit_fpr_t dst, jit_gpr_t base, ptrdiff_t offset) +{ + switch (abi) { + case JIT_OPERAND_ABI_FLOAT: + jit_ldxi_f(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_DOUBLE: + jit_ldxi_d(_jit, dst, base, offset); + break; + default: + abort(); + } +} + +static void +abi_imm_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, + ptrdiff_t offset, intmax_t imm) +{ + ASSERT(!is_fpr_arg(abi)); + + jit_gpr_t tmp = get_temp_gpr(_jit); + abi_imm_to_gpr(_jit, abi, tmp, imm); + abi_gpr_to_mem(_jit, abi, base, offset, tmp); + unget_temp_gpr(_jit); +} + +static void +abi_mem_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, + ptrdiff_t offset, jit_gpr_t src_base, ptrdiff_t src_offset) +{ + if (is_gpr_arg (abi)) { + jit_gpr_t tmp = get_temp_gpr(_jit); + abi_mem_to_gpr(_jit, abi, tmp, src_base, src_offset); + abi_gpr_to_mem(_jit, abi, base, offset, tmp); + unget_temp_gpr(_jit); + } else { + jit_fpr_t tmp = get_temp_xpr(_jit); + abi_mem_to_fpr(_jit, abi, tmp, src_base, src_offset); + abi_fpr_to_mem(_jit, abi, base, offset, tmp); + unget_temp_xpr(_jit); + } +} + +#define MOVE_KIND(a, b) ((((int) a) << 4) | ((int) b)) + +#define MOVE_KIND_ENUM(a, b) \ + MOVE_##a##_TO_##b = MOVE_KIND(JIT_OPERAND_KIND_##a, JIT_OPERAND_KIND_##b) +enum move_kind { + MOVE_KIND_ENUM(IMM, GPR), + MOVE_KIND_ENUM(GPR, GPR), + MOVE_KIND_ENUM(MEM, GPR), + MOVE_KIND_ENUM(FPR, FPR), + MOVE_KIND_ENUM(MEM, FPR), + MOVE_KIND_ENUM(IMM, MEM), + MOVE_KIND_ENUM(GPR, MEM), + MOVE_KIND_ENUM(FPR, MEM), + MOVE_KIND_ENUM(MEM, MEM) +}; +#undef MOVE_KIND_ENUM + +static void +move_operand(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) +{ + switch (MOVE_KIND (src.kind, dst.kind)) { + case MOVE_IMM_TO_GPR: + return abi_imm_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.imm); + + case MOVE_GPR_TO_GPR: + return jit_movr(_jit, dst.loc.gpr.gpr, src.loc.gpr.gpr); + + case MOVE_MEM_TO_GPR: + return abi_mem_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.mem.base, + src.loc.mem.offset); + + case MOVE_FPR_TO_FPR: + return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr); + + case MOVE_MEM_TO_FPR: + return abi_mem_to_fpr(_jit, src.abi, dst.loc.fpr, src.loc.mem.base, + src.loc.mem.offset); + + case MOVE_IMM_TO_MEM: + return abi_imm_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.imm); + + case MOVE_GPR_TO_MEM: + return abi_gpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.gpr.gpr); + + case MOVE_FPR_TO_MEM: + return abi_fpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.fpr); + + case MOVE_MEM_TO_MEM: + return abi_mem_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.mem.base, src.loc.mem.offset); + + default: + abort(); + } +} + +// A direct transliteration of "Tilting at windmills with Coq: formal +// verification of a compilation algorithm for parallel moves" by +// Laurence Rideau, Bernard Paul Serpette, and Xavier Leroy: +// https://xavierleroy.org/publi/parallel-move.pdf + +enum move_status { TO_MOVE, BEING_MOVED, MOVED }; + +static inline int +already_in_place(jit_operand_t src, jit_operand_t dst) +{ + switch (MOVE_KIND(src.kind, dst.kind)) { + case MOVE_GPR_TO_GPR: + return jit_same_gprs (src.loc.gpr.gpr, dst.loc.gpr.gpr); + case MOVE_FPR_TO_FPR: + return jit_same_fprs (src.loc.fpr, dst.loc.fpr); + case MOVE_MEM_TO_MEM: + return jit_same_gprs (src.loc.mem.base, dst.loc.mem.base) && + src.loc.mem.offset == dst.loc.mem.offset; + default: + return 0; + } +} + +static inline int +write_would_clobber(jit_operand_t src, jit_operand_t dst) +{ + if (already_in_place (src, dst)) + return 1; + + if (MOVE_KIND(src.kind, dst.kind) == MOVE_MEM_TO_GPR) + return jit_same_gprs(src.loc.mem.base, dst.loc.gpr.gpr); + + return 0; +} + +static inline ptrdiff_t +operand_addend(jit_operand_t op) +{ + switch (op.kind) { + case JIT_OPERAND_KIND_GPR: + return op.loc.gpr.addend; + case JIT_OPERAND_KIND_MEM: + return op.loc.mem.addend; + default: + abort(); + } +} + +static void +move_one(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, + size_t argc, enum move_status *status, size_t i) +{ + int tmp_gpr = 0, tmp_fpr = 0; + + if (already_in_place(src[i], dst[i])) + return; + + status[i] = BEING_MOVED; + for (size_t j = 0; j < argc; j++) { + if (write_would_clobber(src[j], dst[i])) { + switch (status[j]) { + case TO_MOVE: + move_one(_jit, dst, src, argc, status, j); + break; + case BEING_MOVED: { + jit_operand_t tmp; + if (is_fpr_arg (src[j].kind)) { + tmp_fpr = 1; + tmp = jit_operand_fpr(src[j].abi, get_temp_xpr(_jit)); + } else { + tmp_gpr = 1; + /* Preserve addend, if any, from source operand, to be applied + at the end. */ + tmp = jit_operand_gpr_with_addend(src[j].abi, get_temp_gpr(_jit), + operand_addend(src[j])); + } + move_operand (_jit, tmp, src[j]); + src[j] = tmp; + break; + } + case MOVED: + break; + default: + abort (); + } + } + } + + move_operand (_jit, dst[i], src[i]); + status[i] = MOVED; + if (tmp_gpr) + unget_temp_gpr(_jit); + else if (tmp_fpr) + unget_temp_xpr(_jit); +} + +static void +apply_addend(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) +{ + switch (MOVE_KIND(src.kind, dst.kind)) { + case MOVE_GPR_TO_GPR: + case MOVE_MEM_TO_GPR: + if (operand_addend(src)) + jit_addi(_jit, dst.loc.gpr.gpr, dst.loc.gpr.gpr, operand_addend(src)); + break; + case MOVE_GPR_TO_MEM: + case MOVE_MEM_TO_MEM: + if (operand_addend(src)) { + jit_gpr_t tmp = get_temp_gpr(_jit); + abi_mem_to_gpr(_jit, dst.abi, tmp, dst.loc.mem.base, dst.loc.mem.offset); + jit_addi(_jit, tmp, tmp, operand_addend(src)); + abi_gpr_to_mem(_jit, dst.abi, dst.loc.mem.base, dst.loc.mem.offset, tmp); + } + break; + default: + break; + } +} + +/* Preconditions: No dest operand is IMM. No dest operand aliases + another dest operand. No dest MEM operand uses a base register which + is used as a dest GPR. No dst operand has an addend. The registers + returned by get_temp_gpr and get_temp_fpr do not appear in source or + dest args. */ +void +jit_move_operands(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, + size_t argc) +{ + // Check preconditions, except the condition about tmp registers. + { + uint64_t src_gprs = 0; + uint64_t dst_gprs = 0; + uint64_t dst_fprs = 0; + uint64_t dst_mem_base_gprs = 0; + for (size_t i = 0; i < argc; i++) { + switch (src[i].kind) { + case JIT_OPERAND_KIND_GPR: + src_gprs |= 1ULL << jit_gpr_regno(src[i].loc.gpr.gpr); + break; + case JIT_OPERAND_KIND_FPR: + case JIT_OPERAND_KIND_IMM: + case JIT_OPERAND_KIND_MEM: + break; + default: + abort(); + } + switch (dst[i].kind) { + case JIT_OPERAND_KIND_GPR: { + ASSERT(dst[i].loc.gpr.addend == 0); + uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.gpr.gpr); + ASSERT((dst_gprs & bit) == 0); + dst_gprs |= bit; + break; + } + case JIT_OPERAND_KIND_FPR: { + uint64_t bit = 1ULL << jit_fpr_regno(dst[i].loc.fpr); + ASSERT((dst_fprs & bit) == 0); + dst_fprs |= bit; + break; + } + case JIT_OPERAND_KIND_MEM: { + ASSERT(dst[i].loc.mem.addend == 0); + uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.mem.base); + dst_mem_base_gprs |= bit; + break; + } + case JIT_OPERAND_KIND_IMM: + default: + abort(); + break; + } + } + ASSERT(((src_gprs | dst_gprs) & dst_mem_base_gprs) == 0); + } + + enum move_status status[argc]; + for (size_t i = 0; i < argc; i++) + status[i] = TO_MOVE; + for (size_t i = 0; i < argc; i++) + if (status[i] == TO_MOVE) + move_one(_jit, dst, src, argc, status, i); + + // Apply addends at the end. We could do it earlier in some cases but + // at least at the end we know that an in-place increment of one + // operand won't alias another. + for (size_t i = 0; i < argc; i++) + apply_addend(_jit, dst[i], src[i]); +} + +// Precondition: stack is already aligned. +static size_t +prepare_call_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) +{ + jit_operand_t dst[argc]; + struct abi_arg_iterator iter; + + // Compute shuffle destinations and space for spilled arguments. + reset_abi_arg_iterator(&iter, argc, args); + for (size_t i = 0; i < argc; i++) + next_abi_arg(&iter, &dst[i]); + + size_t stack_size = iter.stack_size; + + // Reserve space for spilled arguments, and fix up SP-relative + // operands. + if (stack_size) + { + // Align stack to 16-byte boundaries on 64-bit targets. + if (__WORDSIZE == 64) + stack_size = (stack_size + 15) & ~15; + jit_subi(_jit, JIT_SP, JIT_SP, stack_size); + for (size_t i = 0; i < argc; i++) { + switch(args[i].kind) { + case JIT_OPERAND_KIND_GPR: + if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) + args[i].loc.gpr.addend += stack_size; + break; + case JIT_OPERAND_KIND_MEM: + if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) + args[i].loc.mem.offset += stack_size; + break; + default: + break; + } + } + } + + jit_move_operands(_jit, dst, args, argc); + + return stack_size; +} + +void +jit_calli(jit_state_t *_jit, jit_pointer_t f, size_t argc, jit_operand_t args[]) +{ + size_t spill_size = prepare_call_args(_jit, argc, args); + + calli(_jit, (jit_word_t)f); + + if (spill_size) + jit_addi(_jit, JIT_SP, JIT_SP, spill_size); +} + +void +jit_callr(jit_state_t *_jit, jit_gpr_t f, size_t argc, jit_operand_t args[]) +{ + size_t spill_size = prepare_call_args(_jit, argc, args); + + callr(_jit, jit_gpr_regno(f)); + + if (spill_size) + jit_addi(_jit, JIT_SP, JIT_SP, spill_size); +} + +void +jit_locate_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) +{ + struct abi_arg_iterator iter; + + reset_abi_arg_iterator(&iter, argc, args); + for (size_t i = 0; i < argc; i++) + next_abi_arg(&iter, &args[i]); +} + +/* Precondition: args are distinct locations of type GPR or FPR. All + addends of arg operands are zero. No GPR arg is SP. */ +void +jit_load_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) +{ + jit_operand_t src[argc]; + + memcpy(src, args, sizeof(src[0]) * argc); + + jit_locate_args(_jit, argc, src); + jit_move_operands(_jit, args, src, argc); +} diff --git a/lightening/x86.c b/lightening/x86.c index 144816dfd..153997d45 100644 --- a/lightening/x86.c +++ b/lightening/x86.c @@ -249,426 +249,6 @@ is_gpr_arg(enum jit_operand_abi arg) return !is_fpr_arg(arg); } -static void -abi_imm_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t dst, - intptr_t imm) -{ - switch (abi) { - case JIT_OPERAND_ABI_UINT8: - ASSERT(0 <= imm); - ASSERT(imm <= UINT8_MAX); - break; - case JIT_OPERAND_ABI_INT8: - ASSERT(INT8_MIN <= imm); - ASSERT(imm <= INT8_MAX); - break; - case JIT_OPERAND_ABI_UINT16: - ASSERT(0 <= imm); - ASSERT(imm <= UINT16_MAX); - break; - case JIT_OPERAND_ABI_INT16: - ASSERT(INT16_MIN <= imm); - ASSERT(imm <= INT16_MAX); - break; - case JIT_OPERAND_ABI_UINT32: - ASSERT(0 <= imm); - ASSERT(imm <= UINT32_MAX); - break; - case JIT_OPERAND_ABI_INT32: - ASSERT(INT32_MIN <= imm); - ASSERT(imm <= INT32_MAX); - break; -#if __WORDSIZE > 32 - case JIT_OPERAND_ABI_UINT64: - case JIT_OPERAND_ABI_INT64: - break; -#endif - case JIT_OPERAND_ABI_POINTER: - break; - default: - abort(); - } - jit_movi (_jit, dst, imm); -} - -static void -abi_gpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, - jit_gpr_t base, ptrdiff_t offset, jit_gpr_t src) -{ - switch (abi) { - case JIT_OPERAND_ABI_UINT8: - case JIT_OPERAND_ABI_INT8: - jit_stxi_c(_jit, offset, base, src); - break; - case JIT_OPERAND_ABI_UINT16: - case JIT_OPERAND_ABI_INT16: - jit_stxi_s(_jit, offset, base, src); - break; - case JIT_OPERAND_ABI_UINT32: - case JIT_OPERAND_ABI_INT32: -#if __WORDSIZE == 32 - case JIT_OPERAND_ABI_POINTER: -#endif - jit_stxi_i(_jit, offset, base, src); - break; -#if __WORDSIZE == 64 - case JIT_OPERAND_ABI_UINT64: - case JIT_OPERAND_ABI_INT64: - case JIT_OPERAND_ABI_POINTER: - jit_stxi_l(_jit, offset, base, src); - break; -#endif - default: - abort(); - } -} - -static void -abi_fpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, - jit_gpr_t base, ptrdiff_t offset, jit_fpr_t src) -{ - switch (abi) { - case JIT_OPERAND_ABI_FLOAT: - jit_stxi_f(_jit, offset, base, src); - break; - case JIT_OPERAND_ABI_DOUBLE: - jit_stxi_d(_jit, offset, base, src); - break; - default: - abort(); - } -} - -static void -abi_mem_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, - jit_gpr_t dst, jit_gpr_t base, ptrdiff_t offset) -{ - switch (abi) { - case JIT_OPERAND_ABI_UINT8: - jit_ldxi_uc(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT8: - jit_ldxi_c(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_UINT16: - jit_ldxi_us(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT16: - jit_ldxi_s(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_UINT32: - jit_ldxi_ui(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT32: - jit_ldxi_i(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_UINT64: - jit_ldxi_l(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT64: - jit_ldxi_l(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_POINTER: - jit_ldxi_l(_jit, dst, base, offset); - break; - default: - abort(); - } -} - -static void -abi_mem_to_fpr(jit_state_t *_jit, enum jit_operand_abi abi, - jit_fpr_t dst, jit_gpr_t base, ptrdiff_t offset) -{ - switch (abi) { - case JIT_OPERAND_ABI_FLOAT: - jit_ldxi_f(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_DOUBLE: - jit_ldxi_d(_jit, dst, base, offset); - break; - default: - abort(); - } -} - -static void -abi_imm_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, - ptrdiff_t offset, intmax_t imm) -{ - ASSERT(!is_fpr_arg(abi)); - - jit_gpr_t tmp = get_temp_gpr(_jit); - abi_imm_to_gpr(_jit, abi, tmp, imm); - abi_gpr_to_mem(_jit, abi, base, offset, tmp); - unget_temp_gpr(_jit); -} - -static void -abi_mem_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, - ptrdiff_t offset, jit_gpr_t src_base, ptrdiff_t src_offset) -{ - if (is_gpr_arg (abi)) { - jit_gpr_t tmp = get_temp_gpr(_jit); - abi_mem_to_gpr(_jit, abi, tmp, src_base, src_offset); - abi_gpr_to_mem(_jit, abi, base, offset, tmp); - unget_temp_gpr(_jit); - } else { - jit_fpr_t tmp = get_temp_xpr(_jit); - abi_mem_to_fpr(_jit, abi, tmp, src_base, src_offset); - abi_fpr_to_mem(_jit, abi, base, offset, tmp); - unget_temp_xpr(_jit); - } -} - -#define MOVE_KIND(a, b) ((((int) a) << 4) | ((int) b)) - -#define MOVE_KIND_ENUM(a, b) \ - MOVE_##a##_TO_##b = MOVE_KIND(JIT_OPERAND_KIND_##a, JIT_OPERAND_KIND_##b) -enum move_kind { - MOVE_KIND_ENUM(IMM, GPR), - MOVE_KIND_ENUM(GPR, GPR), - MOVE_KIND_ENUM(MEM, GPR), - MOVE_KIND_ENUM(FPR, FPR), - MOVE_KIND_ENUM(MEM, FPR), - MOVE_KIND_ENUM(IMM, MEM), - MOVE_KIND_ENUM(GPR, MEM), - MOVE_KIND_ENUM(FPR, MEM), - MOVE_KIND_ENUM(MEM, MEM) -}; -#undef MOVE_KIND_ENUM - -static void -move_operand(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) -{ - switch (MOVE_KIND (src.kind, dst.kind)) { - case MOVE_IMM_TO_GPR: - return abi_imm_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.imm); - - case MOVE_GPR_TO_GPR: - return jit_movr(_jit, dst.loc.gpr.gpr, src.loc.gpr.gpr); - - case MOVE_MEM_TO_GPR: - return abi_mem_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.mem.base, - src.loc.mem.offset); - - case MOVE_FPR_TO_FPR: - return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr); - - case MOVE_MEM_TO_FPR: - return abi_mem_to_fpr(_jit, src.abi, dst.loc.fpr, src.loc.mem.base, - src.loc.mem.offset); - - case MOVE_IMM_TO_MEM: - return abi_imm_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.imm); - - case MOVE_GPR_TO_MEM: - return abi_gpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.gpr.gpr); - - case MOVE_FPR_TO_MEM: - return abi_fpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.fpr); - - case MOVE_MEM_TO_MEM: - return abi_mem_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.mem.base, src.loc.mem.offset); - - default: - abort(); - } -} - -// A direct transliteration of "Tilting at windmills with Coq: formal -// verification of a compilation algorithm for parallel moves" by -// Laurence Rideau, Bernard Paul Serpette, and Xavier Leroy: -// https://xavierleroy.org/publi/parallel-move.pdf - -enum move_status { TO_MOVE, BEING_MOVED, MOVED }; - -static inline int -already_in_place(jit_operand_t src, jit_operand_t dst) -{ - switch (MOVE_KIND(src.kind, dst.kind)) { - case MOVE_GPR_TO_GPR: - return jit_same_gprs (src.loc.gpr.gpr, dst.loc.gpr.gpr); - case MOVE_FPR_TO_FPR: - return jit_same_fprs (src.loc.fpr, dst.loc.fpr); - case MOVE_MEM_TO_MEM: - return jit_same_gprs (src.loc.mem.base, dst.loc.mem.base) && - src.loc.mem.offset == dst.loc.mem.offset; - default: - return 0; - } -} - -static inline int -write_would_clobber(jit_operand_t src, jit_operand_t dst) -{ - if (already_in_place (src, dst)) - return 1; - - if (MOVE_KIND(src.kind, dst.kind) == MOVE_MEM_TO_GPR) - return jit_same_gprs(src.loc.mem.base, dst.loc.gpr.gpr); - - return 0; -} - -static inline ptrdiff_t -operand_addend(jit_operand_t op) -{ - switch (op.kind) { - case JIT_OPERAND_KIND_GPR: - return op.loc.gpr.addend; - case JIT_OPERAND_KIND_MEM: - return op.loc.mem.addend; - default: - abort(); - } -} - -static void -move_one(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, - size_t argc, enum move_status *status, size_t i) -{ - int tmp_gpr = 0, tmp_fpr = 0; - - if (already_in_place(src[i], dst[i])) - return; - - status[i] = BEING_MOVED; - for (size_t j = 0; j < argc; j++) { - if (write_would_clobber(src[j], dst[i])) { - switch (status[j]) { - case TO_MOVE: - move_one(_jit, dst, src, argc, status, j); - break; - case BEING_MOVED: { - jit_operand_t tmp; - if (is_fpr_arg (src[j].kind)) { - tmp_fpr = 1; - tmp = jit_operand_fpr(src[j].abi, get_temp_xpr(_jit)); - } else { - tmp_gpr = 1; - /* Preserve addend, if any, from source operand, to be applied - at the end. */ - tmp = jit_operand_gpr_with_addend(src[j].abi, get_temp_gpr(_jit), - operand_addend(src[j])); - } - move_operand (_jit, tmp, src[j]); - src[j] = tmp; - break; - } - case MOVED: - break; - default: - abort (); - } - } - } - - move_operand (_jit, dst[i], src[i]); - status[i] = MOVED; - if (tmp_gpr) - unget_temp_gpr(_jit); - else if (tmp_fpr) - unget_temp_xpr(_jit); -} - -static void -apply_addend(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) -{ - switch (MOVE_KIND(src.kind, dst.kind)) { - case MOVE_GPR_TO_GPR: - case MOVE_MEM_TO_GPR: - if (operand_addend(src)) - jit_addi(_jit, dst.loc.gpr.gpr, dst.loc.gpr.gpr, operand_addend(src)); - break; - case MOVE_GPR_TO_MEM: - case MOVE_MEM_TO_MEM: - if (operand_addend(src)) { - jit_gpr_t tmp = get_temp_gpr(_jit); - abi_mem_to_gpr(_jit, dst.abi, tmp, dst.loc.mem.base, dst.loc.mem.offset); - jit_addi(_jit, tmp, tmp, operand_addend(src)); - abi_gpr_to_mem(_jit, dst.abi, dst.loc.mem.base, dst.loc.mem.offset, tmp); - } - break; - default: - break; - } -} - -/* Preconditions: No dest operand is IMM. No dest operand aliases - another dest operand. No dest MEM operand uses a base register which - is used as a dest GPR. No dst operand has an addend. The registers - returned by get_temp_gpr and get_temp_fpr do not appear in source or - dest args. */ -void -jit_move_operands(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, - size_t argc) -{ - // Check preconditions, except the condition about tmp registers. - { - uint64_t src_gprs = 0; - uint64_t dst_gprs = 0; - uint64_t dst_fprs = 0; - uint64_t dst_mem_base_gprs = 0; - for (size_t i = 0; i < argc; i++) { - switch (src[i].kind) { - case JIT_OPERAND_KIND_GPR: - src_gprs |= 1ULL << jit_gpr_regno(src[i].loc.gpr.gpr); - break; - case JIT_OPERAND_KIND_FPR: - case JIT_OPERAND_KIND_IMM: - case JIT_OPERAND_KIND_MEM: - break; - default: - abort(); - } - switch (dst[i].kind) { - case JIT_OPERAND_KIND_GPR: { - ASSERT(dst[i].loc.gpr.addend == 0); - uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.gpr.gpr); - ASSERT((dst_gprs & bit) == 0); - dst_gprs |= bit; - break; - } - case JIT_OPERAND_KIND_FPR: { - uint64_t bit = 1ULL << jit_fpr_regno(dst[i].loc.fpr); - ASSERT((dst_fprs & bit) == 0); - dst_fprs |= bit; - break; - } - case JIT_OPERAND_KIND_MEM: { - ASSERT(dst[i].loc.mem.addend == 0); - uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.mem.base); - dst_mem_base_gprs |= bit; - break; - } - case JIT_OPERAND_KIND_IMM: - default: - abort(); - break; - } - } - ASSERT(((src_gprs | dst_gprs) & dst_mem_base_gprs) == 0); - } - - enum move_status status[argc]; - for (size_t i = 0; i < argc; i++) - status[i] = TO_MOVE; - for (size_t i = 0; i < argc; i++) - if (status[i] == TO_MOVE) - move_one(_jit, dst, src, argc, status, i); - - // Apply addends at the end. We could do it earlier in some cases but - // at least at the end we know that an in-place increment of one - // operand won't alias another. - for (size_t i = 0; i < argc; i++) - apply_addend(_jit, dst[i], src[i]); -} - static const jit_gpr_t abi_gpr_args[] = { #if __X32 /* No GPRs in args. */ @@ -771,94 +351,6 @@ next_abi_arg(struct abi_arg_iterator *iter, jit_operand_t *arg) iter->arg_idx++; } -// Precondition: stack is already aligned. -static size_t -prepare_call_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) -{ - jit_operand_t dst[argc]; - struct abi_arg_iterator iter; - - // Compute shuffle destinations and space for spilled arguments. - reset_abi_arg_iterator(&iter, argc, args); - for (size_t i = 0; i < argc; i++) - next_abi_arg(&iter, &dst[i]); - - size_t stack_size = iter.stack_size; - - // Reserve space for spilled arguments, and fix up SP-relative - // operands. - if (stack_size) - { - // Align stack to 16-byte boundaries on 64-bit targets. - if (__WORDSIZE == 64) - stack_size = (stack_size + 15) & ~15; - jit_subi(_jit, JIT_SP, JIT_SP, stack_size); - for (size_t i = 0; i < argc; i++) { - switch(args[i].kind) { - case JIT_OPERAND_KIND_GPR: - if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) - args[i].loc.gpr.addend += stack_size; - break; - case JIT_OPERAND_KIND_MEM: - if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) - args[i].loc.mem.offset += stack_size; - break; - default: - break; - } - } - } - - jit_move_operands(_jit, dst, args, argc); - - return stack_size; -} - -void -jit_calli(jit_state_t *_jit, jit_pointer_t f, size_t argc, jit_operand_t args[]) -{ - size_t spill_size = prepare_call_args(_jit, argc, args); - - calli(_jit, (jit_word_t)f); - - if (spill_size) - jit_addi(_jit, JIT_SP, JIT_SP, spill_size); -} - -void -jit_callr(jit_state_t *_jit, jit_gpr_t f, size_t argc, jit_operand_t args[]) -{ - size_t spill_size = prepare_call_args(_jit, argc, args); - - callr(_jit, jit_gpr_regno(f)); - - if (spill_size) - jit_addi(_jit, JIT_SP, JIT_SP, spill_size); -} - -void -jit_locate_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) -{ - struct abi_arg_iterator iter; - - reset_abi_arg_iterator(&iter, argc, args); - for (size_t i = 0; i < argc; i++) - next_abi_arg(&iter, &args[i]); -} - -/* Precondition: args are distinct locations of type GPR or FPR. All - addends of arg operands are zero. No GPR arg is SP. */ -void -jit_load_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) -{ - jit_operand_t src[argc]; - - memcpy(src, args, sizeof(src[0]) * argc); - - jit_locate_args(_jit, argc, src); - jit_move_operands(_jit, args, src, argc); -} - void jit_flush(void *fptr, void *tptr) {