From 0d78447d85661c917bc8a235251e9bd04e2b0c30 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 31 Oct 2018 19:33:26 +0100 Subject: [PATCH] Trim unneded bits from jit.c --- jit.h | 6 +- jit/jit.c | 3367 +------------------------------------------------ jit/private.h | 605 +-------- 3 files changed, 63 insertions(+), 3915 deletions(-) diff --git a/jit.h b/jit.h index 901321a74..140c390fd 100644 --- a/jit.h +++ b/jit.h @@ -110,9 +110,9 @@ extern void init_jit(void); extern jit_state_t *jit_new_state(void); extern void jit_destroy_state(jit_state_t*); -extern void jit_begin(jit_state_t*); -extern void jit_end(jit_state_t*, jit_addr_t*, size_t*); +extern void jit_begin(jit_state_t*, jit_addr_t, size_t); extern void jit_reset(jit_state_t*); +extern jit_addr_t jit_end(jit_state_t*, size_t*); extern void jit_align(jit_state_t*, unsigned); extern void jit_allocai(jit_state_t*, size_t); @@ -123,7 +123,7 @@ extern void jit_patch_here(jit_state_t*, jit_reloc_t); extern void jit_patch_there(jit_state_t*, jit_reloc_t, jit_pointer_t); extern void jit_calli(jit_state_t *, jit_pointer_t f, - size_t argc, const jit_arg_t *argv); + size_t argc, const jit_arg_t *argv); extern void jit_callr(jit_state_t *, jit_gpr_t f, size_t argc, const jit_arg_t *argv); extern void jit_receive(jit_state_t*, size_t argc, jit_arg_t *argv); diff --git a/jit/jit.c b/jit/jit.c index da630b046..1f53514c7 100644 --- a/jit/jit.c +++ b/jit/jit.c @@ -22,8 +22,8 @@ # include #endif -#include "jit.h" -#include "jit/jit_private.h" +#include "../jit.h" +#include "private.h" #ifndef MAP_ANON # define MAP_ANON MAP_ANONYMOUS @@ -32,886 +32,22 @@ # endif #endif +#if !defined(__sgi) +#define mmap_fd -1 +#endif + #define jit_regload_reload 0 /* convert to reload */ #define jit_regload_delete 1 /* just remove node */ #define jit_regload_isdead 2 /* delete and unset live bit */ -#if __WORDSIZE == 32 -# define bmp_shift 5 -#else -# define bmp_shift 6 -#endif - -/* - * Prototypes - */ -static jit_word_t hash_data(const void*, jit_word_t); - -#define new_pool() _new_pool(_jit) -static void _new_pool(jit_state_t*); - -#define new_node(u) _new_node(_jit, u) -static jit_node_t *_new_node(jit_state_t*, jit_code_t); - -#define link_node(u) _link_node(_jit, u) -static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*); - -#define del_node(u, v) _del_node(_jit, u, v) -static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*); - -#define free_node(u) _free_node(_jit, u) -static inline void _free_node(jit_state_t*, jit_node_t*); - -#define del_label(u, v) _del_label(_jit, u, v) -static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*); - -#define bmp_init() _bmp_init(_jit) -static void _bmp_init(jit_state_t*); - -#define bmp_clear() _bmp_clear(_jit) -static void _bmp_clear(jit_state_t*); - -#define bmp_zero() \ - memset(_jitc->blockmask.ptr, 0, \ - _jitc->blockmask.length * sizeof(jit_word_t)) - -#define bmp_set(bit) _bmp_set(_jit, bit) -static void _bmp_set(jit_state_t*, jit_word_t); - -#define bmp_clr(bit) _bmp_clr(_jit, bit) -static void _bmp_clr(jit_state_t*, jit_word_t) maybe_unused; - -#define bmp_tst(bit) _bmp_tst(_jit, bit) -static jit_bool_t _bmp_tst(jit_state_t*, jit_word_t); - -#define jit_dataset() _jit_dataset(_jit) -static void -_jit_dataset(jit_state_t *_jit); - -#define jit_setup(block) _jit_setup(_jit, block) -static void -_jit_setup(jit_state_t *_jit, jit_block_t *block); - -#define jit_update(node, live, mask, r) _jit_update(_jit, node, live, mask, r) -static void -_jit_update(jit_state_t *_jit, jit_node_t *node, - jit_regset_t *live, jit_regset_t *mask, jit_bool_t recurse); - -#define thread_jumps() _thread_jumps(_jit) -static void -_thread_jumps(jit_state_t *_jit); - -#define sequential_labels() _sequential_labels(_jit) -static void -_sequential_labels(jit_state_t *_jit); - -#define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node) -static jit_bool_t -_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); - -#define redundant_jump(prev, node) _redundant_jump(_jit, prev, node) -static jit_bool_t -_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); - -static jit_code_t -reverse_jump_code(jit_code_t code); - -#define reverse_jump(prev, node) _reverse_jump(_jit, prev, node) -static jit_bool_t -_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); - -#define redundant_store(node, jump) _redundant_store(_jit, node, jump) -static void -_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump); - -#define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s) -static jit_bool_t -_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node, - int32_t kind, jit_int32_t size); - -#define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s) -static jit_bool_t -_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node, - int32_t kind, jit_int32_t size); - -#define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node) -static jit_bool_t -_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); - -#define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node) -static jit_bool_t -_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); - -#define simplify_spill(node, regno) _simplify_spill(_jit, node, regno) -static void -_simplify_spill(jit_state_t *_jit, jit_node_t *node, int32_t regno); - -#define simplify() _simplify(_jit) -static void -_simplify(jit_state_t *_jit); - -#define jit_reg_undef -1 -#define jit_reg_static 0 -#define jit_reg_change 1 -#define register_change_p(n, l, r) _register_change_p(_jit, n, l, r) -static int32_t -_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link, - int32_t regno); - -#define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno) -static jit_bool_t -_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, int32_t regno); - -#define patch_registers() _patch_registers(_jit) -static void -_patch_registers(jit_state_t *_jit); - -#define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p) -static void -_patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link, - int32_t regno, jit_int32_t patch); - -/* - * Initialization - */ -#if !defined(__sgi) -#define mmap_fd -1 -#endif /* * Implementation */ void -init_jit(const char *progname) +init_jit(void) { jit_get_cpu(); - jit_init_debug(progname); - jit_init_size(); -} - -void -finish_jit(void) -{ - jit_finish_debug(); - jit_finish_size(); -} - -int32_t -_jit_get_reg(jit_state_t *_jit, int32_t regspec) -{ - int32_t spec; - int32_t regno; - - spec = regspec & ~(jit_class_chk|jit_class_nospill); - if (spec & jit_class_named) { - regno = jit_regno(spec); - if (jit_regset_tstbit(&_jitc->regsav, regno)) - /* fail if register is spilled */ - goto fail; - if (jit_regset_tstbit(&_jitc->regarg, regno)) - /* fail if register is an argument to current instruction */ - goto fail; - if (jit_regset_tstbit(&_jitc->reglive, regno)) { - if (regspec & jit_class_nospill) - /* fail if register is live and should not spill/reload */ - goto fail; - goto spill; - } - jit_regset_setbit(&_jitc->regarg, regno); - return (regno); - } - else - assert(jit_class(spec) != 0); - - if (_jitc->emit) { - /* search for a free register matching spec */ - for (regno = 0; regno < _jitc->reglen; regno++) { - if ((jit_class(_rvs[regno].spec) & spec) == spec && - !jit_regset_tstbit(&_jitc->regarg, regno) && - !jit_regset_tstbit(&_jitc->reglive, regno)) - goto regarg; - } - - /* search for a register matching spec that is not an argument - * for the current instruction */ - for (regno = 0; regno < _jitc->reglen; regno++) { - if ((jit_class(_rvs[regno].spec) & spec) == spec && - !jit_regset_tstbit(&_jitc->regsav, regno) && - !jit_regset_tstbit(&_jitc->regarg, regno) && - !(regspec & jit_class_nospill)) { - spill: - assert(_jitc->function != NULL); - if (spec & jit_class_gpr) { - if (!_jitc->function->regoff[regno]) { - _jitc->function->regoff[regno] = - jit_allocai(sizeof(jit_word_t)); - _jitc->again = 1; - } -#if DEBUG - /* emit_stxi must not need temporary registers */ - assert(!_jitc->getreg); - _jitc->getreg = 1; -#endif - emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno); -#if DEBUG - _jitc->getreg = 0; -#endif - } - else { - if (!_jitc->function->regoff[regno]) { - _jitc->function->regoff[regno] = - jit_allocai(sizeof(jit_float64_t)); - _jitc->again = 1; - } -#if DEBUG - /* emit_stxi must not need temporary registers */ - assert(!_jitc->getreg); - _jitc->getreg = 1; -#endif - emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno); -#if DEBUG - _jitc->getreg = 0; -#endif - } - jit_regset_setbit(&_jitc->regsav, regno); - regarg: - jit_regset_setbit(&_jitc->regarg, regno); - if (jit_class(_rvs[regno].spec) & jit_class_sav) { - /* if will modify callee save registers without a - * function prolog, better patch this assertion */ - assert(_jitc->function != NULL); - if (!jit_regset_tstbit(&_jitc->function->regset, regno)) { - jit_regset_setbit(&_jitc->function->regset, regno); - _jitc->again = 1; - } - } - return (regno); - } - } - } - else { - /* nospill hint only valid during emit" */ - assert(!(regspec & jit_class_nospill)); - for (regno = 0; regno < _jitc->reglen; regno++) { - if ((jit_class(_rvs[regno].spec) & spec) == spec && - !jit_regset_tstbit(&_jitc->regsav, regno) && - !jit_regset_tstbit(&_jitc->regarg, regno)) { - jit_regset_setbit(&_jitc->regarg, regno); - jit_regset_setbit(&_jitc->regsav, regno); - jit_save(regno); - return (jit_regno_patch|regno); - } - } - } - - /* Out of hardware registers */ -fail: - assert(regspec & jit_class_chk); - return (JIT_NOREG); -} - -void -_jit_unget_reg(jit_state_t *_jit, int32_t regno) -{ - regno = jit_regno(regno); - if (jit_regset_tstbit(&_jitc->regsav, regno)) { - if (_jitc->emit) { -#if DEBUG - /* emit_ldxi must not need a temporary register */ - assert(!_jitc->getreg); - _jitc->getreg = 1; -#endif - if (jit_class(_rvs[regno].spec) & jit_class_gpr) - emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]); - else - emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]); -#if DEBUG - /* emit_ldxi must not need a temporary register */ - _jitc->getreg = 0; -#endif - } - else - jit_load(regno); - jit_regset_clrbit(&_jitc->regsav, regno); - } - assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0); - jit_regset_clrbit(&_jitc->regarg, regno); -} - -jit_bool_t -_jit_callee_save_p(jit_state_t *_jit, int32_t regno) -{ - assert(regno >= 0 && regno < JIT_NOREG); - return (!!(_rvs[regno].spec & jit_class_sav)); -} - -extern jit_bool_t -_jit_pointer_p(jit_state_t *_jit, jit_pointer_t address) -{ - return ((uint8_t *)address >= _jit->code.ptr && - (jit_word_t)address < _jit->pc.w); -} - -#if __ia64__ -void -jit_regset_com(jit_regset_t *u, jit_regset_t *v) -{ - u->rl = ~v->rl; u->rh = ~v->rh; - u->fl = ~v->fl; u->fh = ~v->fh; -} - -void -jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w) -{ - u->rl = v->rl & w->rl; u->rh = v->rh & w->rh; - u->fl = v->fl & w->fl; u->fh = v->fh & w->fh; -} - -void -jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w) -{ - u->rl = v->rl | w->rl; u->rh = v->rh | w->rh; - u->fl = v->fl | w->fl; u->fh = v->fh | w->fh; -} - -void -jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w) -{ - u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh; - u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh; -} - -void -jit_regset_set(jit_regset_t *u, jit_regset_t *v) -{ - u->rl = v->rl; u->rh = v->rh; - u->fl = v->fl; u->fh = v->fh; -} - -void -jit_regset_set_mask(jit_regset_t *u, int32_t v) -{ - jit_bool_t w = !!(v & (v - 1)); - - assert(v >= 0 && v <= 256); - if (v == 0) - u->rl = u->rh = u->fl = u->fh = -1LL; - else if (v <= 64) { - u->rl = w ? (1LL << v) - 1 : -1LL; - u->rh = u->fl = u->fh = 0; - } - else if (v <= 128) { - u->rl = -1LL; - u->rh = w ? (1LL << (v - 64)) - 1 : -1LL; - u->fl = u->fh = 0; - } - else if (v <= 192) { - u->rl = u->rh = -1LL; - u->fl = w ? (1LL << (v - 128)) - 1 : -1LL; - u->fh = 0; - } - else { - u->rl = u->rh = u->fl = -1LL; - u->fh = w ? (1LL << (v - 128)) - 1 : -1LL; - } -} - -jit_bool_t -jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v) -{ - return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0)); -} - -void -jit_regset_set_ui(jit_regset_t *u, jit_word_t v) -{ - u->rl = v; - u->rh = u->fl = u->fh = 0; -} - -jit_bool_t -jit_regset_set_p(jit_regset_t *u) -{ - return (u->rl || u->rh || u->fl || u->fh); -} - -void -jit_regset_clrbit(jit_regset_t *set, int32_t bit) -{ - assert(bit >= 0 && bit <= 255); - if (bit < 64) - set->rl &= ~(1LL << bit); - else if (bit < 128) - set->rh &= ~(1LL << (bit - 64)); - else if (bit < 192) - set->fl &= ~(1LL << (bit - 128)); - else - set->fh &= ~(1LL << (bit - 192)); -} - -void -jit_regset_setbit(jit_regset_t *set, int32_t bit) -{ - assert(bit >= 0 && bit <= 255); - if (bit < 64) - set->rl |= 1LL << bit; - else if (bit < 128) - set->rh |= 1LL << (bit - 64); - else if (bit < 192) - set->fl |= 1LL << (bit - 128); - else - set->fh |= 1LL << (bit - 192); -} - -jit_bool_t -jit_regset_tstbit(jit_regset_t *set, int32_t bit) -{ - assert(bit >= 0 && bit <= 255); - if (bit < 64) - return (!!(set->rl & (1LL << bit))); - else if (bit < 128) - return (!!(set->rh & (1LL << (bit - 64)))); - else if (bit < 192) - return (!!(set->fl & (1LL << (bit - 128)))); - return (!!(set->fh & (1LL << (bit - 192)))); -} - -unsigned long -jit_regset_scan1(jit_regset_t *set, int32_t offset) -{ - assert(offset >= 0 && offset <= 255); - for (; offset < 64; offset++) { - if (set->rl & (1LL << offset)) - return (offset); - } - for (; offset < 128; offset++) { - if (set->rh & (1LL << (offset - 64))) - return (offset); - } - for (; offset < 192; offset++) { - if (set->fl & (1LL << (offset - 128))) - return (offset); - } - for (; offset < 256; offset++) { - if (set->fh & (1LL << (offset - 192))) - return (offset); - } - return (ULONG_MAX); -} - -#elif __sparc__ && __WORDSIZE == 64 -void -jit_regset_com(jit_regset_t *u, jit_regset_t *v) -{ - u->rl = ~v->rl; u->rh = ~v->rh; -} - -void -jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w) -{ - u->rl = v->rl & w->rl; u->rh = v->rh & w->rh; -} - -void -jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w) -{ - u->rl = v->rl | w->rl; u->rh = v->rh | w->rh; -} - -void -jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w) -{ - u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh; -} - -void -jit_regset_set(jit_regset_t *u, jit_regset_t *v) -{ - u->rl = v->rl; u->rh = v->rh; -} - -void -jit_regset_set_mask(jit_regset_t *u, int32_t v) -{ - jit_bool_t w = !!(v & (v - 1)); - - assert(v >= 0 && v <= 128); - if (v == 0) - u->rl = u->rh = -1LL; - else if (v <= 64) { - u->rl = w ? (1LL << v) - 1 : -1LL; - u->rh = 0; - } - else { - u->rl = -1LL; - u->rh = w ? (1LL << (v - 64)) - 1 : -1LL; - } -} - -jit_bool_t -jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v) -{ - return !((u->rl == v && u->rh == 0)); -} - -void -jit_regset_set_ui(jit_regset_t *u, jit_word_t v) -{ - u->rl = v; - u->rh = 0; -} - -jit_bool_t -jit_regset_set_p(jit_regset_t *u) -{ - return (u->rl || u->rh); -} - -void -jit_regset_clrbit(jit_regset_t *set, int32_t bit) -{ - assert(bit >= 0 && bit <= 128); - if (bit < 64) - set->rl &= ~(1LL << bit); - else - set->rh &= ~(1LL << (bit - 64)); -} - -void -jit_regset_setbit(jit_regset_t *set, int32_t bit) -{ - assert(bit >= 0 && bit <= 127); - if (bit < 64) - set->rl |= 1LL << bit; - else - set->rh |= 1LL << (bit - 64); -} - -jit_bool_t -jit_regset_tstbit(jit_regset_t *set, int32_t bit) -{ - assert(bit >= 0 && bit <= 127); - if (bit < 64) - return (!!(set->rl & (1LL << bit))); - else - return (!!(set->rh & (1LL << (bit - 64)))); -} - -unsigned long -jit_regset_scan1(jit_regset_t *set, int32_t offset) -{ - assert(offset >= 0 && offset <= 127); - for (; offset < 64; offset++) { - if (set->rl & (1LL << offset)) - return (offset); - } - for (; offset < 128; offset++) { - if (set->rh & (1LL << (offset - 64))) - return (offset); - } - return (ULONG_MAX); -} - -#else -unsigned long -jit_regset_scan1(jit_regset_t *set, int32_t offset) -{ - jit_regset_t mask; - assert(offset >= 0 && offset <= 63); - if ((mask = *set >> offset)) { - for (;;) { - if (mask & 1) - return (offset); - mask >>= 1; - ++offset; - } - } - return (ULONG_MAX); -} -#endif - -void -_jit_save(jit_state_t *_jit, int32_t reg) -{ - reg = jit_regno(reg); - assert(!_jitc->realize); - _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg); -} - -void -_jit_load(jit_state_t *_jit, int32_t reg) -{ - jit_node_t *node; - - reg = jit_regno(reg); - assert(!_jitc->realize); - assert(_jitc->spill[reg] != NULL); - node = jit_new_node_w(jit_code_load, reg); - /* create a path to flag the save/load is not required */ - node->link = _jitc->spill[reg]; - node->link->link = node; - _jitc->spill[reg] = NULL; -} - -static jit_word_t -hash_data(const void *data, jit_word_t length) -{ - const uint8_t *ptr; - jit_word_t i, key; - for (i = key = 0, ptr = data; i < length; i++) - key = (key << (key & 1)) ^ ptr[i]; - return (key); -} - -jit_pointer_t -_jit_address(jit_state_t *_jit, jit_node_t *node) -{ - assert(_jitc->done); - assert(node != NULL && - /* If a node type that is documented to be a fixed marker */ - (node->code == jit_code_note || node->code == jit_code_name || - /* If another special fixed marker, returned by jit_indirect() */ - (node->code == jit_code_label && (node->flag & jit_flag_use) != 0))); - return ((jit_pointer_t)node->u.w); -} - -jit_node_t * -_jit_data(jit_state_t *_jit, const void *data, - jit_word_t length, int32_t align) -{ - jit_word_t key; - jit_node_t *node; - - assert(!_jitc->realize); - - /* Ensure there is space even if asking for a duplicate */ - if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) { - jit_word_t size; - - size = (_jit->data.length + length + 4096) & - 4095; - assert(size >= _jit->data.length); - if (_jitc->data.ptr == NULL) - jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size); - else - jit_realloc((jit_pointer_t *)&_jitc->data.ptr, - _jit->data.length, size); - _jit->data.length = size; - } - if (_jitc->data.table == NULL) - jit_alloc((jit_pointer_t *)&_jitc->data.table, - (_jitc->data.size = 16) * sizeof(jit_node_t*)); - - key = hash_data(data, length) & (_jitc->data.size - 1); - node = _jitc->data.table[key]; - for (; node; node = node->next) { - if (node->v.w == length && - memcmp(_jitc->data.ptr + node->u.w, data, length) == 0) - break; - } - - if (!node) { - node = jit_new_node_no_link(jit_code_data); - if (!align) - align = length; - switch (align) { - case 0: case 1: - break; - case 2: - _jitc->data.offset = (_jitc->data.offset + 1) & -2; - break; - case 3: case 4: - _jitc->data.offset = (_jitc->data.offset + 3) & -4; - break; - default: - _jitc->data.offset = (_jitc->data.offset + 7) & -8; - break; - } - node->u.w = _jitc->data.offset; - node->v.w = length; - jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length); - _jitc->data.offset += length; - - node->next = _jitc->data.table[key]; - _jitc->data.table[key] = node; - ++_jitc->data.count; - - /* Rehash if more than 75% used table */ - if (_jitc->data.count > - (_jitc->data.size >> 1) + (_jitc->data.size >> 2) && - (_jitc->data.size << 1) > _jitc->data.size) { - jit_word_t i; - jit_node_t **hash; - jit_node_t *next; - jit_node_t *temp; - - jit_alloc((jit_pointer_t *)&hash, - (_jitc->data.size << 1) * sizeof(jit_node_t*)); - for (i = 0; i < _jitc->data.size; i++) { - temp = _jitc->data.table[i]; - for (; temp; temp = next) { - next = temp->next; - key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) & - ((_jitc->data.size << 1) - 1); - temp->next = hash[key]; - hash[key] = temp; - } - } - jit_free((jit_pointer_t *)&_jitc->data.table); - _jitc->data.table = hash; - _jitc->data.size <<= 1; - } - } - - return (node); -} - -static void -_new_pool(jit_state_t *_jit) -{ - jit_node_t *list; - int32_t offset; - - if (_jitc->pool.offset >= _jitc->pool.length) { - int32_t length; - - length = _jitc->pool.length + 16; - jit_realloc((jit_pointer_t *)&_jitc->pool.ptr, - _jitc->pool.length * sizeof(jit_node_t *), - length * sizeof(jit_node_t *)); - _jitc->pool.length = length; - } - jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset), - sizeof(jit_node_t) * 1024); - list = _jitc->pool.ptr[_jitc->pool.offset]; - for (offset = 1; offset < 1024; offset++, list++) - list->next = list + 1; - list->next = _jitc->list; - _jitc->list = _jitc->pool.ptr[_jitc->pool.offset]; - ++_jitc->pool.offset; -} - -static jit_node_t * -_new_node(jit_state_t *_jit, jit_code_t code) -{ - jit_node_t *node; - - if (_jitc->list == NULL) - new_pool(); - node = _jitc->list; - _jitc->list = node->next; - if (_jitc->synth) - node->flag |= jit_flag_synth; - node->next = NULL; - node->code = code; - - return (node); -} - -static inline jit_node_t * -_link_node(jit_state_t *_jit, jit_node_t *node) -{ - if (_jitc->tail) - _jitc->tail->next = node; - else - _jitc->head = node; - return (_jitc->tail = node); -} - -static inline void -_del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) -{ - if (prev == node) { - assert(prev == _jitc->head); - _jitc->head = node->next; - } - else - prev->next = node->next; - memset(node, 0, sizeof(jit_node_t)); - node->next = _jitc->list; - _jitc->list = node; -} - -static inline void -_free_node(jit_state_t *_jit, jit_node_t *node) -{ - memset(node, 0, sizeof(jit_node_t)); - node->next = _jitc->list; - _jitc->list = node; -} - -static void -_del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) -{ - jit_block_t *block; - - /* only allow call to del_label on linked labels */ - block = _jitc->blocks.ptr + node->v.w; - assert(block->label == node); - - /* del_label() should only be called when optimizing. - * This will leave an empty block index */ - jit_regset_del(&block->reglive); - jit_regset_del(&block->regmask); - block->label = NULL; - - /* redundant, should be already true */ - assert(node->link == NULL); - del_node(prev, node); -} - -static void -_bmp_init(jit_state_t *_jit) -{ - _jitc->blockmask.length = 16; - jit_alloc((jit_pointer_t *)&_jitc->blockmask.ptr, - sizeof(jit_word_t) * _jitc->blockmask.length); -} - -static void -_bmp_clear(jit_state_t *_jit) -{ - _jitc->blockmask.length = 0; - jit_free((jit_pointer_t *)&_jitc->blockmask.ptr); -} - -static void -_bmp_set(jit_state_t *_jit, jit_word_t bit) -{ - jit_word_t woff, boff; - - woff = bit >> bmp_shift; - boff = 1LL << (bit & (__WORDSIZE - 1)); - if (woff >= _jitc->blockmask.length) { - jit_word_t length = (woff + 16) & -16; - jit_realloc((jit_pointer_t *)&_jitc->blockmask.ptr, - _jitc->blockmask.length * sizeof(jit_word_t), - length * sizeof(jit_word_t)); - _jitc->blockmask.length = length; - } - _jitc->blockmask.ptr[woff] |= boff; -} - -static void -_bmp_clr(jit_state_t *_jit, jit_word_t bit) -{ - jit_word_t woff, boff; - - woff = bit >> bmp_shift; - boff = 1LL << (bit & (__WORDSIZE - 1)); - if (woff < _jitc->blockmask.length) - _jitc->blockmask.ptr[woff] &= ~boff; -} - -static jit_bool_t -_bmp_tst(jit_state_t *_jit, jit_word_t bit) -{ - jit_word_t woff, boff; - - woff = bit >> bmp_shift; - boff = 1LL << (bit & (__WORDSIZE - 1)); - if (woff < _jitc->blockmask.length) - return ((_jitc->blockmask.ptr[woff] & boff) != 0); - return (0); } jit_state_t * @@ -919,2486 +55,97 @@ jit_new_state(void) { jit_state_t *_jit; - jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t)); - jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t)); - jit_regset_new(&_jitc->regarg); - jit_regset_new(&_jitc->regsav); - jit_regset_new(&_jitc->reglive); - jit_regset_new(&_jitc->regmask); - bmp_init(); + _jit = malloc (sizeof (*_jit)); + if (!_jit) + abort (); - jit_init(); + memset(_jit, 0, sizeof (*_jit)); - jit_alloc((jit_pointer_t *)&_jitc->spill, - _jitc->reglen * sizeof(jit_node_t*)); - jit_alloc((jit_pointer_t *)&_jitc->gen, - _jitc->reglen * sizeof(int32_t)); - jit_alloc((jit_pointer_t *)&_jitc->values, - _jitc->reglen * sizeof(jit_value_t)); + jit_init (_jit); - jit_alloc((jit_pointer_t *)&_jitc->patches.ptr, - (_jitc->patches.length = 1024) * sizeof(jit_patch_t)); - jit_alloc((jit_pointer_t *)&_jitc->functions.ptr, - (_jitc->functions.length = 16) * sizeof(jit_function_t)); - jit_alloc((jit_pointer_t *)&_jitc->pool.ptr, - (_jitc->pool.length = 16) * sizeof(jit_node_t*)); - jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr, - (_jitc->blocks.length = 16) * sizeof(jit_block_t)); -#if __arm__ && DISASSEMBLER - jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr, - (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t)); -#endif - - /* allocate at most one extra note in case jit_name() is - * never called, or called after adding at least one note */ - _jit->note.length = 1; - _jitc->note.size = sizeof(jit_note_t); - - return (_jit); + return _jit; } void -_jit_clear_state(jit_state_t *_jit) +jit_destroy_state(jit_state_t *_jit) { -#if DEVEL_DISASSEMBLER -# define jit_really_clear_state() _jit_really_clear_state(_jit) -} - -void _jit_really_clear_state(jit_state_t *_jit) -{ -#endif - jit_word_t offset; - jit_function_t *function; - - /* release memory not required at jit execution time and set - * pointers to NULL to explicitly know they are released */ - _jitc->head = _jitc->tail = NULL; - - bmp_clear(); - - jit_free((jit_pointer_t *)&_jitc->data.table); - _jitc->data.size = _jitc->data.count = 0; - - jit_free((jit_pointer_t *)&_jitc->spill); - jit_free((jit_pointer_t *)&_jitc->gen); - jit_free((jit_pointer_t *)&_jitc->values); - - jit_free((jit_pointer_t *)&_jitc->blocks.ptr); - - jit_free((jit_pointer_t *)&_jitc->patches.ptr); - _jitc->patches.offset = _jitc->patches.length = 0; - - for (offset = 0; offset < _jitc->functions.offset; offset++) { - function = _jitc->functions.ptr + offset; - jit_free((jit_pointer_t *)&function->regoff); - } - jit_free((jit_pointer_t *)&_jitc->functions.ptr); - _jitc->functions.offset = _jitc->functions.length = 0; - _jitc->function = NULL; - - for (offset = 0; offset < _jitc->pool.offset; offset++) - jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset)); - jit_free((jit_pointer_t *)&_jitc->pool.ptr); - _jitc->pool.offset = _jitc->pool.length = 0; - _jitc->list = NULL; - - _jitc->note.head = _jitc->note.tail = - _jitc->note.name = _jitc->note.note = NULL; - _jitc->note.base = NULL; - -#if __arm__ && DISASSEMBLER - jit_free((jit_pointer_t *)&_jitc->data_info.ptr); -#endif - -#if __powerpc64__ || __ia64__ - jit_free((jit_pointer_t *)&_jitc->prolog.ptr); -#endif - -#if __ia64__ - jit_regset_del(&_jitc->regs); -#endif - - jit_free((jit_pointer_t *)&_jitc); -} - -void -_jit_destroy_state(jit_state_t *_jit) -{ -#if DEVEL_DISASSEMBLER - jit_really_clear_state(); -#endif - if (!_jit->user_code) - munmap(_jit->code.ptr, _jit->code.length); - if (!_jit->user_data) - munmap(_jit->data.ptr, _jit->data.length); - jit_free((jit_pointer_t *)&_jit); -} - -void -_jit_synth_inc(jit_state_t *_jit) -{ - assert(_jitc->synth < 8); - ++_jitc->synth; -} - -jit_node_t * -_jit_new_node(jit_state_t *_jit, jit_code_t code) -{ - assert(!_jitc->realize); - return (link_node(new_node(code))); -} - -jit_node_t * -_jit_new_node_no_link(jit_state_t *_jit, jit_code_t code) -{ - assert(!_jitc->realize); - return (new_node(code)); -} - -void -_jit_link_node(jit_state_t *_jit, jit_node_t *node) -{ - assert(!_jitc->realize); - link_node(node); -} - -void -_jit_synth_dec(jit_state_t *_jit) -{ - assert(_jitc->synth > 0); - --_jitc->synth; -} - -jit_node_t * -_jit_new_node_w(jit_state_t *_jit, jit_code_t code, - jit_word_t u) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_f(jit_state_t *_jit, jit_code_t code, - jit_float32_t u) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.f = u; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_d(jit_state_t *_jit, jit_code_t code, - jit_float64_t u) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.d = u; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_p(jit_state_t *_jit, jit_code_t code, - jit_pointer_t u) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.p = u; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_ww(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_word_t v) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - node->v.w = v; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_wp(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_pointer_t v) -{ - return (jit_new_node_ww(code, u, (jit_word_t)v)); -} - -jit_node_t * -_jit_new_node_fp(jit_state_t *_jit, jit_code_t code, - jit_float32_t u, jit_pointer_t v) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.f = u; - node->v.w = (jit_word_t)v; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_dp(jit_state_t *_jit, jit_code_t code, - jit_float64_t u, jit_pointer_t v) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.d = u; - node->v.w = (jit_word_t)v; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_pw(jit_state_t *_jit, jit_code_t code, - jit_pointer_t u, jit_word_t v) -{ - return (jit_new_node_ww(code, (jit_word_t)u, v)); -} - -jit_node_t * -_jit_new_node_wf(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_float32_t v) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - node->v.f = v; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_wd(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_float64_t v) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - node->v.d = v; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_www(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_word_t v, jit_word_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - node->v.w = v; - node->w.w = w; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_qww(jit_state_t *_jit, jit_code_t code, - int32_t l, jit_int32_t h, - jit_word_t v, jit_word_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - assert(l != h); - node->u.q.l = l; - node->u.q.h = h; - node->v.w = v; - node->w.w = w; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_wwf(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_word_t v, jit_float32_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - node->v.w = v; - node->w.f = w; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_wwd(jit_state_t *_jit, jit_code_t code, - jit_word_t u, jit_word_t v, jit_float64_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.w = u; - node->v.w = v; - node->w.d = w; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_pww(jit_state_t *_jit, jit_code_t code, - jit_pointer_t u, jit_word_t v, jit_word_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.p = u; - node->v.w = v; - node->w.w = w; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_pwf(jit_state_t *_jit, jit_code_t code, - jit_pointer_t u, jit_word_t v, jit_float32_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.p = u; - node->v.w = v; - node->w.f = w; - return (link_node(node)); -} - -jit_node_t * -_jit_new_node_pwd(jit_state_t *_jit, jit_code_t code, - jit_pointer_t u, jit_word_t v, jit_float64_t w) -{ - jit_node_t *node = new_node(code); - assert(!_jitc->realize); - node->u.p = u; - node->v.w = v; - node->w.d = w; - return (link_node(node)); -} - -jit_node_t * -_jit_label(jit_state_t *_jit) -{ - jit_node_t *node; - - if (!(node = _jitc->tail) || node->code != jit_code_label) { - node = jit_forward(); - jit_link(node); - } - - return (node); -} - -jit_node_t * -_jit_forward(jit_state_t *_jit) -{ - return (jit_new_node_no_link(jit_code_label)); -} - -jit_node_t * -_jit_indirect(jit_state_t *_jit) -{ - jit_node_t *node; - - node = jit_label(); - node->flag |= jit_flag_use; - - return (node); -} - -void -_jit_link(jit_state_t *_jit, jit_node_t *node) -{ - jit_block_t *block; - - assert((node->code == jit_code_label || - node->code == jit_code_prolog || - node->code == jit_code_epilog) && !node->next); - jit_link_node(node); - if (_jitc->blocks.offset >= _jitc->blocks.length) { - jit_word_t length; - - length = _jitc->blocks.length + 16; - jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr, - _jitc->blocks.length * sizeof(jit_block_t), - length * sizeof(jit_block_t)); - _jitc->blocks.length = length; - } - block = _jitc->blocks.ptr + _jitc->blocks.offset; - block->label = node; - node->v.w = _jitc->blocks.offset; - jit_regset_new(&block->reglive); - jit_regset_new(&block->regmask); - ++_jitc->blocks.offset; -} - -jit_bool_t -_jit_forward_p(jit_state_t *_jit, jit_node_t *node) -{ - return (node->code == jit_code_label && !node->next && node != _jitc->tail); -} - -jit_bool_t -_jit_indirect_p(jit_state_t *_jit, jit_node_t *node) -{ - return (node->code == jit_code_label && !!(node->flag & jit_flag_use)); -} - -jit_bool_t -_jit_target_p(jit_state_t *_jit, jit_node_t *node) -{ - return (node->code == jit_code_label && !!node->link); -} - -void -_jit_prepare(jit_state_t *_jit) -{ - assert(_jitc->function != NULL); - _jitc->function->call.call = jit_call_default; - _jitc->function->call.argi = - _jitc->function->call.argf = - _jitc->function->call.size = 0; - _jitc->prepare = jit_new_node(jit_code_prepare); -} - -void -_jit_patch(jit_state_t* _jit, jit_node_t *instr) -{ - jit_node_t *label; - - if (!(label = _jitc->tail) || label->code != jit_code_label) - label = jit_label(); - jit_patch_at(instr, label); -} - -int32_t -_jit_classify(jit_state_t *_jit, jit_code_t code) -{ - int32_t mask; - - switch (code) { - case jit_code_data: case jit_code_save: case jit_code_load: - case jit_code_name: case jit_code_label: case jit_code_note: - case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push: - case jit_code_epilog: case jit_code_ret: case jit_code_prepare: - mask = 0; - break; - case jit_code_live: case jit_code_va_end: - case jit_code_retr: case jit_code_retr_f: case jit_code_retr_d: - case jit_code_pushargr: case jit_code_pushargr_f: - case jit_code_pushargr_d: - case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */ - mask = jit_cc_a0_reg; - break; - case jit_code_align: case jit_code_reti: case jit_code_pushargi: - case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */ - mask = jit_cc_a0_int; - break; - case jit_code_reti_f: case jit_code_pushargi_f: - mask = jit_cc_a0_flt; - break; - case jit_code_reti_d: case jit_code_pushargi_d: - mask = jit_cc_a0_dbl; - break; - case jit_code_allocai: - mask = jit_cc_a0_int|jit_cc_a1_int; - break; - case jit_code_arg: case jit_code_arg_f: case jit_code_arg_d: - mask = jit_cc_a0_int|jit_cc_a0_arg; - break; - case jit_code_calli: case jit_code_jmpi: - mask = jit_cc_a0_jmp; - break; - case jit_code_callr: case jit_code_jmpr: - mask = jit_cc_a0_reg|jit_cc_a0_jmp; - break; - case jit_code_retval_c: case jit_code_retval_uc: - case jit_code_retval_s: case jit_code_retval_us: - case jit_code_retval_i: case jit_code_retval_ui: - case jit_code_retval_l: - case jit_code_retval_f: case jit_code_retval_d: - case jit_code_va_start: - mask = jit_cc_a0_reg|jit_cc_a0_chg; - break; - case jit_code_getarg_c: case jit_code_getarg_uc: - case jit_code_getarg_s: case jit_code_getarg_us: - case jit_code_getarg_i: case jit_code_getarg_ui: - case jit_code_getarg_l: - case jit_code_getarg_f: case jit_code_getarg_d: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg; - break; - case jit_code_putargr: case jit_code_putargr_f: - case jit_code_putargr_d: - mask = jit_cc_a0_reg|jit_cc_a1_arg; - break; - case jit_code_putargi: - mask = jit_cc_a0_int|jit_cc_a1_arg; - break; - case jit_code_putargi_f: - mask = jit_cc_a0_flt|jit_cc_a1_arg; - break; - case jit_code_putargi_d: - mask = jit_cc_a0_dbl|jit_cc_a1_arg; - break; - case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc: - case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i: - case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f: - case jit_code_ldi_d: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int; - break; - case jit_code_movi_f: case jit_code_movi_f_w: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt; - break; - case jit_code_movi_d: case jit_code_movi_d_w: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl; - break; - case jit_code_movi_d_ww: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg| - jit_cc_a2_dbl; - break; - case jit_code_negr: case jit_code_comr: case jit_code_movr: - case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s: - case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui: - case jit_code_truncr_f_i: case jit_code_truncr_f_l: - case jit_code_truncr_d_i: case jit_code_truncr_d_l: - case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul: - case jit_code_ldr_c: case jit_code_ldr_uc: - case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i: - case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f: - case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f: - case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f: - case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d: - case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d: - case jit_code_ldr_d: - case jit_code_movr_w_f: case jit_code_movr_f_w: - case jit_code_movr_w_d: case jit_code_movr_d_w: - case jit_code_va_arg: case jit_code_va_arg_d: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg; - break; - case jit_code_movr_d_ww: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg| - jit_cc_a2_reg; - break; - case jit_code_addi: case jit_code_addxi: case jit_code_addci: - case jit_code_subi: case jit_code_subxi: case jit_code_subci: - case jit_code_rsbi: - case jit_code_muli: case jit_code_divi: case jit_code_divi_u: - case jit_code_remi: case jit_code_remi_u: case jit_code_andi: - case jit_code_ori: case jit_code_xori: case jit_code_lshi: - case jit_code_rshi: case jit_code_rshi_u: case jit_code_lti: - case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u: - case jit_code_eqi: case jit_code_gei: case jit_code_gei_u: - case jit_code_gti: case jit_code_gti_u: case jit_code_nei: - case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s: - case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui: - case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int; - break; - case jit_code_qmuli: case jit_code_qmuli_u: - case jit_code_qdivi: case jit_code_qdivi_u: - mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg| - jit_cc_a1_reg|jit_cc_a2_int; - break; - case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f: - case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f: - case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f: - case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f: - case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f: - case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f: - case jit_code_unordi_f: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt; - break; - case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d: - case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d: - case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d: - case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d: - case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d: - case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d: - case jit_code_unordi_d: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl; - break; - case jit_code_addr: case jit_code_addxr: case jit_code_addcr: - case jit_code_subr: case jit_code_subxr: case jit_code_subcr: - case jit_code_mulr: case jit_code_divr: case jit_code_divr_u: - case jit_code_remr: case jit_code_remr_u: case jit_code_andr: - case jit_code_orr: case jit_code_xorr: case jit_code_lshr: - case jit_code_rshr: case jit_code_rshr_u: case jit_code_ltr: - case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u: - case jit_code_eqr: case jit_code_ger: case jit_code_ger_u: - case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner: - case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s: - case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui: - case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f: - case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f: - case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f: - case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f: - case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f: - case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f: - case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d: - case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d: - case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d: - case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d: - case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d: - case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d: - case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d: - case jit_code_movr_ww_d: - mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg; - break; - case jit_code_qmulr: case jit_code_qmulr_u: - case jit_code_qdivr: case jit_code_qdivr_u: - mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg| - jit_cc_a1_reg|jit_cc_a2_reg; - break; - case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i: - case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d: - mask = jit_cc_a0_int|jit_cc_a1_reg; - break; - case jit_code_blti: case jit_code_blti_u: case jit_code_blei: - case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei: - case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u: - case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci: - mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int; - break; - case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f: - case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f: - case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f: - case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f: - case jit_code_bordi_f: case jit_code_bunordi_f: - mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt; - break; - case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d: - case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d: - case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d: - case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d: - case jit_code_bordi_d: case jit_code_bunordi_d: - mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl; - break; - case jit_code_allocar: /* synthesized instructions make it - * equivalent to jit_cc_a0_chg */ - case jit_code_str_c: case jit_code_str_s: case jit_code_str_i: - case jit_code_str_l: case jit_code_str_f: case jit_code_str_d: - mask = jit_cc_a0_reg|jit_cc_a1_reg; - break; - case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i: - case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d: - mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg; - break; - case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler: - case jit_code_bler_u: case jit_code_beqr: case jit_code_bger: - case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u: - case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr: - case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f: - case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f: - case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f: - case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f: - case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d: - case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d: - case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d: - case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d: - case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d: - case jit_code_bunordr_d: - mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg; - break; - case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi: - case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u: - case jit_code_bxsubi: case jit_code_bxsubi_u: - mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int; - break; - case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i: - case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d: - mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg; - break; - case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr: - case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u: - case jit_code_bxsubr: case jit_code_bxsubr_u: - mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg; - break; - default: - abort(); - } - - return (mask); -} - -void -_jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address) -{ - int32_t mask; - - if (instr->code == jit_code_movi) - instr->v.p = address; - else { - mask = jit_classify(instr->code); - assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp); - instr->u.p = address; - } -} - -void -_jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label) -{ - int32_t mask; - - assert(!(instr->flag & jit_flag_node)); - instr->flag |= jit_flag_node; - switch (instr->code) { - case jit_code_movi: - assert(label->code == jit_code_label || - label->code == jit_code_data); - instr->v.n = label; - if (label->code == jit_code_data) - instr->flag |= jit_flag_data; - break; - case jit_code_jmpi: - assert(label->code == jit_code_label || - label->code == jit_code_epilog); - instr->u.n = label; - break; - default: - mask = jit_classify(instr->code); - assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp); - assert(label->code == jit_code_label); - instr->u.n = label; - break; - } - /* link field is used as list of nodes associated with a given label */ - instr->link = label->link; - label->link = instr; -} - -void -_jit_optimize(jit_state_t *_jit) -{ - jit_bool_t jump; - int32_t mask; - jit_node_t *node; - jit_block_t *block; - jit_word_t offset; - - _jitc->function = NULL; - - thread_jumps(); - sequential_labels(); - - /* create initial mapping of live register values - * at the start of a basic block */ - for (offset = 0; offset < _jitc->blocks.offset; offset++) { - block = _jitc->blocks.ptr + offset; - if (!block->label) - continue; - if (block->label->code != jit_code_epilog) { - jit_setup(block); - jit_regset_set(&block->setmask, &block->regmask); - } - } - /* call jit_update resolving undefined values in reverse - * order so that sequential code would find most data already - * resolved when reaching the start of a new basic block */ - for (offset = _jitc->blocks.offset - 1; offset >= 0; offset--) { - block = _jitc->blocks.ptr + offset; - if (!block->label) - continue; - if (block->label->code != jit_code_epilog) { - jit_regset_set(&_jitc->regmask, &block->regmask); - jit_update(block->label->next, &block->reglive, &_jitc->regmask, 1); - } - } - /* do a second pass from start to properly handle some conditions - * of very long living registers that are not referenced for - * several blocks */ - bmp_zero(); - for (offset = 0; offset < _jitc->blocks.offset; offset++) { - block = _jitc->blocks.ptr + offset; - if (!block->label) - continue; - if (block->label->code != jit_code_epilog) { - jit_regset_set(&_jitc->regmask, &block->setmask); - jit_update(block->label->next, &block->reglive, &_jitc->regmask, 1); - } - } - - patch_registers(); - simplify(); - - /* figure out labels that are only reached with a jump - * and is required to do a simple redundant_store removal - * on jit_beqi below */ - jump = 1; - for (node = _jitc->head; node; node = node->next) { - switch (node->code) { - case jit_code_label: - if (!jump) - node->flag |= jit_flag_head; - break; - case jit_code_jmpi: case jit_code_jmpr: - case jit_code_epilog: - jump = 1; - break; - case jit_code_data: case jit_code_note: - break; - default: - jump = 0; - break; - } - } - - for (node = _jitc->head; node; node = node->next) { - mask = jit_classify(node->code); - if (mask & jit_cc_a0_reg) - node->u.w &= ~jit_regno_patch; - if (mask & jit_cc_a1_reg) - node->v.w &= ~jit_regno_patch; - if (mask & jit_cc_a2_reg) - node->w.w &= ~jit_regno_patch; - switch (node->code) { - case jit_code_prolog: - _jitc->function = _jitc->functions.ptr + node->w.w; - break; - case jit_code_epilog: - _jitc->function = NULL; - break; - case jit_code_beqi: - redundant_store(node, 1); - break; - case jit_code_bnei: - redundant_store(node, 0); - break; - default: -#if JIT_HASH_CONSTS - if (mask & jit_cc_a0_flt) { - node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a0_dbl) { - node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a1_flt) { - node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a1_dbl) { - node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a2_flt) { - node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a2_dbl) { - node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8); - node->flag |= jit_flag_node | jit_flag_data; - } -#endif - if (_jitc->function) { - if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) == - (jit_cc_a0_reg|jit_cc_a0_chg)) { - if (mask & jit_cc_a0_rlh) { - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->u.q.l)); - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->u.q.h)); - } - else - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->u.w)); - } - if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) == - (jit_cc_a1_reg|jit_cc_a1_chg)) - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->v.w)); - if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) == - (jit_cc_a2_reg|jit_cc_a2_chg)) - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->w.w)); - } - break; - } - } -} - -void -_jit_reglive(jit_state_t *_jit, jit_node_t *node) -{ - int32_t spec; - int32_t value; - jit_block_t *block; - - switch (node->code) { - case jit_code_label: case jit_code_prolog: case jit_code_epilog: - block = _jitc->blocks.ptr + node->v.w; - jit_regset_set(&_jitc->reglive, &block->reglive); - break; - case jit_code_callr: - value = jit_regno(node->u.w); - if (!(node->u.w & jit_regno_patch)) { - jit_regset_setbit(&_jitc->reglive, value); - } - case jit_code_calli: - for (value = 0; value < _jitc->reglen; value++) { - spec = jit_class(_rvs[value].spec); - if ((spec & jit_class_arg) && jit_regarg_p(node, value)) - jit_regset_setbit(&_jitc->reglive, value); - else if (!(spec & jit_class_sav)) - jit_regset_clrbit(&_jitc->reglive, value); - } - break; - default: - value = jit_classify(node->code); - if (value & jit_cc_a0_reg) { - if (value & jit_cc_a0_rlh) { - if (!(node->u.q.l & jit_regno_patch)) { - if (value & jit_cc_a0_chg) { - jit_regset_clrbit(&_jitc->reglive, node->u.q.l); - jit_regset_setbit(&_jitc->regmask, node->u.q.l); - } - else - jit_regset_setbit(&_jitc->reglive, node->u.q.l); - } - if (!(node->u.q.h & jit_regno_patch)) { - if (value & jit_cc_a0_chg) { - jit_regset_clrbit(&_jitc->reglive, node->u.q.h); - jit_regset_setbit(&_jitc->regmask, node->u.q.h); - } - else - jit_regset_setbit(&_jitc->reglive, node->u.q.h); - } - } - else { - if (!(node->u.w & jit_regno_patch)) { - if (value & jit_cc_a0_chg) { - jit_regset_clrbit(&_jitc->reglive, node->u.w); - jit_regset_setbit(&_jitc->regmask, node->u.w); - } - else - jit_regset_setbit(&_jitc->reglive, node->u.w); - } - } - } - if ((value & jit_cc_a1_reg) && !(node->v.w & jit_regno_patch)) { - if (value & jit_cc_a1_chg) { - jit_regset_clrbit(&_jitc->reglive, node->v.w); - jit_regset_setbit(&_jitc->regmask, node->v.w); - } - else - jit_regset_setbit(&_jitc->reglive, node->v.w); - } - if ((value & jit_cc_a2_reg) && !(node->w.w & jit_regno_patch)) { - if (value & jit_cc_a2_chg) { - jit_regset_clrbit(&_jitc->reglive, node->w.w); - jit_regset_setbit(&_jitc->regmask, node->w.w); - } - else - jit_regset_setbit(&_jitc->reglive, node->w.w); - } - if (jit_regset_set_p(&_jitc->regmask)) { - bmp_zero(); - jit_update(node->next, &_jitc->reglive, &_jitc->regmask, 1); - if (jit_regset_set_p(&_jitc->regmask)) { - /* any unresolved live state is considered as live */ - jit_regset_ior(&_jitc->reglive, - &_jitc->reglive, &_jitc->regmask); - jit_regset_set_ui(&_jitc->regmask, 0); - } - } - break; - } -} - -void -_jit_regarg_set(jit_state_t *_jit, jit_node_t *node, int32_t value) -{ -#if GET_JIT_SIZE - jit_size_prepare(); -#endif - if (value & jit_cc_a0_reg) { - if (value & jit_cc_a0_rlh) { - jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l)); - jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h)); - } - else - jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w)); - } - if (value & jit_cc_a1_reg) - jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w)); - if (value & jit_cc_a2_reg) - jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w)); -} - -void -_jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, int32_t value) -{ -#if GET_JIT_SIZE - jit_size_collect(node); -#endif - if (value & jit_cc_a0_reg) { - if (value & jit_cc_a0_rlh) { - jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l)); - jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h)); - } - else - jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w)); - } - if (value & jit_cc_a1_reg) - jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w)); - if (value & jit_cc_a2_reg) - jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w)); -} - -void -_jit_realize(jit_state_t *_jit) -{ - assert(!_jitc->realize); - if (_jitc->function) - jit_epilog(); - jit_optimize(); - _jitc->realize = 1; - - /* ensure it is aligned */ - _jitc->data.offset = (_jitc->data.offset + 7) & -8; - -#if GET_JIT_SIZE - /* Heuristic to guess code buffer size */ - _jitc->mult = 4; - _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult; -#else - _jit->code.length = jit_get_size(); -#endif -} - -void -_jit_dataset(jit_state_t *_jit) -{ - uint8_t *ptr; - jit_node_t *node; - jit_word_t offset; -#if defined(__sgi) - int mmap_fd; -#endif - - assert(!_jitc->dataset); - if (!_jit->user_data) { - - /* create read only data buffer */ - _jit->data.length = (_jitc->data.offset + - /* reserve space for annotations */ - _jitc->note.size + 4095) & -4096; -#if defined(__sgi) - mmap_fd = open("/dev/zero", O_RDWR); -#endif - _jit->data.ptr = mmap(NULL, _jit->data.length, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, mmap_fd, 0); - assert(_jit->data.ptr != MAP_FAILED); -#if defined(__sgi) - close(mmap_fd); -#endif - } - - if (!_jitc->no_data) - jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset); - - if (_jitc->no_note) { - /* Space for one note is always allocated, so revert it here - * if after jit_new_state was called, it is also requested to - * not generate annotation information */ - _jit->note.length = 0; - _jitc->note.size = 0; - } - else { - _jitc->note.base = _jit->data.ptr; - if (!_jitc->no_data) - _jitc->note.base += _jitc->data.offset; - memset(_jitc->note.base, 0, _jitc->note.size); - } - - if (_jit->user_data) - /* Need the temporary hashed data until jit_emit is finished */ - ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr; - else { - ptr = _jit->data.ptr; - /* Temporary hashed data no longer required */ - jit_free((jit_pointer_t *)&_jitc->data.ptr); - } - - for (offset = 0; offset < _jitc->data.size; offset++) { - for (node = _jitc->data.table[offset]; node; node = node->next) { - node->flag |= jit_flag_patch; - node->u.w = (jit_word_t)(ptr + node->u.w); - } - } - - _jitc->dataset = 1; + free (_jit); } jit_pointer_t -_jit_get_code(jit_state_t *_jit, jit_word_t *length) +jit_address(jit_state_t *_jit) { - assert(_jitc->realize); - if (length) { - if (_jitc->done) - /* If code already generated, return exact size of code */ - *length = _jit->pc.uc - _jit->code.ptr; - else - /* Else return current size of the code buffer */ - *length = _jit->code.length; - } - - return (_jit->code.ptr); + /* TODO: FIXME */ + abort (); } void -_jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length) +jit_begin(jit_state_t *_jit, jit_addr_t addr, size_t length) { - assert(_jitc->realize); - _jit->code.ptr = ptr; - _jit->code.length = length; - _jit->user_code = 1; -} + ASSERT (!_jit->start); -jit_pointer_t -_jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size) -{ - assert(_jitc->realize); - if (data_size) - *data_size = _jitc->data.offset; - if (note_size) - *note_size = _jitc->note.size; - return (_jit->data.ptr); + _jit->start = addr; + _jit->limit = _jit->start + length; + jit_reset(_jit); } void -_jit_set_data(jit_state_t *_jit, jit_pointer_t ptr, - jit_word_t length, jit_word_t flags) +jit_reset(jit_state_t *_jit) { - assert(_jitc->realize); - if (flags & JIT_DISABLE_DATA) - _jitc->no_data = 1; - else - assert(length >= _jitc->data.offset); - if (flags & JIT_DISABLE_NOTE) - _jitc->no_note = 1; - else { - if (flags & JIT_DISABLE_DATA) - assert(length >= _jitc->note.size); - else - assert(length >= _jitc->data.offset + _jitc->note.size); - } - _jit->data.ptr = ptr; - _jit->data.length = length; - _jit->user_data = 1; + ASSERT (_jit->start); + _jit->pc.uc = _jit->start = _jit->limit = NULL; } -jit_pointer_t -_jit_emit(jit_state_t *_jit) +jit_addr_t +jit_end(jit_state_t *_jit, size_t *length) { - jit_pointer_t code; - jit_node_t *node; - size_t length; - int result; -#if defined(__sgi) - int mmap_fd; -#endif + uint8_t *code = _jit->start; + uint8_t *end = _jit->pc.uc; - if (!_jitc->realize) - jit_realize(); + ASSERT (code); + ASSERT (end > code); + ASSERT (end <= _jit->limit); - if (!_jitc->dataset) - jit_dataset(); + jit_flush (code, end); - _jitc->emit = 1; + if (length) { + *length = end - code; + } - if (!_jit->user_code) { -#if defined(__sgi) - mmap_fd = open("/dev/zero", O_RDWR); -#endif - _jit->code.ptr = mmap(NULL, _jit->code.length, - PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, mmap_fd, 0); - assert(_jit->code.ptr != MAP_FAILED); - } - _jitc->code.end = _jit->code.ptr + _jit->code.length - - jit_get_max_instr(); - _jit->pc.uc = _jit->code.ptr; + jit_reset (_jit); - for (;;) { - if ((code = emit_code()) == NULL) { - _jitc->patches.offset = 0; - for (node = _jitc->head; node; node = node->next) { - if (node->link && - (node->code == jit_code_label || - node->code == jit_code_epilog)) - node->flag &= ~jit_flag_patch; - } - if (_jit->user_code) - goto fail; -#if GET_JIT_SIZE - ++_jitc->mult; - length = _jitc->pool.length * 1024 * _jitc->mult; -#else - /* Should only happen on very special cases */ - length = _jit->code.length + 4096; -#endif + return code; +} -#if !HAVE_MREMAP - munmap(_jit->code.ptr, _jit->code.length); -#endif - -#if HAVE_MREMAP -# if __NetBSD__ - _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length, - _jit->code.ptr, length, 0); -# else - _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length, - length, MREMAP_MAYMOVE, NULL); -# endif -#else - _jit->code.ptr = mmap(NULL, length, - PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, mmap_fd, 0); -#endif - - assert(_jit->code.ptr != MAP_FAILED); - _jit->code.length = length; - _jitc->code.end = _jit->code.ptr + _jit->code.length - - jit_get_max_instr(); - _jit->pc.uc = _jit->code.ptr; - } - else - break; - } - -#if defined(__sgi) - if (!_jit->user_code) - close(mmap_fd); -#endif - - _jitc->done = 1; - if (!_jitc->no_note) - jit_annotate(); - - if (_jit->user_data) - jit_free((jit_pointer_t *)&_jitc->data.ptr); - else { - result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ); - assert(result == 0); - } - if (!_jit->user_code) { - result = mprotect(_jit->code.ptr, _jit->code.length, - PROT_READ | PROT_EXEC); - assert(result == 0); - } - - return (_jit->code.ptr); -fail: - return (NULL); +static int +is_power_of_two (unsigned x) +{ + return x && !(x & (x-1)); } void -_jit_frame(jit_state_t *_jit, int32_t frame) +jit_align(jit_state_t *_jit, unsigned align) { - jit_trampoline(frame, 1); + ASSERT (is_power_of_two (align)); + uintptr_t here = _jit->pc.w; + uintptr_t there = (here + align - 1) & ~(align - 1); + if (there - here) + jit_nop(_jit, there - here); } void -_jit_tramp(jit_state_t *_jit, int32_t frame) +jit_patch_here(jit_state_t *_jit, jit_reloc_t *reloc) { - jit_trampoline(frame, 0); + jit_patch_there (_jit, reloc, jit_address (_jit)); } void -_jit_trampoline(jit_state_t *_jit, int32_t frame, jit_bool_t prolog) +jit_patch_there(jit_state_t* _jit, jit_reloc_t *reloc, jit_pointer_t *addr) { - int32_t regno; - - /* Must be called after prolog, actually, just to simplify - * tests and know there is a current function and that - * _jitc->function->self.aoff is at the before any alloca value */ - assert(_jitc->tail && _jitc->tail->code == jit_code_prolog); - - /* + 24 for 3 possible spilled temporaries (that could be a double) */ - frame += 24; -#if defined(__hppa__) - frame += _jitc->function->self.aoff; -#else - frame -= _jitc->function->self.aoff; -#endif - _jitc->function->frame = frame; - if (prolog) - _jitc->function->define_frame = 1; - else - _jitc->function->assume_frame = 1; - for (regno = 0; regno < _jitc->reglen; regno++) - if (jit_class(_rvs[regno].spec) & jit_class_sav) - jit_regset_setbit(&_jitc->function->regset, regno); -} - -/* Compute initial reglive and regmask set values of a basic block. - * reglive is the set of known live registers - * regmask is the set of registers not referenced in the block - */ -static void -_jit_setup(jit_state_t *_jit, jit_block_t *block) -{ -#define reglive block->reglive -#define regmask block->regmask - jit_node_t *node; - jit_bool_t live; - jit_bool_t jump; - unsigned long value; - - jump = 0; - jit_regset_set_mask(®mask, _jitc->reglen); - for (node = block->label->next; node; node = node->next) { - switch (node->code) { - case jit_code_label: case jit_code_prolog: - case jit_code_epilog: - return; - default: - value = jit_classify(node->code); - if (value & jit_cc_a0_reg) { - live = !(value & jit_cc_a0_chg); - if (value & jit_cc_a0_rlh) { - if (!(node->u.q.l & jit_regno_patch) && - jit_regset_tstbit(®mask, node->u.q.l)) { - if (live || !jump) - jit_regset_clrbit(®mask, node->u.q.l); - if (live) - jit_regset_setbit(®live, node->u.q.l); - } - if (!(node->u.q.h & jit_regno_patch) && - jit_regset_tstbit(®mask, node->u.q.h)) { - if (live || !jump) - jit_regset_clrbit(®mask, node->u.q.h); - if (live) - jit_regset_setbit(®live, node->u.q.h); - } - } - else { - if (!(node->u.w & jit_regno_patch) && - jit_regset_tstbit(®mask, node->u.w)) { - if (live || !jump) - jit_regset_clrbit(®mask, node->u.w); - if (live) - jit_regset_setbit(®live, node->u.w); - } - } - } - if ((value & jit_cc_a1_reg) && - !(node->v.w & jit_regno_patch) && - jit_regset_tstbit(®mask, node->v.w)) { - live = !(value & jit_cc_a1_chg); - if (live || !jump) - jit_regset_clrbit(®mask, node->v.w); - if (live) - jit_regset_setbit(®live, node->v.w); - } - if ((value & jit_cc_a2_reg) && - !(node->w.w & jit_regno_patch) && - jit_regset_tstbit(®mask, node->w.w)) { - live = !(value & jit_cc_a2_chg); - if (live || !jump) - jit_regset_clrbit(®mask, node->w.w); - if (live) - jit_regset_setbit(®live, node->w.w); - } - if (value & jit_cc_a0_jmp) - jump = 1; - break; - } - } -#undef regmask -#undef reglive -} - -/* Remove bit of mask argument based on instructions arguments up to end - * of code or finding a basic block boundary, if a value is used as argument, - * also set the live bit to know value cannot be clobbered; if value is - * modified, just remove it from the mask as if it not already in the live - * bitmask, then the value is dead. - */ -static void -_jit_update(jit_state_t *_jit, jit_node_t *node, - jit_regset_t *live, jit_regset_t *mask, jit_bool_t recurse) -{ - int32_t spec; - int32_t regno; - jit_regset_t ztmp; - jit_regset_t zmask; - unsigned long value; - jit_block_t *block; - jit_node_t *label; - - for (; node; node = node->next) { - restart: - if (jit_regset_set_p(mask) == 0) - break; - switch (node->code) { - case jit_code_label: - block = _jitc->blocks.ptr + node->v.w; - jit_regset_and(&ztmp, mask, &block->reglive); - if (jit_regset_set_p(&ztmp)) { - jit_regset_ior(live, live, &ztmp); - jit_regset_com(&ztmp, &ztmp); - jit_regset_and(mask, mask, &ztmp); - } - if (bmp_tst(node->v.w)) - return; - bmp_set(node->v.w); - break; - case jit_code_prolog: - jit_regset_set_ui(mask, 0); - return; - case jit_code_epilog: - jit_regset_set_ui(mask, 0); - return; - case jit_code_callr: - value = jit_regno(node->u.w); - if (!(node->u.w & jit_regno_patch)) { - if (jit_regset_tstbit(mask, value)) { - jit_regset_clrbit(mask, value); - jit_regset_setbit(live, value); - } - } - case jit_code_calli: - for (value = 0; value < _jitc->reglen; ++value) { - value = jit_regset_scan1(mask, value); - if (value >= _jitc->reglen) - break; - spec = jit_class(_rvs[value].spec); - if (!(spec & jit_class_sav)) - jit_regset_clrbit(mask, value); - if ((spec & jit_class_arg) && jit_regarg_p(node, value)) - jit_regset_setbit(live, value); - } - break; - default: - value = jit_classify(node->code); - if (value & jit_cc_a2_reg) { - if (!(node->w.w & jit_regno_patch)) { - if (jit_regset_tstbit(mask, node->w.w)) { - jit_regset_clrbit(mask, node->w.w); - if (!(value & jit_cc_a2_chg)) - jit_regset_setbit(live, node->w.w); - } - } - } - if (value & jit_cc_a1_reg) { - if (!(node->v.w & jit_regno_patch)) { - if (jit_regset_tstbit(mask, node->v.w)) { - jit_regset_clrbit(mask, node->v.w); - if (!(value & jit_cc_a1_chg)) - jit_regset_setbit(live, node->v.w); - } - } - } - if (value & jit_cc_a0_reg) { - if (value & jit_cc_a0_rlh) { - if (!(node->u.q.l & jit_regno_patch)) { - if (jit_regset_tstbit(mask, node->u.q.l)) { - jit_regset_clrbit(mask, node->u.q.l); - if (!(value & jit_cc_a0_chg)) - jit_regset_setbit(live, node->u.q.l); - } - } - if (!(node->u.q.h & jit_regno_patch)) { - if (jit_regset_tstbit(mask, node->u.q.h)) { - jit_regset_clrbit(mask, node->u.q.h); - if (!(value & jit_cc_a0_chg)) - jit_regset_setbit(live, node->u.q.h); - } - } - } - else { - if (!(node->u.w & jit_regno_patch)) { - if (jit_regset_tstbit(mask, node->u.w)) { - jit_regset_clrbit(mask, node->u.w); - if (!(value & jit_cc_a0_chg)) - jit_regset_setbit(live, node->u.w); - } - } - } - } - if (value & jit_cc_a0_jmp) { - if (node->flag & jit_flag_node) { - label = node->u.n; - if (node->code == jit_code_jmpi) { - node = label; - goto restart; - } - block = _jitc->blocks.ptr + label->v.w; - jit_regset_and(&ztmp, mask, &block->reglive); - if (jit_regset_set_p(&ztmp)) { - jit_regset_ior(live, live, &ztmp); - jit_regset_com(&ztmp, &ztmp); - jit_regset_and(mask, mask, &ztmp); - } - if (bmp_tst(label->v.w)) - continue; - bmp_set(label->v.w); - if (jit_regset_set_p(mask) == 0) - return; - /* restore mask if branch is conditional */ - if (recurse) { - jit_regset_set(&zmask, mask); - jit_update(block->label->next, live, &zmask, 0); - } - else { - node = block->label->next; - goto restart; - } - jit_regset_xor(&ztmp, &zmask, mask); - /* remove known live registers from mask */ - if (jit_regset_set_p(&ztmp)) { - jit_regset_and(&ztmp, &ztmp, live); - jit_regset_com(&ztmp, &ztmp); - jit_regset_and(mask, mask, &ztmp); - } - continue; - } - /* Should not really mark as live all registers in unknown - * state if using jit_jmpr(), or jit_jmpi(absolute_address) - * because that would leave the register allocator with - * no options for "nospill" temporaries (other temporaries - * also benefit from not needing to spill/reload), so, the - * user must ensure to either spill/reload, or only leave - * live values on registers that are advertised as - * callee save (as per jit_callee_save_p); on most targets - * these are the JIT_Vn registers. */ - for (regno = 0; regno < _jitc->reglen; regno++) { - spec = jit_class(_rvs[regno].spec); - if (jit_regset_tstbit(mask, regno) && - (spec & (jit_class_gpr|jit_class_fpr)) && - !(spec & jit_class_sav)) - jit_regset_clrbit(mask, regno); - } - /* assume value is live due to jump to unknown location */ - jit_regset_ior(live, live, mask); - jit_regset_set_ui(mask, 0); - return; - } - break; - } - } -} - -static void -_thread_jumps(jit_state_t *_jit) -{ - jit_node_t *prev; - jit_node_t *node; - jit_node_t *next; - int32_t mask; - - for (prev = node = _jitc->head; node;) { - next = node->next; - switch (node->code) { - case jit_code_jmpi: - if (redundant_jump(prev, node)) { - node = prev; - continue; - } - if (shortcut_jump(prev, node)) - continue; - break; - case jit_code_jmpr: - case jit_code_callr: case jit_code_calli: - /* non optimizable jump like code */ - break; - default: - mask = jit_classify(node->code); - if (mask & jit_cc_a0_jmp) { - if (reverse_jump(prev, node) || - shortcut_jump(prev, node)) - continue; - } - break; - } - prev = node; - node = next; - } -} - -static void -_sequential_labels(jit_state_t *_jit) -{ - jit_node_t *jump; - jit_node_t *link; - jit_node_t *prev; - jit_node_t *next; - jit_node_t *node; - - for (prev = node = _jitc->head; node; node = next) { - next = node->next; - if (node->code == jit_code_label) { - if (!node->flag) { - if (!node->link) { - del_label(prev, node); - continue; - } - if (prev != node && prev->code == jit_code_label) { - if ((jump = node->link)) { - for (; jump; jump = link) { - link = jump->link; - jump->u.n = prev; - jump->link = prev->link; - prev->link = jump; - } - node->link = NULL; - } - del_label(prev, node); - continue; - } - } - if (next && next->code == jit_code_label && !next->flag) { - if ((jump = next->link)) { - for (; jump; jump = link) { - link = jump->link; - jump->u.n = node; - jump->link = node->link; - node->link = jump; - } - next->link = NULL; - } - del_label(node, next); - next = node->next; - continue; - } - } - prev = node; - } -} - -static jit_bool_t -_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) -{ - jit_bool_t cond; - jit_node_t *jump; - jit_node_t *next; - jit_node_t *temp; - - if (!(node->flag & jit_flag_node)) - return (0); - assert(node->code != jit_code_jmpr); - cond = node->code != jit_code_jmpi; - jump = node->u.n; - for (next = jump->next; next; next = next->next) { - switch (next->code) { - case jit_code_jmpi: - if (!(next->flag & jit_flag_node)) - return (0); - if (jump->link == node) - jump->link = node->link; - else { - for (temp = jump->link; - temp->link != node; - temp = temp->link) - assert(temp != NULL); - temp->link = node->link; - } - jump = next->u.n; - node->u.n = jump; - node->link = jump->link; - jump->link = node; - return (1); - case jit_code_jmpr: - if (cond) - return (0); - node->code = jit_code_jmpr; - node->u.w = next->u.w; - node->link = NULL; - node->flag &= ~jit_flag_node; - return (1); - case jit_code_note: case jit_code_label: - break; - default: - return (0); - } - } - return (0); -} - -static jit_bool_t -_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) -{ - jit_node_t *local_prev; - jit_node_t *local_next; - - if (!(node->flag & jit_flag_node)) - return (0); - for (local_prev = node, local_next = node->next; - local_next; - local_prev = local_next, local_next = local_next->next) { - - switch (local_next->code) { - case jit_code_label: case jit_code_epilog: - if (node->u.n == local_next) { - if (local_next->link == node) - local_next->link = node->link; - else { - for (local_prev = local_next->link; - local_prev->link != node; - local_prev = local_prev->link) - assert(local_prev != NULL); - local_prev->link = node->link; - } - del_node(prev, node); - return (1); - } - break; - case jit_code_name: case jit_code_note: - case jit_code_align: - break; - default: - return (0); - } - } - return (0); -} - -static jit_code_t -reverse_jump_code(jit_code_t code) -{ - switch (code) { - case jit_code_bltr: return (jit_code_bger); - case jit_code_blti: return (jit_code_bgei); - case jit_code_bltr_u: return (jit_code_bger_u); - case jit_code_blti_u: return (jit_code_bgei_u); - case jit_code_bler: return (jit_code_bgtr); - case jit_code_blei: return (jit_code_bgti); - case jit_code_bler_u: return (jit_code_bgtr_u); - case jit_code_blei_u: return (jit_code_bgti_u); - case jit_code_beqr: return (jit_code_bner); - case jit_code_beqi: return (jit_code_bnei); - case jit_code_bger: return (jit_code_bltr); - case jit_code_bgei: return (jit_code_blti); - case jit_code_bger_u: return (jit_code_bltr_u); - case jit_code_bgei_u: return (jit_code_blti_u); - case jit_code_bgtr: return (jit_code_bler); - case jit_code_bgti: return (jit_code_blei); - case jit_code_bgtr_u: return (jit_code_bler_u); - case jit_code_bgti_u: return (jit_code_blei_u); - case jit_code_bner: return (jit_code_beqr); - case jit_code_bnei: return (jit_code_beqi); - case jit_code_bmsr: return (jit_code_bmcr); - case jit_code_bmsi: return (jit_code_bmci); - case jit_code_bmcr: return (jit_code_bmsr); - case jit_code_bmci: return (jit_code_bmsi); - case jit_code_bltr_f: return (jit_code_bunger_f); - case jit_code_blti_f: return (jit_code_bungei_f); - case jit_code_bler_f: return (jit_code_bungtr_f); - case jit_code_blei_f: return (jit_code_bungti_f); - - case jit_code_beqr_f: return (jit_code_bner_f); - case jit_code_beqi_f: return (jit_code_bnei_f); - - case jit_code_bger_f: return (jit_code_bunltr_f); - case jit_code_bgei_f: return (jit_code_bunlti_f); - case jit_code_bgtr_f: return (jit_code_bunler_f); - case jit_code_bgti_f: return (jit_code_bunlei_f); - - case jit_code_bner_f: return (jit_code_beqr_f); - case jit_code_bnei_f: return (jit_code_beqr_f); - - case jit_code_bunltr_f: return (jit_code_bger_f); - case jit_code_bunlti_f: return (jit_code_bgei_f); - case jit_code_bunler_f: return (jit_code_bgtr_f); - case jit_code_bunlei_f: return (jit_code_bgti_f); - - case jit_code_buneqr_f: return (jit_code_bltgtr_f); - case jit_code_buneqi_f: return (jit_code_bltgti_f); - - case jit_code_bunger_f: return (jit_code_bltr_f); - case jit_code_bungei_f: return (jit_code_blti_f); - case jit_code_bungtr_f: return (jit_code_bler_f); - case jit_code_bungti_f: return (jit_code_blei_f); - - case jit_code_bltgtr_f: return (jit_code_buneqr_f); - case jit_code_bltgti_f: return (jit_code_buneqi_f); - - case jit_code_bordr_f: return (jit_code_bunordr_f); - case jit_code_bordi_f: return (jit_code_bunordi_f); - case jit_code_bunordr_f:return (jit_code_bordr_f); - case jit_code_bunordi_f:return (jit_code_bordi_f); - case jit_code_bltr_d: return (jit_code_bunger_d); - case jit_code_blti_d: return (jit_code_bungei_d); - case jit_code_bler_d: return (jit_code_bungtr_d); - case jit_code_blei_d: return (jit_code_bungti_d); - - case jit_code_beqr_d: return (jit_code_bner_d); - case jit_code_beqi_d: return (jit_code_bnei_d); - - case jit_code_bger_d: return (jit_code_bunltr_d); - case jit_code_bgei_d: return (jit_code_bunlti_d); - case jit_code_bgtr_d: return (jit_code_bunler_d); - case jit_code_bgti_d: return (jit_code_bunlei_d); - - case jit_code_bner_d: return (jit_code_beqr_d); - case jit_code_bnei_d: return (jit_code_beqi_d); - - case jit_code_bunltr_d: return (jit_code_bger_d); - case jit_code_bunlti_d: return (jit_code_bgei_d); - case jit_code_bunler_d: return (jit_code_bgtr_d); - case jit_code_bunlei_d: return (jit_code_bgti_d); - - case jit_code_buneqr_d: return (jit_code_bltgtr_d); - case jit_code_buneqi_d: return (jit_code_bltgti_d); - - case jit_code_bunger_d: return (jit_code_bltr_d); - case jit_code_bungei_d: return (jit_code_blti_d); - case jit_code_bungtr_d: return (jit_code_bler_d); - case jit_code_bungti_d: return (jit_code_blei_d); - - case jit_code_bltgtr_d: return (jit_code_buneqr_d); - case jit_code_bltgti_d: return (jit_code_buneqi_d); - - case jit_code_bordr_d: return (jit_code_bunordr_d); - case jit_code_bordi_d: return (jit_code_bunordi_d); - case jit_code_bunordr_d:return (jit_code_bordr_d); - case jit_code_bunordi_d:return (jit_code_bordi_d); - case jit_code_boaddr: return (jit_code_bxaddr); - case jit_code_boaddi: return (jit_code_bxaddi); - case jit_code_boaddr_u: return (jit_code_bxaddr_u); - case jit_code_boaddi_u: return (jit_code_bxaddi_u); - case jit_code_bxaddr: return (jit_code_boaddr); - case jit_code_bxaddi: return (jit_code_boaddi); - case jit_code_bxaddr_u: return (jit_code_boaddr_u); - case jit_code_bxaddi_u: return (jit_code_boaddi_u); - case jit_code_bosubr: return (jit_code_bxsubr); - case jit_code_bosubi: return (jit_code_bxsubi); - case jit_code_bosubr_u: return (jit_code_bxsubr_u); - case jit_code_bosubi_u: return (jit_code_bxsubi_u); - case jit_code_bxsubr: return (jit_code_bosubr); - case jit_code_bxsubi: return (jit_code_bosubi); - case jit_code_bxsubr_u: return (jit_code_bosubr_u); - case jit_code_bxsubi_u: return (jit_code_bosubi_u); - default: abort(); /* invalid jump code */ - } -} - -/* - * change common pattern: - *