mirror/guile
mirror/guile
1
Fork 0
mirror of https://git.savannah.gnu.org/git/guile.git synced 2025-06-30 15:00:21 +02:00
Code Activity

Improve handling of push/pop/drop in jit

Browse source 
Previously, these instructions were compiled directly.  However, the JIT
also wants to fuse comparisons with branches, and an intervening "drop"
breaks that.  Instead we take a different approach and rely on the
compiler only emitting push/pop/drop in a peephole sort of way, and we
can just simulate the temp stack space and dispatch directly.

* libguile/jit.c (compile_push, compile_push_slow, compile_pop):
(compile_pop_slow, compile_drop, compile_drop_slow): Crash if these are
called.
(COMPILE_WIDE_OP1, COMPILE_WIDE_OP2, COMPILE_WIDE_OP3, COMPILE_WIDE_OP4)
(COMPILE_WIDE_OP5):
(COMPILE_WIDE_DOP1, COMPILE_WIDE_DOP2, COMPILE_WIDE_DOP3)
(COMPILE_WIDE_DOP4, COMPILE_WIDE_DOP5): New helpers.  I didn't manage to
share implementation with COMPILE_*.
(COMPILE_WIDE_*): New variants of compilers that take their operands
from an array instead of parsing the inline operands, relying on
convention for how the compiler emits these instructions.
(parse_wide_operands): New helper to parse out "wide" operands.
(compile1, compile_slow_path): If we see a "push", parse wide operands.
(compile): Move label capture for the slow path into compile_slow_path,
because wide operands may skip some instructions.
This commit is contained in:
Andy Wingo 2023-10-02 13:58:34 +02:00
parent 046378a917
commit 624dd8a17a
1 changed files with 246 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

290
libguile/jit.c
View file

@ -2084,59 +2084,34 @@ compile_reset_frame_slow (scm_jit_state *j, uint32_t nlocals)
static void static void
compile_push (scm_jit_state *j, uint32_t src) compile_push (scm_jit_state *j, uint32_t src)
{ {
jit_gpr_t t = T0; UNREACHABLE ();
jit_subi (j->jit, SP, SP, sizeof (union scm_vm_stack_element));
add_slow_path_patch (j, emit_alloc_frame_for_sp_fast (j, t));
emit_mov (j, 0, src + 1, t);
clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
j->frame_size_min++;
if (j->frame_size_max != INT32_MAX)
j->frame_size_max++;
} }
static void static void
compile_push_slow (scm_jit_state *j, uint32_t src) compile_push_slow (scm_jit_state *j, uint32_t src)
{ {
jit_gpr_t t = T0; UNREACHABLE ();
emit_alloc_frame_for_sp_slow (j, t);
emit_mov (j, 0, src + 1, t);
continue_after_slow_path (j, j->next_ip);
} }
static void static void
compile_pop (scm_jit_state *j, uint32_t dst) compile_pop (scm_jit_state *j, uint32_t dst)
{ {
emit_mov (j, dst + 1, 0, T0); UNREACHABLE ();
jit_addi (j->jit, SP, SP, sizeof (union scm_vm_stack_element));
emit_store_sp (j);
clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
j->frame_size_min--;
if (j->frame_size_max != INT32_MAX)
j->frame_size_max--;
} }
static void static void
compile_pop_slow (scm_jit_state *j, uint32_t dst) compile_pop_slow (scm_jit_state *j, uint32_t dst)
{ {
UNREACHABLE ();
} }
static void static void
compile_drop (scm_jit_state *j, uint32_t nvalues) compile_drop (scm_jit_state *j, uint32_t nvalues)
{ {
jit_addi (j->jit, SP, SP, nvalues * sizeof (union scm_vm_stack_element)); UNREACHABLE ();
emit_store_sp (j);
clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
j->frame_size_min -= nvalues;
if (j->frame_size_max != INT32_MAX)
j->frame_size_max -= nvalues;
} }
static void static void
compile_drop_slow (scm_jit_state *j, uint32_t nvalues) compile_drop_slow (scm_jit_state *j, uint32_t nvalues)
{ {
UNREACHABLE ();
} }
static void static void
@ -5337,15 +5312,34 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
#define COMPILE_DOP4(t0, t1, t2, t3) COMPILE_OP4(t0, t1, t2, t3) #define COMPILE_DOP4(t0, t1, t2, t3) COMPILE_OP4(t0, t1, t2, t3)
#define COMPILE_DOP5(t0, t1, t2, t3, t4) COMPILE_OP5(t0, t1, t2, t3, t4) #define COMPILE_DOP5(t0, t1, t2, t3, t4) COMPILE_OP5(t0, t1, t2, t3, t4)
#define COMPILE_WIDE_OP1(t0) \
COMPILE_WIDE_##t0
#define COMPILE_WIDE_OP2(t0, t1) \
COMPILE_WIDE_##t0##__##t1
#define COMPILE_WIDE_OP3(t0, t1, t2) \
COMPILE_WIDE_##t0##__##t1##__##t2
#define COMPILE_WIDE_OP4(t0, t1, t2, t3) \
COMPILE_WIDE_##t0##__##t1##__##t2##__##t3
#define COMPILE_WIDE_OP5(t0, t1, t2, t3, t4) \
COMPILE_WIDE_##t0##__##t1##__##t2##__##t3##__##t4
#define COMPILE_WIDE_DOP1(t0) COMPILE_WIDE_OP1(t0)
#define COMPILE_WIDE_DOP2(t0, t1) COMPILE_WIDE_OP2(t0, t1)
#define COMPILE_WIDE_DOP3(t0, t1, t2) COMPILE_WIDE_OP3(t0, t1, t2)
#define COMPILE_WIDE_DOP4(t0, t1, t2, t3) COMPILE_WIDE_OP4(t0, t1, t2, t3)
#define COMPILE_WIDE_DOP5(t0, t1, t2, t3, t4) COMPILE_WIDE_OP5(t0, t1, t2, t3, t4)
#define COMPILE_NOP(j, comp) \ #define COMPILE_NOP(j, comp) \
{ \ { \
bad_instruction (j); \ bad_instruction (j); \
} }
#define COMPILE_WIDE_NOP(j, comp) UNREACHABLE()
#define COMPILE_X32(j, comp) \ #define COMPILE_X32(j, comp) \
{ \ { \
comp (j); \ comp (j); \
} }
#define COMPILE_WIDE_X32(j, comp) UNREACHABLE()
#define COMPILE_X8_C24(j, comp) \ #define COMPILE_X8_C24(j, comp) \
{ \ { \
@ -5353,10 +5347,15 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_24 (j->ip[0], a); \ UNPACK_24 (j->ip[0], a); \
comp (j, a); \ comp (j, a); \
} }
#define COMPILE_WIDE_X8_C24(j, comp) UNREACHABLE()
#define COMPILE_X8_F24(j, comp) \ #define COMPILE_X8_F24(j, comp) \
COMPILE_X8_C24 (j, comp) COMPILE_X8_C24 (j, comp)
#define COMPILE_WIDE_X8_F24(j, comp) UNREACHABLE()
#define COMPILE_X8_S24(j, comp) \ #define COMPILE_X8_S24(j, comp) \
COMPILE_X8_C24 (j, comp) COMPILE_X8_C24 (j, comp)
#define COMPILE_WIDE_X8_S24(j, comp) UNREACHABLE()
#define COMPILE_X8_L24(j, comp) \ #define COMPILE_X8_L24(j, comp) \
{ \ { \
@ -5364,18 +5363,36 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
a >>= 8; /* Sign extension. */ \ a >>= 8; /* Sign extension. */ \
comp (j, j->ip + a); \ comp (j, j->ip + a); \
} }
#define COMPILE_WIDE_X8_L24(j, comp) UNREACHABLE()
#define COMPILE_X8_C12_C12(j, comp) \ #define COMPILE_X8_C12_C12(j, comp) \
{ \ { \
uint16_t a, b; \ uint16_t a, b; \
UNPACK_12_12 (j->ip[0], a, b); \ UNPACK_12_12 (j->ip[0], a, b); \
comp (j, a, b); \ comp (j, a, b); \
} }
#define COMPILE_WIDE_X8_C12_C12(j, comp) UNREACHABLE()
#define COMPILE_X8_S12_C12(j, comp) \ #define COMPILE_X8_S12_C12(j, comp) \
COMPILE_X8_C12_C12 (j, comp) COMPILE_X8_C12_C12 (j, comp)
#define COMPILE_WIDE_X8_S12_C12(j, comp) \
{ \
SCM_UNUSED uint16_t a; \
uint16_t b; \
UNPACK_12_12 (j->ip[0], a, b); \
comp (j, wide_operands[0], b); \
}
#define COMPILE_X8_S12_S12(j, comp) \ #define COMPILE_X8_S12_S12(j, comp) \
COMPILE_X8_C12_C12 (j, comp) COMPILE_X8_C12_C12 (j, comp)
#define COMPILE_WIDE_X8_S12_S12(j, comp) \
{ \
comp (j, wide_operands[0], wide_operands[1]); \
}
#define COMPILE_X8_F12_F12(j, comp) \ #define COMPILE_X8_F12_F12(j, comp) \
COMPILE_X8_C12_C12 (j, comp) COMPILE_X8_C12_C12 (j, comp)
#define COMPILE_WIDE_X8_F12_F12(j, comp) UNREACHABLE()
#define COMPILE_X8_S12_Z12(j, comp) \ #define COMPILE_X8_S12_Z12(j, comp) \
{ \ { \
@ -5383,6 +5400,11 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
int16_t b = ((int32_t) j->ip[0]) >> 20; /* Sign extension. */ \ int16_t b = ((int32_t) j->ip[0]) >> 20; /* Sign extension. */ \
comp (j, a, b); \ comp (j, a, b); \
} }
#define COMPILE_WIDE_X8_S12_Z12(j, comp) \
{ \
int16_t b = ((int32_t) j->ip[0]) >> 20; /* Sign extension. */ \
comp (j, wide_operands[0], b); \
}
#define COMPILE_X8_S8_C8_S8(j, comp) \ #define COMPILE_X8_S8_C8_S8(j, comp) \
{ \ { \
@ -5390,10 +5412,30 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_8_8_8 (j->ip[0], a, b, c); \ UNPACK_8_8_8 (j->ip[0], a, b, c); \
comp (j, a, b, c); \ comp (j, a, b, c); \
} }
#define COMPILE_WIDE_X8_S8_C8_S8(j, comp) \
{ \
SCM_UNUSED uint8_t a, c; \
uint8_t b; \
UNPACK_8_8_8 (j->ip[0], a, b, c); \
comp (j, wide_operands[0], b, wide_operands[1]); \
}
#define COMPILE_X8_S8_S8_C8(j, comp) \ #define COMPILE_X8_S8_S8_C8(j, comp) \
COMPILE_X8_S8_C8_S8 (j, comp) COMPILE_X8_S8_C8_S8 (j, comp)
#define COMPILE_WIDE_X8_S8_S8_C8(j, comp) \
{ \
SCM_UNUSED uint8_t a, b; \
uint8_t c; \
UNPACK_8_8_8 (j->ip[0], a, b, c); \
comp (j, wide_operands[0], wide_operands[1], c); \
}
#define COMPILE_X8_S8_S8_S8(j, comp) \ #define COMPILE_X8_S8_S8_S8(j, comp) \
COMPILE_X8_S8_C8_S8 (j, comp) COMPILE_X8_S8_C8_S8 (j, comp)
#define COMPILE_WIDE_X8_S8_S8_S8(j, comp) \
{ \
comp (j, wide_operands[0], wide_operands[1], wide_operands[2]); \
}
#define COMPILE_X8_S8_I16(j, comp) \ #define COMPILE_X8_S8_I16(j, comp) \
{ \ { \
@ -5402,6 +5444,13 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_8_16 (j->ip[0], a, b); \ UNPACK_8_16 (j->ip[0], a, b); \
comp (j, a, SCM_PACK (b)); \ comp (j, a, SCM_PACK (b)); \
} }
#define COMPILE_WIDE_X8_S8_I16(j, comp) \
{ \
SCM_UNUSED uint8_t a; \
scm_t_bits b; \
UNPACK_8_16 (j->ip[0], a, b); \
comp (j, wide_operands[0], SCM_PACK(b)); \
}
#define COMPILE_X8_S8_ZI16(j, comp) \ #define COMPILE_X8_S8_ZI16(j, comp) \
{ \ { \
@ -5410,19 +5459,30 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_8_16 (j->ip[0], a, b); \ UNPACK_8_16 (j->ip[0], a, b); \
comp (j, a, SCM_PACK ((scm_t_signed_bits) b)); \ comp (j, a, SCM_PACK ((scm_t_signed_bits) b)); \
} }
#define COMPILE_WIDE_X8_S8_ZI16(j, comp) \
{ \
SCM_UNUSED uint8_t a; \
int16_t b; \
UNPACK_8_16 (j->ip[0], a, b); \
comp (j, wide_operands[0], SCM_PACK ((scm_t_signed_bits) b)); \
}
#define COMPILE_X32__C32(j, comp) \ #define COMPILE_X32__C32(j, comp) \
{ \ { \
comp (j, j->ip[1]); \ comp (j, j->ip[1]); \
} }
#define COMPILE_WIDE_X32__C32(j, comp) UNREACHABLE()
#define COMPILE_X32__L32(j, comp) \ #define COMPILE_X32__L32(j, comp) \
{ \ { \
int32_t a = j->ip[1]; \ int32_t a = j->ip[1]; \
comp (j, j->ip + a); \ comp (j, j->ip + a); \
} }
#define COMPILE_WIDE_X32__L32(j, comp) UNREACHABLE()
#define COMPILE_X32__N32(j, comp) \ #define COMPILE_X32__N32(j, comp) \
COMPILE_X32__L32 (j, comp) COMPILE_X32__L32 (j, comp)
#define COMPILE_WIDE_X32__N32(j, comp) UNREACHABLE()
#define COMPILE_X8_C24__L32(j, comp) \ #define COMPILE_X8_C24__L32(j, comp) \
{ \ { \
@ -5432,14 +5492,23 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
b = j->ip[1]; \ b = j->ip[1]; \
comp (j, a, j->ip + b); \ comp (j, a, j->ip + b); \
} }
#define COMPILE_WIDE_X8_C24__L32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__L32(j, comp) \ #define COMPILE_X8_S24__L32(j, comp) \
COMPILE_X8_C24__L32 (j, comp) COMPILE_X8_C24__L32 (j, comp)
#define COMPILE_WIDE_X8_S24__L32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__LO32(j, comp) \ #define COMPILE_X8_S24__LO32(j, comp) \
COMPILE_X8_C24__L32 (j, comp) COMPILE_X8_C24__L32 (j, comp)
#define COMPILE_WIDE_X8_S24__LO32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__N32(j, comp) \ #define COMPILE_X8_S24__N32(j, comp) \
COMPILE_X8_C24__L32 (j, comp) COMPILE_X8_C24__L32 (j, comp)
#define COMPILE_WIDE_X8_S24__N32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__R32(j, comp) \ #define COMPILE_X8_S24__R32(j, comp) \
COMPILE_X8_C24__L32 (j, comp) COMPILE_X8_C24__L32 (j, comp)
#define COMPILE_WIDE_X8_S24__R32(j, comp) UNREACHABLE()
#define COMPILE_X8_C24__X8_C24(j, comp) \ #define COMPILE_X8_C24__X8_C24(j, comp) \
{ \ { \
@ -5448,12 +5517,19 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_24 (j->ip[1], b); \ UNPACK_24 (j->ip[1], b); \
comp (j, a, b); \ comp (j, a, b); \
} }
#define COMPILE_WIDE_X8_C24__X8_C24(j, comp) UNREACHABLE()
#define COMPILE_X8_F24__X8_C24(j, comp) \ #define COMPILE_X8_F24__X8_C24(j, comp) \
COMPILE_X8_C24__X8_C24(j, comp) COMPILE_X8_C24__X8_C24(j, comp)
#define COMPILE_WIDE_X8_F24__X8_C24(j, comp) UNREACHABLE()
#define COMPILE_X8_F24__X8_F24(j, comp) \ #define COMPILE_X8_F24__X8_F24(j, comp) \
COMPILE_X8_C24__X8_C24(j, comp) COMPILE_X8_C24__X8_C24(j, comp)
#define COMPILE_WIDE_X8_F24__X8_F24(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__X8_S24(j, comp) \ #define COMPILE_X8_S24__X8_S24(j, comp) \
COMPILE_X8_C24__X8_C24(j, comp) COMPILE_X8_C24__X8_C24(j, comp)
#define COMPILE_WIDE_X8_S24__X8_S24(j, comp) UNREACHABLE()
#define COMPILE_X8_F12_F12__X8_C24(j, comp) \ #define COMPILE_X8_F12_F12__X8_C24(j, comp) \
{ \ { \
@ -5463,6 +5539,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_24 (j->ip[1], c); \ UNPACK_24 (j->ip[1], c); \
comp (j, a, b, c); \ comp (j, a, b, c); \
} }
#define COMPILE_WIDE_X8_F12_F12__X8_C24(j, comp) UNREACHABLE()
#define COMPILE_X8_F24__B1_X7_C24(j, comp) \ #define COMPILE_X8_F24__B1_X7_C24(j, comp) \
{ \ { \
@ -5473,6 +5550,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_24 (j->ip[1], c); \ UNPACK_24 (j->ip[1], c); \
comp (j, a, b, c); \ comp (j, a, b, c); \
} }
#define COMPILE_WIDE_X8_F24__B1_X7_C24(j, comp) UNREACHABLE()
#define COMPILE_X8_S12_S12__C32(j, comp) \ #define COMPILE_X8_S12_S12__C32(j, comp) \
{ \ { \
@ -5482,6 +5560,11 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
c = j->ip[1]; \ c = j->ip[1]; \
comp (j, a, b, c); \ comp (j, a, b, c); \
} }
#define COMPILE_WIDE_X8_S12_S12__C32(j, comp) \
{ \
uint32_t c = j->ip[1]; \
comp (j, wide_operands[0], wide_operands[1], c); \
}
#define COMPILE_X8_S24__C16_C16(j, comp) \ #define COMPILE_X8_S24__C16_C16(j, comp) \
{ \ { \
@ -5491,6 +5574,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_16_16 (j->ip[1], b, c); \ UNPACK_16_16 (j->ip[1], b, c); \
comp (j, a, b, c); \ comp (j, a, b, c); \
} }
#define COMPILE_WIDE_X8_S24__C16_C16(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__C32(j, comp) \ #define COMPILE_X8_S24__C32(j, comp) \
{ \ { \
@ -5499,6 +5583,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
b = j->ip[1]; \ b = j->ip[1]; \
comp (j, a, b); \ comp (j, a, b); \
} }
#define COMPILE_WIDE_X8_S24__C32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__I32(j, comp) \ #define COMPILE_X8_S24__I32(j, comp) \
{ \ { \
@ -5508,19 +5593,43 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
b = j->ip[1]; \ b = j->ip[1]; \
comp (j, a, SCM_PACK (b)); \ comp (j, a, SCM_PACK (b)); \
} }
#define COMPILE_WIDE_X8_S24__I32(j, comp) UNREACHABLE()
#define COMPILE_X8_S8_S8_C8__C32(j, comp) \ #define COMPILE_X8_S8_S8_C8__C32(j, comp) \
{ \ { \
uint8_t a, b, c; \ uint8_t a, b, c; \
uint32_t d; \
UNPACK_8_8_8 (j->ip[0], a, b, c); \ UNPACK_8_8_8 (j->ip[0], a, b, c); \
uint32_t d; \
d = j->ip[1]; \ d = j->ip[1]; \
comp (j, a, b, c, d); \ comp (j, a, b, c, d); \
} }
#define COMPILE_WIDE_X8_S8_S8_C8__C32(j, comp) \
{ \
SCM_UNUSED uint16_t a, b; \
uint8_t c; \
UNPACK_8_8_8 (j->ip[0], a, b, c); \
uint32_t d = j->ip[1]; \
comp (j, wide_operands[0], wide_operands[1], c, d); \
}
#define COMPILE_X8_S8_S8_S8__C32(j, comp) \ #define COMPILE_X8_S8_S8_S8__C32(j, comp) \
COMPILE_X8_S8_S8_C8__C32(j, comp) COMPILE_X8_S8_S8_C8__C32(j, comp)
#define COMPILE_WIDE_X8_S8_S8_S8__C32(j, comp) \
{ \
uint32_t d = j->ip[1]; \
comp (j, wide_operands[0], wide_operands[1], wide_operands[2], d); \
}
#define COMPILE_X8_S8_C8_S8__C32(j, comp) \ #define COMPILE_X8_S8_C8_S8__C32(j, comp) \
COMPILE_X8_S8_S8_C8__C32(j, comp) COMPILE_X8_S8_S8_C8__C32(j, comp)
#define COMPILE_WIDE_X8_S8_C8_S8__C32(j, comp) \
{ \
SCM_UNUSED uint8_t a, c; \
uint8_t b; \
UNPACK_8_8_8 (j->ip[0], a, b, c); \
uint32_t d = j->ip[1]; \
comp (j, wide_operands[0], b, wide_operands[1], d); \
}
#define COMPILE_X8_S24__V32_X8_L24(j, comp) \ #define COMPILE_X8_S24__V32_X8_L24(j, comp) \
{ \ { \
@ -5530,12 +5639,14 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
j->next_ip += len; \ j->next_ip += len; \
comp (j, a, len, j->ip + 2); \ comp (j, a, len, j->ip + 2); \
} }
#define COMPILE_WIDE_X8_S24__V32_X8_L24(j, comp) UNREACHABLE()
#define COMPILE_X32__LO32__L32(j, comp) \ #define COMPILE_X32__LO32__L32(j, comp) \
{ \ { \
int32_t a = j->ip[1], b = j->ip[2]; \ int32_t a = j->ip[1], b = j->ip[2]; \
comp (j, j->ip + a, j->ip + b); \ comp (j, j->ip + a, j->ip + b); \
} }
#define COMPILE_WIDE_X32__LO32__L32(j, comp) UNREACHABLE()
#define COMPILE_X8_F24__X8_C24__L32(j, comp) \ #define COMPILE_X8_F24__X8_C24__L32(j, comp) \
{ \ { \
@ -5546,6 +5657,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
c = j->ip[2]; \ c = j->ip[2]; \
comp (j, a, b, j->ip + c); \ comp (j, a, b, j->ip + c); \
} }
#define COMPILE_WIDE_X8_F24__X8_C24__L32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__A32__B32(j, comp) \ #define COMPILE_X8_S24__A32__B32(j, comp) \
{ \ { \
@ -5556,6 +5668,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
ASSERT (b <= (uint64_t) UINTPTR_MAX); \ ASSERT (b <= (uint64_t) UINTPTR_MAX); \
comp (j, a, SCM_PACK ((uintptr_t) b)); \ comp (j, a, SCM_PACK ((uintptr_t) b)); \
} }
#define COMPILE_WIDE_X8_S24__A32__B32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__AF32__BF32(j, comp) \ #define COMPILE_X8_S24__AF32__BF32(j, comp) \
{ \ { \
@ -5565,6 +5678,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
b.u = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \ b.u = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
comp (j, a, b.d); \ comp (j, a, b.d); \
} }
#define COMPILE_WIDE_X8_S24__AF32__BF32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__AS32__BS32(j, comp) \ #define COMPILE_X8_S24__AS32__BS32(j, comp) \
{ \ { \
@ -5574,6 +5688,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \ b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
comp (j, a, (int64_t) b); \ comp (j, a, (int64_t) b); \
} }
#define COMPILE_WIDE_X8_S24__AS32__BS32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__AU32__BU32(j, comp) \ #define COMPILE_X8_S24__AU32__BU32(j, comp) \
{ \ { \
@ -5583,6 +5698,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \ b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
comp (j, a, b); \ comp (j, a, b); \
} }
#define COMPILE_WIDE_X8_S24__AU32__BU32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__B1_X7_F24__X8_L24(j, comp) \ #define COMPILE_X8_S24__B1_X7_F24__X8_L24(j, comp) \
{ \ { \
@ -5595,6 +5711,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
d = j->ip[2]; d >>= 8; /* Sign extension. */ \ d = j->ip[2]; d >>= 8; /* Sign extension. */ \
comp (j, a, b, c, j->ip + d); \ comp (j, a, b, c, j->ip + d); \
} }
#define COMPILE_WIDE_X8_S24__B1_X7_F24__X8_L24(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__X8_S24__C8_S24(j, comp) \ #define COMPILE_X8_S24__X8_S24__C8_S24(j, comp) \
{ \ { \
@ -5605,6 +5722,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_8_24 (j->ip[2], c, d); \ UNPACK_8_24 (j->ip[2], c, d); \
comp (j, a, b, c, d); \ comp (j, a, b, c, d); \
} }
#define COMPILE_WIDE_X8_S24__X8_S24__C8_S24(j, comp) UNREACHABLE()
#define COMPILE_X8_C24__C8_C24__X8_C24__N32(j, comp) \ #define COMPILE_X8_C24__C8_C24__X8_C24__N32(j, comp) \
{ \ { \
@ -5617,6 +5735,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
e = j->ip[3]; \ e = j->ip[3]; \
comp (j, a, b, c, d, j->ip + e); \ comp (j, a, b, c, d, j->ip + e); \
} }
#define COMPILE_WIDE_X8_C24__C8_C24__X8_C24__N32(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__X8_S24__C8_S24__X8_S24(j, comp) \ #define COMPILE_X8_S24__X8_S24__C8_S24__X8_S24(j, comp) \
{ \ { \
@ -5628,6 +5747,7 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
UNPACK_24 (j->ip[3], e); \ UNPACK_24 (j->ip[3], e); \
comp (j, a, b, c, d, e); \ comp (j, a, b, c, d, e); \
} }
#define COMPILE_WIDE_X8_S24__X8_S24__C8_S24__X8_S24(j, comp) UNREACHABLE()
#define COMPILE_X8_S24__N32__N32__C32(j, comp) \ #define COMPILE_X8_S24__N32__N32__C32(j, comp) \
{ \ { \
@ -5638,6 +5758,54 @@ compile_call_scm_from_scmn_scmn_slow (scm_jit_state *j, uint32_t dst,
uint32_t d = j->ip[3]; \ uint32_t d = j->ip[3]; \
comp (j, a, j->ip + b, j->ip + c, d); \ comp (j, a, j->ip + b, j->ip + c, d); \
} }
#define COMPILE_WIDE_X8_S24__N32__N32__C32(j, comp) UNREACHABLE()
static uint8_t
parse_wide_operands (scm_jit_state *j, uint32_t wide_operands[3])
{
uint8_t opcode = j->ip[0] & 0xff;
uint32_t push_count = 0;
while (opcode == scm_op_push)
{
ASSERT (push_count < 3);
UNPACK_24 (j->ip[0], wide_operands[push_count]);
wide_operands[push_count] -= push_count;
push_count++;
j->ip = j->next_ip;
opcode = j->ip[0] & 0xff;
j->next_ip = j->ip + op_lengths[opcode];
}
ASSERT (push_count > 0);
uint8_t finish_opcode = j->next_ip[0] & 0xff;
uint32_t pop_count = 0;
if (finish_opcode == scm_op_drop)
{
uint32_t count;
UNPACK_24 (j->next_ip[0], count);
pop_count += count;
ASSERT(pop_count <= push_count);
j->next_ip = j->next_ip + op_lengths[finish_opcode];
finish_opcode = j->next_ip[0] & 0xff;
}
if (finish_opcode == scm_op_pop)
{
ASSERT (push_count < 3);
ASSERT (push_count - pop_count == 1);
switch (push_count) {
case 2: wide_operands[2] = wide_operands[1]; /* fall through */
case 1: wide_operands[1] = wide_operands[0]; break;
default: UNREACHABLE ();
}
UNPACK_24 (j->next_ip[0], wide_operands[0]);
pop_count++;
j->next_ip = j->next_ip + op_lengths[finish_opcode];
finish_opcode = j->next_ip[0] & 0xff;
}
ASSERT (pop_count == push_count);
return opcode;
}
static uintptr_t opcodes_seen[256 / (SCM_SIZEOF_UINTPTR_T * 8)]; static uintptr_t opcodes_seen[256 / (SCM_SIZEOF_UINTPTR_T * 8)];
@ -5683,15 +5851,30 @@ compile1 (scm_jit_state *j)
j->next_ip = j->ip + op_lengths[opcode]; j->next_ip = j->ip + op_lengths[opcode];
switch (opcode) if (opcode == scm_op_push)
{ {
uint32_t wide_operands[3];
opcode = parse_wide_operands (j, wide_operands);
switch (opcode)
{
#define COMPILE1(code, cname, name, arity) \ #define COMPILE1(code, cname, name, arity) \
case code: COMPILE_##arity(j, compile_##cname); break; case code: COMPILE_WIDE_##arity(j, compile_##cname); break;
FOR_EACH_VM_OPERATION(COMPILE1) FOR_EACH_VM_OPERATION(COMPILE1)
#undef COMPILE1 #undef COMPILE1
default: default:
UNREACHABLE (); UNREACHABLE ();
}
} }
else
switch (opcode)
{
#define COMPILE1(code, cname, name, arity) \
case code: COMPILE_##arity(j, compile_##cname); break;
FOR_EACH_VM_OPERATION(COMPILE1)
#undef COMPILE1
default:
UNREACHABLE ();
}
j->ip = j->next_ip; j->ip = j->next_ip;
} }
@ -5702,14 +5885,35 @@ compile_slow_path (scm_jit_state *j)
uint8_t opcode = j->ip[0] & 0xff; uint8_t opcode = j->ip[0] & 0xff;
j->next_ip = j->ip + op_lengths[opcode]; j->next_ip = j->ip + op_lengths[opcode];
switch (opcode) if (opcode == scm_op_push)
{ {
uint32_t wide_operands[3];
opcode = parse_wide_operands (j, wide_operands);
ptrdiff_t offset = j->ip - j->start;
j->labels[slow_label_offset (offset)] = jit_address (j->jit);
switch (opcode)
{
#define COMPILE_SLOW(code, cname, name, arity) \ #define COMPILE_SLOW(code, cname, name, arity) \
case code: COMPILE_##arity(j, compile_##cname##_slow); break; case code: COMPILE_WIDE_##arity(j, compile_##cname##_slow); break;
FOR_EACH_VM_OPERATION(COMPILE_SLOW) FOR_EACH_VM_OPERATION(COMPILE_SLOW)
#undef COMPILE_SLOW #undef COMPILE_SLOW
default: default:
UNREACHABLE (); UNREACHABLE ();
}
}
else
{
ptrdiff_t offset = j->ip - j->start;
j->labels[slow_label_offset (offset)] = jit_address (j->jit);
switch (opcode)
{
#define COMPILE_SLOW(code, cname, name, arity) \
case code: COMPILE_##arity(j, compile_##cname##_slow); break;
FOR_EACH_VM_OPERATION(COMPILE_SLOW)
#undef COMPILE_SLOW
default:
UNREACHABLE ();
}
} }
j->ip = j->next_ip; j->ip = j->next_ip;
@ -5838,8 +6042,6 @@ compile (scm_jit_state *j)
j->ip = (uint32_t *) j->start; j->ip = (uint32_t *) j->start;
while (j->ip < j->end) while (j->ip < j->end)
{ {
ptrdiff_t offset = j->ip - j->start;
j->labels[slow_label_offset (offset)] = jit_address (j->jit);
// set register state from j->register_states[offset] ? // set register state from j->register_states[offset] ?
reset_register_state (j, SP_IN_REGISTER); reset_register_state (j, SP_IN_REGISTER);
compile_slow_path (j); compile_slow_path (j);