diff --git a/.gitignore b/.gitignore index 7107bb352..8ff6499d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,22 @@ +* autom4te.cache aclocal.m4 +depcomp INSTALL Makefile.in config.h.in +config.guess +config.sub configure -m4/config.guess -m4/config.sub -m4/depcomp -m4/install-sh -m4/mdate-sh -m4/missing -m4/texinfo.tex -doc/Makefile.in +install-sh +ltmain.sh +missing +m4/libtool.m4 +m4/lt~obsolete.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +doc/mdate-sh +doc/texinfo.tex lightning/Makefile.in tests/Makefile.in diff --git a/ChangeLog b/ChangeLog index fb4afd640..6acdb0b4a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,59 @@ +2012-12-02 Paulo Andrade + + * tests/Makefile.am, tests/3to2.c, tests/3to2.ok, tests/add.c, + tests/add.ok, tests/allocai.c, tests/allocai.ok, tests/bp.c, + tests/bp.ok, tests/divi.c, tests/divi.ok, tests/fib.c, tests/fib.ok, + tests/fibdelay.c, tests/fibdelay.ok, tests/fibit.c, tests/fibit.ok, + tests/funcfp.c, tests/funcfp.ok, tests/incr.c, tests/incr.ok, + tests/ldst.c, tests/ldst.ok, tests/ldxi.c, tests/ldxi.ok, + tests/modi.c, tests/modi.ok, tests/movi.c, tests/movi.ok, + tests/printf.c, tests/printf.ok, tests/printf2.c, tests/printf2.ok, + tests/ret.c, tests/ret.ok, tests/rpn.c, tests/rpn.ok, tests/rpnfp.c, + tests/rpnfp.ok, tests/sete.c, tests/sete.ok, tests/testfp.c, + tests/testfp.ok, tests-run-test: Removed previous test suite, in + favor of a newer one in the check subdirectory. + + * check/3to2.ok, check/3to2.tst, check/add.ok, check/add.tst, + check/allocai.ok, check/allocai.tst, check/bp.ok, check/bp.tst, + check/divi.ok, check/divi.tst, check/fib.ok, check/fib.tst: + New sample input for the new test program, loosely matching + several of the previous test cases. + + * check/Makefile.am: New test suite makefile. + + * check/check.sh, check/run-test: New wrapper files for the + new test suite. + + * check/lightning.c: New file. The main driver of the new test + suite, that compiles to a parser of a very simple assembly like + language, generates jit and executes it. + + * check/all.tst: New file. A generic debug and sample test file + with a directive to prevent it from being executed, and useful to + read disassembly of all possible instructions, using a fixed set + of registers. + + * include/Makefile.am, include/lightning.h, + include/lightning/Makefile.am, include/lightning/jit_arm.h, + include/lightning/jit_mips.h, include/lightning/jit_ppc.h, + include/lightning/jit_private.h, include/lightning/jit_x86.h, + lib/Makefile.am, lib/jit_disasm.c, lib/jit_print.c, + lib/jit_x86-cpu.c, lib/jit_x86-sse.c, lib/jit_x86-x87.c, + lib/jit_x86.c, lib/lightning.c: New files. These files are + written from scratch, only by , and have now + copyright assignment to the FSF. This is the core of the new + lightning rework. Previously it was integrated in code with + a garbage collector and several custom types like vectors and + hash tables, so this first code merge with lightning converts + that code into a library extracting only the jit bits, and at + first only for x86_64 GNU/Linux. + + * lightning.h, m4/lightning.m4: Removed. These are no longer + required in the new lightning code. + + .gitignore, Makefile.am, configure.ac: Update for the new + lightning code. + 2012-12-02 Paulo Andrade * .cvsignore: Removed for extra cleanup. diff --git a/Makefile.am b/Makefile.am index b2c47c4f8..3c3f31d44 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,10 +1,21 @@ -# Automake requirements: -AUTOMAKE_OPTIONS = 1.8 gnu +# +# Copyright 2000, 2001, 2002, 2012 Free Software Foundation, Inc. +# +# This is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = . doc lightning tests - -include_HEADERS = lightning.h - -aclocaldir = $(datadir)/aclocal -dist_aclocal_DATA = m4/lightning.m4 +SUBDIRS = \ + check \ + doc \ + include \ + lib diff --git a/check/3to2.ok b/check/3to2.ok new file mode 100644 index 000000000..de2c0407b --- /dev/null +++ b/check/3to2.ok @@ -0,0 +1,22 @@ +0 +1 +1 +1 +0 +1 +1 +1 +0 +1 +1 +0 +1 +1 +1 +0 +1 +1 +1 +0 +1 +1 diff --git a/check/3to2.tst b/check/3to2.tst new file mode 100644 index 000000000..5804ab6b1 --- /dev/null +++ b/check/3to2.tst @@ -0,0 +1,111 @@ +.data 32 +dfmt: +.c "%1.0f\n" +ifmt: +.c "%d\n" + +.code + jmpi main + +#define def_test_double(a, b, c) \ +test_double_##a##_##b##_##c: \ + prolog \ + arg_d $d0 \ + arg_d $d1 \ + getarg_d %b $d0 \ + getarg_d %c $d1 \ + subr_d %a %b %c \ + retr_d %a \ + epilog +#define test_double(a, b, c, x, y) \ + prepare 0 \ + pushargi_d x \ + pushargi_d y \ + finishi test_double_##a##_##b##_##c \ + prepare 1 \ + pushargi dfmt \ + pushargr_d %fret \ + finishi @printf + +#define def_test_int(a, b, c) \ +test_int_##a##_##b##_##c: \ + prolog \ + arg $i0 \ + arg $i1 \ + getarg %b $i0 \ + getarg %c $i1 \ + subr %a %b %c \ + retr %a \ + epilog +#define test_int(a, b, c, x, y) \ + prepare 0 \ + pushargi x \ + pushargi y \ + finishi test_int_##a##_##b##_##c \ + prepare 1 \ + pushargi ifmt \ + pushargr %ret \ + finishi @printf + +def_test_double(f0, f0, f0) +def_test_double(f0, f0, f1) +def_test_double(f0, f1, f0) +def_test_double(f0, f1, f2) + +def_test_double(f3, f3, f3) +def_test_double(f3, f3, f1) +def_test_double(f3, f1, f3) +def_test_double(f3, f1, f2) + +def_test_double(f3, f0, f0) +def_test_double(f3, f0, f3) +def_test_double(f3, f3, f0) + +def_test_int(r0, r0, r0) +def_test_int(r0, r0, r1) +def_test_int(r0, r1, r0) +def_test_int(r0, r1, r2) + +def_test_int(v0, v0, v0) +def_test_int(v0, v0, r1) +def_test_int(v0, r1, v0) +def_test_int(v0, r1, r2) + +def_test_int(v0, r0, r0) +def_test_int(v0, r0, v0) +def_test_int(v0, v0, r0) + + +main: + prolog + + test_double(f0, f0, f0, 3.0, 2.0) + test_double(f0, f0, f1, 3.0, 2.0) + test_double(f0, f1, f0, 3.0, 2.0) + test_double(f0, f1, f2, 3.0, 2.0) + + test_double(f3, f3, f3, 3.0, 2.0) + test_double(f3, f3, f1, 3.0, 2.0) + test_double(f3, f1, f3, 3.0, 2.0) + test_double(f3, f1, f2, 3.0, 2.0) + + test_double(f3, f0, f0, 3.0, 2.0) + test_double(f3, f0, f3, 3.0, 2.0) + test_double(f3, f3, f0, 3.0, 2.0) + + test_int(r0, r0, r0, 3, 2) + test_int(r0, r0, r1, 3, 2) + test_int(r0, r1, r0, 3, 2) + test_int(r0, r1, r2, 3, 2) + + test_int(v0, v0, v0, 3, 2) + test_int(v0, v0, r1, 3, 2) + test_int(v0, r1, v0, 3, 2) + test_int(v0, r1, r2, 3, 2) + + test_int(v0, r0, r0, 3, 2) + test_int(v0, r0, v0, 3, 2) + test_int(v0, v0, r0, 3, 2) + + ret + epilog diff --git a/check/Makefile.am b/check/Makefile.am new file mode 100644 index 000000000..0c37dcf97 --- /dev/null +++ b/check/Makefile.am @@ -0,0 +1,43 @@ +# +# Copyright 2012 Free Software Foundation, Inc. +# +# This is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +AM_CFLAGS = -I$(top_srcdir)/include -D_GNU_SOURCE + +check_PROGRAMS = lightning + +lightning_LDADD = $(top_srcdir)/lib/liblightning.la -ldl + +lightning_SOURCES = \ + lightning.c + +EXTRA_DIST = \ + 3to2.tst 3to2.ok \ + add.tst add.ok \ + allocai.tst allocai.ok \ + bp.tst bp.ok \ + divi.tst divi.ok \ + fib.tst fib.ok \ + all.tst + +TESTS = 3to2 add allocai bp divi fib + +# Not so pretty but good for a prototype +$(TESTS): check.sh + $(LN_S) check.sh $@ + +TESTS_ENVIRONMENT=$(srcdir)/run-test + +debug: $(check_PROGRAMS) + $(LIBTOOL) --mode=execute gdb $(check_PROGRAMS) + diff --git a/tests/add.ok b/check/add.ok similarity index 100% rename from tests/add.ok rename to check/add.ok diff --git a/check/add.tst b/check/add.tst new file mode 100644 index 000000000..61500cd59 --- /dev/null +++ b/check/add.tst @@ -0,0 +1,31 @@ +.data 32 +fmt: +.c "%d + %d = %d\n" + +.code + jmpi main + +test: + prolog + arg $i0 + arg $i1 + getarg %r0 $i0 + getarg %r1 $i1 + addr %ret %r0 %r1 + ret + epilog + +main: + prolog + prepare 0 + pushargi 5 + pushargi 4 + finishi test + prepare 1 + pushargi fmt + pushargi 5 + pushargi 4 + pushargr %r0 + finishi @printf + ret + epilog diff --git a/check/all.tst b/check/all.tst new file mode 100644 index 000000000..9a2ec4437 --- /dev/null +++ b/check/all.tst @@ -0,0 +1,384 @@ +.disasm // only disassemble +.code + prolog + allocai 32 $buf + arg $c + arg $uc + arg $s + arg $us + arg $i + arg $ui + arg $l + getarg_c %r0 $c + getarg_uc %r0 $uc + getarg_s %r0 $s + getarg_us %r0 $us + getarg_i %r0 $i + getarg_ui %r0 $ui + getarg_l %r0 $l + addr %r0 %r1 %r2 + addi %r0 %r1 2 + addxr %r0 %r1 %r2 + addxi %r0 %r1 2 + addcr %r0 %r1 %r2 + addci %r0 %r1 2 + subr %r0 %r1 %r2 + subi %r0 %r1 2 + subxr %r0 %r1 %r2 + subxi %r0 %r1 2 + subcr %r0 %r1 %r2 + subci %r0 %r1 2 + mulr %r0 %r1 %r2 + muli %r0 %r1 2 + divr %r0 %r1 %r2 + divi %r0 %r1 2 + divr_u %r0 %r1 %r2 + divi_u %r0 %r1 2 + remr %r0 %r1 %r2 + remi %r0 %r1 2 + remr_u %r0 %r1 %r2 + remi_u %r0 %r1 2 + andr %r0 %r1 %r2 + andi %r0 %r1 2 + orr %r0 %r1 %r2 + ori %r0 %r1 2 + xorr %r0 %r1 %r2 + xori %r0 %r1 2 + lshr %r0 %r1 %r2 + lshi %r0 %r1 2 + rshr %r0 %r1 %r2 + rshi %r0 %r1 2 + rshr_u %r0 %r1 %r2 + rshi_u %r0 %r1 2 + negr %r0 %r1 + comr %r0 %r1 + ltr %r0 %r1 %r2 + lti %r0 %r1 2 + ltr_u %r0 %r1 %r2 + lti_u %r0 %r1 2 + ler %r0 %r1 %r2 + lei %r0 %r1 2 + ler_u %r0 %r1 %r2 + lei_u %r0 %r1 2 + eqr %r0 %r1 %r2 + eqi %r0 %r1 2 + ger %r0 %r1 %r2 + gei %r0 %r1 2 + ger_u %r0 %r1 %r2 + gei_u %r0 %r1 2 + gtr %r0 %r1 %r2 + gti %r0 %r1 2 + gtr_u %r0 %r1 %r2 + gti_u %r0 %r1 2 + ner %r0 %r1 %r2 + nei %r0 %r1 2 + movr %r0 %r1 + movi %r0 1 + extr_c %r0 %r1 + extr_uc %r0 %r1 + extr_s %r0 %r1 + extr_us %r0 %r1 + extr_i %r0 %r1 + extr_ui %r0 %r1 + htonr %r0 %r1 + ntohr %r0 %r1 + ldr_c %r0 %r1 + ldi_c %r0 0x80000000 + ldr_uc %r0 %r1 + ldi_uc %r0 0x80000000 + ldr_s %r0 %r1 + ldi_s %r0 0x80000000 + ldr_us %r0 %r1 + ldi_us %r0 0x80000000 + ldr_i %r0 %r1 + ldi_i %r0 0x80000000 + ldr_ui %r0 %r1 + ldi_ui %r0 0x80000000 + ldr_l %r0 %r1 + ldi_l %r0 0x80000000 + ldxr_c %r0 %r1 %r2 + ldxi_c %r0 %r1 1 + ldxr_uc %r0 %r1 %r2 + ldxi_uc %r0 %r1 1 + ldxr_s %r0 %r1 %r2 + ldxi_s %r0 %r1 2 + ldxr_us %r0 %r1 %r2 + ldxi_us %r0 %r1 2 + ldxr_i %r0 %r1 %r2 + ldxi_i %r0 %r1 4 + ldxr_ui %r0 %r1 %r2 + ldxi_ui %r0 %r1 4 + ldxr_l %r0 %r1 %r2 + ldxi_l %r0 %r1 8 + str_c %r1 %r0 + sti_c 0x80000000 %r1 + str_s %r1 %r0 + sti_s 0x80000000 %r1 + str_i %r1 %r0 + sti_i 0x80000000 %r1 + str_l %r1 %r0 + sti_l 0x80000000 %r1 + stxr_c %r2 %r1 %r0 + stxi_c 1 %r1 %r0 + stxr_s %r2 %r1 %r0 + stxi_s 2 %r1 %r0 + stxr_i %r2 %r1 %r0 + stxi_i 4 %r1 %r0 + stxr_l %r2 %r1 %r0 + stxi_l 8 %r1 %r0 +cond: + bltr cond %r0 %r1 +condi: + blti condi %r0 1 +condu: + bltr_u condu %r0 %r1 +condiu: + blti_u condiu %r0 1 + bler cond %r0 %r1 + blei condi %r0 1 + bler_u condu %r0 %r1 + blei_u condiu %r0 1 +bool: + beqr bool %r0 %r1 +booli: + beqi booli %r0 1 + bger cond %r0 %r1 + bgei condi %r0 1 + bger_u condu %r0 %r1 + bgei_u condiu %r0 1 + bgtr cond %r0 %r1 + bgti condi %r0 1 + bgtr_u condu %r0 %r1 + bgti_u condiu %r0 1 + bner bool %r0 %r1 + bnei booli %r0 1 +mask: + bmsr mask %r0 %r1 +maski: + bmsi maski %r0 1 + bmcr mask %r0 %r1 + bmci maski %r0 1 +as: + boaddr as %r0 %r1 +asi: + boaddi asi %r0 1 +asu: + boaddr_u as %r0 %r1 + boaddi_u asi %r0 1 + bxaddr as %r0 %r1 + bxaddi asi %r0 1 + bxaddr_u as %r0 %r1 + bxaddi_u asi %r0 1 + bosubr as %r0 %r1 + bosubi asi %r0 1 + bosubr_u as %r0 %r1 + bosubi_u asi %r0 1 + bxsubr as %r0 %r1 + bxsubi asi %r0 1 + bxsubr_u as %r0 %r1 + bxsubi_u asi %r0 1 +label: + jmpr %r0 + jmpi label + callr %r0 + calli label + prepare 0 + pushargr %r0 + finishr %r0 + prepare 1 + pushargi 1 + finishi 0x80000000 + ret + retr %r1 + reti 2 + retval_c %r1 + retval_uc %r1 + retval_s %r1 + retval_us %r1 + retval_i %r1 + retval_ui %r1 + retval_l %r1 + arg_f $f + getarg_f %f1 $f + addr_f %f0 %f1 %f2 + addi_f %f0 %f1 0.5 + subr_f %f0 %f1 %f2 + subi_f %f0 %f1 0.5 + mulr_f %f0 %f1 %f2 + muli_f %f0 %f1 0.5 + divr_f %f0 %f1 %f2 + divi_f %f0 %f1 0.5 + negr_f %f0 %f1 + absr_f %f0 %f1 + sqrtr_f %f0 %f1 + ltr_f %r0 %f0 %f1 + lti_f %r0 %f0 0.5 + ler_f %r0 %f0 %f1 + lei_f %r0 %f0 0.5 + eqr_f %r0 %f0 %f1 + eqi_f %r0 %f0 0.5 + ger_f %r0 %f0 %f1 + gei_f %r0 %f0 0.5 + gtr_f %r0 %f0 %f1 + gti_f %r0 %f0 0.5 + ner_f %r0 %f0 %f1 + nei_f %r0 %f0 0.5 + unltr_f %r0 %f0 %f1 + unlti_f %r0 %f0 0.5 + unler_f %r0 %f0 %f1 + unlei_f %r0 %f0 0.5 + uneqr_f %r0 %f0 %f1 + uneqi_f %r0 %f0 0.5 + unger_f %r0 %f0 %f1 + ungei_f %r0 %f0 0.5 + ungtr_f %r0 %f0 %f1 + ungti_f %r0 %f0 0.5 + ltgtr_f %r0 %f0 %f1 + ltgti_f %r0 %f0 0.5 + ordr_f %r0 %f0 %f1 + ordi_f %r0 %f0 0.5 + unordr_f %r0 %f0 %f1 + unordi_f %r0 %f0 0.5 + truncr_f_i %r0 %f0 + truncr_f_l %r0 %f0 + extr_f %f0 %r0 + extr_d_f %f0 %f1 + movr_f %f0 %f1 + movi_f %f0 1.5 + ldr_f %f0 %r0 + ldi_f %f0 0x80000000 + ldxr_f %f0 %r0 %r1 + ldxi_f %f0 %r0 4 + str_f %r0 %f0 + sti_f 0x80000000 %f0 + stxr_f %r1 %r0 %f0 + stxi_f 4 %r0 %f0 +ord: + bltr_f ord %f0 %f1 +ordi: + blti_f ordi %f0 0.5 + bler_f ord %f0 %f1 + blei_f ordi %f0 0.5 + beqr_f ord %f0 %f1 + beqi_f ordi %f0 0.5 + bger_f ord %f0 %f1 + bgei_f ordi %f0 0.5 + bgtr_f ord %f0 %f1 + bgti_f ordi %f0 0.5 + bner_f ord %f0 %f1 + bnei_f ordi %f0 0.5 +unord: + bunltr_f unord %f0 %f1 +unordi: + bunlti_f unordi %f0 0.5 + bunler_f unord %f0 %f1 + bunlei_f unordi %f0 0.5 + buneqr_f unord %f0 %f1 + buneqi_f unordi %f0 0.5 + bunger_f unord %f0 %f1 + bungei_f unordi %f0 0.5 + bungtr_f unord %f0 %f1 + bungti_f unordi %f0 0.5 + bltgtr_f unord %f0 %f1 + bltgti_f unordi %f0 0.5 + bordr_f unord %f0 %f1 + bordi_f unordi %f0 0.5 + bunordr_f unord %f0 %f1 + bunordi_f unordi %f0 0.5 + prepare 0 + pushargr_f %f1 + pushargi_f 0.5 + finishi 0x80000000 + retr_f %f1 + reti_f 0.5 + retval_f %f1 + arg_d $f + getarg_d %f1 $f + addr_d %f0 %f1 %f2 + addi_d %f0 %f1 0.5 + subr_d %f0 %f1 %f2 + subi_d %f0 %f1 0.5 + mulr_d %f0 %f1 %f2 + muli_d %f0 %f1 0.5 + divr_d %f0 %f1 %f2 + divi_d %f0 %f1 0.5 + negr_d %f0 %f1 + absr_d %f0 %f1 + sqrtr_d %f0 %f1 + ltr_d %r0 %f0 %f1 + lti_d %r0 %f0 0.5 + ler_d %r0 %f0 %f1 + lei_d %r0 %f0 0.5 + eqr_d %r0 %f0 %f1 + eqi_d %r0 %f0 0.5 + ger_d %r0 %f0 %f1 + gei_d %r0 %f0 0.5 + gtr_d %r0 %f0 %f1 + gti_d %r0 %f0 0.5 + ner_d %r0 %f0 %f1 + nei_d %r0 %f0 0.5 + unltr_d %r0 %f0 %f1 + unlti_d %r0 %f0 0.5 + unler_d %r0 %f0 %f1 + unlei_d %r0 %f0 0.5 + uneqr_d %r0 %f0 %f1 + uneqi_d %r0 %f0 0.5 + unger_d %r0 %f0 %f1 + ungei_d %r0 %f0 0.5 + ungtr_d %r0 %f0 %f1 + ungti_d %r0 %f0 0.5 + ltgtr_d %r0 %f0 %f1 + ltgti_d %r0 %f0 0.5 + ordr_d %r0 %f0 %f1 + ordi_d %r0 %f0 0.5 + unordr_d %r0 %f0 %f1 + unordi_d %r0 %f0 0.5 + truncr_d_i %r0 %f0 + truncr_d_l %r0 %f0 + extr_d %f0 %r0 + extr_f_d %f0 %f1 + movr_d %f0 %f1 + movi_d %f0 1.5 + ldr_d %f0 %r0 + ldi_d %f0 0x80000000 + ldxr_d %f0 %r0 %r1 + ldxi_d %f0 %r0 8 + str_d %r0 %f0 + sti_d 0x80000000 %f0 + stxr_d %r1 %r0 %f0 + stxi_d 8 %r0 %f0 + bltr_d ord %f0 %f1 + blti_d ordi %f0 0.5 + bler_d ord %f0 %f1 + blei_d ordi %f0 0.5 + beqr_d ord %f0 %f1 + beqi_d ordi %f0 0.5 + bger_d ord %f0 %f1 + bgei_d ordi %f0 0.5 + bgtr_d ord %f0 %f1 + bgti_d ordi %f0 0.5 + bner_d ord %f0 %f1 + bnei_d ordi %f0 0.5 + bunltr_d unord %f0 %f1 + bunlti_d unordi %f0 0.5 + bunler_d unord %f0 %f1 + bunlei_d unordi %f0 0.5 + buneqr_d unord %f0 %f1 + buneqi_d unordi %f0 0.5 + bunger_d unord %f0 %f1 + bungei_d unordi %f0 0.5 + bungtr_d unord %f0 %f1 + bungti_d unordi %f0 0.5 + bltgtr_d unord %f0 %f1 + bltgti_d unordi %f0 0.5 + bordr_d unord %f0 %f1 + bordi_d unordi %f0 0.5 + bunordr_d unord %f0 %f1 + bunordi_d unordi %f0 0.5 + prepare 0 + pushargr_d %f1 + pushargi_d 0.5 + finishi 0x80000000 + retr_d %f1 + reti_d 0.5 + retval_d %f1 diff --git a/tests/allocai.ok b/check/allocai.ok similarity index 100% rename from tests/allocai.ok rename to check/allocai.ok diff --git a/check/allocai.tst b/check/allocai.tst new file mode 100644 index 000000000..b613a41b9 --- /dev/null +++ b/check/allocai.tst @@ -0,0 +1,92 @@ +.data 128 +idfmt: +.c "received %d\n" +failure_message: +.c "numbers don't add up to zero\n" +report_message: +.c "failed: got %i instead of %i\n" +succeeded_message: +.c "succeeded\n" + +.code + jmpi main + +/* +static int +identity (int arg) +{ + printf ("received %i\n", arg); + return arg; +} + */ +identify: + prolog + arg $i + getarg %v0 $i + prepare 1 + pushargi idfmt + pushargr %v0 + finishi @printf + retr %v0 + epilog + +identity_func: + prolog + arg $i + getarg %r1 $i + + /* Store the argument on the stack. */ + allocai $(__WORDSIZE >> 3) $off + stxi $off %fp %r1 + + /* Store the negative of the argument on the stack. */ + allocai $(__WORDSIZE >> 3) $neg + negr %r2 %r1 + stxi $neg %fp %r2 + + /* Invoke FUNC. */ + prepare 0 + pushargr %r1 + finishi identify + + /* Ignore the result. */ + + /* Restore the negative and the argument from the stack. */ + ldxi %r2 %fp $neg + ldxi %v1 %fp $off + + /* Make sure they still add to zero. */ + addr %r0 %v1 %r2 + bnei branch %r0 0 + + /* Return it. */ + retr %v1 + + /* Display a failure message. */ +branch: + prepare 1 + pushargi failure_message + finishi @printf + + /* Leave. */ + retr %v1 + epilog + +main: + prolog + prepare 0 + pushargi 7777 + finishi identity_func + beqi succeeded %ret 7777 + prepare 1 + pushargi report_message + pushargr %ret + pushargi 7777 + finishi @printf + reti 1 +succeeded: + prepare 1 + pushargi succeeded_message + finishi @printf + reti 0 + epilog diff --git a/tests/bp.ok b/check/bp.ok similarity index 100% rename from tests/bp.ok rename to check/bp.ok diff --git a/check/bp.tst b/check/bp.tst new file mode 100644 index 000000000..a331244fe --- /dev/null +++ b/check/bp.tst @@ -0,0 +1,44 @@ +.data 32 +fmt: +.c "nfibs(%d) = %d\n" + +.code + jmpi main + +rfibs: + prolog + arg $in + getarg %v0 $in /* V0 = N */ + + blti_u out %v0 2 + subi %v1 %v0 1 /* V1 = N-1 */ + subi %v2 %v0 2 /* V1 = N-2 */ + prepare 0 + pushargr %v1 + finishi rfibs + retval %v1 /* V1 = rfibs(N-1) */ + prepare 0 + pushargr %v2 + finishi rfibs + retval %v2 /* V2 = rfibs(N-2) */ + addi %v1 %v1 1 + addr %ret %v1 %v2 + ret +out: + movi %ret 1 + ret + epilog + +main: + prolog + prepare 0 + pushargi 32 + finishi rfibs + retval %v0 + prepare 1 + pushargi fmt + pushargi 32 + pushargr %v0 + finishi @printf + ret + epilog diff --git a/check/check.sh b/check/check.sh new file mode 100755 index 000000000..54e48c618 --- /dev/null +++ b/check/check.sh @@ -0,0 +1,2 @@ +#!/bin/sh +./lightning `basename $0` diff --git a/tests/divi.ok b/check/divi.ok similarity index 100% rename from tests/divi.ok rename to check/divi.ok diff --git a/check/divi.tst b/check/divi.tst new file mode 100644 index 000000000..6a2115295 --- /dev/null +++ b/check/divi.tst @@ -0,0 +1,79 @@ +.data 128 +small_ops: +.i 40 64 80 +large_ops: +.i 98304 65536 163840 +fmt: +.c "%i/%i = %i (expected %i)\n" +x: +.c "%d\n" +.code + jmpi main + +#define generate_divider(operand) \ +divider_##operand: \ + prolog \ + arg $i \ + getarg %r1 $i \ + divi %r2 %r1 operand \ + retr %r2 \ + epilog +generate_divider(8) +generate_divider(32768) + +#define generate_test_divider(divisor) \ +test_divider_##divisor: \ + prolog \ + allocai 4 $loc \ + arg $p \ + arg $c \ + getarg %v0 $p \ + getarg %v1 $c \ + muli %v1 %v1 4 \ + addr %v1 %v0 %v1 \ +loop_##divisor: \ + bger done_##divisor %v0 %v1 \ + ldr_i %v2 %v0 \ + prepare 0 \ + pushargr %v2 \ + finishi divider_##divisor \ + retval %v2 \ + ldr_i %r2 %v0 \ + divi %r0 %r2 divisor \ + /* save div result */ \ + stxi_i $loc %fp %r0 \ + prepare 1 \ + pushargi fmt \ + pushargr %r2 \ + pushargi divisor \ + pushargr %v2 \ + pushargr %r0 \ + finishi @printf \ + addi %v0 %v0 4 \ + /* reload div result */ \ + ldxi_i %r0 %fp $loc \ + beqr loop_##divisor %r0 %v2 \ + /* return if failed */ \ + reti 1 \ +done_##divisor: \ + reti 0 \ + epilog +generate_test_divider(8) +generate_test_divider(32768) + +main: + prolog + prepare 0 + pushargi small_ops + pushargi 3 + finishi test_divider_8 + bnei fail %ret 0 + prepare 0 + pushargi large_ops + pushargi 3 + finishi test_divider_32768 + bnei fail %ret 0 + reti 0 +fail: + reti 1 + epilog diff --git a/tests/fib.ok b/check/fib.ok similarity index 100% rename from tests/fib.ok rename to check/fib.ok diff --git a/check/fib.tst b/check/fib.tst new file mode 100644 index 000000000..ddca50a1f --- /dev/null +++ b/check/fib.tst @@ -0,0 +1,58 @@ +.data 32 +format: +.c "nfibs(%d) = %d\n" + +.code + jmpi main + +nfibs: + prolog + arg $in + getarg_ui %r2 $in // R2 = n + movi %r1 1 + blti_u ref %r2 2 + subi %r2 %r2 1 + movi %r0 1 +loop: + subi %r2 %r2 1 // decr. counter + addr %v0 %r0 %r1 // V0 = R0 + R1 + movr %r0 %r1 // R0 = R1 + addi %r1 %v0 1 // R1 = V0 + 1 + bnei loop %r2 0 // if (R2) goto loop +ref: + movr %ret %r1 // RET = R1 + ret + epilog + +main: + prolog + arg $argc + arg $argv + + getarg_i %r0 $argc + blei default %r0 1 + getarg %r0 $argv + addi %r0 %r0 $(__WORDSIZE >> 3) + ldr %r0 %r0 + prepare 0 + pushargr %r0 + finishi @atoi + retval %r0 + jmpi call + +default: + movi %r0 32 + +call: + movr %v0 %r0 + prepare 0 + pushargr %r0 + finishi nfibs + retval %r0 + prepare 1 + pushargi format + pushargr %v0 + pushargr %r0 + finishi @printf + ret + epilog diff --git a/check/lightning.c b/check/lightning.c new file mode 100644 index 000000000..43d2bb90c --- /dev/null +++ b/check/lightning.c @@ -0,0 +1,3859 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#include +#include +#include +#include +#include + +#if defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) +# include +#endif + +#if defined(__GNUC__) +# define noreturn __attribute__ ((noreturn)) +# define printf_format(f, v) __attribute__ ((format (printf, f, v))) +# define maybe_unused __attribute__ ((unused)) +#else +# define noreturn /**/ +# define printf_format(f, v) /**/ +# define maybe_unused /**/ +#endif + +#define check_data(length) \ + do { \ + if (data_offset + length >= data_length) \ + error(".data too small (%ld < %ld)", \ + data_length, data_offset + length); \ + } while (0) + +#define get_label_by_name(name) ((label_t *)get_hash(labels, name)) + +#define PARSING_NONE 0 +#define PARSING_DATA 1 +#define PARSING_CODE 2 +#define MAX_IDENTIFIER 256 + +/* + * Types + */ +typedef struct instr instr_t; +typedef union value value_t; +typedef struct parser parser_t; +typedef struct label label_t; +typedef struct patch patch_t; +typedef struct symbol symbol_t; +typedef struct hash hash_t; +typedef struct entry entry_t; +typedef int (*function_t)(int argc, char *argv[]); + +typedef enum { + tok_eof = -1, + tok_symbol, + tok_char, + tok_int, + tok_float, + tok_pointer, + tok_string, + tok_register, + tok_dot, + tok_newline, + tok_semicollon, +} token_t; + +typedef enum { + skip_none, + skip_ws, + skip_nl, +} skip_t; + +typedef enum { + type_none, + type_c, + type_s, + type_i, + type_l, + type_f, + type_d, + type_p, +} type_t; + +#define compose(a, b) (((a) << 8) | b) +typedef enum { + expr_inc = compose('+', '+'), + expr_dec = compose('-', '-'), + expr_not = '!', + expr_com = '~', + expr_mul = '*', + expr_div = '/', + expr_rem = '%', + expr_add = '+', + expr_sub = '-', + expr_lsh = compose('<', '<'), + expr_rsh = compose('>', '>'), + expr_and = '&', + expr_or = '|', + expr_xor = '^', + expr_set = '=', + expr_mulset = compose('*', '='), + expr_divset = compose('/', '='), + expr_remset = compose('%', '='), + expr_addset = compose('+', '='), + expr_subset = compose('-', '='), + expr_lshset = compose(expr_lsh, '='), + expr_rshset = compose(expr_rsh, '='), + expr_andset = compose('&', '='), + expr_orset = compose('|', '='), + expr_xorset = compose('^', '='), + expr_lt = '<', + expr_le = compose('<', '='), + expr_eq = compose('=', '='), + expr_ne = compose('!', '='), + expr_gt = '>', + expr_ge = compose('>', '='), + expr_andand = compose('&', '&'), + expr_oror = compose('|', '|'), + expr_lparen = '(', + expr_rparen = ')', + expr_int = '0', + expr_float = '.', + expr_pointer = '@', + expr_symbol = '$', +} expr_t; +#undef compose + +struct instr { + instr_t *next; + const char *name; + void (*function)(void); + int flag; +}; + +union value { + long i; + unsigned long ui; + float f; + double d; + void *p; + char *cp; + label_t *label; + patch_t *patch; +}; + +struct parser { + FILE *fp; + char name[256]; + int line; + int regval; + type_t regtype; + expr_t expr; + type_t type; + value_t value; + + /* variable length string buffer */ + char *string; + int length; + int offset; + + int newline; + expr_t putback; + int short_circuit; + int parsing; + + struct { + unsigned char buffer[4096]; + int offset; + int length; + } data; +}; + +typedef enum { + label_kind_data, + label_kind_code, + label_kind_code_forward, + label_kind_dynamic, +} label_kind_t; + +struct hash { + entry_t **entries; + int size; + int count; +}; + +struct entry { + entry_t *next; + char *name; + void *value; + int flag; +}; + +struct label { + label_t *next; + char *name; + void *value; + label_kind_t kind; +}; + +typedef enum { + patch_kind_jmp, + patch_kind_mov, + patch_kind_call, +} patch_kind_t; + +struct patch { + patch_t *next; + label_t *label; + void *value; + patch_kind_t kind; +}; + +/* minor support for expressions */ +struct symbol { + symbol_t *next; + char *name; + value_t value; + type_t type; +}; + +/* + * Prototypes + */ +static jit_gpr_t get_ireg(void); +static jit_fpr_t get_freg(void); +static symbol_t *get_symbol(void); +static void jmp_forward(void *value, label_t *label); +static void mov_forward(void *value, label_t *label); +static void call_forward(void *value, label_t *label); +static void make_arg(long value); +static long get_arg(void); +static long get_imm(void); +static void prolog(void); +static void allocai(void); +static void arg(void); +static void getarg_c(void); static void getarg_uc(void); +static void getarg_s(void); static void getarg_us(void); +static void getarg_i(void); static void getarg_ui(void); +static void getarg_l(void); +static void getarg(void); +static void addr(void); static void addi(void); +static void addxr(void); static void addxi(void); +static void addcr(void); static void addci(void); +static void subr(void); static void subi(void); +static void subxr(void); static void subxi(void); +static void subcr(void); static void subci(void); +static void mulr(void); static void muli(void); +static void divr(void); static void divi(void); +static void divr_u(void); static void divi_u(void); +static void remr(void); static void remi(void); +static void remr_u(void); static void remi_u(void); +static void andr(void); static void andi(void); +static void orr(void); static void ori(void); +static void xorr(void); static void xori(void); +static void lshr(void); static void lshi(void); +static void rshr(void); static void rshi(void); +static void rshr_u(void); static void rshi_u(void); +static void negr(void); static void comr(void); +static void ltr(void); static void lti(void); +static void ltr_u(void); static void lti_u(void); +static void ler(void); static void lei(void); +static void ler_u(void); static void lei_u(void); +static void eqr(void); static void eqi(void); +static void ger(void); static void gei(void); +static void ger_u(void); static void gei_u(void); +static void gtr(void); static void gti(void); +static void gtr_u(void); static void gti_u(void); +static void ner(void); static void nei(void); +static void movr(void); static void movi(void); +static void extr_c(void); static void extr_uc(void); +static void extr_s(void); static void extr_us(void); +static void extr_i(void); static void extr_ui(void); +static void htonr(void); static void ntohr(void); +static void ldr_c(void); static void ldi_c(void); +static void ldr_uc(void); static void ldi_uc(void); +static void ldr_s(void); static void ldi_s(void); +static void ldr_us(void); static void ldi_us(void); +static void ldr_i(void); static void ldi_i(void); +static void ldr_ui(void); static void ldi_ui(void); +static void ldr_l(void); static void ldi_l(void); +static void ldr(void); static void ldi(void); +static void ldxr_c(void); static void ldxi_c(void); +static void ldxr_uc(void); static void ldxi_uc(void); +static void ldxr_s(void); static void ldxi_s(void); +static void ldxr_us(void); static void ldxi_us(void); +static void ldxr_i(void); static void ldxi_i(void); +static void ldxr_ui(void); static void ldxi_ui(void); +static void ldxr_l(void); static void ldxi_l(void); +static void ldxr(void); static void ldxi(void); +static void str_c(void); static void sti_c(void); +static void str_s(void); static void sti_s(void); +static void str_i(void); static void sti_i(void); +static void str_l(void); static void sti_l(void); +static void str(void); static void sti(void); +static void stxr_c(void); static void stxi_c(void); +static void stxr_s(void); static void stxi_s(void); +static void stxr_i(void); static void stxi_i(void); +static void stxr_l(void); static void stxi_l(void); +static void stxr(void); static void stxi(void); +static void bltr(void); static void blti(void); +static void bltr_u(void); static void blti_u(void); +static void bler(void); static void blei(void); +static void bler_u(void); static void blei_u(void); +static void beqr(void); static void beqi(void); +static void bger(void); static void bgei(void); +static void bger_u(void); static void bgei_u(void); +static void bgtr(void); static void bgti(void); +static void bgtr_u(void); static void bgti_u(void); +static void bner(void); static void bnei(void); +static void bmsr(void); static void bmsi(void); +static void bmcr(void); static void bmci(void); +static void boaddr(void); static void boaddi(void); +static void boaddr_u(void); static void boaddi_u(void); +static void bxaddr(void); static void bxaddi(void); +static void bxaddr_u(void); static void bxaddi_u(void); +static void bosubr(void); static void bosubi(void); +static void bosubr_u(void); static void bosubi_u(void); +static void bxsubr(void); static void bxsubi(void); +static void bxsubr_u(void); static void bxsubi_u(void); +static void jmpr(void); static void jmpi(void); +static void callr(void); static void calli(void); +static void prepare(void); +static void pushargr(void); static void pushargi(void); +static void finishr(void); static void finishi(void); +static void ret(void); +static void retr(void); static void reti(void); +static void retval_c(void); static void retval_uc(void); +static void retval_s(void); static void retval_us(void); +static void retval_i(void); static void retval_ui(void); +static void retval_l(void); +static void retval(void); +static void epilog(void); +static void arg_f(void); +static void getarg_f(void); +static void addr_f(void); static void addi_f(void); +static void subr_f(void); static void subi_f(void); +static void mulr_f(void); static void muli_f(void); +static void divr_f(void); static void divi_f(void); +static void negr_f(void); static void absr_f(void); +static void sqrtr_f(void); +static void ltr_f(void); static void lti_f(void); +static void ler_f(void); static void lei_f(void); +static void eqr_f(void); static void eqi_f(void); +static void ger_f(void); static void gei_f(void); +static void gtr_f(void); static void gti_f(void); +static void ner_f(void); static void nei_f(void); +static void unltr_f(void); static void unlti_f(void); +static void unler_f(void); static void unlei_f(void); +static void uneqr_f(void); static void uneqi_f(void); +static void unger_f(void); static void ungei_f(void); +static void ungtr_f(void); static void ungti_f(void); +static void ltgtr_f(void); static void ltgti_f(void); +static void ordr_f(void); static void ordi_f(void); +static void unordr_f(void); static void unordi_f(void); +static void truncr_f_i(void); static void truncr_f_l(void); +static void extr_f(void); static void extr_d_f(void); +static void movr_f(void); static void movi_f(void); +static void ldr_f(void); static void ldi_f(void); +static void ldxr_f(void); static void ldxi_f(void); +static void str_f(void); static void sti_f(void); +static void stxr_f(void); static void stxi_f(void); +static void bltr_f(void); static void blti_f(void); +static void bler_f(void); static void blei_f(void); +static void beqr_f(void); static void beqi_f(void); +static void bger_f(void); static void bgei_f(void); +static void bgtr_f(void); static void bgti_f(void); +static void bner_f(void); static void bnei_f(void); +static void bunltr_f(void); static void bunlti_f(void); +static void bunler_f(void); static void bunlei_f(void); +static void buneqr_f(void); static void buneqi_f(void); +static void bunger_f(void); static void bungei_f(void); +static void bungtr_f(void); static void bungti_f(void); +static void bltgtr_f(void); static void bltgti_f(void); +static void bordr_f(void); static void bordi_f(void); +static void bunordr_f(void); static void bunordi_f(void); +static void pushargr_f(void); static void pushargi_f(void); +static void retr_f(void); static void reti_f(void); +static void retval_f(void); +static void arg_d(void); +static void getarg_d(void); +static void addr_d(void); static void addi_d(void); +static void subr_d(void); static void subi_d(void); +static void mulr_d(void); static void muli_d(void); +static void divr_d(void); static void divi_d(void); +static void negr_d(void); static void absr_d(void); +static void sqrtr_d(void); +static void ltr_d(void); static void lti_d(void); +static void ler_d(void); static void lei_d(void); +static void eqr_d(void); static void eqi_d(void); +static void ger_d(void); static void gei_d(void); +static void gtr_d(void); static void gti_d(void); +static void ner_d(void); static void nei_d(void); +static void unltr_d(void); static void unlti_d(void); +static void unler_d(void); static void unlei_d(void); +static void uneqr_d(void); static void uneqi_d(void); +static void unger_d(void); static void ungei_d(void); +static void ungtr_d(void); static void ungti_d(void); +static void ltgtr_d(void); static void ltgti_d(void); +static void ordr_d(void); static void ordi_d(void); +static void unordr_d(void); static void unordi_d(void); +static void truncr_d_i(void); static void truncr_d_l(void); +static void extr_d(void); static void extr_f_d(void); +static void movr_d(void); static void movi_d(void); +static void ldr_d(void); static void ldi_d(void); +static void ldxr_d(void); static void ldxi_d(void); +static void str_d(void); static void sti_d(void); +static void stxr_d(void); static void stxi_d(void); +static void bltr_d(void); static void blti_d(void); +static void bler_d(void); static void blei_d(void); +static void beqr_d(void); static void beqi_d(void); +static void bger_d(void); static void bgei_d(void); +static void bgtr_d(void); static void bgti_d(void); +static void bner_d(void); static void bnei_d(void); +static void bunltr_d(void); static void bunlti_d(void); +static void bunler_d(void); static void bunlei_d(void); +static void buneqr_d(void); static void buneqi_d(void); +static void bunger_d(void); static void bungei_d(void); +static void bungtr_d(void); static void bungti_d(void); +static void bltgtr_d(void); static void bltgti_d(void); +static void bordr_d(void); static void bordi_d(void); +static void bunordr_d(void); static void bunordi_d(void); +static void pushargr_d(void); static void pushargi_d(void); +static void retr_d(void); static void reti_d(void); +static void retval_d(void); + +static void error(const char *format, ...) noreturn printf_format(1, 2); +static void warn(const char *format, ...) printf_format(1, 2) maybe_unused; +static void message(const char *kind, const char *format, va_list ap); + +static int getch(void); +static int getch_noeof(void); +static int ungetch(int ch); +static int skipws(void); +static int skipnl(void); +static int skipct(void); +static int skipcp(void); +static long get_int(skip_t skip); +static unsigned long get_uint(skip_t skip); +static double get_float(skip_t skip); +static void *get_pointer(skip_t skip); +static label_t *get_label(skip_t skip); +static token_t regname(void); +static token_t identifier(int ch); +static void get_data(type_t type); +static void dot(void); +static token_t number(int ch); +static int escape(int ch); +static token_t string(void); +static token_t dynamic(void); +static token_t character(void); +static void expression_prim(void); +static void expression_inc(int pre); +static void expression_dec(int pre); +static void expression_unary(void); +static void expression_mul(void); +static void expression_add(void); +static void expression_shift(void); +static void expression_bit(void); +static void expression_rel(void); +static void expression_cond(void); +static token_t expression(void); +static token_t primary(skip_t skip); +static void parse(void); +static int execute(int argc, char *argv[]); + +static void *xmalloc(size_t size); +static void *xrealloc(void *pointer, size_t size); +static void *xcalloc(size_t nmemb, size_t size); + +static label_t *new_label(label_kind_t kind, char *name, void *value); +static patch_t *new_patch(patch_kind_t kind, label_t *label, void *value); +static int bcmp_symbols(const void *left, const void *right); +static int qcmp_symbols(const void *left, const void *right); +static symbol_t *new_symbol(char *name); +static symbol_t *get_symbol_by_name(char *name); + +static hash_t *new_hash(void); +static int hash_string(char *name); +static void put_hash(hash_t *hash, entry_t *entry); +static entry_t *get_hash(hash_t *hash, char *name); +static void rehash(hash_t *hash); + +/* + * Initialization + */ +static jit_state_t *_jit; +static int flag_verbose; +static int flag_disasm; +static char *progname; +static parser_t parser; +static hash_t *labels; +static int label_offset; +static patch_t *patches; +static symbol_t **symbols; +static int symbol_length; +static int symbol_offset; +static hash_t *instrs; +static char *data; +static size_t data_offset, data_length; +static instr_t instr_vector[] = { +#define entry(value) { NULL, #value, value } + entry(prolog), + entry(allocai), + entry(arg), + entry(getarg_c), entry(getarg_uc), + entry(getarg_s), entry(getarg_us), + entry(getarg_i), entry(getarg_ui), + entry(getarg_l), + entry(getarg), + entry(addr), entry(addi), + entry(addxr), entry(addxi), + entry(addcr), entry(addci), + entry(subr), entry(subi), + entry(subxr), entry(subxi), + entry(subcr), entry(subci), + entry(mulr), entry(muli), + entry(divr), entry(divi), + entry(divr_u), entry(divi_u), + entry(remr), entry(remi), + entry(remr_u), entry(remi_u), + entry(andr), entry(andi), + entry(orr), entry(ori), + entry(xorr), entry(xori), + entry(lshr), entry(lshi), + entry(rshr), entry(rshi), + entry(rshr_u), entry(rshi_u), + entry(negr), entry(comr), + entry(ltr), entry(lti), + entry(ltr_u), entry(lti_u), + entry(ler), entry(lei), + entry(ler_u), entry(lei_u), + entry(eqr), entry(eqi), + entry(ger), entry(gei), + entry(ger_u), entry(gei_u), + entry(gtr), entry(gti), + entry(gtr_u), entry(gti_u), + entry(ner), entry(nei), + entry(movr), entry(movi), + entry(extr_c), entry(extr_uc), + entry(extr_s), entry(extr_us), + entry(extr_i), entry(extr_ui), + entry(htonr), entry(ntohr), + entry(ldr_c), entry(ldi_c), + entry(ldr_uc), entry(ldi_uc), + entry(ldr_s), entry(ldi_s), + entry(ldr_us), entry(ldi_us), + entry(ldr_i), entry(ldi_i), + entry(ldr_ui), entry(ldi_ui), + entry(ldr_l), entry(ldi_l), + entry(ldr), entry(ldi), + entry(ldxr_c), entry(ldxi_c), + entry(ldxr_uc), entry(ldxi_uc), + entry(ldxr_s), entry(ldxi_s), + entry(ldxr_us), entry(ldxi_us), + entry(ldxr_i), entry(ldxi_i), + entry(ldxr_ui), entry(ldxi_ui), + entry(ldxr_l), entry(ldxi_l), + entry(ldxr), entry(ldxi), + entry(str_c), entry(sti_c), + entry(str_s), entry(sti_s), + entry(str_i), entry(sti_i), + entry(str_l), entry(sti_l), + entry(str), entry(sti), + entry(stxr_c), entry(stxi_c), + entry(stxr_s), entry(stxi_s), + entry(stxr_i), entry(stxi_i), + entry(stxr_l), entry(stxi_l), + entry(stxr), entry(stxi), + entry(bltr), entry(blti), + entry(bltr_u), entry(blti_u), + entry(bler), entry(blei), + entry(bler_u), entry(blei_u), + entry(beqr), entry(beqi), + entry(bger), entry(bgei), + entry(bger_u), entry(bgei_u), + entry(bgtr), entry(bgti), + entry(bgtr_u), entry(bgti_u), + entry(bner), entry(bnei), + entry(bmsr), entry(bmsi), + entry(bmcr), entry(bmci), + entry(boaddr), entry(boaddi), + entry(boaddr_u), entry(boaddi_u), + entry(bxaddr), entry(bxaddi), + entry(bxaddr_u), entry(bxaddi_u), + entry(bosubr), entry(bosubi), + entry(bosubr_u), entry(bosubi_u), + entry(bxsubr), entry(bxsubi), + entry(bxsubr_u), entry(bxsubi_u), + entry(jmpr), entry(jmpi), + entry(callr), entry(calli), + entry(prepare), + entry(pushargr), entry(pushargi), + entry(finishr), entry(finishi), + entry(ret), + entry(retr), entry(reti), + entry(retval_c), entry(retval_uc), + entry(retval_s), entry(retval_us), + entry(retval_i), entry(retval_ui), + entry(retval_l), + entry(retval), + entry(epilog), + entry(arg_f), + entry(getarg_f), + entry(addr_f), entry(addi_f), + entry(subr_f), entry(subi_f), + entry(mulr_f), entry(muli_f), + entry(divr_f), entry(divi_f), + entry(negr_f), entry(absr_f), + entry(sqrtr_f), + entry(ltr_f), entry(lti_f), + entry(ler_f), entry(lei_f), + entry(eqr_f), entry(eqi_f), + entry(ger_f), entry(gei_f), + entry(gtr_f), entry(gti_f), + entry(ner_f), entry(nei_f), + entry(unltr_f), entry(unlti_f), + entry(unler_f), entry(unlei_f), + entry(uneqr_f), entry(uneqi_f), + entry(unger_f), entry(ungei_f), + entry(ungtr_f), entry(ungti_f), + entry(ltgtr_f), entry(ltgti_f), + entry(ordr_f), entry(ordi_f), + entry(unordr_f), entry(unordi_f), + entry(truncr_f_i), entry(truncr_f_l), + entry(extr_f), entry(extr_d_f), + entry(movr_f), entry(movi_f), + entry(ldr_f), entry(ldi_f), + entry(ldxr_f), entry(ldxi_f), + entry(str_f), entry(sti_f), + entry(stxr_f), entry(stxi_f), + entry(bltr_f), entry(blti_f), + entry(bler_f), entry(blei_f), + entry(beqr_f), entry(beqi_f), + entry(bger_f), entry(bgei_f), + entry(bgtr_f), entry(bgti_f), + entry(bner_f), entry(bnei_f), + entry(bunltr_f), entry(bunlti_f), + entry(bunler_f), entry(bunlei_f), + entry(buneqr_f), entry(buneqi_f), + entry(bunger_f), entry(bungei_f), + entry(bungtr_f), entry(bungti_f), + entry(bltgtr_f), entry(bltgti_f), + entry(bordr_f), entry(bordi_f), + entry(bunordr_f), entry(bunordi_f), + entry(pushargr_f), entry(pushargi_f), + entry(retr_f), entry(reti_f), + entry(retval_f), + entry(arg_d), + entry(getarg_d), + entry(addr_d), entry(addi_d), + entry(subr_d), entry(subi_d), + entry(mulr_d), entry(muli_d), + entry(divr_d), entry(divi_d), + entry(negr_d), entry(absr_d), + entry(sqrtr_d), + entry(ltr_d), entry(lti_d), + entry(ler_d), entry(lei_d), + entry(eqr_d), entry(eqi_d), + entry(ger_d), entry(gei_d), + entry(gtr_d), entry(gti_d), + entry(ner_d), entry(nei_d), + entry(unltr_d), entry(unlti_d), + entry(unler_d), entry(unlei_d), + entry(uneqr_d), entry(uneqi_d), + entry(unger_d), entry(ungei_d), + entry(ungtr_d), entry(ungti_d), + entry(ltgtr_d), entry(ltgti_d), + entry(ordr_d), entry(ordi_d), + entry(unordr_d), entry(unordi_d), + entry(truncr_d_i), entry(truncr_d_l), + entry(extr_d), entry(extr_f_d), + entry(movr_d), entry(movi_d), + entry(ldr_d), entry(ldi_d), + entry(ldxr_d), entry(ldxi_d), + entry(str_d), entry(sti_d), + entry(stxr_d), entry(stxi_d), + entry(bltr_d), entry(blti_d), + entry(bler_d), entry(blei_d), + entry(beqr_d), entry(beqi_d), + entry(bger_d), entry(bgei_d), + entry(bgtr_d), entry(bgti_d), + entry(bner_d), entry(bnei_d), + entry(bunltr_d), entry(bunlti_d), + entry(bunler_d), entry(bunlei_d), + entry(buneqr_d), entry(buneqi_d), + entry(bunger_d), entry(bungei_d), + entry(bungtr_d), entry(bungti_d), + entry(bltgtr_d), entry(bltgti_d), + entry(bordr_d), entry(bordi_d), + entry(bunordr_d), entry(bunordi_d), + entry(pushargr_d), entry(pushargi_d), + entry(retr_d), entry(reti_d), + entry(retval_d), + +#undef entry +}; + +/* + * Implementation + */ +static jit_gpr_t +get_ireg(void) +{ + if (primary(skip_ws) != tok_register) + error("bad register"); + if (parser.regtype != type_l) + error("bad int register"); + + return ((jit_gpr_t)parser.regval); +} + +static jit_fpr_t +get_freg(void) +{ + if (primary(skip_ws) != tok_register) + error("bad register"); + if (parser.regtype != type_d) + error("bad float register"); + + return ((jit_fpr_t)parser.regval); +} + +static symbol_t * +get_symbol(void) +{ + symbol_t *symbol; + int ch = skipws(); + + if (ch != '$') + error("expecting variable"); + (void)identifier('$'); + if (parser.string[1] == '\0') + error("expecting variable"); + if ((symbol = get_symbol_by_name(parser.string)) == NULL) + symbol = new_symbol(parser.string); + + return (symbol); +} + +static void +jmp_forward(void *value, label_t *label) +{ + (void)new_patch(patch_kind_jmp, label, value); +} + +static void +mov_forward(void *value, label_t *label) +{ + (void)new_patch(patch_kind_mov, label, value); +} + +static void +call_forward(void *value, label_t *label) +{ + (void)new_patch(patch_kind_call, label, value); +} + +static void +make_arg(long value) +{ + symbol_t *symbol = get_symbol(); + + symbol->type = type_l; + symbol->value.i = value; +} + +static long +get_arg(void) +{ + symbol_t *symbol = get_symbol(); + + if (symbol->type != type_l) + error("bad argument %s type", symbol->name); + + return symbol->value.i; +} + +static long +get_imm(void) +{ + int ch; + label_t *label; + long value; + ch = skipws(); + switch (ch) { + case '+': case '-': case '0' ... '9': + ungetch(ch); + value = get_int(skip_none); + break; + case '$': + switch (expression()) { + case tok_int: + case tok_pointer: + value = parser.value.i; + break; + default: + error("expecting immediate"); + } + break; + case '@': + dynamic(); + value = parser.value.p; + break; + default: + ungetch(ch); + label = get_label(skip_none); + if (label->kind == label_kind_data) + value = (long)label->value; + else + error("expecting immediate"); + break; + } + return (value); +} + +#define entry(name) \ +static void \ +name(void) \ +{ \ + jit_##name(); \ +} +#define entry_ca(name) \ +static void \ +name(void) \ +{ \ + make_arg(jit_##name()); \ +} +#define entry_ia(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_int32_t ac = get_arg(); \ + jit_##name(r0, ac); \ +} +#define entry_im(name) \ +static void \ +name(void) \ +{ \ + jit_word_t im = get_imm(); \ + jit_##name(im); \ +} +#define entry_ir(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_##name(r0); \ +} +#define entry_ir_ir_ir(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(), r1 = get_ireg(), r2 = get_ireg(); \ + jit_##name(r0, r1, r2); \ +} +#define entry_ir_ir_im(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(), r1 = get_ireg(); \ + jit_word_t im = get_imm(); \ + jit_##name(r0, r1, im); \ +} +#define entry_ir_ir(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(), r1 = get_ireg(); \ + jit_##name(r0, r1); \ +} +#define entry_ir_im(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_word_t im = get_imm(); \ + jit_##name(r0, im); \ +} +#define entry_ir_pm(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + void *pm = get_pointer(skip_ws); \ + jit_##name(r0, pm); \ +} +#define entry_pm_ir(name) \ +static void \ +name(void) \ +{ \ + void *pm = get_pointer(skip_ws); \ + jit_gpr_t r0 = get_ireg(); \ + jit_##name(pm, r0); \ +} +#define entry_im_ir_ir(name) \ +static void \ +name(void) \ +{ \ + jit_word_t im = get_imm(); \ + jit_gpr_t r0 = get_ireg(), r1 = get_ireg(); \ + (void)jit_##name(im, r0, r1); \ +} +#define entry_lb_ir_ir(name) \ +static void \ +name(void) \ +{ \ + jit_node_t *jmp; \ + label_t *label = get_label(skip_ws); \ + jit_gpr_t r0 = get_ireg(), r1 = get_ireg(); \ + if (label->kind == label_kind_code_forward) \ + jmp_forward((void *)jit_##name(r0, r1), label); \ + else { \ + jmp = jit_##name(r0, r1); \ + jit_patch_at(jmp, (jit_node_t *)label->value); \ + } \ +} +#define entry_lb_ir_im(name) \ +static void \ +name(void) \ +{ \ + jit_node_t *jmp; \ + label_t *label = get_label(skip_ws); \ + jit_gpr_t r0 = get_ireg(); \ + jit_word_t im = get_imm(); \ + if (label->kind == label_kind_code_forward) \ + jmp_forward((void *)jit_##name(r0, im), label); \ + else { \ + jmp = jit_##name(r0, im); \ + jit_patch_at(jmp, (jit_node_t *)label->value); \ + } \ +} +#define entry_lb(name) \ +static void \ +name(void) \ +{ \ + jit_node_t *jmp; \ + label_t *label = get_label(skip_ws); \ + if (label->kind == label_kind_code_forward) \ + jmp_forward((void *)jit_##name(), label); \ + else { \ + jmp = jit_##name(); \ + jit_patch_at(jmp, (jit_node_t *)label->value); \ + } \ +} +#define entry_pm(name) \ +static void \ +name(void) \ +{ \ + void *pm = get_pointer(skip_ws); \ + jit_##name(pm); \ +} +#define entry_fa(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + jit_int32_t ac = get_arg(); \ + jit_##name(r0, ac); \ +} +#define entry_fr_fr_fr(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(), r1 = get_freg(), r2 = get_freg(); \ + jit_##name(r0, r1, r2); \ +} +#define entry_fr_fr_fm(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(), r1 = get_freg(); \ + jit_float64_t im = get_float(skip_ws); \ + jit_##name(r0, r1, im); \ +} +#define entry_fr_fr(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(), r1 = get_freg(); \ + jit_##name(r0, r1); \ +} +#define entry_ir_fr_fr(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_fpr_t r1 = get_freg(), r2 = get_freg(); \ + jit_##name(r0, r1, r2); \ +} +#define entry_ir_fr_fm(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_fpr_t r1 = get_freg(); \ + jit_float64_t im = get_float(skip_ws); \ + jit_##name(r0, r1, im); \ +} +#define entry_ir_fr(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_fpr_t r1 = get_freg(); \ + jit_##name(r0, r1); \ +} +#define entry_fr_ir(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + jit_gpr_t r1 = get_ireg(); \ + jit_##name(r0, r1); \ +} +#define entry_fr_fm(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + jit_float64_t im = get_float(skip_ws); \ + jit_##name(r0, im); \ +} +#define entry_fr_pm(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + void *pm = get_pointer(skip_ws); \ + jit_##name(r0, pm); \ +} +#define entry_fr_ir_ir(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + jit_gpr_t r1 = get_ireg(), r2 = get_ireg(); \ + jit_##name(r0, r1, r2); \ +} +#define entry_fr_ir_im(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + jit_gpr_t r1 = get_ireg(); \ + jit_word_t im = get_imm(); \ + jit_##name(r0, r1, im); \ +} +#define entry_pm_fr(name) \ +static void \ +name(void) \ +{ \ + void *pm = get_pointer(skip_ws); \ + jit_fpr_t r0 = get_freg(); \ + jit_##name(pm, r0); \ +} +#define entry_ir_ir_fr(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(), r1 = get_ireg(); \ + jit_fpr_t r2 = get_freg(); \ + jit_##name(r0, r1, r2); \ +} +#define entry_im_ir_fr(name) \ +static void \ +name(void) \ +{ \ + jit_word_t im = get_imm(); \ + jit_gpr_t r0 = get_ireg(); \ + jit_fpr_t r1 = get_freg(); \ + jit_##name(im, r0, r1); \ +} +#define entry_lb_fr_fr(name) \ +static void \ +name(void) \ +{ \ + jit_node_t *jmp; \ + label_t *label = get_label(skip_ws); \ + jit_fpr_t r0 = get_freg(), r1 = get_freg(); \ + if (label->kind == label_kind_code_forward) \ + jmp_forward((void *)jit_##name(r0, r1), label); \ + else { \ + jmp = jit_##name(r0, r1); \ + jit_patch_at(jmp, (jit_node_t *)label->value); \ + } \ +} +#define entry_lb_fr_fm(name) \ +static void \ +name(void) \ +{ \ + jit_node_t *jmp; \ + label_t *label = get_label(skip_ws); \ + jit_fpr_t r0 = get_freg(); \ + jit_float64_t im = get_float(skip_ws); \ + if (label->kind == label_kind_code_forward) \ + jmp_forward((void *)jit_##name(r0, im), label); \ + else { \ + jmp = jit_##name(r0, im); \ + jit_patch_at(jmp, (jit_node_t *)label->value); \ + } \ +} +#define entry_fr(name) \ +static void \ +name(void) \ +{ \ + jit_fpr_t r0 = get_freg(); \ + jit_##name(r0); \ +} +#define entry_fm(name) \ +static void \ +name(void) \ +{ \ + jit_float64_t im = get_float(skip_ws); \ + jit_##name(im); \ +} +#define entry_fn(name) \ +static void \ +name(void) \ +{ \ + int ch; \ + label_t *label; \ + void *value; \ + ch = skipws(); \ + switch (ch) { \ + case '0' ... '9': \ + ungetch(ch); \ + value = (void *)(long)get_uint(skip_none); \ + break; \ + case '$': \ + switch (expression()) { \ + case tok_int: \ + value = (void *)parser.value.i; \ + break; \ + case tok_pointer: \ + value = parser.value.p; \ + break; \ + default: \ + error("expecting pointer"); \ + } \ + break; \ + case '@': \ + dynamic(); \ + value = parser.value.p; \ + break; \ + default: \ + ungetch(ch); \ + label = get_label(skip_none); \ + if (label->kind == label_kind_code_forward) \ + call_forward((void *)jit_##name(NULL), label); \ + else \ + jit_patch_at(jit_##name(NULL), label->value); \ + return; \ + } \ + jit_##name(value); \ +} + +entry(prolog) +void +allocai(void) { + symbol_t *symbol; + jit_word_t i, im = get_imm(); + i = jit_allocai(im); + symbol = get_symbol(); + symbol->type = type_l; + symbol->value.i = i; +} +entry_ca(arg) +entry_ia(getarg_c) entry_ia(getarg_uc) +entry_ia(getarg_s) entry_ia(getarg_us) +entry_ia(getarg_i) entry_ia(getarg_ui) +entry_ia(getarg_l) +entry_ia(getarg) +entry_ir_ir_ir(addr) entry_ir_ir_im(addi) +entry_ir_ir_ir(addxr) entry_ir_ir_im(addxi) +entry_ir_ir_ir(addcr) entry_ir_ir_im(addci) +entry_ir_ir_ir(subr) entry_ir_ir_im(subi) +entry_ir_ir_ir(subxr) entry_ir_ir_im(subxi) +entry_ir_ir_ir(subcr) entry_ir_ir_im(subci) +entry_ir_ir_ir(mulr) entry_ir_ir_im(muli) +entry_ir_ir_ir(divr) entry_ir_ir_im(divi) +entry_ir_ir_ir(divr_u) entry_ir_ir_im(divi_u) +entry_ir_ir_ir(remr) entry_ir_ir_im(remi) +entry_ir_ir_ir(remr_u) entry_ir_ir_im(remi_u) +entry_ir_ir_ir(andr) entry_ir_ir_im(andi) +entry_ir_ir_ir(orr) entry_ir_ir_im(ori) +entry_ir_ir_ir(xorr) entry_ir_ir_im(xori) +entry_ir_ir_ir(lshr) entry_ir_ir_im(lshi) +entry_ir_ir_ir(rshr) entry_ir_ir_im(rshi) +entry_ir_ir_ir(rshr_u) entry_ir_ir_im(rshi_u) +entry_ir_ir(negr) entry_ir_ir(comr) +entry_ir_ir_ir(ltr) entry_ir_ir_im(lti) +entry_ir_ir_ir(ltr_u) entry_ir_ir_im(lti_u) +entry_ir_ir_ir(ler) entry_ir_ir_im(lei) +entry_ir_ir_ir(ler_u) entry_ir_ir_im(lei_u) +entry_ir_ir_ir(eqr) entry_ir_ir_im(eqi) +entry_ir_ir_ir(ger) entry_ir_ir_im(gei) +entry_ir_ir_ir(ger_u) entry_ir_ir_im(gei_u) +entry_ir_ir_ir(gtr) entry_ir_ir_im(gti) +entry_ir_ir_ir(gtr_u) entry_ir_ir_im(gti_u) +entry_ir_ir_ir(ner) entry_ir_ir_im(nei) +entry_ir_ir(movr) +static void +movi(void) +{ + int ch; + label_t *label; + void *value; + jit_gpr_t r0 = get_ireg(); + ch = skipws(); + switch (ch) { + case '0' ... '9': + ungetch(ch); + value = (void *)(long)get_uint(skip_none); + break; + case '$': + switch (expression()) { + case tok_int: + value = (void *)parser.value.i; + break; + case tok_pointer: + value = parser.value.p; + break; + default: + error("expecting pointer"); + } + break; + case '@': + dynamic(); + value = parser.value.p; + break; + default: + ungetch(ch); + label = get_label(skip_none); + if (label->kind == label_kind_code_forward) + mov_forward((void *)jit_movi(r0, NULL), label); + value = label->value; + break; + } + jit_movi(r0, value); +} +entry_ir_ir(extr_c) entry_ir_ir(extr_uc) +entry_ir_ir(extr_s) entry_ir_ir(extr_us) +entry_ir_ir(extr_i) entry_ir_ir(extr_ui) +entry_ir_ir(htonr) entry_ir_ir(ntohr) +entry_ir_ir(ldr_c) entry_ir_pm(ldi_c) +entry_ir_ir(ldr_uc) entry_ir_pm(ldi_uc) +entry_ir_ir(ldr_s) entry_ir_pm(ldi_s) +entry_ir_ir(ldr_us) entry_ir_pm(ldi_us) +entry_ir_ir(ldr_i) entry_ir_pm(ldi_i) +entry_ir_ir(ldr_ui) entry_ir_pm(ldi_ui) +entry_ir_ir(ldr_l) entry_ir_pm(ldi_l) +entry_ir_ir(ldr) entry_ir_pm(ldi) +entry_ir_ir_ir(ldxr_c) entry_ir_ir_im(ldxi_c) +entry_ir_ir_ir(ldxr_uc) entry_ir_ir_im(ldxi_uc) +entry_ir_ir_ir(ldxr_s) entry_ir_ir_im(ldxi_s) +entry_ir_ir_ir(ldxr_us) entry_ir_ir_im(ldxi_us) +entry_ir_ir_ir(ldxr_i) entry_ir_ir_im(ldxi_i) +entry_ir_ir_ir(ldxr_ui) entry_ir_ir_im(ldxi_ui) +entry_ir_ir_ir(ldxr_l) entry_ir_ir_im(ldxi_l) +entry_ir_ir_ir(ldxr) entry_ir_ir_im(ldxi) +entry_ir_ir(str_c) entry_pm_ir(sti_c) +entry_ir_ir(str_s) entry_pm_ir(sti_s) +entry_ir_ir(str_i) entry_pm_ir(sti_i) +entry_ir_ir(str_l) entry_pm_ir(sti_l) +entry_ir_ir(str) entry_pm_ir(sti) +entry_ir_ir_ir(stxr_c) entry_im_ir_ir(stxi_c) +entry_ir_ir_ir(stxr_s) entry_im_ir_ir(stxi_s) +entry_ir_ir_ir(stxr_i) entry_im_ir_ir(stxi_i) +entry_ir_ir_ir(stxr_l) entry_im_ir_ir(stxi_l) +entry_ir_ir_ir(stxr) entry_im_ir_ir(stxi) +entry_lb_ir_ir(bltr) entry_lb_ir_im(blti) +entry_lb_ir_ir(bltr_u) entry_lb_ir_im(blti_u) +entry_lb_ir_ir(bler) entry_lb_ir_im(blei) +entry_lb_ir_ir(bler_u) entry_lb_ir_im(blei_u) +entry_lb_ir_ir(beqr) entry_lb_ir_im(beqi) +entry_lb_ir_ir(bger) entry_lb_ir_im(bgei) +entry_lb_ir_ir(bger_u) entry_lb_ir_im(bgei_u) +entry_lb_ir_ir(bgtr) entry_lb_ir_im(bgti) +entry_lb_ir_ir(bgtr_u) entry_lb_ir_im(bgti_u) +entry_lb_ir_ir(bner) entry_lb_ir_im(bnei) +entry_lb_ir_ir(bmsr) entry_lb_ir_im(bmsi) +entry_lb_ir_ir(bmcr) entry_lb_ir_im(bmci) +entry_lb_ir_ir(boaddr) entry_lb_ir_im(boaddi) +entry_lb_ir_ir(boaddr_u) entry_lb_ir_im(boaddi_u) +entry_lb_ir_ir(bxaddr) entry_lb_ir_im(bxaddi) +entry_lb_ir_ir(bxaddr_u) entry_lb_ir_im(bxaddi_u) +entry_lb_ir_ir(bosubr) entry_lb_ir_im(bosubi) +entry_lb_ir_ir(bosubr_u) entry_lb_ir_im(bosubi_u) +entry_lb_ir_ir(bxsubr) entry_lb_ir_im(bxsubi) +entry_lb_ir_ir(bxsubr_u) entry_lb_ir_im(bxsubi_u) +entry_ir(jmpr) entry_lb(jmpi) +entry_ir(callr) entry_fn(calli) +entry_im(prepare) +entry_ir(pushargr) entry_im(pushargi) +entry_ir(finishr) entry_fn(finishi) +entry(ret) +entry_ir(retr) entry_im(reti) +entry_ir(retval_c) entry_ir(retval_uc) +entry_ir(retval_s) entry_ir(retval_us) +entry_ir(retval_i) entry_ir(retval_ui) +entry_ir(retval_l) +entry_ir(retval) +entry(epilog) +entry_ca(arg_f) +entry_fa(getarg_f) +entry_fr_fr_fr(addr_f) entry_fr_fr_fm(addi_f) +entry_fr_fr_fr(subr_f) entry_fr_fr_fm(subi_f) +entry_fr_fr_fr(mulr_f) entry_fr_fr_fm(muli_f) +entry_fr_fr_fr(divr_f) entry_fr_fr_fm(divi_f) +entry_fr_fr(negr_f) entry_fr_fr(absr_f) +entry_fr_fr(sqrtr_f) +entry_ir_fr_fr(ltr_f) entry_ir_fr_fm(lti_f) +entry_ir_fr_fr(ler_f) entry_ir_fr_fm(lei_f) +entry_ir_fr_fr(eqr_f) entry_ir_fr_fm(eqi_f) +entry_ir_fr_fr(ger_f) entry_ir_fr_fm(gei_f) +entry_ir_fr_fr(gtr_f) entry_ir_fr_fm(gti_f) +entry_ir_fr_fr(ner_f) entry_ir_fr_fm(nei_f) +entry_ir_fr_fr(unltr_f) entry_ir_fr_fm(unlti_f) +entry_ir_fr_fr(unler_f) entry_ir_fr_fm(unlei_f) +entry_ir_fr_fr(uneqr_f) entry_ir_fr_fm(uneqi_f) +entry_ir_fr_fr(unger_f) entry_ir_fr_fm(ungei_f) +entry_ir_fr_fr(ungtr_f) entry_ir_fr_fm(ungti_f) +entry_ir_fr_fr(ltgtr_f) entry_ir_fr_fm(ltgti_f) +entry_ir_fr_fr(ordr_f) entry_ir_fr_fm(ordi_f) +entry_ir_fr_fr(unordr_f) entry_ir_fr_fm(unordi_f) +entry_ir_fr(truncr_f_i) entry_ir_fr(truncr_f_l) +entry_fr_ir(extr_f) entry_fr_fr(extr_d_f) +entry_fr_fr(movr_f) entry_fr_fm(movi_f) +entry_fr_ir(ldr_f) entry_fr_pm(ldi_f) +entry_fr_ir_ir(ldxr_f) entry_fr_ir_im(ldxi_f) +entry_ir_fr(str_f) entry_pm_fr(sti_f) +entry_ir_ir_fr(stxr_f) entry_im_ir_fr(stxi_f) +entry_lb_fr_fr(bltr_f) entry_lb_fr_fm(blti_f) +entry_lb_fr_fr(bler_f) entry_lb_fr_fm(blei_f) +entry_lb_fr_fr(beqr_f) entry_lb_fr_fm(beqi_f) +entry_lb_fr_fr(bger_f) entry_lb_fr_fm(bgei_f) +entry_lb_fr_fr(bgtr_f) entry_lb_fr_fm(bgti_f) +entry_lb_fr_fr(bner_f) entry_lb_fr_fm(bnei_f) +entry_lb_fr_fr(bunltr_f) entry_lb_fr_fm(bunlti_f) +entry_lb_fr_fr(bunler_f) entry_lb_fr_fm(bunlei_f) +entry_lb_fr_fr(buneqr_f) entry_lb_fr_fm(buneqi_f) +entry_lb_fr_fr(bunger_f) entry_lb_fr_fm(bungei_f) +entry_lb_fr_fr(bungtr_f) entry_lb_fr_fm(bungti_f) +entry_lb_fr_fr(bltgtr_f) entry_lb_fr_fm(bltgti_f) +entry_lb_fr_fr(bordr_f) entry_lb_fr_fm(bordi_f) +entry_lb_fr_fr(bunordr_f) entry_lb_fr_fm(bunordi_f) +entry_fr(pushargr_f) entry_fm(pushargi_f) +entry_fr(retr_f) entry_fm(reti_f) +entry_fr(retval_f) +entry_ca(arg_d) +entry_fa(getarg_d) +entry_fr_fr_fr(addr_d) entry_fr_fr_fm(addi_d) +entry_fr_fr_fr(subr_d) entry_fr_fr_fm(subi_d) +entry_fr_fr_fr(mulr_d) entry_fr_fr_fm(muli_d) +entry_fr_fr_fr(divr_d) entry_fr_fr_fm(divi_d) +entry_fr_fr(negr_d) entry_fr_fr(absr_d) +entry_fr_fr(sqrtr_d) +entry_ir_fr_fr(ltr_d) entry_ir_fr_fm(lti_d) +entry_ir_fr_fr(ler_d) entry_ir_fr_fm(lei_d) +entry_ir_fr_fr(eqr_d) entry_ir_fr_fm(eqi_d) +entry_ir_fr_fr(ger_d) entry_ir_fr_fm(gei_d) +entry_ir_fr_fr(gtr_d) entry_ir_fr_fm(gti_d) +entry_ir_fr_fr(ner_d) entry_ir_fr_fm(nei_d) +entry_ir_fr_fr(unltr_d) entry_ir_fr_fm(unlti_d) +entry_ir_fr_fr(unler_d) entry_ir_fr_fm(unlei_d) +entry_ir_fr_fr(uneqr_d) entry_ir_fr_fm(uneqi_d) +entry_ir_fr_fr(unger_d) entry_ir_fr_fm(ungei_d) +entry_ir_fr_fr(ungtr_d) entry_ir_fr_fm(ungti_d) +entry_ir_fr_fr(ltgtr_d) entry_ir_fr_fm(ltgti_d) +entry_ir_fr_fr(ordr_d) entry_ir_fr_fm(ordi_d) +entry_ir_fr_fr(unordr_d) entry_ir_fr_fm(unordi_d) +entry_ir_fr(truncr_d_i) entry_ir_fr(truncr_d_l) +entry_fr_ir(extr_d) entry_fr_fr(extr_f_d) +entry_fr_fr(movr_d) entry_fr_fm(movi_d) +entry_fr_ir(ldr_d) entry_fr_pm(ldi_d) +entry_fr_ir_ir(ldxr_d) entry_fr_ir_im(ldxi_d) +entry_ir_fr(str_d) entry_pm_fr(sti_d) +entry_ir_ir_fr(stxr_d) entry_im_ir_fr(stxi_d) +entry_lb_fr_fr(bltr_d) entry_lb_fr_fm(blti_d) +entry_lb_fr_fr(bler_d) entry_lb_fr_fm(blei_d) +entry_lb_fr_fr(beqr_d) entry_lb_fr_fm(beqi_d) +entry_lb_fr_fr(bger_d) entry_lb_fr_fm(bgei_d) +entry_lb_fr_fr(bgtr_d) entry_lb_fr_fm(bgti_d) +entry_lb_fr_fr(bner_d) entry_lb_fr_fm(bnei_d) +entry_lb_fr_fr(bunltr_d) entry_lb_fr_fm(bunlti_d) +entry_lb_fr_fr(bunler_d) entry_lb_fr_fm(bunlei_d) +entry_lb_fr_fr(buneqr_d) entry_lb_fr_fm(buneqi_d) +entry_lb_fr_fr(bunger_d) entry_lb_fr_fm(bungei_d) +entry_lb_fr_fr(bungtr_d) entry_lb_fr_fm(bungti_d) +entry_lb_fr_fr(bltgtr_d) entry_lb_fr_fm(bltgti_d) +entry_lb_fr_fr(bordr_d) entry_lb_fr_fm(bordi_d) +entry_lb_fr_fr(bunordr_d) entry_lb_fr_fm(bunordi_d) +entry_fr(pushargr_d) entry_fm(pushargi_d) +entry_fr(retr_d) entry_fm(reti_d) +entry_fr(retval_d) + +#undef entry_fn +#undef entry_fm +#undef entry_lb_fr_fm +#undef entry_lb_fr_fr +#undef entry_im_ir_fr +#undef entry_ir_ir_fr +#undef entry_pm_fr +#undef entry_fr_ir_ir +#undef entry_fr_ir_im +#undef entry_fr_pm +#undef entry_fr_fm +#undef entry_fr_ir +#undef entry_ir_fr +#undef entry_ir_fr_fm +#undef entry_ir_fr_fr +#undef entry_fr_fr +#undef entry_fr_fr_fm +#undef entry_fr_fr_fr +#undef entry_fa +#undef entry_pm +#undef entry_lb +#undef entry_lb_ir_im +#undef entry_lb_ir_ir +#undef entry_im_ir_ir +#undef entry_pm_ir +#undef entry_ir_pm +#undef entry_ir_im +#undef entry_ir_ir +#undef entry_ir_ir_im +#undef entry_ir_ir_ir +#undef entry_ir +#undef entry_im +#undef entry_ia +#undef entry_ca +#undef entry + +static void +error(const char *format, ...) +{ + va_list ap; + int length; + char *string; + + va_start(ap, format); + message("error", format, ap); + va_end(ap); + length = parser.data.length - parser.data.offset; + string = (char *)(parser.data.buffer + parser.data.offset - 1); + if (length > 77) + strcpy(string + 74, "..."); + else + parser.data.buffer[parser.data.length - 1] = '\0'; + fprintf(stderr, "(%s)\n", string); + exit(-1); +} + +static void +warn(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + message("warning", format, ap); + va_end(ap); +} + +static void +message(const char *kind, const char *format, va_list ap) +{ + fprintf(stderr, "%s:%d: %s: ", parser.name, + parser.line - parser.newline, kind); + vfprintf(stderr, format, ap); + fputc('\n', stderr); +} + +static int +getch(void) +{ + int ch; + + if (parser.data.offset < parser.data.length) + ch = parser.data.buffer[parser.data.offset++]; + else { + /* keep first offset for ungetch */ + if ((parser.data.length = fread(parser.data.buffer + 1, 1, + sizeof(parser.data.buffer) - 1, + parser.fp) + 1) <= 1) { + ch = EOF; + parser.data.offset = 1; + } + else { + ch = parser.data.buffer[1]; + parser.data.offset = 2; + } + } + if ((parser.newline = ch == '\n')) + ++parser.line; + + return (ch); +} + +static int +getch_noeof(void) +{ + int ch = getch(); + + if (ch == EOF) + error("unexpected end of file"); + + return (ch); +} + +static int +ungetch(int ch) +{ + if ((parser.newline = ch == '\n')) + --parser.line; + + if (parser.data.offset) + parser.data.buffer[--parser.data.offset] = ch; + else + /* overwrite */ + parser.data.buffer[0] = ch; + + return (ch); +} + +static int +skipws(void) +{ + int ch; + + for (ch = getch();; ch = getch()) { + switch (ch) { + case '/': + ch = skipct(); + break; + case '#': + ch = skipcp(); + break; + } + switch (ch) { + case ' ': case '\f': case '\r': case '\t': + break; + default: + return (ch); + } + } +} + +static int +skipnl(void) +{ + int ch; + + for (ch = getch();; ch = getch()) { + switch (ch) { + case '/': + ch = skipct(); + break; + case '#': + ch = skipcp(); + break; + } + switch (ch) { + case ' ': case '\f': case '\n': case '\r': case '\t': + break; + /* handle as newline */ + case ';': + break; + default: + return (ch); + } + } +} + +static int +skipct(void) +{ + int ch; + + ch = getch(); + switch (ch) { + case '/': + for (ch = getch(); ch != '\n' && ch != EOF; ch = getch()) + ; + return (ch); + case '*': + for (; ch != '/';) { + while (getch_noeof() != '*') + ; + while ((ch = getch_noeof()) == '*') + ; + } + return (getch()); + default: + ungetch(ch); + return ('/'); + } +} + +static int +skipcp(void) +{ + int ch; + + for (ch = getch(); ch != '\n' && ch != EOF; ch = getch()) { + switch (ch) { + case '0' ... '9': + if ((number(ch)) == tok_int) + parser.line = parser.value.i - 1; + break; + case '"': + string(); + if (parser.offset >= (int)sizeof(parser.name)) { + strncpy(parser.name, parser.string, sizeof(parser.name)); + parser.name[sizeof(parser.name) - 1] = '\0'; + } + else + strcpy(parser.name, parser.string); + break; + default: + break; + } + } + + return (ch); +} + +static long +get_int(skip_t skip) +{ + switch (primary(skip)) { + case tok_int: + break; + case tok_pointer: + parser.type = type_l; + parser.value.i = (long)parser.value.p; + break; + default: + error("expecting integer"); + } + + return (parser.value.i); +} + +static unsigned long +get_uint(skip_t skip) +{ + switch (primary(skip)) { + case tok_char: case tok_int: + break; + case tok_pointer: + parser.type = type_l; + parser.value.ui = (unsigned long)parser.value.p; + break; + default: + error("expecting integer"); + } + + return (parser.value.ui); +} + +static double +get_float(skip_t skip) +{ + switch (primary(skip)) { + case tok_char: + case tok_int: + parser.type = type_d; + parser.value.d = parser.value.i; + break; + case tok_float: + break; + default: + error("expecting float"); + } + + return (parser.value.d); +} + +static void * +get_pointer(skip_t skip) +{ + label_t *label; + token_t token = primary(skip); + + switch (token) { + case tok_symbol: + label = get_label_by_name(parser.string); + if (label == NULL) + error("bad identifier %s", parser.string); + switch (label->kind) { + case label_kind_data: + case label_kind_code: + break; + case label_kind_code_forward: + /* as expression arguments */ + error("forward references not implemented"); + break; + case label_kind_dynamic: + break; + } + parser.type = type_p; + return (parser.value.p = label->value); + case tok_int: + parser.type = type_p; + return (parser.value.p = (void *)parser.value.ui); + case tok_pointer: + return (parser.value.p); + default: error("bad pointer"); + } +} + +static label_t * +get_label(skip_t skip) +{ + label_t *label; + int ch = skipws(); + + switch (ch) { + case '@': + (void)dynamic(); + break; + case 'a' ... 'z': case 'A' ... 'Z': case '_': + (void)identifier(ch); + break; + default: + error("expecting label"); + } + if ((label = get_label_by_name(parser.string)) == NULL) + label = new_label(label_kind_code_forward, + parser.string, jit_forward()); + + return (label); +} + +static token_t +regname(void) +{ + long num; + int check = 1, ch = getch(); + + switch (ch) { + case 'r': + parser.regtype = type_l; + switch (ch = getch()) { + case '0': parser.regval = JIT_R0; break; + case '1': parser.regval = JIT_R1; break; + case '2': parser.regval = JIT_R2; break; + case 'e': + if (getch() != 't') goto fail; + parser.regval = JIT_RET; + break; + case '(': + num = get_int(skip_none); + if (num < 0 || num >= JIT_R_NUM) goto fail; + parser.regval = JIT_R(num); + if (getch() != ')') goto fail; + check = 0; + break; + default: goto fail; + } + break; + case 'v': + parser.regtype = type_l; + switch (ch = getch()) { + case '0': parser.regval = JIT_V0; break; + case '1': parser.regval = JIT_V1; break; + case '2': parser.regval = JIT_V2; break; + default: goto fail; + case '(': + num = get_int(skip_none); + if (num < 0 || num >= JIT_V_NUM) goto fail; + parser.regval = JIT_V(num); + if (getch() != ')') goto fail; + check = 0; + break; + } + break; + case 'f': + parser.regtype = type_d; + switch (ch = getch()) { + case '0': parser.regval = JIT_F0; break; + case '1': parser.regval = JIT_F1; break; + case '2': parser.regval = JIT_F2; break; + case '3': parser.regval = JIT_F3; break; + case '4': parser.regval = JIT_F4; break; + case '5': parser.regval = JIT_F5; break; + case 'p': + parser.regtype = type_l; /* oops */ + parser.regval = JIT_FP; break; + case 'r': + if (getch() != 'e' || getch() != 't') goto fail; + parser.regval = JIT_FRET; + break; + case '(': + num = get_int(skip_none); + if (num < 0 || num >= JIT_F_NUM) goto fail; + parser.regval = JIT_F(num); + if (getch() != ')') goto fail; + check = 0; + break; + default: goto fail; + } + break; + case 's': + parser.regtype = type_l; + if (getch() != 'p') + goto fail; + parser.regval = JIT_SP; + break; + default: + fail: + error("bad register"); + } + if (check) { + ch = getch(); + if ((ch >= 'a' && ch <= 'z') || + (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || + ch == '_') + goto fail; + ungetch(ch); + } + + return (tok_register); +} + +static token_t +identifier(int ch) +{ + parser.string[0] = ch; + for (parser.offset = 1;;) { + switch ((ch = getch())) { + case 'a' ... 'z': case 'A' ... 'Z': case '0' ... '9' : case '_': + if (parser.offset + 1 >= MAX_IDENTIFIER) { + parser.string[parser.offset] = '\0'; + error("bad identifier %s", parser.string); + } + parser.string[parser.offset++] = ch; + break; + default: + parser.string[parser.offset] = '\0'; + ungetch(ch); + return (tok_symbol); + } + } +} + +static void +get_data(type_t type) +{ + int ch; + token_t token; + char *test = data; + + for (;;) { + switch (type) { + case type_c: + switch (token = primary(skip_ws)) { + case tok_char: case tok_int: + check_data(sizeof(signed char)); + *(signed char *)(data + data_offset) = parser.value.i; + data_offset += sizeof(char); + break; + case tok_string: + check_data(parser.offset); + memcpy(data + data_offset, parser.string, + parser.offset); + data_offset += parser.offset; + break; + case tok_newline: + case tok_semicollon: + if (test == data) error("syntax error"); + return; + default: error("bad initializer"); + } + break; + case type_s: + check_data(sizeof(signed short)); + *(signed short *)(data + data_offset) = get_int(skip_ws); + data_offset += sizeof(short); + break; + case type_i: + check_data(sizeof(signed int)); + *(signed int *)(data + data_offset) = get_int(skip_ws); + data_offset += sizeof(int); + break; + case type_l: + check_data(sizeof(signed long)); + *(signed long *)(data + data_offset) = get_int(skip_ws); + data_offset += sizeof(long); + break; + case type_f: + check_data(sizeof(float)); + *(float *)(data + data_offset) = get_float(skip_ws); + data_offset += sizeof(float); + break; + case type_d: + check_data(sizeof(double)); + *(double *)(data + data_offset) = get_float(skip_ws); + data_offset += sizeof(double); + break; + case type_p: + /* FIXME **patch if realloc** */ + check_data(sizeof(void*)); + *(void **)(data + data_offset) = get_pointer(skip_ws); + data_offset += sizeof(void*); + break; + default: + abort(); + } + ch = skipws(); + if (ch == '\n' || ch == ';' || ch == EOF) + break; + ungetch(ch); + } +} + +static void +dot(void) +{ + int ch; + size_t offset, length; + + switch (ch = getch_noeof()) { + case '$': + /* use .$(expression) for non side effects expression */ + (void)expression(); + return; + case 'a' ... 'z': case 'A' ... 'Z': case '_': + (void)identifier(ch); + break; + default: + ungetch(ch); + if (skipws() != '$') + error("expecting symbol"); + /* allow spaces before an expression */ + (void)expression(); + return; + } + if (parser.string[1] == '\0') { + switch (parser.string[0]) { + case 'c': get_data(type_c); break; + case 's': get_data(type_s); break; + case 'i': get_data(type_i); break; + case 'l': get_data(type_l); break; + case 'f': get_data(type_f); break; + case 'd': get_data(type_d); break; + case 'p': get_data(type_p); break; + default: error("bad type .%c", parser.string[0]); + } + } + else if (strcmp(parser.string, "data") == 0) { + if (parser.parsing != PARSING_NONE) + error(".data must be specified once and be the first section"); + parser.parsing = PARSING_DATA; + data_length = get_int(skip_ws); + data = (char *)xcalloc(1, data_length); + } + else if (strcmp(parser.string, "code") == 0) { + if (parser.parsing != PARSING_NONE && + parser.parsing != PARSING_DATA) + error(".code must be specified once only"); + parser.parsing = PARSING_CODE; + } + else if (strcmp(parser.string, "align") == 0) { + length = get_int(skip_ws); + if (parser.parsing != PARSING_DATA) + error(".align must be in .data"); + if (length > 1 && length <= 4096 && !(length & (length - 1))) { + offset = data_offset; + offset += length - ((offset + length) % length); + check_data(offset - data_offset); + data_offset = offset; + } + else + error("bad .align %ld (must be a power of 2, >= 2 && <= 4096)", + (long)length); + } + else if (strcmp(parser.string, "size") == 0) { + length = get_int(skip_ws); + if (parser.parsing != PARSING_DATA) + error(".size must be in .data"); + check_data(length); + data_offset += length; + } + else if (strcmp(parser.string, "cpu") == 0) { + if (primary(skip_ws) != tok_symbol) + error("expecting cpu flag"); + ch = get_int(skip_ws); + if (strcmp(parser.string, "sse2") == 0) +#if defined(__i386__) + /* only meaningful for i386 as there is no x87 path for x86_64 + * and should only use just after jit_prolog and not mix with + * code that uses xmm registers */ + jit_cpu.sse2 = !!ch +#endif + ; + else if (strcmp(parser.string, "sse4_1") == 0) +#if defined(__i386__) || defined(__x86_64__) + jit_cpu.sse4_1 = !!ch +#endif + ; + else if (strcmp(parser.string, "version") == 0) +#if defined(__arm__) + jit_cpu.version = ch +#endif + ; + else if (strcmp(parser.string, "thumb") == 0) +#if defined(__arm__) + jit_cpu.thumb = ch +#endif + ; + else if (strcmp(parser.string, "vfp") == 0) +#if defined(__arm__) + jit_cpu.vfp = ch +#endif + ; + else if (strcmp(parser.string, "neon") == 0) +#if defined(__arm__) + jit_cpu.neon = !!ch +#endif + ; + else + warn("ignoring \".cpu %s %d\"", parser.string, ch); + } + else if (strcmp(parser.string, "disasm") == 0) + flag_disasm = 1; + else + error("unknown command .%s", parser.string); +} + +static token_t +number(int ch) +{ + char buffer[1024], *endptr; + int integer = 1, offset = 0, neg = 0, e = 0, d = 0, base = 10; + + for (;; ch = getch()) { + switch (ch) { + case '-': + if (offset == 0) { + neg = 1; + continue; + } + if (offset && buffer[offset - 1] != 'e') { + ungetch(ch); + goto done; + } + break; + case '+': + if (offset == 0) + continue; + if (offset && buffer[offset - 1] != 'e') { + ungetch(ch); + goto done; + } + break; + case '.': + if (d) + goto fail; + d = 1; + base = 10; + integer = 0; + break; + case '0': + if (offset == 0 && base == 10) { + base = 8; + continue; + } + break; + case 'b': + if (offset == 0 && base == 8) { + base = 2; + continue; + } + if (base != 16) + goto fail; + break; + case '1': + break; + case '2' ... '7': + if (base < 8) + goto fail; + break; + case '8': case '9': + if (base < 10) + goto fail; + break; + case 'x': + if (offset == 0 && base == 8) { + base = 16; + continue; + } + goto fail; + case 'a': case 'c': case 'd': case 'f': + if (base < 16) + goto fail; + break; + case 'e': + if (e) + goto fail; + if (base != 16) { + e = 1; + base = 10; + integer = 0; + } + break; + case '_': case 'g' ... 'w': case 'y': case 'z': case 'A' ... 'Z': + fail: + buffer[offset++] = '\0'; + error("bad constant %s", buffer); + default: + ungetch(ch); + goto done; + } + if (offset + 1 >= (int)sizeof(buffer)) + goto fail; + buffer[offset++] = ch; + } +done: + /* check for literal 0 */ + if (offset == 0 && base == 8) buffer[offset++] = '0'; + buffer[offset] = '\0'; + if (integer) { + parser.value.ui = strtoul(buffer, &endptr, base); + parser.type = type_l; + if (neg) + parser.value.i = -parser.value.i; + } + else { + parser.type = type_d; + parser.value.d = strtod(buffer, &endptr); + if (neg) + parser.value.d = -parser.value.d; + } + if (*endptr) + goto fail; + + return (integer ? tok_int : tok_float); +} + +static int +escape(int ch) +{ + switch (ch) { + case 'a': ch = '\a'; break; + case 'b': ch = '\b'; break; + case 'f': ch = '\f'; break; + case 'n': ch = '\n'; break; + case 'r': ch = '\r'; break; + case 't': ch = '\t'; break; + case 'v': ch = '\v'; break; + default: break; + } + + return (ch); +} + +static token_t +string(void) +{ + int ch, esc = 0; + + for (parser.offset = 0;;) { + switch (ch = getch_noeof()) { + case '\\': + if (esc) goto append; + esc = 1; + break; + case '"': + if (!esc) { + parser.string[parser.offset++] = '\0'; + parser.value.p = parser.string; + parser.type = type_p; + return (tok_string); + } + /* FALLTHROUGH */ + default: + append: + if (esc) { + ch = escape(ch); + esc = 0; + } + if (parser.offset + 1 >= parser.length) { + parser.length += 4096; + parser.string = (char *)xrealloc(parser.string, + parser.length); + } + parser.string[parser.offset++] = ch; + break; + } + } +} + +static token_t +character(void) +{ + int ch, esc = 0; + + if ((ch = getch_noeof()) == '\\') { + esc = 1; + ch = getch(); + } + if (getch_noeof() != '\'') + error("bad single byte char"); + if (esc) + ch = escape(ch); + parser.type = type_l; + parser.value.i = ch & 0xff; + + return (tok_char); +} + +static token_t +dynamic(void) +{ + label_t *label; + void *value; + char *string; + (void)identifier('@'); + if ((label = get_label_by_name(parser.string)) == NULL) { + value = dlsym(RTLD_DEFAULT, parser.string + 1); + if ((string = dlerror())) + error("%s", string); + label = new_label(label_kind_dynamic, parser.string, value); + } + parser.type = type_p; + parser.value.p = label->value; + + return (tok_pointer); +} + +static void +expression_prim(void) +{ + int ch; + token_t token; + label_t *label; + symbol_t *symbol; + + if (parser.putback) { + parser.expr = parser.putback; + parser.putback = (expr_t)0; + return; + } + switch (ch = skipws()) { + case '!': + if ((ch = getch_noeof()) == '=') parser.expr = expr_ne; + else { + ungetch(ch); parser.expr = expr_not; + } + break; + case '~': parser.expr = expr_com; + break; + case '*': + if ((ch = getch_noeof()) == '=') parser.expr = expr_mulset; + else { + ungetch(ch); parser.expr = expr_mul; + } + break; + case '/': + if ((ch = getch_noeof()) == '=') parser.expr = expr_divset; + else { + ungetch(ch); parser.expr = expr_div; + } + break; + case '%': + if ((ch = getch_noeof()) == '=') parser.expr = expr_remset; + else { + ungetch(ch); parser.expr = expr_rem; + } + break; + case '+': + switch (ch = getch_noeof()) { + case '+': parser.expr = expr_inc; + break; + case '=': parser.expr = expr_addset; + break; + default: ungetch(ch); parser.expr = expr_add; + break; + } + break; + case '-': + switch (ch = getch_noeof()) { + case '-': parser.expr = expr_dec; + break; + case '=': parser.expr = expr_subset; + break; + default: ungetch(ch); parser.expr = expr_sub; + break; + } + break; + case '<': + switch (ch = getch_noeof()) { + case '=': parser.expr = expr_le; + break; + case '<': ch = getch_noeof(); + if (ch == '=') parser.expr = expr_lshset; + else { + ungetch(ch); parser.expr = expr_lsh; + } + break; + default: ungetch(ch); parser.expr = expr_lt; + break; + } + break; + case '>': + switch (ch = getch_noeof()) { + case '=': parser.expr = expr_ge; + break; + case '>': ch = getch_noeof(); + if (ch == '=') parser.expr = expr_rshset; + else { + ungetch(ch); parser.expr = expr_rsh; + } + break; + default: ungetch(ch); parser.expr = expr_gt; + break; + } + break; + case '&': + switch (ch = getch_noeof()) { + case '=': parser.expr = expr_andset; + break; + case '&': parser.expr = expr_andand; + break; + default: ungetch(ch); parser.expr = expr_and; + break; + } + break; + case '|': + switch (ch = getch_noeof()) { + case '=': parser.expr = expr_orset; + break; + case '|': parser.expr = expr_oror; + break; + default: ungetch(ch); parser.expr = expr_or; + break; + } + break; + case '^': + if ((ch = getch_noeof()) == '=') parser.expr = expr_xorset; + else { + ungetch(ch); parser.expr = expr_xor; + } + break; + case '=': + if ((ch = getch_noeof()) == '=') parser.expr = expr_eq; + else { + ungetch(ch); parser.expr = expr_set; + } + break; + case '(': parser.expr = expr_lparen; + break; + case ')': parser.expr = expr_rparen; + break; + case '0' ... '9': + token = number(ch); + parser.expr = token == tok_int ? expr_int : expr_float; + break; + case '@': + (void)dynamic(); + parser.expr = expr_pointer; + break; + case '$': + identifier('$'); + /* no support for nested expressions */ + if (parser.string[0] == '\0') + error("syntax error"); + parser.expr = expr_symbol; + if ((symbol = get_symbol_by_name(parser.string)) != NULL) { + parser.type = symbol->type; + parser.value = symbol->value; + } + else + /* only create symbol on assignment */ + parser.type = type_none; + break; + case 'a' ... 'z': case 'A' ... 'Z': case '_': + identifier(ch); + if ((label = get_label_by_name(parser.string))) { + if (label->kind == label_kind_code_forward) + error("forward value for %s not supported", + parser.string); + parser.expr = expr_pointer; + parser.type = type_p; + parser.value.p = label->value; + } + else + error("invalid identifier %s", parser.string); + break; + case '\'': + character(); + parser.expr = expr_int; + break; + case '"': + /* not smart enough to put it in data and/or relocate it, etc */ + error("must declare strings as data"); + default: + error("syntax error"); + } +} + +static void +expression_inc(int pre) +{ + symbol_t *symbol; + + if (pre) { + expression_prim(); + if (parser.expr != expr_symbol) + error("syntax error"); + } + if ((symbol = get_symbol_by_name(parser.string)) == NULL) { + if (!parser.short_circuit) + error("undefined symbol %s", symbol->name); + } + if (!parser.short_circuit) { + parser.type = symbol->type; + if (!pre) + parser.value = symbol->value; + switch (symbol->type) { + case type_l: + ++symbol->value.i; + break; + case type_d: + /* should really be an error */ + symbol->value.d += 1.0; + break; + default: + ++parser.value.cp; + break; + } + if (pre) + parser.value = symbol->value; + } + expression_prim(); +} + +static void +expression_dec(int pre) +{ + symbol_t *symbol; + + if (pre) { + expression_prim(); + if (parser.expr != expr_symbol) + error("syntax error"); + } + if ((symbol = get_symbol_by_name(parser.string)) == NULL) { + if (!parser.short_circuit) + error("undefined symbol %s", symbol->name); + } + if (!parser.short_circuit) { + parser.type = symbol->type; + if (!pre) + parser.value = symbol->value; + switch (symbol->type) { + case type_l: + --symbol->value.i; + break; + case type_d: + /* should really be an error */ + symbol->value.d -= 1.0; + break; + default: + --parser.value.cp; + break; + } + if (pre) + parser.value = symbol->value; + } + expression_prim(); +} + +static void +expression_unary(void) +{ + symbol_t *symbol; + char buffer[256]; + + expression_prim(); + switch (parser.expr) { + case expr_add: + expression_unary(); + switch (parser.type) { + case type_l: + case type_d: + break; + default: + error("syntax error"); + } + break; + case expr_sub: + expression_unary(); + switch (parser.type) { + case type_l: + parser.value.i = -parser.value.i; + break; + case type_d: + parser.value.d = -parser.value.d; + break; + default: + error("syntax error"); + } + break; + case expr_inc: + expression_inc(1); + break; + case expr_dec: + expression_dec(1); + break; + case expr_not: + expression_unary(); + switch (parser.type) { + case type_l: + parser.value.i = !parser.value.i; + break; + case type_d: + parser.value.i = parser.value.d != 0; + break; + case type_p: + parser.value.i = parser.value.p != NULL; + break; + default: + error("syntax error"); + } + parser.type = type_l; + break; + case expr_com: + expression_unary(); + if (parser.type != type_l) + error("syntax error"); + parser.value.i = ~parser.value.i; + break; + case expr_lparen: + expression_cond(); + if (parser.expr != expr_rparen) + error("syntax error"); + expression_prim(); + break; + case expr_symbol: + strcpy(buffer, parser.string); + expression_prim(); + switch (parser.expr) { + case expr_set: + if ((symbol = get_symbol_by_name(buffer)) == NULL) { + if (!parser.short_circuit) + symbol = new_symbol(buffer); + } + expression_cond(); + set: + if (!parser.short_circuit) { + if (symbol == NULL) + error("syntax error"); + symbol->type = parser.type; + symbol->value = parser.value; + } + break; + case expr_mulset: parser.putback = expr_mul; + goto check; + case expr_divset: parser.putback = expr_div; + goto check; + case expr_remset: parser.putback = expr_rem; + goto check; + case expr_addset: parser.putback = expr_add; + goto check; + case expr_subset: parser.putback = expr_sub; + goto check; + case expr_lshset: parser.putback = expr_lsh; + goto check; + case expr_rshset: parser.putback = expr_rsh; + goto check; + case expr_andset: parser.putback = expr_and; + goto check; + case expr_orset: parser.putback = expr_or; + goto check; + case expr_xorset: parser.putback = expr_xor; + check: + if ((symbol = get_symbol_by_name(buffer)) == NULL) { + if (!parser.short_circuit) + error("undefined symbol %s", buffer); + parser.type = type_l; + parser.value.i = 1; + } + switch (parser.putback) { + case expr_mul: case expr_div: case expr_rem: + expression_mul(); + break; + case expr_add: case expr_sub: + expression_add(); + break; + case expr_lsh: case expr_rsh: + expression_shift(); + break; + case expr_and: case expr_or: case expr_xor: + expression_bit(); + break; + default: + abort(); + } + goto set; + case expr_inc: + expression_inc(0); + break; + case expr_dec: + expression_dec(0); + break; + default: + break; + } + break; + case expr_int: + case expr_float: + case expr_pointer: + /* make next token available */ + expression_prim(); + default: + break; + } +} + +static void +expression_mul(void) +{ + type_t type; + value_t value; + + expression_unary(); + switch (parser.type) { + case type_l: case type_d: case type_p: break; + default: return; + } + for (;;) { + switch (parser.expr) { + case expr_mul: + type = parser.type, value = parser.value; + expression_unary(); + switch (parser.type) { + case type_l: + if (type == type_l) + value.i *= parser.value.i; + else + value.d *= parser.value.i; + break; + case type_d: + if (type == type_l) { + type = type_d; + value.d = value.i; + } + value.d *= parser.value.d; + break; + default: + error("invalid operand"); + } + parser.type = type, parser.value = value; + break; + case expr_div: + type = parser.type, value = parser.value; + expression_unary(); + switch (parser.type) { + case type_l: + if (type == type_l) { + if (parser.value.i == 0) + error("divide by zero"); + value.i /= parser.value.i; + } + else + value.d /= parser.value.i; + break; + case type_d: + if (type == type_l) { + type = type_d; + value.d = value.i; + } + value.d /= parser.value.d; + break; + default: + error("invalid operand"); + } + parser.type = type, parser.value = value; + break; + case expr_rem: + type = parser.type, value = parser.value; + expression_unary(); + switch (parser.type) { + case type_l: + if (type == type_l) { + if (parser.value.i == 0) + error("divide by zero"); + value.i %= parser.value.i; + } + else + error("invalid operand"); + break; + default: + error("invalid operand"); + } + parser.type = type, parser.value = value; + break; + default: + return; + } + } +} + +static void +expression_add(void) +{ + type_t type; + value_t value; + + expression_mul(); + switch (parser.type) { + case type_l: case type_d: case type_p: break; + default: return; + } + for (;;) { + switch (parser.expr) { + case expr_add: + type = parser.type, value = parser.value; + expression_mul(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i += parser.value.i; + break; + case type_d: + value.d += parser.value.i; + break; + default: + value.cp += parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + type = type_d; + value.d = value.i; + break; + case type_d: + break; + default: + error("invalid operand"); + } + value.d += parser.value.d; + break; + case type_p: + switch (type) { + case type_l: + type = type_p; + value.cp = value.i + parser.value.cp; + break; + default: + error("invalid operand"); + } + break; + default: + error("invalid operand"); + } + parser.type = type, parser.value = value; + break; + case expr_sub: + type = parser.type, value = parser.value; + expression_mul(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i -= parser.value.i; + break; + case type_d: + value.d -= parser.value.i; + break; + default: + value.cp -= parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + type = type_d; + value.d = value.i; + break; + case type_d: + break; + default: + error("invalid operand"); + } + value.d -= parser.value.d; + break; + case type_p: + switch (type) { + case type_p: + type = type_l; + value.i = value.cp - parser.value.cp; + break; + default: + error("invalid operand"); + } + break; + default: + error("invalid operand"); + } + parser.type = type, parser.value = value; + break; + default: + return; + } + } +} + +static void +expression_shift(void) +{ + long value; + expression_add(); + + switch (parser.type) { + case type_l: case type_d: case type_p: break; + default: return; + } + for (;;) { + switch (parser.expr) { + case expr_lsh: + value = parser.value.i; + if (parser.type != type_l) + error("invalid operand"); + expression_add(); + if (parser.type != type_l) + error("invalid operand"); + value <<= parser.value.i; + parser.value.i = value; + break; + case expr_rsh: + value = parser.value.i; + if (parser.type != type_l) + error("invalid operand"); + expression_add(); + if (parser.type != type_l) + error("invalid operand"); + value >>= parser.value.i; + parser.value.i = value; + break; + default: + return; + } + } +} + +static void +expression_bit(void) +{ + long i; + + expression_shift(); + switch (parser.type) { + case type_l: case type_d: case type_p: break; + default: return; + } + for (;;) { + switch (parser.expr) { + case expr_and: + if (parser.type != type_l) + error("invalid operand"); + i = parser.value.i; + expression_shift(); + if (parser.type != type_l) + error("invalid operand"); + i &= parser.value.i; + parser.value.i = i; + break; + case expr_or: + if (parser.type != type_l) + error("invalid operand"); + i = parser.value.i; + expression_shift(); + if (parser.type != type_l) + error("invalid operand"); + i |= parser.value.i; + parser.value.i = i; + break; + case expr_xor: + if (parser.type != type_l) + error("invalid operand"); + i = parser.value.i; + expression_shift(); + if (parser.type != type_l) + error("invalid operand"); + i ^= parser.value.i; + parser.value.i = i; + break; + default: + return; + } + } +} + +static void +expression_rel(void) +{ + type_t type; + value_t value; + + expression_bit(); + switch (parser.type) { + case type_l: case type_d: case type_p: break; + default: return; + } + for (;;) { + switch (parser.expr) { + case expr_lt: + type = parser.type, value = parser.value; + expression_bit(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i < parser.value.i; + break; + case type_d: + value.i = value.d < parser.value.i; + break; + default: + value.i = (long)value.p < parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i < parser.value.d; + break; + case type_d: + value.i = value.d < parser.value.d; + break; + default: + error("invalid operand"); + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i < (long)parser.value.p; + break; + case type_d: + error("invalid operand"); + default: + value.i = (long)value.p < (long)parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value = value; + break; + case expr_le: + type = parser.type, value = parser.value; + expression_bit(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i <= parser.value.i; + break; + case type_d: + value.i = value.d <= parser.value.i; + break; + default: + value.i = (long)value.p <= parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i <= parser.value.d; + break; + case type_d: + value.i = value.d <= parser.value.d; + break; + default: + value.i = (long)value.p <= parser.value.d; + break; + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i <= (long)parser.value.p; + break; + case type_d: + error("invalid operand"); + default: + value.i = (long)value.p <= (long)parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value = value; + break; + case expr_eq: + type = parser.type, value = parser.value; + expression_bit(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i == parser.value.i; + break; + case type_d: + value.i = value.d == parser.value.i; + break; + default: + value.i = (long)value.p == parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i == parser.value.d; + break; + case type_d: + value.i = value.d == parser.value.d; + break; + default: + error("invalid operand"); + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i == (long)parser.value.p; + break; + case type_d: + error("invalid operand"); + default: + value.i = value.p == parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value = value; + break; + case expr_ge: + type = parser.type, value = parser.value; + expression_bit(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i >= parser.value.i; + break; + case type_d: + value.i = value.d >= parser.value.i; + break; + default: + value.i = (long)value.p >= parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i >= parser.value.d; + break; + case type_d: + value.i = value.d >= parser.value.d; + break; + default: + error("invalid operand"); + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i >= (long)parser.value.p; + break; + case type_d: + error("invalid operand"); + default: + value.i = (long)value.p >= (long)parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value = value; + break; + case expr_gt: + type = parser.type, value = parser.value; + expression_bit(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i > parser.value.i; + break; + case type_d: + value.i = value.d > parser.value.i; + break; + default: + value.i = (long)value.p > parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i > parser.value.d; + break; + case type_d: + value.i = value.d > parser.value.d; + break; + default: + error("invalid operand"); + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i > (long)parser.value.p; + break; + case type_d: + error("invalid operand"); + default: + value.i = (long)value.p > (long)parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value = value; + break; + case expr_ne: + type = parser.type, value = parser.value; + expression_bit(); + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i != parser.value.i; + break; + case type_d: + value.i = value.d != parser.value.i; + break; + default: + value.i = (long)value.p != parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i != parser.value.d; + break; + case type_d: + value.i = value.d != parser.value.d; + break; + default: + error("invalid operand"); + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i != (long)parser.value.p; + break; + case type_d: + error("invalid operand"); + default: + value.i = value.p != parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value = value; + break; + default: + return; + } + } +} + +static void +expression_cond(void) +{ + type_t type; + value_t value; + int short_circuit; + + expression_rel(); + switch (parser.type) { + case type_l: case type_d: case type_p: break; + default: return; + } + for (;;) { + switch (parser.expr) { + case expr_andand: + type = parser.type, value = parser.value; + switch (type) { + case type_l: + short_circuit = value.i == 0; + break; + case type_d: + short_circuit = value.d == 0.0; + break; + default: + short_circuit = value.p == NULL; + break; + } + parser.short_circuit += short_circuit; + expression_rel(); + parser.short_circuit -= short_circuit; + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i && parser.value.i; + break; + case type_d: + value.i = value.d && parser.value.i; + break; + default: + value.i = value.p && parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i && parser.value.d; + break; + case type_d: + value.i = value.d && parser.value.d; + break; + default: + value.i = value.p && parser.value.d; + break; + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i && parser.value.p; + break; + case type_d: + value.i = value.d && parser.value.p; + break; + default: + value.i = value.p && parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value.i = value.i; + break; + case expr_oror: + type = parser.type, value = parser.value; + switch (type) { + case type_l: + short_circuit = value.i != 0; + break; + case type_d: + short_circuit = value.d != 0.0; + break; + default: + short_circuit = value.p != NULL; + break; + } + parser.short_circuit += short_circuit; + expression_rel(); + parser.short_circuit -= short_circuit; + switch (parser.type) { + case type_l: + switch (type) { + case type_l: + value.i = value.i || parser.value.i; + break; + case type_d: + value.i = value.d || parser.value.i; + break; + default: + value.i = value.p || parser.value.i; + break; + } + break; + case type_d: + switch (type) { + case type_l: + value.i = value.i || parser.value.d; + break; + case type_d: + value.i = value.d || parser.value.d; + break; + default: + value.i = value.p || parser.value.d; + break; + } + break; + case type_p: + switch (type) { + case type_l: + value.i = value.i || parser.value.p; + break; + case type_d: + value.i = value.d || parser.value.p; + break; + default: + value.i = value.p || parser.value.p; + break; + } + break; + default: + error("invalid operand"); + } + parser.type = type_l, parser.value.i = value.i; + break; + default: + return; + } + } +} + +static token_t +expression(void) +{ + symbol_t *symbol; + + (void)identifier('$'); + if (parser.string[1] == '\0') { + if (getch_noeof() != '(') + error("bad symbol or expression"); + parser.type = type_none; + expression_cond(); + if (parser.expr != expr_rparen) + error("bad expression"); + switch (parser.type) { + case type_l: + return (tok_int); + case type_d: + return (tok_float); + case type_p: + return (tok_pointer); + default: + error("bad expression"); + } + } + else if ((symbol = get_symbol_by_name(parser.string))) { + switch (parser.type = symbol->type) { + case type_l: + parser.value.i = symbol->value.i; + return (tok_int); + case type_d: + parser.value.d = symbol->value.d; + return (tok_float); + default: + parser.value.p = symbol->value.p; + return (tok_pointer); + } + } + else + error("undefined symbol %s", parser.string); +} + +static token_t +primary(skip_t skip) +{ + int ch; + + switch (skip) { + case skip_none: ch = getch(); break; + case skip_ws: ch = skipws(); break; + case skip_nl: ch = skipnl(); break; + default: abort(); + } + switch (ch) { + case '%': + return (regname()); + case 'a' ... 'z': case 'A' ... 'Z': case '_': + return (identifier(ch)); + case '0' ... '9': case '+': case '-': + return (number(ch)); + case '.': + return (tok_dot); + case '"': + return (string()); + case '\'': + return (character()); + case '@': + return (dynamic()); + case '$': + return (expression()); + case EOF: + return (tok_eof); + case '\n': + return (tok_newline); + case ';': + return (tok_semicollon); + default: + error("syntax error"); + } +} + +static void +parse(void) +{ + int ch; + label_kind_t kind; + token_t token; + instr_t *instr; + label_t *label; + void *value; + + for (;;) { + switch (token = primary(skip_nl)) { + case tok_symbol: + ch = getch_noeof(); + if (ch == ':') { + if ((label = get_label_by_name(parser.string))) { + if (label->kind == label_kind_code_forward) { + label->kind = label_kind_code; + label->value = jit_label(); + } + else + error("label %s: redefined", parser.string); + } + else { + if (parser.parsing == PARSING_DATA) { + kind = label_kind_data; + value = data + data_offset; + } + else if (parser.parsing == PARSING_CODE) { + kind = label_kind_code; + value = jit_label(); + } + else + error("label not in .code or .data"); + label = new_label(kind, parser.string, value); + } + break; + } + ungetch(ch); + if ((instr = + (instr_t *)get_hash(instrs, parser.string)) == NULL) + error("unhandled symbol %s", parser.string); + if (parser.parsing != PARSING_CODE) + error(".code must be specified before instructions"); + (*instr->function)(); + break; + case tok_dot: + dot(); + break; + case tok_eof: + return; + default: + error("syntax error"); + } + } +} + +static int +execute(int argc, char *argv[]) +{ + label_t *label; + function_t function; + patch_t *patch, *next; + + for (patch = patches; patch; patch = next) { + next = patch->next; + label = patch->label; + if (label->kind == label_kind_code_forward) + error("undefined label %s", label->name); + switch (patch->kind) { + case patch_kind_jmp: + case patch_kind_mov: + case patch_kind_call: + jit_patch_at(patch->value, label->value); + break; + default: + abort(); + } + free(patch); + patch = next; + } + + function = jit_emit(); + if (flag_verbose > 1 || flag_disasm) { + jit_print(); + fprintf(stdout, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); + } + if (flag_verbose > 0 || flag_disasm) { + jit_disassemble(); + fprintf(stdout, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); + } + if (flag_disasm) + return (0); + return ((*function)(argc, argv)); +} + +static void * +xmalloc(size_t size) +{ + void *pointer = malloc(size); + + if (pointer == NULL) + error("out of memory"); + + return (pointer); +} + +static void * +xrealloc(void *pointer, size_t size) +{ + pointer = realloc(pointer, size); + + if (pointer == NULL) + error("out of memory"); + + return (pointer); +} + +static void * +xcalloc(size_t nmemb, size_t size) +{ + void *pointer = calloc(nmemb, size); + + if (pointer == NULL) + error("out of memory"); + + return (pointer); +} + +static label_t * +new_label(label_kind_t kind, char *name, void *value) +{ + label_t *label; + + label = (label_t *)xmalloc(sizeof(label_t)); + label->kind = kind; + label->name = strdup(name); + label->value = value; + put_hash(labels, (entry_t *)label); + label_offset++; + return (label); +} + +static patch_t * +new_patch(patch_kind_t kind, label_t *label, void *value) +{ + patch_t *patch = (patch_t *)xmalloc(sizeof(patch_t)); + patch->kind = kind; + patch->label = label; + patch->value = value; + patch->next = patches; + patches = patch; + + return (patch); +} + +static int +bcmp_symbols(const void *left, const void *right) +{ + return (strcmp((char *)left, (*(symbol_t **)right)->name)); +} + +static int +qcmp_symbols(const void *left, const void *right) +{ + return (strcmp((*(symbol_t **)left)->name, (*(symbol_t **)right)->name)); +} + +static symbol_t * +new_symbol(char *name) +{ + symbol_t *symbol; + + if ((symbol_offset & 15) == 0) { + if ((symbol_length += 16) == 16) + symbols = (symbol_t **)xmalloc(sizeof(symbol_t *) * + symbol_length); + else + symbols = (symbol_t **)xrealloc(symbols, sizeof(symbol_t *) * + symbol_length); + } + symbol = (symbol_t *)xmalloc(sizeof(symbol_t)); + symbol->name = strdup(name); + symbols[symbol_offset++] = symbol; + qsort(symbols, symbol_offset, sizeof(symbol_t *), qcmp_symbols); + + return (symbol); +} + +static symbol_t * +get_symbol_by_name(char *name) +{ + symbol_t **symbol_pointer; + + symbol_pointer = (symbol_t **)bsearch(name, symbols, symbol_offset, + sizeof(symbol_t *), bcmp_symbols); + + return (symbol_pointer ? *symbol_pointer : NULL); +} + +static hash_t * +new_hash(void) +{ + hash_t *hash; + + hash = (hash_t *)xmalloc(sizeof(hash_t)); + hash->count = 0; + hash->entries = (entry_t **)xcalloc(hash->size = 32, sizeof(void *)); + + return (hash); +} + +static int +hash_string(char *name) +{ + char *ptr; + int key; + + for (key = 0, ptr = name; *ptr; ptr++) + key = (key << (key & 1)) ^ *ptr; + + return (key); +} + +static void +put_hash(hash_t *hash, entry_t *entry) +{ + entry_t *prev, *ptr; + int key = hash_string(entry->name) & (hash->size - 1); + + for (prev = ptr = hash->entries[key]; ptr; prev = ptr, ptr = ptr->next) { + if (strcmp(entry->name, ptr->name) == 0) + error("duplicated entry %s", entry->name); + } + if (prev == NULL) + hash->entries[key] = entry; + else + prev->next = entry; + entry->next = NULL; + ++hash->count; + if (hash->count > hash->size * 0.75) + rehash(hash); +} + +static entry_t * +get_hash(hash_t *hash, char *name) +{ + entry_t *entry; + int key = hash_string(name) & (hash->size - 1); + + for (entry = hash->entries[key]; entry; entry = entry->next) { + if (strcmp(entry->name, name) == 0) + return (entry); + } + return (NULL); +} + +static void +rehash(hash_t *hash) +{ + int i, size, key; + entry_t *entry, *next, **entries; + + entries = (entry_t **)xcalloc(size = hash->size * 2, sizeof(void *)); + for (i = 0; i < hash->size; i++) { + for (entry = hash->entries[i]; entry; entry = next) { + next = entry->next; + key = hash_string(entry->name) & (size - 1); + entry->next = entries[key]; + entries[key] = entry; + } + } + free(hash->entries); + hash->entries = entries; + hash->size = size; +} + +static void +usage(void) +{ + fprintf(stderr, "\ +Usage: %s [jit-assembler-options] file [jit-program-options]\n\ +Options:\n\ + -help Display this information\n\ + -v[0-3] Verbose output level\n\ + -D[=] Preprocessor options\n" + , progname); + exit(1); +} + +int +main(int argc, char *argv[]) +{ + static const char *short_options = "v::"; + static struct option long_options[] = { + { "help", 0, 0, 'h' }, + { 0, 0, 0, 0 } + }; + int offset; + char *endptr; + int opt_index; + int opt_short; + char cmdline[8192]; + + progname = argv[0]; + for (;;) { + if ((opt_short = getopt_long_only(argc, argv, short_options, + long_options, &opt_index)) < 0) + break; + switch (opt_short) { + case 'h': + default: + usage(); + break; + case 'v': + if (optarg) { + flag_verbose = strtol(optarg, &endptr, 10); + if (*endptr || flag_verbose < 0) + usage(); + } + else + flag_verbose = 1; + break; + } + } + + opt_index = optind; + if (opt_index < 0 || opt_index >= argc) + usage(); + if (strcmp(argv[opt_index], "-") == 0) + strcpy(parser.name, ""); + else { + if ((endptr = strrchr(argv[opt_index], '/')) == NULL) + endptr = argv[opt_index]; + else + ++endptr; + strncpy(parser.name, endptr, sizeof(parser.name)); + parser.name[sizeof(parser.name) - 1] = '\0'; + } + opt_short = snprintf(cmdline, sizeof(cmdline), "gcc -E -x c %s", argv[opt_index]); + for (++opt_index; opt_index < argc; opt_index++) { + if (argv[opt_index][0] == '-') + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " %s", argv[opt_index]); + else { + --opt_index; + break; + } + } + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__WORDSIZE=%d", __WORDSIZE); + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__LITTLE_ENDIAN=%d", __LITTLE_ENDIAN); + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__BIG_ENDIAN=%d", __BIG_ENDIAN); + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__BYTE_ORDER=%d", __BYTE_ORDER); +#if defined(__i386__) + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__i386__=1"); +#endif +#if defined(__x86_64__) + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__x86_64__=1"); +#endif +#if defined(__mips__) + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__mips__=1"); +#endif +#if defined(__arm__) + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__arm__=1"); +#endif + if ((parser.fp = popen(cmdline, "r")) == NULL) + error("cannot execute %s", cmdline); + + parser.line = 1; + parser.string = (char *)xmalloc(parser.length = 4096); + + labels = new_hash(); + +#if defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) + /* double precision 0x200 + * round nearest 0x000 + * invalid operation mask 0x001 + * denormalized operand mask 0x002 + * zero divide mask 0x004 + * precision (inexact) mask 0x020 + */ + { + fpu_control_t fpu_control = 0x027f; + _FPU_SETCW(fpu_control); + } +#endif + + init_jit(); + _jit = jit_new_state(); + + instrs = new_hash(); + for (offset = 0; + offset < (int)(sizeof(instr_vector) / sizeof(instr_vector[0])); + offset++) + put_hash(instrs, (entry_t *)(instr_vector + offset)); + + labels = new_hash(); + + parse(); + pclose(parser.fp); + parser.fp = NULL; + + for (opt_short = 0; opt_index < argc; opt_short++, opt_index++) + argv[opt_short] = argv[opt_index]; + argv[opt_short] = NULL; + argc = opt_short; + execute(argc, argv); + + finish_jit(); + + return (0); +} diff --git a/tests/run-test b/check/run-test similarity index 63% rename from tests/run-test rename to check/run-test index 3f588ccce..931abc7bc 100755 --- a/tests/run-test +++ b/check/run-test @@ -1,10 +1,6 @@ #! /bin/sh -./$1 | tr -d \\r > $1.log -if test $? = 77; then - exit 77 -fi - +./lightning $1.tst | tr -d \\r > $1.log if cmp -s $srcdir/$1.ok $1.log; then rm $1.log else diff --git a/configure.ac b/configure.ac index ce7e0c0f4..1d8dd3ef2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,70 +1,76 @@ -dnl Hey Emacs, I want this in -*- autoconf -*- mode, please. - -dnl Copyright 2000, 2001, 2002 Free Software Foundation, Inc. -dnl Please see COPYING for a description your rights and responsibilities -dnl with this software. -dnl Process this file with autoconf to produce a configure script. - -dnl ----------------------------- HOST SYSTEM ----------------------------------- - -AC_PREREQ(2.54) -AC_INIT([GNU lightning], 1.2c, bonzini@gnu.org, lightning) -AC_CONFIG_AUX_DIR(m4) -AC_CONFIG_MACRO_DIR(m4) -AC_CONFIG_SRCDIR([lightning.h]) +dnl +dnl Copyright 2000, 2001, 2002, 2012 Free Software Foundation, Inc. +dnl +dnl This is free software; you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation; either version 3 of the License, or +dnl (at your option) any later version. +dnl +dnl This software is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU General Public License for more details. +dnl +AC_PREREQ(2.57) +AC_INIT([GNU lightning], 2.0, pcpa@gnu.org, lightning) AC_CANONICAL_TARGET -AC_CONFIG_HEADERS(config.h) -AM_INIT_AUTOMAKE +AC_CONFIG_SRCDIR([Makefile.am]) +AM_INIT_AUTOMAKE([dist-bzip2]) +AC_CONFIG_MACRO_DIR(m4) + +AM_CONFIG_HEADER(config.h) AC_PROG_CC -AC_PROG_CPP -AC_PROG_LN_S -AC_PROG_RANLIB AC_PROG_INSTALL -AC_PROG_MAKE_SET -AC_PATH_PROG(INSTALL_INFO, install-info, :, $PATH:/sbin) -AC_EXEEXT +AC_PROG_LIBTOOL -BACKENDS="LIGHTNING_BACKENDS" -AC_SUBST(BACKENDS) +AC_CHECK_LIB(gmp, __gmpz_init, , + [AC_MSG_ERROR([GNU MP not found, see http://gmplib.org/])]) -case "$target_cpu" in - i?86) LIGHTNING_TARGET=LIGHTNING_I386 ;; - x86_64) LIGHTNING_TARGET=LIGHTNING_X86_64 ;; - sparc*) LIGHTNING_TARGET=LIGHTNING_SPARC ;; - powerpc) LIGHTNING_TARGET=LIGHTNING_PPC ;; - *) ;; -esac -LIGHTNING_CONFIGURE_LINKS( - [AC_DEFINE_UNQUOTED(LIGHTNING_TARGET, [AS_TR_CPP([$LIGHTNING_TARGET])], - [Used to pick the appropriate disassembler, for debugging])], - [AC_MSG_ERROR([cpu $target_cpu not supported])]) - -AC_SUBST(cpu) -AM_CONDITIONAL(LIGHTNING_MAIN, :) - -AM_CONDITIONAL(REGRESSION_TESTING, test "$host_cpu" = "$target_cpu") - -dnl ---------------------------- COMMAND LINE --------------------------------- - -AC_ARG_ENABLE( assertions, -[ --enable-assertions perform internal consistency checks], -, enable_assertions=no) - -if test "$enable_assertions" != no; then - AC_DEFINE(_ASM_SAFETY, 1, [Define to enable assertions]) +AC_ARG_ENABLE(disassembler, + AS_HELP_STRING([--enable-disassembler], + [Enable jit disassembler using binutils]), + [DISASSEMBLER=$enableval], [DISASSEMBLER=auto]) +if test "x$DISASSEMBLER" != "xno"; then + # FIXME need to check for libiberty first or will fail to link + AC_CHECK_LIB(iberty, htab_try_create, , + [HAVE_IBERTY="no"]) + AC_CHECK_LIB(bfd, bfd_init, , + [HAVE_BFD="no"]) + AC_CHECK_LIB(opcodes, init_disassemble_info, , + [HAVE_OPCODES="no"]) + if test "x$HAVE_IBERTY" = "xno" -o \ + "x$HAVE_BFD" = "xno" -o \ + "x$HAVE_OPCODES" = "xno"; then + if test "x$DISASSEMBLER" != "xauto"; then + AC_MSG_ERROR([binutils not found, see http://www.gnu.org/software/binutils/]) + else + AC_MSG_WARN([binutils not found, see http://www.gnu.org/software/binutils/]) + DISASSEMBLER="no" + fi + fi +fi +AM_CONDITIONAL(with_disassembler, [test "x$DISASSEMBLER" != "xno"]) +if test "x$DISASSEMBLER" != "xno"; then + LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DDISASSEMBLER=1" fi -dnl --------------------------- PRODUCE OUTPUT -------------------------------- +cpu= +case "$target_cpu" in + i?x86|x86_64) cpu=x86 ;; + *) ;; +esac +if test x$cpu = x; then + AC_MSG_ERROR([cpu $target_cpu not supported]) +fi +AM_CONDITIONAL(cpu_x86, [test cpu-$cpu = cpu-x86]) -AC_CONFIG_FILES(Makefile doc/Makefile tests/Makefile - lightning/Makefile) +AC_SUBST([LIGHTNING_CFLAGS]) -AC_OUTPUT - -# A small sanity check -echo "#include " > confdefs.h # dummy input file -CPPFLAGS="$CPPFLAGS -I. -I$srcdir -I$srcdir/lightning/$cpu" -AC_TRY_COMPILE([#include "lightning.h"], , , - AC_MSG_WARN(the compiler that was found could not compile GNU lightning)) +AC_OUTPUT([Makefile + doc/Makefile + include/Makefile + include/lightning/Makefile + lib/Makefile + check/Makefile]) diff --git a/include/Makefile.am b/include/Makefile.am new file mode 100644 index 000000000..1405eb177 --- /dev/null +++ b/include/Makefile.am @@ -0,0 +1,17 @@ +# +# Copyright 2000, 2001, 2002, 2012 Free Software Foundation, Inc. +# +# This is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +include_HEADERS = lightning.h +SUBDIRS = \ + lightning diff --git a/include/lightning.h b/include/lightning.h new file mode 100644 index 000000000..6a0ca56c5 --- /dev/null +++ b/include/lightning.h @@ -0,0 +1,852 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _lightning_h +#define _lightning_h + +#include +#include +#include +#include + +#ifndef __WORDSIZE +# define __WORDSIZE WORDSIZE +#endif +#ifndef __BYTE_ORDER +# define __BYTE_ORDER BYTE_ORDER +#endif +#ifndef __LITTLE_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +#endif +#ifndef __BIG_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +#endif + +typedef signed char jit_int8_t; +typedef unsigned char jit_uint8_t; +typedef signed short jit_int16_t; +typedef unsigned short jit_uint16_t; +typedef signed int jit_int32_t; +typedef unsigned int jit_uint32_t; +#if __WORDSIZE == 32 +typedef signed long long jit_int64_t; +typedef unsigned long long jit_uint64_t; +typedef jit_int32_t jit_word_t; +typedef jit_uint32_t jit_uword_t; +#else +typedef signed long jit_int64_t; +typedef unsigned long jit_uint64_t; +typedef jit_int64_t jit_word_t; +typedef jit_uint64_t jit_uword_t; +#endif +typedef float jit_float32_t; +typedef double jit_float64_t; +typedef void* jit_pointer_t; +typedef jit_int32_t jit_bool_t; +typedef jit_int32_t jit_gpr_t; +typedef jit_int32_t jit_fpr_t; + +#if defined(__i386__) || defined(__x86_64__) +# include +#elif defined(__mips__) +# include +#elif defined(__arm__) +# include +#elif defined(__ppc__) +# include +#endif + +#define jit_flag_node 0x00000001 /* patch node not absolute */ +#define jit_flag_patch 0x00000002 /* jump already patched */ +#define jit_flag_data 0x00000004 /* data in the constant pool */ +#define jit_flag_use 0x00000008 /* do not remove marker label */ +#define jit_flag_head 0x00100000 /* label reached by normal flow */ + +#define JIT_R(index) jit_r(index) +#define JIT_V(index) jit_v(index) +#define JIT_F(index) jit_f(index) +#define JIT_R_NUM jit_r_num() +#define JIT_V_NUM jit_v_num() +#define JIT_F_NUM jit_f_num() + +#define jit_class_chk 0x02000000 /* just checking */ +#define jit_class_arg 0x08000000 /* argument register */ +#define jit_class_sav 0x10000000 /* callee save */ +#define jit_class_gpr 0x20000000 /* general purpose */ +#define jit_class_fpr 0x40000000 /* float */ +#define jit_class(reg) ((reg) & 0xffff0000) +#define jit_regno(reg) ((reg) & 0x00007fff) + +#define jit_call_default 0 +/* assume only varags functions called are printf like, that is, + * without a declared float/double argument */ +/* FIXME currently no way to create a varargs (or non varargs) jit function + * if calling sequence changes for float/double arguments */ +#define jit_call_varargs 1 + +typedef struct jit_node jit_node_t; +typedef struct jit_state jit_state_t; + +typedef enum { +#define jit_data(u,v) _jit_data(_jit,u,v) + jit_code_data, + jit_code_save, jit_code_load, +#define jit_note(u) jit_new_node_ww(jit_code_note,0,(jit_word_t)u) +#define jit_label() _jit_label(_jit) +#define jit_forward() _jit_forward(_jit) +#define jit_link(u) _jit_link(_jit,u) + jit_code_note, jit_code_label, + +#define jit_prolog() _jit_prolog(_jit) + jit_code_prolog, + +#define jit_allocai(u) _jit_allocai(_jit,u) + +#define jit_arg() _jit_arg(_jit) +#define jit_getarg_c(u,v) _jit_getarg_c(_jit,u,v) +#define jit_getarg_uc(u,v) _jit_getarg_uc(_jit,u,v) +#define jit_getarg_s(u,v) _jit_getarg_s(_jit,u,v) +#define jit_getarg_us(u,v) _jit_getarg_us(_jit,u,v) +#define jit_getarg_i(u,v) _jit_getarg_i(_jit,u,v) + /* >> 64 bit */ +#define jit_getarg_ui(u,v) _jit_getarg_ui(_jit,u,v) +#define jit_getarg_l(u,v) _jit_getarg_l(_jit,u,v) + /* << 64 bit */ +#if __WORDSIZE == 32 +# define jit_getarg(u,v) jit_getarg_i(u,v) +#else +# define jit_getarg(u,v) jit_getarg_l(u,v) +#endif + +#define jit_addr(u,v,w) jit_new_node_www(jit_code_addr,u,v,w) +#define jit_addi(u,v,w) jit_new_node_www(jit_code_addi,u,v,w) + jit_code_addr, jit_code_addi, +#define jit_addxr(u,v,w) jit_new_node_www(jit_code_addxr,u,v,w) +#define jit_addxi(u,v,w) jit_new_node_www(jit_code_addxi,u,v,w) + jit_code_addxr, jit_code_addxi, +#define jit_addcr(u,v,w) jit_new_node_www(jit_code_addcr,u,v,w) +#define jit_addci(u,v,w) jit_new_node_www(jit_code_addci,u,v,w) + jit_code_addcr, jit_code_addci, +#define jit_subr(u,v,w) jit_new_node_www(jit_code_subr,u,v,w) +#define jit_subi(u,v,w) jit_new_node_www(jit_code_subi,u,v,w) + jit_code_subr, jit_code_subi, +#define jit_subxr(u,v,w) jit_new_node_www(jit_code_subxr,u,v,w) +#define jit_subxi(u,v,w) jit_new_node_www(jit_code_subxi,u,v,w) + jit_code_subxr, jit_code_subxi, +#define jit_subcr(u,v,w) jit_new_node_www(jit_code_subcr,u,v,w) +#define jit_subci(u,v,w) jit_new_node_www(jit_code_subci,u,v,w) + jit_code_subcr, jit_code_subci, + +#define jit_mulr(u,v,w) jit_new_node_www(jit_code_mulr,u,v,w) +#define jit_muli(u,v,w) jit_new_node_www(jit_code_muli,u,v,w) + jit_code_mulr, jit_code_muli, +#define jit_divr(u,v,w) jit_new_node_www(jit_code_divr,u,v,w) +#define jit_divi(u,v,w) jit_new_node_www(jit_code_divi,u,v,w) + jit_code_divr, jit_code_divi, +#define jit_divr_u(u,v,w) jit_new_node_www(jit_code_divr_u,u,v,w) +#define jit_divi_u(u,v,w) jit_new_node_www(jit_code_divi_u,u,v,w) + jit_code_divr_u, jit_code_divi_u, +#define jit_remr(u,v,w) jit_new_node_www(jit_code_remr,u,v,w) +#define jit_remi(u,v,w) jit_new_node_www(jit_code_remi,u,v,w) + jit_code_remr, jit_code_remi, +#define jit_remr_u(u,v,w) jit_new_node_www(jit_code_remr_u,u,v,w) +#define jit_remi_u(u,v,w) jit_new_node_www(jit_code_remi_u,u,v,w) + jit_code_remr_u, jit_code_remi_u, + +#define jit_andr(u,v,w) jit_new_node_www(jit_code_andr,u,v,w) +#define jit_andi(u,v,w) jit_new_node_www(jit_code_andi,u,v,w) + jit_code_andr, jit_code_andi, +#define jit_orr(u,v,w) jit_new_node_www(jit_code_orr,u,v,w) +#define jit_ori(u,v,w) jit_new_node_www(jit_code_ori,u,v,w) + jit_code_orr, jit_code_ori, +#define jit_xorr(u,v,w) jit_new_node_www(jit_code_xorr,u,v,w) +#define jit_xori(u,v,w) jit_new_node_www(jit_code_xori,u,v,w) + jit_code_xorr, jit_code_xori, + +#define jit_lshr(u,v,w) jit_new_node_www(jit_code_lshr,u,v,w) +#define jit_lshi(u,v,w) jit_new_node_www(jit_code_lshi,u,v,w) + jit_code_lshr, jit_code_lshi, +#define jit_rshr(u,v,w) jit_new_node_www(jit_code_rshr,u,v,w) +#define jit_rshi(u,v,w) jit_new_node_www(jit_code_rshi,u,v,w) + jit_code_rshr, jit_code_rshi, +#define jit_rshr_u(u,v,w) jit_new_node_www(jit_code_rshr_u,u,v,w) +#define jit_rshi_u(u,v,w) jit_new_node_www(jit_code_rshi_u,u,v,w) + jit_code_rshr_u, jit_code_rshi_u, + +#define jit_negr(u,v) jit_new_node_ww(jit_code_negr,u,v) +#define jit_comr(u,v) jit_new_node_ww(jit_code_comr,u,v) + jit_code_negr, jit_code_comr, + +#define jit_ltr(u,v,w) jit_new_node_www(jit_code_ltr,u,v,w) +#define jit_lti(u,v,w) jit_new_node_www(jit_code_lti,u,v,w) + jit_code_ltr, jit_code_lti, +#define jit_ltr_u(u,v,w) jit_new_node_www(jit_code_ltr_u,u,v,w) +#define jit_lti_u(u,v,w) jit_new_node_www(jit_code_lti_u,u,v,w) + jit_code_ltr_u, jit_code_lti_u, +#define jit_ler(u,v,w) jit_new_node_www(jit_code_ler,u,v,w) +#define jit_lei(u,v,w) jit_new_node_www(jit_code_lei,u,v,w) + jit_code_ler, jit_code_lei, +#define jit_ler_u(u,v,w) jit_new_node_www(jit_code_ler_u,u,v,w) +#define jit_lei_u(u,v,w) jit_new_node_www(jit_code_lei_u,u,v,w) + jit_code_ler_u, jit_code_lei_u, +#define jit_eqr(u,v,w) jit_new_node_www(jit_code_eqr,u,v,w) +#define jit_eqi(u,v,w) jit_new_node_www(jit_code_eqi,u,v,w) + jit_code_eqr, jit_code_eqi, +#define jit_ger(u,v,w) jit_new_node_www(jit_code_ger,u,v,w) +#define jit_gei(u,v,w) jit_new_node_www(jit_code_gei,u,v,w) + jit_code_ger, jit_code_gei, +#define jit_ger_u(u,v,w) jit_new_node_www(jit_code_ger_u,u,v,w) +#define jit_gei_u(u,v,w) jit_new_node_www(jit_code_gei_u,u,v,w) + jit_code_ger_u, jit_code_gei_u, +#define jit_gtr(u,v,w) jit_new_node_www(jit_code_gtr,u,v,w) +#define jit_gti(u,v,w) jit_new_node_www(jit_code_gti,u,v,w) + jit_code_gtr, jit_code_gti, +#define jit_gtr_u(u,v,w) jit_new_node_www(jit_code_gtr_u,u,v,w) +#define jit_gti_u(u,v,w) jit_new_node_www(jit_code_gti_u,u,v,w) + jit_code_gtr_u, jit_code_gti_u, +#define jit_ner(u,v,w) jit_new_node_www(jit_code_ner,u,v,w) +#define jit_nei(u,v,w) jit_new_node_www(jit_code_nei,u,v,w) + jit_code_ner, jit_code_nei, + +#define jit_movr(u,v) jit_new_node_ww(jit_code_movr,u,v) +#define jit_movi(u,v) jit_new_node_ww(jit_code_movi,u,v) + jit_code_movr, jit_code_movi, +#define jit_extr_c(u,v) jit_new_node_ww(jit_code_extr_c,u,v) +#define jit_extr_uc(u,v) jit_new_node_ww(jit_code_extr_uc,u,v) + jit_code_extr_c, jit_code_extr_uc, +#define jit_extr_s(u,v) jit_new_node_ww(jit_code_extr_s,u,v) +#define jit_extr_us(u,v) jit_new_node_ww(jit_code_extr_us,u,v) + jit_code_extr_s, jit_code_extr_us, + /* >> 64 bit */ +#define jit_extr_i(u,v) jit_new_node_ww(jit_code_extr_i,u,v) +#define jit_extr_ui(u,v) jit_new_node_ww(jit_code_extr_ui,u,v) + jit_code_extr_i, jit_code_extr_ui, + /* << 64 bit */ +#define jit_htonr(u,v) jit_new_node_ww(jit_code_htonr,u,v) +#define jit_ntohr(u,v) jit_new_node_ww(jit_code_htonr,u,v) + jit_code_htonr, + +#define jit_ldr_c(u,v) jit_new_node_ww(jit_code_ldr_c,u,v) +#define jit_ldi_c(u,v) jit_new_node_ww(jit_code_ldi_c,u,v) + jit_code_ldr_c, jit_code_ldi_c, +#define jit_ldr_uc(u,v) jit_new_node_ww(jit_code_ldr_uc,u,v) +#define jit_ldi_uc(u,v) jit_new_node_ww(jit_code_ldi_uc,u,v) + jit_code_ldr_uc, jit_code_ldi_uc, +#define jit_ldr_s(u,v) jit_new_node_ww(jit_code_ldr_s,u,v) +#define jit_ldi_s(u,v) jit_new_node_ww(jit_code_ldi_s,u,v) + jit_code_ldr_s, jit_code_ldi_s, +#define jit_ldr_us(u,v) jit_new_node_ww(jit_code_ldr_us,u,v) +#define jit_ldi_us(u,v) jit_new_node_ww(jit_code_ldi_us,u,v) + jit_code_ldr_us, jit_code_ldi_us, +#define jit_ldr_i(u,v) jit_new_node_ww(jit_code_ldr_i,u,v) +#define jit_ldi_i(u,v) jit_new_node_ww(jit_code_ldi_i,u,v) + jit_code_ldr_i, jit_code_ldi_i, +#if __WORDSIZE == 32 +# define jit_ldr(u,v) jit_ldr_i(u,v) +# define jit_ldi(u,v) jit_ldi_i(u,v) +#else +# define jit_ldr(u,v) jit_ldr_l(u,v) +# define jit_ldi(u,v) jit_ldi_l(u,v) +#endif + /* >> 64 bit */ +#define jit_ldr_ui(u,v) jit_new_node_ww(jit_code_ldr_ui,u,v) +#define jit_ldi_ui(u,v) jit_new_node_ww(jit_code_ldi_ui,u,v) + jit_code_ldr_ui, jit_code_ldi_ui, +#define jit_ldr_l(u,v) jit_new_node_ww(jit_code_ldr_l,u,v) +#define jit_ldi_l(u,v) jit_new_node_ww(jit_code_ldi_l,u,v) + jit_code_ldr_l, jit_code_ldi_l, + /* << 64 bit */ + +#define jit_ldxr_c(u,v,w) jit_new_node_www(jit_code_ldxr_c,u,v,w) +#define jit_ldxi_c(u,v,w) jit_new_node_www(jit_code_ldxi_c,u,v,w) + jit_code_ldxr_c, jit_code_ldxi_c, +#define jit_ldxr_uc(u,v,w) jit_new_node_www(jit_code_ldxr_uc,u,v,w) +#define jit_ldxi_uc(u,v,w) jit_new_node_www(jit_code_ldxi_uc,u,v,w) + jit_code_ldxr_uc, jit_code_ldxi_uc, +#define jit_ldxr_s(u,v,w) jit_new_node_www(jit_code_ldxr_s,u,v,w) +#define jit_ldxi_s(u,v,w) jit_new_node_www(jit_code_ldxi_s,u,v,w) + jit_code_ldxr_s, jit_code_ldxi_s, +#define jit_ldxr_us(u,v,w) jit_new_node_www(jit_code_ldxr_us,u,v,w) +#define jit_ldxi_us(u,v,w) jit_new_node_www(jit_code_ldxi_us,u,v,w) + jit_code_ldxr_us, jit_code_ldxi_us, +#define jit_ldxr_i(u,v,w) jit_new_node_www(jit_code_ldxr_i,u,v,w) +#define jit_ldxi_i(u,v,w) jit_new_node_www(jit_code_ldxi_i,u,v,w) + jit_code_ldxr_i, jit_code_ldxi_i, +#if __WORDSIZE == 32 +# define jit_ldxr(u,v,w) jit_ldxr_i(u,v,w) +# define jit_ldxi(u,v,w) jit_ldxi_i(u,v,w) +#else +# define jit_ldxr_ui(u,v,w) jit_new_node_www(jit_code_ldxr_ui,u,v,w) +# define jit_ldxi_ui(u,v,w) jit_new_node_www(jit_code_ldxi_ui,u,v,w) +# define jit_ldxr_l(u,v,w) jit_new_node_www(jit_code_ldxr_l,u,v,w) +# define jit_ldxi_l(u,v,w) jit_new_node_www(jit_code_ldxi_l,u,v,w) +# define jit_ldxr(u,v,w) jit_ldxr_l(u,v,w) +# define jit_ldxi(u,v,w) jit_ldxi_l(u,v,w) +#endif + jit_code_ldxr_ui, jit_code_ldxi_ui, + jit_code_ldxr_l, jit_code_ldxi_l, + +#define jit_str_c(u,v) jit_new_node_ww(jit_code_str_c,u,v) +#define jit_sti_c(u,v) jit_new_node_ww(jit_code_sti_c,u,v) + jit_code_str_c, jit_code_sti_c, +#define jit_str_s(u,v) jit_new_node_ww(jit_code_str_s,u,v) +#define jit_sti_s(u,v) jit_new_node_ww(jit_code_sti_s,u,v) + jit_code_str_s, jit_code_sti_s, +#define jit_str_i(u,v) jit_new_node_ww(jit_code_str_i,u,v) +#define jit_sti_i(u,v) jit_new_node_ww(jit_code_sti_i,u,v) + jit_code_str_i, jit_code_sti_i, +#if __WORDSIZE == 32 +# define jit_str(u,v) jit_str_i(u,v) +# define jit_sti(u,v) jit_sti_i(u,v) +#else +#define jit_str(u,v) jit_str_l(u,v) +#define jit_sti(u,v) jit_sti_l(u,v) +#endif + /* >> 64 bit */ +#define jit_str_l(u,v) jit_new_node_ww(jit_code_str_l,u,v) +#define jit_sti_l(u,v) jit_new_node_ww(jit_code_sti_l,u,v) + jit_code_str_l, jit_code_sti_l, + /* << 64 bit */ + +#define jit_stxr_c(u,v,w) jit_new_node_www(jit_code_stxr_c,u,v,w) +#define jit_stxi_c(u,v,w) jit_new_node_www(jit_code_stxi_c,u,v,w) + jit_code_stxr_c, jit_code_stxi_c, +#define jit_stxr_s(u,v,w) jit_new_node_www(jit_code_stxr_s,u,v,w) +#define jit_stxi_s(u,v,w) jit_new_node_www(jit_code_stxi_s,u,v,w) + jit_code_stxr_s, jit_code_stxi_s, +#define jit_stxr_i(u,v,w) jit_new_node_www(jit_code_stxr_i,u,v,w) +#define jit_stxi_i(u,v,w) jit_new_node_www(jit_code_stxi_i,u,v,w) + jit_code_stxr_i, jit_code_stxi_i, +#if __WORDSIZE == 32 +# define jit_stxr(u,v,w) jit_stxr_i(u,v,w) +# define jit_stxi(u,v,w) jit_stxi_i(u,v,w) +#else +# define jit_stxr_l(u,v,w) jit_new_node_www(jit_code_stxr_l,u,v,w) +# define jit_stxi_l(u,v,w) jit_new_node_www(jit_code_stxi_l,u,v,w) +#endif + /* >> 64 bit */ +#define jit_stxr(u,v,w) jit_stxr_l(u,v,w) +#define jit_stxi(u,v,w) jit_stxi_l(u,v,w) + jit_code_stxr_l, jit_code_stxi_l, + /* << 64 bit */ + +#define jit_bltr(v,w) jit_new_node_pww(jit_code_bltr,NULL,v,w) +#define jit_blti(v,w) jit_new_node_pww(jit_code_blti,NULL,v,w) + jit_code_bltr, jit_code_blti, +#define jit_bltr_u(v,w) jit_new_node_pww(jit_code_bltr_u,NULL,v,w) +#define jit_blti_u(v,w) jit_new_node_pww(jit_code_blti_u,NULL,v,w) + jit_code_bltr_u, jit_code_blti_u, +#define jit_bler(v,w) jit_new_node_pww(jit_code_bler,NULL,v,w) +#define jit_blei(v,w) jit_new_node_pww(jit_code_blei,NULL,v,w) + jit_code_bler, jit_code_blei, +#define jit_bler_u(v,w) jit_new_node_pww(jit_code_bler_u,NULL,v,w) +#define jit_blei_u(v,w) jit_new_node_pww(jit_code_blei_u,NULL,v,w) + jit_code_bler_u, jit_code_blei_u, +#define jit_beqr(v,w) jit_new_node_pww(jit_code_beqr,NULL,v,w) +#define jit_beqi(v,w) jit_new_node_pww(jit_code_beqi,NULL,v,w) + jit_code_beqr, jit_code_beqi, +#define jit_bger(v,w) jit_new_node_pww(jit_code_bger,NULL,v,w) +#define jit_bgei(v,w) jit_new_node_pww(jit_code_bgei,NULL,v,w) + jit_code_bger, jit_code_bgei, +#define jit_bger_u(v,w) jit_new_node_pww(jit_code_bger_u,NULL,v,w) +#define jit_bgei_u(v,w) jit_new_node_pww(jit_code_bgei_u,NULL,v,w) + jit_code_bger_u, jit_code_bgei_u, +#define jit_bgtr(v,w) jit_new_node_pww(jit_code_bgtr,NULL,v,w) +#define jit_bgti(v,w) jit_new_node_pww(jit_code_bgti,NULL,v,w) + jit_code_bgtr, jit_code_bgti, +#define jit_bgtr_u(v,w) jit_new_node_pww(jit_code_bgtr_u,NULL,v,w) +#define jit_bgti_u(v,w) jit_new_node_pww(jit_code_bgti_u,NULL,v,w) + jit_code_bgtr_u, jit_code_bgti_u, +#define jit_bner(v,w) jit_new_node_pww(jit_code_bner,NULL,v,w) +#define jit_bnei(v,w) jit_new_node_pww(jit_code_bnei,NULL,v,w) + jit_code_bner, jit_code_bnei, + +#define jit_bmsr(v,w) jit_new_node_pww(jit_code_bmsr,NULL,v,w) +#define jit_bmsi(v,w) jit_new_node_pww(jit_code_bmsi,NULL,v,w) + jit_code_bmsr, jit_code_bmsi, +#define jit_bmcr(v,w) jit_new_node_pww(jit_code_bmcr,NULL,v,w) +#define jit_bmci(v,w) jit_new_node_pww(jit_code_bmci,NULL,v,w) + jit_code_bmcr, jit_code_bmci, + +#define jit_boaddr(v,w) jit_new_node_pww(jit_code_boaddr,NULL,v,w) +#define jit_boaddi(v,w) jit_new_node_pww(jit_code_boaddi,NULL,v,w) + jit_code_boaddr, jit_code_boaddi, +#define jit_boaddr_u(v,w) jit_new_node_pww(jit_code_boaddr_u,NULL,v,w) +#define jit_boaddi_u(v,w) jit_new_node_pww(jit_code_boaddi_u,NULL,v,w) + jit_code_boaddr_u, jit_code_boaddi_u, +#define jit_bxaddr(v,w) jit_new_node_pww(jit_code_bxaddr,NULL,v,w) +#define jit_bxaddi(v,w) jit_new_node_pww(jit_code_bxaddi,NULL,v,w) + jit_code_bxaddr, jit_code_bxaddi, +#define jit_bxaddr_u(v,w) jit_new_node_pww(jit_code_bxaddr_u,NULL,v,w) +#define jit_bxaddi_u(v,w) jit_new_node_pww(jit_code_bxaddi_u,NULL,v,w) + jit_code_bxaddr_u, jit_code_bxaddi_u, +#define jit_bosubr(v,w) jit_new_node_pww(jit_code_bosubr,NULL,v,w) +#define jit_bosubi(v,w) jit_new_node_pww(jit_code_bosubi,NULL,v,w) + jit_code_bosubr, jit_code_bosubi, +#define jit_bosubr_u(v,w) jit_new_node_pww(jit_code_bosubr_u,NULL,v,w) +#define jit_bosubi_u(v,w) jit_new_node_pww(jit_code_bosubi_u,NULL,v,w) + jit_code_bosubr_u, jit_code_bosubi_u, +#define jit_bxsubr(v,w) jit_new_node_pww(jit_code_bxsubr,NULL,v,w) +#define jit_bxsubi(v,w) jit_new_node_pww(jit_code_bxsubi,NULL,v,w) + jit_code_bxsubr, jit_code_bxsubi, +#define jit_bxsubr_u(v,w) jit_new_node_pww(jit_code_bxsubr_u,NULL,v,w) +#define jit_bxsubi_u(v,w) jit_new_node_pww(jit_code_bxsubi_u,NULL,v,w) + jit_code_bxsubr_u, jit_code_bxsubi_u, + +#define jit_jmpr(u) jit_new_node_w(jit_code_jmpr,u) +#define jit_jmpi() jit_new_node_p(jit_code_jmpi,NULL) + jit_code_jmpr, jit_code_jmpi, +#define jit_callr(u) jit_new_node_w(jit_code_callr,u) +#define jit_calli(u) jit_new_node_p(jit_code_calli,u) + jit_code_callr, jit_code_calli, + +#define jit_prepare(u) _jit_prepare(_jit,u) +#define jit_pushargr(u) _jit_pushargr(_jit,u) +#define jit_pushargi(u) _jit_pushargi(_jit,u) +#define jit_finishr(u) _jit_finishr(_jit,u) +#define jit_finishi(u) _jit_finishi(_jit,u) +#define jit_ret() _jit_ret(_jit) +#define jit_retr(u) _jit_retr(_jit,u) +#define jit_reti(u) _jit_reti(_jit,u) +#define jit_retval_c(u) _jit_retval_c(_jit,u) +#define jit_retval_uc(u) _jit_retval_uc(_jit,u) +#define jit_retval_s(u) _jit_retval_s(_jit,u) +#define jit_retval_us(u) _jit_retval_us(_jit,u) +#define jit_retval_i(u) _jit_retval_i(_jit,u) + /* >> 64 bit */ +#define jit_retval_ui(u) _jit_retval_ui(_jit,u) +#define jit_retval_l(u) _jit_retval_l(_jit,u) + /* << 64 bit */ +#if __WORDSIZE == 32 +# define jit_retval(u) jit_retval_i(u) +#else +# define jit_retval(u) jit_retval_l(u) +#endif + /* Usually should not need to call directly, but useful if need + * to get a label just before a jit_prolog() call */ +#define jit_epilog() _jit_epilog(_jit) + jit_code_epilog, + +#define jit_arg_f() _jit_arg_f(_jit) +#define jit_getarg_f(u,v) _jit_getarg_f(_jit,u,v) + +#define jit_addr_f(u,v,w) jit_new_node_www(jit_code_addr_f,u,v,w) +#define jit_addi_f(u,v,w) jit_new_node_wwf(jit_code_addi_f,u,v,w) + jit_code_addr_f, jit_code_addi_f, +#define jit_subr_f(u,v,w) jit_new_node_www(jit_code_subr_f,u,v,w) +#define jit_subi_f(u,v,w) jit_new_node_wwf(jit_code_subi_f,u,v,w) + jit_code_subr_f, jit_code_subi_f, +#define jit_mulr_f(u,v,w) jit_new_node_www(jit_code_mulr_f,u,v,w) +#define jit_muli_f(u,v,w) jit_new_node_wwf(jit_code_muli_f,u,v,w) + jit_code_mulr_f, jit_code_muli_f, +#define jit_divr_f(u,v,w) jit_new_node_www(jit_code_divr_f,u,v,w) +#define jit_divi_f(u,v,w) jit_new_node_wwf(jit_code_divi_f,u,v,w) + jit_code_divr_f, jit_code_divi_f, +#define jit_negr_f(u,v) jit_new_node_ww(jit_code_negr_f,u,v) +#define jit_absr_f(u,v) jit_new_node_ww(jit_code_absr_f,u,v) +#define jit_sqrtr_f(u,v) jit_new_node_ww(jit_code_sqrtr_f,u,v) + jit_code_negr_f, jit_code_absr_f, jit_code_sqrtr_f, + +#define jit_ltr_f(u,v,w) jit_new_node_www(jit_code_ltr_f,u,v,w) +#define jit_lti_f(u,v,w) jit_new_node_wwf(jit_code_lti_f,u,v,w) + jit_code_ltr_f, jit_code_lti_f, +#define jit_ler_f(u,v,w) jit_new_node_www(jit_code_ler_f,u,v,w) +#define jit_lei_f(u,v,w) jit_new_node_wwf(jit_code_lei_f,u,v,w) + jit_code_ler_f, jit_code_lei_f, +#define jit_eqr_f(u,v,w) jit_new_node_www(jit_code_eqr_f,u,v,w) +#define jit_eqi_f(u,v,w) jit_new_node_wwf(jit_code_eqi_f,u,v,w) + jit_code_eqr_f, jit_code_eqi_f, +#define jit_ger_f(u,v,w) jit_new_node_www(jit_code_ger_f,u,v,w) +#define jit_gei_f(u,v,w) jit_new_node_wwf(jit_code_gei_f,u,v,w) + jit_code_ger_f, jit_code_gei_f, +#define jit_gtr_f(u,v,w) jit_new_node_www(jit_code_gtr_f,u,v,w) +#define jit_gti_f(u,v,w) jit_new_node_wwf(jit_code_gti_f,u,v,w) + jit_code_gtr_f, jit_code_gti_f, +#define jit_ner_f(u,v,w) jit_new_node_www(jit_code_ner_f,u,v,w) +#define jit_nei_f(u,v,w) jit_new_node_wwf(jit_code_nei_f,u,v,w) + jit_code_ner_f, jit_code_nei_f, +#define jit_unltr_f(u,v,w) jit_new_node_www(jit_code_unltr_f,u,v,w) +#define jit_unlti_f(u,v,w) jit_new_node_wwf(jit_code_unlti_f,u,v,w) + jit_code_unltr_f, jit_code_unlti_f, +#define jit_unler_f(u,v,w) jit_new_node_www(jit_code_unler_f,u,v,w) +#define jit_unlei_f(u,v,w) jit_new_node_wwf(jit_code_unlei_f,u,v,w) + jit_code_unler_f, jit_code_unlei_f, +#define jit_uneqr_f(u,v,w) jit_new_node_www(jit_code_uneqr_f,u,v,w) +#define jit_uneqi_f(u,v,w) jit_new_node_wwf(jit_code_uneqi_f,u,v,w) + jit_code_uneqr_f, jit_code_uneqi_f, +#define jit_unger_f(u,v,w) jit_new_node_www(jit_code_unger_f,u,v,w) +#define jit_ungei_f(u,v,w) jit_new_node_wwf(jit_code_ungei_f,u,v,w) + jit_code_unger_f, jit_code_ungei_f, +#define jit_ungtr_f(u,v,w) jit_new_node_www(jit_code_ungtr_f,u,v,w) +#define jit_ungti_f(u,v,w) jit_new_node_wwf(jit_code_ungti_f,u,v,w) + jit_code_ungtr_f, jit_code_ungti_f, +#define jit_ltgtr_f(u,v,w) jit_new_node_www(jit_code_ltgtr_f,u,v,w) +#define jit_ltgti_f(u,v,w) jit_new_node_wwf(jit_code_ltgti_f,u,v,w) + jit_code_ltgtr_f, jit_code_ltgti_f, +#define jit_ordr_f(u,v,w) jit_new_node_www(jit_code_ordr_f,u,v,w) +#define jit_ordi_f(u,v,w) jit_new_node_wwf(jit_code_ordi_f,u,v,w) + jit_code_ordr_f, jit_code_ordi_f, +#define jit_unordr_f(u,v,w) jit_new_node_www(jit_code_unordr_f,u,v,w) +#define jit_unordi_f(u,v,w) jit_new_node_wwf(jit_code_unordi_f,u,v,w) + jit_code_unordr_f, jit_code_unordi_f, + +#define jit_truncr_f_i(u,v) jit_new_node_ww(jit_code_truncr_f_i,u,v) + jit_code_truncr_f_i, +#if __WODSIZE == 32 +# define jit_truncr_f(u,v) jit_truncr_f_i(u,v) +#else +# define jit_truncr_f(u,v) jit_truncr_f_l(u,v) +#endif + /* >> 64 bit */ +#define jit_truncr_f_l(u,v) jit_new_node_ww(jit_code_truncr_f_l,u,v) + jit_code_truncr_f_l, + /* << 64 bit */ +#define jit_extr_f(u,v) jit_new_node_ww(jit_code_extr_f,u,v) +#define jit_extr_d_f(u,v) jit_new_node_ww(jit_code_extr_d_f,u,v) + jit_code_extr_f, jit_code_extr_d_f, +#define jit_movr_f(u,v) jit_new_node_ww(jit_code_movr_f,u,v) +#define jit_movi_f(u,v) jit_new_node_wf(jit_code_movi_f,u,v) + jit_code_movr_f, jit_code_movi_f, + +#define jit_ldr_f(u,v) jit_new_node_ww(jit_code_ldr_f,u,v) +#define jit_ldi_f(u,v) jit_new_node_ww(jit_code_ldi_f,u,v) + jit_code_ldr_f, jit_code_ldi_f, +#define jit_ldxr_f(u,v,w) jit_new_node_www(jit_code_ldxr_f,u,v,w) +#define jit_ldxi_f(u,v,w) jit_new_node_www(jit_code_ldxi_f,u,v,w) + jit_code_ldxr_f, jit_code_ldxi_f, +#define jit_str_f(u,v) jit_new_node_ww(jit_code_str_f,u,v) +#define jit_sti_f(u,v) jit_new_node_ww(jit_code_sti_f,u,v) + jit_code_str_f, jit_code_sti_f, +#define jit_stxr_f(u,v,w) jit_new_node_www(jit_code_stxr_f,u,v,w) +#define jit_stxi_f(u,v,w) jit_new_node_www(jit_code_stxi_f,u,v,w) + jit_code_stxr_f, jit_code_stxi_f, + +#define jit_bltr_f(v,w) jit_new_node_pww(jit_code_bltr_f,NULL,v,w) +#define jit_blti_f(v,w) jit_new_node_pwf(jit_code_blti_f,NULL,v,w) + jit_code_bltr_f, jit_code_blti_f, +#define jit_bler_f(v,w) jit_new_node_pww(jit_code_bler_f,NULL,v,w) +#define jit_blei_f(v,w) jit_new_node_pwf(jit_code_blei_f,NULL,v,w) + jit_code_bler_f, jit_code_blei_f, +#define jit_beqr_f(v,w) jit_new_node_pww(jit_code_beqr_f,NULL,v,w) +#define jit_beqi_f(v,w) jit_new_node_pwf(jit_code_beqi_f,NULL,v,w) + jit_code_beqr_f, jit_code_beqi_f, +#define jit_bger_f(v,w) jit_new_node_pww(jit_code_bger_f,NULL,v,w) +#define jit_bgei_f(v,w) jit_new_node_pwf(jit_code_bgei_f,NULL,v,w) + jit_code_bger_f, jit_code_bgei_f, +#define jit_bgtr_f(v,w) jit_new_node_pww(jit_code_bgtr_f,NULL,v,w) +#define jit_bgti_f(v,w) jit_new_node_pwf(jit_code_bgti_f,NULL,v,w) + jit_code_bgtr_f, jit_code_bgti_f, +#define jit_bner_f(v,w) jit_new_node_pww(jit_code_bner_f,NULL,v,w) +#define jit_bnei_f(v,w) jit_new_node_pwf(jit_code_bnei_f,NULL,v,w) + jit_code_bner_f, jit_code_bnei_f, +#define jit_bunltr_f(v,w) jit_new_node_pww(jit_code_bunltr_f,NULL,v,w) +#define jit_bunlti_f(v,w) jit_new_node_pwf(jit_code_bunlti_f,NULL,v,w) + jit_code_bunltr_f, jit_code_bunlti_f, +#define jit_bunler_f(v,w) jit_new_node_pww(jit_code_bunler_f,NULL,v,w) +#define jit_bunlei_f(v,w) jit_new_node_pwf(jit_code_bunlei_f,NULL,v,w) + jit_code_bunler_f, jit_code_bunlei_f, +#define jit_buneqr_f(v,w) jit_new_node_pww(jit_code_buneqr_f,NULL,v,w) +#define jit_buneqi_f(v,w) jit_new_node_pwf(jit_code_buneqi_f,NULL,v,w) + jit_code_buneqr_f, jit_code_buneqi_f, +#define jit_bunger_f(v,w) jit_new_node_pww(jit_code_bunger_f,NULL,v,w) +#define jit_bungei_f(v,w) jit_new_node_pwf(jit_code_bungei_f,NULL,v,w) + jit_code_bunger_f, jit_code_bungei_f, +#define jit_bungtr_f(v,w) jit_new_node_pww(jit_code_bungtr_f,NULL,v,w) +#define jit_bungti_f(v,w) jit_new_node_pwf(jit_code_bungti_f,NULL,v,w) + jit_code_bungtr_f, jit_code_bungti_f, +#define jit_bltgtr_f(v,w) jit_new_node_pww(jit_code_bltgtr_f,NULL,v,w) +#define jit_bltgti_f(v,w) jit_new_node_pwf(jit_code_bltgti_f,NULL,v,w) + jit_code_bltgtr_f, jit_code_bltgti_f, +#define jit_bordr_f(v,w) jit_new_node_pww(jit_code_bordr_f,NULL,v,w) +#define jit_bordi_f(v,w) jit_new_node_pwf(jit_code_bordi_f,NULL,v,w) + jit_code_bordr_f, jit_code_bordi_f, +#define jit_bunordr_f(v,w) jit_new_node_pww(jit_code_bunordr_f,NULL,v,w) +#define jit_bunordi_f(v,w) jit_new_node_pwf(jit_code_bunordi_f,NULL,v,w) + jit_code_bunordr_f, jit_code_bunordi_f, + +#define jit_pushargr_f(u) _jit_pushargr_f(_jit,u) +#define jit_pushargi_f(u) _jit_pushargi_f(_jit,u) +#define jit_retr_f(u) _jit_retr_f(_jit,u) +#define jit_reti_f(u) _jit_reti_f(_jit,u) +#define jit_retval_f(u) _jit_retval_f(_jit,u) + jit_code_retval_f, + +#define jit_arg_d() _jit_arg_d(_jit) +#define jit_getarg_d(u,v) _jit_getarg_d(_jit,u,v) + +#define jit_addr_d(u,v,w) jit_new_node_www(jit_code_addr_d,u,v,w) +#define jit_addi_d(u,v,w) jit_new_node_wwd(jit_code_addi_d,u,v,w) + jit_code_addr_d, jit_code_addi_d, +#define jit_subr_d(u,v,w) jit_new_node_www(jit_code_subr_d,u,v,w) +#define jit_subi_d(u,v,w) jit_new_node_wwd(jit_code_subi_d,u,v,w) + jit_code_subr_d, jit_code_subi_d, +#define jit_mulr_d(u,v,w) jit_new_node_www(jit_code_mulr_d,u,v,w) +#define jit_muli_d(u,v,w) jit_new_node_wwd(jit_code_muli_d,u,v,w) + jit_code_mulr_d, jit_code_muli_d, +#define jit_divr_d(u,v,w) jit_new_node_www(jit_code_divr_d,u,v,w) +#define jit_divi_d(u,v,w) jit_new_node_wwd(jit_code_divi_d,u,v,w) + jit_code_divr_d, jit_code_divi_d, + +#define jit_negr_d(u,v) jit_new_node_ww(jit_code_negr_d,u,v) +#define jit_absr_d(u,v) jit_new_node_ww(jit_code_absr_d,u,v) +#define jit_sqrtr_d(u,v) jit_new_node_ww(jit_code_sqrtr_d,u,v) + jit_code_negr_d, jit_code_absr_d, jit_code_sqrtr_d, + +#define jit_ltr_d(u,v,w) jit_new_node_www(jit_code_ltr_d,u,v,w) +#define jit_lti_d(u,v,w) jit_new_node_wwd(jit_code_lti_d,u,v,w) + jit_code_ltr_d, jit_code_lti_d, +#define jit_ler_d(u,v,w) jit_new_node_www(jit_code_ler_d,u,v,w) +#define jit_lei_d(u,v,w) jit_new_node_wwd(jit_code_lei_d,u,v,w) + jit_code_ler_d, jit_code_lei_d, +#define jit_eqr_d(u,v,w) jit_new_node_www(jit_code_eqr_d,u,v,w) +#define jit_eqi_d(u,v,w) jit_new_node_wwd(jit_code_eqi_d,u,v,w) + jit_code_eqr_d, jit_code_eqi_d, +#define jit_ger_d(u,v,w) jit_new_node_www(jit_code_ger_d,u,v,w) +#define jit_gei_d(u,v,w) jit_new_node_wwd(jit_code_gei_d,u,v,w) + jit_code_ger_d, jit_code_gei_d, +#define jit_gtr_d(u,v,w) jit_new_node_www(jit_code_gtr_d,u,v,w) +#define jit_gti_d(u,v,w) jit_new_node_wwd(jit_code_gti_d,u,v,w) + jit_code_gtr_d, jit_code_gti_d, +#define jit_ner_d(u,v,w) jit_new_node_www(jit_code_ner_d,u,v,w) +#define jit_nei_d(u,v,w) jit_new_node_wwd(jit_code_nei_d,u,v,w) + jit_code_ner_d, jit_code_nei_d, +#define jit_unltr_d(u,v,w) jit_new_node_www(jit_code_unltr_d,u,v,w) +#define jit_unlti_d(u,v,w) jit_new_node_wwd(jit_code_unlti_d,u,v,w) + jit_code_unltr_d, jit_code_unlti_d, +#define jit_unler_d(u,v,w) jit_new_node_www(jit_code_unler_d,u,v,w) +#define jit_unlei_d(u,v,w) jit_new_node_wwd(jit_code_unlei_d,u,v,w) + jit_code_unler_d, jit_code_unlei_d, +#define jit_uneqr_d(u,v,w) jit_new_node_www(jit_code_uneqr_d,u,v,w) +#define jit_uneqi_d(u,v,w) jit_new_node_wwd(jit_code_uneqi_d,u,v,w) + jit_code_uneqr_d, jit_code_uneqi_d, +#define jit_unger_d(u,v,w) jit_new_node_www(jit_code_unger_d,u,v,w) +#define jit_ungei_d(u,v,w) jit_new_node_wwd(jit_code_ungei_d,u,v,w) + jit_code_unger_d, jit_code_ungei_d, +#define jit_ungtr_d(u,v,w) jit_new_node_www(jit_code_ungtr_d,u,v,w) +#define jit_ungti_d(u,v,w) jit_new_node_wwd(jit_code_ungti_d,u,v,w) + jit_code_ungtr_d, jit_code_ungti_d, +#define jit_ltgtr_d(u,v,w) jit_new_node_www(jit_code_ltgtr_d,u,v,w) +#define jit_ltgti_d(u,v,w) jit_new_node_wwd(jit_code_ltgti_d,u,v,w) + jit_code_ltgtr_d, jit_code_ltgti_d, +#define jit_ordr_d(u,v,w) jit_new_node_www(jit_code_ordr_d,u,v,w) +#define jit_ordi_d(u,v,w) jit_new_node_wwd(jit_code_ordi_d,u,v,w) + jit_code_ordr_d, jit_code_ordi_d, +#define jit_unordr_d(u,v,w) jit_new_node_www(jit_code_unordr_d,u,v,w) +#define jit_unordi_d(u,v,w) jit_new_node_wwd(jit_code_unordi_d,u,v,w) + jit_code_unordr_d, jit_code_unordi_d, + +#define jit_truncr_d_i(u,v) jit_new_node_ww(jit_code_truncr_d_i,u,v) + jit_code_truncr_d_i, +#if __WODSIZE == 32 +# define jit_truncr_d(u,v) jit_truncr_d_i(u,v) +#else +# define jit_truncr_d(u,v) jit_truncr_d_l(u,v) +#endif + /* >> 64 bit */ +#define jit_truncr_d_l(u,v) jit_new_node_ww(jit_code_truncr_d_l,u,v) + jit_code_truncr_d_l, + /* << 64 bit */ +#define jit_extr_d(u,v) jit_new_node_ww(jit_code_extr_f,u,v) +#define jit_extr_f_d(u,v) jit_new_node_ww(jit_code_extr_f_d,u,v) + jit_code_extr_d, jit_code_extr_f_d, +#define jit_movr_d(u,v) jit_new_node_ww(jit_code_movr_d,u,v) +#define jit_movi_d(u,v) jit_new_node_wd(jit_code_movi_d,u,v) + jit_code_movr_d, jit_code_movi_d, + +#define jit_ldr_d(u,v) jit_new_node_ww(jit_code_ldr_d,u,v) +#define jit_ldi_d(u,v) jit_new_node_ww(jit_code_ldi_d,u,v) + jit_code_ldr_d, jit_code_ldi_d, +#define jit_ldxr_d(u,v,w) jit_new_node_www(jit_code_ldxr_d,u,v,w) +#define jit_ldxi_d(u,v,w) jit_new_node_www(jit_code_ldxi_d,u,v,w) + jit_code_ldxr_d, jit_code_ldxi_d, +#define jit_str_d(u,v) jit_new_node_ww(jit_code_str_d,u,v) +#define jit_sti_d(u,v) jit_new_node_ww(jit_code_sti_d,u,v) + jit_code_str_d, jit_code_sti_d, +#define jit_stxr_d(u,v,w) jit_new_node_www(jit_code_stxr_d,u,v,w) +#define jit_stxi_d(u,v,w) jit_new_node_www(jit_code_stxi_d,u,v,w) + jit_code_stxr_d, jit_code_stxi_d, + +#define jit_bltr_d(v,w) jit_new_node_pww(jit_code_bltr_d,NULL,v,w) +#define jit_blti_d(v,w) jit_new_node_pwd(jit_code_blti_d,NULL,v,w) + jit_code_bltr_d, jit_code_blti_d, +#define jit_bler_d(v,w) jit_new_node_pww(jit_code_bler_d,NULL,v,w) +#define jit_blei_d(v,w) jit_new_node_pwd(jit_code_blei_d,NULL,v,w) + jit_code_bler_d, jit_code_blei_d, +#define jit_beqr_d(v,w) jit_new_node_pww(jit_code_beqr_d,NULL,v,w) +#define jit_beqi_d(v,w) jit_new_node_pwd(jit_code_beqi_d,NULL,v,w) + jit_code_beqr_d, jit_code_beqi_d, +#define jit_bger_d(v,w) jit_new_node_pww(jit_code_bger_d,NULL,v,w) +#define jit_bgei_d(v,w) jit_new_node_pwd(jit_code_bgei_d,NULL,v,w) + jit_code_bger_d, jit_code_bgei_d, +#define jit_bgtr_d(v,w) jit_new_node_pww(jit_code_bgtr_d,NULL,v,w) +#define jit_bgti_d(v,w) jit_new_node_pwd(jit_code_bgti_d,NULL,v,w) + jit_code_bgtr_d, jit_code_bgti_d, +#define jit_bner_d(v,w) jit_new_node_pww(jit_code_bner_d,NULL,v,w) +#define jit_bnei_d(v,w) jit_new_node_pwd(jit_code_bnei_d,NULL,v,w) + jit_code_bner_d, jit_code_bnei_d, +#define jit_bunltr_d(v,w) jit_new_node_pww(jit_code_bunltr_d,NULL,v,w) +#define jit_bunlti_d(v,w) jit_new_node_pwd(jit_code_bunlti_d,NULL,v,w) + jit_code_bunltr_d, jit_code_bunlti_d, +#define jit_bunler_d(v,w) jit_new_node_pww(jit_code_bunler_d,NULL,v,w) +#define jit_bunlei_d(v,w) jit_new_node_pwd(jit_code_bunlei_d,NULL,v,w) + jit_code_bunler_d, jit_code_bunlei_d, +#define jit_buneqr_d(v,w) jit_new_node_pww(jit_code_buneqr_d,NULL,v,w) +#define jit_buneqi_d(v,w) jit_new_node_pwd(jit_code_buneqi_d,NULL,v,w) + jit_code_buneqr_d, jit_code_buneqi_d, +#define jit_bunger_d(v,w) jit_new_node_pww(jit_code_bunger_d,NULL,v,w) +#define jit_bungei_d(v,w) jit_new_node_pwd(jit_code_bungei_d,NULL,v,w) + jit_code_bunger_d, jit_code_bungei_d, +#define jit_bungtr_d(v,w) jit_new_node_pww(jit_code_bungtr_d,NULL,v,w) +#define jit_bungti_d(v,w) jit_new_node_pwd(jit_code_bungti_d,NULL,v,w) + jit_code_bungtr_d, jit_code_bungti_d, +#define jit_bltgtr_d(v,w) jit_new_node_pww(jit_code_bltgtr_d,NULL,v,w) +#define jit_bltgti_d(v,w) jit_new_node_pwd(jit_code_bltgti_d,NULL,v,w) + jit_code_bltgtr_d, jit_code_bltgti_d, +#define jit_bordr_d(v,w) jit_new_node_pww(jit_code_bordr_d,NULL,v,w) +#define jit_bordi_d(v,w) jit_new_node_pwd(jit_code_bordi_d,NULL,v,w) + jit_code_bordr_d, jit_code_bordi_d, +#define jit_bunordr_d(v,w) jit_new_node_pww(jit_code_bunordr_d,NULL,v,w) +#define jit_bunordi_d(v,w) jit_new_node_pwd(jit_code_bunordi_d,NULL,v,w) + jit_code_bunordr_d, jit_code_bunordi_d, + +#define jit_pushargr_d(u) _jit_pushargr_d(_jit,u) +#define jit_pushargi_d(u) _jit_pushargi_d(_jit,u) +#define jit_retr_d(u) _jit_retr_d(_jit,u) +#define jit_reti_d(u) _jit_reti_d(_jit,u) +#define jit_retval_d(u) _jit_retval_d(_jit,u) + jit_code_retval_d, +} jit_code_t; + +/* + * Prototypes + */ +extern void init_jit(void); +extern void finish_jit(void); + +extern jit_state_t *jit_new_state(void); + +extern jit_node_t *_jit_data(jit_state_t*, jit_pointer_t, jit_word_t); +extern jit_node_t *_jit_note(jit_state_t*, jit_pointer_t); + +extern jit_node_t *_jit_label(jit_state_t*); +extern jit_node_t *_jit_forward(jit_state_t*); +extern void _jit_link(jit_state_t*, jit_node_t*); + +extern void _jit_prolog(jit_state_t*); + +extern jit_int32_t _jit_allocai(jit_state_t*, jit_int32_t); + +extern jit_int32_t _jit_arg(jit_state_t*); +extern void _jit_getarg_c(jit_state_t*, jit_gpr_t, jit_int32_t); +extern void _jit_getarg_uc(jit_state_t*, jit_gpr_t, jit_int32_t); +extern void _jit_getarg_s(jit_state_t*, jit_gpr_t, jit_int32_t); +extern void _jit_getarg_us(jit_state_t*, jit_gpr_t, jit_int32_t); +extern void _jit_getarg_i(jit_state_t*, jit_gpr_t, jit_int32_t); +#if __WORDSIZE == 64 +extern void _jit_getarg_ui(jit_state_t*, jit_gpr_t, jit_int32_t); +extern void _jit_getarg_l(jit_state_t*, jit_gpr_t, jit_int32_t); +#endif + +extern void _jit_prepare(jit_state_t*, jit_int32_t); +extern void _jit_pushargr(jit_state_t*, jit_gpr_t); +extern void _jit_pushargi(jit_state_t*, jit_word_t); +extern void _jit_finishr(jit_state_t*, jit_gpr_t); +extern jit_node_t *_jit_finishi(jit_state_t*, jit_pointer_t); +extern void _jit_ret(jit_state_t*); +extern void _jit_retr(jit_state_t*, jit_gpr_t); +extern void _jit_reti(jit_state_t*, jit_word_t); +extern void _jit_retval_c(jit_state_t*, jit_gpr_t); +extern void _jit_retval_uc(jit_state_t*, jit_gpr_t); +extern void _jit_retval_s(jit_state_t*, jit_gpr_t); +extern void _jit_retval_us(jit_state_t*, jit_gpr_t); +extern void _jit_retval_i(jit_state_t*, jit_gpr_t); +#if __WORDSIZE == 64 +extern void _jit_retval_ui(jit_state_t*, jit_gpr_t); +extern void _jit_retval_l(jit_state_t*, jit_gpr_t); +#endif +extern void _jit_epilog(jit_state_t*); + +#define jit_patch(u) _jit_patch(_jit,u) +extern void _jit_patch(jit_state_t*, jit_node_t*); +#define jit_patch_at(u,v) _jit_patch_at(_jit,u,v) +extern void _jit_patch_at(jit_state_t*, jit_node_t*, jit_node_t*); +#define jit_patch_abs(u,v) _jit_patch_abs(_jit,u,v) +extern void _jit_patch_abs(jit_state_t*, jit_node_t*, jit_pointer_t); +#define jit_emit() _jit_emit(_jit) +extern jit_pointer_t _jit_emit(jit_state_t*); + +#define jit_print() _jit_print(_jit) +extern void _jit_print(jit_state_t*); + +extern jit_int32_t _jit_arg_f(jit_state_t*); +extern void _jit_getarg_f(jit_state_t*, jit_fpr_t, jit_int32_t); + +extern void _jit_pushargr_f(jit_state_t*, jit_fpr_t); +extern void _jit_pushargi_f(jit_state_t*, jit_float32_t); +extern void _jit_retr_f(jit_state_t*, jit_fpr_t); +extern void _jit_reti_f(jit_state_t*, jit_float32_t); +extern void _jit_retval_f(jit_state_t*, jit_fpr_t); + +extern jit_int32_t _jit_arg_d(jit_state_t*); +extern void _jit_getarg_d(jit_state_t*, jit_fpr_t, jit_int32_t); + +extern void _jit_pushargr_d(jit_state_t*, jit_fpr_t); +extern void _jit_pushargi_d(jit_state_t*, jit_float64_t); +extern void _jit_retr_d(jit_state_t*, jit_fpr_t); +extern void _jit_reti_d(jit_state_t*, jit_float64_t); +extern void _jit_retval_d(jit_state_t*, jit_fpr_t); + +#define jit_new_node(c) _jit_new_node(_jit,c) +extern jit_node_t *_jit_new_node(jit_state_t*, jit_code_t); +#define jit_new_node_w(c,u) _jit_new_node_w(_jit,c,u) +extern jit_node_t *_jit_new_node_w(jit_state_t*, jit_code_t, + jit_word_t); +#define jit_new_node_p(c,u) _jit_new_node_p(_jit,c,u) +extern jit_node_t *_jit_new_node_p(jit_state_t*, jit_code_t, + jit_pointer_t); +#define jit_new_node_ww(c,u,v) _jit_new_node_ww(_jit,c,u,v) +extern jit_node_t *_jit_new_node_ww(jit_state_t*,jit_code_t, + jit_word_t, jit_word_t); +#define jit_new_node_wf(c,u,v) _jit_new_node_wf(_jit,c,u,v) +extern jit_node_t *_jit_new_node_wf(jit_state_t*, jit_code_t, + jit_word_t, jit_float32_t); +#define jit_new_node_wd(c,u,v) _jit_new_node_wd(_jit,c,u,v) +extern jit_node_t *_jit_new_node_wd(jit_state_t*, jit_code_t, + jit_word_t, jit_float64_t); +#define jit_new_node_www(c,u,v,w) _jit_new_node_www(_jit,c,u,v,w) +extern jit_node_t *_jit_new_node_www(jit_state_t*, jit_code_t, + jit_word_t, jit_word_t, jit_word_t); +#define jit_new_node_wwf(c,u,v,w) _jit_new_node_wwf(_jit,c,u,v,w) +extern jit_node_t *_jit_new_node_wwf(jit_state_t*, jit_code_t, + jit_word_t, jit_word_t, jit_float32_t); +#define jit_new_node_wwd(c,u,v,w) _jit_new_node_wwd(_jit,c,u,v,w) +extern jit_node_t *_jit_new_node_wwd(jit_state_t*, jit_code_t, + jit_word_t, jit_word_t, jit_float64_t); +#define jit_new_node_pww(c,u,v,w) _jit_new_node_pww(_jit,c,u,v,w) +extern jit_node_t *_jit_new_node_pww(jit_state_t*, jit_code_t, + jit_pointer_t, jit_word_t, jit_word_t); +#define jit_new_node_pwf(c,u,v,w) _jit_new_node_pwf(_jit,c,u,v,w) +extern jit_node_t *_jit_new_node_pwf(jit_state_t*, jit_code_t, + jit_pointer_t, jit_word_t, jit_float32_t); +#define jit_new_node_pwd(c,u,v,w) _jit_new_node_pwd(_jit,c,u,v,w) +extern jit_node_t *_jit_new_node_pwd(jit_state_t*, jit_code_t, + jit_pointer_t, jit_word_t, jit_float64_t); + +#define jit_disassemble() _jit_disassemble(_jit) +extern void _jit_disassemble(jit_state_t*); + +#endif /* _lightning_h */ diff --git a/include/lightning/Makefile.am b/include/lightning/Makefile.am new file mode 100644 index 000000000..ab5b2f311 --- /dev/null +++ b/include/lightning/Makefile.am @@ -0,0 +1,23 @@ +# +# Copyright 2000, 2001, 2002, 2012 Free Software Foundation, Inc. +# +# This is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +includedir = $(includedir)/lightning + +EXTRA_DIST = \ + jit_private.h + +if cpu_x86 +include_HEADERS = \ + jit_x86.h +endif diff --git a/include/lightning/jit_arm.h b/include/lightning/jit_arm.h new file mode 100644 index 000000000..9ee7b2c96 --- /dev/null +++ b/include/lightning/jit_arm.h @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _jit_arm_h +#define _jit_arm_h + +#define JIT_HASH_CONSTS 0 +#define JIT_NUM_OPERANDS 3 + +/* + * Types + */ +#define jit_swf_p() (jit_cpu.vfp == 0) +#define jit_hardfp_p() jit_cpu.abi + +#define JIT_RET _R0 +#define JIT_SP _R13 +#define JIT_FP _R11 +typedef enum { +#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4) +#define jit_r(i) (_R4 + (i)) +#define jit_r_num() 3 +#define jit_v(i) (_R7 + (i)) +#define jit_v_num() 3 +#define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 4) +#define jit_f(i) (jit_cpu.abi ? _D8 + (i) : _D7 + (i)) +#define jit_f_num() (jit_cpu.vfp ? 16 : 8) + _R12, /* ip - temporary */ +#define JIT_R0 _R4 +#define JIT_R1 _R5 +#define JIT_R2 _R6 + _R4, /* r4 - variable */ + _R5, /* r5 - variable */ + _R6, /* r6 - variable */ +#define JIT_V0 _R7 +#define JIT_V1 _R8 +#define JIT_V2 _R9 + _R7, /* r7 - variable */ + _R8, /* r8 - variable */ + _R9, /* r9 - variable */ + _R10, /* sl - stack limit */ + _R11, /* fp - frame pointer */ + _R13, /* sp - stack pointer */ + _R14, /* lr - link register */ + _R15, /* pc - program counter */ +#define JIT_RA0 _R0 +#define JIT_RA1 _R1 +#define JIT_RA2 _R2 +#define JIT_RA3 _R3 + _R3, /* r3 - argument/result */ + _R2, /* r2 - argument/result */ + _R1, /* r1 - argument/result */ + _R0, /* r0 - argument/result */ +#if defined(__ARM_PCS_VFP) +# define JIT_FRET _D0 +#else +# define JIT_FRET _R0 +#endif +#define JIT_F0 (jit_hardfp_p() ? _D8 : _D0) +#define JIT_F1 (jit_hardfp_p() ? _D9 : _D1) +#define JIT_F2 (jit_hardfp_p() ? _D10 : _D2) +#define JIT_F3 (jit_hardfp_p() ? _D11 : _D3) +#define JIT_F4 (jit_hardfp_p() ? _D12 : _D4) +#define JIT_F5 (jit_hardfp_p() ? _D13 : _D5) +#define JIT_F6 (jit_hardfp_p() ? _D14 : _D6) +#define JIT_F7 (jit_hardfp_p() ? _D15 : _D7) + _S16, _D8 = _S16, _Q4 = _D8, + _S17, + _S18, _D9 = _S18, + _S19, + _S20, _D10 = _S20, _Q5 = _D10, + _S21, + _S22, _D11 = _S22, + _S23, + _S24, _D12 = _S24, _Q6 = _D12, + _S25, + _S26, _D13 = _S26, + _S27, + _S28, _D14 = _S28, _Q7 = _D14, + _S29, + _S30, _D15 = _S30, + _S31, +#define JIT_FA0 _D0 +#define JIT_FA1 _D1 +#define JIT_FA2 _D2 +#define JIT_FA3 _D3 +#define JIT_FA4 _D4 +#define JIT_FA5 _D5 +#define JIT_FA6 _D6 +#define JIT_FA7 _D7 + _S15, + _S14, _D7 = _S14, + _S13, + _S12, _D6 = _S12, _Q3 = _D6, + _S11, + _S10, _D5 = _S10, + _S9, + _S8, _D4 = _S8, _Q2 = _D4, + _S7, + _S6, _D3 = _S6, + _S5, + _S4, _D2 = _S4, _Q1 = _D2, + _S3, + _S2, _D1 = _S2, + _S1, + _S0, _D0 = _S0, _Q0 = _D0, + _NOREG, +#define JIT_NOREG _NOREG +} jit_reg_t; + +typedef struct { + jit_uint32_t version : 4; + jit_uint32_t extend : 1; + /* only generate thumb instructions for thumb2 */ + jit_uint32_t thumb : 1; + jit_uint32_t vfp : 3; + jit_uint32_t neon : 1; + jit_uint32_t abi : 2; +} jit_cpu_t; + +typedef struct { + /* prevent using thumb instructions that set flags? */ + jit_uint32_t no_set_flags : 1; +} jit_flags_t; + +typedef jit_int64_t jit_regset_t; + +/* + * Initialization + */ +extern jit_cpu_t jit_cpu; + +#endif /* _jit_arm_h */ diff --git a/include/lightning/jit_mips.h b/include/lightning/jit_mips.h new file mode 100644 index 000000000..cb7235fc1 --- /dev/null +++ b/include/lightning/jit_mips.h @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _jit_mips_h +#define _jit_mips_h + +#define JIT_HASH_CONSTS 1 +#define JIT_NUM_OPERANDS 3 + +/* + * Types + */ +#define JIT_RET _V0 +#define JIT_FRET _F0 +#define JIT_SP _SP +#define JIT_FP _FP +typedef enum { +#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4) +#define jit_r(i) (_V0 + (i)) +#define jit_r_num() 12 +#define jit_v(i) (_S0 + (i)) +#define jit_r_num() 8 +#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4) +#define jit_f(i) (_F0 + (i)) +#define jit_f_num() 14 + _AT, +#define JIT_R0 _V0 +#define JIT_R1 _V1 +#define JIT_R2 _T0 +#define JIT_R3 _T1 +#define JIT_R4 _T2 +#define JIT_R5 _T3 +#define JIT_R6 _T4 +#define JIT_R7 _T5 +#define JIT_R8 _T6 +#define JIT_R9 _T7 +#define JIT_R10 _T8 +#define JIT_R11 _T9 /* must point to PIC function */ + _V0, _V1, _T0, _T1, _T2, _T3, _T4, _T5, _T6, _T7, _T8, _T9, +#define JIT_V0 _S0 +#define JIT_V1 _S1 +#define JIT_V2 _S2 +#define JIT_V3 _S3 +#define JIT_V4 _S4 +#define JIT_V5 _S5 +#define JIT_V6 _S6 +#define JIT_V7 _S7 + _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7, + _ZERO, _K0, _K1, _RA, + _GP, /* FIXME use to point to jit data */ + _SP, _FP, +# define JIT_RA0 _A0 +# define JIT_RA1 _A1 +# define JIT_RA2 _A2 +# define JIT_RA3 _A3 + _A3, _A2, _A1, _A0, + +#define JIT_F0 _F0 +#define JIT_F1 _F2 +#define JIT_F2 _F4 +#define JIT_F3 _F6 +#define JIT_F4 _F8 +#define JIT_F5 _F10 + _F0, _F2, _F4, _F6, _F8, _F10, + /* callee save float registers */ +#define JIT_FS0 _F16 +#define JIT_FS1 _F18 +#define JIT_FS2 _F20 +#define JIT_FS3 _F22 +#define JIT_FS4 _F24 +#define JIT_FS5 _F26 +#define JIT_FS6 _F28 +#define JIT_FS7 _F30 + _F16, _F18, _F20, _F22, _F24, _F26, _F28, _F30, +#define JIT_FA0 _F12 +#define JIT_FA1 _F14 + _F12, _F14, +#define JIT_NOREG _NOREG + _NOREG, +} jit_reg_t; + +typedef jit_int64_t jit_regset_t; + +#endif /* _jit_mips_h */ diff --git a/include/lightning/jit_ppc.h b/include/lightning/jit_ppc.h new file mode 100644 index 000000000..52111c8f0 --- /dev/null +++ b/include/lightning/jit_ppc.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _jit_ppc_h +#define _jit_ppc_h + +#define JIT_HASH_CONSTS 1 +#define JIT_NUM_OPERANDS 3 + +/* + * Types + */ +typedef enum { +#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8) +#define jit_r(i) (_R11 + (i)) +#define jit_r_num() 3 +#define jit_v(i) (_R30 - (i)) +#define jit_r_num() 17 +#define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) +#define jit_f(i) (_F0 + (i)) +#define jit_f_num() 6 + _R0, +#define JIT_R0 _R11 +#define JIT_R1 _R12 +#define JIT_R2 _R13 +#define JIT_R3 _R2 + _R11, _R12, _R13, _R2, +#define JIT_V0 _R30 +#define JIT_V1 _R29 +#define JIT_V2 _R28 +#define JIT_V3 _R28 +#define JIT_V4 _R26 +#define JIT_V5 _R25 +#define JIT_V6 _R24 +#define JIT_V7 _R23 +#define JIT_V8 _R22 +#define JIT_V9 _R21 +#define JIT_V10 _R20 +#define JIT_V11 _R19 +#define JIT_V12 _R18 +#define JIT_V13 _R17 +#define JIT_V14 _R16 +#define JIT_V15 _R15 +#define JIT_V16 _R14 + _R14, _R15, _R16, _R17, _R18, _R19, _R20, _R21, + _R22, _R23, _R24, _R25, _R26, _R27, _R28, _R29, + _R30, +#define JIT_SP _R1 + _R1, +#define JIT_FP _R31 + _R31, +#define JIT_RET _R3 +#define JIT_RA0 _R3 +#define JIT_RA1 _R4 +#define JIT_RA2 _R5 +#define JIT_RA3 _R6 +#define JIT_RA4 _R7 +#define JIT_RA5 _R8 +#define JIT_RA6 _R9 +#define JIT_RA7 _R10 + _R10, _R9, _R8, _R7, _R6, _R5, _R4, _R3, +# define JIT_F0 _F0 +# define JIT_F1 _F8 +# define JIT_F2 _F9 +# define JIT_F3 _F10 +# define JIT_F4 _F11 +# define JIT_F5 _F12 + _F0, _F9, _F10, _F11, _F12, _F13, +#define JIT_FS0 _F14 +#define JIT_FS1 _F15 +#define JIT_FS2 _F16 +#define JIT_FS3 _F17 +#define JIT_FS4 _F18 +#define JIT_FS5 _F19 +#define JIT_FS6 _F20 +#define JIT_FS7 _F21 +#define JIT_FS8 _F22 +#define JIT_FS9 _F23 +#define JIT_FS10 _F24 +#define JIT_FS11 _F25 +#define JIT_FS12 _F26 +#define JIT_FS13 _F27 +#define JIT_FS14 _F28 +#define JIT_FS15 _F29 +#define JIT_FS16 _F30 +#define JIT_FS17 _F31 + _F14, _F15, _F16, _F17, _F18, _F19, _F20, + _F21, _F22, _F23, _F24, _F25, _F26, _F27, + _F28, _F29, _F30, _F31, +#define JIT_FRET _F1 +#define JIT_FA0 _F1 +#define JIT_FA1 _F2 +#define JIT_FA2 _F3 +#define JIT_FA3 _F4 +#define JIT_FA4 _F5 +#define JIT_FA5 _F6 +#define JIT_FA6 _F7 +#define JIT_FA7 _F8 + _F8, _F7, _F6, _F5, _F4, _F3, _F2, _F1, + _NOREG, +#define JIT_NOREG _NOREG +} jit_reg_t; + +typedef jit_int64_t jit_regset_t; + +#endif /* _jit_ppc_h */ diff --git a/include/lightning/jit_private.h b/include/lightning/jit_private.h new file mode 100644 index 000000000..0899c76b8 --- /dev/null +++ b/include/lightning/jit_private.h @@ -0,0 +1,363 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _jit_private_h +#define _jit_private_h + +#include +#include +#include +#include + +#if defined(__GNUC__) +# define maybe_unused __attribute__ ((unused)) +# define unlikely(exprn) __builtin_expect(!!(exprn), 0) +# define likely(exprn) __builtin_expect(!!(exprn), 1) +# if (__GNUC__ >= 4) +# define PUBLIC __attribute__ ((visibility("default"))) +# define HIDDEN __attribute__ ((visibility("hidden"))) +# else +# define PUBLIC /**/ +# define HIDDEN /**/ +# endif +#else +# define maybe_unused /**/ +# define unlikely(exprn) exprn +# define likely(exprn) exprn +# define PUBLIC /**/ +# define HIDDEN /**/ +#endif + +#define jit_size(vector) (sizeof(vector) / sizeof((vector)[0])) + +/* + * Private jit_class bitmasks + */ +#define jit_class_nospill 0x00800000 /* hint to fail if need spill */ +#define jit_class_sft 0x01000000 /* not a hardware register */ +#define jit_class_rg8 0x04000000 /* x86 8 bits */ +#define jit_class_xpr 0x80000000 /* float / vector */ +#define jit_regno_patch 0x00008000 /* this is a register + * returned by a "user" call + * to jit_get_reg() */ + +#define jit_kind_register 1 +#define jit_kind_code 2 +#define jit_kind_word 3 +#define jit_kind_float32 4 +#define jit_kind_float64 5 + +#define jit_cc_a0_reg 0x00000001 /* arg0 is a register */ +#define jit_cc_a0_chg 0x00000002 /* arg0 is modified */ +#define jit_cc_a0_jmp 0x00000004 /* arg0 is a jump target */ +#define jit_cc_a0_int 0x00000010 /* arg0 is immediate word */ +#define jit_cc_a0_flt 0x00000020 /* arg0 is immediate float */ +#define jit_cc_a0_dbl 0x00000040 /* arg0 is immediate double */ +#define jit_cc_a1_reg 0x00000100 /* arg1 is a register */ +#define jit_cc_a1_chg 0x00000200 /* arg1 is modified */ +#define jit_cc_a1_int 0x00001000 /* arg1 is immediate word */ +#define jit_cc_a1_flt 0x00002000 /* arg1 is immediate float */ +#define jit_cc_a1_dbl 0x00004000 /* arg1 is immediate double */ +#define jit_cc_a2_reg 0x00010000 /* arg2 is a register */ +#define jit_cc_a2_chg 0x00020000 /* arg2 is modified */ +#define jit_cc_a2_int 0x00100000 /* arg2 is immediate word */ +#define jit_cc_a2_flt 0x00200000 /* arg2 is immediate float */ +#define jit_cc_a2_dbl 0x00400000 /* arg2 is immediate double */ + +#define jit_regset_com(u, v) ((u) = ~(v)) +#define jit_regset_and(u, v, w) ((u) = (v) & (w)) +#define jit_regset_ior(u, v, w) ((u) = (v) | (w)) +#define jit_regset_xor(u, v, w) ((u) = (v) ^ (w)) +#define jit_regset_set(u, v) ((u) = (v)) +#define jit_regset_cmp_ui(u, v) ((u) != (v)) +#define jit_regset_set_ui(u, v) ((u) = (v)) +#define jit_regset_set_p(set) (set) +#if DEBUG +# define jit_regset_clrbit(set, bit) \ + (assert(bit >= 0 && bit < (sizeof(jit_regset_t) << 3)), \ + (set) &= ~(1LL << (bit))) +# define jit_regset_setbit(set, bit) \ + (assert(bit >= 0 && bit < (sizeof(jit_regset_t) << 3)), \ + (set) |= 1LL << (bit)) +# define jit_regset_tstbit(set, bit) \ + (assert(bit >= 0 && bit < (sizeof(jit_regset_t) << 3)), \ + (set) & (1LL << (bit))) +#else +# define jit_regset_clrbit(set, bit) ((set) &= ~(1LL << (bit))) +# define jit_regset_setbit(set, bit) ((set) |= 1LL << (bit)) +# define jit_regset_tstbit(set, bit) ((set) & (1LL << (bit))) +#endif +#define jit_regset_new(set) ((set) = 0) +#define jit_regset_del(set) ((set) = 0) +extern unsigned long +jit_regset_scan1(jit_regset_t, jit_int32_t); + +#define jit_reglive_setup() \ + do { \ + jit_regset_set_ui(_jit->reglive, 0); \ + jit_regset_set_ui(_jit->regmask, 0); \ + } while (0) + +/* + * Types + */ +typedef union jit_data jit_data_t; +typedef struct jit_block jit_block_t; +typedef struct jit_value jit_value_t; +typedef struct jit_function jit_function_t; +typedef struct jit_register jit_register_t; +#if __arm__ +# if DISASSEMBLER +typedef struct jit_data_info jit_data_info_t; +# endif +#endif + +union jit_data { + struct { +#if __BYTE_ORDER == __LITTLE_ENDIAN + jit_int32_t l; + jit_int32_t h; +#else + jit_int32_t h; + jit_int32_t l; +#endif + } pair; + jit_word_t w; + jit_float32_t f; + jit_float64_t d; + jit_pointer_t p; + jit_node_t *n; +}; + +struct jit_node { + jit_node_t *next; + jit_code_t code; + jit_int32_t flag; + jit_data_t u; + jit_data_t v; + jit_data_t w; + jit_node_t *link; +}; + +struct jit_block { + jit_node_t *label; + jit_regset_t reglive; + jit_regset_t regmask; +}; + +struct jit_value { + jit_int32_t kind; + jit_code_t code; + jit_data_t base; + jit_data_t disp; +}; + +typedef struct { + jit_word_t inst; + jit_node_t *node; +} jit_patch_t; + +#if __arm__ && DISASSEMBLER +struct jit_data_info { + jit_uword_t code; /* pointer in code buffer */ + jit_word_t length; /* length of constant vector */ +}; +#endif + +struct jit_function { + struct { + jit_int32_t argi; + jit_int32_t argf; + jit_int32_t size; + jit_int32_t aoff; + jit_int32_t alen; + } self; + struct { + jit_int32_t argi; + jit_int32_t argf; + jit_int32_t size; + jit_int32_t kind; + } call; + jit_node_t *prolog; + jit_node_t *epilog; + jit_int32_t *regoff; + jit_regset_t regset; + jit_int32_t stack; +}; + +struct jit_state { + union { + jit_uint8_t *uc; + jit_uint16_t *us; + jit_uint32_t *ui; + jit_uint64_t *ul; + jit_word_t w; + } pc; + jit_node_t *head; + jit_node_t *tail; + jit_uint32_t emit : 1; /* emit state entered */ + jit_uint32_t again : 1; /* start over emiting function */ + jit_int32_t reglen; /* number of registers */ + jit_regset_t regarg; /* cannot allocate */ + jit_regset_t regsav; /* automatic spill only once */ + jit_regset_t reglive; /* known live registers at some point */ + jit_regset_t regmask; /* register mask to update reglive */ + mpz_t blockmask; /* mask of visited basic blocks */ + struct { + jit_uint8_t *ptr; + jit_word_t length; + } code; + struct { + jit_uint8_t *ptr; /* constant pool */ + jit_node_t **table; /* very simple hash table */ + jit_word_t size; /* number of vectors in table */ + jit_word_t count; /* number of hash table entries */ + jit_word_t offset; /* offset in bytes in ptr */ + jit_word_t length; /* length in bytes of ptr */ + } data; + jit_node_t **spill; + jit_int32_t *gen; /* ssa like "register version" */ + jit_value_t *values; /* temporary jit_value_t vector */ + struct { + jit_block_t *ptr; + jit_word_t offset; + jit_word_t length; + } blocks; /* basic blocks */ + struct { + jit_patch_t *ptr; + jit_word_t offset; + jit_word_t length; + } patches; /* forward patch information */ + jit_function_t *function; /* current function */ + struct { + jit_function_t *ptr; + jit_word_t offset; + jit_word_t length; + } functions; /* prolog/epilogue offsets in code */ + struct { + jit_node_t **ptr; + jit_word_t offset; + jit_word_t length; + } pool; + jit_node_t *list; +#if __arm__ +# if DISASSEMBLER + struct { + jit_data_info_t *ptr; + it_word_t offset; + jit_word_t length; + } data_info; /* constant pools information */ +# endif + struct { + jit_uint8_t *data; /* pointer to code */ + jit_word_t size; /* size data */ + jit_word_t offset; /* pending patches */ + jit_word_t length; /* number of pending constants */ + jit_int32_t values[1024]; /* pending constants */ + jit_word_t patches[2048]; + } consts; +#endif +}; + +struct jit_register { + jit_reg_t spec; + char *name; +}; + +/* + * Prototypes + */ +extern void jit_get_cpu(void); + +#define jit_init() _jit_init(_jit) +extern void _jit_init(jit_state_t*); + +#define jit_new_node_no_link(u) _jit_new_node_no_link(_jit, u) +extern jit_node_t *_jit_new_node_no_link(jit_state_t*, jit_code_t); + +#define jit_link_node(u) _jit_link_node(_jit, u) +extern void _jit_link_node(jit_state_t*, jit_node_t*); + +#define jit_link_label(l) _jit_link_label(_jit,l) +extern void +_jit_link_label(jit_state_t*,jit_node_t*); + +#define jit_reglive(node) _jit_reglive(_jit, node) +extern void +_jit_reglive(jit_state_t*, jit_node_t*); + +#define jit_regarg_set(n,v) _jit_regarg_set(_jit,n,v) +extern void +_jit_regarg_set(jit_state_t*, jit_node_t*, jit_int32_t); + +#define jit_regarg_clr(n,v) _jit_regarg_clr(_jit,n,v) +extern void +_jit_regarg_clr(jit_state_t*, jit_node_t*, jit_int32_t); + +#define jit_get_reg(s) _jit_get_reg(_jit,s) +extern jit_int32_t +_jit_get_reg(jit_state_t*, jit_int32_t); + +#define jit_unget_reg(r) _jit_unget_reg(_jit,r) +extern void +_jit_unget_reg(jit_state_t*, jit_int32_t); + +#define jit_save(reg) _jit_save(_jit, reg) +extern void +_jit_save(jit_state_t*, jit_int32_t); + +#define jit_load(reg) _jit_load(_jit, reg) +extern void +_jit_load(jit_state_t*, jit_int32_t); + +#define jit_optimize() _jit_optimize(_jit) +extern void +_jit_optimize(jit_state_t*); + +#define jit_classify(code) _jit_classify(_jit, code) +extern jit_int32_t +_jit_classify(jit_state_t*, jit_code_t); + +#define jit_regarg_p(n, r) _jit_regarg_p(_jit, n, r) +extern jit_bool_t +_jit_regarg_p(jit_state_t*, jit_node_t*, jit_int32_t); + +#define emit_ldxi(r0, r1, i0) _emit_ldxi(_jit, r0, r1, i0) +extern void +_emit_ldxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); + +#define emit_stxi(i0, r0, r1) _emit_stxi(_jit, i0, r0, r1) +extern void +_emit_stxi(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); + +#define emit_ldxi_d(r0, r1, i0) _emit_ldxi_d(_jit, r0, r1, i0) +extern void +_emit_ldxi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); + +#define emit_stxi_d(i0, r0, r1) _emit_stxi(_jit, i0, r0, r1) +extern void +_emit_stxi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); + +extern void jit_init_debug(); +extern void jit_finish_debug(); + +/* + * Externs + */ +extern jit_register_t _rvs[]; + +#endif /* _jit_private_h */ diff --git a/include/lightning/jit_x86.h b/include/lightning/jit_x86.h new file mode 100644 index 000000000..5091d5bf2 --- /dev/null +++ b/include/lightning/jit_x86.h @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _jit_x86_h +#define _jit_x86_h + +#define JIT_HASH_CONSTS 1 +#define JIT_NUM_OPERANDS 2 + +/* + * Types + */ +#define jit_sse2_p() jit_cpu.sse2 +#define jit_x87_reg_p(reg) ((reg) >= _ST0 && (reg) <= _ST7) + +#define JIT_RET _RAX +#define JIT_SP _RSP +#define JIT_FP _RBP +typedef enum { +#if __WORDSIZE == 32 +# define jit_arg_reg_p(i) 0 +# define jit_r(i) (_RAX + (i)) +# define jit_r_num() 3 +# define jit_v(i) (_RBX + (i)) +# define jit_v_num() 3 +# define jit_arg_reg_p(i) 0 +# define jit_f(i) (jit_cpu.sse2 ? _XMM0 + (i) : _ST0 + (i)) +# define jit_f_num() (jit_cpu.sse2 ? 8 : 6) +# define JIT_FRET _ST0 +# define JIT_R0 _RAX +# define JIT_R1 _RCX +# define JIT_R2 _RDX + _RAX, _RCX, _RDX, +# define JIT_V0 _RBX +# define JIT_V1 _RSI +# define JIT_V2 _RDI + _RBX, _RSI, _RDI, + _RSP, _RBP, +# define JIT_F0 (jit_sse2_p() ? _XMM0 : _ST0) +# define JIT_F1 (jit_sse2_p() ? _XMM1 : _ST1) +# define JIT_F2 (jit_sse2_p() ? _XMM2 : _ST2) +# define JIT_F3 (jit_sse2_p() ? _XMM3 : _ST3) +# define JIT_F4 (jit_sse2_p() ? _XMM4 : _ST4) +# define JIT_F5 (jit_sse2_p() ? _XMM5 : _ST5) +# define JIT_F6 (jit_sse2_p() ? _XMM6 : _ST6) +# define JIT_F7 (jit_sse2_p() ? _XMM7 : _ST7) + _XMM0, _XMM1, _XMM2, _XMM3, _XMM4, _XMM5, _XMM6, _XMM7, +# define jit_sse_reg_p(reg) ((reg) >= _XMM0 && (reg) <= _XMM7) +#else +# define jit_arg_reg_p(i) ((i) >= 0 && (i) < 6) +# define jit_r(i) (_RAX + (i)) +# define jit_r_num() 4 +# define jit_v(i) (_RBX + (i)) +# define jit_v_num() 4 +# define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) +# define jit_f(index) (_XMM0 + (index)) +# define jit_f_num() 8 +# define JIT_FRET _XMM0 +# define JIT_R0 _RAX +# define JIT_R1 _R10 +# define JIT_R2 _R11 +# define JIT_R3 _R12 + _RAX, _R10, _R11, _R12, +# define JIT_V0 _RBX +# define JIT_V1 _R13 +# define JIT_V2 _R14 +# define JIT_V3 _R15 + _RBX, _R13, _R14, _R15, +# define JIT_RA0 _RDI +# define JIT_RA1 _RSI +# define JIT_RA2 _RDX +# define JIT_RA3 _RCX +# define JIT_RA4 _R8 +# define JIT_RA5 _R9 + _R9, _R8, _RCX, _RDX, _RSI, _RDI, + _RSP, _RBP, +# define JIT_F0 _XMM8 +# define JIT_F1 _XMM9 +# define JIT_F2 _XMM10 +# define JIT_F3 _XMM11 +# define JIT_F4 _XMM12 +# define JIT_F5 _XMM13 +# define JIT_F6 _XMM14 +# define JIT_F7 _XMM15 + _XMM8, _XMM9, _XMM10, _XMM11, _XMM12, _XMM13, _XMM14, _XMM15, +# define JIT_FA0 _XMM0 +# define JIT_FA1 _XMM1 +# define JIT_FA2 _XMM2 +# define JIT_FA3 _XMM3 +# define JIT_FA4 _XMM4 +# define JIT_FA5 _XMM5 +# define JIT_FA6 _XMM6 +# define JIT_FA7 _XMM7 + _XMM7, _XMM6, _XMM5, _XMM4, _XMM3, _XMM2, _XMM1, _XMM0, +# define jit_sse_reg_p(reg) ((reg) >= _XMM8 && (reg) <= _XMM0) +#endif + _ST0, _ST1, _ST2, _ST3, _ST4, _ST5, _ST6, _ST7, +# define JIT_NOREG _NOREG + _NOREG, +} jit_reg_t; + +typedef struct { + /* x87 present */ + jit_uint32_t fpu : 1; + /* cmpxchg8b instruction */ + jit_uint32_t cmpxchg8b : 1; + /* cmov and fcmov branchless conditional mov */ + jit_uint32_t cmov : 1; + /* mmx registers/instructions available */ + jit_uint32_t mmx : 1; + /* sse registers/instructions available */ + jit_uint32_t sse : 1; + /* sse2 registers/instructions available */ + jit_uint32_t sse2 : 1; + /* sse3 instructions available */ + jit_uint32_t sse3 : 1; + /* pcmulqdq instruction */ + jit_uint32_t pclmulqdq : 1; + /* ssse3 suplemental sse3 instructions available */ + jit_uint32_t ssse3 : 1; + /* fused multiply/add using ymm state */ + jit_uint32_t fma : 1; + /* cmpxchg16b instruction */ + jit_uint32_t cmpxchg16b : 1; + /* sse4.1 instructions available */ + jit_uint32_t sse4_1 : 1; + /* sse4.2 instructions available */ + jit_uint32_t sse4_2 : 1; + /* movbe instruction available */ + jit_uint32_t movbe : 1; + /* popcnt instruction available */ + jit_uint32_t popcnt : 1; + /* aes instructions available */ + jit_uint32_t aes : 1; + /* avx instructions available */ + jit_uint32_t avx : 1; + /* lahf/sahf available in 64 bits mode */ + jit_uint32_t lahf : 1; +} jit_cpu_t; + +#if __WORDSIZE == 32 +typedef jit_int32_t jit_regset_t; +#else +typedef jit_int64_t jit_regset_t; +#endif + +/* + * Initialization + */ +extern jit_cpu_t jit_cpu; + +#endif /* _jit_x86_h */ diff --git a/lib/Makefile.am b/lib/Makefile.am new file mode 100644 index 000000000..a656f614e --- /dev/null +++ b/lib/Makefile.am @@ -0,0 +1,28 @@ +# +# Copyright 2000, 2001, 2002, 2012 Free Software Foundation, Inc. +# +# This is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# + +AM_CFLAGS = -I$(top_srcdir)/include -D_GNU_SOURCE $(LIGHTNING_CFLAGS) +liblightning_LTLIBRARIES = liblightning.la + +liblightningdir = $(libdir) +liblightning_la_SOURCES = \ + jit_disasm.c \ + jit_print.c \ + lightning.c + +EXTRA_DIST = \ + jit_x86.c \ + jit_x86-cpu.c \ + jit_x86-sse.c \ + jit_x86-x87.c diff --git a/lib/jit_disasm.c b/lib/jit_disasm.c new file mode 100644 index 000000000..8e231ba12 --- /dev/null +++ b/lib/jit_disasm.c @@ -0,0 +1,297 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#include +#include +#include + +/* + * Prototypes + */ +#if DISASSEMBLER +static int +disasm_compare_symbols(const void *ap, const void *bp); + +static void +disasm_print_address(bfd_vma addr, struct disassemble_info *info); + +static void +disassemble(jit_pointer_t code, jit_int32_t length); +#endif + +/* + * Initialization + */ +#if DISASSEMBLER +static bfd *disasm_bfd; +static disassemble_info disasm_info; +static disassembler_ftype disasm_print; +static asymbol **disasm_symbols; +static asymbol *disasm_synthetic; +static long disasm_num_symbols; +static long disasm_num_synthetic; +#define disasm_stream stdout +#endif + +/* + * Implementation + */ +void +jit_init_debug(void) +{ +#if DISASSEMBLER + bfd_init(); + + /* FIXME */ + disasm_bfd = bfd_openr("/proc/self/exe", NULL); + + assert(disasm_bfd); + bfd_check_format(disasm_bfd, bfd_object); + bfd_check_format(disasm_bfd, bfd_archive); + disasm_print = disassembler(disasm_bfd); + assert(disasm_print); + INIT_DISASSEMBLE_INFO(disasm_info, disasm_stream, fprintf); +# if defined(__i386__) || defined(__x86_64__) + disasm_info.arch = bfd_arch_i386; +# if defined(__x86_64__) + disasm_info.mach = bfd_mach_x86_64; +# else + disasm_info.mach = bfd_mach_i386_i386; +# endif +# endif +# if defined(__arm__) + /* FIXME add mapping for prolog switching to arm and possible jump + * before first prolog also in arm mode */ + if (jit_cpu.thumb) + disasm_info.disassembler_options = "force-thumb"; +# endif + disasm_info.print_address_func = disasm_print_address; + + if (bfd_get_file_flags(disasm_bfd) & HAS_SYMS) { + asymbol **in; + asymbol **out; + asymbol *symbol; + long offset; + long sym_count; + long dyn_count; + long sym_storage; + long dyn_storage; + + sym_storage = bfd_get_symtab_upper_bound(disasm_bfd); + assert(sym_storage >= 0); + + if (bfd_get_file_flags(disasm_bfd) & DYNAMIC) { + dyn_storage = bfd_get_dynamic_symtab_upper_bound(disasm_bfd); + assert(dyn_storage >= 0); + } + else + dyn_storage = 0; + + disasm_symbols = malloc(sym_storage + dyn_storage); + sym_count = bfd_canonicalize_symtab(disasm_bfd, disasm_symbols); + assert(sym_count >= 0); + if (dyn_storage) { + dyn_count = bfd_canonicalize_dynamic_symtab(disasm_bfd, + disasm_symbols + + sym_count); + assert(dyn_count >= 0); + } + else + dyn_count = 0; + disasm_num_symbols = sym_count + dyn_count; + + disasm_num_synthetic = bfd_get_synthetic_symtab(disasm_bfd, + sym_count, + disasm_symbols, + dyn_count, + disasm_symbols + + sym_count, + &disasm_synthetic); + if (disasm_num_synthetic > 0) { + disasm_symbols = realloc(disasm_symbols, + sym_storage + dyn_storage + + disasm_num_synthetic * sizeof(asymbol *)); + for (offset = 0; offset < disasm_num_synthetic; offset++) + disasm_symbols[disasm_num_symbols++] = + disasm_synthetic + offset; + } + + /* remove symbols not useful for disassemble */ + in = out = disasm_symbols; + for (offset = 0; offset < disasm_num_symbols; offset++) { + symbol = *in++; + if (symbol->name && + symbol->name[0] != '\0' && + !(symbol->flags & (BSF_DEBUGGING | BSF_SECTION_SYM)) && + !bfd_is_und_section(symbol->section) && + !bfd_is_com_section(symbol->section)) + *out++ = symbol; + } + disasm_num_symbols = out - disasm_symbols; + qsort(disasm_symbols, disasm_num_symbols, + sizeof(asymbol *), disasm_compare_symbols); + } +#endif +} + +void +jit_finish_debug(void) +{ +#if DISASSEMBLER + if (disasm_synthetic) + free(disasm_synthetic); + if (disasm_symbols) + free(disasm_symbols); +#endif +} + +void +_jit_disassemble(jit_state_t *_jit) +{ +#if DISASSEMBLER + disassemble(_jit->code.ptr, _jit->pc.uc - _jit->code.ptr); +#endif +} + +#if DISASSEMBLER +/* Based on objdump source */ +static int +disasm_compare_symbols(const void *ap, const void *bp) +{ + const asymbol *a = *(const asymbol **)ap; + const asymbol *b = *(const asymbol **)bp; + + if (bfd_asymbol_value(a) > bfd_asymbol_value(b)) + return (1); + if (bfd_asymbol_value(a) < bfd_asymbol_value(b)) + return (-1); + return (0); +} + +#if __WORDSIZE == 32 +# define address_buffer_length 16 +# define address_buffer_format "%llx" +#else +# define address_buffer_length 32 +# define address_buffer_format "%lx" +#endif +static void +disasm_print_address(bfd_vma addr, struct disassemble_info *info) +{ + char buffer[address_buffer_length]; + + sprintf(buffer, address_buffer_format, (jit_word_t)addr); + (*info->fprintf_func)(info->stream, "0x%s", buffer); + + if (disasm_num_symbols) { + long low; + long high; + long offset; + asymbol *symbol; + + low = 0; + high = disasm_num_symbols; + do { + offset = (low + high) >> 1; + symbol = disasm_symbols[offset]; + if (bfd_asymbol_value(symbol) > addr) + high = offset - 1; + else if (bfd_asymbol_value(symbol) < addr) + low = offset + 1; + else + break; + } while (low < high); + + if (offset >= 0 && offset < disasm_num_symbols) { + if (bfd_asymbol_value(symbol) < addr) { + while (++offset < disasm_num_symbols) { + symbol = disasm_symbols[offset]; + if (bfd_asymbol_value(symbol) >= addr) + break; + } + } + else if (bfd_asymbol_value(symbol) > addr) { + while (offset--) { + if (bfd_asymbol_value(disasm_symbols[offset]) < addr) + break; + symbol = disasm_symbols[offset]; + } + } + if (bfd_asymbol_value(symbol) == addr) + (*info->fprintf_func)(info->stream, " # %s", symbol->name); + } + } +} + +static void +disassemble(jit_pointer_t code, jit_int32_t length) +{ + int bytes; +#if __arm__ + jit_data_info_t *data_info; + jit_int32_t data_offset; +#endif + bfd_vma pc = (jit_uword_t)code; + bfd_vma end = (jit_uword_t)code + length; + char buffer[address_buffer_length]; + +#if __arm__ + data_info = _jit->data_info; + data_offset = 0; +#endif + disasm_info.buffer = code; + disasm_info.buffer_vma = (jit_uword_t)code; + disasm_info.buffer_length = length; + while (pc < end) { +#if __arm__ + again: + if (data_info) { + while (data_info.ptr[data_offset].code < pc) { + data_offset += 2; + if (data_offset >= data_info.length) { + data_info = NULL; + goto again; + } + } + if (pc == data_info.ptr[data_offset].code) { + line = data_info.ptr[data_offset].length; + for (; line >= 4; line -= 4, pc += 4) { + bytes = sprintf(buffer, address_buffer_format, pc); + (*disasm_info.fprintf_func)(disasm_stream, + "%*c0x%s\t.data\t0x%08x\n", + 16 - bytes, ' ', buffer, + *(jit_uint32_t *) + (jit_uint32_t)pc); + } + /* reset disassemble information instead of attempting + * to hack the arm specific backend data structures to + * tell it to forward the required number of bytes. */ + disasm_info.buffer = (jit_pointer_t)(jit_uint32_t)pc; + disasm_info.buffer_vma = (jit_uword_t)pc; + if ((disasm_info.buffer_length = end - pc) <= 0) + break; + } + } +#endif + bytes = sprintf(buffer, address_buffer_format, (jit_word_t)pc); + (*disasm_info.fprintf_func)(disasm_stream, "%*c0x%s\t", + 16 - bytes, ' ', buffer); + pc += (*disasm_print)(pc, &disasm_info); + putc('\n', disasm_stream); + } +} +#endif diff --git a/lib/jit_print.c b/lib/jit_print.c new file mode 100644 index 000000000..01a5c783d --- /dev/null +++ b/lib/jit_print.c @@ -0,0 +1,427 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#include +#include + +#define print_chr(value) fputc(value, stdout) +#define print_hex(value) fprintf(stdout, "0x%lx", value) +#define print_dec(value) fprintf(stdout, "%ld", value) +#define print_flt(value) fprintf(stdout, "%g", value) +#define print_str(value) fprintf(stdout, "%s", value) +#define print_ptr(value) fprintf(stdout, "%p", value) +#define print_reg(value) \ + do { \ + if ((value) & jit_regno_patch) \ + print_chr('?'); \ + print_str(_rvs[jit_regno(value)].name); \ + } while (0) + +/* + * Initialization + */ +static char *code_name[] = { + "data", + "save", "load", + "#note", + "label", + "prolog", + "addr", "addi", + "addxr", "addxi", + "addcr", "addci", + "subr", "subi", + "subxr", "subxi", + "subcr", "subci", + "mulr", "muli", + "divr", "divi", + "divr_u", "divi_u", + "remr", "remi", + "remr_u", "remi_u", + "andr", "andi", + "orr", "ori", + "xorr", "xori", + "lshr", "lshi", + "rshr", "rshi", + "rshr_u", "rshi_u", + "negr", "comr", + "ltr", "lti", + "ltr_u", "lti_u", + "ler", "lei", + "ler_u", "lei_u", + "eqr", "eqi", + "ger", "gei", + "ger_u", "gei_u", + "gtr", "gti", + "gtr_u", "gti_u", + "ner", "nei", + "movr", "movi", + "extr_c", "extr_uc", + "extr_s", "extr_us", + "extr_i", "extr_ui", + "htonr", + "ldr_c", "ldi_c", + "ldr_uc", "ldi_uc", + "ldr_s", "ldi_s", + "ldr_us", "ldi_us", + "ldr_i", "ldi_i", + "ldr_ui", "ldi_ui", + "ldr_l", "ldi_l", + "ldxr_c", "ldxi_c", + "ldxr_uc", "ldxi_uc", + "ldxr_s", "ldxi_s", + "ldxr_us", "ldxi_us", + "ldxr_i", "ldxi_i", + "ldxr_ui", "ldxi_ui", + "ldxr_l", "ldxi_l", + "str_c", "sti_c", + "str_s", "sti_s", + "str_i", "sti_i", + "str_l", "sti_l", + "stxr_c", "stxi_c", + "stxr_s", "stxi_s", + "stxr_i", "stxi_i", + "stxr_l", "stxi_l", + "bltr", "blti", + "bltr_u", "blti_u", + "bler", "blei", + "bler_u", "blei_u", + "beqr", "beqi", + "bger", "bgei", + "bger_u", "bgei_u", + "bgtr", "bgti", + "bgtr_u", "bgti_u", + "bner", "bnei", + "bmsr", "bmsi", + "bmcr", "bmci", + "boaddr", "boaddi", + "boaddr_u", "boaddi_u", + "bxaddr", "bxaddi", + "bxaddr_u", "bxaddi_u", + "bosubr", "bosubi", + "bosubr_u", "bosubi_u", + "bxsubr", "bxsubi", + "bxsubr_u", "bxsubi_u", + "jmpr", "jmpi", + "callr", "calli", + "epilog", + "addr_f", "addi_f", + "subr_f", "subi_f", + "mulr_f", "muli_f", + "divr_f", "divi_f", + "negr_f", "absr_f", + "sqrtr_f", + "ltr_f", "lti_f", + "ler_f", "lei_f", + "eqr_f", "eqi_f", + "ger_f", "gei_f", + "gtr_f", "gti_f", + "ner_f", "nei_f", + "unltr_f", "unlti_f", + "unler_f", "unlei_f", + "uneqr_f", "uneqi_f", + "unger_f", "ungei_f", + "ungtr_f", "ungti_f", + "ltgtr_f", "ltgti_f", + "ordr_f", "ordi_f", + "unordr_f", "unordi_f", + "truncr_f_i", "truncr_f_l", + "extr_f", "extr_d_f", + "movr_f", "movi_f", + "ldr_f", "ldi_f", + "ldxr_f", "ldxi_f", + "str_f", "sti_f", + "stxr_f", "stxi_f", + "bltr_f", "blti_f", + "bler_f", "blei_f", + "beqr_f", "beqi_f", + "bger_f", "bgei_f", + "bgtr_f", "bgti_f", + "bner_f", "bnei_f", + "bunltr_f", "bunlti_f", + "bunler_f", "bunlei_f", + "buneqr_f", "buneqi_f", + "bunger_f", "bungei_f", + "bungtr_f", "bungti_f", + "bltgtr_f", "bltgti_f", + "bordr_f", "bordi_f", + "bunordr_f", "bunordi_f", + "retval_f", + "addr_d", "addi_d", + "subr_d", "subi_d", + "mulr_d", "muli_d", + "divr_d", "divi_d", + "negr_d", "absr_d", + "sqrtr_d", + "ltr_d", "lti_d", + "ler_d", "lei_d", + "eqr_d", "eqi_d", + "ger_d", "gei_d", + "gtr_d", "gti_d", + "ner_d", "nei_d", + "unltr_d", "unlti_d", + "unler_d", "unlei_d", + "uneqr_d", "uneqi_d", + "unger_d", "ungei_d", + "ungtr_d", "ungti_d", + "ltgtr_d", "ltgti_d", + "ordr_d", "ordi_d", + "unordr_d", "unordi_d", + "truncr_d_i", "truncr_d_l", + "extr_d", "extr_f_d", + "movr_d", "movi_d", + "ldr_d", "ldi_d", + "ldxr_d", "ldxi_d", + "str_d", "sti_d", + "stxr_d", "stxi_d", + "bltr_d", "blti_d", + "bler_d", "blei_d", + "beqr_d", "beqi_d", + "bger_d", "bgei_d", + "bgtr_d", "bgti_d", + "bner_d", "bnei_d", + "bunltr_d", "bunlti_d", + "bunler_d", "bunlei_d", + "buneqr_d", "buneqi_d", + "bunger_d", "bungei_d", + "bungtr_d", "bungti_d", + "bltgtr_d", "bltgti_d", + "bordr_d", "bordi_d", + "bunordr_d", "bunordi_d", + "retval_d", +}; + +/* + * Implementation + */ +void +_jit_print(jit_state_t *_jit) +{ + jit_node_t *node; + jit_block_t *block; + jit_bool_t first; + jit_int32_t value; + jit_int32_t offset; + + first = 0; + for (node = _jit->head; node; node = node->next) { + if (!first) + print_chr('\n'); + else + first = 0; + if (node->code == jit_code_label || + node->code == jit_code_prolog || node->code == jit_code_epilog) { + print_chr('L'); + print_dec(node->v.w); + print_chr(':'); + block = _jit->blocks.ptr + node->v.w; + for (offset = 0; offset < _jit->reglen; offset++) { + if (jit_regset_tstbit(block->reglive, offset)) { + print_chr(' '); + print_reg(offset); + } + } + if (node->code == jit_code_prolog || + node->code == jit_code_epilog) { + print_str(" /* "); + print_str(code_name[node->code]); + print_str(" */"); + } + continue; + } + value = jit_classify(node->code) & + (jit_cc_a0_int|jit_cc_a0_jmp|jit_cc_a0_reg| + jit_cc_a1_reg|jit_cc_a1_int|jit_cc_a1_flt|jit_cc_a1_dbl| + jit_cc_a2_reg|jit_cc_a2_int|jit_cc_a2_flt|jit_cc_a2_dbl); + if (value & jit_cc_a0_jmp) + print_str(" "); + else + print_chr('\t'); + print_str(code_name[node->code]); + switch (node->code) { + r: + print_chr(' '); print_reg(node->u.w); continue; + n: + print_chr(' '); + if (!(node->flag & jit_flag_node)) + print_ptr(node->u.p); + else { + print_chr('L'); + print_dec(node->u.n->v.w); + } + continue; + r_r: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_reg(node->v.w); continue; + r_w: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_hex(node->v.w); continue; + r_f: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); + if (node->flag & jit_flag_data) + print_flt(*(jit_float32_t *)node->v.n->u.w); + else + print_flt(node->v.f); + continue; + r_d: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); + if (node->flag & jit_flag_data) + print_flt(*(jit_float64_t *)node->v.n->u.w); + else + print_flt(node->v.d); + continue; + w_r: + print_chr(' '); print_hex(node->u.w); + print_chr(' '); print_reg(node->v.w); continue; + r_r_r: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_reg(node->v.w); + print_chr(' '); print_reg(node->w.w); continue; + r_r_w: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_reg(node->v.w); + print_chr(' '); print_hex(node->w.w); continue; + r_r_f: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_reg(node->v.w); + print_chr(' '); + if (node->flag & jit_flag_data) + print_flt(*(jit_float32_t *)node->w.n->u.w); + else + print_flt(node->w.f); + continue; + r_r_d: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_reg(node->v.w); + print_chr(' '); + if (node->flag & jit_flag_data) + print_flt(*(jit_float64_t *)node->w.n->u.w); + else + print_flt(node->w.d); + continue; + w_r_r: + print_chr(' '); print_hex(node->u.w); + print_chr(' '); print_reg(node->v.w); + print_chr(' '); print_reg(node->w.w); continue; + n_r_r: + print_chr(' '); + if (!(node->flag & jit_flag_node)) + print_ptr(node->u.p); + else { + print_chr('L'); + print_dec(node->u.n->v.w); + } + print_chr(' '); print_reg(node->v.w); + print_chr(' '); print_reg(node->w.w); continue; + n_r_w: + print_chr(' '); + if (!(node->flag & jit_flag_node)) + print_ptr(node->u.p); + else { + print_chr('L'); + print_dec(node->u.n->v.w); + } + print_chr(' '); print_reg(node->v.w); + print_chr(' '); print_hex(node->w.w); continue; + n_r_f: + print_chr(' '); + if (!(node->flag & jit_flag_node)) + print_ptr(node->u.p); + else{ + print_chr('L'); + print_dec(node->u.n->v.w); + } + print_chr(' '); print_reg(node->v.w); + print_chr(' '); + if (node->flag & jit_flag_data) + print_flt(*(jit_float32_t *)node->w.n->u.w); + else + print_flt(node->w.f); + continue; + n_r_d: + print_chr(' '); + if (!(node->flag & jit_flag_node)) + print_ptr(node->u.p); + else { + print_chr('L'); + print_dec(node->u.n->v.w); + } + print_chr(' '); print_reg(node->v.w); + print_chr(' '); + if (node->flag & jit_flag_data) + print_flt(*(jit_float64_t *)node->w.n->u.w); + else + print_flt(node->w.d); + continue; + + case jit_code_note: + /* FIXME should be name:line information */ + print_chr(' '); + print_ptr(node->v.p); + break; + + case jit_code_data: + case jit_code_label: + case jit_code_prolog: case jit_code_epilog: + break; + case jit_code_save: case jit_code_load: + goto r; + default: + switch (value) { + case jit_cc_a0_reg: + case jit_cc_a0_reg|jit_cc_a0_chg: + case jit_cc_a0_reg|jit_cc_a0_jmp: + goto r; + case jit_cc_a0_jmp: + goto n; + case jit_cc_a0_reg|jit_cc_a1_reg: + goto r_r; + case jit_cc_a0_reg|jit_cc_a1_int: + goto r_w; + case jit_cc_a0_reg|jit_cc_a1_flt: + goto r_f; + case jit_cc_a0_reg|jit_cc_a1_dbl: + goto r_d; + case jit_cc_a0_int|jit_cc_a1_reg: + goto w_r; + case jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg: + goto r_r_r; + case jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_int: + goto r_r_w; + case jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_flt: + goto r_r_f; + case jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_dbl: + goto r_r_d; + case jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg: + goto w_r_r; + case jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg: + goto n_r_r; + case jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int: + goto n_r_w; + case jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt: + goto n_r_f; + case jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl: + goto n_r_d; + default: + abort(); + } + break; + } + } + print_chr('\n'); +} diff --git a/lib/jit_x86-cpu.c b/lib/jit_x86-cpu.c new file mode 100644 index 000000000..b40523ac1 --- /dev/null +++ b/lib/jit_x86-cpu.c @@ -0,0 +1,3118 @@ +/* + * Copyright (C) 2011 Paulo Cesar Pereira de Andrade. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +/* avoid using it due to partial stalls */ +#define USE_INC_DEC 0 + +#if PROTO +# if __WORDSIZE == 32 +# define stack_alignment 4 +# define stack_framesize 20 +# define ldi(u, v) ldi_i(u, v) +# define ldxi(u, v, w) ldxi_i(u, v, w) +# define sti(u, v) sti_i(u, v) +# define stxi(u, v, w) stxi_i(u, v, w) +# define can_sign_extend_int_p(im) true +# define can_zero_extend_int_p(im) true +# define fits_uint32_p(im) true +# define reg8_p(rn) \ + ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO) +# else +# define stack_alignment 8 +# define stack_framesize 56 +# define ldi(u, v) ldi_l(u, v) +# define ldxi(u, v, w) ldxi_l(u, v, w) +# define sti(u, v) sti_l(u, v) +# define stxi(u, v, w) stxi_l(u, v, w) +# define can_sign_extend_int_p(im) \ + (((im) >= 0 && (long)(im) <= 0x7fffffffL) || \ + ((im) < 0 && (long)(im) >= -0x80000000L)) +# define can_zero_extend_int_p(im) \ + ((im) >= 0 && (im) < 0x80000000L) +# define fits_uint32_p(im) ((im & 0xffffffff00000000L) == 0) +# define reg8_p(rn) 1 +# endif +# define _RAX_REGNO 0 +# define _RCX_REGNO 1 +# define _RDX_REGNO 2 +# define _RBX_REGNO 3 +# define _RSP_REGNO 4 +# define _RBP_REGNO 5 +# define _RSI_REGNO 6 +# define _RDI_REGNO 7 +# define _R8_REGNO 8 +# define _R9_REGNO 9 +# define _R10_REGNO 10 +# define _R11_REGNO 11 +# define _R12_REGNO 12 +# define _R13_REGNO 13 +# define _R14_REGNO 14 +# define _R15_REGNO 15 +# define r7(reg) (reg & 7) +# define _SCL1 0x00 +# define _SCL2 0x01 +# define _SCL4 0x02 +# define _SCL8 0x03 +# define X86_ADD 0 +# define X86_OR 1 << 3 +# define X86_ADC 2 << 3 +# define X86_SBB 3 << 3 +# define X86_AND 4 << 3 +# define X86_SUB 5 << 3 +# define X86_XOR 6 << 3 +# define X86_CMP 7 << 3 +# define X86_ROL 0 +# define X86_ROR 1 +# define X86_RCL 2 +# define X86_RCR 3 +# define X86_SHL 4 +# define X86_SHR 5 +# define X86_SAR 7 +# define X86_NOT 2 +# define X86_NEG 3 +# define X86_MUL 4 +# define X86_IMUL 5 +# define X86_DIV 6 +# define X86_IDIV 7 +# define X86_CC_O 0x0 +# define X86_CC_NO 0x1 +# define X86_CC_NAE 0x2 +# define X86_CC_B 0x2 +# define X86_CC_C 0x2 +# define X86_CC_AE 0x3 +# define X86_CC_NB 0x3 +# define X86_CC_NC 0x3 +# define X86_CC_E 0x4 +# define X86_CC_Z 0x4 +# define X86_CC_NE 0x5 +# define X86_CC_NZ 0x5 +# define X86_CC_BE 0x6 +# define X86_CC_NA 0x6 +# define X86_CC_A 0x7 +# define X86_CC_NBE 0x7 +# define X86_CC_S 0x8 +# define X86_CC_NS 0x9 +# define X86_CC_P 0xa +# define X86_CC_PE 0xa +# define X86_CC_NP 0xb +# define X86_CC_PO 0xb +# define X86_CC_L 0xc +# define X86_CC_NGE 0xc +# define X86_CC_GE 0xd +# define X86_CC_NL 0xd +# define X86_CC_LE 0xe +# define X86_CC_NG 0xe +# define X86_CC_G 0xf +# define X86_CC_NLE 0xf +# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m +# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b +# define ic(c) *_jit->pc.uc++ = c +# define is(s) *_jit->pc.us++ = s +# define ii(i) *_jit->pc.ui++ = i +# if __WORDSIZE == 64 +# define il(l) *_jit->pc.ul++ = l +# endif +# define patch_abs(instr, label) \ + *(jit_word_t *)(instr - sizeof(jit_word_t)) = label +# define patch_rel(instr, label) \ + *(jit_int32_t *)(instr - 4) = label - instr +# define patch_rel_char(instr, label) \ + *(jit_int8_t *)(instr - 1) = label - instr +# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b) +static void +_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms) +static void +_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define nop(n) _nop(_jit, n) +static void _nop(jit_state_t*, jit_int32_t); +# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd) +static void +_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define pushr(r0) _pushr(_jit, r0) +static void _pushr(jit_state_t*, jit_int32_t) maybe_unused; +# define popr(r0) _popr(_jit, r0) +static void _popr(jit_state_t*, jit_int32_t) maybe_unused; +# define xchgr(r0, r1) _xchgr(_jit, r0, r1) +static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t); +# define testr(r0, r1) _testr(_jit, r0, r1) +static void _testr(jit_state_t*, jit_int32_t, jit_int32_t); +# define testi(r0, i0) _testi(_jit, r0, i0) +static void _testi(jit_state_t*, jit_int32_t, jit_word_t); +# define cc(code, r0) _cc(_jit, code, r0) +static void _cc(jit_state_t*, jit_int32_t, jit_int32_t); +# define icmpr(r0, r1) alur(X86_CMP, r0, r1) +# define alur(code, r0, r1) _alur(_jit, code, r0, r1) +static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define icmpi(r0, i0) alui(X86_CMP, r0, i0) +# define alui(code, r0, i0) _alui(_jit, code, r0, i0) +static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define iaddr(r0, r1) alur(X86_ADD, r0, r1) +# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2) +static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define iaddi(r0, i0) alui(X86_ADD, r0, i0) +# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0) +static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2) +static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0) +static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define iaddxr(r0, r1) alur(X86_ADC, r0, r1) +# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2) +static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define iaddxi(r0, i0) alui(X86_ADC, r0, i0) +# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0) +static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define isubr(r0, r1) alur(X86_SUB, r0, r1) +# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2) +static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define isubi(r0, i0) alui(X86_SUB, r0, i0) +# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0) +static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2) +static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0) +static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); +# define isubxr(r0, r1) alur(X86_SBB, r0, r1) +# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2) +static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define isubxi(r0, i0) alui(X86_SBB, r0, i0) +# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0) +static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); +# define imulr(r0, r1) _imulr(_jit, r0, r1) +static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t); +# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0) +static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2) +static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0) +static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define idivr(r0) unr(X86_IDIV, r0) +# define idivr_u(r0) unr(X86_DIV, r0) +# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit) +static void _sign_extend_rdx_rax(jit_state_t*); +# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1) +static void +_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t, + jit_bool_t,jit_bool_t); +# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2) +static void +_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t); +# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1) +# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1) +# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1) +# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1) +# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0) +# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0) +# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0) +# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0) +# define iandr(r0, r1) alur(X86_AND, r0, r1) +# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2) +static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define iandi(r0, i0) alui(X86_AND, r0, i0) +# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0) +static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t); +# define iorr(r0, r1) alur(X86_OR, r0, r1) +# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2) +static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t); +# define iori(r0, i0) alui(X86_OR, r0, i0) +# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0) +static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t); +# define ixorr(r0, r1) alur(X86_XOR, r0, r1) +# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2) +static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t); +# define ixori(r0, i0) alui(X86_XOR, r0, i0) +# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0) +static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t); +# define irotshr(code, r0) _irotshr(_jit, code, r0) +static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t); +# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2) +static void +_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0) +static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0) +static void +_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t); +# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2) +# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0) +static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2) +# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0) +# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2) +# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0) +# define unr(code, r0) _unr(_jit, code, r0) +static void _unr(jit_state_t*, jit_int32_t, jit_int32_t); +# define inegr(r0) unr(X86_NEG, r0) +# define negr(r0, r1) _negr(_jit, r0, r1) +static void _negr(jit_state_t*, jit_int32_t, jit_int32_t); +# define icomr(r0) unr(X86_NOT, r0) +# define comr(r0, r1) _comr(_jit, r0, r1) +static void _comr(jit_state_t*, jit_int32_t, jit_int32_t); +# if USE_INC_DEC +# define incr(r0, r1) _incr(_jit, r0, r1) +static void _incr(jit_state_t*, jit_int32_t, jit_int32_t); +# define decr(r0, r1) _decr(_jit, r0, r1) +static void _decr(jit_state_t*, jit_int32_t, jit_int32_t); +# endif +# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2) +static void +_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t); +# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0) +static void +_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t); +# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1) +static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2) +static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0) +static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2) +static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0) +# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2) +static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0) +# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2) +static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0) +static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2) +static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0) +static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2) +static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0) +static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2) +static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0) +static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2) +static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0) +# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2) +static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0) +static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2) +static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0) +static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define movr(r0, r1) _movr(_jit, r0, r1) +static void _movr(jit_state_t*, jit_int32_t, jit_int32_t); +# define imovi(r0, i0) _imovi(_jit, r0, i0) +static void _imovi(jit_state_t*, jit_int32_t, jit_word_t); +# define movi(r0, i0) _movi(_jit, r0, i0) +static void _movi(jit_state_t*, jit_int32_t, jit_word_t); +# define movi_p(r0, i0) _movi_p(_jit, r0, i0) +static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t); +# define movcr(r0, r1) _movcr(_jit, r0, r1) +static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t); +# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1) +static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t); +# define movsr(r0, r1) _movsr(_jit, r0, r1) +static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t); +# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1) +static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t); +# if __WORDSIZE == 64 +# define movir(r0, r1) _movir(_jit, r0, r1) +static void _movir(jit_state_t*,jit_int32_t,jit_int32_t); +# define movir_u(r0, r1) _movir_u(_jit, r0, r1) +static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t); +# endif +#define htonr(r0, r1) _htonr(_jit, r0, r1) +static void _htonr(jit_state_t*,jit_int32_t,jit_int32_t); +# define extr_c(r0, r1) _extr_c(_jit, r0, r1) +static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); +# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1) +static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t); +# define extr_s(r0, r1) movsr(r0, r1) +# define extr_us(r0, r1) movsr_u(r0, r1) +# if __WORDSIZE == 64 +# define extr_i(r0, r1) movir(r0, r1) +# define extr_ui(r0, r1) movir_u(r0, r1) +# endif +# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1) +static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0) +static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1) +static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0) +static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1) +static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0) +static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1) +static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0) +static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1) +static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0) +static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t); +# if __WORDSIZE == 64 +# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1) +static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0) +static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1) +static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0) +static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t); +# endif +# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2) +static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0) +static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2) +static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0) +static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2) +static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0) +static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2) +static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0) +static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2) +static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0) +static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# if __WORDSIZE == 64 +# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2) +static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0) +static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2) +static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0) +static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# endif +# define str_c(r0, r1) _str_c(_jit, r0, r1) +static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t); +# define sti_c(i0, r0) _sti_c(_jit, i0, r0) +static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t); +# define str_s(r0, r1) _str_s(_jit, r0, r1) +static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t); +# define sti_s(i0, r0) _sti_s(_jit, i0, r0) +static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t); +# define str_i(r0, r1) _str_i(_jit, r0, r1) +static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t); +# define sti_i(i0, r0) _sti_i(_jit, i0, r0) +static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t); +# if __WORDSIZE == 64 +# define str_l(r0, r1) _str_l(_jit, r0, r1) +static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t); +# define sti_l(i0, r0) _sti_l(_jit, i0, r0) +static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t); +# endif +# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2) +static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1) +static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2) +static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1) +static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2) +static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1) +static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# if __WORDSIZE == 64 +# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2) +static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1) +static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# endif +# define jcc(code, i0) _jcc(_jit, code, i0) +# define jo(i0) jcc(X86_CC_O, i0) +# define jno(i0) jcc(X86_CC_NO, i0) +# define jnae(i0) jcc(X86_CC_NAE, i0) +# define jb(i0) jcc(X86_CC_B, i0) +# define jc(i0) jcc(X86_CC_C, i0) +# define jae(i0) jcc(X86_CC_AE, i0) +# define jnb(i0) jcc(X86_CC_NB, i0) +# define jnc(i0) jcc(X86_CC_NC, i0) +# define je(i0) jcc(X86_CC_E, i0) +# define jz(i0) jcc(X86_CC_Z, i0) +# define jne(i0) jcc(X86_CC_NE, i0) +# define jnz(i0) jcc(X86_CC_NZ, i0) +# define jbe(i0) jcc(X86_CC_BE, i0) +# define jna(i0) jcc(X86_CC_NA, i0) +# define ja(i0) jcc(X86_CC_A, i0) +# define jnbe(i0) jcc(X86_CC_NBE, i0) +# define js(i0) jcc(X86_CC_S, i0) +# define jns(i0) jcc(X86_CC_NS, i0) +# define jp(i0) jcc(X86_CC_P, i0) +# define jpe(i0) jcc(X86_CC_PE, i0) +# define jnp(i0) jcc(X86_CC_NP, i0) +# define jpo(i0) jcc(X86_CC_PO, i0) +# define jl(i0) jcc(X86_CC_L, i0) +# define jnge(i0) jcc(X86_CC_NGE, i0) +# define jge(i0) jcc(X86_CC_GE, i0) +# define jnl(i0) jcc(X86_CC_NL, i0) +# define jle(i0) jcc(X86_CC_LE, i0) +# define jng(i0) jcc(X86_CC_NG, i0) +# define jg(i0) jcc(X86_CC_G, i0) +# define jnle(i0) jcc(X86_CC_NLE, i0) +static void _jcc(jit_state_t*, jit_int32_t, jit_word_t); +# define jccs(code, i0) _jccs(_jit, code, i0) +# define jos(i0) jccs(X86_CC_O, i0) +# define jnos(i0) jccs(X86_CC_NO, i0) +# define jnaes(i0) jccs(X86_CC_NAE, i0) +# define jbs(i0) jccs(X86_CC_B, i0) +# define jcs(i0) jccs(X86_CC_C, i0) +# define jaes(i0) jccs(X86_CC_AE, i0) +# define jnbs(i0) jccs(X86_CC_NB, i0) +# define jncs(i0) jccs(X86_CC_NC, i0) +# define jes(i0) jccs(X86_CC_E, i0) +# define jzs(i0) jccs(X86_CC_Z, i0) +# define jnes(i0) jccs(X86_CC_NE, i0) +# define jnzs(i0) jccs(X86_CC_NZ, i0) +# define jbes(i0) jccs(X86_CC_BE, i0) +# define jnas(i0) jccs(X86_CC_NA, i0) +# define jas(i0) jccs(X86_CC_A, i0) +# define jnbes(i0) jccs(X86_CC_NBE, i0) +# define jss(i0) jccs(X86_CC_S, i0) +# define jnss(i0) jccs(X86_CC_NS, i0) +# define jps(i0) jccs(X86_CC_P, i0) +# define jpes(i0) jccs(X86_CC_PE, i0) +# define jnps(i0) jccs(X86_CC_NP, i0) +# define jpos(i0) jccs(X86_CC_PO, i0) +# define jls(i0) jccs(X86_CC_L, i0) +# define jnges(i0) jccs(X86_CC_NGE, i0) +# define jges(i0) jccs(X86_CC_GE, i0) +# define jnls(i0) jccs(X86_CC_NL, i0) +# define jles(i0) jccs(X86_CC_LE, i0) +# define jngs(i0) jccs(X86_CC_NG, i0) +# define jgs(i0) jccs(X86_CC_G, i0) +# define jnles(i0) jccs(X86_CC_NLE, i0) +static void _jccs(jit_state_t*, jit_int32_t, jit_word_t); +# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1) +static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t); +# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1) +static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t); +# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0) +static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t); +# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1) +static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1) +static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1) +static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1) +static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1) +static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1) +static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1) +static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1) +static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1) +static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1) +static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1) +static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1) +static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1) +static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1) +static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1) +static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1) +static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1) +static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1) +static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1) +static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1) +static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1) +static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1) +static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1) +static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1) +static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1) +static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1) +static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1) +static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1) +static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1) +static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1) +static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1) +static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1) +static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1) +static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1) +static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1) +static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1) +static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1) +static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1) +static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1) +static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1) +static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define callr(r0) _callr(_jit, r0) +static void _callr(jit_state_t*, jit_int32_t); +# define calli(i0) _calli(_jit, i0) +static jit_word_t _calli(jit_state_t*, jit_word_t); +# define jmpr(r0) _jmpr(_jit, r0) +static void _jmpr(jit_state_t*, jit_int32_t); +# define jmpi(i0) _jmpi(_jit, i0) +static jit_word_t _jmpi(jit_state_t*, jit_word_t); +# define prolog(node) _prolog(_jit, node) +static void _prolog(jit_state_t*, jit_node_t*); +# define epilog(node) _epilog(_jit, node) +static void _epilog(jit_state_t*, jit_node_t*); +# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label) +static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t); +#endif + +#if CODE +static void +_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w, + jit_int32_t r, jit_int32_t x, jit_int32_t b) +{ +#if __WORDSIZE == 64 + jit_int32_t v = 0x40 | (w << 3); + + if (r != _NOREG) + v |= (r & 8) >> 1; + if (x != _NOREG) + v |= (x & 8) >> 2; + if (b != _NOREG) + v |= (b & 8) >> 3; + if (l || v != 0x40) + ic(v); +#endif +} + +static void +_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md, + jit_int32_t rb, jit_int32_t ri, jit_int32_t ms) +{ + if (ri == _NOREG) { + if (rb == _NOREG) { +#if __WORDSIZE == 32 + mrm(0x00, r7(rd), 0x05); +#else + mrm(0x00, r7(rd), 0x04); + sib(_SCL1, 0x04, 0x05); +#endif + ii(md); + } + else if (r7(rb) == _RSP_REGNO) { + if (md == 0) { + mrm(0x00, r7(rd), 0x04); + sib(ms, 0x04, 0x04); + } + else if ((jit_int8_t)md == md) { + mrm(0x01, r7(rd), 0x04); + sib(ms, 0x04, 0x04); + ic(md); + } + else { + mrm(0x02, r7(rd), 0x04); + sib(ms, 0x04, 0x04); + ii(md); + } + } + else { + if (md == 0 && r7(rb) != _RBP_REGNO) + mrm(0x00, r7(rd), r7(rb)); + else if ((jit_int8_t)md == md) { + mrm(0x01, r7(rd), r7(rb)); + ic(md); + } + else { + mrm(0x02, r7(rd), r7(rb)); + ii(md); + } + } + } + else if (rb == _NOREG) { + mrm(0x00, r7(rd), 0x04); + sib(ms, r7(ri), 0x05); + ii(md); + } + else if (r7(ri) != _RSP_REGNO) { + if (md == 0 && r7(rb) != _RBP_REGNO) { + mrm(0x00, r7(rd), 0x04); + sib(ms, r7(ri), r7(rb)); + } + else if ((jit_int8_t)md == md) { + mrm(0x01, r7(rd), 0x04); + sib(ms, r7(ri), r7(rb)); + ic(md); + } + else { + mrm(0x02, r7(rd), 0x04); + sib(ms, r7(ri), r7(rb)); + ic(md); + } + } + else { + fprintf(stderr, "illegal index register"); + abort(); + } +} + +static void +_nop(jit_state_t *_jit, jit_int32_t count) +{ + switch (count) { + case 0: + break; + case 1: /* NOP */ + ic(0x90); break; + case 2: /* 66 NOP */ + ic(0x66); ic(0x90); + break; + case 3: /* NOP DWORD ptr [EAX] */ + ic(0x0f); ic(0x1f); ic(0x00); + break; + case 4: /* NOP DWORD ptr [EAX + 00H] */ + ic(0x0f); ic(0x1f); ic(0x40); ic(0x00); + break; + case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */ + ic(0x0f); ic(0x1f); ic(0x44); ic(0x00); + ic(0x00); + break; + case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */ + ic(0x66); ic(0x0f); ic(0x1f); ic(0x44); + ic(0x00); ic(0x00); + break; + case 7: /* NOP DWORD ptr [EAX + 00000000H] */ + ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000); + break; + case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */ + ic(0x0f); ic(0x1f); ic(0x84); ic(0x00); + ii(0x0000); + break; + case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */ + ic(0x66); ic(0x0f); ic(0x1f); ic(0x84); + ic(0x00); ii(0x0000); + break; + default: + abort(); + } +} + +static void +_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb, + jit_int32_t ri, jit_int32_t ms, jit_int32_t rd) +{ + rex(0, 1, rd, ri, rb); + ic(0x8d); + rx(rd, md, rb, ri, ms); +} + +static void +_pushr(jit_state_t *_jit, jit_int32_t r0) +{ + rex(0, 0, 0, 0, r0); + ic(0x50 | r7(r0)); +} + +static void +_popr(jit_state_t *_jit, jit_int32_t r0) +{ + rex(0, 0, 0, 0, r0); + ic(0x58 | r7(r0)); +} + +static void +_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r1, _NOREG, r0); + ic(0x87); + mrm(0x03, r7(r1), r7(r0)); +} + +static void +_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r1, _NOREG, r0); + ic(0x85); + mrm(0x03, r7(r1), r7(r0)); +} + +static void +_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + rex(0, 1, _NOREG, _NOREG, r0); + if (r0 == _RAX_REGNO) + ic(0xa9); + else { + ic(0xf7); + mrm(0x03, 0x00, r7(r0)); + } + ii(i0); +} + +static void +_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0) +{ + rex(0, 0, _NOREG, _NOREG, r0); + ic(0x0f); + ic(0x90 | code); + mrm(0x03, 0x00, r7(r0)); +} + +static void +_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r1, _NOREG, r0); + ic(code | 0x01); + mrm(0x03, r7(r1), r7(r0)); +} + +static void +_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, _NOREG, _NOREG, r0); + if ((jit_int8_t)i0 == i0) { + ic(0x83); + ic(0xc0 | code | r7(r0)); + ic(i0); + } + else { + if (r0 == _RAX_REGNO) + ic(code | 0x05); + else { + ic(0x81); + ic(0xc0 | code | r7(r0)); + } + ii(i0); + } + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + alur(code, r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) + iaddr(r0, r2); + else if (r0 == r2) + iaddr(r0, r1); + else + lea(0, r1, r2, _SCL1, r0); +} + +static void +_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + movr(r0, r1); +#if USE_INC_DEC + else if (i0 == 1) + incr(r0, r1); + else if (i0 == -1) + decr(r0, r1); +#endif + else if (can_sign_extend_int_p(i0)) { + if (r0 == r1) + iaddi(r0, i0); + else + lea(i0, r1, _NOREG, _SCL1, r0); + } + else if (r0 != r1) { + movi(r0, i0); + iaddr(r0, r1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iaddr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r2) + iaddr(r0, r1); + else { + movr(r0, r1); + iaddr(r0, r2); + } +} + +static void +_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + movr(r0, r1); + iaddi(r0, i0); + } + else if (r0 == r1) { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iaddr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + movi(r0, i0); + iaddr(r0, r1); + } +} + +static void +_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r2) + iaddxr(r0, r1); + else { + movr(r0, r1); + iaddxr(r0, r2); + } +} + +static void +_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + movr(r0, r1); + iaddxi(r0, i0); + } + else if (r0 == r1) { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iaddxr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + movi(r0, i0); + iaddxr(r0, r1); + } +} + +static void +_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + ixorr(r0, r0); + else if (r0 == r2) { + isubr(r0, r1); + inegr(r0); + } + else { + movr(r0, r1); + isubr(r0, r2); + } +} + +static void +_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + movr(r0, r1); +#if USE_INC_DEC + else if (i0 == 1) + decr(r0, r1); + else if (i0 == -1) + incr(r0, r1); +#endif + else if (can_sign_extend_int_p(i0)) { + if (r0 == r1) + isubi(r0, i0); + else + lea(-i0, r1, _NOREG, _SCL1, r0); + } + else if (r0 != r1) { + movi(r0, i0); + isubr(r0, r1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + isubr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (r0 == r2 && r0 != r1) { + reg = jit_get_reg(jit_class_gpr); + movr(rn(reg), r0); + movr(r0, r1); + isubr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + movr(r0, r1); + isubr(r0, r2); + } +} + +static void +_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + movr(r0, r1); + if (can_sign_extend_int_p(i0)) + isubi(r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + isubr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (r0 == r2 && r0 != r1) { + reg = jit_get_reg(jit_class_gpr); + movr(rn(reg), r0); + movr(r0, r1); + isubxr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + movr(r0, r1); + isubxr(r0, r2); + } +} + +static void +_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + movr(r0, r1); + if (can_sign_extend_int_p(i0)) + isubxi(r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + imovi(rn(reg), i0); + isubxr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r1, _NOREG, r0); + ic(0x0f); + ic(0xaf); + mrm(0x03, r7(r1), r7(r0)); +} + +static void +_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r1, _NOREG, r0); + if ((jit_int8_t)i0 == i0) { + ic(0x6b); + mrm(0x03, r7(r0), r7(r1)); + ic(i0); + } + else { + ic(0x69); + mrm(0x03, r7(r0), r7(r1)); + ii(i0); + } + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + imulr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) + imulr(r0, r2); + else if (r0 == r2) + imulr(r0, r1); + else { + movr(r0, r1); + imulr(r0, r2); + } +} + +static void +_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + switch (i0) { + case 0: + ixorr(r0, r0); + break; + case 1: + movr(r0, r1); + break; + case -1: + negr(r0, r1); + break; + case 2: + lea(0, _NOREG, r1, _SCL2, r0); + break; + case 4: + lea(0, _NOREG, r1, _SCL4, r0); + break; + case 8: + lea(0, _NOREG, r1, _SCL8, r0); + break; + default: + if (i0 > 0 && !(i0 & (i0 - 1))) + lshi(r0, r1, ffsl(i0) - 1); + else if (can_sign_extend_int_p(i0)) + imuli(r0, r1, i0); + else if (r0 != r1) { + movi(r0, i0); + imulr(r0, r1); + } + else + imuli(r0, r0, i0); + break; + } +} + +static void +_sign_extend_rdx_rax(jit_state_t *_jit) +{ + rex(0, 1, 0, 0, 0); + ic(0x99); +} + +static void +_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, + jit_bool_t sign, jit_bool_t divide) +{ + jit_int32_t div; + jit_int32_t reg; + + if (r0 != _RDX_REGNO) + (void)jit_get_reg(_RDX|jit_class_gpr); + if (r0 != _RAX_REGNO) + (void)jit_get_reg(_RAX|jit_class_gpr); + + if (r2 == _RAX_REGNO) { + if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) { + if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG) + reg = jit_get_reg(r1 == _RCX_REGNO ? _RBX : _RCX); + div = rn(reg); + movr(div, _RAX_REGNO); + if (r1 != _RAX_REGNO) + movr(_RAX_REGNO, r1); + } + else { + if (r0 == r1) + xchgr(r0, _RAX_REGNO); + else { + if (r0 != _RAX_REGNO) + movr(r0, _RAX_REGNO); + if (r1 != _RAX_REGNO) + movr(_RAX_REGNO, r1); + } + div = r0; + reg = 0; + } + } + else if (r2 == _RDX_REGNO) { + if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) { + if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG) + reg = jit_get_reg(r1 == _RCX_REGNO ? _RBX : _RCX); + div = rn(reg); + movr(div, _RDX_REGNO); + if (r1 != _RAX_REGNO) + movr(_RAX_REGNO, r1); + } + else { + if (r1 != _RAX_REGNO) + movr(_RAX_REGNO, r1); + movr(r0, _RDX_REGNO); + div = r0; + reg = 0; + } + } + else { + if (r1 != _RAX_REGNO) + movr(_RAX_REGNO, r1); + div = r2; + reg = 0; + } + + if (sign) { + sign_extend_rdx_rax(); + idivr(div); + } + else { + ixorr(_RDX_REGNO, _RDX_REGNO); + idivr_u(div); + } + + if (reg) + jit_unget_reg(reg); + + if (r0 != _RAX_REGNO) { + if (divide) + movr(r0, _RAX_REGNO); + jit_unget_reg(_RAX); + } + if (r0 != _RDX_REGNO) { + if (!divide) + movr(r0, _RDX_REGNO); + jit_unget_reg(_RDX); + } +} + +static void +_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0, + jit_bool_t sign, jit_bool_t divide) +{ + jit_int32_t reg; + jit_int32_t div; + + if (divide) { + switch (i0) { + case 1: + movr(r0, r1); + return; + case -1: + if (sign) { + negr(r0, r1); + return; + } + break; + default: + if (i0 > 0 && !(i0 & (i0 - 1))) { + movr(r0, r1); + if (sign) + rshi(r0, r0, ffsl(i0) - 1); + else + rshi_u(r0, r0, ffsl(i0) - 1); + return; + } + break; + } + } + else if (i0 == 1 || (sign && i0 == -1)) { + ixorr(r0, r0); + return; + } + else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) { + if (can_sign_extend_int_p(i0)) { + movr(r0, r1); + iandi(r0, i0 - 1); + } + else if (r0 != r1) { + movi(r0, i0 - 1); + iandr(r0, r1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0 - 1); + iandr(r0, rn(reg)); + jit_unget_reg(reg); + } + return; + } + + if (r0 != _RDX_REGNO) + (void)jit_get_reg(_RDX|jit_class_gpr); + if (r0 != _RAX_REGNO) + (void)jit_get_reg(_RAX|jit_class_gpr); + + if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) { + if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG) + reg = jit_get_reg(_RCX); + div = rn(reg); + } + else { + reg = 0; + div = r0; + } + + movi(div, i0); + movr(_RAX, r1); + + if (sign) { + sign_extend_rdx_rax(); + idivr(div); + } + else { + ixorr(_RDX_REGNO, _RDX_REGNO); + idivr_u(div); + } + + if (reg) + jit_unget_reg(reg); + + if (r0 != _RAX_REGNO) { + if (divide) + movr(r0, _RAX_REGNO); + jit_unget_reg(_RAX); + } + if (r0 != _RDX_REGNO) { + if (!divide) + movr(r0, _RDX_REGNO); + jit_unget_reg(_RDX); + } +} + +static void +_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movr(r0, r1); + else if (r0 == r1) + iandr(r0, r2); + else if (r0 == r2) + iandr(r0, r1); + else { + movr(r0, r1); + iandr(r0, r2); + } +} + +static void +_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + + if (i0 == 0) + ixorr(r0, r0); + else if (i0 == -1) + movr(r0, r1); + else if (r0 == r1) { + if (can_sign_extend_int_p(i0)) + iandi(r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iandr(r0, rn(reg)); + jit_unget_reg(r0); + } + } + else { + movi(r0, i0); + iandr(r0, r1); + } +} + +static void +_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movr(r0, r1); + else if (r0 == r1) + iorr(r0, r2); + else if (r0 == r2) + iorr(r0, r1); + else { + movr(r0, r1); + iorr(r0, r2); + } +} + +static void +_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + movr(r0, r1); + else if (i0 == -1) + movi(r0, -1); + else if (can_sign_extend_int_p(i0)) { + movr(r0, r1); + iori(r0, i0); + } + else if (r0 != r1) { + movi(r0, i0); + ixorr(r0, r1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iorr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + ixorr(r0, r0); + else if (r0 == r1) + ixorr(r0, r2); + else if (r0 == r2) + ixorr(r0, r1); + else { + movr(r0, r1); + ixorr(r0, r2); + } +} + +static void +_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + movr(r0, r1); + else if (i0 == -1) + comr(r0, r1); + else if (can_sign_extend_int_p(i0)) { + movr(r0, r1); + ixori(r0, i0); + } + else if (r0 != r1) { + movi(r0, i0); + ixorr(r0, r1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ixorr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0) +{ + rex(0, 1, _RCX_REGNO, _NOREG, r0); + ic(0xd3); + mrm(0x03, code, r7(r0)); +} + +static void +_rotshr(jit_state_t *_jit, jit_int32_t code, + jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + + if (r0 == _RCX_REGNO) { + reg = jit_get_reg(jit_class_gpr); + movr(rn(reg), r1); + if (r2 != _RCX_REGNO) + movr(_RCX_REGNO, r2); + irotshr(code, rn(reg)); + movr(_RCX_REGNO, rn(reg)); + jit_unget_reg(reg); + } + else if (r2 != _RCX_REGNO) { + reg = jit_get_reg(jit_class_gpr); + movr(rn(reg), _RCX_REGNO); + movr(_RCX_REGNO, r2); + movr(r0, r1); + irotshr(code, r0); + movr(_RCX_REGNO, rn(reg)); + jit_unget_reg(reg); + } + else { + movr(r0, r1); + irotshr(code, r0); + } +} + +static void +_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0) +{ + rex(0, 1, _NOREG, _NOREG, r0); + if (i0 == 1) { + ic(0xd1); + mrm(0x03, code, r7(r0)); + } + else { + ic(0xc1); + mrm(0x03, code, r7(r0)); + ic(i0); + } +} + +static void +_rotshi(jit_state_t *_jit, jit_int32_t code, + jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + movr(r0, r1); + if (i0) + irotshi(code, r0, i0); +} + +static void +_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0 == 0) + movr(r0, r1); + else if (i0 <= 3) + lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0); + else + rotshi(X86_SHL, r0, r1, i0); +} + +static void +_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0) +{ + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xf7); + mrm(0x03, code, r7(r0)); +} + +static void +_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + inegr(r0); + else { + ixorr(r0, r0); + isubr(r0, r1); + } +} + +static void +_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + movr(r0, r1); + icomr(r0); +} + +#if USE_INC_DEC +static void +_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + movr(r0, r1); +# if __WORDSIZE == 64 + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xff); + ic(0xc0 | r7(r0)); +# else + ic(0x40 | r7(r0)); +# endif +} + +static void +_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + movr(r0, r1); +# if __WORDSIZE == 64 + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xff); + ic(0xc8 | r7(r0)); +# else + ic(0x48 | r7(r0)); +# endif +} +#endif + +static void +_cr(jit_state_t *_jit, + jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + jit_bool_t same; + if (reg8_p(r0)) { + same = r0 == r1 || r0 == r2; + if (!same) + ixorr(r0, r0); + icmpr(r1, r2); + if (same) + imovi(r0, 0); + cc(code, r0); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + ixorr(rn(reg), rn(reg)); + icmpr(r1, r2); + cc(code, rn(reg)); + movr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ci(jit_state_t *_jit, + jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + jit_bool_t same; + if (reg8_p(r0)) { + same = r0 == r1; + if (!same) + ixorr(r0, r0); + icmpi(r1, i0); + if (same) + imovi(r0, 0); + cc(code, r0); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + ixorr(rn(reg), rn(reg)); + icmpi(r1, i0); + cc(code, reg); + movr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + jit_bool_t same; + if (reg8_p(r0)) { + same = r0 == r1; + if (!same) + ixorr(r0, r0); + testr(r1, r1); + if (same) + imovi(r0, 0); + cc(code, r0); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + ixorr(rn(reg), rn(reg)); + testr(r1, r1); + cc(code, rn(reg)); + movr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 0); + else + cr(X86_CC_L, r0, r1, r2); +} + +static void +_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_L, r0, r1, i0); + else + ci0(X86_CC_S, r0, r1); +} + +static void +_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 0); + else + cr(X86_CC_B, r0, r1, r2); +} + +static void +_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + cr(X86_CC_LE, r0, r1, r2); +} + +static void +_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + cr(X86_CC_BE, r0, r1, r2); +} + +static void +_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_BE, r0, r1, i0); + else + ci0(X86_CC_E, r0, r1); +} + +static void +_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + cr(X86_CC_E, r0, r1, r2); +} + +static void +_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_E, r0, r1, i0); + else + ci0(X86_CC_E, r0, r1); +} + +static void +_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + cr(X86_CC_GE, r0, r1, r2); +} + +static void +_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_GE, r0, r1, i0); + else + ci0(X86_CC_NS, r0, r1); +} + +static void +_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + cr(X86_CC_AE, r0, r1, r2); +} + +static void +_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_AE, r0, r1, i0); + else + ci0(X86_CC_NB, r0, r1); +} + +static void +_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 0); + else + cr(X86_CC_G, r0, r1, r2); +} + +static void +_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 0); + else + cr(X86_CC_A, r0, r1, r2); +} + +static void +_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_A, r0, r1, i0); + else + ci0(X86_CC_NE, r0, r1); +} + +static void +_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 0); + else + cr(X86_CC_NE, r0, r1, r2); +} + +static void +_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) + ci(X86_CC_NE, r0, r1, i0); + else + ci0(X86_CC_NE, r0, r1); +} + +static void +_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 != r1) { + rex(0, 1, r1, _NOREG, r0); + ic(0x89); + ic(0xc0 | (r1 << 3) | r7(r0)); + } +} + +static void +_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ +#if __WORDSIZE == 64 + if (fits_uint32_p(i0)) { + rex(0, 0, _NOREG, _NOREG, r0); + ic(0xb8 | r7(r0)); + ii(i0); + } + else { + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xb8 | r7(r0)); + il(i0); + } +#else + ic(0xb8 | r7(r0)); + ii(i0); +#endif +} + +static void +_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + if (i0) + imovi(r0, i0); + else + ixorr(r0, r0); +} + +static jit_word_t +_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ +#if __WORDSIZE == 64 + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xb8 | r7(r0)); + il(i0); +#else + ic(0xb8 | r7(r0)); + ii(i0); +#endif + return (_jit->pc.w); +} + +static void +_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbe); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbe); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbf); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xb7); + mrm(0x03, r7(r0), r7(r1)); +} + +#if __WORDSIZE == 64 +static void +_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x63); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 0, r1, _NOREG, r0); + ic(0x89); + ic(0xc0 | (r1 << 3) | r7(r0)); +} +#endif + +static void +_htonr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + movr(r0, r1); + rex(0, 1, _NOREG, _NOREG, r0); + ic(0x0f); + ic(0xc8 | r7(r0)); +} + +static void +_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (reg8_p(r1)) + movcr(r0, r1); + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r1); + movcr(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (reg8_p(r1)) + movcr_u(r0, r1); + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r1); + movcr_u(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbe); + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xbe); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_c(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbe); + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xb6); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_uc(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbf); + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xbf); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_s(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xb7); + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xb7); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_us(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if __WORDSIZE == 64 + rex(0, 1, r0, _NOREG, r1); + ic(0x63); +#else + ic(0x8b); +#endif + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { +#if __WORDSIZE == 64 + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x63); +#else + ic(0x8b); +#endif + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_i(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +#if __WORDSIZE == 64 +static void +_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 0, r0, _NOREG, r1); + ic(0x63); + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x63); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_ui(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r0, _NOREG, r1); + ic(0x8b); + rx(r0, 0, r1, _NOREG, _SCL1); +} + +static void +_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x8b); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldr_l(r0, rn(reg)); + jit_unget_reg(reg); + } +} +#endif + +static void +_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 1, r0, r1, r2); + ic(0x0f); + ic(0xbe); + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbe); + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_c(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 1, r0, r1, r2); + ic(0x0f); + ic(0xb6); + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xb6); + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_uc(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 1, r0, r1, r2); + ic(0x0f); + ic(0xbf); + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xbf); + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_s(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 1, r0, r1, r2); + ic(0x0f); + ic(0xb7); + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, r1); + ic(0x0f); + ic(0xb7); + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_us(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ +#if __WORDSIZE == 64 + rex(0, 1, r0, r1, r2); + ic(0x63); +#else + ic(0x8b); +#endif + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { +#if __WORDSIZE == 64 + rex(0, 1, r0, _NOREG, r1); + ic(0x63); +#else + ic(0x8b); +#endif + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_i(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +#if __WORDSIZE == 64 +static void +_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 0, r0, r1, r2); + ic(0x8b); + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 0, r0, _NOREG, r1); + ic(0x8b); + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_ui(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 1, r0, r1, r2); + ic(0x8b); + rx(r0, 0, r2, r1, _SCL1); +} + +static void +_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, r1); + ic(0x8b); + rx(r0, i0, r1, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ldxr_l(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} +#endif + +static void +_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (reg8_p(r1)) { + rex(0, 0, r1, r0, _NOREG); + ic(0x88); + rx(r1, 0, r0, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r1); + ic(0x88); + rx(rn(reg), 0, r0, _NOREG, _SCL1); + jit_unget_reg(reg); + } +} + +static void +_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + if (reg8_p(r0)) { + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x88); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r0); + ic(0x88); + rx(rn(reg), i0, _NOREG, _NOREG, _SCL1); + jit_unget_reg(reg); + } + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + str_c(rn(reg), r0); + jit_unget_reg(reg); + } +} + +static void +_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + ic(0x66); + rex(0, 0, r1, r0, _NOREG); + ic(0x89); + rx(r1, 0, r0, _NOREG, _SCL1); +} + +static void +_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + ic(0x66); + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x89); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + str_s(rn(reg), r0); + jit_unget_reg(reg); + } +} + +static void +_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 0, r1, r0, _NOREG); + ic(0x89); + rx(r1, 0, r0, _NOREG, _SCL1); +} + +static void +_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x89); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + str_i(rn(reg), r0); + jit_unget_reg(reg); + } +} + +#if __WORDSIZE == 64 +static void +_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 1, r1, r0, _NOREG); + ic(0x89); + rx(r1, 0, r0, _NOREG, _SCL1); +} + +static void +_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r0, _NOREG, _NOREG); + ic(0x89); + rx(r0, i0, _NOREG, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + str_l(rn(reg), r0); + jit_unget_reg(reg); + } +} +#endif + +static void +_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (reg8_p(r2)) { + rex(0, 0, r2, r0, r1); + ic(0x88); + rx(r2, 0, r0, r1, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r2); + ic(0x88); + rx(rn(reg), 0, r0, r1, _SCL1); + jit_unget_reg(reg); + } +} + +static void +_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + if (reg8_p(r1)) { + rex(0, 0, r1, _NOREG, r0); + ic(0x88); + rx(r1, i0, r0, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r1); + ic(0x88); + rx(rn(reg), i0, r0, _NOREG, _SCL1); + jit_unget_reg(reg); + } + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + stxr_c(rn(reg), r0, r1); + jit_unget_reg(reg); + } +} + +static void +_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + ic(0x66); + rex(0, 0, r2, r0, r1); + ic(0x89); + rx(r2, 0, r0, r1, _SCL1); +} + +static void +_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + ic(0x66); + rex(0, 0, r1, _NOREG, r0); + ic(0x89); + rx(r1, i0, r0, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + stxr_s(rn(reg), r0, r1); + jit_unget_reg(reg); + } +} + +static void +_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 0, r2, r0, r1); + ic(0x89); + rx(r2, 0, r0, r1, _SCL1); +} + +static void +_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 0, r1, _NOREG, r0); + ic(0x89); + rx(r1, i0, r0, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + stxr_i(rn(reg), r0, r1); + jit_unget_reg(reg); + } +} + +#if __WORDSIZE == 64 +static void +_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + rex(0, 1, r2, r0, r1); + ic(0x89); + rx(r2, 0, r0, r1, _SCL1); +} + +static void +_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + rex(0, 1, r1, _NOREG, r0); + ic(0x89); + rx(r1, i0, r0, _NOREG, _SCL1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + stxr_l(rn(reg), r0, r1); + jit_unget_reg(reg); + } +} +#endif + +static void +_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0) +{ + ic(0x70 | code); + ic(i0 - (_jit->pc.w + 1)); +} + +static void +_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0) +{ + ic(0x0f); + ic(0x80 | code); + ii(i0 - (_jit->pc.w + 4)); +} + +static void +_jcr(jit_state_t *_jit, + jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + alur(X86_CMP, r0, r1); + jcc(code, i0); +} + +static void +_jci(jit_state_t *_jit, + jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + alui(X86_CMP, r0, i1); + jcc(code, i0); +} + +static void +_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0) +{ + testr(r0, r0); + jcc(code, i0); +} + +static jit_word_t +_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jcr(X86_CC_L, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_L, i0, r0, i1); + else jci0(X86_CC_S, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jcr(X86_CC_B, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_B, i0, r0, i1); + else jci0(X86_CC_B, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) jmpi(i0); + else jcr (X86_CC_LE, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_LE, i0, r0, i1); + else jci0(X86_CC_LE, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) jmpi(i0); + else jcr (X86_CC_BE, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_BE, i0, r0, i1); + else jci0(X86_CC_BE, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) jmpi(i0); + else jcr (X86_CC_E, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_E, i0, r0, i1); + else jci0(X86_CC_E, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) jmpi(i0); + else jcr (X86_CC_GE, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_GE, i0, r0, i1); + else jci0(X86_CC_NS, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) jmpi(i0); + else jcr (X86_CC_AE, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_AE, i0, r0, i1); + else jmpi(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jcr(X86_CC_G, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jci(X86_CC_G, i0, r0, i1); + return (_jit->pc.w); +} + +static jit_word_t +_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jcr(X86_CC_A, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_A, i0, r0, i1); + else jci0(X86_CC_NE, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jcr(X86_CC_NE, i0, r0, r1); + return (_jit->pc.w); +} + +static jit_word_t +_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + if (i1) jci (X86_CC_NE, i0, r0, i1); + else jci0(X86_CC_NE, i0, r0); + return (_jit->pc.w); +} + +static jit_word_t +_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + testr(r0, r1); + jnz(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_zero_extend_int_p(i1)) + testi(r0, i1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + testr(r0, rn(reg)); + } + jnz(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + testr(r0, r1); + jz(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_zero_extend_int_p(i1)) + testi(r0, i1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + testr(r0, rn(reg)); + } + jz(i0); + return (_jit->pc.w); +} + +static jit_word_t +_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + iaddr(r0, r1); + jo(i0); + return (_jit->pc.w); +} + +static jit_word_t +_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + iaddi(r0, i1); + jo(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (boaddr(i0, r0, rn(reg))); +} + +static jit_word_t +_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + iaddr(r0, r1); + jc(i0); + return (_jit->pc.w); +} + +static jit_word_t +_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + iaddi(r0, i1); + jc(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (boaddr_u(i0, r0, rn(reg))); +} + +static jit_word_t +_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + iaddr(r0, r1); + jno(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + iaddi(r0, i1); + jno(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (bxaddr(i0, r0, rn(reg))); +} + +static jit_word_t +_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + iaddr(r0, r1); + jnc(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + iaddi(r0, i1); + jnc(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (bxaddr_u(i0, r0, rn(reg))); +} + +static jit_word_t +_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + isubr(r0, r1); + jo(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + isubi(r0, i1); + jo(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (bosubr(i0, r0, rn(reg))); +} + +static jit_word_t +_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + isubr(r0, r1); + jc(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + isubi(r0, i1); + jc(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (bosubr_u(i0, r0, rn(reg))); +} + +static jit_word_t +_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + isubr(r0, r1); + jno(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + isubi(r0, i1); + jno(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (bxsubr(i0, r0, rn(reg))); +} + +static jit_word_t +_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + isubr(r0, r1); + jnc(i0); + return (_jit->pc.w); +} + +static jit_word_t +_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i1)) { + isubi(r0, i1); + jnc(i0); + return (_jit->pc.w); + } + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i1); + jit_unget_reg(reg); + return (bxsubr_u(i0, r0, rn(reg))); +} + +static void +_callr(jit_state_t *_jit, jit_int32_t r0) +{ + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xff); + mrm(0x03, 0x02, r7(r0)); +} + +static jit_word_t +_calli(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t word; +#if __WORDSIZE == 64 + jit_int32_t reg; + + reg = jit_get_reg(jit_class_gpr); + word = movi_p(rn(reg), i0); + callr(rn(reg)); + jit_unget_reg(reg); +#else + ic(0xe8); + ii(i0 - (_jit->pc.w + 4)); + word = _jit->pc.w; +#endif + return (word); +} + +static void +_jmpr(jit_state_t *_jit, jit_int32_t r0) +{ + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xff); + mrm(0x03, 0x04, r7(r0)); +} + +static jit_word_t +_jmpi(jit_state_t *_jit, jit_word_t i0) +{ + ic(0xe9); + ii(i0 - (_jit->pc.w + 4)); + return (_jit->pc.w); +} + +static void +_prolog(jit_state_t *_jit, jit_node_t *node) +{ + jit_function_t *function; + + function = _jit->functions.ptr + node->u.w; + + /* callee save registers */ + subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - sizeof(jit_word_t)); +#if __WORDSIZE == 32 + if (jit_regset_tstbit(function->regset, _RDI)) + stxi(12, _RSP_REGNO, _RDI_REGNO); + if (jit_regset_tstbit(function->regset, _RSI)) + stxi( 8, _RSP_REGNO, _RSI_REGNO); + if (jit_regset_tstbit(function->regset, _RBX)) + stxi( 4, _RSP_REGNO, _RBX_REGNO); +#else + if (jit_regset_tstbit(function->regset, _RBX)) + stxi(40, _RSP_REGNO, _RBX_REGNO); + if (jit_regset_tstbit(function->regset, _R12)) + stxi(32, _RSP_REGNO, _R12_REGNO); + if (jit_regset_tstbit(function->regset, _R13)) + stxi(24, _RSP_REGNO, _R13_REGNO); + if (jit_regset_tstbit(function->regset, _R14)) + stxi(16, _RSP_REGNO, _R14_REGNO); + if (jit_regset_tstbit(function->regset, _R15)) + stxi( 8, _RSP_REGNO, _R15_REGNO); +#endif + stxi(0, _RSP_REGNO, _RBP_REGNO); + movr(_RBP_REGNO, _RSP_REGNO); + + /* alloca */ + subi(_RSP_REGNO, _RSP_REGNO, function->stack); +} + +static void +_epilog(jit_state_t *_jit, jit_node_t *node) +{ + jit_function_t *function; + + function = _jit->functions.ptr + node->w.w; + + /* callee save registers */ + movr(_RSP_REGNO, _RBP_REGNO); +#if __WORDSIZE == 32 + if (jit_regset_tstbit(function->regset, _RDI)) + ldxi(_RDI_REGNO, _RSP_REGNO, 12); + if (jit_regset_tstbit(function->regset, _RSI)) + ldxi(_RSI_REGNO, _RSP_REGNO, 8); + if (jit_regset_tstbit(function->regset, _RBX)) + ldxi(_RBX_REGNO, _RSP_REGNO, 4); +#else + if (jit_regset_tstbit(function->regset, _RBX)) + ldxi(_RBX_REGNO, _RSP_REGNO, 40); + if (jit_regset_tstbit(function->regset, _R12)) + ldxi(_R12_REGNO, _RSP_REGNO, 32); + if (jit_regset_tstbit(function->regset, _R13)) + ldxi(_R13_REGNO, _RSP_REGNO, 24); + if (jit_regset_tstbit(function->regset, _R14)) + ldxi(_R14_REGNO, _RSP_REGNO, 16); + if (jit_regset_tstbit(function->regset, _R15)) + ldxi(_R15_REGNO, _RSP_REGNO, 8); +#endif + ldxi(_RBP_REGNO, _RSP_REGNO, 0); + addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - sizeof(jit_word_t)); + + ic(0xc3); +} + +static void +_patch_at(jit_state_t *_jit, jit_node_t *node, + jit_word_t instr, jit_word_t label) +{ + switch (node->code) { +#if __WORDSIZE == 64 + case jit_code_calli: +#endif + case jit_code_movi: + patch_abs(instr, label); + break; + default: + patch_rel(instr, label); + break; + } +} +#endif diff --git a/lib/jit_x86-sse.c b/lib/jit_x86-sse.c new file mode 100644 index 000000000..1f56b4ca9 --- /dev/null +++ b/lib/jit_x86-sse.c @@ -0,0 +1,1439 @@ +/* + * Copyright (C) 2011 Paulo Cesar Pereira de Andrade. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#if PROTO +#define X86_SSE_MOV 0x10 +#define X86_SSE_MOV1 0x11 +#define X86_SSE_MOVLP 0x12 +#define X86_SSE_MOVHP 0x16 +#define X86_SSE_MOVA 0x28 +#define X86_SSE_CVTIS 0x2a +#define X86_SSE_CVTTSI 0x2c +#define X86_SSE_CVTSI 0x2d +#define X86_SSE_UCOMI 0x2e +#define X86_SSE_COMI 0x2f +#define X86_SSE_ROUND 0x3a +#define X86_SSE_SQRT 0x51 +#define X86_SSE_RSQRT 0x52 +#define X86_SSE_RCP 0x53 +#define X86_SSE_AND 0x54 +#define X86_SSE_ANDN 0x55 +#define X86_SSE_OR 0x56 +#define X86_SSE_XOR 0x57 +#define X86_SSE_ADD 0x58 +#define X86_SSE_MUL 0x59 +#define X86_SSE_CVTSD 0x5a +#define X86_SSE_CVTDT 0x5b +#define X86_SSE_SUB 0x5c +#define X86_SSE_MIN 0x5d +#define X86_SSE_DIV 0x5e +#define X86_SSE_MAX 0x5f +#define X86_SSE_X2G 0x6e +#define X86_SSE_EQB 0x74 +#define X86_SSE_EQW 0x75 +#define X86_SSE_EQD 0x76 +#define X86_SSE_G2X 0x7e +#define X86_SSE_MOV2 0xd6 +# define sser(c,r0,r1) _sser(_jit,c,r0,r1) +static void _sser(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define ssexr(p,c,r0,r1) _ssexr(_jit,p,c,r0,r1) +static void _ssexr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ssexi(c,r0,m,i) _ssexi(_jit,c,r0,m,i) +static void _ssexi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define addssr(r0, r1) ssexr(0xf3, X86_SSE_ADD, r0, r1) +# define addsdr(r0, r1) ssexr(0xf2, X86_SSE_ADD, r0, r1) +# define subssr(r0, r1) ssexr(0xf3, X86_SSE_SUB, r0, r1) +# define subsdr(r0, r1) ssexr(0xf2, X86_SSE_SUB, r0, r1) +# define mulssr(r0, r1) ssexr(0xf3, X86_SSE_MUL, r0, r1) +# define mulsdr(r0, r1) ssexr(0xf2, X86_SSE_MUL, r0, r1) +# define divssr(r0, r1) ssexr(0xf3, X86_SSE_DIV, r0, r1) +# define divsdr(r0, r1) ssexr(0xf2, X86_SSE_DIV, r0, r1) +# define andpsr(r0, r1) sser( X86_SSE_AND, r0, r1) +# define andpdr(r0, r1) ssexr(0x66, X86_SSE_AND, r0, r1) +# define sse_truncr_f_i(r0, r1) ssexr(0xf3, X86_SSE_CVTTSI, r0, r1) +# define sse_truncr_d_i(r0, r1) ssexr(0xf2, X86_SSE_CVTTSI, r0, r1) +# if __WORDSIZE == 64 +# define sse_truncr_f_l(r0, r1) sselxr(0xf3, X86_SSE_CVTTSI, r0, r1) +# define sse_truncr_d_l(r0, r1) sselxr(0xf2, X86_SSE_CVTTSI, r0, r1) +# endif +# define sse_extr_f(r0, r1) sselxr(0xf3, X86_SSE_CVTIS, r0, r1) +# define sse_extr_d(r0, r1) sselxr(0xf2, X86_SSE_CVTIS, r0, r1) +# define sse_extr_f_d(r0, r1) ssexr(0xf3, X86_SSE_CVTSD, r0, r1) +# define sse_extr_d_f(r0, r1) ssexr(0xf2, X86_SSE_CVTSD, r0, r1) +# define ucomissr(r0,r1) sser(X86_SSE_UCOMI,r0,r1) +# define ucomisdr(r0,r1) ssexr(0x66,X86_SSE_UCOMI,r0,r1) +# define xorpsr(r0,r1) sser(X86_SSE_XOR,r0,r1) +# define xorpdr(r0,r1) ssexr(0x66,X86_SSE_XOR,r0,r1) +# define movdlxr(r0,r1) ssexr(0x66, X86_SSE_X2G,r0,r1) +# define pcmpeqlr(r0, r1) ssexr(0x66, X86_SSE_EQD, r0, r1) +# define psrl(r0, i0) ssexi(0x72, r0, 0x02, i0) +# define psrq(r0, i0) ssexi(0x73, r0, 0x02, i0) +# define psll(r0, i0) ssexi(0x72, r0, 0x06, i0) +# define pslq(r0, i0) ssexi(0x73, r0, 0x06, i0) +# define movdqxr(r0,r1) sselxr(0x66,X86_SSE_X2G,r0,r1) +# if __WORDSIZE == 64 +# define sselxr(p,c,r0,r1) _sselxr(_jit,p,c,r0,r1) +static void +_sselxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t); +# else +# define sselxr(p,c,r0,r1) ssexr(p,c,r0,r1) +# endif +# define ssexrx(p,c,md,rb,ri,ms,rd) _ssexrx(_jit,p,c,md,rb,ri,ms,rd) +# define movssmr(md,rb,ri,ms,rd) ssexrx(0xf3,X86_SSE_MOV,md,rb,ri,ms,rd) +# define movsdmr(md,rb,ri,ms,rd) ssexrx(0xf2,X86_SSE_MOV,md,rb,ri,ms,rd) +# define movssrm(rs,md,mb,mi,ms) ssexrx(0xf3,X86_SSE_MOV1,md,mb,mi,ms,rs) +# define movsdrm(rs,md,mb,mi,ms) ssexrx(0xf2,X86_SSE_MOV1,md,mb,mi,ms,rs) +static void +_ssexrx(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, + jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_addr_f(r0, r1, r2) _sse_addr_f(_jit, r0, r1, r2) +static void _sse_addr_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_addi_f(r0, r1, i0) _sse_addi_f(_jit, r0, r1, i0) +static void _sse_addi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_addr_d(r0, r1, r2) _sse_addr_d(_jit, r0, r1, r2) +static void _sse_addr_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_addi_d(r0, r1, i0) _sse_addi_d(_jit, r0, r1, i0) +static void _sse_addi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_subr_f(r0, r1, r2) _sse_subr_f(_jit, r0, r1, r2) +static void _sse_subr_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_subi_f(r0, r1, i0) _sse_subi_f(_jit, r0, r1, i0) +static void _sse_subi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_subr_d(r0, r1, r2) _sse_subr_d(_jit, r0, r1, r2) +static void _sse_subr_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_subi_d(r0, r1, i0) _sse_subi_d(_jit, r0, r1, i0) +static void _sse_subi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_mulr_f(r0, r1, r2) _sse_mulr_f(_jit, r0, r1, r2) +static void _sse_mulr_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_muli_f(r0, r1, i0) _sse_muli_f(_jit, r0, r1, i0) +static void _sse_muli_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_mulr_d(r0, r1, r2) _sse_mulr_d(_jit, r0, r1, r2) +static void _sse_mulr_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_muli_d(r0, r1, i0) _sse_muli_d(_jit, r0, r1, i0) +static void _sse_muli_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_divr_f(r0, r1, r2) _sse_divr_f(_jit, r0, r1, r2) +static void _sse_divr_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_divi_f(r0, r1, i0) _sse_divi_f(_jit, r0, r1, i0) +static void _sse_divi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_divr_d(r0, r1, r2) _sse_divr_d(_jit, r0, r1, r2) +static void _sse_divr_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define sse_divi_d(r0, r1, i0) _sse_divi_d(_jit, r0, r1, i0) +static void _sse_divi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_absr_f(r0, r1) _sse_absr_f(_jit, r0, r1) +static void _sse_absr_f(jit_state_t*,jit_int32_t,jit_int32_t); +# define sse_absr_d(r0, r1) _sse_absr_d(_jit, r0, r1) +static void _sse_absr_d(jit_state_t*,jit_int32_t,jit_int32_t); +# define sse_negr_f(r0, r1) _sse_negr_f(_jit, r0, r1) +static void _sse_negr_f(jit_state_t*,jit_int32_t,jit_int32_t); +# define sse_negr_d(r0, r1) _sse_negr_d(_jit, r0, r1) +static void _sse_negr_d(jit_state_t*,jit_int32_t,jit_int32_t); +# define sse_sqrtr_f(r0, r1) ssexr(0xf3, X86_SSE_SQRT, r0, r1) +# define sse_sqrtr_d(r0, r1) ssexr(0xf2, X86_SSE_SQRT, r0, r1) +# define ssecmpf(code, r0, r1, r2) _ssecmp(_jit, 0, code, r0, r1, r2) +# define ssecmpd(code, r0, r1, r2) _ssecmp(_jit, 1, code, r0, r1, r2) +static void +_ssecmp(jit_state_t*, jit_bool_t, jit_int32_t, + jit_int32_t, jit_int32_t, jit_int32_t); +#define sse_movr_f(r0,r1) _sse_movr_f(_jit,r0,r1) +static void _sse_movr_f(jit_state_t*, jit_int32_t, jit_int32_t); +#define sse_movi_f(r0,i0) _sse_movi_f(_jit,r0,i0) +static void _sse_movi_f(jit_state_t*, jit_int32_t, jit_float32_t*); +# define sse_lti_f(r0, r1, i0) _sse_lti_f(_jit, r0, r1, i0) +static void _sse_lti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ltr_f(r0, r1, r2) ssecmpf(X86_CC_A, r0, r1, r2) +# define sse_lei_f(r0, r1, i0) _sse_lei_f(_jit, r0, r1, i0) +static void _sse_lei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ler_f(r0, r1, r2) ssecmpf(X86_CC_AE, r0, r1, r2) +# define sse_eqi_f(r0, r1, i0) _sse_eqi_f(_jit, r0, r1, i0) +static void _sse_eqi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_eqr_f(r0, r1, r2) _sse_eqr_f(_jit, r0, r1, r2) +static void _sse_eqr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_gei_f(r0, r1, i0) _sse_gei_f(_jit, r0, r1, i0) +static void _sse_gei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ger_f(r0, r1, r2) ssecmpf(X86_CC_AE, r0, r2, r1) +# define sse_gti_f(r0, r1, i0) _sse_gti_f(_jit, r0, r1, i0) +static void _sse_gti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_gtr_f(r0, r1, r2) ssecmpf(X86_CC_A, r0, r2, r1) +# define sse_nei_f(r0, r1, i0) _sse_nei_f(_jit, r0, r1, i0) +static void _sse_nei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ner_f(r0, r1, r2) _sse_ner_f(_jit, r0, r1, r2) +static void _sse_ner_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_unlti_f(r0, r1, i0) _sse_unlti_f(_jit, r0, r1, i0) +static void _sse_unlti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_unltr_f(r0, r1, r2) ssecmpf(X86_CC_NAE, r0, r2, r1) +# define sse_unlei_f(r0, r1, i0) _sse_unlei_f(_jit, r0, r1, i0) +static void _sse_unlei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_unler_f(r0, r1, r2) _sse_unler_f(_jit, r0, r1, r2) +# define sse_uneqi_f(r0, r1, i0) _sse_uneqi_f(_jit, r0, r1, i0) +static void _sse_uneqi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +static void _sse_unler_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_uneqr_f(r0, r1, r2) _sse_uneqr_f(_jit, r0, r1, r2) +static void _sse_uneqr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_ungei_f(r0, r1, i0) _sse_ungei_f(_jit, r0, r1, i0) +static void _sse_ungei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_unger_f(r0, r1, r2) _sse_unger_f(_jit, r0, r1, r2) +static void _sse_unger_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_ungti_f(r0, r1, i0) _sse_ungti_f(_jit, r0, r1, i0) +static void _sse_ungti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ungtr_f(r0, r1, r2) ssecmpf(X86_CC_NAE, r0, r2, r1) +# define sse_ltgti_f(r0, r1, i0) _sse_ltgti_f(_jit, r0, r1, i0) +static void _sse_ltgti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ltgtr_f(r0, r1, r2) _sse_ltgtr_f(_jit, r0, r1, r2) +static void _sse_ltgtr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_ordi_f(r0, r1, i0) _sse_ordi_f(_jit, r0, r1, i0) +static void _sse_ordi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_ordr_f(r0, r1, r2) ssecmpf(X86_CC_NP, r0, r2, r1) +# define sse_unordi_f(r0, r1, i0) _sse_unordi_f(_jit, r0, r1, i0) +static void _sse_unordi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define sse_unordr_f(r0, r1, r2) ssecmpf(X86_CC_P, r0, r2, r1) +# define sse_ldr_f(r0, r1) movssmr(0, r1, _NOREG, _SCL1, r0) +# define sse_ldi_f(r0, i0) _sse_ldi_f(_jit, r0, i0) +static void _sse_ldi_f(jit_state_t*, jit_int32_t, jit_word_t); +# define sse_ldxr_f(r0, r1, r2) movssmr(0, r1, r2, _SCL1, r0) +# define sse_ldxi_f(r0, r1, i0) _sse_ldxi_f(_jit, r0, r1, i0) +static void _sse_ldxi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define sse_str_f(r0, r1) movssrm(r1, 0, r0, _NOREG, _SCL1) +# define sse_sti_f(i0, r0) _sse_sti_f(_jit, i0, r0) +static void _sse_sti_f(jit_state_t*, jit_word_t,jit_int32_t); +# define sse_stxr_f(r0, r1, r2) movssrm(r2, 0, r0, r1, _SCL1) +# define sse_stxi_f(i0, r0, r1) _sse_stxi_f(_jit, i0, r0, r1) +static void _sse_stxi_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bltr_f(i0, r0, r1) _sse_bltr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bltr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_blti_f(i0, r0, i1) _sse_blti_f(_jit, i0, r0, i1) +static jit_word_t +_sse_blti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bler_f(i0, r0, r1) _sse_bler_f(_jit, i0, r0, r1) +static jit_word_t _sse_bler_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_blei_f(i0, r0, i1) _sse_blei_f(_jit, i0, r0, i1) +static jit_word_t +_sse_blei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_beqr_f(i0, r0, r1) _sse_beqr_f(_jit, i0, r0, r1) +static jit_word_t _sse_beqr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_beqi_f(i0, r0, i1) _sse_beqi_f(_jit, i0, r0, i1) +static jit_word_t +_sse_beqi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bger_f(i0, r0, r1) _sse_bger_f(_jit, i0, r0, r1) +static jit_word_t _sse_bger_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bgei_f(i0, r0, i1) _sse_bgei_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bgei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bgtr_f(i0, r0, r1) _sse_bgtr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bgtr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bgti_f(i0, r0, i1) _sse_bgti_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bgti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bner_f(i0, r0, r1) _sse_bner_f(_jit, i0, r0, r1) +static jit_word_t _sse_bner_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bnei_f(i0, r0, i1) _sse_bnei_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bnei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bunltr_f(i0, r0, r1) _sse_bunltr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bunltr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bunlti_f(i0, r0, i1) _sse_bunlti_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bunlti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bunler_f(i0, r0, r1) _sse_bunler_f(_jit, i0, r0, r1) +static jit_word_t _sse_bunler_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bunlei_f(i0, r0, i1) _sse_bunlei_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bunlei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_buneqr_f(i0, r0, r1) _sse_buneqr_f(_jit, i0, r0, r1) +static jit_word_t _sse_buneqr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_buneqi_f(i0, r0, i1) _sse_buneqi_f(_jit, i0, r0, i1) +static jit_word_t +_sse_buneqi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bunger_f(i0, r0, r1) _sse_bunger_f(_jit, i0, r0, r1) +static jit_word_t _sse_bunger_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bungei_f(i0, r0, i1) _sse_bungei_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bungei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bungtr_f(i0, r0, r1) _sse_bungtr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bungtr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bungti_f(i0, r0, i1) _sse_bungti_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bungti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bltgtr_f(i0, r0, r1) _sse_bltgtr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bltgtr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bltgti_f(i0, r0, i1) _sse_bltgti_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bltgti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bordr_f(i0, r0, r1) _sse_bordr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bordr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bordi_f(i0, r0, i1) _sse_bordi_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bordi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define sse_bunordr_f(i0, r0, r1) _sse_bunordr_f(_jit, i0, r0, r1) +static jit_word_t _sse_bunordr_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bunordi_f(i0, r0, i1) _sse_bunordi_f(_jit, i0, r0, i1) +static jit_word_t +_sse_bunordi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +#define sse_movr_d(r0,r1) _sse_movr_d(_jit,r0,r1) +static void _sse_movr_d(jit_state_t*, jit_int32_t, jit_int32_t); +#define sse_movi_d(r0,i0) _sse_movi_d(_jit,r0,i0) +static void _sse_movi_d(jit_state_t*, jit_int32_t, jit_float64_t*); +# define sse_ltr_d(r0, r1, r2) ssecmpd(X86_CC_A, r0, r1, r2) +# define sse_lti_d(r0, r1, i0) _sse_lti_d(_jit, r0, r1, i0) +static void _sse_lti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ler_d(r0, r1, r2) ssecmpd(X86_CC_AE, r0, r1, r2) +# define sse_lei_d(r0, r1, i0) _sse_lei_d(_jit, r0, r1, i0) +static void _sse_lei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_eqr_d(r0, r1, r2) _sse_eqr_d(_jit, r0, r1, r2) +static void _sse_eqr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_eqi_d(r0, r1, i0) _sse_eqi_d(_jit, r0, r1, i0) +static void _sse_eqi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ger_d(r0, r1, r2) ssecmpd(X86_CC_AE, r0, r2, r1) +# define sse_gei_d(r0, r1, i0) _sse_gei_d(_jit, r0, r1, i0) +static void _sse_gei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_gtr_d(r0, r1, r2) ssecmpd(X86_CC_A, r0, r2, r1) +# define sse_gti_d(r0, r1, i0) _sse_gti_d(_jit, r0, r1, i0) +static void _sse_gti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ner_d(r0, r1, r2) _sse_ner_d(_jit, r0, r1, r2) +static void _sse_ner_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_nei_d(r0, r1, i0) _sse_nei_d(_jit, r0, r1, i0) +static void _sse_nei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_unltr_d(r0, r1, r2) ssecmpd(X86_CC_NAE, r0, r2, r1) +# define sse_unlti_d(r0, r1, i0) _sse_unlti_d(_jit, r0, r1, i0) +static void _sse_unlti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_unler_d(r0, r1, r2) _sse_unler_d(_jit, r0, r1, r2) +static void _sse_unler_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_unlei_d(r0, r1, i0) _sse_unlei_d(_jit, r0, r1, i0) +static void _sse_unlei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_uneqr_d(r0, r1, r2) _sse_uneqr_d(_jit, r0, r1, r2) +static void _sse_uneqr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_uneqi_d(r0, r1, i0) _sse_uneqi_d(_jit, r0, r1, i0) +static void _sse_uneqi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_unger_d(r0, r1, r2) _sse_unger_d(_jit, r0, r1, r2) +static void _sse_unger_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_ungei_d(r0, r1, i0) _sse_ungei_d(_jit, r0, r1, i0) +static void _sse_ungei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ungtr_d(r0, r1, r2) ssecmpd(X86_CC_NAE, r0, r2, r1) +# define sse_ungti_d(r0, r1, i0) _sse_ungti_d(_jit, r0, r1, i0) +static void _sse_ungti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ltgtr_d(r0, r1, r2) _sse_ltgtr_d(_jit, r0, r1, r2) +static void _sse_ltgtr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define sse_ltgti_d(r0, r1, i0) _sse_ltgti_d(_jit, r0, r1, i0) +static void _sse_ltgti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ordr_d(r0, r1, r2) ssecmpd(X86_CC_NP, r0, r2, r1) +# define sse_ordi_d(r0, r1, i0) _sse_ordi_d(_jit, r0, r1, i0) +static void _sse_ordi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_unordr_d(r0, r1, r2) ssecmpd(X86_CC_P, r0, r2, r1) +# define sse_unordi_d(r0, r1, i0) _sse_unordi_d(_jit, r0, r1, i0) +static void _sse_unordi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define sse_ldr_d(r0, r1) movsdmr(0, r1, _NOREG, _SCL1, r0) +# define sse_ldi_d(r0, i0) _sse_ldi_d(_jit, r0, i0) +static void _sse_ldi_d(jit_state_t*, jit_int32_t, jit_word_t); +# define sse_ldxr_d(r0, r1, r2) movsdmr(0, r1, r2, _SCL1, r0) +# define sse_ldxi_d(r0, r1, i0) _sse_ldxi_d(_jit, r0, r1, i0) +static void _sse_ldxi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define sse_bltr_d(i0, r0, r1) _sse_bltr_d(_jit, i0, r0, r1) +# define sse_str_d(r0, r1) movsdrm(r1, 0, r0, _NOREG, _SCL1) +# define sse_sti_d(i0, r0) _sse_sti_d(_jit, i0, r0) +static void _sse_sti_d(jit_state_t*, jit_word_t,jit_int32_t); +# define sse_stxr_d(r0, r1, r2) movsdrm(r2, 0, r0, r1, _SCL1) +# define sse_stxi_d(i0, r0, r1) _sse_stxi_d(_jit, i0, r0, r1) +static void _sse_stxi_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +static jit_word_t _sse_bltr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_blti_d(i0, r0, i1) _sse_blti_d(_jit, i0, r0, i1) +static jit_word_t +_sse_blti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bler_d(i0, r0, r1) _sse_bler_d(_jit, i0, r0, r1) +static jit_word_t _sse_bler_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_blei_d(i0, r0, i1) _sse_blei_d(_jit, i0, r0, i1) +static jit_word_t +_sse_blei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_beqr_d(i0, r0, r1) _sse_beqr_d(_jit, i0, r0, r1) +static jit_word_t _sse_beqr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_beqi_d(i0, r0, i1) _sse_beqi_d(_jit, i0, r0, i1) +static jit_word_t +_sse_beqi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bger_d(i0, r0, r1) _sse_bger_d(_jit, i0, r0, r1) +static jit_word_t _sse_bger_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bgei_d(i0, r0, i1) _sse_bgei_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bgei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bgtr_d(i0, r0, r1) _sse_bgtr_d(_jit, i0, r0, r1) +static jit_word_t _sse_bgtr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bgti_d(i0, r0, i1) _sse_bgti_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bgti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bner_d(i0, r0, r1) _sse_bner_d(_jit, i0, r0, r1) +static jit_word_t _sse_bner_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bnei_d(i0, r0, i1) _sse_bnei_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bnei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bunltr_d(i0, r0, r1) _sse_bunltr_d(_jit, i0, r0, r1) +static jit_word_t _sse_bunltr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bunlti_d(i0, r0, i1) _sse_bunlti_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bunlti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bunler_d(i0, r0, r1) _sse_bunler_d(_jit, i0, r0, r1) +static jit_word_t _sse_bunler_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bunlei_d(i0, r0, i1) _sse_bunlei_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bunlei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_buneqr_d(i0, r0, r1) _sse_buneqr_d(_jit, i0, r0, r1) +static jit_word_t _sse_buneqr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_buneqi_d(i0, r0, i1) _sse_buneqi_d(_jit, i0, r0, i1) +static jit_word_t +_sse_buneqi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bunger_d(i0, r0, r1) _sse_bunger_d(_jit, i0, r0, r1) +static jit_word_t _sse_bunger_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bungei_d(i0, r0, i1) _sse_bungei_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bungei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bungtr_d(i0, r0, r1) _sse_bungtr_d(_jit, i0, r0, r1) +static jit_word_t _sse_bungtr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bungti_d(i0, r0, i1) _sse_bungti_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bungti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bltgtr_d(i0, r0, r1) _sse_bltgtr_d(_jit, i0, r0, r1) +static jit_word_t _sse_bltgtr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bltgti_d(i0, r0, i1) _sse_bltgti_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bltgti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bordr_d(i0, r0, r1) _sse_bordr_d(_jit, i0, r0, r1) +static jit_word_t _sse_bordr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bordi_d(i0, r0, i1) _sse_bordi_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bordi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define sse_bunordr_d(i0, r0, r1) _sse_bunordr_d(_jit, i0, r0, r1) +static jit_word_t _sse_bunordr_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define sse_bunordi_d(i0, r0, i1) _sse_bunordi_d(_jit, i0, r0, i1) +static jit_word_t +_sse_bunordi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +#endif + +#if CODE +# define fpr_opi(name, type, size) \ +static void \ +_sse_##name##i_##type(jit_state_t *_jit, \ + jit_int32_t r0, jit_int32_t r1, \ + jit_float##size##_t *i0) \ +{ \ + jit_int32_t reg = jit_get_reg(jit_class_fpr|jit_class_xpr); \ + assert(jit_sse_reg_p(reg)); \ + sse_movi_##type(rn(reg), i0); \ + sse_##name##r_##type(r0, r1, rn(reg)); \ + jit_unget_reg(reg); \ +} +# define fpr_bopi(name, type, size) \ +static jit_word_t \ +_sse_b##name##i_##type(jit_state_t *_jit, \ + jit_word_t i0, jit_int32_t r0, \ + jit_float##size##_t *i1) \ +{ \ + jit_word_t word; \ + jit_int32_t reg = jit_get_reg(jit_class_fpr|jit_class_xpr); \ + assert(jit_sse_reg_p(reg)); \ + sse_movi_##type(rn(reg), i1); \ + word = sse_b##name##r_##type(i0, r0, rn(reg)); \ + jit_unget_reg(reg); \ + return (word); \ +} +# define fopi(name) fpr_opi(name, f, 32) +# define fbopi(name) fpr_bopi(name, f, 32) +# define dopi(name) fpr_opi(name, d, 64) +# define dbopi(name) fpr_bopi(name, d, 64) +static void +_sser(jit_state_t *_jit, jit_int32_t c, jit_int32_t r0, jit_int32_t r1) +{ + rex(0, 0, r1, 0, r0); + ic(0x0f); + ic(c); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_ssexr(jit_state_t *_jit, jit_int32_t p, jit_int32_t c, + jit_int32_t r0, jit_int32_t r1) +{ + ic(p); + rex(0, 0, r0, 0, r1); + ic(0x0f); + ic(c); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_ssexi(jit_state_t *_jit, jit_int32_t c, jit_int32_t r0, + jit_int32_t m, jit_int32_t i) +{ + ic(0x66); + rex(0, 0, 0, 0, r0); + ic(0x0f); + ic(c); + mrm(0x03, r7(m), r7(r0)); + ic(i); +} + +#if __WORDSIZE == 64 +static void +_sselxr(jit_state_t *_jit, jit_int32_t p, jit_int32_t c, + jit_int32_t r0, jit_int32_t r1) +{ + ic(p); + rex(0, 1, r0, 0, r1); + ic(0x0f); + ic(c); + mrm(0x03, r7(r0), r7(r1)); +} +#endif + +static void +_ssexrx(jit_state_t *_jit, jit_int32_t px, jit_int32_t code, jit_int32_t md, + jit_int32_t rb, jit_int32_t ri, jit_int32_t ms, jit_int32_t rd) +{ + ic(px); + rex(0, 0, rd, ri, rb); + ic(0x0f); + ic(code); + rx(rd, md, rb, ri, ms); +} + +static void +_sse_addr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) + addssr(r0, r2); + else if (r0 == r2) + addssr(r0, r1); + else { + sse_movr_f(r0, r1); + addssr(r0, r2); + } +} + +fopi(add) + +static void +_sse_addr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) + addsdr(r0, r2); + else if (r0 == r2) + addsdr(r0, r1); + else { + sse_movr_d(r0, r1); + addsdr(r0, r2); + } +} + +dopi(add) + +static void +_sse_subr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (r0 == r1) + subssr(r0, r2); + else if (r0 == r2) { + reg = jit_get_reg(jit_class_fpr|jit_class_xpr); + sse_movr_f(rn(reg), r0); + sse_movr_f(r0, r1); + subssr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + sse_movr_f(r0, r1); + subssr(r0, r2); + } +} + +fopi(sub) + +static void +_sse_subr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (r0 == r1) + subsdr(r0, r2); + else if (r0 == r2) { + reg = jit_get_reg(jit_class_fpr|jit_class_xpr); + sse_movr_d(rn(reg), r0); + sse_movr_d(r0, r1); + subsdr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + sse_movr_d(r0, r1); + subsdr(r0, r2); + } +} + +dopi(sub) + +static void +_sse_mulr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) + mulssr(r0, r2); + else if (r0 == r2) + mulssr(r0, r1); + else { + sse_movr_f(r0, r1); + mulssr(r0, r2); + } +} + +fopi(mul) + +static void +_sse_mulr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) + mulsdr(r0, r2); + else if (r0 == r2) + mulsdr(r0, r1); + else { + sse_movr_d(r0, r1); + mulsdr(r0, r2); + } +} + +dopi(mul) + +static void +_sse_divr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (r0 == r1) + divssr(r0, r2); + else if (r0 == r2) { + reg = jit_get_reg(jit_class_fpr|jit_class_xpr); + sse_movr_f(rn(reg), r0); + sse_movr_f(r0, r1); + divssr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + sse_movr_f(r0, r1); + divssr(r0, r2); + } +} + +fopi(div) + +static void +_sse_divr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + if (r0 == r1) + divsdr(r0, r2); + else if (r0 == r2) { + reg = jit_get_reg(jit_class_fpr|jit_class_xpr); + sse_movr_d(rn(reg), r0); + sse_movr_d(r0, r1); + divsdr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + sse_movr_d(r0, r1); + divsdr(r0, r2); + } +} + +dopi(div) + +static void +_sse_absr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (r0 == r1) { + reg = jit_get_reg(jit_class_fpr|jit_class_xpr); + pcmpeqlr(rn(reg), rn(reg)); + psrl(rn(reg), 1); + andpsr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + pcmpeqlr(r0, r0); + psrl(r0, 1); + andpsr(r0, r1); + } +} + +static void +_sse_absr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (r0 == r1) { + reg = jit_get_reg(jit_class_fpr|jit_class_xpr); + pcmpeqlr(rn(reg), rn(reg)); + psrq(rn(reg), 1); + andpdr(r0, rn(reg)); + jit_unget_reg(reg); + } + else { + pcmpeqlr(r0, r0); + psrq(r0, 1); + andpdr(r0, r1); + } +} + +static void +_sse_negr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t freg, ireg; + ireg = jit_get_reg(jit_class_gpr); + imovi(rn(ireg), 0x80000000); + if (r0 == r1) { + freg = jit_get_reg(jit_class_fpr|jit_class_xpr); + movdlxr(rn(freg), rn(ireg)); + xorpsr(r0, rn(freg)); + jit_unget_reg(freg); + } + else { + movdlxr(r0, rn(ireg)); + xorpsr(r0, r1); + } + jit_unget_reg(ireg); +} + +static void +_sse_negr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t freg, ireg; + ireg = jit_get_reg(jit_class_gpr); + imovi(rn(ireg), 0x80000000); + if (r0 == r1) { + freg = jit_get_reg(jit_class_fpr|jit_class_xpr); + movdlxr(rn(freg), rn(ireg)); + pslq(rn(freg), 32); + xorpdr(r0, rn(freg)); + jit_unget_reg(freg); + } + else { + movdlxr(r0, rn(ireg)); + pslq(r0, 32); + xorpdr(r0, r1); + } + jit_unget_reg(ireg); +} + +static void +_ssecmp(jit_state_t *_jit, jit_bool_t d, jit_int32_t code, + jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, reg); + } + ixorr(reg, reg); + if (d) + ucomisdr(r2, r1); + else + ucomissr(r2, r1); + cc(code, reg); + if (!rc) + xchgr(r0, reg); +} + +static void +_sse_movr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 != r1) + ssexr(0xf3, X86_SSE_MOV, r0, r1); +} + +static void +_sse_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t *i0) +{ + union { + jit_int32_t i; + jit_float32_t f; + } data; + + data.f = *i0; + if (data.f == 0.0 && !(data.i & 0x80000000)) + xorpsr(r0, r0); + else { +#if __WORDSIZE == 64 + if (can_sign_extend_int_p((jit_word_t)i0)) + sse_ldi_f(r0, (jit_word_t)i0); + else { + /* if will allocate a register for offset, just use immediate */ + jit_int32_t reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), data.i); + movdlxr(r0, rn(reg)); + jit_unget_reg(reg); + } +#else + sse_ldi_f(r0, (jit_word_t)i0); +#endif + } +} + +fopi(lt) +fopi(le) + +static void +_sse_eqr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + jit_word_t jp_code; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, _RAX_REGNO); + } + ixorr(reg, reg); + ucomissr(r2, r1); + jpes(0); + jp_code = _jit->pc.w; + cc(X86_CC_E, reg); + patch_rel_char(jp_code, _jit->pc.w); + if (!rc) + xchgr(r0, reg); +} + +fopi(eq) +fopi(ge) +fopi(gt) + +static void +_sse_ner_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + jit_word_t jp_code; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, _RAX_REGNO); + } + imovi(reg, 1); + ucomissr(r2, r1); + jpes(0); + jp_code = _jit->pc.w; + cc(X86_CC_NE, reg); + patch_rel_char(jp_code, _jit->pc.w); + if (!rc) + xchgr(r0, reg); +} + +fopi(ne) +fopi(unlt) + +static void +_sse_unler_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + ssecmpf(X86_CC_NA, r0, r2, r1); +} + +fopi(unle) + +static void +_sse_uneqr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + ssecmpf(X86_CC_E, r0, r1, r2); +} + +fopi(uneq) + +static void +_sse_unger_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + ssecmpf(X86_CC_NA, r0, r1, r2); +} + +fopi(unge) +fopi(ungt) + +static void +_sse_ltgtr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + ixorr(r0, r0); + else + ssecmpf(X86_CC_NE, r0, r1, r2); +} + +fopi(ltgt) +fopi(ord) +fopi(unord) + +static void +_sse_ldi_f(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movssmr(i0, _NOREG, _NOREG, _SCL1, r0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_ldr_f(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_sse_ldxi_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movssmr(i0, r1, _NOREG, _SCL1, r0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_ldxr_f(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_sse_sti_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movssrm(r0, i0, _NOREG, _NOREG, _SCL1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_str_f(rn(reg), r0); + jit_unget_reg(reg); + } +} + +static void +_sse_stxi_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movssrm(r1, i0, r0, _NOREG, _SCL1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_stxr_f(rn(reg), r0, r1); + jit_unget_reg(reg); + } +} + +static jit_word_t +_sse_bltr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r1, r0); + ja(i0); + return (_jit->pc.w); +} +fbopi(lt) + +static jit_word_t +_sse_bler_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r1, r0); + jae(i0); + return (_jit->pc.w); +} +fbopi(le) + +static jit_word_t +_sse_beqr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t jp_code; + ucomissr(r0, r1); + jps(0); + jp_code = _jit->pc.w; + je(i0); + patch_rel_char(jp_code, _jit->pc.w); + return (_jit->pc.w); +} +fbopi(eq) + +static jit_word_t +_sse_bger_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r0, r1); + jae(i0); + return (_jit->pc.w); +} +fbopi(ge) + +static jit_word_t +_sse_bgtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r0, r1); + ja(i0); + return (_jit->pc.w); +} +fbopi(gt) + +static jit_word_t +_sse_bner_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t jp_code; + jit_word_t jz_code; + ucomissr(r0, r1); + jps(0); + jp_code = _jit->pc.w; + jzs(0); + jz_code = _jit->pc.w; + patch_rel_char(jp_code, _jit->pc.w); + jmpi(i0); + patch_rel_char(jz_code, _jit->pc.w); + return (_jit->pc.w); +} +fbopi(ne) + +static jit_word_t +_sse_bunltr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r0, r1); + jnae(i0); + return (_jit->pc.w); +} +fbopi(unlt) + +static jit_word_t +_sse_bunler_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + jmpi(i0); + else { + ucomissr(r0, r1); + jna(i0); + } + return (_jit->pc.w); +} +fbopi(unle) + +static jit_word_t +_sse_buneqr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + jmpi(i0); + else { + ucomissr(r0, r1); + je(i0); + } + return (_jit->pc.w); +} +fbopi(uneq) + +static jit_word_t +_sse_bunger_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + jmpi(i0); + else { + ucomissr(r1, r0); + jna(i0); + } + return (_jit->pc.w); +} +fbopi(unge) + +static jit_word_t +_sse_bungtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r1, r0); + jnae(i0); + return (_jit->pc.w); +} +fbopi(ungt) + +static jit_word_t +_sse_bltgtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r0, r1); + jne(i0); + return (_jit->pc.w); +} +fbopi(ltgt) + +static jit_word_t +_sse_bordr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r0, r1); + jnp(i0); + return (_jit->pc.w); +} +fbopi(ord) + +static jit_word_t +_sse_bunordr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomissr(r0, r1); + jp(i0); + return (_jit->pc.w); +} +fbopi(unord) + +dopi(lt) +dopi(le) + +static void +_sse_eqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + jit_word_t jp_code; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, _RAX_REGNO); + } + ixorr(reg, reg); + ucomisdr(r2, r1); + jpes(0); + jp_code = _jit->pc.w; + cc(X86_CC_E, reg); + patch_rel_char(jp_code, _jit->pc.w); + if (!rc) + xchgr(r0, reg); +} + +dopi(eq) +dopi(ge) +dopi(gt) + +static void +_sse_ner_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + jit_word_t jp_code; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, _RAX_REGNO); + } + imovi(reg, 1); + ucomisdr(r2, r1); + jpes(0); + jp_code = _jit->pc.w; + cc(X86_CC_NE, reg); + patch_rel_char(jp_code, _jit->pc.w); + if (!rc) + xchgr(r0, reg); +} + +dopi(ne) +dopi(unlt) + +static void +_sse_unler_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + ssecmpd(X86_CC_NA, r0, r2, r1); +} + +dopi(unle) + +static void +_sse_uneqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + ssecmpd(X86_CC_E, r0, r1, r2); +} + +dopi(uneq) + +static void +_sse_unger_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + ssecmpd(X86_CC_NA, r0, r1, r2); +} + +dopi(unge) +dopi(ungt) + +static void +_sse_ltgtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + ixorr(r0, r0); + else + ssecmpd(X86_CC_NE, r0, r1, r2); +} + +dopi(ltgt) +dopi(ord) +dopi(unord) + +static void +_sse_movr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 != r1) + ssexr(0xf2, X86_SSE_MOV, r0, r1); +} + +static void +_sse_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t *i0) +{ + union { + jit_int32_t ii[2]; + jit_word_t w; + jit_float64_t d; + } data; + + data.d = *i0; + if (data.d == 0.0 && !(data.ii[1] & 0x80000000)) + xorpdr(r0, r0); + else { +#if __WORDSIZE == 64 + if (can_sign_extend_int_p((jit_word_t)i0)) + sse_ldi_d(r0, (jit_word_t)i0); + else { + /* if will allocate a register for offset, just use immediate */ + jit_int32_t reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), data.w); + movdqxr(r0, rn(reg)); + jit_unget_reg(reg); + } +#else + sse_ldi_d(r0, (jit_word_t)i0); +#endif + } +} + +static void +_sse_ldi_d(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movsdmr(i0, _NOREG, _NOREG, _SCL1, r0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_ldr_d(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_sse_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movsdmr(i0, r1, _NOREG, _SCL1, r0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_ldxr_f(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_sse_sti_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movsdrm(r0, i0, _NOREG, _NOREG, _SCL1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_str_d(rn(reg), r0); + jit_unget_reg(reg); + } +} + +static void +_sse_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) + movsdrm(r1, i0, r0, _NOREG, _SCL1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + sse_stxr_f(rn(reg), r0, r1); + jit_unget_reg(reg); + } +} + +static jit_word_t +_sse_bltr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r1, r0); + ja(i0); + return (_jit->pc.w); +} +dbopi(lt) + +static jit_word_t +_sse_bler_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r1, r0); + jae(i0); + return (_jit->pc.w); +} +dbopi(le) + +static jit_word_t +_sse_beqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t jp_code; + ucomisdr(r0, r1); + jps(0); + jp_code = _jit->pc.w; + je(i0); + patch_rel_char(jp_code, _jit->pc.w); + return (_jit->pc.w); +} +dbopi(eq) + +static jit_word_t +_sse_bger_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r0, r1); + jae(i0); + return (_jit->pc.w); +} +dbopi(ge) + +static jit_word_t +_sse_bgtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r0, r1); + ja(i0); + return (_jit->pc.w); +} +dbopi(gt) + +static jit_word_t +_sse_bner_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t jp_code; + jit_word_t jz_code; + ucomisdr(r0, r1); + jps(0); + jp_code = _jit->pc.w; + jzs(0); + jz_code = _jit->pc.w; + patch_rel_char(jp_code, _jit->pc.w); + jmpi(i0); + patch_rel_char(jz_code, _jit->pc.w); + return (_jit->pc.w); +} +dbopi(ne) + +static jit_word_t +_sse_bunltr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r0, r1); + jnae(i0); + return (_jit->pc.w); +} +dbopi(unlt) + +static jit_word_t +_sse_bunler_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + jmpi(i0); + else { + ucomisdr(r0, r1); + jna(i0); + } + return (_jit->pc.w); +} +dbopi(unle) + +static jit_word_t +_sse_buneqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + jmpi(i0); + else { + ucomisdr(r0, r1); + je(i0); + } + return (_jit->pc.w); +} +dbopi(uneq) + +static jit_word_t +_sse_bunger_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) + jmpi(i0); + else { + ucomisdr(r1, r0); + jna(i0); + } + return (_jit->pc.w); +} +dbopi(unge) + +static jit_word_t +_sse_bungtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r1, r0); + jnae(i0); + return (_jit->pc.w); +} +dbopi(ungt) + +static jit_word_t +_sse_bltgtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r0, r1); + jne(i0); + return (_jit->pc.w); +} +dbopi(ltgt) + +static jit_word_t +_sse_bordr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r0, r1); + jnp(i0); + return (_jit->pc.w); +} +dbopi(ord) + +static jit_word_t +_sse_bunordr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + ucomisdr(r0, r1); + jp(i0); + return (_jit->pc.w); +} +dbopi(unord) +# undef fopi +# undef fbopi +# undef bopi +# undef dbopi +# undef fpr_bopi +# undef fpr_opi +#endif diff --git a/lib/jit_x86-x87.c b/lib/jit_x86-x87.c new file mode 100644 index 000000000..1e5d154a1 --- /dev/null +++ b/lib/jit_x86-x87.c @@ -0,0 +1,1215 @@ +/* + * Copyright (C) 2011 Paulo Cesar Pereira de Andrade. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#if PROTO +# define _ST0_REGNO 0 +# define _ST1_REGNO 1 +# define _ST2_REGNO 2 +# define _ST3_REGNO 3 +# define _ST4_REGNO 4 +# define _ST5_REGNO 5 +# define _ST6_REGNO 6 +# define _ST7_REGNO 7 +# define x87rx(code, md, rb, ri, ms) _x87rx(_jit, code, md, rb, ri, ms) +# define fldsm(md, rb, ri, ms) x87rx(010, md, rb, ri, ms) +# define fstsm(md, rb, ri, ms) x87rx(012, md, rb, ri, ms) +# define fldlm(md, rb, ri, ms) x87rx(050, md, rb, ri, ms) +# define fstlm(md, rb, ri, ms) x87rx(052, md, rb, ri, ms) +# define fisttplm(md, rb, ri, ms) x87rx(031, md, rb, ri, ms) +# define fisttpqm(md, rb, ri, ms) x87rx(071, md, rb, ri, ms) +# define fildlm(md, rb, ri, ms) x87rx(030, md, rb,ri, ms) +# define fildqm(md, rb, ri, ms) x87rx(075, md, rb,ri, ms) +static void +_x87rx(jit_state_t*, jit_int32_t, jit_int32_t, + jit_int32_t, jit_int32_t, jit_int32_t); +# define x87ri(cc,r0) _x87ri(_jit,cc,r0) +# define fchs_() x87ri(014, 0) +# define fabs_() x87ri(014, 1) +# define fld1() x87ri(015, 0) +# define fldl2t() x87ri(015, 1) +# define fldl2e() x87ri(015, 2) +# define fldpi() x87ri(015, 3) +# define fldlg2() x87ri(015, 4) +# define fldln2() x87ri(015, 4) +# define fldz() x87ri(015, 6) +# define fsqrt_() x87ri(017, 2) +# define fldr(r0) x87ri(010, r0) +# define fxchr(r0) x87ri(011, r0) +# define fstr(r0) x87ri(052, r0) +# define fstpr(r0) x87ri(053, r0) +# define fucomir(r0) x87ri(035, r0) +# define fucomipr(r0) x87ri(075, r0) +static void _x87ri(jit_state_t*, jit_int32_t, jit_int32_t); +# define faddr(r0, r1) x87rri(000, r0, r1) +# define fmulr(r0, r1) x87rri(001, r0, r1) +# define fsubr(r0, r1) x87rri(004, r0, r1) +# define fsubrr(r0, r1) x87rri(005, r0, r1) +# define fdivr(r0, r1) x87rri(006, r0, r1) +# define fdivrr(r0, r1) x87rri(007, r0, r1) +# define x87rri(cc, r0, r1) _x87rri(_jit, cc, r0, r1) +static void _x87rri(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_addr_f(r0, r1, r2) _x87_addr_d(_jit, r0, r1, r2) +# define x87_addi_f(r0, r1, i0) _x87_addi_f(_jit, r0, r1, i0) +static void _x87_addi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_addr_d(r0, r1, r2) _x87_addr_d(_jit, r0, r1, r2) +static void _x87_addr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_addi_d(r0, r1, i0) _x87_addi_d(_jit, r0, r1, i0) +static void _x87_addi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_subr_f(r0, r1, r2) _x87_subr_d(_jit, r0, r1, r2) +# define x87_subi_f(r0, r1, i0) _x87_subi_f(_jit, r0, r1, i0) +static void _x87_subi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_subr_d(r0, r1, r2) _x87_subr_d(_jit, r0, r1, r2) +static void _x87_subr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_subi_d(r0, r1, i0) _x87_subi_d(_jit, r0, r1, i0) +static void _x87_subi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_mulr_f(r0, r1, r2) _x87_mulr_d(_jit, r0, r1, r2) +# define x87_muli_f(r0, r1, i0) _x87_muli_f(_jit, r0, r1, i0) +static void _x87_muli_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_mulr_d(r0, r1, r2) _x87_mulr_d(_jit, r0, r1, r2) +static void _x87_mulr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_muli_d(r0, r1, i0) _x87_muli_d(_jit, r0, r1, i0) +static void _x87_muli_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_divr_f(r0, r1, r2) _x87_divr_d(_jit, r0, r1, r2) +# define x87_divi_f(r0, r1, i0) _x87_divi_f(_jit, r0, r1, i0) +static void _x87_divi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_divr_d(r0, r1, r2) _x87_divr_d(_jit, r0, r1, r2) +static void _x87_divr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_divi_d(r0, r1, i0) _x87_divi_d(_jit, r0, r1, i0) +static void _x87_divi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_absr_f(r0, r1) _x87_absr_d(_jit, r0, r1) +# define x87_absr_d(r0, r1) _x87_absr_d(_jit, r0, r1) +static void _x87_absr_d(jit_state_t*, jit_int32_t, jit_int32_t); +# define x87_negr_f(r0, r1) _x87_negr_d(_jit, r0, r1) +# define x87_negr_d(r0, r1) _x87_negr_d(_jit, r0, r1) +static void _x87_negr_d(jit_state_t*, jit_int32_t, jit_int32_t); +# define x87_sqrtr_f(r0, r1) _x87_sqrtr_d(_jit, r0, r1) +# define x87_sqrtr_d(r0, r1) _x87_sqrtr_d(_jit, r0, r1) +static void _x87_sqrtr_d(jit_state_t*, jit_int32_t, jit_int32_t); +# define x87_truncr_f_i(r0, r1) _x87_truncr_d_i(_jit, r0, r1) +# define x87_truncr_d_i(r0, r1) _x87_truncr_d_i(_jit, r0, r1) +static void _x87_truncr_d_i(jit_state_t*, jit_int32_t, jit_int32_t); +# if __WORDSIZE == 64 +# define x87_truncr_f_l(r0, r1) _x87_truncr_d_l(_jit, r0, r1) +# define x87_truncr_d_l(r0, r1) _x87_truncr_d_l(_jit, r0, r1) +static void _x87_truncr_d_l(jit_state_t*, jit_int32_t, jit_int32_t); +# endif +# define x87_extr_f(r0, r1) _x87_extr_d(_jit, r0, r1) +# define x87_extr_d(r0, r1) _x87_extr_d(_jit, r0, r1) +# define x87_extr_f_d(r0, r1) x87_movr_d(r0, r1) +# define x87_extr_d_f(r0, r1) x87_movr_d(r0, r1) +static void _x87_extr_d(jit_state_t*, jit_int32_t, jit_int32_t); +# define x87cmp(code, r0, r1, r2) _x87cmp(_jit, code, r0, r1, r2) +static void +_x87cmp(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87cmp2(code, r0, r1, r2) _x87cmp2(_jit, code, r0, r1, r2) +static void +_x87cmp2(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87jcc(code, i0, r0, r1) _x87jcc(_jit, code, i0, r0, r1) +static jit_word_t +_x87jcc(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t, jit_int32_t); +# define x87jcc2(code, i0, r0, r1) _x87jcc2(_jit, code, i0, r0, r1) +static jit_word_t +_x87jcc2(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t, jit_int32_t); +#define x87_movi_f(r0,i0) _x87_movi_f(_jit,r0,i0) +static void _x87_movi_f(jit_state_t*, jit_int32_t, jit_float32_t*); +# define x87_ldr_f(r0, r1) _x87_ldr_f(_jit, r0, r1) +static void _x87_ldr_f(jit_state_t*, jit_int32_t, jit_int32_t); +# define x87_ldi_f(r0, i0) _x87_ldi_f(_jit, r0, i0) +static void _x87_ldi_f(jit_state_t*, jit_int32_t, jit_word_t); +# define x87_ldxr_f(r0, r1, r2) _x87_ldxr_f(_jit, r0, r1, r2) +static void _x87_ldxr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_ldxi_f(r0, r1, i0) _x87_ldxi_f(_jit, r0, r1, i0) +static void _x87_ldxi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define x87_str_f(r0, r1) _x87_str_f(_jit, r0, r1) +static void _x87_str_f(jit_state_t*,jit_int32_t,jit_int32_t); +# define x87_sti_f(i0, r0) _x87_sti_f(_jit, i0, r0) +static void _x87_sti_f(jit_state_t*,jit_word_t, jit_int32_t); +# define x87_stxr_f(r0, r1, r2) _x87_stxr_f(_jit, r0, r1, r2) +static void _x87_stxr_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define x87_stxi_f(i0, r0, r1) _x87_stxi_f(_jit, i0, r0, r1) +static void _x87_stxi_f(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define x87_ltr_f(r0, r1, r2) x87cmp(X86_CC_A, r0, r2, r1) +# define x87_lti_f(r0, r1, i0) _x87_lti_f(_jit, r0, r1, i0) +static void _x87_lti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ler_f(r0, r1, r2) x87cmp(X86_CC_AE, r0, r2, r1) +# define x87_lei_f(r0, r1, i0) _x87_lei_f(_jit, r0, r1, i0) +static void _x87_lei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_eqr_f(r0, r1, r2) x87_eqr_d(r0, r2, r1) +# define x87_eqi_f(r0, r1, i0) _x87_eqi_f(_jit, r0, r1, i0) +static void _x87_eqi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ger_f(r0, r1, r2) x87cmp(X86_CC_AE, r0, r1, r2) +# define x87_gei_f(r0, r1, i0) _x87_gei_f(_jit, r0, r1, i0) +static void _x87_gei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_gtr_f(r0, r1, r2) x87cmp(X86_CC_A, r0, r1, r2) +# define x87_gti_f(r0, r1, i0) _x87_gti_f(_jit, r0, r1, i0) +static void _x87_gti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ner_f(r0, r1, r2) x87_ner_d(r0, r2, r1) +# define x87_nei_f(r0, r1, i0) _x87_nei_f(_jit, r0, r1, i0) +static void _x87_nei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_unltr_f(r0, r1, r2) x87cmp(X86_CC_NAE, r0, r1, r2) +# define x87_unlti_f(r0, r1, i0) _x87_unlti_f(_jit, r0, r1, i0) +static void _x87_unlti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_unler_f(r0, r1, r2) x87cmp(X86_CC_NA, r0, r1, r2) +# define x87_unlei_f(r0, r1, i0) _x87_unlei_f(_jit, r0, r1, i0) +static void _x87_unlei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_uneqr_f(r0, r1, r2) x87cmp2(X86_CC_E, r0, r1, r1) +# define x87_uneqi_f(r0, r1, i0) _x87_uneqi_f(_jit, r0, r1, i0) +static void _x87_uneqi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_unger_f(r0, r1, r2) x87cmp(X86_CC_NA, r0, r2, r1) +# define x87_ungei_f(r0, r1, i0) _x87_ungei_f(_jit, r0, r1, i0) +static void _x87_ungei_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ungtr_f(r0, r1, r2) x87cmp(X86_CC_NAE, r0, r2, r1) +# define x87_ungti_f(r0, r1, i0) _x87_ungti_f(_jit, r0, r1, i0) +static void _x87_ungti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ltgtr_f(r0, r1, r2) x87_ltgtr_d(r0, r1, r2) +# define x87_ltgti_f(r0, r1, i0) _x87_ltgti_f(_jit, r0, r1, i0) +static void _x87_ltgti_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ordr_f(r0, r1, r2) x87cmp2(X86_CC_NP, r0, r2, r1) +# define x87_ordi_f(r0, r1, i0) _x87_ordi_f(_jit, r0, r1, i0) +static void _x87_ordi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_unordr_f(r0, r1, r2) x87cmp2(X86_CC_P, r0, r2, r1) +# define x87_unordi_f(r0, r1, i0) _x87_unordi_f(_jit, r0, r1, i0) +static void _x87_unordi_f(jit_state_t*,jit_int32_t,jit_int32_t,jit_float32_t*); +# define x87_ltr_d(r0, r1, r2) x87cmp(X86_CC_A, r0, r2, r1) +# define x87_lti_d(r0, r1, i0) _x87_lti_d(_jit, r0, r1, i0) +static void _x87_lti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_ler_d(r0, r1, r2) x87cmp(X86_CC_AE, r0, r2, r1) +# define x87_lei_d(r0, r1, i0) _x87_lei_d(_jit, r0, r1, i0) +static void _x87_lei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_eqr_d(r0, r1, r2) _x87_eqr_d(_jit, r0, r2, r1) +static void _x87_eqr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_eqi_d(r0, r1, i0) _x87_eqi_d(_jit, r0, r1, i0) +static void _x87_eqi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_ger_d(r0, r1, r2) x87cmp(X86_CC_AE, r0, r1, r2) +# define x87_gei_d(r0, r1, i0) _x87_gei_d(_jit, r0, r1, i0) +static void _x87_gei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_gtr_d(r0, r1, r2) x87cmp(X86_CC_A, r0, r1, r2) +# define x87_gti_d(r0, r1, i0) _x87_gti_d(_jit, r0, r1, i0) +static void _x87_gti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_ner_d(r0, r1, r2) _x87_ner_d(_jit, r0, r2, r1) +static void _x87_ner_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_nei_d(r0, r1, i0) _x87_nei_d(_jit, r0, r1, i0) +static void _x87_nei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_unltr_d(r0, r1, r2) x87cmp(X86_CC_NAE, r0, r1, r2) +# define x87_unlti_d(r0, r1, i0) _x87_unlti_d(_jit, r0, r1, i0) +static void _x87_unlti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_unler_d(r0, r1, r2) x87cmp(X86_CC_NA, r0, r1, r2) +# define x87_unlei_d(r0, r1, i0) _x87_unlei_d(_jit, r0, r1, i0) +static void _x87_unlei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_uneqr_d(r0, r1, r2) x87cmp2(X86_CC_E, r0, r1, r1) +# define x87_uneqi_d(r0, r1, i0) _x87_uneqi_d(_jit, r0, r1, i0) +static void _x87_uneqi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_unger_d(r0, r1, r2) x87cmp(X86_CC_NA, r0, r2, r1) +# define x87_ungei_d(r0, r1, i0) _x87_ungei_d(_jit, r0, r1, i0) +static void _x87_ungei_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_ungtr_d(r0, r1, r2) x87cmp(X86_CC_NAE, r0, r2, r1) +# define x87_ungti_d(r0, r1, i0) _x87_ungti_d(_jit, r0, r1, i0) +static void _x87_ungti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_ltgtr_d(r0, r1, r2) _x87_ltgtr_d(_jit, r0, r1, r2) +static void _x87_ltgtr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_ltgti_d(r0, r1, i0) _x87_ltgti_d(_jit, r0, r1, i0) +static void _x87_ltgti_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_ordr_d(r0, r1, r2) x87cmp2(X86_CC_NP, r0, r2, r1) +# define x87_ordi_d(r0, r1, i0) _x87_ordi_d(_jit, r0, r1, i0) +static void _x87_ordi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +# define x87_unordr_d(r0, r1, r2) x87cmp2(X86_CC_P, r0, r2, r1) +# define x87_unordi_d(r0, r1, i0) _x87_unordi_d(_jit, r0, r1, i0) +static void _x87_unordi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); +#define x87_movr_f(r0,r1) _x87_movr_d(_jit,r0,r1) +#define x87_movr_d(r0,r1) _x87_movr_d(_jit,r0,r1) +static void _x87_movr_d(jit_state_t*, jit_int32_t, jit_int32_t); +#define x87_movi_d(r0,i0) _x87_movi_d(_jit,r0,i0) +static void _x87_movi_d(jit_state_t*, jit_int32_t, jit_float64_t*); +# define x87_ldr_d(r0, r1) _x87_ldr_d(_jit, r0, r1) +static void _x87_ldr_d(jit_state_t*, jit_int32_t, jit_int32_t); +# define x87_ldi_d(r0, i0) _x87_ldi_d(_jit, r0, i0) +static void _x87_ldi_d(jit_state_t*, jit_int32_t, jit_word_t); +# define x87_ldxr_d(r0, r1, r2) _x87_ldxr_d(_jit, r0, r1, r2) +static void _x87_ldxr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define x87_ldxi_d(r0, r1, i0) _x87_ldxi_d(_jit, r0, r1, i0) +static void _x87_ldxi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define x87_str_d(r0, r1) _x87_str_d(_jit, r0, r1) +static void _x87_str_d(jit_state_t*,jit_int32_t,jit_int32_t); +# define x87_sti_d(i0, r0) _x87_sti_d(_jit, i0, r0) +static void _x87_sti_d(jit_state_t*,jit_word_t,jit_int32_t); +# define x87_stxr_d(r0, r1, r2) _x87_stxr_d(_jit, r0, r1, r2) +static void _x87_stxr_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define x87_stxi_d(i0, r0, r1) _x87_stxi_d(_jit, i0, r0, r1) +static void _x87_stxi_d(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define x87_bltr_f(i0, r0, r1) x87jcc(X86_CC_A, i0, r1, r0) +# define x87_blti_f(i0, r0, i1) _x87_blti_f(_jit, i0, r0, i1) +static jit_word_t +_x87_blti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bler_f(i0, r0, r1) x87jcc(X86_CC_AE, i0, r1, r0) +# define x87_blei_f(i0, r0, i1) _x87_blei_f(_jit, i0, r0, i1) +static jit_word_t +_x87_blei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_beqr_f(i0, r0, r1) _x87_beqr_d(_jit, i0, r0, r1) +# define x87_beqi_f(i0, r0, i1) _x87_beqi_f(_jit, i0, r0, i1) +static jit_word_t +_x87_beqi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bger_f(i0, r0, r1) x87jcc(X86_CC_AE, i0, r0, r1) +# define x87_bgei_f(i0, r0, i1) _x87_bgei_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bgei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bgtr_f(i0, r0, r1) x87jcc(X86_CC_A, i0, r0, r1) +# define x87_bgti_f(i0, r0, i1) _x87_bgti_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bgti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bner_f(i0, r0, r1) _x87_bner_d(_jit, i0, r0, r1) +# define x87_bnei_f(i0, r0, i1) _x87_bnei_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bnei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bunltr_f(i0, r0, r1) x87jcc(X86_CC_NAE, i0, r0, r1) +# define x87_bunlti_f(i0, r0, i1) _x87_bunlti_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bunlti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bunler_f(i0, r0, r1) x87jcc(X86_CC_NA, i0, r0, r1) +# define x87_bunlei_f(i0, r0, i1) _x87_bunlei_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bunlei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_buneqr_f(i0, r0, r1) x87jcc2(X86_CC_E, i0, r0, r1) +# define x87_buneqi_f(i0, r0, i1) _x87_buneqi_f(_jit, i0, r0, i1) +static jit_word_t +_x87_buneqi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bunger_f(i0, r0, r1) x87jcc(X86_CC_NA, i0, r1, r0) +# define x87_bungei_f(i0, r0, i1) _x87_bungei_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bungei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bungtr_f(i0, r0, r1) x87jcc(X86_CC_NAE, i0, r1, r0) +# define x87_bungti_f(i0, r0, i1) _x87_bungti_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bungti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bltgtr_f(i0, r0, r1) x87jcc2(X86_CC_NE, i0, r0, r1) +# define x87_bltgti_f(i0, r0, i1) _x87_bltgti_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bltgti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bordr_f(i0, r0, r1) x87jcc2(X86_CC_NP, i0, r0, r1) +# define x87_bordi_f(i0, r0, i1) _x87_bordi_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bordi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bunordr_f(i0, r0, r1) x87jcc2(X86_CC_P, i0, r0, r1) +# define x87_bunordi_f(i0, r0, i1) _x87_bunordi_f(_jit, i0, r0, i1) +static jit_word_t +_x87_bunordi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t*); +# define x87_bltr_d(i0, r0, r1) x87jcc(X86_CC_A, i0, r1, r0) +# define x87_blti_d(i0, r0, i1) _x87_blti_d(_jit, i0, r0, i1) +static jit_word_t +_x87_blti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bler_d(i0, r0, r1) x87jcc(X86_CC_AE, i0, r1, r0) +# define x87_blei_d(i0, r0, i1) _x87_blei_d(_jit, i0, r0, i1) +static jit_word_t +_x87_blei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_beqr_d(i0, r0, r1) _x87_beqr_d(_jit, i0, r0, r1) +static jit_word_t +_x87_beqr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define x87_beqi_d(i0, r0, i1) _x87_beqi_d(_jit, i0, r0, i1) +static jit_word_t +_x87_beqi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bger_d(i0, r0, r1) x87jcc(X86_CC_AE, i0, r0, r1) +# define x87_bgei_d(i0, r0, i1) _x87_bgei_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bgei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bgtr_d(i0, r0, r1) x87jcc(X86_CC_A, i0, r0, r1) +# define x87_bgti_d(i0, r0, i1) _x87_bgti_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bgti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bner_d(i0, r0, r1) _x87_bner_d(_jit, i0, r0, r1) +static jit_word_t +_x87_bner_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define x87_bnei_d(i0, r0, i1) _x87_bnei_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bnei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bunltr_d(i0, r0, r1) x87jcc(X86_CC_NAE, i0, r0, r1) +# define x87_bunlti_d(i0, r0, i1) _x87_bunlti_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bunlti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bunler_d(i0, r0, r1) x87jcc(X86_CC_NA, i0, r0, r1) +# define x87_bunlei_d(i0, r0, i1) _x87_bunlei_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bunlei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_buneqr_d(i0, r0, r1) x87jcc2(X86_CC_E, i0, r0, r1) +# define x87_buneqi_d(i0, r0, i1) _x87_buneqi_d(_jit, i0, r0, i1) +static jit_word_t +_x87_buneqi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bunger_d(i0, r0, r1) x87jcc(X86_CC_NA, i0, r1, r0) +# define x87_bungei_d(i0, r0, i1) _x87_bungei_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bungei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bungtr_d(i0, r0, r1) x87jcc(X86_CC_NAE, i0, r1, r0) +# define x87_bungti_d(i0, r0, i1) _x87_bungti_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bungti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bltgtr_d(i0, r0, r1) x87jcc2(X86_CC_NE, i0, r0, r1) +# define x87_bltgti_d(i0, r0, i1) _x87_bltgti_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bltgti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bordr_d(i0, r0, r1) x87jcc2(X86_CC_NP, i0, r0, r1) +# define x87_bordi_d(i0, r0, i1) _x87_bordi_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bordi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +# define x87_bunordr_d(i0, r0, r1) x87jcc2(X86_CC_P, i0, r0, r1) +# define x87_bunordi_d(i0, r0, i1) _x87_bunordi_d(_jit, i0, r0, i1) +static jit_word_t +_x87_bunordi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t*); +#endif + +#if CODE +# define fpr_opi(name, type, size) \ +static void \ +_x87_##name##i_##type(jit_state_t *_jit, \ + jit_int32_t r0, jit_int32_t r1, \ + jit_float##size##_t *i0) \ +{ \ + jit_int32_t reg = jit_get_reg(jit_class_fpr); \ + assert(jit_x87_reg_p(reg)); \ + x87_movi_##type(rn(reg), i0); \ + x87_##name##r_##type(r0, r1, rn(reg)); \ + jit_unget_reg(reg); \ +} +# define fpr_bopi(name, type, size) \ +static jit_word_t \ +_x87_b##name##i_##type(jit_state_t *_jit, \ + jit_word_t i0, jit_int32_t r0, \ + jit_float##size##_t *i1) \ +{ \ + jit_word_t word; \ + jit_int32_t reg = jit_get_reg(jit_class_fpr); \ + assert(jit_x87_reg_p(reg)); \ + x87_movi_##type(rn(reg), i1); \ + word = x87_b##name##r_##type(i0, r0, rn(reg)); \ + jit_unget_reg(reg); \ + return (word); \ +} +# define fopi(name) fpr_opi(name, f, 32) +# define fbopi(name) fpr_bopi(name, f, 32) +# define dopi(name) fpr_opi(name, d, 64) +# define dbopi(name) fpr_bopi(name, d, 64) + +static void +_x87rx(jit_state_t *_jit, jit_int32_t code, jit_int32_t md, + jit_int32_t rb, jit_int32_t ri, jit_int32_t ms) +{ + rex(0, 1, rb, ri, _NOREG); + ic(0xd8 | (code >> 3)); + rx((code & 7), md, rb, ri, ms); +} + +static void +_x87ri(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0) +{ + ic(0xd8 | (code >> 3)); + mrm(0x03, (code & 7), r0); +} + +static void +_x87rri(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1) +{ + if (r1 == _ST0_REGNO) + x87ri(code | 040, r0); + else { + assert(r0 == _ST0_REGNO); + x87ri(code, r1); + } +} + +fopi(add) +fopi(sub) +fopi(mul) +fopi(div) + +static void +_x87_addr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) { + if (r2 == _ST0_REGNO) + faddr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + faddr(_ST0_REGNO, r2); + else { + fxchr(r0); + faddr(_ST0_REGNO, r2); + fxchr(r0); + } + } + else if (r0 == r2) { + if (r1 == _ST0_REGNO) + faddr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + faddr(_ST0_REGNO, r1); + else { + fxchr(r0); + faddr(_ST0_REGNO, r1); + fxchr(r0); + } + } + else { + fldr(r1); + faddr(_ST0_REGNO, r2 + 1); + fstpr(r0 + 1); + } +} + +dopi(add) + +static void +_x87_subr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) { + if (r2 == _ST0_REGNO) + fsubrr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + fsubr(_ST0_REGNO, r2); + else { + fxchr(r0); + fsubr(_ST0_REGNO, r0); + fxchr(r0); + } + } + else if (r0 == r2) { + if (r1 == _ST0_REGNO) + fsubr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + fsubrr(_ST0_REGNO, r1); + else { + fxchr(r0); + fsubrr(_ST0_REGNO, r1); + fxchr(r0); + } + } + else { + fldr(r1); + fsubr(_ST0_REGNO, r2 + 1); + fstpr(r0 + 1); + } +} + +dopi(sub) + +static void +_x87_mulr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) { + if (r2 == _ST0_REGNO) + fmulr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + fmulr(_ST0_REGNO, r2); + else { + fxchr(r0); + fmulr(_ST0_REGNO, r2); + fxchr(r0); + } + } + else if (r0 == r2) { + if (r1 == _ST0_REGNO) + fmulr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + fmulr(_ST0_REGNO, r1); + else { + fxchr(r0); + fmulr(_ST0_REGNO, r1); + fxchr(r0); + } + } + else { + fldr(r1); + fmulr(_ST0_REGNO, r2 + 1); + fstpr(r0 + 1); + } +} + +dopi(mul) + +static void +_x87_divr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r0 == r1) { + if (r2 == _ST0_REGNO) + fdivrr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + fdivr(_ST0_REGNO, r2); + else { + fxchr(r0); + fdivr(_ST0_REGNO, r0); + fxchr(r0); + } + } + else if (r0 == r2) { + if (r1 == _ST0_REGNO) + fdivr(r0, _ST0_REGNO); + else if (r0 == _ST0_REGNO) + fsubrr(_ST0_REGNO, r1); + else { + fxchr(r0); + fdivrr(_ST0_REGNO, r1); + fxchr(r0); + } + } + else { + fldr(r1); + fdivr(_ST0_REGNO, r2 + 1); + fstpr(r0 + 1); + } +} + +dopi(div) + +static void +_x87_absr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) { + if (r1 == _ST0_REGNO) + fabs_(); + else { + fxchr(r0); + fabs_(); + fxchr(r0); + } + } + else { + fldr(r1); + fabs_(); + fstpr(r0 + 1); + } +} + +static void +_x87_negr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) { + if (r1 == _ST0_REGNO) + fchs_(); + else { + fxchr(r0); + fchs_(); + fxchr(r0); + } + } + else { + fldr(r1); + fchs_(); + fstpr(r0 + 1); + } +} + +static void +_x87_sqrtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == r1) { + if (r1 == _ST0_REGNO) + fsqrt_(); + else { + fxchr(r0); + fsqrt_(); + fxchr(r0); + } + } + else { + fldr(r1); + fsqrt_(); + fstpr(r0 + 1); + } +} + +static void +_x87_truncr_d_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + pushr(_RAX_REGNO); + fldr(r1); + fisttplm(0, _RSP_REGNO, _NOREG, _SCL1); + popr(r0); +} + +# if __WORDSIZE == 64 +static void +_x87_truncr_d_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + pushr(_RAX_REGNO); + fldr(r1); + fisttpqm(0, _RSP_REGNO, _NOREG, _SCL1); + popr(r0); +} +# endif + +static void +_x87_extr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + pushr(r1); +# if __WORDSIZE == 32 + fildlm(0, _RSP_REGNO, _NOREG, _SCL1); +# else + fildqm(0, _RSP_REGNO, _NOREG, _SCL1); +# endif + fstpr(r0 + 1); + popr(r1); +} + +static void +_x87cmp(jit_state_t *_jit, jit_int32_t code, + jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, reg); + } + ixorr(reg, reg); + if (r1 == _ST0_REGNO) + fucomir(r2); + else { + fldr(r1); + fucomipr(r2 + 1); + } + cc(code, reg); + if (!rc) + xchgr(r0, reg); +} + +static void +_x87cmp2(jit_state_t *_jit, jit_int32_t code, + jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_int32_t reg; + jit_int32_t f1, f2; + if (r2 == _ST0_REGNO) f1 = r2, f2 = r1; + else f1 = r1, f2 = r2; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, reg); + } + ixorr(reg, reg); + if (f1 == _ST0_REGNO) + fucomir(f2); + else { + fldr(f1); + fucomipr(f2 + 1); + } + cc(code, reg); + if (!rc) + xchgr(r0, reg); +} + +static jit_word_t +_x87jcc(jit_state_t *_jit, jit_int32_t code, + jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 == _ST0_REGNO) + fucomir(r1); + else { + fldr(r0); + fucomipr(r1 + 1); + } + jcc(code, i0); + return (_jit->pc.w); +} + +static jit_word_t +_x87jcc2(jit_state_t *_jit, jit_int32_t code, + jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t f0, f1; + if (r1 == _ST0_REGNO) f0 = r1, f1 = r0; + else f0 = r0, f1 = r1; + if (f0 == _ST0_REGNO) + fucomir(f1); + else { + fldr(f0); + fucomipr(f1 + 1); + } + jcc(code, i0); + return (_jit->pc.w); +} + +fopi(lt) +fopi(le) +fopi(eq) +fopi(ge) +fopi(gt) +fopi(ne) +fopi(unlt) +fopi(unle) +fopi(uneq) +fopi(unge) +fopi(ungt) +fopi(ltgt) +fopi(ord) +fopi(unord) +fbopi(lt) +fbopi(le) +fbopi(eq) +fbopi(ge) +fbopi(gt) +fbopi(ne) +fbopi(unlt) +fbopi(unle) +fbopi(uneq) +fbopi(unge) +fbopi(ungt) +fbopi(ltgt) +fbopi(ord) +fbopi(unord) + +static void +_x87_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t *i0) +{ + union { + jit_int32_t i; + jit_float32_t f; + } data; + + data.f = *i0; + if (data.f == 0.0 && !(data.i & 0x80000000)) + fldz(); + else if (data.f == 1.0) + fld1(); + else if (data.f == 3.3219280948873623478703195458468f) + fldl2t(); + else if (data.f == 1.4426950408889634073599246886656f) + fldl2e(); + else if (data.f == 3.1415926535897932384626421096161f) + fldpi(); + else if (data.f == 0.3010299956639811952137387498515f) + fldlg2(); + else if (data.f == 0.6931471805599453094172323683399f) + fldln2(); + else { + x87_ldi_f(r0, (jit_word_t)i0); + return; + } + fstpr(r0 + 1); +} + +static void +_x87_ldr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + fldsm(0, r1, _NOREG, _SCL1); + fstpr(r0 + 1); +} + +static void +_x87_ldi_f(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + fldsm(i0, _NOREG, _NOREG, _SCL1); + fstpr(r0 + 1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + x87_ldr_f(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_x87_ldxr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + fldsm(0, r1, r2, _SCL1); + fstpr(r0 + 1); +} + +static void +_x87_ldxi_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + fldsm(i0, r1, _NOREG, _SCL1); + fstpr(r0 + 1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + x87_ldxr_f(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_x87_str_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r1 == _ST0_REGNO) + fstsm(0, r0, _NOREG, _SCL1); + else { + fxchr(r1); + fstsm(0, r0, _NOREG, _SCL1); + fxchr(r1); + } +} + +static void +_x87_sti_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (!can_sign_extend_int_p(i0)) { + reg = jit_get_reg(jit_class_gpr); + jit_movi(rn(reg), i0); + x87_str_f(rn(reg), r0); + jit_unget_reg(reg); + } + else if (r0 == _ST0_REGNO) + fstsm(i0, _NOREG, _NOREG, _SCL1); + else { + fxchr(r0); + fstsm(i0, _NOREG, _NOREG, _SCL1); + fxchr(r0); + } +} + +static void +_x87_stxr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r2 == _ST0_REGNO) + fstsm(0, r0, r1, _SCL1); + else { + fxchr(r2); + fstsm(0, r0, r1, _SCL1); + fxchr(r2); + } +} + +static void +_x87_stxi_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (!can_sign_extend_int_p(i0)) { + reg = jit_get_reg(jit_class_gpr); + jit_movi(rn(reg), i0); + x87_stxr_f(rn(reg), r0, r1); + jit_unget_reg(reg); + } + else if (r0 == _ST0_REGNO) + fstsm(i0, r0, _NOREG, _SCL1); + else { + fxchr(r1); + fstsm(i0, r0, _NOREG, _SCL1); + fxchr(r1); + } +} + +static void +_x87_movr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 != r1) { + if (r1 == _ST0) + fstr(r0); + else if (r0 == _ST0) { + fxchr(r1); + fstr(r1); + } + else { + fldr(r1); + fstpr(r0 + 1); + } + } +} + +static void +_x87_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t *i0) +{ + union { + jit_int32_t ii[2]; + jit_word_t w; + jit_float64_t d; + } data; + + data.d = *i0; + if (data.d == 0.0 && !(data.ii[1] & 0x80000000)) + fldz(); + else if (data.d == 1.0) + fld1(); + else if (data.d == 3.3219280948873623478703195458468) + fldl2t(); + else if (data.d == 1.4426950408889634073599246886656) + fldl2e(); + else if (data.d == 3.1415926535897932384626421096161) + fldpi(); + else if (data.d == 0.3010299956639811952137387498515) + fldlg2(); + else if (data.d == 0.6931471805599453094172323683399) + fldln2(); + else { + x87_ldi_d(r0, (jit_word_t)i0); + return; + } + fstpr(r0 + 1); +} + +dopi(lt) +dopi(le) + +static void +_x87_eqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_word_t jp_code; + jit_int32_t reg, f1, f2; + if (r2 == _ST0_REGNO) f1 = r2, f2 = r1; + else f1 = r1, f2 = r2; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, reg); + } + ixorr(reg, reg); + if (f1 == _ST0_REGNO) + fucomir(f2); + else { + fldr(f1); + fucomipr(f2 + 1); + } + jpes(0); + jp_code = _jit->pc.w; + cc(X86_CC_E, reg); + patch_rel_char(jp_code, _jit->pc.w); + if (!rc) + xchgr(r0, reg); +} + +dopi(eq) +dopi(ge) +dopi(gt) + +static void +_x87_ner_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_bool_t rc; + jit_word_t jp_code; + jit_int32_t reg, f1, f2; + if (r2 == _ST0_REGNO) f1 = r2, f2 = r1; + else f1 = r1, f2 = r2; + if ((rc = reg8_p(r0))) + reg = r0; + else { + reg = _RAX_REGNO; + movr(r0, reg); + } + imovi(reg, 1); + if (f1 == _ST0_REGNO) + fucomir(f2); + else { + fldr(f1); + fucomipr(f2 + 1); + } + jpes(0); + jp_code = _jit->pc.w; + cc(X86_CC_NE, reg); + patch_rel_char(jp_code, _jit->pc.w); + if (!rc) + xchgr(r0, reg); +} + +dopi(ne) +dopi(unlt) +dopi(unle) +dopi(uneq) +dopi(unge) +dopi(ungt) + +static void +_x87_ltgtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r1 == r2) + movi(r0, 1); + else + x87cmp2(X86_CC_NE, r0, r1, r2); +} + +dopi(ltgt) +dopi(ord) +dopi(unord) + +static void +_x87_ldr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + fldlm(0, r1, _NOREG, _SCL1); + fstpr(r0 + 1); +} + +static void +_x87_ldi_d(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + fldlm(i0, _NOREG, _NOREG, _SCL1); + fstpr(r0 + 1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + x87_ldr_f(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_x87_ldxr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + fldlm(0, r1, r2, _SCL1); + fstpr(r0 + 1); +} + +static void +_x87_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_int_p(i0)) { + fldlm(i0, r1, _NOREG, _SCL1); + fstpr(r0 + 1); + } + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + x87_ldxr_d(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_x87_str_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r1 == _ST0_REGNO) + fstlm(0, r0, _NOREG, _SCL1); + else { + fxchr(r1); + fstlm(0, r0, _NOREG, _SCL1); + fxchr(r1); + } +} + +static void +_x87_sti_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (!can_sign_extend_int_p(i0)) { + reg = jit_get_reg(jit_class_gpr); + jit_movi(rn(reg), i0); + x87_str_d(rn(reg), r0); + jit_unget_reg(reg); + } + else if (r0 == _ST0_REGNO) + fstlm(i0, _NOREG, _NOREG, _SCL1); + else { + fxchr(r0); + fstlm(i0, _NOREG, _NOREG, _SCL1); + fxchr(r0); + } +} + +static void +_x87_stxr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + if (r2 == _ST0_REGNO) + fstlm(0, r0, r1, _SCL1); + else { + fxchr(r2); + fstlm(0, r0, r1, _SCL1); + fxchr(r2); + } +} + +static void +_x87_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (!can_sign_extend_int_p(i0)) { + reg = jit_get_reg(jit_class_gpr); + jit_movi(rn(reg), i0); + x87_stxr_d(rn(reg), r0, r1); + jit_unget_reg(reg); + } + else if (r1 == _ST0_REGNO) + fstlm(i0, r0, _NOREG, _SCL1); + else { + fxchr(r1); + fstlm(i0, r0, _NOREG, _SCL1); + fxchr(r1); + } +} + +dbopi(lt) +dbopi(le) + +static jit_word_t +_x87_beqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t f0, f1; + jit_word_t jp_code; + if (r1 == _ST0_REGNO) f0 = r1, f1 = r0; + else f0 = r0, f1 = r1; + if (f0 == _ST0_REGNO) + fucomir(f1); + else { + fldr(f0); + fucomipr(f1 + 1); + } + jpes(0); + jp_code = _jit->pc.w; + jcc(X86_CC_E, i0); + patch_rel_char(jp_code, _jit->pc.w); + return (_jit->pc.w); +} +dbopi(eq) +dbopi(ge) +dbopi(gt) + +static jit_word_t +_x87_bner_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t f0, f1; + jit_word_t jp_code; + jit_word_t jz_code; + if (r1 == _ST0_REGNO) f0 = r1, f1 = r0; + else f0 = r0, f1 = r1; + if (f0 == _ST0_REGNO) + fucomir(f1); + else { + fldr(f0); + fucomipr(f1 + 1); + } + jpes(0); + jp_code = _jit->pc.w; + jzs(0); + jz_code = _jit->pc.w; + patch_rel_char(jp_code, _jit->pc.w); + jmpi(i0); + patch_rel_char(jz_code, _jit->pc.w); + return (_jit->pc.w); +} +dbopi(ne) +dbopi(unlt) +dbopi(unle) +dbopi(uneq) +dbopi(unge) +dbopi(ungt) +dbopi(ltgt) +dbopi(ord) +dbopi(unord) +# undef fopi +# undef fbopi +# undef dopi +# undef dbopi +# undef fpr_bopi +# undef fpr_opi +#endif diff --git a/lib/jit_x86.c b/lib/jit_x86.c new file mode 100644 index 000000000..939e0c97e --- /dev/null +++ b/lib/jit_x86.c @@ -0,0 +1,1699 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#include +#include + +#define rc(value) jit_class_##value +#define rn(reg) (jit_regno(_rvs[jit_regno(reg)].spec)) + +#if __WORDSIZE == 32 +# define stack_alignment 4 +# define stack_framesize 20 +#else +# define stack_alignment 8 +# define stack_framesize 56 +#endif + +/* + * Prototypes + */ +#define patch(instr, node) _patch(_jit, instr, node) +static void _patch(jit_state_t*,jit_word_t,jit_node_t*); +#define sse_from_x87_f(r0, r1) _sse_from_x87_f(_jit, r0, r1) +static void _sse_from_x87_f(jit_state_t*,jit_int32_t,jit_int32_t); +#define sse_from_x87_d(r0, r1) _sse_from_x87_d(_jit, r0, r1) +static void _sse_from_x87_d(jit_state_t*,jit_int32_t,jit_int32_t); +#define x87_from_sse_f(r0, r1) _x87_from_sse_f(_jit, r0, r1) +static void _x87_from_sse_f(jit_state_t*,jit_int32_t,jit_int32_t); +#define x87_from_sse_d(r0, r1) _x87_from_sse_d(_jit, r0, r1) +static void _x87_from_sse_d(jit_state_t*,jit_int32_t,jit_int32_t); + +#define PROTO 1 +# include "jit_x86-cpu.c" +# include "jit_x86-sse.c" +# include "jit_x86-x87.c" +#undef PROTO + +/* + * Initialization + */ +jit_cpu_t jit_cpu; +jit_register_t _rvs[] = { +#if __WORDSIZE == 32 + { rc(gpr) | rc(rg8) | 0, "%eax" }, + { rc(gpr) | rc(rg8) | 1, "%ecx" }, + { rc(gpr) | rc(rg8) | 2, "%edx" }, + { rc(sav) | rc(rg8) | rc(gpr) | 3, "%ebx" }, + { rc(sav) | rc(gpr) | 6, "%esi" }, + { rc(sav) | rc(gpr) | 7, "%edi" }, + { rc(sav) | 4, "%esp" }, + { rc(sav) | 5, "%ebp" }, + { rc(xpr) | rc(fpr) | 0, "%xmm0" }, + { rc(xpr) | rc(fpr) | 1, "%xmm1" }, + { rc(xpr) | rc(fpr) | 2, "%xmm2" }, + { rc(xpr) | rc(fpr) | 3, "%xmm3" }, + { rc(xpr) | rc(fpr) | 4, "%xmm4" }, + { rc(xpr) | rc(fpr) | 5, "%xmm5" }, + { rc(xpr) | rc(fpr) | 6, "%xmm6" }, + { rc(xpr) | rc(fpr) | 7, "%xmm7" }, + { rc(fpr) | 0, "st(0)" }, + { rc(fpr) | 1, "st(1)" }, + { rc(fpr) | 2, "st(2)" }, + { rc(fpr) | 3, "st(3)" }, + { rc(fpr) | 4, "st(4)" }, + { rc(fpr) | 5, "st(5)" }, + { rc(fpr) | 6, "st(6)" }, + { rc(fpr) | 7, "st(7)" }, +#else + /* %rax is a pseudo flag argument for varargs functions */ + { rc(arg) | rc(gpr) | rc(rg8) | 0, "%rax" }, + { rc(gpr) | rc(rg8) | 10, "%r10" }, + { rc(gpr) | rc(rg8) | 11, "%r11" }, + { rc(gpr) | rc(rg8) | 12, "%r12" }, + { rc(sav) | rc(rg8) | rc(gpr) | 3, "%rbx" }, + { rc(sav) | rc(rg8) | rc(gpr) | 13, "%r13" }, + { rc(sav) | rc(rg8) | rc(gpr) | 14, "%r14" }, + { rc(sav) | rc(rg8) | rc(gpr) | 15, "%r15" }, + { rc(arg) | rc(rg8) | rc(gpr) | 9, "%r9" }, + { rc(arg) | rc(rg8) | rc(gpr) | 8, "%r8" }, + { rc(arg) | rc(rg8) | rc(gpr) | 1, "%rcx" }, + { rc(arg) | rc(rg8) | rc(gpr) | 2, "%rdx" }, + { rc(arg) | rc(rg8) | rc(gpr) | 6, "%rsi" }, + { rc(arg) | rc(rg8) | rc(gpr) | 7, "%rdi" }, + { rc(sav) | 4, "%rsp" }, + { rc(sav) | 5, "%rbp" }, + { rc(xpr) | rc(fpr) | 8, "%xmm8" }, + { rc(xpr) | rc(fpr) | 9, "%xmm9" }, + { rc(xpr) | rc(fpr) | 10, "%xmm10" }, + { rc(xpr) | rc(fpr) | 11, "%xmm11" }, + { rc(xpr) | rc(fpr) | 12, "%xmm12" }, + { rc(xpr) | rc(fpr) | 13, "%xmm13" }, + { rc(xpr) | rc(fpr) | 14, "%xmm14" }, + { rc(xpr) | rc(fpr) | 15, "%xmm15" }, + { rc(xpr) | rc(arg) | rc(fpr) | 7, "%xmm7" }, + { rc(xpr) | rc(arg) | rc(fpr) | 6, "%xmm6" }, + { rc(xpr) | rc(arg) | rc(fpr) | 5, "%xmm5" }, + { rc(xpr) | rc(arg) | rc(fpr) | 4, "%xmm4" }, + { rc(xpr) | rc(arg) | rc(fpr) | 3, "%xmm3" }, + { rc(xpr) | rc(arg) | rc(fpr) | 2, "%xmm2" }, + { rc(xpr) | rc(arg) | rc(fpr) | 1, "%xmm1" }, + { rc(xpr) | rc(arg) | rc(fpr) | 0, "%xmm0" }, + { rc(fpr) | 0, "st(0)" }, + { rc(fpr) | 1, "st(1)" }, + { rc(fpr) | 2, "st(2)" }, + { rc(fpr) | 3, "st(3)" }, + { rc(fpr) | 4, "st(4)" }, + { rc(fpr) | 5, "st(5)" }, + { rc(fpr) | 6, "st(6)" }, + { rc(fpr) | 7, "st(7)" }, +#endif + { _NOREG, "" }, +}; + +/* + * Implementation + */ +void +jit_get_cpu(void) +{ + union { + struct { + jit_uint32_t sse3 : 1; + jit_uint32_t pclmulqdq : 1; + jit_uint32_t dtes64 : 1; /* amd reserved */ + jit_uint32_t monitor : 1; + jit_uint32_t ds_cpl : 1; /* amd reserved */ + jit_uint32_t vmx : 1; /* amd reserved */ + jit_uint32_t smx : 1; /* amd reserved */ + jit_uint32_t est : 1; /* amd reserved */ + jit_uint32_t tm2 : 1; /* amd reserved */ + jit_uint32_t ssse3 : 1; + jit_uint32_t cntx_id : 1; /* amd reserved */ + jit_uint32_t __reserved0 : 1; + jit_uint32_t fma : 1; + jit_uint32_t cmpxchg16b : 1; + jit_uint32_t xtpr : 1; /* amd reserved */ + jit_uint32_t pdcm : 1; /* amd reserved */ + jit_uint32_t __reserved1 : 1; + jit_uint32_t pcid : 1; /* amd reserved */ + jit_uint32_t dca : 1; /* amd reserved */ + jit_uint32_t sse4_1 : 1; + jit_uint32_t sse4_2 : 1; + jit_uint32_t x2apic : 1; /* amd reserved */ + jit_uint32_t movbe : 1; /* amd reserved */ + jit_uint32_t popcnt : 1; + jit_uint32_t tsc : 1; /* amd reserved */ + jit_uint32_t aes : 1; + jit_uint32_t xsave : 1; + jit_uint32_t osxsave : 1; + jit_uint32_t avx : 1; + jit_uint32_t __reserved2 : 1; /* amd F16C */ + jit_uint32_t __reserved3 : 1; + jit_uint32_t __alwayszero : 1; /* amd RAZ */ + } bits; + jit_uint32_t cpuid; + } ecx; + union { + struct { + jit_uint32_t fpu : 1; + jit_uint32_t vme : 1; + jit_uint32_t de : 1; + jit_uint32_t pse : 1; + jit_uint32_t tsc : 1; + jit_uint32_t msr : 1; + jit_uint32_t pae : 1; + jit_uint32_t mce : 1; + jit_uint32_t cmpxchg8b : 1; + jit_uint32_t apic : 1; + jit_uint32_t __reserved0 : 1; + jit_uint32_t sep : 1; + jit_uint32_t mtrr : 1; + jit_uint32_t pge : 1; + jit_uint32_t mca : 1; + jit_uint32_t cmov : 1; + jit_uint32_t pat : 1; + jit_uint32_t pse36 : 1; + jit_uint32_t psn : 1; /* amd reserved */ + jit_uint32_t clfsh : 1; + jit_uint32_t __reserved1 : 1; + jit_uint32_t ds : 1; /* amd reserved */ + jit_uint32_t acpi : 1; /* amd reserved */ + jit_uint32_t mmx : 1; + jit_uint32_t fxsr : 1; + jit_uint32_t sse : 1; + jit_uint32_t sse2 : 1; + jit_uint32_t ss : 1; /* amd reserved */ + jit_uint32_t htt : 1; + jit_uint32_t tm : 1; /* amd reserved */ + jit_uint32_t __reserved2 : 1; + jit_uint32_t pbe : 1; /* amd reserved */ + } bits; + jit_uint32_t cpuid; + } edx; +#if __WORDSIZE == 32 + int ac, flags; +#endif + jit_uint32_t eax, ebx; + +#if __WORDSIZE == 32 + /* adapted from glibc __sysconf */ + __asm__ volatile ("pushfl;\n\t" + "popl %0;\n\t" + "movl $0x240000, %1;\n\t" + "xorl %0, %1;\n\t" + "pushl %1;\n\t" + "popfl;\n\t" + "pushfl;\n\t" + "popl %1;\n\t" + "xorl %0, %1;\n\t" + "pushl %0;\n\t" + "popfl" + : "=r" (flags), "=r" (ac)); + + /* i386 or i486 without cpuid */ + if ((ac & (1 << 21)) == 0) + /* probably without x87 as well */ + return; +#endif + + /* query %eax = 1 function */ + __asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1" + : "=a" (eax), "=r" (ebx), + "=c" (ecx.cpuid), "=d" (edx.cpuid) + : "0" (1)); + + jit_cpu.fpu = edx.bits.fpu; + jit_cpu.cmpxchg8b = edx.bits.cmpxchg8b; + jit_cpu.cmov = edx.bits.cmov; + jit_cpu.mmx = edx.bits.mmx; + jit_cpu.sse = edx.bits.sse; + jit_cpu.sse2 = edx.bits.sse2; + jit_cpu.sse3 = ecx.bits.sse3; + jit_cpu.pclmulqdq = ecx.bits.pclmulqdq; + jit_cpu.ssse3 = ecx.bits.ssse3; + jit_cpu.fma = ecx.bits.fma; + jit_cpu.cmpxchg16b = ecx.bits.cmpxchg16b; + jit_cpu.sse4_1 = ecx.bits.sse4_1; + jit_cpu.sse4_2 = ecx.bits.sse4_2; + jit_cpu.movbe = ecx.bits.movbe; + jit_cpu.popcnt = ecx.bits.popcnt; + jit_cpu.aes = ecx.bits.aes; + jit_cpu.avx = ecx.bits.avx; + +#if __WORDSIZE == 64 + /* query %eax = 0x80000001 function */ + __asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1" + : "=a" (eax), "=r" (ebx), + "=c" (ecx.cpuid), "=d" (edx.cpuid) + : "0" (0x80000001)); + jit_cpu.lahf = ecx.cpuid & 1; +#endif +} + +void +_jit_init(jit_state_t *_jit) +{ + _jit->reglen = jit_size(_rvs) - 1; +#if __WORDSIZE == 32 + if (!jit_cpu.sse2) { + jit_int32_t regno; + + for (regno = _jit->reglen; regno >= 0; regno--) { + if (_rvs[regno].spec & jit_class_xpr) + _rvs[regno].spec = 0; + } + } +#endif +} + +void +_jit_prolog(jit_state_t *_jit) +{ + jit_int32_t offset; + + if (_jit->function) + jit_epilog(); + assert(jit_regset_cmp_ui(_jit->regarg, 0) == 0); + jit_regset_set_ui(_jit->regsav, 0); + offset = _jit->functions.offset; + if (offset >= _jit->functions.length) { + _jit->functions.ptr = realloc(_jit->functions.ptr, + (_jit->functions.length + 16) * + sizeof(jit_function_t)); + memset(_jit->functions.ptr + _jit->functions.length, 0, + 16 * sizeof(jit_function_t)); + _jit->functions.length += 16; + } + _jit->function = _jit->functions.ptr + _jit->functions.offset++; + _jit->function->self.size = stack_framesize; + _jit->function->self.argi = _jit->function->self.argf = + _jit->function->self.aoff = _jit->function->self.alen = 0; + /* sse/x87 conversion */ + _jit->function->self.aoff = -8; + _jit->function->regoff = calloc(_jit->reglen, sizeof(jit_int32_t)); + + _jit->function->prolog = jit_new_node_no_link(jit_code_prolog); + jit_link(_jit->function->prolog); + _jit->function->prolog->w.w = offset; + _jit->function->epilog = jit_new_node_no_link(jit_code_epilog); + /* u: label value + * v: offset in blocks vector + * w: offset in functions vector + */ + _jit->function->epilog->w.w = offset; + + jit_regset_new(_jit->function->regset); +} + +jit_int32_t +_jit_allocai(jit_state_t *_jit, jit_int32_t length) +{ + assert(_jit->function); + switch (length) { + case 0: case 1: break; + case 2: _jit->function->self.aoff &= -2; break; + case 3: case 4: _jit->function->self.aoff &= -4; break; + default: _jit->function->self.aoff &= -8; break; + } + _jit->function->self.aoff -= length; + return (_jit->function->self.aoff); +} + +void +_jit_ret(jit_state_t *_jit) +{ + jit_node_t *instr; + + assert(_jit->function); + + /* jump to epilog */ + instr = jit_jmpi(); + jit_patch_at(instr, _jit->function->epilog); +} + +void +_jit_retr(jit_state_t *_jit, jit_int32_t u) +{ + if (JIT_RET != u) + jit_movr(JIT_RET, u); + jit_ret(); +} + +void +_jit_reti(jit_state_t *_jit, jit_word_t u) +{ + jit_movi(JIT_RET, u); + jit_ret(); +} + +void +_jit_retr_f(jit_state_t *_jit, jit_int32_t u) +{ + if (JIT_FRET != u) + jit_movr_f(JIT_FRET, u); + jit_ret(); +} + +void +_jit_reti_f(jit_state_t *_jit, jit_float32_t u) +{ + jit_movi_f(JIT_FRET, u); + jit_ret(); +} + +void +_jit_retr_d(jit_state_t *_jit, jit_int32_t u) +{ + if (JIT_FRET != u) + jit_movr_d(JIT_FRET, u); + jit_ret(); +} + +void +_jit_reti_d(jit_state_t *_jit, jit_float64_t u) +{ + jit_movi_d(JIT_FRET, u); + jit_ret(); +} + +void +_jit_epilog(jit_state_t *_jit) +{ + assert(_jit->function); +#if __WORDSIZE == 32 + _jit->function->stack = (((_jit->function->self.alen - + _jit->function->self.aoff) + 15) & -16) + 12; +#else + _jit->function->stack = (((_jit->function->self.alen - + _jit->function->self.aoff) + 15) & -16) + 8; +#endif + assert(_jit->function->epilog->next == NULL); + jit_link(_jit->function->epilog); + _jit->function = NULL; +} + +jit_int32_t +_jit_arg(jit_state_t *_jit) +{ + jit_int32_t offset; + + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->self.argi < 6) + return (_jit->function->self.argi++); +#endif + offset = _jit->function->self.size; + _jit->function->self.size += stack_alignment; + return (offset); +} + +jit_bool_t +_jit_arg_reg_p(jit_state_t *_jit, jit_int32_t offset) +{ +#if __WORDSIZE == 32 + return (0); +#else + return (offset >= 0 && offset < 6); +#endif +} + +jit_int32_t +_jit_arg_f(jit_state_t *_jit) +{ + jit_int32_t offset; + + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->self.argf < 8) + return (_jit->function->self.argf++); +#endif + offset = _jit->function->self.size; + _jit->function->self.size += stack_alignment; + return (offset); +} + +jit_bool_t +_jit_arg_f_reg_p(jit_state_t *_jit, jit_int32_t offset) +{ +#if __WORDSIZE == 32 + return (0); +#else + return (offset >= 0 && offset < 8); +#endif +} + +jit_int32_t +_jit_arg_d(jit_state_t *_jit) +{ +#if __WORDSIZE == 64 + return (jit_arg_f()); +#else + jit_int32_t offset; + + assert(_jit->function); + offset = _jit->function->self.size; + _jit->function->self.size += 8; + return (offset); +#endif +} + +jit_bool_t +_jit_arg_d_reg_p(jit_state_t *_jit, jit_int32_t offset) +{ + return (jit_arg_f_reg_p(offset)); +} + +void +_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 6) + jit_extr_c(u, _RDI - v); + else +#endif + jit_ldxi_c(u, _RBP, v); +} + +void +_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 6) + jit_extr_uc(u, _RDI - v); + else +#endif + jit_ldxi_uc(u, _RBP, v); +} + +void +_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 6) + jit_extr_s(u, _RDI - v); + else +#endif + jit_ldxi_s(u, _RBP, v); +} + +void +_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 6) + jit_extr_us(u, _RDI - v); + else +#endif + jit_ldxi_us(u, _RBP, v); +} + +void +_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 6) + jit_extr_i(u, _RDI - v); + else +#endif + jit_ldxi_i(u, _RBP, v); +} + +#if __WORDSIZE == 64 +void +_jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ + if (v < 6) + jit_extr_ui(u, _RDI - v); + else + jit_ldxi_ui(u, _RBP, v); +} + +void +_jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ + if (v < 6) + jit_movr(u, _RDI - v); + else + jit_ldxi_l(u, _RBP, v); +} +#endif + +void +_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 8) + jit_movr_f(u, _XMM0 - v); + else +#endif + jit_ldxi_f(u, _RBP, v); +} + +void +_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ +#if __WORDSIZE == 64 + if (v < 8) + jit_movr_d(u, _XMM0 - v); + else +#endif + jit_ldxi_d(u, _RBP, v); +} + +void +_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +{ + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->call.argi < 6) { + jit_movr(_RDI - _jit->function->call.argi, u); + ++_jit->function->call.argi; + } + else +#endif + { + jit_stxi(_jit->function->call.size, _RSP, u); + _jit->function->call.size += sizeof(jit_word_t); + } +} + +void +_jit_pushargi(jit_state_t *_jit, jit_word_t u) +{ + jit_int32_t regno; + + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->call.argi < 6) { + jit_movi(_RDI - _jit->function->call.argi, u); + ++_jit->function->call.argi; + } + else +#endif + { + regno = jit_get_reg(jit_class_gpr); + jit_movi(regno, u); + jit_stxi(_jit->function->call.size, _RSP, regno); + _jit->function->call.size += sizeof(jit_word_t); + jit_unget_reg(regno); + } +} + +void +_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) +{ + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->call.argf < 8) { + jit_movr_f(_XMM0 - _jit->function->call.argf, u); + ++_jit->function->call.argf; + } + else +#endif + { + jit_stxi_f(_jit->function->call.size, _RSP, u); + _jit->function->call.size += sizeof(jit_word_t); + } +} + +void +_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) +{ + jit_int32_t regno; + + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->call.argf < 8) { + jit_movi_f(_XMM0 - _jit->function->call.argf, u); + ++_jit->function->call.argf; + } + else +#endif + { + regno = jit_get_reg(jit_class_fpr); + jit_movi_f(regno, u); + jit_stxi_f(_jit->function->call.size, _RSP, regno); + _jit->function->call.size += sizeof(jit_word_t); + jit_unget_reg(regno); + } +} + +void +_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) +{ + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->call.argf < 8) { + jit_movr_d(_XMM0 - _jit->function->call.argf, u); + ++_jit->function->call.argf; + } + else +#endif + { + jit_stxi_d(_jit->function->call.size, _RSP, u); + _jit->function->call.size += sizeof(jit_float64_t); + } +} + +void +_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) +{ + jit_int32_t regno; + + assert(_jit->function); +#if __WORDSIZE == 64 + if (_jit->function->call.argf < 8) { + jit_movi_d(_XMM0 - _jit->function->call.argf, u); + ++_jit->function->call.argf; + } + else +#endif + { + regno = jit_get_reg(jit_class_fpr); + jit_movi_d(regno, u); + jit_stxi_d(_jit->function->call.size, _RSP, regno); + _jit->function->call.size += sizeof(jit_float64_t); + jit_unget_reg(regno); + } +} + +jit_bool_t +_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno) +{ +#if __WORDSIZE == 64 + jit_int32_t spec; + + spec = jit_class(_rvs[regno].spec); + if (spec & jit_class_arg) { + if (spec & jit_class_gpr) { + regno = _RDI - regno; + if (regno >= 0 && regno < node->v.w) + return (1); + } + else if (spec & jit_class_fpr) { + regno = _XMM0 - regno; + if (regno >= 0 && regno < node->w.w) + return (1); + } + } +#endif + return (0); +} + +void +_jit_finishr(jit_state_t *_jit, jit_int32_t r0) +{ + jit_int32_t reg; + jit_node_t *call; + + reg = r0; + assert(_jit->function); + if (_jit->function->self.alen < _jit->function->call.size) + _jit->function->self.alen = _jit->function->call.size; +#if __WORDSIZE == 64 + if (_jit->function->call.kind & jit_call_varargs) { + if (jit_regno(reg) == _RAX) { + reg = jit_get_reg(jit_class_gpr); + jit_movr(reg, _RAX); + } + if (_jit->function->call.argf) + jit_movi(_RAX, _jit->function->call.argf); + else + jit_movi(_RAX, 0); + if (reg != r0) + jit_unget_reg(reg); + } +#endif + call = jit_callr(reg); + call->v.w = _jit->function->call.argi; + call->w.w = _jit->function->call.argf; + _jit->function->call.argi = _jit->function->call.argf = + _jit->function->call.size = 0; +} + +jit_node_t * +_jit_finishi(jit_state_t *_jit, jit_pointer_t i0) +{ +#if __WORDSIZE == 64 + jit_int32_t reg; +#endif + jit_node_t *node; + + assert(_jit->function); + if (_jit->function->self.alen < _jit->function->call.size) + _jit->function->self.alen = _jit->function->call.size; +#if __WORDSIZE == 64 + if (_jit->function->call.kind & jit_call_varargs) + jit_regset_setbit(_jit->regarg, _RAX); + reg = jit_get_reg(jit_class_gpr); + node = jit_movi(reg, (jit_word_t)i0); + jit_finishr(reg); + jit_unget_reg(reg); + if (_jit->function->call.kind & jit_call_varargs) + jit_regset_clrbit(_jit->regarg, _RAX); +#else + node = jit_calli(i0); + node->v.w = _jit->function->call.argi; + node->w.w = _jit->function->call.argf; +#endif + _jit->function->call.argi = _jit->function->call.argf = + _jit->function->call.size = 0; + return (node); +} + +void +_jit_retval_c(jit_state_t *_jit, jit_int32_t r0) +{ + jit_extr_c(r0, JIT_RET); +} + +void +_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0) +{ + jit_extr_uc(r0, JIT_RET); +} + +void +_jit_retval_s(jit_state_t *_jit, jit_int32_t r0) +{ + jit_extr_s(r0, JIT_RET); +} + +void +_jit_retval_us(jit_state_t *_jit, jit_int32_t r0) +{ + jit_extr_us(r0, JIT_RET); +} + +void +_jit_retval_i(jit_state_t *_jit, jit_int32_t r0) +{ +#if __WORDSIZE == 32 + if (r0 != JIT_RET) + jit_movr(r0, JIT_RET); +#else + jit_extr_i(r0, JIT_RET); +#endif +} + +#if __WORDSIZE == 64 +void +_jit_retval_ui(jit_state_t *_jit, jit_int32_t r0) +{ + jit_extr_ui(r0, JIT_RET); +} + +void +_jit_retval_l(jit_state_t *_jit, jit_int32_t r0) +{ + if (r0 != JIT_RET) + jit_movr(r0, JIT_RET); +} +#endif + +void +_jit_retval_f(jit_state_t *_jit, jit_int32_t r0) +{ +# if __WORDSIZE == 32 + jit_new_node_w(jit_code_retval_f, r0); +# else + if (r0 != JIT_FRET) + jit_movr_f(r0, JIT_FRET); +# endif +} + +void +_jit_retval_d(jit_state_t *_jit, jit_int32_t r0) +{ +# if __WORDSIZE == 32 + jit_new_node_w(jit_code_retval_d, r0); +# else + if (r0 != JIT_FRET) + jit_movr_d(r0, JIT_FRET); +# endif +} + +jit_pointer_t +_jit_emit(jit_state_t *_jit) +{ + jit_node_t *node; + jit_node_t *temp; + jit_word_t word; + jit_int32_t value; + jit_int32_t offset; + struct { + jit_node_t *node; + jit_word_t word; + jit_int32_t patch_offset; + } undo; + + if (_jit->function) + jit_epilog(); + jit_optimize(); + + _jit->emit = 1; + + _jit->code.length = 16 * 1024 * 1024; + _jit->code.ptr = mmap(NULL, _jit->code.length, + PROT_EXEC | PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + assert(_jit->code.ptr != MAP_FAILED); + _jit->pc.uc = _jit->code.ptr; + + /* clear jit_flag_patch from label nodes if reallocating buffer + * and starting over + */ + + _jit->function = NULL; + + jit_reglive_setup(); + + undo.word = 0; + undo.node = NULL; + undo.patch_offset = 0; +#define case_rr(name, type) \ + case jit_code_##name##r##type: \ + name##r##type(rn(node->u.w), rn(node->v.w)); \ + break +#define case_rw(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(rn(node->u.w), node->v.w); \ + break +#define case_rf(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->v.w)) \ + x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \ + break +#define case_fr(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->u.w)) \ + x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \ + break +#define case_fw(name, type) \ + case jit_code_##name##i##type: \ + if (jit_x87_reg_p(node->u.w)) \ + x87_##name##i##type(rn(node->u.w), node->v.w); \ + else \ + sse_##name##i##type(rn(node->u.w), node->v.w); \ + break +#define case_wr(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(node->u.w, rn(node->v.w)); \ + break +#define case_wf(name, type) \ + case jit_code_##name##i##type: \ + if (jit_x87_reg_p(node->v.w)) \ + x87_##name##i##type(node->u.w, rn(node->v.w)); \ + else \ + sse_##name##i##type(node->u.w, rn(node->v.w)); \ + break +#define case_ff(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->u.w) && \ + jit_x87_reg_p(node->v.w)) \ + x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \ + break; +#define case_rrr(name, type) \ + case jit_code_##name##r##type: \ + name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_frr(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->u.w)) \ + x87_##name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_rrf(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->u.w)) \ + x87_##name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_rrw(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \ + break +#define case_frw(name, type) \ + case jit_code_##name##i##type: \ + if (jit_x87_reg_p(node->u.w)) \ + x87_##name##i##type(rn(node->u.w), \ + rn(node->v.w), node->w.w); \ + else \ + sse_##name##i##type(rn(node->u.w), \ + rn(node->v.w), node->w.w); \ + break +#define case_wrr(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \ + break +#define case_wrf(name, type) \ + case jit_code_##name##i##type: \ + if (jit_x87_reg_p(node->w.w)) \ + x87_##name##i##type(node->u.w, \ + rn(node->v.w), rn(node->w.w)); \ + else \ + sse_##name##i##type(node->u.w, \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_brr(name, type) \ + case jit_code_##name##r##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##r##type(temp->u.w, rn(node->v.w), \ + rn(node->w.w)); \ + else { \ + word = name##r##type(_jit->pc.w, \ + rn(node->v.w), rn(node->w.w)); \ + patch(word, node); \ + } \ + break +#define case_brw(name, type) \ + case jit_code_##name##i##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##i##type(temp->u.w, \ + rn(node->v.w), node->w.w); \ + else { \ + word = name##i##type(_jit->pc.w, \ + rn(node->v.w), node->w.w); \ + patch(word, node); \ + } \ + break +#define case_rff(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->v.w) && \ + jit_x87_reg_p(node->w.w)) \ + x87_##name##r##type(rn(node->u.w), rn(node->v.w), \ + rn(node->w.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), rn(node->v.w), \ + rn(node->w.w)); \ + break; +#define case_rfw(name, type, size) \ + case jit_code_##name##i##type: \ + assert(node->flag & jit_flag_data); \ + if (jit_x87_reg_p(node->v.w)) \ + x87_##name##i##type(rn(node->u.w), rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + else \ + sse_##name##i##type(rn(node->u.w), rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + break +#define case_fff(name, type) \ + case jit_code_##name##r##type: \ + if (jit_x87_reg_p(node->u.w) && \ + jit_x87_reg_p(node->v.w) && \ + jit_x87_reg_p(node->w.w)) \ + x87_##name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + else \ + sse_##name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_ffw(name, type, size) \ + case jit_code_##name##i##type: \ + assert(node->flag & jit_flag_data); \ + if (jit_x87_reg_p(node->u.w) && \ + jit_x87_reg_p(node->v.w)) \ + x87_##name##i##type(rn(node->u.w), rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + else \ + sse_##name##i##type(rn(node->u.w), rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + break +#define case_bff(name, type) \ + case jit_code_b##name##r##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) { \ + if (jit_x87_reg_p(node->v.w) && \ + jit_x87_reg_p(node->w.w)) \ + x87_b##name##r##type(temp->u.w, \ + rn(node->v.w), rn(node->w.w)); \ + else \ + sse_b##name##r##type(temp->u.w, \ + rn(node->v.w), rn(node->w.w)); \ + } \ + else { \ + if (jit_x87_reg_p(node->v.w) && \ + jit_x87_reg_p(node->w.w)) \ + word = x87_b##name##r##type(_jit->pc.w, \ + rn(node->v.w), rn(node->w.w)); \ + else \ + word = sse_b##name##r##type(_jit->pc.w, \ + rn(node->v.w), rn(node->w.w)); \ + patch(word, node); \ + } \ + break +#define case_bfw(name, type, size) \ + case jit_code_b##name##i##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) { \ + if (jit_x87_reg_p(node->v.w)) \ + x87_b##name##i##type(temp->u.w, \ + rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + else \ + sse_b##name##i##type(temp->u.w, \ + rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + } \ + else { \ + if (jit_x87_reg_p(node->v.w)) \ + word = x87_b##name##i##type(_jit->pc.w, \ + rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + else \ + word = sse_b##name##i##type(_jit->pc.w, \ + rn(node->v.w), \ + (jit_float##size##_t *)node->w.n->u.w); \ + patch(word, node); \ + } \ + break + for (node = _jit->head; node; node = node->next) { + value = jit_classify(node->code); + jit_regarg_set(node, value); + switch (node->code) { + case jit_code_note: + node->u.w = _jit->pc.w; + break; + case jit_code_label: + if (node->link && + (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) + nop(sizeof(jit_word_t) - word); + /* remember label is defined */ + node->flag |= jit_flag_patch; + node->u.w = _jit->pc.w; + break; + case_rrr(add,); + case_rrw(add,); + case_rrr(addx,); + case_rrw(addx,); + case_rrr(addc,); + case_rrw(addc,); + case_rrr(sub,); + case_rrw(sub,); + case_rrr(subx,); + case_rrw(subx,); + case_rrr(subc,); + case_rrw(subc,); + case_rrr(mul,); + case_rrw(mul,); + case_rrr(div,); + case_rrw(div,); + case_rrr(div, _u); + case_rrw(div, _u); + case_rrr(rem,); + case_rrw(rem,); + case_rrr(rem, _u); + case_rrw(rem, _u); + case_rrr(and,); + case_rrw(and,); + case_rrr(or,); + case_rrw(or,); + case_rrr(xor,); + case_rrw(xor,); + case_rrr(lsh,); + case_rrw(lsh,); + case_rrr(rsh,); + case_rrw(rsh,); + case_rrr(rsh, _u); + case_rrw(rsh, _u); + case_rr(neg,); + case_rr(com,); + case_rrr(lt,); + case_rrw(lt,); + case_rrr(lt, _u); + case_rrw(lt, _u); + case_rrr(le,); + case_rrw(le,); + case_rrr(le, _u); + case_rrw(le, _u); + case_rrr(eq,); + case_rrw(eq,); + case_rrr(ge,); + case_rrw(ge,); + case_rrr(ge, _u); + case_rrw(ge, _u); + case_rrr(gt,); + case_rrw(gt,); + case_rrr(gt, _u); + case_rrw(gt, _u); + case_rrr(ne,); + case_rrw(ne,); + case_rr(mov,); + case jit_code_movi: + if (node->flag & jit_flag_node) { + temp = node->v.n; + if (temp->code == jit_code_data || + (temp->code == jit_code_label && + (temp->flag & jit_flag_patch))) + movi(rn(node->u.w), temp->u.w); + else { + assert(temp->code == jit_code_label || + temp->code == jit_code_epilog); + word = movi_p(rn(node->u.w), node->v.w); + patch(word, node); + } + } + else + movi(rn(node->u.w), node->v.w); + break; + case_rr(hton,); + case_rr(ext, _c); + case_rr(ext, _uc); + case_rr(ext, _s); + case_rr(ext, _us); +#if __WORDSIZE == 64 + case_rr(ext, _i); + case_rr(ext, _ui); +#endif + case_rf(trunc, _f_i); + case_rf(trunc, _d_i); +#if __WORDSIZE == 64 + case_rf(trunc, _f_l); + case_rf(trunc, _d_l); +#endif + case_rr(ld, _c); + case_rw(ld, _c); + case_rr(ld, _uc); + case_rw(ld, _uc); + case_rr(ld, _s); + case_rw(ld, _s); + case_rr(ld, _us); + case_rw(ld, _us); + case_rr(ld, _i); + case_rw(ld, _i); +#if __WORDSIZE == 64 + case_rr(ld, _ui); + case_rw(ld, _ui); + case_rr(ld, _l); + case_rw(ld, _l); +#endif + case_rrr(ldx, _c); + case_rrw(ldx, _c); + case_rrr(ldx, _uc); + case_rrw(ldx, _uc); + case_rrr(ldx, _s); + case_rrw(ldx, _s); + case_rrr(ldx, _us); + case_rrw(ldx, _us); + case_rrr(ldx, _i); + case_rrw(ldx, _i); +#if __WORDSIZE == 64 + case_rrr(ldx, _ui); + case_rrw(ldx, _ui); + case_rrr(ldx, _l); + case_rrw(ldx, _l); +#endif + case_rr(st, _c); + case_wr(st, _c); + case_rr(st, _s); + case_wr(st, _s); + case_rr(st, _i); + case_wr(st, _i); +#if __WORDSIZE == 64 + case_rr(st, _l); + case_wr(st, _l); +#endif + case_rrr(stx, _c); + case_wrr(stx, _c); + case_rrr(stx, _s); + case_wrr(stx, _s); + case_rrr(stx, _i); + case_wrr(stx, _i); +#if __WORDSIZE == 64 + case_rrr(stx, _l); + case_wrr(stx, _l); +#endif + case_brr(blt,); + case_brw(blt,); + case_brr(blt, _u); + case_brw(blt, _u); + case_brr(ble,); + case_brw(ble,); + case_brr(ble, _u); + case_brw(ble, _u); + case_brr(beq,); + case_brw(beq,); + case_brr(bge,); + case_brw(bge,); + case_brr(bge, _u); + case_brw(bge, _u); + case_brr(bgt,); + case_brw(bgt,); + case_brr(bgt, _u); + case_brw(bgt, _u); + case_brr(bne,); + case_brw(bne,); + case_brr(bms,); + case_brw(bms,); + case_brr(bmc,); + case_brw(bmc,); + case_brr(boadd,); + case_brw(boadd,); + case_brr(boadd, _u); + case_brw(boadd, _u); + case_brr(bxadd,); + case_brw(bxadd,); + case_brr(bxadd, _u); + case_brw(bxadd, _u); + case_brr(bosub,); + case_brw(bosub,); + case_brr(bosub, _u); + case_brw(bosub, _u); + case_brr(bxsub,); + case_brw(bxsub,); + case_brr(bxsub, _u); + case_brw(bxsub, _u); + case_fff(add, _f); + case_ffw(add, _f, 32); + case_fff(sub, _f); + case_ffw(sub, _f, 32); + case_fff(mul, _f); + case_ffw(mul, _f, 32); + case_fff(div, _f); + case_ffw(div, _f, 32); + case_ff(abs, _f); + case_ff(neg, _f); + case_ff(sqrt, _f); + case_fr(ext, _f); + case_fr(ext, _d_f); + case_rff(lt, _f); + case_rfw(lt, _f, 32); + case_rff(le, _f); + case_rfw(le, _f, 32); + case_rff(eq, _f); + case_rfw(eq, _f, 32); + case_rff(ge, _f); + case_rfw(ge, _f, 32); + case_rff(gt, _f); + case_rfw(gt, _f, 32); + case_rff(ne, _f); + case_rfw(ne, _f, 32); + case_rff(unlt, _f); + case_rfw(unlt, _f, 32); + case_rff(unle, _f); + case_rfw(unle, _f, 32); + case_rff(uneq, _f); + case_rfw(uneq, _f, 32); + case_rff(unge, _f); + case_rfw(unge, _f, 32); + case_rff(ungt, _f); + case_rfw(ungt, _f, 32); + case_rff(ltgt, _f); + case_rfw(ltgt, _f, 32); + case_rff(ord, _f); + case_rfw(ord, _f, 32); + case_rff(unord, _f); + case_rfw(unord, _f, 32); + case jit_code_movr_f: + if (jit_x87_reg_p(node->u.w)) { + if (jit_x87_reg_p(node->v.w)) + x87_movr_f(rn(node->u.w), rn(node->v.w)); + else + x87_from_sse_f(rn(node->u.w), rn(node->v.w)); + } + else { + if (jit_sse_reg_p(node->v.w)) + sse_movr_f(rn(node->u.w), rn(node->v.w)); + else + sse_from_x87_f(rn(node->u.w), rn(node->v.w)); + } + break; + case jit_code_movi_f: + assert(node->flag & jit_flag_data); + if (jit_x87_reg_p(node->u.w)) + x87_movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w); + else + sse_movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w); + break; + case_fr(ld, _f); + case_fw(ld, _f); + case_frr(ldx, _f); + case_frw(ldx, _f); + case_rf(st, _f); + case_wf(st, _f); + case_rrf(stx, _f); + case_wrf(stx, _f); + case_bff(lt, _f); + case_bfw(lt, _f, 32); + case_bff(le, _f); + case_bfw(le, _f, 32); + case_bff(eq, _f); + case_bfw(eq, _f, 32); + case_bff(ge, _f); + case_bfw(ge, _f, 32); + case_bff(gt, _f); + case_bfw(gt, _f, 32); + case_bff(ne, _f); + case_bfw(ne, _f, 32); + case_bff(unlt, _f); + case_bfw(unlt, _f, 32); + case_bff(unle, _f); + case_bfw(unle, _f, 32); + case_bff(uneq, _f); + case_bfw(uneq, _f, 32); + case_bff(unge, _f); + case_bfw(unge, _f, 32); + case_bff(ungt, _f); + case_bfw(ungt, _f, 32); + case_bff(ltgt, _f); + case_bfw(ltgt, _f, 32); + case_bff(ord, _f); + case_bfw(ord, _f, 32); + case_bff(unord, _f); + case_bfw(unord, _f, 32); + case_fff(add, _d); + case_ffw(add, _d, 64); + case_fff(sub, _d); + case_ffw(sub, _d, 64); + case_fff(mul, _d); + case_ffw(mul, _d, 64); + case_fff(div, _d); + case_ffw(div, _d, 64); + case_ff(abs, _d); + case_ff(neg, _d); + case_ff(sqrt, _d); + case_fr(ext, _d); + case_fr(ext, _f_d); + case_rff(lt, _d); + case_rfw(lt, _d, 64); + case_rff(le, _d); + case_rfw(le, _d, 64); + case_rff(eq, _d); + case_rfw(eq, _d, 64); + case_rff(ge, _d); + case_rfw(ge, _d, 64); + case_rff(gt, _d); + case_rfw(gt, _d, 64); + case_rff(ne, _d); + case_rfw(ne, _d, 64); + case_rff(unlt, _d); + case_rfw(unlt, _d, 64); + case_rff(unle, _d); + case_rfw(unle, _d, 64); + case_rff(uneq, _d); + case_rfw(uneq, _d, 64); + case_rff(unge, _d); + case_rfw(unge, _d, 64); + case_rff(ungt, _d); + case_rfw(ungt, _d, 64); + case_rff(ltgt, _d); + case_rfw(ltgt, _d, 64); + case_rff(ord, _d); + case_rfw(ord, _d, 64); + case_rff(unord, _d); + case_rfw(unord, _d, 64); + case jit_code_movr_d: + if (jit_x87_reg_p(node->u.w)) { + if (jit_x87_reg_p(node->v.w)) + x87_movr_d(rn(node->u.w), rn(node->v.w)); + else + x87_from_sse_d(rn(node->u.w), rn(node->v.w)); + } + else { + if (jit_sse_reg_p(node->v.w)) + sse_movr_d(rn(node->u.w), rn(node->v.w)); + else + sse_from_x87_d(rn(node->u.w), rn(node->v.w)); + } + break; + case jit_code_movi_d: + assert(node->flag & jit_flag_data); + if (jit_x87_reg_p(node->u.w)) + x87_movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w); + else + sse_movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w); + break; + case_fr(ld, _d); + case_fw(ld, _d); + case_frr(ldx, _d); + case_frw(ldx, _d); + case_rf(st, _d); + case_wf(st, _d); + case_rrf(stx, _d); + case_wrf(stx, _d); + case_bff(lt, _d); + case_bfw(lt, _d, 64); + case_bff(le, _d); + case_bfw(le, _d, 64); + case_bff(eq, _d); + case_bfw(eq, _d, 64); + case_bff(ge, _d); + case_bfw(ge, _d, 64); + case_bff(gt, _d); + case_bfw(gt, _d, 64); + case_bff(ne, _d); + case_bfw(ne, _d, 64); + case_bff(unlt, _d); + case_bfw(unlt, _d, 64); + case_bff(unle, _d); + case_bfw(unle, _d, 64); + case_bff(uneq, _d); + case_bfw(uneq, _d, 64); + case_bff(unge, _d); + case_bfw(unge, _d, 64); + case_bff(ungt, _d); + case_bfw(ungt, _d, 64); + case_bff(ltgt, _d); + case_bfw(ltgt, _d, 64); + case_bff(ord, _d); + case_bfw(ord, _d, 64); + case_bff(unord, _d); + case_bfw(unord, _d, 64); + case jit_code_jmpr: + jmpr(rn(node->u.w)); + break; + case jit_code_jmpi: + temp = node->u.n; + assert(temp->code == jit_code_label || + temp->code == jit_code_epilog); + if (temp->flag & jit_flag_patch) + jmpi(temp->u.w); + else { + word = jmpi(_jit->pc.w); + patch(word, node); + } + break; + case jit_code_callr: + callr(rn(node->u.w)); + break; + case jit_code_calli: + if (node->flag & jit_flag_node) { + temp = node->u.n; + assert(temp->code == jit_code_label || + temp->code == jit_code_epilog); + word = calli(temp->u.w); + if (!(temp->flag & jit_flag_patch)) + patch(word, node); + } + else + calli(node->u.w); + break; + case jit_code_prolog: + _jit->function = _jit->functions.ptr + node->u.w; + undo.node = node; + undo.word = _jit->pc.w; + undo.patch_offset = _jit->patches.offset; + restart_function: + _jit->again = 0; + prolog(node); + break; + case jit_code_epilog: + if (_jit->again) { + for (temp = undo.node->next; + temp != node; temp = temp->next) { + if (temp->code == jit_code_label || + temp->code == jit_code_epilog) + temp->flag &= ~jit_flag_patch; + } + node = undo.node; + _jit->pc.w = undo.word; + _jit->patches.offset = undo.patch_offset; + goto restart_function; + } + if (node->link && + (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) + nop(sizeof(jit_word_t) - word); + /* remember label is defined */ + node->flag |= jit_flag_patch; + node->u.w = _jit->pc.w; + epilog(node); + _jit->function = NULL; + break; +#if __WORDSIZE == 32 + case jit_code_retval_f: + if (jit_sse_reg_p(node->u.w)) { + fstpr(_ST1_REGNO); + sse_from_x87_f(rn(node->u.w), _ST0_REGNO); + } + else + fstpr(rn(node->u.w) + 1); + break; + case jit_code_retval_d: + if (jit_sse_reg_p(node->u.w)) { + fstpr(_ST1_REGNO); + sse_from_x87_d(rn(node->u.w), _ST0_REGNO); + } + else + fstpr(rn(node->u.w) + 1); + break; +#endif + default: + abort(); + } + jit_regarg_clr(node, value); + /* update register live state */ + jit_reglive(node); + } +#undef case_bfw +#undef case_bff +#undef case_ffw +#undef case_rfw +#undef case_rff +#undef case_brw +#undef case_brr +#undef case_wrf +#undef case_wrr +#undef case_frw +#undef case_rrf +#undef case_rrw +#undef case_frr +#undef case_rrr +#undef case_wf +#undef case_fw +#undef case_fr +#undef case_rr + + for (offset = 0; offset < _jit->patches.offset; offset++) { + node = _jit->patches.ptr[offset].node; + word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w; + patch_at(node, _jit->patches.ptr[offset].inst, word); + } + + return (_jit->code.ptr); +} + +#define CODE 1 +# include "jit_x86-cpu.c" +# include "jit_x86-sse.c" +# include "jit_x86-x87.c" +#undef CODE + +void +_emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0) +{ + ldxi(rn(r0), rn(r1), i0); +} + +void +_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1) +{ + stxi(i0, rn(r0), rn(r1)); +} + +void +_emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0) +{ + if (jit_x87_reg_p(r0)) + x87_ldxi_d(rn(r0), rn(r1), i0); + else + sse_ldxi_d(rn(r0), rn(r1), i0); +} + +void +_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1) +{ + if (jit_x87_reg_p(r1)) + x87_stxi_d(i0, rn(r0), rn(r1)); + else + sse_stxi_d(i0, rn(r0), rn(r1)); +} + +static void +_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) +{ + jit_int32_t flag; + + assert(node->flag & jit_flag_node); + if (node->code == jit_code_movi) + flag = node->v.n->flag; + else + flag = node->u.n->flag; + assert(!(flag & jit_flag_patch)); + if (_jit->patches.offset >= _jit->patches.length) { + _jit->patches.ptr = realloc(_jit->patches.ptr, + (_jit->patches.length + 1024) * + sizeof(jit_patch_t)); + memset(_jit->patches.ptr + _jit->patches.length, 0, + 1024 * sizeof(jit_patch_t)); + _jit->patches.length += 1024; + } + _jit->patches.ptr[_jit->patches.offset].inst = instr; + _jit->patches.ptr[_jit->patches.offset].node = node; + ++_jit->patches.offset; +} + +static void +_sse_from_x87_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + x87_stxi_f(-8, _RBP_REGNO, r1); + sse_ldxi_f(r0, _RBP_REGNO, -8); +} + +static void +_sse_from_x87_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + x87_stxi_d(-8, _RBP_REGNO, r1); + sse_ldxi_d(r0, _RBP_REGNO, -8); +} + +static void +_x87_from_sse_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + sse_stxi_f(-8, _RBP_REGNO, r1); + x87_ldxi_f(r0, _RBP_REGNO, -8); +} + +static void +_x87_from_sse_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + sse_stxi_d(-8, _RBP_REGNO, r1); + x87_ldxi_d(r0, _RBP_REGNO, -8); +} diff --git a/lib/lightning.c b/lib/lightning.c new file mode 100644 index 000000000..75afeba0b --- /dev/null +++ b/lib/lightning.c @@ -0,0 +1,2306 @@ +/* + * Copyright (C) 2012 Free Software Foundation, Inc. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#include +#include +#include + +#define jit_regload_reload 0 /* convert to reload */ +#define jit_regload_delete 1 /* just remove node */ +#define jit_regload_isdead 2 /* delete and unset live bit */ + +/* + * Prototypes + */ +static jit_word_t hash_data(jit_pointer_t, jit_word_t); + +#define new_pool() _new_pool(_jit) +static void _new_pool(jit_state_t*); + +#define new_node(u) _new_node(_jit, u) +static jit_node_t *_new_node(jit_state_t*, jit_code_t); + +#define link_node(u) _link_node(_jit, u) +static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*); + +#define del_node(u, v) _del_node(_jit, u, v) +static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*); + +#define del_label(u, v) _del_label(_jit, u, v) +static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*); + +#define jit_setup(block) _jit_setup(_jit, block) +static void +_jit_setup(jit_state_t *_jit, jit_block_t *block); + +#define jit_update(setup,node,live,mask) _jit_update(_jit,setup,node,live,mask) +static void +_jit_update(jit_state_t *_jit, jit_bool_t setup, jit_node_t *node, + jit_regset_t *live, jit_regset_t *mask); + +#define thread_jumps() _thread_jumps(_jit) +static void +_thread_jumps(jit_state_t *_jit); + +#define sequential_labels() _sequential_labels(_jit) +static void +_sequential_labels(jit_state_t *_jit); + +#define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node) +static jit_bool_t +_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); + +#define redundant_jump(prev, node) _redundant_jump(_jit, prev, node) +static jit_bool_t +_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); + +static jit_code_t +reverse_jump_code(jit_code_t code); + +#define reverse_jump(prev, node) _reverse_jump(_jit, prev, node) +static jit_bool_t +_reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); + +#define redundant_store(node, jump) _redundant_store(_jit, node, jump) +static void +_redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump); + +#define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s) +static jit_bool_t +_simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node, + jit_int32_t kind, jit_int32_t size); + +#define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s) +static jit_bool_t +_simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node, + jit_int32_t kind, jit_int32_t size); + +#define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node) +static jit_bool_t +_simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); + +#define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node) +static jit_bool_t +_simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); + +#define simplify_spill(node, regno) _simplify_spill(_jit, node, regno) +static void +_simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno); + +#define simplify() _simplify(_jit) +static void +_simplify(jit_state_t *_jit); + +#define jit_reg_undef -1 +#define jit_reg_static 0 +#define jit_reg_change 1 +#define register_change_p(n, l, r) _register_change_p(_jit, n, l, r) +static jit_int32_t +_register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link, + jit_int32_t regno); + +#define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno) +static jit_bool_t +_spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno); + +#define patch_registers() _patch_registers(_jit) +static void +_patch_registers(jit_state_t *_jit); + +#define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p) +static void +_patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link, + jit_int32_t regno, jit_int32_t patch); + +/* + * Implementation + */ +void +init_jit(void) +{ + jit_get_cpu(); + jit_init_debug(); +} + +void +finish_jit(void) +{ + jit_finish_debug(); +} + +jit_int32_t +_jit_get_reg(jit_state_t *_jit, jit_int32_t regspec) +{ + jit_int32_t spec; + jit_int32_t regno; + + /* if asking for an explicit register value, assume it will + * properly handle the case of the register also being an + * argument for the instruction, or the register value + * being live */ + spec = regspec & ~(jit_class_chk|jit_class_nospill); + if ((regno = jit_regno(spec))) { + if (jit_regset_tstbit(_jit->regsav, regno)) + /* fail if register is spilled */ + goto fail; + if (jit_regset_tstbit(_jit->regarg, regno)) { + if (regspec & jit_class_nospill) + goto fail; + goto spill; + } + jit_regset_setbit(_jit->regarg, regno); + return (regno); + } + else + assert(jit_class(spec) != 0); + + if (_jit->emit) { + /* search for a free register matching spec */ + for (regno = 0; regno < _jit->reglen; regno++) { + if ((jit_class(_rvs[regno].spec) & spec) == spec && + !jit_regset_tstbit(_jit->regarg, regno) && + !jit_regset_tstbit(_jit->reglive, regno)) + goto regarg; + } + + /* search for a register matching spec that is not an argument + * for the current instruction */ + for (regno = 0; regno < _jit->reglen; regno++) { + if ((jit_class(_rvs[regno].spec) & spec) == spec && + !jit_regset_tstbit(_jit->regsav, regno) && + !jit_regset_tstbit(_jit->regarg, regno) && + !(regspec & jit_class_nospill)) { + spill: + assert(_jit->function); + if (spec & jit_class_gpr) { + if (!_jit->function->regoff[regno]) { + _jit->function->regoff[regno] = + jit_allocai(sizeof(jit_word_t)); + _jit->again = 1; + } + emit_stxi(_jit->function->regoff[regno], JIT_FP, regno); + } + else { + if (!_jit->function->regoff[regno]) { + _jit->function->regoff[regno] = + jit_allocai(sizeof(jit_float64_t)); + _jit->again = 1; + } + emit_stxi_d(_jit->function->regoff[regno], JIT_FP, regno); + } + jit_regset_setbit(_jit->regsav, regno); + regarg: + jit_regset_setbit(_jit->regarg, regno); + if (jit_class(_rvs[regno].spec) & jit_class_sav) { + /* if will modify callee save registers without a + * function prolog, better patch this assertion */ + assert(_jit->function); + if (!jit_regset_tstbit(_jit->function->regset, regno)) { + jit_regset_setbit(_jit->function->regset, regno); + _jit->again = 1; + } + } + return (regno); + } + } + } + else { + /* nospill hint only valid during emit" */ + assert(!(regspec & jit_class_nospill)); + for (regno = 0; regno < _jit->reglen; regno++) { + if ((jit_class(_rvs[regno].spec) & spec) == spec && + !jit_regset_tstbit(_jit->regsav, regno) && + !jit_regset_tstbit(_jit->regarg, regno)) { + jit_regset_setbit(_jit->regarg, regno); + jit_regset_setbit(_jit->regsav, regno); + jit_save(regno); + return (jit_regno_patch|regno); + } + } + } + + /* Out of hardware registers */ +fail: + assert(regspec & jit_class_chk); + return (JIT_NOREG); +} + +void +_jit_unget_reg(jit_state_t *_jit, jit_int32_t regno) +{ + regno = jit_regno(regno); + if (jit_regset_tstbit(_jit->regsav, regno)) { + if (_jit->emit) { + if (jit_class(_rvs[regno].spec) & jit_class_gpr) + emit_ldxi(regno, JIT_FP, _jit->function->regoff[regno]); + else + emit_ldxi_d(regno, JIT_FP, _jit->function->regoff[regno]); + } + else + jit_load(regno); + jit_regset_clrbit(_jit->regsav, regno); + } + assert(jit_regset_tstbit(_jit->regarg, regno)); + jit_regset_clrbit(_jit->regarg, regno); +} + +unsigned long +jit_regset_scan1(jit_regset_t set, jit_int32_t offset) +{ + jit_int32_t index; + jit_int32_t length; + union { + jit_uint64_t ul; + jit_uint8_t uc[8]; + } data; + + assert(offset >= 0 && offset <= 63); + data.ul = set; + if (data.uc[index = offset >> 3]) { + length = (index + 1) << 3; + for (; offset < length; offset++) { + if (set & (1LL << offset)) + return (offset); + } + } + for (index++; index < 8; index++) { + if (data.uc[index]) { + offset = index << 3; + length = (index + 1) << 3; + for (; offset < length; offset++) { + if (set & (1LL << offset)) + return (offset); + } + } + } + return (ULONG_MAX); +} + +void +_jit_save(jit_state_t *_jit, jit_int32_t reg) +{ + reg = jit_regno(reg); + assert(!_jit->emit); + _jit->spill[reg] = jit_new_node_w(jit_code_save, reg); +} + +void +_jit_load(jit_state_t *_jit, jit_int32_t reg) +{ + jit_node_t *node; + + reg = jit_regno(reg); + assert(!_jit->emit); + assert(_jit->spill[reg]); + node = jit_new_node_w(jit_code_load, reg); + /* create a path to flag the save/load is not required */ + node->link = _jit->spill[reg]; + node->link->link = node; + _jit->spill[reg] = NULL; +} + +static jit_word_t +hash_data(jit_pointer_t data, jit_word_t length) +{ + jit_word_t i, key; + union { + jit_uint8_t *c; + jit_word_t *w; + jit_pointer_t p; + } ptr; + + for (i = 0, key = 0, ptr.p = data; i < length / sizeof(jit_word_t); i++) + key = (key << (key & 1)) ^ ptr.w[i]; + for (i *= sizeof(jit_word_t); i < length; i++) + key = (key << (key & 1)) ^ ptr.c[i]; + + return (key); +} + +jit_node_t * +_jit_data(jit_state_t *_jit, jit_pointer_t data, jit_word_t length) +{ + jit_word_t key; + jit_node_t *node; + + assert(!_jit->emit); + + /* Ensure there is space even if asking for a duplicate */ + if (((_jit->data.offset + 7) & -8) + length > _jit->data.length) { + jit_word_t size; + + size = (_jit->data.length + length + 4096) & - 4095; + assert(size >= _jit->data.length); + if (_jit->data.ptr == NULL) + _jit->data.ptr = calloc(1, size); + else { + _jit->data.ptr = realloc(_jit->data.ptr, size); + memset(_jit->data.ptr + _jit->data.length, 0, + size - _jit->data.length); + } + _jit->data.length = size; + } + if (_jit->data.table == NULL) + _jit->data.table = calloc(_jit->data.size = 16, sizeof(jit_node_t*)); + + key = hash_data(data, length) & (_jit->data.size - 1); + node = _jit->data.table[key]; + for (; node; node = node->next) { + if (node->v.w == length && + memcmp(_jit->data.ptr + node->u.w, data, length) == 0) + break; + } + + if (!node) { + node = jit_new_node_no_link(jit_code_data); + switch (length) { + case 0: case 1: + break; + case 2: + _jit->data.offset = (_jit->data.offset + 1) & -2; + break; + case 3: case 4: + _jit->data.offset = (_jit->data.offset + 3) & -4; + break; + default: + _jit->data.offset = (_jit->data.offset + 7) & -8; + break; + } + node->u.w = _jit->data.offset; + node->v.w = length; + memcpy(_jit->data.ptr + _jit->data.offset, data, length); + _jit->data.offset += length; + + node->next = _jit->data.table[key]; + _jit->data.table[key] = node; + ++_jit->data.count; + + /* Rehash if more than 75% used table */ + if (_jit->data.count > + (_jit->data.size >> 1) + (_jit->data.size >> 2) && + (_jit->data.size << 1) > _jit->data.size) { + jit_word_t i; + jit_node_t **hash; + jit_node_t *next; + jit_node_t *temp; + + hash = calloc(_jit->data.size << 1, sizeof(jit_node_t*)); + for (i = 0; i < _jit->data.size; i++) { + temp = _jit->data.table[i]; + for (; temp; temp = next) { + next = temp->next; + key = hash_data(_jit->data.ptr + temp->u.w, temp->v.w) & + ((_jit->data.size << 1) - 1); + temp->next = hash[key]; + hash[key] = temp; + } + } + free(_jit->data.table); + _jit->data.table = hash; + _jit->data.size <<= 1; + } + } + + return (node); +} + +static void +_new_pool(jit_state_t *_jit) +{ + jit_node_t *list; + jit_int32_t offset; + + if (_jit->pool.offset >= _jit->pool.length) { + jit_node_t **ptr; + jit_int32_t length; + + length = _jit->pool.length + 16; + ptr = realloc(_jit->pool.ptr, length * sizeof(jit_node_t)); + memset(ptr + _jit->pool.length, 0, 16 * sizeof(jit_node_t)); + _jit->pool.ptr = ptr; + _jit->pool.length = length; + } + _jit->pool.ptr[_jit->pool.offset] = calloc(sizeof(jit_node_t), 1024); + list = _jit->pool.ptr[_jit->pool.offset]; + for (offset = 1; offset < 1024; offset++, list++) + list->next = list + 1; + list->next = _jit->list; + _jit->list = _jit->pool.ptr[_jit->pool.offset]; + ++_jit->pool.offset; +} + +static jit_node_t * +_new_node(jit_state_t *_jit, jit_code_t code) +{ + jit_node_t *node; + + if (_jit->list == NULL) + new_pool(); + node = _jit->list; + _jit->list = node->next; + node->next = NULL; + node->code = code; + + return (node); +} + +static inline jit_node_t * +_link_node(jit_state_t *_jit, jit_node_t *node) +{ + if (_jit->tail) + _jit->tail->next = node; + else + _jit->head = node; + return (_jit->tail = node); +} + +static inline void +_del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) +{ + if (prev == node) { + assert(prev == _jit->head); + _jit->head = node->next; + } + else + prev->next = node->next; + memset(node, 0, sizeof(jit_node_t)); + node->next = _jit->list; + _jit->list = node; +} + +static void +_del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) +{ + jit_block_t *block; + + /* only allow call to del_label on linked labels */ + block = _jit->blocks.ptr + node->v.w; + assert(block->label == node); + + /* del_label() should only be called when optimizing. + * This will leave an empty block index */ + jit_regset_del(block->reglive); + jit_regset_del(block->regmask); + block->label = NULL; + + /* redundant, should be already true */ + assert(node->link == NULL); + del_node(prev, node); +} + +jit_state_t * +jit_new_state(void) +{ + jit_state_t *_jit; + + _jit = calloc(1, sizeof(jit_state_t)); + jit_regset_new(_jit->regarg); + jit_regset_new(_jit->regsav); + jit_regset_new(_jit->reglive); + jit_regset_new(_jit->regmask); + mpz_init(_jit->blockmask); + + jit_init(); + + _jit->spill = calloc(_jit->reglen, sizeof(jit_node_t*)); + _jit->gen = calloc(_jit->reglen, sizeof(jit_int32_t)); + _jit->values = calloc(_jit->reglen, sizeof(jit_value_t)); + + _jit->patches.ptr = calloc(_jit->patches.length = 1024, + sizeof(jit_patch_t)); + _jit->functions.ptr = calloc(_jit->functions.length = 16, + sizeof(jit_function_t)); + _jit->pool.ptr = calloc(_jit->pool.length = 16, + sizeof(jit_node_t*)); + _jit->blocks.ptr = calloc(_jit->blocks.length = 16, + sizeof(jit_block_t)); +#if __arm__ && DISASSEMBLER + _jit->data_info.ptr = calloc(_jit->data_info.length = 1024, + sizeof(jit_data_info_t)); +#endif + + return (_jit); +} + +jit_node_t * +_jit_new_node(jit_state_t *_jit, jit_code_t code) +{ + assert(!_jit->emit); + return (link_node(new_node(code))); +} + +jit_node_t * +_jit_new_node_no_link(jit_state_t *_jit, jit_code_t code) +{ + assert(!_jit->emit); + return (new_node(code)); +} + +void +_jit_link_node(jit_state_t *_jit, jit_node_t *node) +{ + assert(!_jit->emit); + link_node(node); +} + +jit_node_t * +_jit_new_node_w(jit_state_t *_jit, jit_code_t code, + jit_word_t u) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_p(jit_state_t *_jit, jit_code_t code, + jit_pointer_t u) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.p = u; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_ww(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_word_t v) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + node->v.w = v; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_wf(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_float32_t v) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + node->v.f = v; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_wd(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_float64_t v) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + node->v.d = v; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_www(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_word_t v, jit_word_t w) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + node->v.w = v; + node->w.w = w; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_wwf(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_word_t v, jit_float32_t w) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + node->v.w = v; + node->w.f = w; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_wwd(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_word_t v, jit_float64_t w) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.w = u; + node->v.w = v; + node->w.d = w; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_pww(jit_state_t *_jit, jit_code_t code, + jit_pointer_t u, jit_word_t v, jit_word_t w) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.p = u; + node->v.w = v; + node->w.w = w; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_pwf(jit_state_t *_jit, jit_code_t code, + jit_pointer_t u, jit_word_t v, jit_float32_t w) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.p = u; + node->v.w = v; + node->w.f = w; + return (link_node(node)); +} + +jit_node_t * +_jit_new_node_pwd(jit_state_t *_jit, jit_code_t code, + jit_pointer_t u, jit_word_t v, jit_float64_t w) +{ + jit_node_t *node = new_node(code); + assert(!_jit->emit); + node->u.p = u; + node->v.w = v; + node->w.d = w; + return (link_node(node)); +} + +jit_node_t * +_jit_label(jit_state_t *_jit) +{ + jit_node_t *node; + + if (!(node = _jit->tail) || node->code != jit_code_label) { + node = jit_forward(); + jit_link(node); + } + + return (node); +} + +jit_node_t * +_jit_forward(jit_state_t *_jit) +{ + return (jit_new_node_no_link(jit_code_label)); +} + +void +_jit_link(jit_state_t *_jit, jit_node_t *node) +{ + jit_block_t *block; + + assert((node->code == jit_code_label || + node->code == jit_code_prolog || + node->code == jit_code_epilog) && !node->next); + jit_link_node(node); + if (_jit->blocks.offset >= _jit->blocks.length) { + jit_word_t length; + + length = _jit->blocks.length + 16; + block = realloc(_jit->blocks.ptr, length * sizeof(jit_block_t)); + memset(block + _jit->blocks.length, 0, 16 * sizeof(jit_block_t)); + _jit->blocks.ptr = block; + _jit->blocks.length = length; + } + block = _jit->blocks.ptr + _jit->blocks.offset; + block->label = node; + node->v.w = _jit->blocks.offset; + jit_regset_new(block->reglive); + jit_regset_new(block->regmask); + ++_jit->blocks.offset; +} + +void +_jit_prepare(jit_state_t *_jit, jit_int32_t kind) +{ + assert(_jit->function); + _jit->function->call.kind = kind; + _jit->function->call.argi = + _jit->function->call.argf = + _jit->function->call.size = 0; +} + +void +_jit_patch(jit_state_t* _jit, jit_node_t *instr) +{ + jit_node_t *label; + + if (!(label = _jit->tail) || + (label->code != jit_code_label && label->code != jit_code_epilog)) + label = jit_label(); + jit_patch_at(instr, label); +} + +jit_int32_t +_jit_classify(jit_state_t *_jit, jit_code_t code) +{ + jit_int32_t mask; + + switch (code) { + case jit_code_data: case jit_code_save: case jit_code_load: + case jit_code_label: case jit_code_note: case jit_code_prolog: + case jit_code_epilog: + mask = 0; + break; + case jit_code_calli: case jit_code_jmpi: + mask = jit_cc_a0_jmp; + break; + case jit_code_callr: case jit_code_jmpr: + mask = jit_cc_a0_reg|jit_cc_a0_jmp; + break; + case jit_code_retval_f: case jit_code_retval_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg; + break; + case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc: + case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i: + case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f: + case jit_code_ldi_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int; + break; + case jit_code_movi_f: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt; + break; + case jit_code_movi_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl; + break; + case jit_code_negr: case jit_code_comr: case jit_code_movr: + case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s: + case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui: + case jit_code_truncr_f_i: case jit_code_truncr_f_l: + case jit_code_truncr_d_i: case jit_code_truncr_d_l: + case jit_code_htonr: case jit_code_ldr_c: case jit_code_ldr_uc: + case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i: + case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f: + case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f: + case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f: + case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d: + case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d: + case jit_code_ldr_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg; + break; + case jit_code_addi: case jit_code_addxi: case jit_code_addci: + case jit_code_subi: case jit_code_subxi: case jit_code_subci: + case jit_code_muli: case jit_code_divi: case jit_code_divi_u: + case jit_code_remi: case jit_code_remi_u: case jit_code_andi: + case jit_code_ori: case jit_code_xori: case jit_code_lshi: + case jit_code_rshi: case jit_code_rshi_u: case jit_code_lti: + case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u: + case jit_code_eqi: case jit_code_gei: case jit_code_gei_u: + case jit_code_gti: case jit_code_gti_u: case jit_code_nei: + case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s: + case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui: + case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int; + break; + case jit_code_addi_f: case jit_code_subi_f: case jit_code_muli_f: + case jit_code_divi_f: case jit_code_lti_f: case jit_code_lei_f: + case jit_code_eqi_f: case jit_code_gei_f: case jit_code_gti_f: + case jit_code_nei_f: case jit_code_unlti_f: case jit_code_unlei_f: + case jit_code_uneqi_f: case jit_code_ungei_f: case jit_code_ungti_f: + case jit_code_ltgti_f: case jit_code_ordi_f: case jit_code_unordi_f: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt; + break; + case jit_code_addi_d: case jit_code_subi_d: case jit_code_muli_d: + case jit_code_divi_d: case jit_code_lti_d: case jit_code_lei_d: + case jit_code_eqi_d: case jit_code_gei_d: case jit_code_gti_d: + case jit_code_nei_d: case jit_code_unlti_d: case jit_code_unlei_d: + case jit_code_uneqi_d: case jit_code_ungei_d: case jit_code_ungti_d: + case jit_code_ltgti_d: case jit_code_ordi_d: case jit_code_unordi_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl; + break; + case jit_code_addr: case jit_code_addxr: case jit_code_addcr: + case jit_code_subr: case jit_code_subxr: case jit_code_subcr: + case jit_code_mulr: case jit_code_divr: case jit_code_divr_u: + case jit_code_remr: case jit_code_remr_u: case jit_code_andr: + case jit_code_orr: case jit_code_xorr: case jit_code_lshr: + case jit_code_rshr: case jit_code_rshr_u: case jit_code_ltr: + case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u: + case jit_code_eqr: case jit_code_ger: case jit_code_ger_u: + case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner: + case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s: + case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui: + case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f: + case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f: + case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f: + case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f: + case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f: + case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f: + case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d: + case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d: + case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d: + case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d: + case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d: + case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d: + case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg; + break; + case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i: + case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d: + mask = jit_cc_a0_int|jit_cc_a1_reg; + break; + case jit_code_blti: case jit_code_blti_u: case jit_code_blei: + case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei: + case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u: + case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci: + mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int; + break; + case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f: + case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f: + case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f: + case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f: + case jit_code_bordi_f: case jit_code_bunordi_f: + mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt; + break; + case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d: + case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d: + case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d: + case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d: + case jit_code_bordi_d: case jit_code_bunordi_d: + mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl; + break; + case jit_code_str_c: case jit_code_str_s: case jit_code_str_i: + case jit_code_str_l: case jit_code_str_f: case jit_code_str_d: + mask = jit_cc_a0_reg|jit_cc_a1_reg; + break; + case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i: + case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d: + mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg; + break; + case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler: + case jit_code_bler_u: case jit_code_beqr: case jit_code_bger: + case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u: + case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr: + case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f: + case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f: + case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f: + case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f: + case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d: + case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d: + case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d: + case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d: + case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d: + case jit_code_bunordr_d: + mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg; + break; + case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi: + case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u: + case jit_code_bxsubi: case jit_code_bxsubi_u: + mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int; + break; + case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i: + case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d: + mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg; + break; + case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr: + case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u: + case jit_code_bxsubr: case jit_code_bxsubr_u: + mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg; + break; + default: + abort(); + } + + return (mask); +} + +void +_jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address) +{ + jit_int32_t mask; + + if (instr->code == jit_code_movi) + instr->v.p = address; + else { + mask = jit_classify(instr->code); + assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp); + instr->u.p = address; + } +} + +void +_jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label) +{ + jit_int32_t mask; + + instr->flag |= jit_flag_node; + switch (instr->code) { + case jit_code_movi: + assert(label->code == jit_code_label || + label->code == jit_code_data); + instr->v.n = label; + if (label->code == jit_code_data) + instr->flag |= jit_flag_data; + break; + case jit_code_jmpi: + assert(label->code == jit_code_label || + label->code == jit_code_epilog); + instr->u.n = label; + break; + default: + mask = jit_classify(instr->code); + assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp); + assert(label->code == jit_code_label); + instr->u.n = label; + break; + } + /* link field is used as list of nodes associated with a given label */ + instr->link = label->link; + label->link = instr; +} + +void +_jit_optimize(jit_state_t *_jit) +{ + jit_bool_t jump; + jit_int32_t mask; + jit_node_t *node; + jit_block_t *block; + jit_word_t offset; + + _jit->function = NULL; + + thread_jumps(); + sequential_labels(); + simplify(); + + /* create initial mapping of live register values + * at the start of a basic block */ + for (offset = 0; offset < _jit->blocks.offset; offset++) { + block = _jit->blocks.ptr + offset; + if (!block->label) + continue; + if (block->label->code != jit_code_epilog) + jit_setup(block); + } + /* call jit_update resolving undefined values in reverse + * order so that sequential code would find most data already + * resolved when reaching the start of a new basic block */ + for (offset = _jit->blocks.offset - 1; offset >= 0; offset--) { + block = _jit->blocks.ptr + offset; + if (!block->label) + continue; + if (block->label->code != jit_code_epilog) { + jit_regset_set(_jit->regmask, block->regmask); + jit_update(1, block->label->next, &block->reglive, &_jit->regmask); + } + } + + patch_registers(); + + /* figure out labels that are only reached with a jump + * and is required to do a simple redundant_store removal + * on jit_beqi below */ + jump = 1; + for (node = _jit->head; node; node = node->next) { + switch (node->code) { + case jit_code_label: + if (!jump) + node->flag |= jit_flag_head; + break; + case jit_code_jmpi: case jit_code_jmpr: + case jit_code_epilog: + jump = 1; + break; + case jit_code_data: case jit_code_note: + break; + default: + jump = 0; + break; + } + } + + for (node = _jit->head; node; node = node->next) { + switch (node->code) { + case jit_code_prolog: + _jit->function = _jit->functions.ptr + node->w.w; + break; + case jit_code_epilog: + _jit->function = NULL; + break; + case jit_code_beqi: + redundant_store(node, 1); + break; + case jit_code_bnei: + redundant_store(node, 0); + break; + default: + mask = jit_classify(node->code); +#if JIT_HASH_CONSTS + if (mask & jit_cc_a1_flt) { + node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t)); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a1_dbl) { + node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t)); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a2_flt) { + node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t)); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a2_dbl) { + node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t)); + node->flag |= jit_flag_node | jit_flag_data; + } +#endif + if (_jit->function) { + if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) == + (jit_cc_a0_reg|jit_cc_a0_chg)) + jit_regset_setbit(_jit->function->regset, + jit_regno(node->u.w)); + if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) == + (jit_cc_a1_reg|jit_cc_a1_chg)) + jit_regset_setbit(_jit->function->regset, + jit_regno(node->v.w)); + } + break; + } + } + +#if JIT_HASH_CONSTS + /* create read only data buffer */ + if ((_jit->data.length = (_jit->data.offset + 4095) & -4096)) { + jit_uint8_t *ptr; + + ptr = mmap(NULL, _jit->data.length, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + assert(ptr != MAP_FAILED); + memcpy(ptr, _jit->data.ptr, _jit->data.offset); + free(_jit->data.ptr); + _jit->data.ptr = ptr; + for (offset = 0; offset < _jit->data.size; offset++) { + for (node = _jit->data.table[offset]; node; node = node->next) { + node->flag |= jit_flag_patch; + node->u.w = (jit_word_t)(_jit->data.ptr + node->u.w); + } + } + } +#endif +} + +void +_jit_reglive(jit_state_t *_jit, jit_node_t *node) +{ + jit_int32_t spec; + jit_int32_t value; + jit_block_t *block; + + switch (node->code) { + case jit_code_label: case jit_code_prolog: case jit_code_epilog: + block = _jit->blocks.ptr + node->v.w; + jit_regset_set(_jit->reglive, block->reglive); + break; + case jit_code_callr: + value = jit_regno(node->u.w); + if (!(node->u.w & jit_regno_patch)) { + jit_regset_setbit(_jit->reglive, value); + } + case jit_code_calli: + for (value = 0; value < _jit->reglen; value++) { + spec = jit_class(_rvs[value].spec); + if ((spec & jit_class_arg) && jit_regarg_p(node, value)) + jit_regset_setbit(_jit->reglive, value); + else if (!(spec & jit_class_sav)) + jit_regset_clrbit(_jit->reglive, value); + } +#if defined(JIT_RET) + /* Explicitly set return registers as live, because retval + * should be free to not create a note, and/or user not + * call jit_retval (but not a good idea to expect JIT_R0 + * to match JIT_RET) */ + jit_regset_setbit(_jit->reglive, JIT_RET); +# if __arm__ + /* FIXME need a better logic (and r2-r3 may contain results) */ + jit_regset_setbit(_jit->reglive, _R1); +# endif +#endif +#if defined(JIT_FRET) + jit_regset_setbit(_jit->reglive, JIT_FRET); +#endif + break; + default: + value = jit_classify(node->code); + if ((value & jit_cc_a0_reg) && !(node->u.w & jit_regno_patch)) { + if (value & jit_cc_a0_chg) { + jit_regset_clrbit(_jit->reglive, node->u.w); + jit_regset_setbit(_jit->regmask, node->u.w); + } + else + jit_regset_setbit(_jit->reglive, node->u.w); + } + if ((value & jit_cc_a1_reg) && !(node->v.w & jit_regno_patch)) { + if (value & jit_cc_a1_chg) { + jit_regset_clrbit(_jit->reglive, node->v.w); + jit_regset_setbit(_jit->regmask, node->v.w); + } + else + jit_regset_setbit(_jit->reglive, node->v.w); + } + if ((value & jit_cc_a2_reg) && !(node->w.w & jit_regno_patch)) + jit_regset_setbit(_jit->reglive, node->w.w); + if (jit_regset_set_p(_jit->regmask)) { + mpz_set_ui(_jit->blockmask, 0); + jit_update(0, node->next, &_jit->reglive, &_jit->regmask); + if (jit_regset_set_p(_jit->regmask)) { + /* any unresolved live state is considered as live */ + jit_regset_ior(_jit->reglive, _jit->reglive, _jit->regmask); + jit_regset_set_ui(_jit->regmask, 0); + } + } + break; + } +} + +void +_jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value) +{ + if (value & jit_cc_a0_reg) + jit_regset_setbit(_jit->regarg, jit_regno(node->u.w)); + if (value & jit_cc_a1_reg) + jit_regset_setbit(_jit->regarg, jit_regno(node->v.w)); + if (value & jit_cc_a2_reg) + jit_regset_setbit(_jit->regarg, jit_regno(node->w.w)); +} + +void +_jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value) +{ + if (value & jit_cc_a0_reg) + jit_regset_clrbit(_jit->regarg, jit_regno(node->u.w)); + if (value & jit_cc_a1_reg) + jit_regset_clrbit(_jit->regarg, jit_regno(node->v.w)); + if (value & jit_cc_a2_reg) + jit_regset_clrbit(_jit->regarg, jit_regno(node->w.w)); +} + +/* Compute initial reglive set of a basic block, keeping values not + * known in the regmask set. + */ +static void +_jit_setup(jit_state_t *_jit, jit_block_t *block) +{ +#define reglive block->reglive +#define regmask block->regmask + jit_node_t *node; + jit_int32_t spec; + unsigned long value; + + regmask = (1LL << _jit->reglen) - 1; + for (node = block->label->next; node; node = node->next) { + switch (node->code) { + case jit_code_label: case jit_code_prolog: + case jit_code_epilog: + return; + case jit_code_callr: + if (!(node->u.w & jit_regno_patch) && + jit_regset_tstbit(regmask, node->u.w)) { + jit_regset_clrbit(regmask, node->u.w); + jit_regset_setbit(reglive, node->u.w); + } + case jit_code_calli: + for (value = jit_regset_scan1(regmask, 0); value != ULONG_MAX; + value = jit_regset_scan1(regmask, value + 1)) { + spec = jit_class(_rvs[value].spec); + if (!(spec & jit_class_sav)) + jit_regset_clrbit(regmask, value); + if ((spec & jit_class_arg) && jit_regarg_p(node, value)) + jit_regset_setbit(reglive, value); + } + /* If result not already marked as live, record it may + * be used, so that subsequent call to jit_update + * will verify it. */ +#if defined(JIT_RET) + if (!jit_regset_tstbit(reglive, JIT_RET)) + jit_regset_setbit(regmask, JIT_RET); +# if __arm__ + if (!jit_regset_tstbit(reglive, _R1)) + jit_regset_setbit(regmask, _R1); +# endif +#endif +#if defined(JIT_FRET) + if (!jit_regset_tstbit(reglive, JIT_FRET)) + jit_regset_setbit(regmask, JIT_FRET); +#endif + break; + default: + value = jit_classify(node->code); + if ((value & jit_cc_a0_reg) && + !(node->u.w & jit_regno_patch) && + jit_regset_tstbit(regmask, node->u.w)) { + jit_regset_clrbit(regmask, node->u.w); + if (!(value & jit_cc_a0_chg)) + jit_regset_setbit(reglive, node->u.w); + } + if ((value & jit_cc_a1_reg) && + !(node->v.w & jit_regno_patch) && + jit_regset_tstbit(regmask, node->v.w)) { + jit_regset_clrbit(regmask, node->v.w); + if (!(value & jit_cc_a1_chg)) + jit_regset_setbit(reglive, node->v.w); + } + if ((value & jit_cc_a2_reg) && + !(node->w.w & jit_regno_patch) && + jit_regset_tstbit(regmask, node->w.w)) { + jit_regset_clrbit(regmask, node->w.w); + jit_regset_setbit(reglive, node->w.w); + } + break; + } + } +#undef regmask +#undef reglive +} + +/* remove bit of mask argument based on instructions arguments up to end + * of code or finding a basic block boundary, if a value is used as argument, + * also set the live bit to known value cannot be clobbered; if value is + * modified, just remove it from the mask as if it not already in the live + * bitmask, then the value is dead + * FIXME it should really stop on basic block boundaries, but for now + * at least, keep parsing nodes to avoid incorrectly deciding a register + * is not live, in case the initial live state is not consistent (changes + * caused by temporary register allocation should not cross basic block + * boundaries, so, initial computation by jit_setup+jit_update should + * hold up to end of jit generation, but, some things like simplify() + * or any other kind of register patching may have side effects that + * could only be properly handled by doing a second jit_setup+jit_update + * sequence, what is not cheap...) + */ +static void +_jit_update(jit_state_t *_jit, jit_bool_t setup, jit_node_t *node, + jit_regset_t *live, jit_regset_t *mask) +{ + jit_int32_t spec; + jit_regset_t ztmp; + jit_regset_t zmask; + unsigned long value; + jit_block_t *block; + jit_node_t *label; + + for (; node; node = node->next) { + restart: + if (jit_regset_set_p(*mask) == 0) + break; + switch (node->code) { + case jit_code_label: + if (setup) + return; + block = _jit->blocks.ptr + node->v.w; + if (mpz_tstbit(_jit->blockmask, node->v.w)) + return; + mpz_setbit(_jit->blockmask, node->v.w); + jit_regset_and(ztmp, *mask, block->reglive); + if (jit_regset_set_p(ztmp)) { + jit_regset_ior(*live, *live, ztmp); + jit_regset_com(ztmp, ztmp); + jit_regset_and(*mask, *mask, ztmp); + } + break; + case jit_code_prolog: case jit_code_epilog: + jit_regset_set_ui(*mask, 0); + return; + case jit_code_callr: + value = jit_regno(node->u.w); + if (!(node->u.w & jit_regno_patch)) { + if (jit_regset_tstbit(*mask, value)) { + jit_regset_clrbit(*mask, value); + jit_regset_setbit(*live, value); + } + } + case jit_code_calli: +#if defined(JIT_RET) + if (jit_regset_tstbit(*mask, JIT_RET)) { + jit_regset_setbit(_jit->reglive, JIT_RET); + jit_regset_clrbit(*mask, JIT_RET); + } +# if __arm__ + if (jit_regset_tstbit(*mask, _R1)) { + jit_regset_setbit(_jit->reglive, _R1); + jit_regset_clrbit(*mask, _R1); + } +# endif +#endif +#if defined(JIT_FRET) + if (jit_regset_tstbit(*mask, JIT_FRET)) { + jit_regset_setbit(_jit->reglive, JIT_FRET); + jit_regset_clrbit(*mask, JIT_FRET); + } +#endif + for (value = jit_regset_scan1(*mask, 0); value != ULONG_MAX; + value = jit_regset_scan1(*mask, value + 1)) { + spec = jit_class(_rvs[value].spec); + if (!(spec & jit_class_sav)) + jit_regset_clrbit(*mask, value); + if ((spec & jit_class_arg) && jit_regarg_p(node, value)) + jit_regset_setbit(*live, value); + } + break; + default: + value = jit_classify(node->code); + if (value & jit_cc_a2_reg) { + if (!(node->w.w & jit_regno_patch)) { + if (jit_regset_tstbit(*mask, node->w.w)) { + jit_regset_clrbit(*mask, node->w.w); + jit_regset_setbit(*live, node->w.w); + } + } + } + if (value & jit_cc_a1_reg) { + if (!(node->v.w & jit_regno_patch)) { + if (jit_regset_tstbit(*mask, node->v.w)) { + jit_regset_clrbit(*mask, node->v.w); + if (!(value & jit_cc_a1_chg)) + jit_regset_setbit(*live, node->v.w); + } + } + } + if (value & jit_cc_a0_reg) { + if (!(node->u.w & jit_regno_patch)) { + if (jit_regset_tstbit(*mask, node->u.w)) { + jit_regset_clrbit(*mask, node->u.w); + if (!(value & jit_cc_a0_chg)) + jit_regset_setbit(*live, node->u.w); + } + } + } + if (value & jit_cc_a0_jmp) { + if (node->flag & jit_flag_node) { + label = node->u.n; + if (node->code == jit_code_jmpi) { + node = label; + goto restart; + } + if (setup) + continue; + if (label->code == jit_code_label) { + block = _jit->blocks.ptr + label->v.w; + if (mpz_tstbit(_jit->blockmask, label->v.w)) + continue; + mpz_setbit(_jit->blockmask, label->v.w); + jit_regset_and(ztmp, *mask, block->reglive); + if (jit_regset_set_p(ztmp)) { + jit_regset_ior(*live, *live, ztmp); + jit_regset_com(ztmp, ztmp); + jit_regset_and(*mask, *mask, ztmp); + } + if (jit_regset_set_p(*mask) == 0) + return; + /* restore mask if branch is conditional */ + zmask = *mask; + jit_update(0, block->label->next, live, &zmask); + jit_regset_xor(ztmp, zmask, *mask); + /* remove known live registers from mask */ + if (jit_regset_set_p(ztmp)) { + jit_regset_and(ztmp, ztmp, *live); + jit_regset_com(ztmp, ztmp); + jit_regset_and(*mask, *mask, ztmp); + } + continue; + } + } + /* assume value is live due to jump to unknown location */ + jit_regset_ior(*live, *live, *mask); + jit_regset_set_ui(*mask, 0); + return; + } + break; + } + } +} + +static void +_thread_jumps(jit_state_t *_jit) +{ + jit_node_t *prev; + jit_node_t *node; + jit_node_t *next; + jit_int32_t mask; + + for (prev = node = _jit->head; node;) { + next = node->next; + switch (node->code) { + case jit_code_jmpi: + if (redundant_jump(prev, node)) { + node = prev; + continue; + } + if (shortcut_jump(prev, node)) + continue; + break; + case jit_code_jmpr: + case jit_code_callr: case jit_code_calli: + /* non optimizable jump like code */ + break; + default: + mask = jit_classify(node->code); + if (mask & jit_cc_a0_jmp) { + if (reverse_jump(prev, node) || + shortcut_jump(prev, node)) + continue; + } + break; + } + prev = node; + node = next; + } +} + +static void +_sequential_labels(jit_state_t *_jit) +{ + jit_node_t *jump; + jit_node_t *link; + jit_node_t *prev; + jit_node_t *next; + jit_node_t *node; + + for (prev = node = _jit->head; node; node = next) { + next = node->next; + if (node->code == jit_code_label) { + if (!node->flag) { + if (!node->link) { + del_label(prev, node); + continue; + } + if (prev != node && prev->code == jit_code_label) { + if ((jump = node->link)) { + for (; jump; jump = link) { + link = jump->link; + jump->u.n = prev; + jump->link = prev->link; + prev->link = jump; + } + node->link = NULL; + } + del_label(prev, node); + continue; + } + } + if (next && next->code == jit_code_label && !next->flag) { + if ((jump = next->link)) { + for (; jump; jump = link) { + link = jump->link; + jump->u.n = node; + jump->link = node->link; + node->link = jump; + } + next->link = NULL; + } + del_label(node, next); + next = node->next; + continue; + } + } + prev = node; + } +} + +static jit_bool_t +_shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) +{ + jit_bool_t cond; + jit_node_t *jump; + jit_node_t *next; + jit_node_t *temp; + + if (!(node->flag & jit_flag_node)) + return (0); + assert(node->code != jit_code_jmpr); + cond = node->code != jit_code_jmpi; + jump = node->u.n; + for (next = jump->next; next; next = next->next) { + switch (next->code) { + case jit_code_jmpi: + if (jump->link == node) + jump->link = node->link; + else { + for (temp = jump->link; + temp->link != node; + temp = temp->link) + assert(temp != NULL); + temp->link = node->link; + } + jump = next->u.n; + node->u.n = jump; + node->link = jump->link; + jump->link = node; + return (1); + case jit_code_jmpr: + if (cond) + return (0); + node->code = jit_code_jmpr; + node->u.w = next->u.w; + node->link = NULL; + node->flag &= ~jit_flag_node; + return (1); + case jit_code_note: case jit_code_label: + break; + default: + return (0); + } + } + return (0); +} + +static jit_bool_t +_redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) +{ + jit_node_t *local_prev; + jit_node_t *local_next; + + if (!(node->flag & jit_flag_node)) + return (0); + for (local_prev = node, local_next = node->next; + local_next; + local_prev = local_next, local_next = local_next->next) { + + switch (local_next->code) { + case jit_code_label: case jit_code_epilog: + if (node->u.n == local_next) { + if (local_next->link == node) + local_next->link = node->link; + else { + for (local_prev = local_next->link; + local_prev->link != node; + local_prev = local_prev->link) + assert(local_prev != NULL); + local_prev->link = node->link; + } + del_node(prev, node); + return (1); + } + break; + case jit_code_note: + break; + default: + return (0); + } + } + return (0); +} + +static jit_code_t +reverse_jump_code(jit_code_t code) +{ + switch (code) { + case jit_code_bltr: return (jit_code_bger); + case jit_code_blti: return (jit_code_bgei); + case jit_code_bltr_u: return (jit_code_bger_u); + case jit_code_blti_u: return (jit_code_bgei_u); + case jit_code_bler: return (jit_code_bgtr); + case jit_code_blei: return (jit_code_bgti); + case jit_code_bler_u: return (jit_code_bgtr_u); + case jit_code_blei_u: return (jit_code_bgti_u); + case jit_code_beqr: return (jit_code_bner); + case jit_code_beqi: return (jit_code_bnei); + case jit_code_bger: return (jit_code_bltr); + case jit_code_bgei: return (jit_code_blti); + case jit_code_bger_u: return (jit_code_bltr_u); + case jit_code_bgei_u: return (jit_code_blti_u); + case jit_code_bgtr: return (jit_code_bler); + case jit_code_bgti: return (jit_code_blei); + case jit_code_bgtr_u: return (jit_code_bler_u); + case jit_code_bgti_u: return (jit_code_blei_u); + case jit_code_bner: return (jit_code_beqr); + case jit_code_bnei: return (jit_code_beqi); + case jit_code_bmsr: return (jit_code_bmcr); + case jit_code_bmsi: return (jit_code_bmci); + case jit_code_bmcr: return (jit_code_bmsr); + case jit_code_bmci: return (jit_code_bmsi); + case jit_code_bltr_f: return (jit_code_bunger_f); + case jit_code_blti_f: return (jit_code_bungei_f); + case jit_code_bler_f: return (jit_code_bungtr_f); + case jit_code_blei_f: return (jit_code_bungti_f); + case jit_code_beqr_f: return (jit_code_bltgtr_f); + case jit_code_beqi_f: return (jit_code_bltgti_f); + case jit_code_bger_f: return (jit_code_bunltr_f); + case jit_code_bgei_f: return (jit_code_bunlti_f); + case jit_code_bgtr_f: return (jit_code_bunler_f); + case jit_code_bgti_f: return (jit_code_bunlei_f); + case jit_code_bner_f: return (jit_code_buneqr_f); + case jit_code_bnei_f: return (jit_code_buneqi_f); + case jit_code_bunltr_f: return (jit_code_bger_f); + case jit_code_bunlti_f: return (jit_code_bgei_f); + case jit_code_bunler_f: return (jit_code_bgtr_f); + case jit_code_bunlei_f: return (jit_code_bgti_f); + case jit_code_buneqr_f: return (jit_code_bgtr_f); + case jit_code_buneqi_f: return (jit_code_bgti_f); + case jit_code_bunger_f: return (jit_code_bltr_f); + case jit_code_bungei_f: return (jit_code_blti_f); + case jit_code_bungtr_f: return (jit_code_bler_f); + case jit_code_bungti_f: return (jit_code_blei_f); + case jit_code_bltgtr_f: return (jit_code_beqr_f); + case jit_code_bltgti_f: return (jit_code_beqi_f); + case jit_code_bordr_f: return (jit_code_bunordr_f); + case jit_code_bordi_f: return (jit_code_bunordi_f); + case jit_code_bunordr_f:return (jit_code_bordr_f); + case jit_code_bunordi_f:return (jit_code_bordi_f); + case jit_code_bltr_d: return (jit_code_bunger_d); + case jit_code_blti_d: return (jit_code_bungei_d); + case jit_code_bler_d: return (jit_code_bungtr_d); + case jit_code_blei_d: return (jit_code_bungti_d); + case jit_code_beqr_d: return (jit_code_bltgtr_d); + case jit_code_beqi_d: return (jit_code_bltgti_d); + case jit_code_bger_d: return (jit_code_bunltr_d); + case jit_code_bgei_d: return (jit_code_bunlti_d); + case jit_code_bgtr_d: return (jit_code_bunler_d); + case jit_code_bgti_d: return (jit_code_bunlei_d); + case jit_code_bner_d: return (jit_code_buneqr_d); + case jit_code_bnei_d: return (jit_code_buneqi_d); + case jit_code_bunltr_d: return (jit_code_bger_d); + case jit_code_bunlti_d: return (jit_code_bgei_d); + case jit_code_bunler_d: return (jit_code_bgtr_d); + case jit_code_bunlei_d: return (jit_code_bgti_d); + case jit_code_buneqr_d: return (jit_code_bgtr_d); + case jit_code_buneqi_d: return (jit_code_bgti_d); + case jit_code_bunger_d: return (jit_code_bltr_d); + case jit_code_bungei_d: return (jit_code_blti_d); + case jit_code_bungtr_d: return (jit_code_bler_d); + case jit_code_bungti_d: return (jit_code_blei_d); + case jit_code_bltgtr_d: return (jit_code_beqr_d); + case jit_code_bltgti_d: return (jit_code_beqi_d); + case jit_code_bordr_d: return (jit_code_bunordr_d); + case jit_code_bordi_d: return (jit_code_bunordi_d); + case jit_code_bunordr_d:return (jit_code_bordr_d); + case jit_code_bunordi_d:return (jit_code_bordi_d); + case jit_code_boaddr: return (jit_code_bxaddr); + case jit_code_boaddi: return (jit_code_bxaddi); + case jit_code_boaddr_u: return (jit_code_bxaddr_u); + case jit_code_boaddi_u: return (jit_code_bxaddi_u); + case jit_code_bxaddr: return (jit_code_boaddr); + case jit_code_bxaddi: return (jit_code_boaddi); + case jit_code_bxaddr_u: return (jit_code_boaddr_u); + case jit_code_bxaddi_u: return (jit_code_boaddi_u); + case jit_code_bosubr: return (jit_code_bxsubr); + case jit_code_bosubi: return (jit_code_bxsubi); + case jit_code_bosubr_u: return (jit_code_bxsubr_u); + case jit_code_bosubi_u: return (jit_code_bxsubi_u); + case jit_code_bxsubr: return (jit_code_bosubr); + case jit_code_bxsubi: return (jit_code_bosubi); + case jit_code_bxsubr_u: return (jit_code_bosubr_u); + case jit_code_bxsubi_u: return (jit_code_bosubi_u); + default: abort(); /* invalid jump code */ + } +} + +/* + * change common pattern: + *