diff --git a/parallel-rsp/jit_allocator.cpp b/parallel-rsp/jit_allocator.cpp index 1c30958..4207fab 100644 --- a/parallel-rsp/jit_allocator.cpp +++ b/parallel-rsp/jit_allocator.cpp @@ -14,7 +14,11 @@ namespace RSP { namespace JIT { +#if TARGET_OS_TV // tvOS is 64bit but does not allow an infinite amount of VA space +static constexpr bool huge_va = false; +#else static constexpr bool huge_va = std::numeric_limits::max() > 0x100000000ull; +#endif // On 64-bit systems, we will never allocate more than one block, this is important since we must ensure that // relative jumps are reachable in 32-bits. // We won't actually allocate 1 GB on 64-bit, but just reserve VA space for it, which we have basically an infinite amount of. diff --git a/parallel-rsp/lightning/.gitignore b/parallel-rsp/lightning/.gitignore index 62ca42a..bc7e971 100644 --- a/parallel-rsp/lightning/.gitignore +++ b/parallel-rsp/lightning/.gitignore @@ -1,4 +1,15 @@ +/build-aux +* + +*.o +*.lo +*.la + +.libs/ +.deps/ +*/.libs/ +*/.deps/ + autom4te.cache aclocal.m4 depcomp @@ -20,14 +31,14 @@ missing size stamp-h1 test-driver -check/.deps -doc/.deps -lib/.deps + m4/libtool.m4 m4/lt~obsolete.m4 m4/ltoptions.m4 m4/ltsugar.m4 m4/ltversion.m4 -doc/mdate-sh -doc/texinfo.tex + lightning.pc +include/lightning.h + +build-aux/ diff --git a/parallel-rsp/lightning/.gitmodules b/parallel-rsp/lightning/.gitmodules new file mode 100644 index 0000000..acb2669 --- /dev/null +++ b/parallel-rsp/lightning/.gitmodules @@ -0,0 +1,3 @@ +[submodule "gnulib"] + path = gnulib + url = git://git.sv.gnu.org/gnulib.git diff --git a/parallel-rsp/lightning/.gitrepo b/parallel-rsp/lightning/.gitrepo index fe7174d..8200a32 100644 --- a/parallel-rsp/lightning/.gitrepo +++ b/parallel-rsp/lightning/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://git.savannah.gnu.org/git/lightning.git branch = master - commit = b0b8eb5e856c0d29053dc842e1919a2eb58c8cda - parent = 9f797430963d9cf0fcef7d963466f9cac7026de2 + commit = 75e5274ba59feab99a2afcf329d890911ed01eaf + parent = b7d23a9d07791c7006d57c58ab1e442103741cd6 method = merge - cmdver = 0.4.1 + cmdver = 0.4.5 diff --git a/parallel-rsp/lightning/ChangeLog b/parallel-rsp/lightning/ChangeLog index 76cac91..6a30fd4 100644 --- a/parallel-rsp/lightning/ChangeLog +++ b/parallel-rsp/lightning/ChangeLog @@ -1,3 +1,307 @@ +2023-02-09 Paulo Andrade + + * check/Makefile.am: Update for new bit.tst test, to check the + new clor, clzr, ctor and ctzr instructions. + * check/all.tst: Update to verify encoding of new instructions. + * check/lightning.c: Update to have the lightning "assembler" + understanding the new instructions. + * include/lightning.h.in: Define new codes for new instructions. + * lib/jit_aarch64.c, lib/jit_alpha.c, lib/jit_arm.c, lib/jit_hppa.c, + lib/jit_ia64.c, lib/jit_loongarch.c, lib/jit_mips.c, lib/jit_ppc.c, + lib/jit_riscv.c, lib/jit_s390.c, lib/jit_sparc.c, lib/jit_x86.c: + Implement fallback version of new instructions. + * lib/jit_fallback.c: Actual implementation of the fallbacks of + the new instructions. + * lib/jit_names.c: Update to print debug information of new + instructions. + +2023-01-26 Paulo Andrade + + * check/riprel.c, check/riprel.ok: New check files. + * check/Makefile.am: Support for new riprel test. + * lib/jit_x86-cpu.c, lib/jit_x86-sse.c, lib/jit_x86.c: Implement + %rip relative addressing when reliable. Currently disabled for + x32 and _WIN32; could be added for positive relative addresses + only where it should work. + * lib/lightning.c: Correct problem added in previous patch due + to not testing on a 32 bit environment. + +2023-01-23 Paulo Andrade + + * lib/jit_mips-cpu.c, lib/jit_mips-cpu.c: Use pseudo instructions + "b" (BEQ(0,0,disp)) and "bal" (BGEZAL(0,disp)) for mips2, when an + unconditional branch or function call is known to be in range of a + relative jump. This should significantly reduce jit size generation. + +2023-01-20 Paulo Andrade + + * lib/jit_mips-cpu.c, lib/jit_mips.c, lib/jit_rewind.c: Adapt + code to implement a variable framesize and optimize frame pointer + for simple leaf functions. + +2023-01-19 Paulo Andrade + + * lib/jit_riscv.c, lib/jit_riscv-cpu.c: Adapt code to use a + variable framesize. Previously it was aligning the stack at + 8 bytes, not 16. Now functions are called with a 16 byte aligned + stack. + +2023-01-18 Paulo Andrade + + * include/lightning/jit_private.h: Include new framesize field + of jit_compiler_t; add new alist field for jit_function_t; add + new cvt_offset and need_stack fields specific to x86. + * lib/jit_x86.c, lib/jit_x86-cpu: Rewrite code to create stack + frames, so that less stack space can be used if no, or very few + callee save registers are modified in a function. + * jit_x86-sse.c, jit_x86-x87.c: Make CVT_OFFSET variable, and + dynamically allocated; this is required to avoid needing to + modify twice %rsp at function prologs, even if no stack space + is used. + +2022-11-09 Paulo Andrade + + * configure.ac: Add new --enable-devel-strong-type-checking + option. + * include/lightning.h.in: Rework to not need to know if + PACKED_STACK is defined, and add a new argument to _jit_arg, + _jit_putarg{r,i}, _jit_pusharg{r,i} and _jit_ret{r,i} to have + the same code path if PACKED_STACK is defined or not, and also + to implement STRONG_TYPE_CHECK enabled with the new + --enable-devel-strong-type-checking. + * include/lightning/jit_private.h: Add new macros to add assertions + for STRONG_TYPE_CHECK and avoid pasting tokens in jit_inc_synth* + when the token is not a static known value. + * lib/jit_aarch64.c: The first implementation of the new code, + working correctly in Apple M1 and with and without STRONG_TYPE_CHECK + in Linux. + +2022-11-08 Paulo Andrade + + Add support for packed stack arguments as used by Apple M1 + aarch64 cpus. This requires a major redesign in how Lightning + works, because contrary to all other supported ports, in this + case arguments must be truncated and sign/zero extended if + passed in registers, but when receiving the argument, there + is no need to truncate and sign/zero extend. + Return values are also treated this way. The callee must + truncate sign/zero extend, not the caller. + check/Makefile.am: Add LIGHTNING_CFLAGS to AM_CFLAGS. + check/all.tst: Implement paired arg/getarg/pusharg/putarg/ret + codes to validate they do not generate assertions. + * check/allocar.tst, check/call.tst, check/fib.tst, check/put.tst, + check/stack.tst: Update to pass in all build types. + check/lightning.c: Add new codes for extra codes to handle + packed stack. + * configure.ac: Add a preprocessor define to know if packed stack + need is required. This is not really used, as it was moved to + jit_aarch64.h. + * doc/Makefile.am: Add LIGHTNING_CFLAGS to AM_CFLAGS. + * doc/rpn.c: Update to pass in all build types. + include/lightning.h.in: Add new codes and reorder enum. + * include/lightning/jit_aarch64.h: Detect condition of needing + a packed stack. + * lib/jit_aarch64-sz.c: Regenerate. + * lib/jit_aarch64.c: Major updates for packed stack. + * lib/jit_names.c: Updates for debug output. + * lib/lightning.c: Update for new codes. + +2022-10-31 Marc Nieper-Wißkirchen + + Add new skip instruction. + * .gitignore: Update from Gnulib. + * check/Makefile.am: Add tests. + * check/lightning.c: Handle skip instructions. + * check/protect.c: Rewrite with skip. + * check/skip.ok: New test. + * check/skip.tst: New test. + * doc/body.texi: Document the skip instruction. + * include/lightning.h.in: Add the skip instruction. + * lib/jit_aarch64-sz.c: Update for skip instruction. + * lib/jit_aarch64.c: Implement skip instruction. + * lib/jit_alpha-sz.c: Update for skip instruction. + * lib/jit_alpha.c: Implement skip instruction. + * lib/jit_arm-sz.c: Update for skip instruction. + * lib/jit_arm.c: Implement skip instruction. + * lib/jit_hppa-sz.c: Update for skip instruction. + * lib/jit_hppa.c: Implement skip instruction. + * lib/jit_ia64-sz.c: Update for skip instruction. + * lib/jit_ia64.c: Implement skip instruction. + * lib/jit_loongarch-sz.c: Update for skip instruction. + * lib/jit_loongarch.c: Implement skip instruction. + * lib/jit_mips-sz.c: Update for skip instruction. + * lib/jit_mips.c: Implement skip instruction. + * lib/jit_names.c: Update for skip instruction. + * lib/jit_ppc-sz.c: Update for skip instruction. + * lib/jit_ppc.c: Implement skip instruction. + * lib/jit_riscv-sz.c: Update for skip instruction. + * lib/jit_riscv.c: Implement skip instruction. + * lib/jit_s390-sz.c: Update for skip instruction. + * lib/jit_s390.c: Implement skip instruction. + * lib/jit_size.c: Treat align and skip in a special way. + * lib/jit_sparc-sz.c: Update for skip instruction. + * lib/jit_sparc.c: Implement skip instruction. + * lib/jit_x86-sz.c: Update for skip instruction. + * lib/jit_x86.c: Implement skip instruction. + * lib/lightning.c: Classify skip instruction. + +2022-10-30 Marc Nieper-Wißkirchen + + Add user-visible functions jit_protect and jit_unprotect. + * check/Makefile.am: Add test for jit_protect and jit_unprotect. + * check/protect.c: New test. + * doc/body.texi: Add documentation for jit_protect and + jit_unprotect. + * include/lightning.h.in: Add prototypes for jit_protect and + jit_unprotect. + * include/lightning/jit_private.h: Add a field to store the size + of the protected memory. + * lib/lightning.c: Remember the size of the protected memory and + implement the two new functions. + +2022-10-12 Paulo Andrade + + * include/lightning/jit_loongarch.h, lib/jit_loongarch-cpu.c, + lib/jit_loongarch-fpu.c, lib/jit_loongarch-sz.c, lib/jit_loongarch.c: + New files implementing the first version of the new loongarch port. + * check/float.tst: Add preprocessor checks for NaN and +-Inf + values converted to integers for loongarch. + * configure.ac: Add check and conditionals for new architecture. + * include/lightning.h.in, check/lightning.c, + include/lightning/Makefile.am, include/lightning/jit_private.h, + lib/Makefile.am, lib/jit_size.c, lib/lightning.c: Update for new + port. + +2022-10-05 Paulo Andrade + + * check/lightning.c: Remove -Dmacro=value from usage and attempt + to parse it. It was buggy and not properly implemented. Now + it pass any extra options to the generated jit. To pass any + option starting with '-' need to also use '--'. + * check/collatz.e: New sample file showing an example of jit + generation. + +2022-10-04 Paulo Andrade + + * include/lightning/jit_private.h: Add new flag to jit_block_t. + * lib/lightning.c: Rewrite register liveness and state at block + entry code to avoid a very expensive and non scaling code path. + Now it attempts to do as few as possible recomputations when + merging state of adjacent blocks, still doing one extra nop pass + (in the sense that it will not find any changes) to make sure the + logic is correct. + +2022-09-30 Paulo Andrade + + * include/lightning/jit_private.h: Implement new data structures + specific to riscv. + * lib/jit_disasm.c: Extra disassemble code for riscv constant pool. + * lib/jit_riscv-cpu.c: Modify movi to use constant pool if 3 or + more instructions are required to construct constant and modify + movi_p to use a pc relative load from a constant pool. + lib/jit_riscv-sz.c: Update for new constant pool code. Most + instructions that need 64 bit constants are significantly reduced. + * lib/jit_riscv.c: Implement most of the constant pool code. + * lib/jit_size.c: Update for estimate of code generation size. + * lib/lightning.c: Update for riscv specific code, and also make + sure to mprotect the constant pool as executable. + +2022-09-08 Paulo Andrade + + * lib/jit_fallback.c: Implement fallback compare and swap with + pthreads. + * check/Makefile.am: Update for new cas{r,i} simple test. + * check/catomic.c, check/catomic.ok: New test case for + simple compare and swap atomic operation. + * check/lightning.c: Add entries to be able to use + the new compare and swap atomic operation. Still missing + a general test, only the basic C version. + * include/lightning.h.in: Include pthread.h, even if not + needing a fallback compare and swap. + * include/lightning/jit_private.h: Add support for a register pair + in second argument. Required by the new casr and casi operations. + * lib/jit_aarch64-cpu.c, lib/jit_aarch64-sz.c, lib/jit_aarch64.c, + lib/jit_ppc-cpu.c, lib/jit_ppc-sz.c, lib/jit_ppc.c, lib/jit_x86-cpu.c, + lib/jit_x86-sz.c, lib/jit_x86.c: Implement inline code for compare + and swap. + * lib/jit_arm-cpu.c, lib/jit_arm-sz.c, lib/jit_arm.c: Implement + inline code for compare and swap if cpu is armv7, otherwise, use + a fallback with pthreads. + * lib/jit_alpha-cpu.c, lib/jit_alpha-sz.c, lib/jit_alpha.c, + lib/jit_hppa-cpu.c, lib/jit_hppa-sz.c, lib/jit_hppa.c, + lib/jit_ia64-cpu.c, lib/jit_ia64-sz.c, lib/jit_ia64.c, + lib/jit_mips-cpu.c, lib/jit_mips-sz.c, lib/jit_mips.c, + lib/jit_riscv-cpu.c, lib/jit_riscv-sz.c, lib/jit_riscv.c, + lib/jit_s390-cpu.c, lib/jit_s390-sz.c, lib/jit_s390.c, + lib/jit_sparc-cpu.c, lib/jit_sparc-sz.c, lib/jit_sparc.c: Implement + fallback compare and swap with pthreads. At least some of these + should be updated for inline code generation. + * lib/jit_names.c, lib/jit_print.c: lib/lightning.c: Update for the + new compare and swap operation. + * doc/body.texi: Add simple documentation of the compare and swap + new operation. + +2022-08-12 Marc Nieper-Wißkirchen + + Document jit_align. + * doc/body.texi: Add documentation for jit_align. + +2022-05-14 Paulo Andrade + + * include/lightning.h.in: Reorder jit_mov{n,z}r in instruction list. + * lib/jit_alpha.c, lib/jit_alpha-cpu.c, lib/jit_hppa.c, + lib/jit_hppa-cpu.c, lib/jit_ia64.c, lib/jit_ia64-cpu.c, + lib/jit_riscv.c, lib/jit_riscv-cpu.c, lib/jit_s390.c, + lib/jit_s390-cpu.c, lib/jit_sparc.c, lib/jit_sparc-cpu.c: + Implement fallback jit_mov{n,z}r. These are a somewhat cheap + implementation, but should be reviewed for the arches that already + have a proper conditional move. + * lib/jit_arm-sz.c, lib/jit_mips-sz.c: Add missing maximum size + estimative and reorder. + * lib/jit_aarch64-sz.c, lib/jit_x86-sz.c, lib/jit_ppc-sz.c: + Reorder entry to match definition order. + * lib/jit_aarch64-sz.c, lib/jit_alpha-sz.c, lib/jit_hppa-sz.c, + lib/jit_ia64-sz.c, lib/jit_riscv-sz.c, lib/jit_s390-sz.c, + lib/jit_sparc-sz.c: Add heuristic value, basically the sum of + the cost of a movr + beqr. + * lib/jit_names.c: Add entries for debug output of mov{n,z}r. + * lib/lightning.c: Use proper bitmask in jit_classify. + +2021-04-03 Marc Nieper-Wißkirchen + + * check/Makefile.am: Add test for the live instruction. + * check/live.ok: New file. + * check/live.tst: New file. + * doc/body.texi: Add documentation for the live instruction and + for jit_get_reg/jit_get_unreg. Fix menu entries. + * include/lightning.h.in (jit_get_reg, jit_unget_reg): Expose the + macros in the public header file. + * include/lightning/jit_private.h (jit_get_reg, jit_unget_reg): + Remove the macros from the private header file. + +2021-04-03 Marc Nieper-Wißkirchen + + * Makefile.am, check/Makefile.am, doc/Makefile.am, + lib/Makefile.am: Include $(top_builddir)/include in include paths + for the autoconf-generated header file lightning.h. + +2021-04-03 Marc Nieper-Wißkirchen + + * doc/.gitignore: Add version.texi to list of ignored files. + * doc/version.texi: Remove file from version control. + +2020-18-04 Paulo Andrade + + * lib/jit_x86-cpu.c, lib/jit_x86.c: Implement %rip relative + calls and jumps on x86_64. Currently very conservative, assuming + a jit block can be larger than 2G, so, if a jump or call is in + the same jit generation, but target is unknown, use an indirect + branch (could have an option to assume a jit code block is + never larger than 2G). Also a deoptimization is that now does + not always generate ip relative jmpi; previously implicitly + assumed jmpi would never be larger than 2G. Overall still an + optimization. + 2020-23-01 Paulo Andrade * lib/lightning.c: Add a proper fix to the condition of considering @@ -26,7 +330,7 @@ to the set of registers to scan for live range, what might consume a lot of cpu time, doing nothing. -2019-09-16 Marc Nieper-WiÃkirchen +2019-09-16 Marc Nieper-Wißkirchen * include/lightning/jit_x86.h, lib/jit_x86.c: Correct x86_64 backend, made %r12 a callee-save register as dictated by the @@ -65,7 +369,7 @@ https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=925129 * THANKS: update. -2019-08-29 Marc Nieper-WiÃkirchen +2019-08-29 Marc Nieper-Wißkirchen * include/lightning/jit_private.h: Move definition of offsetof from the public header file here. @@ -1415,7 +1719,7 @@ * include/lightning/jit_arm.h: Correct wrong jit_f macro definition. - * include/lightning/jit_ia64.h, include/lightning/jit_ppc.h: + * include/lightning/jit_ia64.h, include/lightning/jit_ppc.h: Correct wrong jit_r macro definition. * lib/jit_x86-x87.c, lib/jit_x86.c: Actually use the @@ -1884,7 +2188,7 @@ divr_f and divrd_d implementation. * check/lightning.c: Add __ia64__ preprocessor define - on Itanium. + on Itanium. * check/alu.inc, check/clobber.tst, check/float.tst: Define several macros conditionally to __ia64__. This is required @@ -2991,7 +3295,7 @@ * include/lightning/jit_x86.h: Correct typo in macro name. * lib/jit_arm.c, lib/jit_arm-cpu.c, lib/jit_mips.c, - lib/jit_mips-cpu.c, lib/jit_ppc.c, lib/jit_ppc-cpu.c, + lib/jit_mips-cpu.c, lib/jit_ppc.c, lib/jit_ppc-cpu.c, lib/jit_x86.c, lib/jit_x86-cpu.c: Correct wrong code to get current jit function pointer. @@ -3225,7 +3529,7 @@ * lightning/i386/core.h (jit_ldr_c, jit_ldxr_c, jit_ldr_s, jit_ldxr_s): Move... * lightning/i386/core-32.h: ... here. - * lightning/i386/core-64.h (jit_ldr_c, jit_ldxr_c, jit_ldr_s, + * lightning/i386/core-64.h (jit_ldr_c, jit_ldxr_c, jit_ldr_s): Use movsbq and movswq. 2010-08-10 Paulo César Pereira de Andrade @@ -3306,11 +3610,11 @@ * lightning/ppc/funcs.h (jit_flush_code): modified the computation of start/end. The pointer arithmetic was done without casting. It - prevented compilation with recent gcc versions. + prevented compilation with recent gcc versions. * lightning/ppc/core.h (jit_pushr_i): The offset for the store was incorrect. Should have been 4 bytes below SP (not above). - * lightning/ppc/core.h (jit_popr_i): The offset for the load was - incorrect. Should have been 0 (not +8). + * lightning/ppc/core.h (jit_popr_i): The offset for the load was + incorrect. Should have been 0 (not +8). 2008-06-17 Paolo Bonzini @@ -3589,7 +3893,7 @@ * lightning/i386/fp-32.h: ... here. * lightning/i386/fp-64.h: Write the code. * lightning/sparc/fp.h: Fix jit_extr_{f_d,d_f} register order. - + 2006-11-22 Paolo Bonzini * lightning/i386/asm-i386.h: Move x86-64 instructions... @@ -3803,7 +4107,7 @@ 2006-01-23 Paolo Bonzini * configure.ac: Fix comments in config.h.in. - + 2005-11-25 Paolo Bonzini * lightning/sparc/fp.h: Fix header comment. @@ -4123,7 +4427,7 @@ * lightning/ppc/funcs.h: correctly align stack pointer No changelogs for the assemblers (lightning directory) until 1.0 - + 2003-03-27 Paolo Bonzini * tests/printf2.c: new test @@ -4135,7 +4439,7 @@ No changelogs for the assemblers (lightning directory) until 1.0 2001-01-19 Paolo Bonzini * configure.in: support cross-assembling - + * disass/bfd.h, disass/dis-asm.h, disass/dis-buf.c, disass/i386-dis.c, disass/i386.h, disass/ppc-dis.c, disass/ppc.h, disass/ppc-opc.c, disass/sparc-dis.c, diff --git a/parallel-rsp/lightning/Makefile.am b/parallel-rsp/lightning/Makefile.am index c921901..8dbbaef 100644 --- a/parallel-rsp/lightning/Makefile.am +++ b/parallel-rsp/lightning/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright 2000, 2001, 2002, 2012-2019 Free Software Foundation, Inc. +# Copyright 2000, 2001, 2002, 2012-2023 Free Software Foundation, Inc. # # This file is part of GNU lightning. # @@ -17,18 +17,22 @@ ACLOCAL_AMFLAGS = -I m4 SUBDIRS = \ + gnulib-lib \ check \ doc \ include \ lib +EXTRA_DIST = m4/gnulib-cache.m4 + pkgconfiglibdir = $(libdir)/pkgconfig pkgconfiglib_DATA = lightning.pc if get_jit_size JIT_SIZE_PATH = "$(top_builddir)/jit_$(cpu)-sz.c" AM_CPPFLAGS=-DGET_JIT_SIZE=1 -DJIT_SIZE_PATH='$(JIT_SIZE_PATH)' -AM_CFLAGS = -I$(top_srcdir)/include -D_GNU_SOURCE $(LIGHTNING_CFLAGS) +AM_CFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -D_GNU_SOURCE $(LIGHTNING_CFLAGS) noinst_PROGRAMS = size size_LDADD = $(top_builddir)/lib/liblightning.la -lm $(SHLIB) diff --git a/parallel-rsp/lightning/README b/parallel-rsp/lightning/README index ae36ea5..7e3df42 100644 --- a/parallel-rsp/lightning/README +++ b/parallel-rsp/lightning/README @@ -1,3 +1,5 @@ GNU lightning is a library to aid in making portable programs that compile assembly code at run time. For more information, look at the info documentation. + +For help building lightning, see README-hacking. diff --git a/parallel-rsp/lightning/README-hacking b/parallel-rsp/lightning/README-hacking new file mode 100644 index 0000000..cc61598 --- /dev/null +++ b/parallel-rsp/lightning/README-hacking @@ -0,0 +1,106 @@ +This README-hacking file describes the development environment. + +Everything related to the development of GNU lightning is on Savannah: +https://savannah.gnu.org/projects/lightning/. + + +* Working from the Repository + +** Autotools + +This distribution uses the latest stable versions of Automake, Autoconf. If +you are getting the sources from git (or change configure.ac), you'll need +to have these tools installed to (re)build. All +of these programs are available from ftp://ftp.gnu.org/gnu. + +If you're using a GNU/Linux distribution, the easiest way to install these +packages depends on your system. The following shell command should work +for Debian-based systems such as Ubuntu: + + $ sudo apt-get install autoconf automake + + +** Building + +If you intend to do development work with lightning, it's useful to build +lightning with its disassembler feature enabled. This optional feature +requires additional dependencies. On Ubuntu, this command should work: + + $ sudo apt-get install binutils-dev libiberty-dev zlib1g-dev + +After getting the git sources, and installing the tools above, you can run + + $ ./bootstrap + $ ./configure + $ make + $ make check + +to do a fresh build. At this point, there should be no difference between +your local copy, and the master copy: + + $ git diff + +should output no difference. + +After that first time, running make should suffice. + +To install lightning: + + $ sudo make install + +** Gnulib + +This distribution also uses Gnulib (https://www.gnu.org/software/gnulib) to +share common files, stored as a submodule in git. + +** Updating + + $ git pull + $ git submodule update + +** Updating a submodule + +To update a submodule, say gnulib, do as follows: + +Get the most recent version of the master branch from git. + + $ cd gnulib + $ git pull + +Make sure GNU lightning can live with that version of gnulib. + + $ cd .. + $ ./bootstrap + $ make distcheck + +Register your changes. + + $ git commit ... + + +---- + + +Copyright 2021 Free Software Foundation, Inc. + +This file is part of GNU lightning. + +GNU lightning is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation; either version 3, or (at +your option) any later version. + +GNU lightning is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this program. If not, see +. + + +Local Variables: +mode: outline +fill-column: 76 +End: diff --git a/parallel-rsp/lightning/THANKS b/parallel-rsp/lightning/THANKS index 0e0f1a9..d5737af 100644 --- a/parallel-rsp/lightning/THANKS +++ b/parallel-rsp/lightning/THANKS @@ -19,3 +19,4 @@ Holger Hans Peter Freyther Jon Arintok Bruno Haible Marc Nieper-Wißkirchen +Paul Cercueil diff --git a/parallel-rsp/lightning/TODO b/parallel-rsp/lightning/TODO index 676af02..8b13789 100644 --- a/parallel-rsp/lightning/TODO +++ b/parallel-rsp/lightning/TODO @@ -1,28 +1 @@ - * Validate that divrem in jit_x86-cpu.c is not modifying - the non result arguments. This is not verified by clobber.tst, - as it only checks registers not involved in the operation - (because it does not know about values being set as input - for the the operation). - * Write a simple higher level language implementation generating - jit with lightning, that could be some lisp or C like language. - - * rerun ./configure --enable-devel-get-jit-size and regenerate - the related jit_$arch-sz.c for the ports where nodata is - meaningful: - hppa (done) - i586 (done) - ia64 - mips o32 (done) - mips n32 - mips n64 - powerpc 32 (done) - powerpc 64 (done) - ppc - s390x (done) - sparc (done) - x86_64 (done) - Missing ones are due to no longer (remote) access to such hosts - and may be broken with jit_set_data(..., JIT_DISABLE_DATA). - (ia64 hp-ux or linx), (irix mips for 32 or 64 abi), and - (darwin ppc). diff --git a/parallel-rsp/lightning/bootstrap b/parallel-rsp/lightning/bootstrap new file mode 100755 index 0000000..c17a36f --- /dev/null +++ b/parallel-rsp/lightning/bootstrap @@ -0,0 +1,1090 @@ +#! /bin/sh +# Print a version string. +scriptversion=2021-01-10.00; # UTC + +# Bootstrap this package from checked-out sources. + +# Copyright (C) 2003-2021 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Originally written by Paul Eggert. The canonical version of this +# script is maintained as build-aux/bootstrap in gnulib, however, to +# be useful to your project, you should place a copy of it under +# version control in the top-level directory of your project. The +# intent is that all customization can be done with a bootstrap.conf +# file also maintained in your version control; gnulib comes with a +# template build-aux/bootstrap.conf to get you started. + +# Please report bugs or propose patches to bug-gnulib@gnu.org. + +nl=' +' + +# Ensure file names are sorted consistently across platforms. +LC_ALL=C +export LC_ALL + +# Ensure that CDPATH is not set. Otherwise, the output from cd +# would cause trouble in at least one use below. +(unset CDPATH) >/dev/null 2>&1 && unset CDPATH + +local_gl_dir=gl + +# Honor $PERL, but work even if there is none. +PERL="${PERL-perl}" + +me=$0 + +default_gnulib_url=https://git.savannah.gnu.org/git/gnulib.git + +usage() { + cat <&2 +} + +# warn_ WORD1... +warn_ () +{ + # If IFS does not start with ' ', set it and emit the warning in a subshell. + case $IFS in + ' '*) warnf_ '%s\n' "$*";; + *) (IFS=' '; warn_ "$@");; + esac +} + +# die WORD1... +die() { warn_ "$@"; exit 1; } + +# Configuration. + +# Name of the Makefile.am +gnulib_mk=gnulib.mk + +# List of gnulib modules needed. +gnulib_modules= + +# Any gnulib files needed that are not in modules. +gnulib_files= + +: ${AUTOPOINT=autopoint} +: ${AUTORECONF=autoreconf} + +# A function to be called for each unrecognized option. Returns 0 if +# the option in $1 has been processed by the function. Returns 1 if +# the option has not been processed by the function. Override it via +# your own definition in bootstrap.conf + +bootstrap_option_hook() { return 1; } + +# A function to be called in order to print the --help information +# corresponding to user-defined command-line options. + +bootstrap_print_option_usage_hook() { :; } + +# A function to be called right after gnulib-tool is run. +# Override it via your own definition in bootstrap.conf. +bootstrap_post_import_hook() { :; } + +# A function to be called after everything else in this script. +# Override it via your own definition in bootstrap.conf. +bootstrap_epilogue() { :; } + +# The command to download all .po files for a specified domain into a +# specified directory. Fill in the first %s with the destination +# directory and the second with the domain name. +po_download_command_format=\ +"wget --mirror --level=1 -nd -nv -A.po -P '%s' \ + https://translationproject.org/latest/%s/" + +# Prefer a non-empty tarname (4th argument of AC_INIT if given), else +# fall back to the package name (1st argument with munging). +extract_package_name=' + /^AC_INIT(\[*/{ + s/// + /^[^,]*,[^,]*,[^,]*,[ []*\([^][ ,)]\)/{ + s//\1/ + s/[],)].*// + p + q + } + s/[],)].*// + s/^GNU // + y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ + s/[^abcdefghijklmnopqrstuvwxyz0123456789_]/-/g + p + } +' +package=$(${AUTOCONF:-autoconf} --trace AC_INIT:\$4 configure.ac 2>/dev/null) +if test -z "$package"; then + package=$(sed -n "$extract_package_name" configure.ac) \ + || die 'cannot find package name in configure.ac' +fi +gnulib_name=lib$package + +build_aux=build-aux +source_base=lib +m4_base=m4 +doc_base=doc +tests_base=tests +gnulib_extra_files=" + build-aux/install-sh + build-aux/mdate-sh + build-aux/texinfo.tex + build-aux/depcomp + build-aux/config.guess + build-aux/config.sub + doc/INSTALL +" + +# Additional gnulib-tool options to use. Use "\newline" to break lines. +gnulib_tool_option_extras= + +# Other locale categories that need message catalogs. +EXTRA_LOCALE_CATEGORIES= + +# Additional xgettext options to use. Use "\\\newline" to break lines. +XGETTEXT_OPTIONS='\\\ + --flag=_:1:pass-c-format\\\ + --flag=N_:1:pass-c-format\\\ + --flag=error:3:c-format --flag=error_at_line:5:c-format\\\ +' + +# Package bug report address and copyright holder for gettext files +COPYRIGHT_HOLDER='Free Software Foundation, Inc.' +MSGID_BUGS_ADDRESS=bug-$package@gnu.org + +# Files we don't want to import. +excluded_files= + +# File that should exist in the top directory of a checked out hierarchy, +# but not in a distribution tarball. +checkout_only_file=README-hacking + +# Whether to use copies instead of symlinks. +copy=false + +# Set this to '.cvsignore .gitignore' in bootstrap.conf if you want +# those files to be generated in directories like lib/, m4/, and po/. +# Or set it to 'auto' to make this script select which to use based +# on which version control system (if any) is used in the source directory. +vc_ignore=auto + +# Set this to true in bootstrap.conf to enable --bootstrap-sync by +# default. +bootstrap_sync=false + +# Use git to update gnulib sources +use_git=true + +check_exists() { + if test "$1" = "--verbose"; then + ($2 --version /dev/null 2>&1 + if test $? -ge 126; then + # If not found, run with diagnostics as one may be + # presented with env variables to set to find the right version + ($2 --version /dev/null 2>&1 + fi + + test $? -lt 126 +} + +# find_tool ENVVAR NAMES... +# ------------------------- +# Search for a required program. Use the value of ENVVAR, if set, +# otherwise find the first of the NAMES that can be run. +# If found, set ENVVAR to the program name, die otherwise. +# +# FIXME: code duplication, see also gnu-web-doc-update. +find_tool () +{ + find_tool_envvar=$1 + shift + find_tool_names=$@ + eval "find_tool_res=\$$find_tool_envvar" + if test x"$find_tool_res" = x; then + for i; do + if check_exists $i; then + find_tool_res=$i + break + fi + done + fi + if test x"$find_tool_res" = x; then + warn_ "one of these is required: $find_tool_names;" + die "alternatively set $find_tool_envvar to a compatible tool" + fi + eval "$find_tool_envvar=\$find_tool_res" + eval "export $find_tool_envvar" +} + +# Override the default configuration, if necessary. +# Make sure that bootstrap.conf is sourced from the current directory +# if we were invoked as "sh bootstrap". +case "$0" in + */*) test -r "$0.conf" && . "$0.conf" ;; + *) test -r "$0.conf" && . ./"$0.conf" ;; +esac + +if test "$vc_ignore" = auto; then + vc_ignore= + test -d .git && vc_ignore=.gitignore + test -d CVS && vc_ignore="$vc_ignore .cvsignore" +fi + +if test x"$gnulib_modules$gnulib_files$gnulib_extra_files" = x; then + use_gnulib=false +else + use_gnulib=true +fi + +# Translate configuration into internal form. + +# Parse options. + +for option +do + case $option in + --help) + usage + exit;; + --gnulib-srcdir=*) + GNULIB_SRCDIR=${option#--gnulib-srcdir=};; + --skip-po) + SKIP_PO=t;; + --force) + checkout_only_file=;; + --copy) + copy=true;; + --bootstrap-sync) + bootstrap_sync=true;; + --no-bootstrap-sync) + bootstrap_sync=false;; + --no-git) + use_git=false;; + *) + bootstrap_option_hook $option || die "$option: unknown option";; + esac +done + +$use_git || test -d "$GNULIB_SRCDIR" \ + || die "Error: --no-git requires --gnulib-srcdir" + +if test -n "$checkout_only_file" && test ! -r "$checkout_only_file"; then + die "Bootstrapping from a non-checked-out distribution is risky." +fi + +# Strip blank and comment lines to leave significant entries. +gitignore_entries() { + sed '/^#/d; /^$/d' "$@" +} + +# If $STR is not already on a line by itself in $FILE, insert it at the start. +# Entries are inserted at the start of the ignore list to ensure existing +# entries starting with ! are not overridden. Such entries support +# whitelisting exceptions after a more generic blacklist pattern. +insert_if_absent() { + file=$1 + str=$2 + test -f $file || touch $file + test -r $file || die "Error: failed to read ignore file: $file" + duplicate_entries=$(gitignore_entries $file | sort | uniq -d) + if [ "$duplicate_entries" ] ; then + die "Error: Duplicate entries in $file: " $duplicate_entries + fi + linesold=$(gitignore_entries $file | wc -l) + linesnew=$( { echo "$str"; cat $file; } | gitignore_entries | sort -u | wc -l) + if [ $linesold != $linesnew ] ; then + { echo "$str" | cat - $file > $file.bak && mv $file.bak $file; } \ + || die "insert_if_absent $file $str: failed" + fi +} + +# Adjust $PATTERN for $VC_IGNORE_FILE and insert it with +# insert_if_absent. +insert_vc_ignore() { + vc_ignore_file="$1" + pattern="$2" + case $vc_ignore_file in + *.gitignore) + # A .gitignore entry that does not start with '/' applies + # recursively to subdirectories, so prepend '/' to every + # .gitignore entry. + pattern=$(echo "$pattern" | sed s,^,/,);; + esac + insert_if_absent "$vc_ignore_file" "$pattern" +} + +# Die if there is no AC_CONFIG_AUX_DIR($build_aux) line in configure.ac. +found_aux_dir=no +grep '^[ ]*AC_CONFIG_AUX_DIR(\['"$build_aux"'\])' configure.ac \ + >/dev/null && found_aux_dir=yes +grep '^[ ]*AC_CONFIG_AUX_DIR('"$build_aux"')' configure.ac \ + >/dev/null && found_aux_dir=yes +test $found_aux_dir = yes \ + || die "configure.ac lacks 'AC_CONFIG_AUX_DIR([$build_aux])'; add it" + +# If $build_aux doesn't exist, create it now, otherwise some bits +# below will malfunction. If creating it, also mark it as ignored. +if test ! -d $build_aux; then + mkdir $build_aux + for dot_ig in x $vc_ignore; do + test $dot_ig = x && continue + insert_vc_ignore $dot_ig $build_aux + done +fi + +# Note this deviates from the version comparison in automake +# in that it treats 1.5 < 1.5.0, and treats 1.4.4a < 1.4-p3a +# but this should suffice as we won't be specifying old +# version formats or redundant trailing .0 in bootstrap.conf. +# If we did want full compatibility then we should probably +# use m4_version_compare from autoconf. +sort_ver() { # sort -V is not generally available + ver1="$1" + ver2="$2" + + # split on '.' and compare each component + i=1 + while : ; do + p1=$(echo "$ver1" | cut -d. -f$i) + p2=$(echo "$ver2" | cut -d. -f$i) + if [ ! "$p1" ]; then + echo "$1 $2" + break + elif [ ! "$p2" ]; then + echo "$2 $1" + break + elif [ ! "$p1" = "$p2" ]; then + if [ "$p1" -gt "$p2" ] 2>/dev/null; then # numeric comparison + echo "$2 $1" + elif [ "$p2" -gt "$p1" ] 2>/dev/null; then # numeric comparison + echo "$1 $2" + else # numeric, then lexicographic comparison + lp=$(printf "$p1\n$p2\n" | LANG=C sort -n | tail -n1) + if [ "$lp" = "$p2" ]; then + echo "$1 $2" + else + echo "$2 $1" + fi + fi + break + fi + i=$(($i+1)) + done +} + +get_version_sed=' +# Move version to start of line. +s/.*[v ]\([0-9]\)/\1/ + +# Skip lines that do not start with version. +/^[0-9]/!d + +# Remove characters after the version. +s/[^.a-z0-9-].*// + +# The first component must be digits only. +s/^\([0-9]*\)[a-z-].*/\1/ + +#the following essentially does s/5.005/5.5/ +s/\.0*\([1-9]\)/.\1/g +p +q' + +get_version() { + app=$1 + + $app --version >/dev/null 2>&1 || { $app --version; return 1; } + + $app --version 2>&1 | sed -n "$get_version_sed" +} + +check_versions() { + ret=0 + + while read app req_ver; do + # We only need libtoolize from the libtool package. + if test "$app" = libtool; then + app=libtoolize + fi + # Exempt git if --no-git is in effect. + if test "$app" = git; then + $use_git || continue + fi + # Honor $APP variables ($TAR, $AUTOCONF, etc.) + appvar=$(echo $app | LC_ALL=C tr '[a-z]-' '[A-Z]_') + test "$appvar" = TAR && appvar=AMTAR + case $appvar in + GZIP) ;; # Do not use $GZIP: it contains gzip options. + PERL::*) ;; # Keep perl modules as-is + *) eval "app=\${$appvar-$app}" ;; + esac + + # Handle the still-experimental Automake-NG programs specially. + # They remain named as the mainstream Automake programs ("automake", + # and "aclocal") to avoid gratuitous incompatibilities with + # pre-existing usages (by, say, autoreconf, or custom autogen.sh + # scripts), but correctly identify themselves (as being part of + # "GNU automake-ng") when asked their version. + case $app in + automake-ng|aclocal-ng) + app=${app%-ng} + ($app --version | grep '(GNU automake-ng)') >/dev/null 2>&1 || { + warn_ "Error: '$app' not found or not from Automake-NG" + ret=1 + continue + } ;; + # Another check is for perl modules. These can be written as + # e.g. perl::XML::XPath in case of XML::XPath module, etc. + perl::*) + # Extract module name + app="${app#perl::}" + if ! $PERL -m"$app" -e 'exit 0' >/dev/null 2>&1; then + warn_ "Error: perl module '$app' not found" + ret=1 + fi + continue + ;; + esac + if [ "$req_ver" = "-" ]; then + # Merely require app to exist; not all prereq apps are well-behaved + # so we have to rely on $? rather than get_version. + if ! check_exists --verbose $app; then + warn_ "Error: '$app' not found" + ret=1 + fi + else + # Require app to produce a new enough version string. + inst_ver=$(get_version $app) + if [ ! "$inst_ver" ]; then + warn_ "Error: '$app' not found" + ret=1 + else + latest_ver=$(sort_ver $req_ver $inst_ver | cut -d' ' -f2) + if [ ! "$latest_ver" = "$inst_ver" ]; then + warnf_ '%s\n' \ + "Error: '$app' version == $inst_ver is too old" \ + " '$app' version >= $req_ver is required" + ret=1 + fi + fi + fi + done + + return $ret +} + +print_versions() { + echo "Program Min_version" + echo "----------------------" + printf %s "$buildreq" + echo "----------------------" + # can't depend on column -t +} + +# Find sha1sum, named gsha1sum on MacPorts, shasum on Mac OS X 10.6. +# Also find the compatible sha1 utility on the BSDs +if test x"$SKIP_PO" = x; then + find_tool SHA1SUM sha1sum gsha1sum shasum sha1 +fi + +use_libtool=0 +# We'd like to use grep -E, to see if any of LT_INIT, +# AC_PROG_LIBTOOL, AM_PROG_LIBTOOL is used in configure.ac, +# but that's not portable enough (e.g., for Solaris). +grep '^[ ]*A[CM]_PROG_LIBTOOL' configure.ac >/dev/null \ + && use_libtool=1 +grep '^[ ]*LT_INIT' configure.ac >/dev/null \ + && use_libtool=1 +if test $use_libtool = 1; then + find_tool LIBTOOLIZE glibtoolize libtoolize +fi + +# gnulib-tool requires at least automake and autoconf. +# If either is not listed, add it (with minimum version) as a prerequisite. +case $buildreq in + *automake*) ;; + *) buildreq="automake 1.9 +$buildreq" ;; +esac +case $buildreq in + *autoconf*) ;; + *) buildreq="autoconf 2.59 +$buildreq" ;; +esac + +# When we can deduce that gnulib-tool will require patch, +# and when patch is not already listed as a prerequisite, add it, too. +if test -d "$local_gl_dir" \ + && ! find "$local_gl_dir" -name '*.diff' -exec false {} +; then + case $buildreq in + *patch*) ;; + *) buildreq="patch - +$buildreq" ;; + esac +fi + +if ! printf "$buildreq" | check_versions; then + echo >&2 + if test -f README-prereq; then + die "See README-prereq for how to get the prerequisite programs" + else + die "Please install the prerequisite programs" + fi +fi + +# Warn the user if autom4te appears to be broken; this causes known +# issues with at least gettext 0.18.3. +probe=$(echo 'm4_quote([hi])' | autom4te -l M4sugar -t 'm4_quote:$%' -) +if test "x$probe" != xhi; then + warn_ "WARNING: your autom4te wrapper eats stdin;" + warn_ "if bootstrap fails, consider upgrading your autotools" +fi + +echo "$0: Bootstrapping from checked-out $package sources..." + +# See if we can use gnulib's git-merge-changelog merge driver. +if $use_git && test -d .git && check_exists git; then + if git config merge.merge-changelog.driver >/dev/null ; then + : + elif check_exists git-merge-changelog; then + echo "$0: initializing git-merge-changelog driver" + git config merge.merge-changelog.name 'GNU-style ChangeLog merge driver' + git config merge.merge-changelog.driver 'git-merge-changelog %O %A %B' + else + echo "$0: consider installing git-merge-changelog from gnulib" + fi +fi + + +cleanup_gnulib() { + status=$? + rm -fr "$gnulib_path" + exit $status +} + +git_modules_config () { + test -f .gitmodules && git config --file .gitmodules "$@" +} + +if $use_gnulib; then + if $use_git; then + gnulib_path=$(git_modules_config submodule.gnulib.path) + test -z "$gnulib_path" && gnulib_path=gnulib + fi + + # Get gnulib files. Populate $GNULIB_SRCDIR, possibly updating a + # submodule, for use in the rest of the script. + + case ${GNULIB_SRCDIR--} in + -) + # Note that $use_git is necessarily true in this case. + if git_modules_config submodule.gnulib.url >/dev/null; then + echo "$0: getting gnulib files..." + git submodule init -- "$gnulib_path" || exit $? + git submodule update -- "$gnulib_path" || exit $? + + elif [ ! -d "$gnulib_path" ]; then + echo "$0: getting gnulib files..." + + trap cleanup_gnulib 1 2 13 15 + + shallow= + if test -z "$GNULIB_REVISION"; then + git clone -h 2>&1 | grep -- --depth > /dev/null && shallow='--depth 2' + fi + git clone $shallow ${GNULIB_URL:-$default_gnulib_url} "$gnulib_path" \ + || cleanup_gnulib + + trap - 1 2 13 15 + fi + GNULIB_SRCDIR=$gnulib_path + ;; + *) + # Use GNULIB_SRCDIR directly or as a reference. + if $use_git && test -d "$GNULIB_SRCDIR"/.git && \ + git_modules_config submodule.gnulib.url >/dev/null; then + echo "$0: getting gnulib files..." + if git submodule -h|grep -- --reference > /dev/null; then + # Prefer the one-liner available in git 1.6.4 or newer. + git submodule update --init --reference "$GNULIB_SRCDIR" \ + "$gnulib_path" || exit $? + else + # This fallback allows at least git 1.5.5. + if test -f "$gnulib_path"/gnulib-tool; then + # Since file already exists, assume submodule init already complete. + git submodule update -- "$gnulib_path" || exit $? + else + # Older git can't clone into an empty directory. + rmdir "$gnulib_path" 2>/dev/null + git clone --reference "$GNULIB_SRCDIR" \ + "$(git_modules_config submodule.gnulib.url)" "$gnulib_path" \ + && git submodule init -- "$gnulib_path" \ + && git submodule update -- "$gnulib_path" \ + || exit $? + fi + fi + GNULIB_SRCDIR=$gnulib_path + fi + ;; + esac + + if test -d "$GNULIB_SRCDIR"/.git && test -n "$GNULIB_REVISION" \ + && ! git_modules_config submodule.gnulib.url >/dev/null; then + (cd "$GNULIB_SRCDIR" && git checkout "$GNULIB_REVISION") || cleanup_gnulib + fi + + # $GNULIB_SRCDIR now points to the version of gnulib to use, and + # we no longer need to use git or $gnulib_path below here. + + if $bootstrap_sync; then + cmp -s "$0" "$GNULIB_SRCDIR/build-aux/bootstrap" || { + echo "$0: updating bootstrap and restarting..." + case $(sh -c 'echo "$1"' -- a) in + a) ignored=--;; + *) ignored=ignored;; + esac + exec sh -c \ + 'cp "$1" "$2" && shift && exec "${CONFIG_SHELL-/bin/sh}" "$@"' \ + $ignored "$GNULIB_SRCDIR/build-aux/bootstrap" \ + "$0" "$@" --no-bootstrap-sync + } + fi + + gnulib_tool=$GNULIB_SRCDIR/gnulib-tool + <$gnulib_tool || exit $? +fi + +# Get translations. + +download_po_files() { + subdir=$1 + domain=$2 + echo "$me: getting translations into $subdir for $domain..." + cmd=$(printf "$po_download_command_format" "$subdir" "$domain") + eval "$cmd" +} + +# Mirror .po files to $po_dir/.reference and copy only the new +# or modified ones into $po_dir. Also update $po_dir/LINGUAS. +# Note po files that exist locally only are left in $po_dir but will +# not be included in LINGUAS and hence will not be distributed. +update_po_files() { + # Directory containing primary .po files. + # Overwrite them only when we're sure a .po file is new. + po_dir=$1 + domain=$2 + + # Mirror *.po files into this dir. + # Usually contains *.s1 checksum files. + ref_po_dir="$po_dir/.reference" + + test -d $ref_po_dir || mkdir $ref_po_dir || return + download_po_files $ref_po_dir $domain \ + && ls "$ref_po_dir"/*.po 2>/dev/null | + sed 's|.*/||; s|\.po$||' > "$po_dir/LINGUAS" || return + + langs=$(cd $ref_po_dir && echo *.po | sed 's/\.po//g') + test "$langs" = '*' && langs=x + for po in $langs; do + case $po in x) continue;; esac + new_po="$ref_po_dir/$po.po" + cksum_file="$ref_po_dir/$po.s1" + if ! test -f "$cksum_file" || + ! test -f "$po_dir/$po.po" || + ! $SHA1SUM -c "$cksum_file" < "$new_po" > /dev/null 2>&1; then + echo "$me: updated $po_dir/$po.po..." + cp "$new_po" "$po_dir/$po.po" \ + && $SHA1SUM < "$new_po" > "$cksum_file" || return + fi + done +} + +case $SKIP_PO in +'') + if test -d po; then + update_po_files po $package || exit + fi + + if test -d runtime-po; then + update_po_files runtime-po $package-runtime || exit + fi;; +esac + +symlink_to_dir() +{ + src=$1/$2 + dst=${3-$2} + + test -f "$src" && { + + # If the destination directory doesn't exist, create it. + # This is required at least for "lib/uniwidth/cjk.h". + dst_dir=$(dirname "$dst") + if ! test -d "$dst_dir"; then + mkdir -p "$dst_dir" + + # If we've just created a directory like lib/uniwidth, + # tell version control system(s) it's ignorable. + # FIXME: for now, this does only one level + parent=$(dirname "$dst_dir") + for dot_ig in x $vc_ignore; do + test $dot_ig = x && continue + ig=$parent/$dot_ig + insert_vc_ignore $ig "${dst_dir##*/}" + done + fi + + if $copy; then + { + test ! -h "$dst" || { + echo "$me: rm -f $dst" && + rm -f "$dst" + } + } && + test -f "$dst" && + cmp -s "$src" "$dst" || { + echo "$me: cp -fp $src $dst" && + cp -fp "$src" "$dst" + } + else + # Leave any existing symlink alone, if it already points to the source, + # so that broken build tools that care about symlink times + # aren't confused into doing unnecessary builds. Conversely, if the + # existing symlink's timestamp is older than the source, make it afresh, + # so that broken tools aren't confused into skipping needed builds. See + # . + test -h "$dst" && + src_ls=$(ls -diL "$src" 2>/dev/null) && set $src_ls && src_i=$1 && + dst_ls=$(ls -diL "$dst" 2>/dev/null) && set $dst_ls && dst_i=$1 && + test "$src_i" = "$dst_i" && + both_ls=$(ls -dt "$src" "$dst") && + test "X$both_ls" = "X$dst$nl$src" || { + dot_dots= + case $src in + /*) ;; + *) + case /$dst/ in + *//* | */../* | */./* | /*/*/*/*/*/) + die "invalid symlink calculation: $src -> $dst";; + /*/*/*/*/) dot_dots=../../../;; + /*/*/*/) dot_dots=../../;; + /*/*/) dot_dots=../;; + esac;; + esac + + echo "$me: ln -fs $dot_dots$src $dst" && + ln -fs "$dot_dots$src" "$dst" + } + fi + } +} + +version_controlled_file() { + parent=$1 + file=$2 + if test -d .git; then + git rm -n "$file" > /dev/null 2>&1 + elif test -d .svn; then + svn log -r HEAD "$file" > /dev/null 2>&1 + elif test -d CVS; then + grep -F "/${file##*/}/" "$parent/CVS/Entries" 2>/dev/null | + grep '^/[^/]*/[0-9]' > /dev/null + else + warn_ "no version control for $file?" + false + fi +} + +# NOTE: we have to be careful to run both autopoint and libtoolize +# before gnulib-tool, since gnulib-tool is likely to provide newer +# versions of files "installed" by these two programs. +# Then, *after* gnulib-tool (see below), we have to be careful to +# run autoreconf in such a way that it does not run either of these +# two just-pre-run programs. + +# Import from gettext. +with_gettext=yes +grep '^[ ]*AM_GNU_GETTEXT_VERSION(' configure.ac >/dev/null || \ + with_gettext=no + +if test $with_gettext = yes || test $use_libtool = 1; then + + tempbase=.bootstrap$$ + trap "rm -f $tempbase.0 $tempbase.1" 1 2 13 15 + + > $tempbase.0 > $tempbase.1 && + find . ! -type d -print | sort > $tempbase.0 || exit + + if test $with_gettext = yes; then + # Released autopoint has the tendency to install macros that have been + # obsoleted in current gnulib, so run this before gnulib-tool. + echo "$0: $AUTOPOINT --force" + $AUTOPOINT --force || exit + fi + + # Autoreconf runs aclocal before libtoolize, which causes spurious + # warnings if the initial aclocal is confused by the libtoolized + # (or worse out-of-date) macro directory. + # libtoolize 1.9b added the --install option; but we support back + # to libtoolize 1.5.22, where the install action was default. + if test $use_libtool = 1; then + install= + case $($LIBTOOLIZE --help) in + *--install*) install=--install ;; + esac + echo "running: $LIBTOOLIZE $install --copy" + $LIBTOOLIZE $install --copy + fi + + find . ! -type d -print | sort >$tempbase.1 + old_IFS=$IFS + IFS=$nl + for file in $(comm -13 $tempbase.0 $tempbase.1); do + IFS=$old_IFS + parent=${file%/*} + version_controlled_file "$parent" "$file" || { + for dot_ig in x $vc_ignore; do + test $dot_ig = x && continue + ig=$parent/$dot_ig + insert_vc_ignore "$ig" "${file##*/}" + done + } + done + IFS=$old_IFS + + rm -f $tempbase.0 $tempbase.1 + trap - 1 2 13 15 +fi + +# Import from gnulib. + +if $use_gnulib; then + gnulib_tool_options="\ + --no-changelog\ + --aux-dir=$build_aux\ + --doc-base=$doc_base\ + --lib=$gnulib_name\ + --m4-base=$m4_base/\ + --source-base=$source_base/\ + --tests-base=$tests_base\ + --local-dir=$local_gl_dir\ + $gnulib_tool_option_extras\ + " + if test $use_libtool = 1; then + case "$gnulib_tool_options " in + *' --libtool '*) ;; + *) gnulib_tool_options="$gnulib_tool_options --libtool" ;; + esac + fi + echo "$0: $gnulib_tool $gnulib_tool_options --import ..." + $gnulib_tool $gnulib_tool_options --import $gnulib_modules \ + || die "gnulib-tool failed" + + for file in $gnulib_files; do + symlink_to_dir "$GNULIB_SRCDIR" $file \ + || die "failed to symlink $file" + done +fi + +bootstrap_post_import_hook \ + || die "bootstrap_post_import_hook failed" + +# Don't proceed if there are uninitialized submodules. In particular, +# the next step will remove dangling links, which might be links into +# uninitialized submodules. +# +# Uninitialized submodules are listed with an initial dash. +if $use_git && git submodule | grep '^-' >/dev/null; then + die "some git submodules are not initialized. " \ + "Run 'git submodule update --init' and bootstrap again." +fi + +# Remove any dangling symlink matching "*.m4" or "*.[ch]" in some +# gnulib-populated directories. Such .m4 files would cause aclocal to fail. +# The following requires GNU find 4.2.3 or newer. Considering the usual +# portability constraints of this script, that may seem a very demanding +# requirement, but it should be ok. Ignore any failure, which is fine, +# since this is only a convenience to help developers avoid the relatively +# unusual case in which a symlinked-to .m4 file is git-removed from gnulib +# between successive runs of this script. +find "$m4_base" "$source_base" \ + -depth \( -name '*.m4' -o -name '*.[ch]' \) \ + -type l -xtype l -delete > /dev/null 2>&1 + +# Invoke autoreconf with --force --install to ensure upgrades of tools +# such as ylwrap. +AUTORECONFFLAGS="--verbose --install --force -I $m4_base $ACLOCAL_FLAGS" + +# Some systems (RHEL 5) are using ancient autotools, for which the +# --no-recursive option had not been invented. Detect that lack and +# omit the option when it's not supported. FIXME in 2017: remove this +# hack when RHEL 5 autotools are updated, or when they become irrelevant. +case $($AUTORECONF --help) in + *--no-recursive*) AUTORECONFFLAGS="$AUTORECONFFLAGS --no-recursive";; +esac + +# Tell autoreconf not to invoke autopoint or libtoolize; they were run above. +echo "running: AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS" +AUTOPOINT=true LIBTOOLIZE=true $AUTORECONF $AUTORECONFFLAGS \ + || die "autoreconf failed" + +# Get some extra files from gnulib, overriding existing files. +for file in $gnulib_extra_files; do + case $file in + */INSTALL) dst=INSTALL;; + build-aux/*) dst=$build_aux/${file#build-aux/};; + *) dst=$file;; + esac + symlink_to_dir "$GNULIB_SRCDIR" $file $dst \ + || die "failed to symlink $file" +done + +if test $with_gettext = yes; then + # Create gettext configuration. + echo "$0: Creating po/Makevars from po/Makevars.template ..." + rm -f po/Makevars + sed ' + /^EXTRA_LOCALE_CATEGORIES *=/s/=.*/= '"$EXTRA_LOCALE_CATEGORIES"'/ + /^COPYRIGHT_HOLDER *=/s/=.*/= '"$COPYRIGHT_HOLDER"'/ + /^MSGID_BUGS_ADDRESS *=/s|=.*|= '"$MSGID_BUGS_ADDRESS"'| + /^XGETTEXT_OPTIONS *=/{ + s/$/ \\/ + a\ + '"$XGETTEXT_OPTIONS"' $${end_of_xgettext_options+} + } + ' po/Makevars.template >po/Makevars \ + || die 'cannot generate po/Makevars' + + # If the 'gettext' module is in use, grab the latest Makefile.in.in. + # If only the 'gettext-h' module is in use, assume autopoint already + # put the correct version of this file into place. + case $gnulib_modules in + *gettext-h*) ;; + *gettext*) + cp $GNULIB_SRCDIR/build-aux/po/Makefile.in.in po/Makefile.in.in \ + || die "cannot create po/Makefile.in.in" + ;; + esac + + if test -d runtime-po; then + # Similarly for runtime-po/Makevars, but not quite the same. + rm -f runtime-po/Makevars + sed ' + /^DOMAIN *=.*/s/=.*/= '"$package"'-runtime/ + /^subdir *=.*/s/=.*/= runtime-po/ + /^MSGID_BUGS_ADDRESS *=/s/=.*/= bug-'"$package"'@gnu.org/ + /^XGETTEXT_OPTIONS *=/{ + s/$/ \\/ + a\ + '"$XGETTEXT_OPTIONS_RUNTIME"' $${end_of_xgettext_options+} + } + ' po/Makevars.template >runtime-po/Makevars \ + || die 'cannot generate runtime-po/Makevars' + + # Copy identical files from po to runtime-po. + (cd po && cp -p Makefile.in.in *-quot *.header *.sed *.sin ../runtime-po) + fi +fi + +bootstrap_epilogue + +echo "$0: done. Now you can run './configure'." + +# Local variables: +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC0" +# time-stamp-end: "; # UTC" +# End: diff --git a/parallel-rsp/lightning/bootstrap.conf b/parallel-rsp/lightning/bootstrap.conf new file mode 100644 index 0000000..423491b --- /dev/null +++ b/parallel-rsp/lightning/bootstrap.conf @@ -0,0 +1,76 @@ +# Bootstrap configuration. + +# Copyright (C) 2006-2020 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +# gnulib modules used by this package. +gnulib_modules=" +" + +# gnulib library name. +gnulib_name=libgnu + +# directories. +source_base=gnulib-lib +doc_base=gnulib-doc + +# Additional xgettext options to use. Use "\\\newline" to break lines. +XGETTEXT_OPTIONS=$XGETTEXT_OPTIONS'\\\ + --from-code=UTF-8\\\ + --flag=asprintf:2:c-format --flag=vasprintf:2:c-format\\\ + --flag=asnprintf:3:c-format --flag=vasnprintf:3:c-format\\\ + --flag=wrapf:1:c-format\\\ +' + +# If "AM_GNU_GETTEXT(external" or "AM_GNU_GETTEXT([external]" +# appears in configure.ac, exclude some unnecessary files. +# Without grep's -E option (not portable enough, pre-configure), +# the following test is ugly. Also, this depends on the existence +# of configure.ac, not the obsolescent-named configure.in. But if +# you're using this infrastructure, you should care about such things. + +gettext_external=0 +grep '^[ ]*AM_GNU_GETTEXT(external\>' configure.ac > /dev/null && + gettext_external=1 +grep '^[ ]*AM_GNU_GETTEXT(\[external\]' configure.ac > /dev/null && + gettext_external=1 + +if test $gettext_external = 1; then + # Gettext supplies these files, but we don't need them since + # we don't have an intl subdirectory. + excluded_files=' + m4/glibc2.m4 + m4/intdiv0.m4 + m4/lcmessage.m4 + m4/lock.m4 + m4/printf-posix.m4 + m4/size_max.m4 + m4/uintmax_t.m4 + m4/ulonglong.m4 + m4/visibility.m4 + m4/xsize.m4 + ' +fi + +# Build prerequisites +buildreq="\ +autoconf 2.59 +automake 1.9.6 +git 1.5.5 +tar - +" + +bootstrap_sync=true diff --git a/parallel-rsp/lightning/check/.gitignore b/parallel-rsp/lightning/check/.gitignore new file mode 100644 index 0000000..3ca81ec --- /dev/null +++ b/parallel-rsp/lightning/check/.gitignore @@ -0,0 +1,67 @@ +*.nodata +nodata +*.log +*.trs + +3to2 +bswap +add +align +allocai +allocar +alu_add +alu_and +alu_com +alu_div +alu_lsh +alu_mul +alu_neg +alu_or +alu_rem +alu_rsb +alu_rsh +alu_sub +alu_xor +alux_add +alux_sub +bp +branch +call +carg +carry +catomic +ccall +clobber +ctramp +cva_list +cvt +divi +fib +float +fop_abs +fop_sqrt +hton +jmpr +ldsti +ldstr +ldstr-c +ldstxi +ldstxi-c +ldstxr +ldstxr-c +lightning +live +movzr +put +qalu_div +qalu_mul +range +ranger +ret +rpn +self +setcode +stack +tramp +va_list +varargs diff --git a/parallel-rsp/lightning/check/Makefile.am b/parallel-rsp/lightning/check/Makefile.am index e04f7ac..c77f5cd 100644 --- a/parallel-rsp/lightning/check/Makefile.am +++ b/parallel-rsp/lightning/check/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright 2012-2019 Free Software Foundation, Inc. +# Copyright 2012-2023 Free Software Foundation, Inc. # # This file is part of GNU lightning. # @@ -14,9 +14,11 @@ # License for more details. # -AM_CFLAGS = -I$(top_srcdir)/include -D_GNU_SOURCE +AM_CFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -D_GNU_SOURCE $(LIGHTNING_CFLAGS) -check_PROGRAMS = lightning ccall self setcode nodata ctramp carg cva_list +check_PROGRAMS = lightning ccall self setcode nodata ctramp carg cva_list \ + catomic protect riprel lightning_LDADD = $(top_builddir)/lib/liblightning.la -lm $(SHLIB) lightning_SOURCES = lightning.c @@ -42,6 +44,15 @@ carg_SOURCES = carg.c cva_list_LDADD = $(top_builddir)/lib/liblightning.la -lm $(SHLIB) cva_list_SOURCES = cva_list.c +catomic_LDADD = $(top_builddir)/lib/liblightning.la -lm -lpthread $(SHLIB) +catomic_SOURCES = catomic.c + +protect_LDADD = $(top_builddir)/lib/liblightning.la -lm $(SHLIB) +protect_SOURCES = protect.c + +riprel_LDADD = $(top_builddir)/lib/liblightning.la -lm $(SHLIB) +riprel_SOURCES = riprel.c + $(top_builddir)/lib/liblightning.la: cd $(top_builddir)/lib; $(MAKE) $(AM_MAKEFLAGS) liblightning.la @@ -65,6 +76,7 @@ EXTRA_DIST = \ ldstxi-c.tst ldstxi-c.ok \ cvt.tst cvt.ok \ hton.tst hton.ok \ + bswap.tst bswap.ok \ branch.tst branch.ok \ alu.inc \ alu_add.tst alu_add.ok \ @@ -82,6 +94,7 @@ EXTRA_DIST = \ alu_rsh.tst alu_rsh.ok \ alu_com.tst alu_com.ok \ alu_neg.tst alu_neg.ok \ + movzr.tst movzr.ok \ fop_abs.tst fop_abs.ok \ fop_sqrt.tst fop_sqrt.ok \ varargs.tst varargs.ok \ @@ -91,6 +104,7 @@ EXTRA_DIST = \ call.tst call.ok \ float.tst float.ok \ jmpr.tst jmpr.ok \ + live.tst live.ok \ put.tst put.ok \ qalu.inc \ qalu_mul.tst qalu_mul.ok \ @@ -98,8 +112,10 @@ EXTRA_DIST = \ range.tst range.ok \ ranger.tst ranger.ok \ ret.tst ret.ok \ + skip.tst skip.ok \ tramp.tst tramp.ok \ va_list.tst va_list.ok \ + bit.tst bit.ok \ check.sh \ check.x87.sh \ check.arm.sh check.swf.sh \ @@ -107,7 +123,8 @@ EXTRA_DIST = \ check.arm4.swf.sh \ check.nodata.sh \ check.x87.nodata.sh \ - run-test all.tst + run-test all.tst \ + collatz.tst factorial.tst base_TESTS = \ 3to2 add align allocai \ @@ -115,20 +132,21 @@ base_TESTS = \ ldstr ldsti \ ldstxr ldstxi \ ldstr-c ldstxr-c ldstxi-c \ - cvt hton branch \ + cvt hton bswap branch \ alu_add alux_add \ alu_sub alux_sub alu_rsb \ alu_mul alu_div alu_rem \ alu_and alu_or alu_xor \ alu_lsh alu_rsh \ alu_com alu_neg \ + movzr \ fop_abs fop_sqrt \ varargs stack \ clobber carry call \ - float jmpr put \ + float jmpr live put \ qalu_mul qalu_div \ - range ranger ret tramp \ - va_list + range ranger ret skip tramp \ + va_list bit $(base_TESTS): check.sh $(LN_S) $(srcdir)/check.sh $@ @@ -150,6 +168,7 @@ x87_TESTS = \ alu_and.x87 alu_or.x87 alu_xor.x87 \ alu_lsh.x87 alu_rsh.x87 \ alu_com.x87 alu_neg.x87 \ + movzr.x87 \ fop_abs.x87 fop_sqrt.x87 \ varargs.x87 stack.x87 \ clobber.x87 carry.x87 call.x87 \ @@ -173,6 +192,7 @@ x87_nodata_TESTS = \ alu_and.x87.nodata alu_or.x87.nodata alu_xor.x87.nodata \ alu_lsh.x87.nodata alu_rsh.x87.nodata \ alu_com.x87.nodata alu_neg.x87.nodata \ + movzr.x87.nodata \ fop_abs.x87.nodata fop_sqrt.x87.nodata \ varargs.x87.nodata stack.x87.nodata \ clobber.x87.nodata carry.x87.nodata call.x87.nodata \ @@ -191,13 +211,14 @@ arm_TESTS = \ rpn.arm ldstr.arm ldsti.arm \ ldstxr.arm ldstxi.arm \ ldstr-c.arm ldstxr-c.arm ldstxi-c.arm \ - cvt.arm hton.arm branch.arm \ + cvt.arm hton.arm bswap.arm branch.arm \ alu_add.arm alux_add.arm \ alu_sub.arm alux_sub.arm alu_rsb.arm \ alu_mul.arm alu_div.arm alu_rem.arm \ alu_and.arm alu_or.arm alu_xor.arm \ alu_lsh.arm alu_rsh.arm \ alu_com.arm alu_neg.arm \ + movzr.arm \ fop_abs.arm fop_sqrt.arm \ varargs.arm stack.arm \ clobber.arm carry.arm call.arm \ @@ -216,13 +237,14 @@ swf_TESTS = \ rpn.swf ldstr.swf ldsti.swf \ ldstxr.swf ldstxi.swf \ ldstr-c.swf ldstxr-c.swf ldstxi-c.swf \ - cvt.swf hton.swf branch.swf \ + cvt.swf hton.swf bswap.swf branch.swf \ alu_add.swf alux_add.swf \ alu_sub.swf alux_sub.swf alu_rsb.swf \ alu_mul.swf alu_div.swf alu_rem.swf \ alu_and.swf alu_or.swf alu_xor.swf \ alu_lsh.swf alu_rsh.swf \ alu_com.swf alu_neg.swf \ + movzr.swf \ fop_abs.swf fop_sqrt.swf \ varargs.swf stack.swf \ clobber.swf carry.swf call.swf \ @@ -239,13 +261,14 @@ arm_swf_TESTS = \ rpn.arm.swf ldstr.arm.swf ldsti.arm.swf \ ldstxr.arm.swf ldstxi.arm.swf \ ldstr-c.arm.swf ldstxr-c.arm.swf ldstxi-c.arm.swf \ - cvt.arm.swf hton.arm.swf branch.arm.swf \ + cvt.arm.swf hton.arm.swf bswap.arm.swf branch.arm.swf \ alu_add.arm.swf alux_add.arm.swf \ alu_sub.arm.swf alux_sub.arm.swf alu_rsb.arm.swf \ alu_mul.arm.swf alu_div.arm.swf alu_rem.arm.swf \ alu_and.arm.swf alu_or.arm.swf alu_xor.arm.swf \ alu_lsh.arm.swf alu_rsh.arm.swf \ alu_com.arm.swf alu_neg.arm.swf \ + movzr.arm.swf \ fop_abs.arm.swf fop_sqrt.arm.swf \ varargs.arm.swf stack.arm.swf \ clobber.arm.swf carry.arm.swf call.arm.swf \ @@ -263,13 +286,14 @@ arm4_swf_TESTS = \ rpn.arm4.swf ldstr.arm4.swf ldsti.arm4.swf \ ldstxr.arm4.swf ldstxi.arm4.swf \ ldstr-c.arm4.swf ldstxr-c.arm4.swf ldstxi-c.arm4.swf \ - cvt.arm4.swf hton.arm4.swf branch.arm4.swf \ - alu_add.arm4.swf alux_add.arm4.swf \ + cvt.arm4.swf hton.arm4.swf bswap.arm4.swf \ + branch.arm4.swf alu_add.arm4.swf alux_add.arm4.swf \ alu_sub.arm4.swf alux_sub.arm4.swf alu_rsb.arm4.swf \ alu_mul.arm4.swf alu_div.arm4.swf alu_rem.arm4.swf \ alu_and.arm4.swf alu_or.arm4.swf alu_xor.arm4.swf \ alu_lsh.arm4.swf alu_rsh.arm4.swf \ alu_com.arm4.swf alu_neg.arm4.swf \ + movzr.arm4.swf \ fop_abs.arm4.swf fop_sqrt.arm4.swf \ varargs.arm4.swf stack.arm4.swf \ clobber.arm4.swf carry.arm4.swf call.arm4.swf \ @@ -297,22 +321,23 @@ nodata_TESTS = \ alu_and.nodata alu_or.nodata alu_xor.nodata \ alu_lsh.nodata alu_rsh.nodata \ alu_com.nodata alu_neg.nodata \ + movzr.nodata \ fop_abs.nodata fop_sqrt.nodata \ varargs.nodata stack.nodata \ clobber.nodata carry.nodata call.nodata \ float.nodata jmpr.nodata tramp.nodata \ range.nodata ranger.nodata put.nodata \ - va_list.nodata + va_list.nodata bit.nodata $(nodata_TESTS): check.nodata.sh $(LN_S) $(srcdir)/check.nodata.sh $@ TESTS += $(nodata_TESTS) endif -TESTS += ccall self setcode nodata ctramp carg cva_list +TESTS += ccall self setcode nodata ctramp carg cva_list catomic \ + protect riprel CLEANFILES = $(TESTS) #TESTS_ENVIRONMENT=$(srcdir)/run-test; debug: lightning $(LIBTOOL) --mode=execute gdb lightning - diff --git a/parallel-rsp/lightning/check/align.tst b/parallel-rsp/lightning/check/align.tst index 5d5348c..7bc1010 100644 --- a/parallel-rsp/lightning/check/align.tst +++ b/parallel-rsp/lightning/check/align.tst @@ -17,7 +17,7 @@ L2: align $(__WORDSIZE / 8) /* possible nops */ L3: jmpi L1 - align $(__WORDSIZE / 8) /* possible nops */ + align 32 /* Force nops */ L4: prepare pushargi fmt diff --git a/parallel-rsp/lightning/check/all.tst b/parallel-rsp/lightning/check/all.tst index ac4fc97..d24f7ae 100644 --- a/parallel-rsp/lightning/check/all.tst +++ b/parallel-rsp/lightning/check/all.tst @@ -2,15 +2,16 @@ .code prolog allocai 32 $buf - arg $c - arg $uc - arg $s - arg $us - arg $i + arg_c $c + arg_c $uc + arg_s $s + arg_s $us + arg_i $i + arg_i $ui #if __WORDSIZE == 64 - arg $ui - arg $l + arg_l $l #endif + arg $a getarg_c %r0 $c getarg_uc %r0 $uc getarg_s %r0 $s @@ -20,6 +21,25 @@ getarg_ui %r0 $ui getarg_l %r0 $l #endif + getarg %r0 $a + putargr_c %r0 $c + putargi_c 1 $c + putargr_uc %r0 $uc + putargi_uc 1 $uc + putargr_s %r0 $s + putargi_s 1 $s + putargr_us %r0 $us + putargi_us 1 $us + putargr_i %r0 $i + putargi_i 1 $ui +#if __WORDSIZE == 64 + putargr_ui %r0 $ui + putargi_ui 1 $ui + putargr_l %r0 $l + putargi_l 1 $l +#endif + putargr %r0 $a + putargi 1 $a addr %r0 %r1 %r2 addi %r0 %r1 2 addcr %r0 %r1 %r2 @@ -64,6 +84,10 @@ rshi_u %r0 %r1 2 negr %r0 %r1 comr %r0 %r1 + clor %r0 %r1 + clzr %r0 %r1 + ctor %r0 %r1 + ctzr %r0 %r1 ltr %r0 %r1 %r2 lti %r0 %r1 2 ltr_u %r0 %r1 %r2 @@ -205,6 +229,15 @@ label: callr %r0 calli label prepare + pushargr_c %r0 + pushargr_uc %r0 + pushargr_s %r0 + pushargr_us %r0 + pushargr_i %r0 +#if __WORDSIZE == 64 + pushargr_ui %r0 + pushargr_l %r0 +#endif pushargr %r0 finishr %r0 prepare @@ -212,6 +245,15 @@ label: ellipsis finishi 0x80000000 ret + retr_c %r1 + retr_uc %r1 + retr_s %r1 + retr_us %r1 + retr_i %r1 +#if __WORDSIZE == 64 + retr_ui %r1 + retr_l %r1 +#endif retr %r1 reti 2 retval_c %r1 @@ -225,6 +267,8 @@ label: #endif arg_f $f getarg_f %f1 $f + putargr_f %f1 $f + putargi_f 1.0 $f addr_f %f0 %f1 %f2 addi_f %f0 %f1 0.5 subr_f %f0 %f1 %f2 @@ -323,6 +367,8 @@ unordi: retval_f %f1 arg_d $f getarg_d %f1 $f + putargr_d %f1 $f + putargi_d 1.0 $f addr_d %f0 %f1 %f2 addi_d %f0 %f1 0.5 subr_d %f0 %f1 %f2 diff --git a/parallel-rsp/lightning/check/allocar.tst b/parallel-rsp/lightning/check/allocar.tst index e3ee010..1bffef8 100644 --- a/parallel-rsp/lightning/check/allocar.tst +++ b/parallel-rsp/lightning/check/allocar.tst @@ -55,7 +55,7 @@ fill##T##done: \ #define fill_us fill_s #define fill_ui fill_i -#define ARG( T, N) arg $arg##T##N +#define ARG( T, N) arg##T $arg##T##N #define ARGF( T, N) arg##T $arg##T##N #define ARG1( K, T) ARG##K(T, 0) #define ARG2( K, T) ARG1( K, T) ARG##K(T, 1) @@ -74,56 +74,56 @@ fill##T##done: \ #define ARG15(K, T) ARG14(K, T) ARG##K(T, 14) #define ARG16(K, T) ARG15(K, T) ARG##K(T, 15) #define ARG_c(N) ARG##N( , _c) -#define ARG_uc(N) ARG##N( , _uc) +#define ARG_uc(N) ARG##N( , _c) #define ARG_s(N) ARG##N( , _s) -#define ARG_us(N) ARG##N( , _us) +#define ARG_us(N) ARG##N( , _s) #define ARG_i(N) ARG##N( , _i) -#define ARG_ui(N) ARG##N( , _ui) +#define ARG_ui(N) ARG##N( , _i) #define ARG_l(N) ARG##N( , _l) #define ARG_f(N) ARG##N(F, _f) #define ARG_d(N) ARG##N(F, _d) -#define CHK(N, T, V) \ - getarg %r0 $arg##T##V \ +#define CHK(N, T, TT, V) \ + getarg##T %r0 $arg##TT##V \ ldxi##T %r1 %v0 $(V * szof##T) \ beqr N##T##V %r0 %r1 \ calli @abort \ N##T##V: -#define CHKF(N, T, V) \ - getarg##T %f0 $arg##T##V \ +#define CHKF(N, T, TT, V) \ + getarg##T %f0 $arg##TT##V \ ldxi##T %f1 %v0 $(V * szof##T) \ beqr##T N##T##V %f0 %f1 \ calli @abort \ N##T##V: -#define GET1( K, N, T, V) CHK##K(N, T, 0) -#define GET2( K, N, T, V) GET1( K, N, T, V) CHK##K(N, T, 1) -#define GET3( K, N, T, V) GET2( K, N, T, V) CHK##K(N, T, 2) -#define GET4( K, N, T, V) GET3( K, N, T, V) CHK##K(N, T, 3) -#define GET5( K, N, T, V) GET4( K, N, T, V) CHK##K(N, T, 4) -#define GET6( K, N, T, V) GET5( K, N, T, V) CHK##K(N, T, 5) -#define GET7( K, N, T, V) GET6( K, N, T, V) CHK##K(N, T, 6) -#define GET8( K, N, T, V) GET7( K, N, T, V) CHK##K(N, T, 7) -#define GET9( K, N, T, V) GET8( K, N, T, V) CHK##K(N, T, 8) -#define GET10(K, N, T, V) GET9( K, N, T, V) CHK##K(N, T, 9) -#define GET11(K, N, T, V) GET10(K, N, T, V) CHK##K(N, T, 10) -#define GET12(K, N, T, V) GET11(K, N, T, V) CHK##K(N, T, 11) -#define GET13(K, N, T, V) GET12(K, N, T, V) CHK##K(N, T, 12) -#define GET14(K, N, T, V) GET13(K, N, T, V) CHK##K(N, T, 13) -#define GET15(K, N, T, V) GET14(K, N, T, V) CHK##K(N, T, 14) -#define GET16(K, N, T, V) GET15(K, N, T, V) CHK##K(N, T, 15) +#define GET1( K, N, T, TT, V) CHK##K(N, T, TT, 0) +#define GET2( K, N, T, TT, V) GET1( K, N, T, TT, V) CHK##K(N, T, TT, 1) +#define GET3( K, N, T, TT, V) GET2( K, N, T, TT, V) CHK##K(N, T, TT, 2) +#define GET4( K, N, T, TT, V) GET3( K, N, T, TT, V) CHK##K(N, T, TT, 3) +#define GET5( K, N, T, TT, V) GET4( K, N, T, TT, V) CHK##K(N, T, TT, 4) +#define GET6( K, N, T, TT, V) GET5( K, N, T, TT, V) CHK##K(N, T, TT, 5) +#define GET7( K, N, T, TT, V) GET6( K, N, T, TT, V) CHK##K(N, T, TT, 6) +#define GET8( K, N, T, TT, V) GET7( K, N, T, TT, V) CHK##K(N, T, TT, 7) +#define GET9( K, N, T, TT, V) GET8( K, N, T, TT, V) CHK##K(N, T, TT, 8) +#define GET10(K, N, T, TT, V) GET9( K, N, T, TT, V) CHK##K(N, T, TT, 9) +#define GET11(K, N, T, TT, V) GET10(K, N, T, TT, V) CHK##K(N, T, TT, 10) +#define GET12(K, N, T, TT, V) GET11(K, N, T, TT, V) CHK##K(N, T, TT, 11) +#define GET13(K, N, T, TT, V) GET12(K, N, T, TT, V) CHK##K(N, T, TT, 12) +#define GET14(K, N, T, TT, V) GET13(K, N, T, TT, V) CHK##K(N, T, TT, 13) +#define GET15(K, N, T, TT, V) GET14(K, N, T, TT, V) CHK##K(N, T, TT, 14) +#define GET16(K, N, T, TT, V) GET15(K, N, T, TT, V) CHK##K(N, T, TT, 15) -#define GET_c(N, M) GET##N( , c##N, _c, M) -#define GET_uc(N, M) GET##N( , uc##N, _uc, M) -#define GET_s(N, M) GET##N( , s##N, _s, M) -#define GET_us(N, M) GET##N( , us##N, _us, M) -#define GET_i(N, M) GET##N( , i##N, _i, M) -#define GET_ui(N, M) GET##N( , ui##N, _ui, M) -#define GET_l(N, M) GET##N( , l##N, _l, M) -#define GET_f(N, M) GET##N(F, f##N, _f, M) -#define GET_d(N, M) GET##N(F, d##N, _d, M) +#define GET_c(N, M) GET##N( , c##N, _c, _c, M) +#define GET_uc(N, M) GET##N( , uc##N, _uc, _c, M) +#define GET_s(N, M) GET##N( , s##N, _s, _s, M) +#define GET_us(N, M) GET##N( , us##N, _us, _s, M) +#define GET_i(N, M) GET##N( , i##N, _i, _i, M) +#define GET_ui(N, M) GET##N( , ui##N, _ui, _i, M) +#define GET_l(N, M) GET##N( , l##N, _l, _l, M) +#define GET_f(N, M) GET##N(F, f##N, _f, _f, M) +#define GET_d(N, M) GET##N(F, d##N, _d, _d, M) -#define PUSH( T, V) pushargi V +#define PUSH( T, V) pushargi##T V #define PUSHF( T, V) pushargi##T V #define PUSH0( K, T) /**/ #define PUSH1( K, T) PUSH##K(T, 0) @@ -161,14 +161,14 @@ test##T##_0: \ ret \ epilog -#define DEFN(N, M, T) \ +#define DEFN(N, M, T, TT) \ name test##T##_##N \ test##T##_##N: \ prolog \ arg $argp \ /* stack buffer in %v0 */ \ getarg %v0 $argp \ - ARG##T(N) \ + ARG##TT(N) \ /* validate arguments */ \ GET##T(N, M) \ /* heap buffer in %v1 */ \ @@ -260,24 +260,24 @@ test##T##_17_done: \ ret \ epilog -#define DEF( T) \ +#define DEF( T, TT) \ DEF0( T) \ - DEFN( 1, 0, T) \ - DEFN( 2, 1, T) \ - DEFN( 3, 2, T) \ - DEFN( 4, 3, T) \ - DEFN( 5, 4, T) \ - DEFN( 6, 5, T) \ - DEFN( 7, 6, T) \ - DEFN( 8, 7, T) \ - DEFN( 9, 8, T) \ - DEFN(10, 9, T) \ - DEFN(11, 10, T) \ - DEFN(12, 11, T) \ - DEFN(13, 12, T) \ - DEFN(14, 13, T) \ - DEFN(15, 14, T) \ - DEFN(16, 15, T) \ + DEFN( 1, 0, T, TT) \ + DEFN( 2, 1, T, TT) \ + DEFN( 3, 2, T, TT) \ + DEFN( 4, 3, T, TT) \ + DEFN( 5, 4, T, TT) \ + DEFN( 6, 5, T, TT) \ + DEFN( 7, 6, T, TT) \ + DEFN( 8, 7, T, TT) \ + DEFN( 9, 8, T, TT) \ + DEFN(10, 9, T, TT) \ + DEFN(11, 10, T, TT) \ + DEFN(12, 11, T, TT) \ + DEFN(13, 12, T, TT) \ + DEFN(14, 13, T, TT) \ + DEFN(15, 14, T, TT) \ + DEFN(16, 15, T, TT) \ DEFX(T) #define CALL(T) calli test##T##_17 @@ -323,17 +323,17 @@ memcpy_done: FILLF(_f) FILLF(_d) - DEF(_c) - DEF(_uc) - DEF(_s) - DEF(_us) - DEF(_i) + DEF(_c, _c) + DEF(_uc, _c) + DEF(_s, _s) + DEF(_us, _s) + DEF(_i, _i) #if __WORDSIZE == 64 - DEF(_ui) - DEF(_l) + DEF(_ui, _i) + DEF(_l, _l) #endif - DEF(_f) - DEF(_d) + DEF(_f, _f) + DEF(_d, _d) name main main: diff --git a/parallel-rsp/lightning/check/bit.ok b/parallel-rsp/lightning/check/bit.ok new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/parallel-rsp/lightning/check/bit.ok @@ -0,0 +1 @@ +ok diff --git a/parallel-rsp/lightning/check/bit.tst b/parallel-rsp/lightning/check/bit.tst new file mode 100644 index 0000000..b721d5c --- /dev/null +++ b/parallel-rsp/lightning/check/bit.tst @@ -0,0 +1,881 @@ +/* If the fallback clor, clzr, ctor and ctzr are used, it might be better + * to implement it as functions, as inlined it is almost as large as a + * function. + * Below is an example of how to do it. + */ + +.data 4096 +str_clo: +.c "clo" +str_clz: +.c "clz" +str_cto: +.c "cto" +str_ctz: +.c "ctz" +print_fmt: +#if __WORDSIZE == 64 +.c "%s (0x%016lx) %s = %d\n" +#else +.c "%s (0x%08lx) %s = %d\n" +#endif +ok: +.c "ok\n" + +#define BIT2(OP, ARG, RES, R0, R1) \ + movi %R1 ARG \ + OP##r %R0 %R1 \ + beqi OP##R0##R1##ARG %R0 RES \ + calli @abort \ +OP##R0##R1##ARG: + +#define BIT1(OP, ARG, RES, V0, V1, V2, R0, R1, R2) \ + BIT2(OP, ARG, RES, V0, V0) \ + BIT2(OP, ARG, RES, V0, V1) \ + BIT2(OP, ARG, RES, V0, V2) \ + BIT2(OP, ARG, RES, V0, R0) \ + BIT2(OP, ARG, RES, V0, R1) \ + BIT2(OP, ARG, RES, V0, R2) + +#define BIT(OP, ARG, RES, V0, V1, V2, R0, R1, R2) \ + BIT1(OP, ARG, RES, V1, V2, R0, R1, R2, V0) \ + BIT1(OP, ARG, RES, V2, R0, R1, R2, V0, V1) \ + BIT1(OP, ARG, RES, R0, R1, R2, V0, V1, V2) \ + BIT1(OP, ARG, RES, R1, R2, V0, V1, V2, R0) \ + BIT1(OP, ARG, RES, R2, V0, V1, V2, R0, R1) + +#define CLO(ARG, RES) \ + BIT(clo, ARG, RES, v0, v1, v2, r0, r1, r2) +#define CLZ(ARG, RES) \ + BIT(clz, ARG, RES, v0, v1, v2, r0, r1, r2) +#define CTO(ARG, RES) \ + BIT(cto, ARG, RES, v0, v1, v2, r0, r1, r2) +#define CTZ(ARG, RES) \ + BIT(ctz, ARG, RES, v0, v1, v2, r0, r1, r2) + +.code + jmpi main +/* + jit_uword_t cto(jit_uword_t r0) { + r0 = ~r0; + if (r0 == 0) + r0 = __WORDSIZE; + else + r0 = ctz(r0); + return r0; + } + */ +name cto +cto: + prolog + arg $in + getarg %r0 $in + comr %r0 %r0 + bnei do_cto %r0 0 + movi %r0 __WORDSIZE + jmpi done_cto +do_cto: + prepare + pushargr %r0 + finishi ctz + retval %r0 +done_cto: + retr %r0 + epilog + +/* + jit_uword_t clo(jit_uword_t r0) { + r0 = ~r0; + if (r0 == 0) + r0 = __WORDSIZE; + else + r0 = clz(r0); + return r0; + } + */ +name clo +clo: + prolog + arg $in + getarg %r0 $in + comr %r0 %r0 + bnei do_clo %r0 0 + movi %r0 __WORDSIZE + jmpi done_clo +do_clo: + prepare + pushargr %r0 + finishi clz + retval %r0 +done_clo: + retr %r0 + epilog + +/* + jit_uword_t clz(jit_word_t r1) { + jit_uword_t r0, r2; + if (r1 == 0) + r0 = __WORDSIZE; + else { + r0 = 0; + #if __WORDSIZE == 64 + r2 = 0xffffffff00000000UL; + if (!(r1 & r2)) { + r1 <<= 32; + r0 += 32; + } + r2 <<= 16; + #else + r2 = 0xffff0000UL; + #endif + if (!(r1 & r2)) { + r1 <<= 16; + r0 += 16; + } + r2 <<= 8; + if (!(r1 & r2)) { + r1 <<= 8; + r0 += 8; + } + r2 <<= 4; + if (!(r1 & r2)) { + r1 <<= 4; + r0 += 4; + } + r2 <<= 2; + if (!(r1 & r2)) { + r1 <<= 2; + r0 += 2; + } + r2 <<= 1; + if (!(r1 & r2)) + r0 += 1; + } + return r0; + } + */ +name clz +clz: + prolog + arg $in + getarg %r1 $in + bnei lun %r1 0 + reti __WORDSIZE +lun: + movi %r0 0 +#if __WORDSIZE == 64 + movi %r2 0xffffffff00000000 + bmsr l32 %r1 %r2 + lshi %r1 %r1 32 + addi %r0 %r0 32 +l32: + lshi %r2 %r2 16 +#else + movi %r2 0xffff0000 +#endif + bmsr l16 %r1 %r2 + lshi %r1 %r1 16 + addi %r0 %r0 16 +l16: + lshi %r2 %r2 8 + bmsr l8 %r1 %r2 + lshi %r1 %r1 8 + addi %r0 %r0 8 +l8: + lshi %r2 %r2 4 + bmsr l4 %r1 %r2 + lshi %r1 %r1 4 + addi %r0 %r0 4 +l4: + lshi %r2 %r2 2 + bmsr l2 %r1 %r2 + lshi %r1 %r1 2 + addi %r0 %r0 2 +l2: + lshi %r2 %r2 1 + bmsr l1 %r1 %r2 + addi %r0 %r0 1 +l1: + retr %r0 + epilog + +/* + jit_uword_t ctz(jit_uword_t r1) { + jit_uword_t r0, r2; + if (r1 == 0) + r0 = __WORDSIZE; + else { + r0 = 0; + #if __WORDSIZE == 64 + r2 = 0xffffffffUL;; + if (!(r1 & r2)) { + r1 >>= 32; + r0 += 32; + } + r2 >>= 16; + #else + r2 = 0xffffUL;; + #endif + if (!(r1 & r2)) { + r1 >>= 16; + r0 += 16; + } + r2 >>= 8; + if (!(r1 & r2)) { + r1 >>= 8; + r0 += 8; + } + r2 >>= 4; + if (!(r1 & r2)) { + r1 >>= 4; + r0 += 4; + } + r2 >>= 2; + if (!(r1 & r2)) { + r1 >>= 2; + r0 += 2; + } + r2 >>= 1; + if (!(r1 & r2)) + r0 += 1; + } + return r0; + } +*/ +name ctz +ctz: + prolog + arg $in + getarg %r1 $in + bnei tun %r1 0 + reti __WORDSIZE +tun: +#if __WORDSIZE == 64 + movi %r0 0 + movi %r2 0xffffffff + bmsr t32 %r1 %r2 + rshi_u %r1 %r1 32 + addi %r0 %r0 32 +t32: + rshi %r2 %r2 16 +#else + movi %r2 0xffff +#endif + bmsr t16 %r1 %r2 + rshi_u %r1 %r1 16 + addi %r0 %r0 16 +t16: + rshi %r2 %r2 8 + bmsr t8 %r1 %r2 + rshi_u %r1 %r1 8 + addi %r0 %r0 8 +t8: + rshi %r2 %r2 4 + bmsr t4 %r1 %r2 + rshi_u %r1 %r1 4 + addi %r0 %r0 4 +t4: + rshi %r2 %r2 2 + bmsr t2 %r1 %r2 + rshi_u %r1 %r1 2 + addi %r0 %r0 2 +t2: + rshi %r2 %r2 1 + bmsr t1 %r1 %r2 + addi %r0 %r0 1 +t1: + retr %r0 + epilog + +/* + char *bitsprint(char *v0, jit_uword_t v1) { + jit_uword_t r0, r1; + memset(v0, '0', __WORDSIZE); + v0[__WORDSIZE] = 0; + for (r0 = 1L << (__WORDSIZE - 1), r1 = 0; r0; r0 >>= 1, ++r1) { + if (v1 & r0) + v0[r1] = '1'; + } + return v0; + } + */ +name bitsprint +bitsprint: + prolog + arg $buf + arg $val + getarg %v0 $buf + getarg %v1 $val + prepare + pushargr %v0 + pushargi '0' + pushargi __WORDSIZE + finishi @memset + movi %r0 0 + addi %r1 %v0 __WORDSIZE + str_c %r1 %r0 + movi %r0 $(1 << (__WORDSIZE - 1)) + movi %r1 0 + movi %r2 '1' +bitloop: + bmcr bitzero %v1 %r0 + stxr_c %r1 %v0 %r2 +bitzero: + addi %r1 %r1 1 + rshi_u %r0 %r0 1 + bnei bitloop %r0 0 + retr %v0 + epilog + +/* + #if 0 + int main(int argc, char *argv[]) { + jit_uword_t r0, v0, v1, v2; + char buf[80]; + #if __WORDSIZE == 64 + char *fmt = "%s (0x%016lx) %s = %d\n"; + v0 = 0x8000000000000000UL; + v2 = 0xffffffffffffffffUL; + #else + char *fmt = "%s (0x%08lx) %s = %d\n"; + v0 = 0x80000000UL; + v2 = 0xffffffffUL; + #endif + do { + v1 = v0 - 1; + r0 = clz(v0); + bitsprint(buf, v0); + printf(fmt, "clz", v0, buf, r0); + r0 = clo(v2); + bitsprint(buf, v2); + printf(fmt, "clo", v2, buf, r0); + r0 = ctz(v0); + bitsprint(buf, v0); + printf(fmt, "ctz", v0, buf, r0); + r0 = cto(v1); + bitsprint(buf, v1); + printf(fmt, "cto", v1, buf, r0); + v0 >>= 1; + v2 <<= 1; + } while ((jit_word_t)v1 > -1); + return 0; + } + #endif + */ + +/* Make it "#if 1" for a "debug mode", that helps in regenerating tables, + * or temporary state while implementing optimized port specific versions. */ +#if 0 +#define CALL_FUNC 1 + name main +main: + prolog + allocai 80 $buf +#if __WORDSIZE == 64 + movi %v0 0x8000000000000000 + movi %v2 0xffffffffffffffff +#else + movi %v0 0x80000000 + movi %v2 0xffffffff +#endif +loop: + subi %v1 %v0 1 + addi %r1 %fp $buf + prepare + pushargr %r1 + pushargr %v0 + finishi bitsprint +#if CALL_FUNC + prepare + pushargr %v0 + finishi clz + retval %r0 +#else + clzr %r0 %v0 +#endif + addi %r1 %fp $buf + prepare + pushargi print_fmt + ellipsis + pushargi str_clz + pushargr %v0 + pushargr %r1 + pushargr %r0 + finishi @printf + addi %r1 %fp $buf + prepare + pushargr %r1 + pushargr %v2 + finishi bitsprint +#if CALL_FUNC + prepare + pushargr %v2 + finishi clo + retval %r0 +#else + clor %r0 %v2 +#endif + addi %r1 %fp $buf + prepare + pushargi print_fmt + ellipsis + pushargi str_clo + pushargr %v2 + pushargr %r1 + pushargr %r0 + finishi @printf + addi %r1 %fp $buf + prepare + pushargr %r1 + pushargr %v0 + finishi bitsprint +#if CALL_FUNC + prepare + pushargr %v0 + finishi ctz + retval %r0 +#else + ctzr %r0 %v0 +#endif + addi %r1 %fp $buf + prepare + pushargi print_fmt + ellipsis + pushargi str_ctz + pushargr %v0 + pushargr %r1 + pushargr %r0 + finishi @printf + addi %r1 %fp $buf + prepare + pushargr %r1 + pushargr %v1 + finishi bitsprint +#if CALL_FUNC + prepare + pushargr %v1 + finishi cto + retval %r0 +#else + ctor %r0 %v1 +#endif + addi %r1 %fp $buf + prepare + pushargi print_fmt + ellipsis + pushargi str_cto + pushargr %v1 + pushargr %r1 + pushargr %r0 + finishi @printf + rshi_u %v0 %v0 1 + lshi %v2 %v2 1 + bgti loop %v1 -1 + ret + epilog +#else + + name main +main: + prolog +#if __WORDSIZE == 32 + CLZ(0x80000000, 0) + CLO(0xffffffff, 32) + CTZ(0x80000000, 31) + CTO(0x7fffffff, 31) + CLZ(0x40000000, 1) + CLO(0xfffffffe, 31) + CTZ(0x40000000, 30) + CTO(0x3fffffff, 30) + CLZ(0x20000000, 2) + CLO(0xfffffffc, 30) + CTZ(0x20000000, 29) + CTO(0x1fffffff, 29) + CLZ(0x10000000, 3) + CLO(0xfffffff8, 29) + CTZ(0x10000000, 28) + CTO(0x0fffffff, 28) + CLZ(0x08000000, 4) + CLO(0xfffffff0, 28) + CTZ(0x08000000, 27) + CTO(0x07ffffff, 27) + CLZ(0x04000000, 5) + CLO(0xffffffe0, 27) + CTZ(0x04000000, 26) + CTO(0x03ffffff, 26) + CLZ(0x02000000, 6) + CLO(0xffffffc0, 26) + CTZ(0x02000000, 25) + CTO(0x01ffffff, 25) + CLZ(0x01000000, 7) + CLO(0xffffff80, 25) + CTZ(0x01000000, 24) + CTO(0x00ffffff, 24) + CLZ(0x00800000, 8) + CLO(0xffffff00, 24) + CTZ(0x00800000, 23) + CTO(0x007fffff, 23) + CLZ(0x00400000, 9) + CLO(0xfffffe00, 23) + CTZ(0x00400000, 22) + CTO(0x003fffff, 22) + CLZ(0x00200000, 10) + CLO(0xfffffc00, 22) + CTZ(0x00200000, 21) + CTO(0x001fffff, 21) + CLZ(0x00100000, 11) + CLO(0xfffff800, 21) + CTZ(0x00100000, 20) + CTO(0x000fffff, 20) + CLZ(0x00080000, 12) + CLO(0xfffff000, 20) + CTZ(0x00080000, 19) + CTO(0x0007ffff, 19) + CLZ(0x00040000, 13) + CLO(0xffffe000, 19) + CTZ(0x00040000, 18) + CTO(0x0003ffff, 18) + CLZ(0x00020000, 14) + CLO(0xffffc000, 18) + CTZ(0x00020000, 17) + CTO(0x0001ffff, 17) + CLZ(0x00010000, 15) + CLO(0xffff8000, 17) + CTZ(0x00010000, 16) + CTO(0x0000ffff, 16) + CLZ(0x00008000, 16) + CLO(0xffff0000, 16) + CTZ(0x00008000, 15) + CTO(0x00007fff, 15) + CLZ(0x00004000, 17) + CLO(0xfffe0000, 15) + CTZ(0x00004000, 14) + CTO(0x00003fff, 14) + CLZ(0x00002000, 18) + CLO(0xfffc0000, 14) + CTZ(0x00002000, 13) + CTO(0x00001fff, 13) + CLZ(0x00001000, 19) + CLO(0xfff80000, 13) + CTZ(0x00001000, 12) + CTO(0x00000fff, 12) + CLZ(0x00000800, 20) + CLO(0xfff00000, 12) + CTZ(0x00000800, 11) + CTO(0x000007ff, 11) + CLZ(0x00000400, 21) + CLO(0xffe00000, 11) + CTZ(0x00000400, 10) + CTO(0x000003ff, 10) + CLZ(0x00000200, 22) + CLO(0xffc00000, 10) + CTZ(0x00000200, 9) + CTO(0x000001ff, 9) + CLZ(0x00000100, 23) + CLO(0xff800000, 9) + CTZ(0x00000100, 8) + CTO(0x000000ff, 8) + CLZ(0x00000080, 24) + CLO(0xff000000, 8) + CTZ(0x00000080, 7) + CTO(0x0000007f, 7) + CLZ(0x00000040, 25) + CLO(0xfe000000, 7) + CTZ(0x00000040, 6) + CTO(0x0000003f, 6) + CLZ(0x00000020, 26) + CLO(0xfc000000, 6) + CTZ(0x00000020, 5) + CTO(0x0000001f, 5) + CLZ(0x00000010, 27) + CLO(0xf8000000, 5) + CTZ(0x00000010, 4) + CTO(0x0000000f, 4) + CLZ(0x00000008, 28) + CLO(0xf0000000, 4) + CTZ(0x00000008, 3) + CTO(0x00000007, 3) + CLZ(0x00000004, 29) + CLO(0xe0000000, 3) + CTZ(0x00000004, 2) + CTO(0x00000003, 2) + CLZ(0x00000002, 30) + CLO(0xc0000000, 2) + CTZ(0x00000002, 1) + CTO(0x00000001, 1) + CLZ(0x00000001, 31) + CLO(0x80000000, 1) + CTZ(0x00000001, 0) + CTO(0x00000000, 0) + CLZ(0x00000000, 32) + CLO(0x00000000, 0) + CTZ(0x00000000, 32) + CTO(0xffffffff, 32) +#else + CLZ(0x8000000000000000, 0) + CLO(0xffffffffffffffff, 64) + CTZ(0x8000000000000000, 63) + CTO(0x7fffffffffffffff, 63) + CLZ(0x4000000000000000, 1) + CLO(0xfffffffffffffffe, 63) + CTZ(0x4000000000000000, 62) + CTO(0x3fffffffffffffff, 62) + CLZ(0x2000000000000000, 2) + CLO(0xfffffffffffffffc, 62) + CTZ(0x2000000000000000, 61) + CTO(0x1fffffffffffffff, 61) + CLZ(0x1000000000000000, 3) + CLO(0xfffffffffffffff8, 61) + CTZ(0x1000000000000000, 60) + CTO(0x0fffffffffffffff, 60) + CLZ(0x0800000000000000, 4) + CLO(0xfffffffffffffff0, 60) + CTZ(0x0800000000000000, 59) + CTO(0x07ffffffffffffff, 59) + CLZ(0x0400000000000000, 5) + CLO(0xffffffffffffffe0, 59) + CTZ(0x0400000000000000, 58) + CTO(0x03ffffffffffffff, 58) + CLZ(0x0200000000000000, 6) + CLO(0xffffffffffffffc0, 58) + CTZ(0x0200000000000000, 57) + CTO(0x01ffffffffffffff, 57) + CLZ(0x0100000000000000, 7) + CLO(0xffffffffffffff80, 57) + CTZ(0x0100000000000000, 56) + CTO(0x00ffffffffffffff, 56) + CLZ(0x0080000000000000, 8) + CLO(0xffffffffffffff00, 56) + CTZ(0x0080000000000000, 55) + CTO(0x007fffffffffffff, 55) + CLZ(0x0040000000000000, 9) + CLO(0xfffffffffffffe00, 55) + CTZ(0x0040000000000000, 54) + CTO(0x003fffffffffffff, 54) + CLZ(0x0020000000000000, 10) + CLO(0xfffffffffffffc00, 54) + CTZ(0x0020000000000000, 53) + CTO(0x001fffffffffffff, 53) + CLZ(0x0010000000000000, 11) + CLO(0xfffffffffffff800, 53) + CTZ(0x0010000000000000, 52) + CTO(0x000fffffffffffff, 52) + CLZ(0x0008000000000000, 12) + CLO(0xfffffffffffff000, 52) + CTZ(0x0008000000000000, 51) + CTO(0x0007ffffffffffff, 51) + CLZ(0x0004000000000000, 13) + CLO(0xffffffffffffe000, 51) + CTZ(0x0004000000000000, 50) + CTO(0x0003ffffffffffff, 50) + CLZ(0x0002000000000000, 14) + CLO(0xffffffffffffc000, 50) + CTZ(0x0002000000000000, 49) + CTO(0x0001ffffffffffff, 49) + CLZ(0x0001000000000000, 15) + CLO(0xffffffffffff8000, 49) + CTZ(0x0001000000000000, 48) + CTO(0x0000ffffffffffff, 48) + CLZ(0x0000800000000000, 16) + CLO(0xffffffffffff0000, 48) + CTZ(0x0000800000000000, 47) + CTO(0x00007fffffffffff, 47) + CLZ(0x0000400000000000, 17) + CLO(0xfffffffffffe0000, 47) + CTZ(0x0000400000000000, 46) + CTO(0x00003fffffffffff, 46) + CLZ(0x0000200000000000, 18) + CLO(0xfffffffffffc0000, 46) + CTZ(0x0000200000000000, 45) + CTO(0x00001fffffffffff, 45) + CLZ(0x0000100000000000, 19) + CLO(0xfffffffffff80000, 45) + CTZ(0x0000100000000000, 44) + CTO(0x00000fffffffffff, 44) + CLZ(0x0000080000000000, 20) + CLO(0xfffffffffff00000, 44) + CTZ(0x0000080000000000, 43) + CTO(0x000007ffffffffff, 43) + CLZ(0x0000040000000000, 21) + CLO(0xffffffffffe00000, 43) + CTZ(0x0000040000000000, 42) + CTO(0x000003ffffffffff, 42) + CLZ(0x0000020000000000, 22) + CLO(0xffffffffffc00000, 42) + CTZ(0x0000020000000000, 41) + CTO(0x000001ffffffffff, 41) + CLZ(0x0000010000000000, 23) + CLO(0xffffffffff800000, 41) + CTZ(0x0000010000000000, 40) + CTO(0x000000ffffffffff, 40) + CLZ(0x0000008000000000, 24) + CLO(0xffffffffff000000, 40) + CTZ(0x0000008000000000, 39) + CTO(0x0000007fffffffff, 39) + CLZ(0x0000004000000000, 25) + CLO(0xfffffffffe000000, 39) + CTZ(0x0000004000000000, 38) + CTO(0x0000003fffffffff, 38) + CLZ(0x0000002000000000, 26) + CLO(0xfffffffffc000000, 38) + CTZ(0x0000002000000000, 37) + CTO(0x0000001fffffffff, 37) + CLZ(0x0000001000000000, 27) + CLO(0xfffffffff8000000, 37) + CTZ(0x0000001000000000, 36) + CTO(0x0000000fffffffff, 36) + CLZ(0x0000000800000000, 28) + CLO(0xfffffffff0000000, 36) + CTZ(0x0000000800000000, 35) + CTO(0x00000007ffffffff, 35) + CLZ(0x0000000400000000, 29) + CLO(0xffffffffe0000000, 35) + CTZ(0x0000000400000000, 34) + CTO(0x00000003ffffffff, 34) + CLZ(0x0000000200000000, 30) + CLO(0xffffffffc0000000, 34) + CTZ(0x0000000200000000, 33) + CTO(0x00000001ffffffff, 33) + CLZ(0x0000000100000000, 31) + CLO(0xffffffff80000000, 33) + CTZ(0x0000000100000000, 32) + CTO(0x00000000ffffffff, 32) + CLZ(0x0000000080000000, 32) + CLO(0xffffffff00000000, 32) + CTZ(0x0000000080000000, 31) + CTO(0x000000007fffffff, 31) + CLZ(0x0000000040000000, 33) + CLO(0xfffffffe00000000, 31) + CTZ(0x0000000040000000, 30) + CTO(0x000000003fffffff, 30) + CLZ(0x0000000020000000, 34) + CLO(0xfffffffc00000000, 30) + CTZ(0x0000000020000000, 29) + CTO(0x000000001fffffff, 29) + CLZ(0x0000000010000000, 35) + CLO(0xfffffff800000000, 29) + CTZ(0x0000000010000000, 28) + CTO(0x000000000fffffff, 28) + CLZ(0x0000000008000000, 36) + CLO(0xfffffff000000000, 28) + CTZ(0x0000000008000000, 27) + CTO(0x0000000007ffffff, 27) + CLZ(0x0000000004000000, 37) + CLO(0xffffffe000000000, 27) + CTZ(0x0000000004000000, 26) + CTO(0x0000000003ffffff, 26) + CLZ(0x0000000002000000, 38) + CLO(0xffffffc000000000, 26) + CTZ(0x0000000002000000, 25) + CTO(0x0000000001ffffff, 25) + CLZ(0x0000000001000000, 39) + CLO(0xffffff8000000000, 25) + CTZ(0x0000000001000000, 24) + CTO(0x0000000000ffffff, 24) + CLZ(0x0000000000800000, 40) + CLO(0xffffff0000000000, 24) + CTZ(0x0000000000800000, 23) + CTO(0x00000000007fffff, 23) + CLZ(0x0000000000400000, 41) + CLO(0xfffffe0000000000, 23) + CTZ(0x0000000000400000, 22) + CTO(0x00000000003fffff, 22) + CLZ(0x0000000000200000, 42) + CLO(0xfffffc0000000000, 22) + CTZ(0x0000000000200000, 21) + CTO(0x00000000001fffff, 21) + CLZ(0x0000000000100000, 43) + CLO(0xfffff80000000000, 21) + CTZ(0x0000000000100000, 20) + CTO(0x00000000000fffff, 20) + CLZ(0x0000000000080000, 44) + CLO(0xfffff00000000000, 20) + CTZ(0x0000000000080000, 19) + CTO(0x000000000007ffff, 19) + CLZ(0x0000000000040000, 45) + CLO(0xffffe00000000000, 19) + CTZ(0x0000000000040000, 18) + CTO(0x000000000003ffff, 18) + CLZ(0x0000000000020000, 46) + CLO(0xffffc00000000000, 18) + CTZ(0x0000000000020000, 17) + CTO(0x000000000001ffff, 17) + CLZ(0x0000000000010000, 47) + CLO(0xffff800000000000, 17) + CTZ(0x0000000000010000, 16) + CTO(0x000000000000ffff, 16) + CLZ(0x0000000000008000, 48) + CLO(0xffff000000000000, 16) + CTZ(0x0000000000008000, 15) + CTO(0x0000000000007fff, 15) + CLZ(0x0000000000004000, 49) + CLO(0xfffe000000000000, 15) + CTZ(0x0000000000004000, 14) + CTO(0x0000000000003fff, 14) + CLZ(0x0000000000002000, 50) + CLO(0xfffc000000000000, 14) + CTZ(0x0000000000002000, 13) + CTO(0x0000000000001fff, 13) + CLZ(0x0000000000001000, 51) + CLO(0xfff8000000000000, 13) + CTZ(0x0000000000001000, 12) + CTO(0x0000000000000fff, 12) + CLZ(0x0000000000000800, 52) + CLO(0xfff0000000000000, 12) + CTZ(0x0000000000000800, 11) + CTO(0x00000000000007ff, 11) + CLZ(0x0000000000000400, 53) + CLO(0xffe0000000000000, 11) + CTZ(0x0000000000000400, 10) + CTO(0x00000000000003ff, 10) + CLZ(0x0000000000000200, 54) + CLO(0xffc0000000000000, 10) + CTZ(0x0000000000000200, 9) + CTO(0x00000000000001ff, 9) + CLZ(0x0000000000000100, 55) + CLO(0xff80000000000000, 9) + CTZ(0x0000000000000100, 8) + CTO(0x00000000000000ff, 8) + CLZ(0x0000000000000080, 56) + CLO(0xff00000000000000, 8) + CTZ(0x0000000000000080, 7) + CTO(0x000000000000007f, 7) + CLZ(0x0000000000000040, 57) + CLO(0xfe00000000000000, 7) + CTZ(0x0000000000000040, 6) + CTO(0x000000000000003f, 6) + CLZ(0x0000000000000020, 58) + CLO(0xfc00000000000000, 6) + CTZ(0x0000000000000020, 5) + CTO(0x000000000000001f, 5) + CLZ(0x0000000000000010, 59) + CLO(0xf800000000000000, 5) + CTZ(0x0000000000000010, 4) + CTO(0x000000000000000f, 4) + CLZ(0x0000000000000008, 60) + CLO(0xf000000000000000, 4) + CTZ(0x0000000000000008, 3) + CTO(0x0000000000000007, 3) + CLZ(0x0000000000000004, 61) + CLO(0xe000000000000000, 3) + CTZ(0x0000000000000004, 2) + CTO(0x0000000000000003, 2) + CLZ(0x0000000000000002, 62) + CLO(0xc000000000000000, 2) + CTZ(0x0000000000000002, 1) + CTO(0x0000000000000001, 1) + CLZ(0x0000000000000001, 63) + CLO(0x8000000000000000, 1) + CTZ(0x0000000000000001, 0) + CTO(0x0000000000000000, 0) + CLZ(0x0000000000000000, 64) + CLO(0x0000000000000000, 0) + CTZ(0x0000000000000000, 64) + CTO(0xffffffffffffffff, 64) +#endif + prepare + pushargi ok + finishi @printf + reti 0 + epilog +#endif diff --git a/parallel-rsp/lightning/check/bswap.ok b/parallel-rsp/lightning/check/bswap.ok new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/parallel-rsp/lightning/check/bswap.ok @@ -0,0 +1 @@ +ok diff --git a/parallel-rsp/lightning/check/bswap.tst b/parallel-rsp/lightning/check/bswap.tst new file mode 100644 index 0000000..f123e95 --- /dev/null +++ b/parallel-rsp/lightning/check/bswap.tst @@ -0,0 +1,154 @@ +.data 16 +ok: +.c "ok\n" + +#define us12_i 0x1234 +#define us7f_i 0x7ff7 +#define us80_i 0x8008 +#define usff_i 0xffff +#define ui12_i 0x01234567 +#define ui7f_i 0x7f7ff7f7 +#define ui80_i 0x80800808 +#define uiff_i 0xffffffff +#define ul12_i 0x0123456789abcdef +#define ul7f_i 0x7f7f7f7ff7f7f7f7 +#define ul80_i 0x8080808008080808 +#define ulff_i 0xffffffffffffffff + +#if __WORDSIZE == 32 +# define xus12_i 0xffff1234 +# define xus7f_i 0x10107ff7 +# define xus80_i 0x81188008 +# define xusff_i 0xeaaeffff +#else +# define xus12_i 0xffffffffffff1234 +# define xus7f_i 0x1010100101017ff7 +# define xus80_i 0x8181811818818008 +# define xusff_i 0xeaeaeaaeaeaeffff +# define xui12_i 0xffffffff01234567 +# define xui7f_i 0x101001017f7ff7f7 +# define xui80_i 0x8181181880800808 +# define xuiff_i 0xeaeaaeaeffffffff +#endif + +# define us12_o 0x3412 +# define us7f_o 0xf77f +# define us80_o 0x0880 +# define usff_o 0xffff +# define ui12_o 0x67452301 +# define ui7f_o 0xf7f77f7f +# define ui80_o 0x08088080 +# define uiff_o 0xffffffff +# define ul12_o 0xefcdab8967452301 +# define ul7f_o 0xf7f7f7f77f7f7f7f +# define ul80_o 0x0808080880808080 +# define ulff_o 0xffffffffffffffff + +#define BSWAP4(I, O, T, R0, R1) \ + movi %R0 I \ + bswapr_##T %R1 %R0 \ + beqi T##R0##R1##I %R1 O \ + calli @abort \ +T##R0##R1##I: + +#define BSWAP3(T, R0, R1) \ + BSWAP4(T##12_i, T##12_o, T, R0, R1) \ + BSWAP4(x##T##12_i, T##12_o, T, R0, R1) \ + BSWAP4(T##7f_i, T##7f_o, T, R0, R1) \ + BSWAP4(x##T##7f_i, T##7f_o, T, R0, R1) \ + BSWAP4(T##80_i, T##80_o, T, R0, R1) \ + BSWAP4(x##T##80_i, T##80_o, T, R0, R1) \ + BSWAP4(T##ff_i, T##ff_o, T, R0, R1) \ + BSWAP4(x##T##ff_i, T##ff_o, T, R0, R1) + +#define BSWAP3x(T, R0, R1) \ + BSWAP4(T##12_i, T##12_o, T, R0, R1) \ + BSWAP4(T##7f_i, T##7f_o, T, R0, R1) \ + BSWAP4(T##80_i, T##80_o, T, R0, R1) \ + BSWAP4(T##ff_i, T##ff_o, T, R0, R1) + +#define BSWAP2(T, V0, V1, V2, R0, R1, R2) \ + BSWAP3(T, V0, V0) \ + BSWAP3(T, V0, V1) \ + BSWAP3(T, V0, V2) \ + BSWAP3(T, V0, R0) \ + BSWAP3(T, V0, R1) \ + BSWAP3(T, V0, R2) \ + +#define BSWAP2x(T, V0, V1, V2, R0, R1, R2) \ + BSWAP3x(T, V0, V0) \ + BSWAP3x(T, V0, V1) \ + BSWAP3x(T, V0, V2) \ + BSWAP3x(T, V0, R0) \ + BSWAP3x(T, V0, R1) \ + BSWAP3x(T, V0, R2) \ + +#define BSWAP1(T, V0, V1, V2, R0, R1, R2) \ + BSWAP2(T, V0, V1, V2, R0, R1, R2) \ + BSWAP2(T, V1, V2, R0, R1, R2, V0) \ + BSWAP2(T, V2, R0, R1, R2, V0, V1) \ + BSWAP2(T, R0, R1, R2, V0, V1, V2) \ + BSWAP2(T, R1, R2, V0, V1, V2, R0) \ + BSWAP2(T, R2, V0, V1, V2, R0, R1) + +#define BSWAP1x(T, V0, V1, V2, R0, R1, R2) \ + BSWAP2x(T, V0, V1, V2, R0, R1, R2) \ + BSWAP2x(T, V1, V2, R0, R1, R2, V0) \ + BSWAP2x(T, V2, R0, R1, R2, V0, V1) \ + BSWAP2x(T, R0, R1, R2, V0, V1, V2) \ + BSWAP2x(T, R1, R2, V0, V1, V2, R0) \ + BSWAP2x(T, R2, V0, V1, V2, R0, R1) + +#if __WORDSIZE == 32 +# define BSWAP(V0, V1, V2, R0, R1, R2) \ + BSWAP1(us, V0, V1, V2, R0, R1, R2) \ + BSWAP1x(ui, V0, V1, V2, R0, R1, R2) +#else +# define BSWAP(V0, V1, V2, R0, R1, R2) \ + BSWAP1(us, V0, V1, V2, R0, R1, R2) \ + BSWAP1(ui, V0, V1, V2, R0, R1, R2) \ + BSWAP1x(ul, V0, V1, V2, R0, R1, R2) +#endif + +.code + prolog + /* simple sequence for easier disassembly reading and encoding check */ + movi %r0 us12_i + bswapr_us %r1 %r0 + beqi us %r1 us12_o + calli @abort +us: + + movi %r0 xus12_i + bswapr_us %r1 %r0 + beqi xus %r1 us12_o + calli @abort +xus: + movi %r0 ui12_i + bswapr_ui %r1 %r0 + beqi ui %r1 ui12_o + calli @abort +ui: +#if __WORDSIZE == 64 + movi %r0 xui12_i + bswapr_ui %r1 %r0 + beqi xui %r1 ui12_o + calli @abort +xui: + movi %r0 ul12_i + bswapr_ul %r1 %r0 + beqi ul %r1 ul12_o + calli @abort +ul: +#endif + + BSWAP(v0, v1, v2, r0, r1, r2) + + // just to know did not abort + prepare + pushargi ok + ellipsis + finishi @printf + + ret + epilog diff --git a/parallel-rsp/lightning/check/call.tst b/parallel-rsp/lightning/check/call.tst index 21068b6..40fb041 100644 --- a/parallel-rsp/lightning/check/call.tst +++ b/parallel-rsp/lightning/check/call.tst @@ -1,10 +1,10 @@ -#define def_wi(i) \ +#define def_wi(i, ii) \ name _w##i \ _w##i: \ prolog \ - arg $arg##i \ + arg##ii $arg##i \ getarg##i %r0 $arg##i \ - retr %r0 \ + retr##i %r0 \ epilog #define def_wf(f) \ name _w##f \ @@ -15,11 +15,11 @@ _w##f: \ truncr##f %r0 %f0 \ retr %r0 \ epilog -#define def_fi(f, i) \ +#define def_fi(f, i, ii) \ name f##i \ f##i: \ prolog \ - arg $arg##i \ + arg##ii $arg##i \ getarg##i %r0 $arg##i \ extr##f %f0 %r0 \ retr##f %f0 \ @@ -52,33 +52,33 @@ bstr: .code jmpi main - def_wi(_c) - def_wi(_uc) - def_wi(_s) - def_wi(_us) + def_wi(_c, _c) + def_wi(_uc, _c) + def_wi(_s, _s) + def_wi(_us, _s) #if __WORDSIZE == 64 - def_wi(_i) - def_wi(_ui) + def_wi(_i, _i) + def_wi(_ui, _i) #endif def_wf(_f) def_wf(_d) - def_fi(_f, _c) - def_fi(_f, _uc) - def_fi(_f, _s) - def_fi(_f, _us) - def_fi(_f, _i) + def_fi(_f, _c, _c) + def_fi(_f, _uc, _c) + def_fi(_f, _s, _s) + def_fi(_f, _us, _s) + def_fi(_f, _i, _i) #if __WORDSIZE == 64 - def_fi(_f, _ui) - def_fi(_f, _l) + def_fi(_f, _ui, _i) + def_fi(_f, _l, _l) #endif - def_fi(_d, _c) - def_fi(_d, _uc) - def_fi(_d, _s) - def_fi(_d, _us) - def_fi(_d, _i) + def_fi(_d, _c, _c) + def_fi(_d, _uc, _c) + def_fi(_d, _s, _s) + def_fi(_d, _us, _s) + def_fi(_d, _i, _i) #if __WORDSIZE == 64 - def_fi(_d, _ui) - def_fi(_d, _l) + def_fi(_d, _ui, _i) + def_fi(_d, _l, _l) #endif def_f(_f) def_f(_d) @@ -91,7 +91,7 @@ main: #define _call_w(n, i, a, r) \ prepare \ - pushargi a \ + pushargi##i a \ finishi _w##i \ retval %r0 \ extr##i %r0 %r0 \ @@ -111,7 +111,7 @@ _w##f##_##n: #define call_wf(n, f, a, r) _call_wf(n, f, a, r) #define _call_fi(n, f, i, a, r) \ prepare \ - pushargi a \ + pushargi##i a \ finishi f##i \ retval##f %f0 \ beqi##f f##i##n %f0 r \ @@ -196,6 +196,7 @@ f##g##n: call_wf(__LINE__, _d, c7f, f7f) call_wf(__LINE__, _d, wc80, f80) call_wf(__LINE__, _d, wc81, f81) + call_fi(__LINE__, _f, _c, c7f, f7f) call_fi(__LINE__, _f, _c, c80, f80) call_fi(__LINE__, _f, _uc, c7f, f7f) diff --git a/parallel-rsp/lightning/check/carg.c b/parallel-rsp/lightning/check/carg.c index 35b897e..6992db4 100644 --- a/parallel-rsp/lightning/check/carg.c +++ b/parallel-rsp/lightning/check/carg.c @@ -58,8 +58,8 @@ int main(int argc, char *argv[]) { void (*code)(void); - jit_node_t *jmp, *pass; - jit_node_t *jw, *jf, *jd; + jit_node_t *jmp, *pass, *fail; + jit_node_t *jw, *jf, *jd; jit_int32_t s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15, s16; jit_node_t *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, @@ -172,10 +172,11 @@ main(int argc, char *argv[]) LOAD_ARG(16); #undef LOAD_ARG pass = jit_forward(); + fail = jit_forward(); #define CHECK_ARG(N) \ do { \ jit_getarg(JIT_R0, a##N); \ - jit_patch_at(jit_beqi(JIT_R0, 17 - N), pass); \ + jit_patch_at(jit_bnei(JIT_R0, 17 - N), fail); \ } while (0) CHECK_ARG(1); CHECK_ARG(2); @@ -194,6 +195,8 @@ main(int argc, char *argv[]) CHECK_ARG(15); CHECK_ARG(16); #undef CHECK_ARG + jit_patch_at(jit_jmpi(), pass); + jit_link(fail); jit_calli(abort); jit_link(pass); jit_ret(); @@ -300,10 +303,11 @@ main(int argc, char *argv[]) LOAD_ARG(16); #undef LOAD_ARG pass = jit_forward(); + fail = jit_forward(); #define CHECK_ARG(N) \ do { \ jit_getarg_f(JIT_F0, a##N); \ - jit_patch_at(jit_beqi_f(JIT_F0, 17 - N), pass); \ + jit_patch_at(jit_bnei_f(JIT_F0, 17 - N), fail); \ } while (0) CHECK_ARG(1); CHECK_ARG(2); @@ -322,6 +326,8 @@ main(int argc, char *argv[]) CHECK_ARG(15); CHECK_ARG(16); #undef CHECK_ARG + jit_patch_at(jit_jmpi(), pass); + jit_link(fail); jit_calli(abort); jit_link(pass); jit_ret(); @@ -428,10 +434,11 @@ main(int argc, char *argv[]) LOAD_ARG(16); #undef LOAD_ARG pass = jit_forward(); + fail = jit_forward(); #define CHECK_ARG(N) \ do { \ jit_getarg_d(JIT_F0, a##N); \ - jit_patch_at(jit_beqi_d(JIT_F0, 17 - N), pass); \ + jit_patch_at(jit_bnei_d(JIT_F0, 17 - N), fail); \ } while (0) CHECK_ARG(1); CHECK_ARG(2); @@ -450,6 +457,8 @@ main(int argc, char *argv[]) CHECK_ARG(15); CHECK_ARG(16); #undef CHECK_ARG + jit_patch_at(jit_jmpi(), pass); + jit_link(fail); jit_calli(abort); jit_link(pass); jit_ret(); @@ -484,6 +493,7 @@ main(int argc, char *argv[]) jit_pushargi(1); } jit_patch_at(jit_finishi(NULL), jw); + jit_prepare(); { jit_pushargi_f(16); @@ -504,6 +514,7 @@ main(int argc, char *argv[]) jit_pushargi_f(1); } jit_patch_at(jit_finishi(NULL), jf); + jit_prepare(); { jit_pushargi_d(16); diff --git a/parallel-rsp/lightning/check/catomic.c b/parallel-rsp/lightning/check/catomic.c new file mode 100644 index 0000000..e1e2ea7 --- /dev/null +++ b/parallel-rsp/lightning/check/catomic.c @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include + +#if DEBUG +volatile +#endif +jit_word_t lock; +pthread_t tids[4]; + +#if DEBUG +int debug_offset(void) +{ + int i; + pthread_t self = pthread_self(); + for (i = 0; i < 4; ++i) + if (tids[i] == self) + return i; + return -1; +} + +void debug_spin(void) +{ + printf(" spin %d : %ld\n", debug_offset(), lock); +} + +void debug_lock(void) +{ + printf(" lock %d : %ld\n", debug_offset(), lock); +} + +void debug_unlock(void) +{ + printf("unlock %d : %ld\n", debug_offset(), lock); +} +#define DEBUG_SPIN() jit_calli(debug_spin) +#define DEBUG_LOCK() jit_calli(debug_lock) +#define DEBUG_UNLOCK() jit_calli(debug_unlock) +#else +#define DEBUG_SPIN() /**/ +#define DEBUG_LOCK() /**/ +#define DEBUG_UNLOCK() /**/ +#endif + +void alarm_handler(int unused) +{ + _exit(1); +} + +int +main(int argc, char *argv[]) +{ + jit_state_t *_jit; + void (*code)(void); + jit_node_t *jmpi_main, *label; + jit_node_t *func0, *func1, *func2, *func3; + jit_node_t *patch0, *patch1, *patch2, *patch3; + + /* If there is any bug, do not hang in "make check" */ + signal(SIGALRM, alarm_handler); + alarm(5); + + init_jit(argv[0]); + _jit = jit_new_state(); + + jmpi_main = jit_jmpi(); + +#define defun(name, line) \ + jit_name(#name); \ + jit_note("catomic.c", line); \ + name = jit_label(); \ + jit_prolog(); \ + jit_movi(JIT_V0, (jit_word_t)&lock); \ + jit_movi(JIT_V1, 0); \ + jit_movi(JIT_V2, line); \ + /* spin until get the lock */ \ + DEBUG_SPIN(); \ + label = jit_label(); \ + jit_casr(JIT_R0, JIT_V0, JIT_V1, JIT_V2); \ + jit_patch_at(jit_beqi(JIT_R0, 0), label); \ + /* lock acquired */ \ + DEBUG_LOCK(); \ + jit_prepare(); \ + /* pretend to be doing something useful for 0.01 sec + * while holding the lock */ \ + jit_pushargi(10000); \ + jit_finishi(usleep); \ + /* release lock */ \ + DEBUG_UNLOCK(); \ + jit_movi(JIT_V1, 0); \ + jit_str(JIT_V0, JIT_V1); \ + /* Now test casi */ \ + jit_movi(JIT_V1, 0); \ + jit_movi(JIT_V2, line); \ + /* spin until get the lock */ \ + DEBUG_SPIN(); \ + label = jit_label(); \ + jit_casi(JIT_R0, (jit_word_t)&lock, JIT_V1, JIT_V2); \ + jit_patch_at(jit_beqi(JIT_R0, 0), label); \ + /* lock acquired */ \ + DEBUG_LOCK(); \ + jit_prepare(); \ + /* pretend to be doing something useful for 0.01 sec + * while holding the lock */ \ + jit_pushargi(10000); \ + jit_finishi(usleep); \ + jit_prepare(); \ + /* for make check, just print "ok" */ \ + jit_pushargi((jit_word_t)"ok"); \ + /*jit_pushargi((jit_word_t)#name);*/ \ + jit_finishi(puts); \ + /* release lock */ \ + DEBUG_UNLOCK(); \ + jit_movi(JIT_V1, 0); \ + jit_str(JIT_V0, JIT_V1); \ + jit_ret(); \ + jit_epilog(); + defun(func0, __LINE__); + defun(func1, __LINE__); + defun(func2, __LINE__); + defun(func3, __LINE__); + + jit_patch(jmpi_main); + jit_name("main"); + jit_note("catomic.c", __LINE__); + jit_prolog(); + +#define start(tid) \ + /* set JIT_R0 to thread function */ \ + jit_patch_at(jit_movi(JIT_R0, 0), func##tid); \ + jit_prepare(); \ + /* pthread_t first argument */ \ + jit_pushargi((jit_word_t)(tids + tid)); \ + /* pthread_attr_t second argument */ \ + jit_pushargi((jit_word_t)NULL); \ + /* start routine third argument */ \ + jit_pushargr(JIT_R0); \ + /* argument to start routine fourth argument */ \ + jit_pushargi((jit_word_t)NULL); \ + /* start thread */ \ + jit_finishi(pthread_create); + /* spawn four threads */ + start(0); + start(1); + start(2); + start(3); + +#define join(tid) \ + /* load pthread_t value in JIT_R0 */ \ + jit_movi(JIT_R0, (jit_word_t)tids); \ + if (__WORDSIZE == 64 && sizeof(pthread_t) == 4) \ + jit_ldxi_i(JIT_R0, JIT_R0, tid * sizeof(pthread_t)); \ + else \ + jit_ldxi(JIT_R0, JIT_R0, tid * sizeof(pthread_t)); \ + jit_prepare(); \ + jit_pushargr(JIT_R0); \ + jit_pushargi((jit_word_t)NULL); \ + jit_finishi(pthread_join); + /* wait for threads to finish */ + join(0); + join(1); + join(2); + join(3); + + jit_prepare(); + jit_pushargi((jit_word_t)"ok"); + jit_finishi(puts); + + jit_ret(); + jit_epilog(); + + code = jit_emit(); + +#if DEBUG + jit_disassemble(); +#endif + + jit_clear_state(); + + /* let first thread acquire the lock */ + lock = 0; + + (*code)(); + jit_destroy_state(); + + finish_jit(); + + return (0); +} diff --git a/parallel-rsp/lightning/check/catomic.ok b/parallel-rsp/lightning/check/catomic.ok new file mode 100644 index 0000000..b130552 --- /dev/null +++ b/parallel-rsp/lightning/check/catomic.ok @@ -0,0 +1,5 @@ +ok +ok +ok +ok +ok diff --git a/parallel-rsp/lightning/check/ccall.c b/parallel-rsp/lightning/check/ccall.c index 9dae256..3491f2e 100644 --- a/parallel-rsp/lightning/check/ccall.c +++ b/parallel-rsp/lightning/check/ccall.c @@ -132,6 +132,16 @@ # define _l15 _w15 #endif +#ifndef jit_arg_uc +# define jit_arg_uc jit_arg_c +#endif +#ifndef jit_arg_us +# define jit_arg_us jit_arg_s +#endif +#ifndef jit_arg_ui +# define jit_arg_ui jit_arg_i +#endif + /* * Types */ @@ -624,7 +634,7 @@ main(int argc, char *argv[]) #define arg15(T) arg14(T) a15 = jit_arg##T(); #define get0(B,T,R) jit_movi##B(R##0,0); -#define get1(B,T,R) jit_getarg##B(R##0,a##1); +#define get1(B,T,R) jit_getarg##T(R##0,a##1); #define get2(B,T,R) \ get1(B,T,R); \ jit_movr##B(R##1, R##0); \ @@ -707,7 +717,7 @@ main(int argc, char *argv[]) n##T##N = jit_name(strfy(n##T##N)); \ jit_note("ccall.c", __LINE__); \ jit_prolog(); \ - arg##N(); \ + arg##N(T); \ get##N(,T,JIT_R) \ jit_extr##T(JIT_R0, JIT_R0); \ jit_retr(JIT_R0); \ @@ -777,7 +787,7 @@ main(int argc, char *argv[]) #define calin(T,N) \ jit_prepare(); \ - push##N() \ + push##N(T) \ jit_finishi(C##T##N); \ jit_retval##T(JIT_R0); \ jmp = jit_beqi(JIT_R0, T##N); \ @@ -826,7 +836,7 @@ main(int argc, char *argv[]) #undef calfn #define calin(T,N) \ jit_prepare(); \ - push##N() \ + push##N(T) \ jit_finishi(CJ##T##N); \ jit_retval##T(JIT_R0); \ jmp = jit_beqi(JIT_R0, T##N); \ diff --git a/parallel-rsp/lightning/check/collatz.tst b/parallel-rsp/lightning/check/collatz.tst new file mode 100644 index 0000000..85c4d41 --- /dev/null +++ b/parallel-rsp/lightning/check/collatz.tst @@ -0,0 +1,79 @@ +.data 32 +str: +.c "%lu\n" +.code + jmpi main +/* + * unsigned long collatz(unsigned long n) { + * unsigned long r = n; + * if (!(r & 1)) { + * r = r / 2; + * return r; + * } + * r = r * 3; + * r = r + 1; + * return r; + * } + */ +collatz: + prolog + arg $n + getarg %r0 $n + bmsi odd %r0 1 + //divi_u %r0 %r0 2 + rshi_u %r0 %r0 1 + retr %r0 +odd: + muli %r0 %r0 3 + addi %r0 %r0 1 + retr %r0 + epilog + +/* + * int main(int argc, char *argv[]) { + * unsigned long v; + * if (argc == 2) + * v = strtoul(argv[1], NULL, 0); + * else + * v = (1L << __WORDSIZE / 2) - 1; + * while (1) { + * printf("%ld\n", v); + * if (v <= 1) + * break; + * v = collatz(v); + * } + * return 0; + * } + */ +main: + prolog + arg $argc + arg $argv + getarg %r0 $argc + bnei default %r0 2 + getarg %v0 $argv + ldxi %r0 %v0 $(__WORDSIZE >> 3) + prepare + pushargr %r0 + pushargi 0 + pushargi 0 + finishi @strtoul + retval %v0 + jmpi loop +default: + movi %v0 $((1 << __WORDSIZE / 2) - 1) +loop: + prepare + pushargi str + ellipsis + pushargr %v0 + finishi @printf + blei_u done %v0 1 + prepare + pushargr %v0 + finishi collatz + retval %v0 + jmpi loop +done: + reti 0 + epilog diff --git a/parallel-rsp/lightning/check/factorial.tst b/parallel-rsp/lightning/check/factorial.tst new file mode 100644 index 0000000..68adbb2 --- /dev/null +++ b/parallel-rsp/lightning/check/factorial.tst @@ -0,0 +1,73 @@ +.data 32 +str: +.c "%.0lf\n" +.code + jmpi main +/* + * double factorial(unsigned long n) { + * double r = 1; + * while (n > 1) { + * r *= n; + * --n; + * } + * return r; + * } + */ +factorial: + prolog + arg $n + getarg %r0 $n + movi_d %f0 1.0 + extr_d %f1 %r0 + movr_d %f2 %f0 +loop: + bltr_d done %f1 %f2 + mulr_d %f0 %f0 %f1 + subr_d %f1 %f1 %f2 + jmpi loop +done: + retr_d %f0 + epilog + +/* + * int main(int argc, char *argv[]) { + * unsigned long v; + * double d; + * if (argc == 2) + * v = strtoul(argv[1], NULL, 0); + * else + * v = 32; + * d = factorial(v); + * printf("%.0lf\n", d); + * return 0; + * } + */ +main: + prolog + arg $argc + arg $argv + getarg %r0 $argc + bnei default %r0 2 + getarg %v0 $argv + ldxi %r0 %v0 $(__WORDSIZE >> 3) + prepare + pushargr %r0 + pushargi 0 + pushargi 0 + finishi @strtoul + retval %v0 + jmpi call +default: + movi %v0 32 +call: + prepare + pushargr %v0 + finishi factorial + retval_d %f0 + prepare + pushargi str + ellipsis + pushargr_d %f0 + finishi @printf + reti 0 + epilog diff --git a/parallel-rsp/lightning/check/fib.tst b/parallel-rsp/lightning/check/fib.tst index 0835323..926ee81 100644 --- a/parallel-rsp/lightning/check/fib.tst +++ b/parallel-rsp/lightning/check/fib.tst @@ -32,7 +32,7 @@ main: arg $argc arg $argv - getarg_i %r0 $argc + getarg %r0 $argc blei default %r0 1 getarg %r0 $argv addi %r0 %r0 $(__WORDSIZE >> 3) diff --git a/parallel-rsp/lightning/check/float.tst b/parallel-rsp/lightning/check/float.tst index ff5606a..05a0889 100644 --- a/parallel-rsp/lightning/check/float.tst +++ b/parallel-rsp/lightning/check/float.tst @@ -16,12 +16,12 @@ ok: #if __mips__ || __sparc__ || __hppa__ || __riscv # define wnan x7f -#elif __arm__ || __aarch64__ || __alpha__ +#elif __arm__ || __aarch64__ || __alpha__ || __loongarch__ # define wnan 0 #else # define wnan x80 #endif -#if __mips__ || __arm__ || __ppc__ || __sparc__ || __hppa__ || __aarch64__ || __s390__ || __riscv +#if __mips__ || __arm__ || __ppc__ || __sparc__ || __hppa__ || __aarch64__ || __s390__ || __riscv || __loongarch__ # define wpinf x7f #elif __alpha__ /* (at least) bug compatible with gcc 4.2.3 -ieee */ diff --git a/parallel-rsp/lightning/check/lightning.c b/parallel-rsp/lightning/check/lightning.c index e60ef05..d57fd73 100644 --- a/parallel-rsp/lightning/check/lightning.c +++ b/parallel-rsp/lightning/check/lightning.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2022 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -30,6 +30,7 @@ #include #include #include +#include #if defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) # include @@ -269,10 +270,16 @@ static jit_pointer_t get_arg(void); static jit_word_t get_imm(void); static void live(void); static void align(void); static void name(void); +static void skip(void); static void prolog(void); static void frame(void); static void tramp(void); static void ellipsis(void); static void allocai(void); static void allocar(void); +static void arg_c(void); static void arg_s(void); +static void arg_i(void); +#if __WORDSIZE == 64 +static void arg_l(void); +#endif static void arg(void); static void getarg_c(void); static void getarg_uc(void); static void getarg_s(void); static void getarg_us(void); @@ -281,6 +288,15 @@ static void getarg_i(void); static void getarg_ui(void); static void getarg_l(void); #endif static void getarg(void); +static void putargr_c(void); static void putargi_c(void); +static void putargr_uc(void); static void putargi_uc(void); +static void putargr_s(void); static void putargi_s(void); +static void putargr_us(void); static void putargi_us(void); +static void putargr_i(void); static void putargi_i(void); +#if __WORDSIZE == 64 +static void putargr_ui(void); static void putargi_ui(void); +static void putargr_l(void); static void putargi_l(void); +#endif static void putargr(void); static void putargi(void); static void addr(void); static void addi(void); static void addxr(void); static void addxi(void); @@ -305,6 +321,8 @@ static void lshr(void); static void lshi(void); static void rshr(void); static void rshi(void); static void rshr_u(void); static void rshi_u(void); static void negr(void); static void comr(void); +static void clor(void); static void clzr(void); +static void ctor(void); static void ctzr(void); static void ltr(void); static void lti(void); static void ltr_u(void); static void lti_u(void); static void ler(void); static void lei(void); @@ -315,6 +333,7 @@ static void ger_u(void); static void gei_u(void); static void gtr(void); static void gti(void); static void gtr_u(void); static void gti_u(void); static void ner(void); static void nei(void); +static void casr(void); static void casi(void); static void movr(void); static void movi(void); static void extr_c(void); static void extr_uc(void); static void extr_s(void); static void extr_us(void); @@ -327,6 +346,12 @@ static void htonr_ui(void); static void ntohr_ui(void); static void htonr_ul(void); static void ntohr_ul(void); #endif static void htonr(void); static void ntohr(void); +static void bswapr_us(void); static void bswapr_ui(void); +#if __WORDSIZE == 64 +static void bswapr_ul(void); +#endif +static void bswapr(void); +static void movnr(void); static void movzr(void); static void ldr_c(void); static void ldi_c(void); static void ldr_uc(void); static void ldi_uc(void); static void ldr_s(void); static void ldi_s(void); @@ -384,9 +409,30 @@ static void bxsubr_u(void); static void bxsubi_u(void); static void jmpr(void); static void jmpi(void); static void callr(void); static void calli(void); static void prepare(void); + +static void pushargr_c(void); static void pushargi_c(void); +static void pushargr_uc(void); static void pushargi_uc(void); +static void pushargr_s(void); static void pushargi_s(void); +static void pushargr_us(void); static void pushargi_us(void); +static void pushargr_i(void); static void pushargi_i(void); +#if __WORDSIZE == 64 +static void pushargr_ui(void); static void pushargi_ui(void); +static void pushargr_l(void); static void pushargi_l(void); +#endif static void pushargr(void); static void pushargi(void); + static void finishr(void); static void finishi(void); static void ret(void); + +static void retr_c(void); static void reti_c(void); +static void retr_uc(void); static void reti_uc(void); +static void retr_s(void); static void reti_s(void); +static void retr_us(void); static void reti_us(void); +static void retr_i(void); static void reti_i(void); +#if __WORDSIZE == 64 +static void retr_ui(void); static void reti_ui(void); +static void retr_l(void); static void reti_l(void); +#endif static void retr(void); static void reti(void); static void retval_c(void); static void retval_uc(void); static void retval_s(void); static void retval_us(void); @@ -583,10 +629,16 @@ static instr_t instr_vector[] = { #define entry2(name, function) { NULL, name, function } entry(live), entry(align), entry(name), + entry(skip), entry(prolog), entry(frame), entry(tramp), entry(ellipsis), entry(allocai), entry(allocar), + entry(arg_c), entry(arg_s), + entry(arg_i), +#if __WORDSIZE == 64 + entry(arg_l), +#endif entry(arg), entry(getarg_c), entry(getarg_uc), entry(getarg_s), entry(getarg_us), @@ -595,6 +647,16 @@ static instr_t instr_vector[] = { entry(getarg_ui), entry(getarg_l), #endif entry(getarg), + + entry(putargr_c), entry(putargi_c), + entry(putargr_uc), entry(putargi_uc), + entry(putargr_s), entry(putargi_s), + entry(putargr_us), entry(putargi_us), + entry(putargr_i), entry(putargi_i), +#if __WORDSIZE == 64 + entry(putargr_ui), entry(putargi_ui), + entry(putargr_l), entry(putargi_l), +#endif entry(putargr), entry(putargi), entry(addr), entry(addi), entry(addxr), entry(addxi), @@ -619,6 +681,8 @@ static instr_t instr_vector[] = { entry(rshr), entry(rshi), entry(rshr_u), entry(rshi_u), entry(negr), entry(comr), + entry(clor), entry(clzr), + entry(ctor), entry(ctzr), entry(ltr), entry(lti), entry(ltr_u), entry(lti_u), entry(ler), entry(lei), @@ -629,6 +693,7 @@ static instr_t instr_vector[] = { entry(gtr), entry(gti), entry(gtr_u), entry(gti_u), entry(ner), entry(nei), + entry(casr), entry(casi), entry(movr), entry(movi), entry(extr_c), entry(extr_uc), entry(extr_s), entry(extr_us), @@ -641,6 +706,12 @@ static instr_t instr_vector[] = { entry(htonr_ul), entry(ntohr_ul), #endif entry(htonr), entry(ntohr), + entry(bswapr_us), entry(bswapr_ui), +#if __WORDSIZE == 64 + entry(bswapr_ul), +#endif + entry(bswapr), + entry(movnr), entry(movzr), entry(ldr_c), entry(ldi_c), entry(ldr_uc), entry(ldi_uc), entry(ldr_s), entry(ldi_s), @@ -698,9 +769,27 @@ static instr_t instr_vector[] = { entry(jmpr), entry(jmpi), entry(callr), entry(calli), entry(prepare), + entry(pushargr_c), entry(pushargi_c), + entry(pushargr_uc), entry(pushargi_uc), + entry(pushargr_s), entry(pushargi_s), + entry(pushargr_us), entry(pushargi_us), + entry(pushargr_i), entry(pushargi_i), +#if __WORDSIZE == 64 + entry(pushargr_ui), entry(pushargi_ui), + entry(pushargr_l), entry(pushargi_l), +#endif entry(pushargr), entry(pushargi), entry(finishr), entry(finishi), entry(ret), + entry(retr_c), entry(reti_c), + entry(retr_uc), entry(reti_uc), + entry(retr_s), entry(reti_s), + entry(retr_us), entry(reti_us), + entry(retr_i), entry(reti_i), +#if __WORDSIZE == 64 + entry(retr_ui), entry(reti_ui), + entry(retr_l), entry(reti_l), +#endif entry(retr), entry(reti), entry(retval_c), entry(retval_uc), entry(retval_s), entry(retval_us), @@ -1015,6 +1104,16 @@ name(void) \ jit_word_t im = get_imm(); \ jit_##name(r0, r1, r2, im); \ } +#define entry_ir_im_ir_ir(name) \ +static void \ +name(void) \ +{ \ + jit_gpr_t r0 = get_ireg(); \ + jit_word_t im = get_imm(); \ + jit_gpr_t r1 = get_ireg(), r2 = get_ireg(); \ + jit_##name(r0, im, r1, r2); \ +} + #define entry_ir_ir(name) \ static void \ name(void) \ @@ -1375,6 +1474,7 @@ live(void) { jit_live(parser.regval); } entry_im(align) +entry_im(skip) entry(prolog) entry_im(frame) entry_im(tramp) entry(ellipsis) @@ -1388,6 +1488,11 @@ allocai(void) { symbol->value.i = i; } entry_ir_ir(allocar) +entry_ca(arg_c) entry_ca(arg_s) +entry_ca(arg_i) +#if __WORDSIZE == 64 +entry_ca(arg_l) +#endif entry_ca(arg) entry_ia(getarg_c) entry_ia(getarg_uc) entry_ia(getarg_s) entry_ia(getarg_us) @@ -1396,6 +1501,15 @@ entry_ia(getarg_i) entry_ia(getarg_ui) entry_ia(getarg_l) #endif entry_ia(getarg) +entry_ia(putargr_c) entry_ima(putargi_c) +entry_ia(putargr_uc) entry_ima(putargi_uc) +entry_ia(putargr_s) entry_ima(putargi_s) +entry_ia(putargr_us) entry_ima(putargi_us) +entry_ia(putargr_i) entry_ima(putargi_i) +#if __WORDSIZE == 64 +entry_ia(putargr_ui) entry_ima(putargi_ui) +entry_ia(putargr_l) entry_ima(putargi_l) +#endif entry_ia(putargr) entry_ima(putargi) entry_ir_ir_ir(addr) entry_ir_ir_im(addi) entry_ir_ir_ir(addxr) entry_ir_ir_im(addxi) @@ -1420,6 +1534,8 @@ entry_ir_ir_ir(lshr) entry_ir_ir_im(lshi) entry_ir_ir_ir(rshr) entry_ir_ir_im(rshi) entry_ir_ir_ir(rshr_u) entry_ir_ir_im(rshi_u) entry_ir_ir(negr) entry_ir_ir(comr) +entry_ir_ir(clor) entry_ir_ir(clzr) +entry_ir_ir(ctor) entry_ir_ir(ctzr) entry_ir_ir_ir(ltr) entry_ir_ir_im(lti) entry_ir_ir_ir(ltr_u) entry_ir_ir_im(lti_u) entry_ir_ir_ir(ler) entry_ir_ir_im(lei) @@ -1430,6 +1546,7 @@ entry_ir_ir_ir(ger_u) entry_ir_ir_im(gei_u) entry_ir_ir_ir(gtr) entry_ir_ir_im(gti) entry_ir_ir_ir(gtr_u) entry_ir_ir_im(gti_u) entry_ir_ir_ir(ner) entry_ir_ir_im(nei) +entry_ir_ir_ir_ir(casr) entry_ir_im_ir_ir(casi) entry_ir_ir(movr) static void movi(void) @@ -1489,6 +1606,12 @@ entry_ir_ir(htonr_ui) entry_ir_ir(ntohr_ui) entry_ir_ir(htonr_ul) entry_ir_ir(ntohr_ul) #endif entry_ir_ir(htonr) entry_ir_ir(ntohr) +entry_ir_ir(bswapr_us) entry_ir_ir(bswapr_ui) +#if __WORDSIZE == 64 +entry_ir_ir(bswapr_ul) +#endif +entry_ir_ir(bswapr) +entry_ir_ir_ir(movnr) entry_ir_ir_ir(movzr) entry_ir_ir(ldr_c) entry_ir_pm(ldi_c) entry_ir_ir(ldr_uc) entry_ir_pm(ldi_uc) entry_ir_ir(ldr_s) entry_ir_pm(ldi_s) @@ -1546,9 +1669,27 @@ entry_lb_ir_ir(bxsubr_u) entry_lb_ir_im(bxsubi_u) entry_ir(jmpr) entry_lb(jmpi) entry_ir(callr) entry_fn(calli) entry(prepare) +entry_ir(pushargr_c) entry_im(pushargi_c) +entry_ir(pushargr_uc) entry_im(pushargi_uc) +entry_ir(pushargr_s) entry_im(pushargi_s) +entry_ir(pushargr_us) entry_im(pushargi_us) +entry_ir(pushargr_i) entry_im(pushargi_i) +#if __WORDSIZE == 64 +entry_ir(pushargr_ui) entry_im(pushargi_ui) +entry_ir(pushargr_l) entry_im(pushargi_l) +#endif entry_ir(pushargr) entry_im(pushargi) entry_ir(finishr) entry_fn(finishi) entry(ret) +entry_ir(retr_c) entry_im(reti_c) +entry_ir(retr_uc) entry_im(reti_uc) +entry_ir(retr_s) entry_im(reti_s) +entry_ir(retr_us) entry_im(reti_us) +entry_ir(retr_i) entry_im(reti_i) +#if __WORDSIZE == 64 +entry_ir(retr_ui) entry_im(reti_ui) +entry_ir(retr_l) entry_im(reti_l) +#endif entry_ir(retr) entry_im(reti) entry_ir(retval_c) entry_ir(retval_uc) entry_ir(retval_s) entry_ir(retval_us) @@ -3791,11 +3932,16 @@ execute(int argc, char *argv[]) function = jit_emit(); if (flag_verbose > 1 || flag_disasm) { jit_print(); - fprintf(stdout, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); + fprintf(stderr, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); } if (flag_verbose > 0 || flag_disasm) { jit_disassemble(); - fprintf(stdout, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); + fprintf(stderr, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); + } + if (flag_verbose && argc) { + for (result = 0; result < argc; result++) + printf("argv[%d] = %s\n", result, argv[result]); + fprintf(stderr, " - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n"); } jit_clear_state(); @@ -3996,12 +4142,11 @@ usage(void) { #if HAVE_GETOPT_LONG_ONLY fprintf(stderr, "\ -Usage: %s [jit assembler options] file [jit program options]\n\ +Usage: %s [jit assembler options] file [--] [jit program options]\n\ Jit assembler options:\n\ -help Display this information\n\ -v[0-3] Verbose output level\n\ - -d Do not use a data buffer\n\ - -D[=] Preprocessor options\n" + -d Do not use a data buffer\n" # if defined(__i386__) && __WORDSIZE == 32 " -mx87=1 Force using x87 when sse2 available\n" # endif @@ -4017,11 +4162,10 @@ Jit assembler options:\n\ , progname); #else fprintf(stderr, "\ -Usage: %s [jit assembler options] file [jit program options]\n\ +Usage: %s [jit assembler options] file [--] [jit program options]\n\ Jit assembler options:\n\ -h Display this information\n\ - -v Verbose output level\n\ - -D[=] Preprocessor options\n", progname); + -v Verbose output level\n", progname); #endif finish_jit(); exit(1); @@ -4196,16 +4340,6 @@ main(int argc, char *argv[]) # define cc "gcc" #endif opt_short = snprintf(cmdline, sizeof(cmdline), cc " -E -x c %s", argv[opt_index]); - for (++opt_index; opt_index < argc; opt_index++) { - if (argv[opt_index][0] == '-') - opt_short += snprintf(cmdline + opt_short, - sizeof(cmdline) - opt_short, - " %s", argv[opt_index]); - else { - --opt_index; - break; - } - } opt_short += snprintf(cmdline + opt_short, sizeof(cmdline) - opt_short, " -D__WORDSIZE=%d", __WORDSIZE); @@ -4282,6 +4416,11 @@ main(int argc, char *argv[]) opt_short += snprintf(cmdline + opt_short, sizeof(cmdline) - opt_short, " -D__alpha__=1"); +#endif +#if defined(__loongarch__) + opt_short += snprintf(cmdline + opt_short, + sizeof(cmdline) - opt_short, + " -D__loongarch__=1"); #endif if ((parser.fp = popen(cmdline, "r")) == NULL) error("cannot execute %s", cmdline); diff --git a/parallel-rsp/lightning/check/live.ok b/parallel-rsp/lightning/check/live.ok new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/parallel-rsp/lightning/check/live.ok @@ -0,0 +1 @@ +ok diff --git a/parallel-rsp/lightning/check/live.tst b/parallel-rsp/lightning/check/live.tst new file mode 100644 index 0000000..f082a66 --- /dev/null +++ b/parallel-rsp/lightning/check/live.tst @@ -0,0 +1,33 @@ +.data 16 +ok: +.c "ok" + +.code + jmpi main + +check_r0: + prolog + movi %v0 exit_r0 + movi %r0 1 + movi %r2 10 + // on x86 this changes %rax on other arches could use %r0 as temporary + divi %r1 %r2 3 + live %r0 + // %r0 must still be 1 + jmpr %v0 +exit_r0: + retr %r0 + epilog + +main: + prolog + calli check_r0 + retval %r1 + beqi r0_ok %r1 1 + calli @abort +r0_ok: + prepare + pushargi ok + finishi @puts + ret + epilog diff --git a/parallel-rsp/lightning/check/movzr.ok b/parallel-rsp/lightning/check/movzr.ok new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/parallel-rsp/lightning/check/movzr.ok @@ -0,0 +1 @@ +ok diff --git a/parallel-rsp/lightning/check/movzr.tst b/parallel-rsp/lightning/check/movzr.tst new file mode 100644 index 0000000..baa3ff8 --- /dev/null +++ b/parallel-rsp/lightning/check/movzr.tst @@ -0,0 +1,62 @@ +.data 8 +ok: +.c "ok\n" + +#define CMOVR(N, T, OP, I0, I1, V, R0, R1, R2) \ + movi %R1 I0 \ + movi %R2 I1 \ + movi %R0 V \ + OP##r##T %R0 %R1 %R2 \ + beqi OP##T##N##r_##R0##R1##R2 %R0 V \ + calli @abort \ +OP##T##N##r_##R0##R1##R2: + +/* reg0 = reg1 op reg0 */ +#define CMOVR1(N, T, OP, I0, I1, V, R0, R1, R2) \ + movi %R0 I1 \ + movi %R1 I0 \ + movi %R2 V \ + OP##r##T %R0 %R1 %R0 \ + beqr OP##T##N##r_1##R0##R1##R2 %R0 %R2 \ + calli @abort \ +OP##T##N##r_1##R0##R1##R2: + +#define TEST_CMOV1(N, OP, I0, I1, V, R0, R1, R2) \ + CMOVR(N, , OP, I0, I1, V, R0, R1, R2) \ + CMOVR1(N, , OP, I0, I1, V, R0, R1, R2) \ + +#define TEST_CMOV(N, OP, I0, I1, V) \ + TEST_CMOV1(N, OP, I0, I1, V, v0, v1, v2) \ + TEST_CMOV1(N, OP, I0, I1, V, v0, v1, r0) \ + TEST_CMOV1(N, OP, I0, I1, V, v0, v1, r1) \ + TEST_CMOV1(N, OP, I0, I1, V, v0, v1, r2) \ + TEST_CMOV1(N, OP, I0, I1, V, v1, v2, r1) \ + TEST_CMOV1(N, OP, I0, I1, V, v1, v2, r2) \ + TEST_CMOV1(N, OP, I0, I1, V, v2, r0, r1) \ + TEST_CMOV1(N, OP, I0, I1, V, v2, r0, r2) \ + TEST_CMOV1(N, OP, I0, I1, V, r0, r1, r2) + +#define MOVZR(N, I0, I1, V) TEST_CMOV(N, movz, I0, I1, V) +#define MOVNR(N, I0, I1, V) TEST_CMOV(N, movn, I0, I1, V) + +.code + prolog + + MOVZR(0, 0x0, 0x0, 0x0) + MOVZR(1, 0xf7de, 0x0, 0xf7de) + + MOVZR(2, 0x0, 0xdead, 0xdead) + MOVZR(3, 0xf7de, 0xdead, 0xdead) + + MOVNR(0, 0x0, 0x0, 0x0) + MOVNR(1, 0xf7de, 0x0, 0x0) + + MOVNR(2, 0x0, 0xdead, 0x0) + MOVNR(3, 0xf7de, 0xdead, 0xf7de) + + prepare + pushargi ok + ellipsis + finishi @printf + ret + epilog diff --git a/parallel-rsp/lightning/check/protect.c b/parallel-rsp/lightning/check/protect.c new file mode 100644 index 0000000..f5ec740 --- /dev/null +++ b/parallel-rsp/lightning/check/protect.c @@ -0,0 +1,59 @@ +/* + * Simple test of (un)protecting a code buffer. + */ + +#include +#include +#include + +#define MARKER 10 + +int +main(int argc, char *argv[]) +{ + jit_state_t *_jit; + jit_node_t *load, *label, *ok; + unsigned char *ptr; + void (*function)(void); + int mmap_prot, mmap_flags; + + init_jit(argv[0]); + _jit = jit_new_state(); + + jit_prolog(); + + load = jit_movi(JIT_R0, 0); + jit_ldr_c(JIT_R0, JIT_R0); + ok = jit_forward(); + jit_patch_at(jit_beqi(JIT_R0, MARKER), ok); + jit_prepare(); + jit_pushargi(1); + jit_finishi(exit); + label = jit_indirect(); + jit_skip(1); /* Reserves enough space for a byte. */ + jit_patch_at(load, label); + jit_link(ok); + jit_prepare(); + jit_pushargi((jit_word_t)"%s\n"); + jit_ellipsis(); + jit_pushargi((jit_word_t)"ok"); + jit_finishi(printf); + + function = jit_emit(); + if (function == NULL) + abort(); + + jit_unprotect (); + ptr = jit_address (label); + *ptr = MARKER; + jit_protect (); + + jit_clear_state(); + + (*function)(); + + jit_destroy_state(); + finish_jit(); + + return (0); +} diff --git a/parallel-rsp/lightning/check/put.tst b/parallel-rsp/lightning/check/put.tst index a7e39e1..65f1580 100644 --- a/parallel-rsp/lightning/check/put.tst +++ b/parallel-rsp/lightning/check/put.tst @@ -9,49 +9,49 @@ ok: putr: prolog frame 160 - arg $ac - arg $auc - arg $as - arg $aus - arg $ai + arg_c $ac + arg_c $auc + arg_s $as + arg_s $aus + arg_i $ai #if __WORDSIZE == 64 - arg $aui - arg $al + arg_i $aui + arg_l $al #endif arg_f $af arg_d $ad arg $a #if __WORDSIZE == 64 - arg $_l - arg $_ui + arg_l $_l + arg_i $_ui #endif - arg $_i - arg $_us - arg $_s - arg $_uc - arg $_c + arg_i $_i + arg_s $_us + arg_s $_s + arg_c $_uc + arg_c $_c getarg_c %r0 $ac negr %r0 %r0 - putargr %r0 $ac + putargr_c %r0 $ac getarg_uc %r0 $auc negr %r0 %r0 - putargr %r0 $auc + putargr_uc %r0 $auc getarg_s %r0 $as negr %r0 %r0 - putargr %r0 $as + putargr_s %r0 $as getarg_us %r0 $aus negr %r0 %r0 - putargr %r0 $aus + putargr_us %r0 $aus getarg_i %r0 $ai negr %r0 %r0 - putargr %r0 $ai + putargr_i %r0 $ai #if __WORDSIZE == 64 getarg_ui %r0 $aui negr %r0 %r0 - putargr %r0 $aui + putargr_ui %r0 $aui getarg_l %r0 $al negr %r0 %r0 - putargr %r0 $al + putargr_l %r0 $al #endif getarg_f %f0 $af negr_f %f0 %f0 @@ -65,49 +65,49 @@ putr: #if __WORDSIZE == 64 getarg_l %r0 $_l negr %r0 %r0 - putargr %r0 $_l + putargr_l %r0 $_l getarg_ui %r0 $_ui negr %r0 %r0 - putargr %r0 $_ui + putargr_ui %r0 $_ui #endif getarg_i %r0 $_i negr %r0 %r0 - putargr %r0 $_i + putargr_i %r0 $_i getarg_us %r0 $_us negr %r0 %r0 - putargr %r0 $_us + putargr_us %r0 $_us getarg_s %r0 $_s negr %r0 %r0 - putargr %r0 $_s + putargr_s %r0 $_s getarg_uc %r0 $_uc negr %r0 %r0 - putargr %r0 $_uc + putargr_uc %r0 $_uc getarg_c %r0 $_c negr %r0 %r0 - putargr %r0 $_c + putargr_c %r0 $_c jmpi _putr rputr: - putargi 17 $ac - putargi 16 $auc - putargi 15 $as - putargi 14 $aus - putargi 13 $ai + putargi_c 17 $ac + putargi_uc 16 $auc + putargi_s 15 $as + putargi_us 14 $aus + putargi_i 13 $ai #if __WORDSIZE == 64 - putargi 12 $aui - putargi 11 $al + putargi_ui 12 $aui + putargi_l 11 $al #endif putargi_f 10 $af putargi_d 9 $ad putargi 8 $a #if __WORDSIZE == 64 - putargi 7 $_l - putargi 6 $_ui + putargi_l 7 $_l + putargi_ui 6 $_ui #endif - putargi 5 $_i - putargi 4 $_us - putargi 3 $_s - putargi 2 $_uc - putargi 1 $_c + putargi_i 5 $_i + putargi_us 4 $_us + putargi_s 3 $_s + putargi_uc 2 $_uc + putargi_c 1 $_c jmpi _puti rputi: ret @@ -117,27 +117,27 @@ rputi: _putr: prolog tramp 160 - arg $ac - arg $auc - arg $as - arg $aus - arg $ai + arg_c $ac + arg_c $auc + arg_s $as + arg_s $aus + arg_i $ai #if __WORDSIZE == 64 - arg $aui - arg $al + arg_i $aui + arg_l $al #endif arg_f $af arg_d $ad arg $a #if __WORDSIZE == 64 - arg $_l - arg $_ui + arg_l $_l + arg_i $_ui #endif - arg $_i - arg $_us - arg $_s - arg $_uc - arg $_c + arg_i $_i + arg_s $_us + arg_s $_s + arg_c $_uc + arg_c $_c getarg_c %r0 $ac beqi rac %r0 -1 calli @abort @@ -181,7 +181,7 @@ rad: calli @abort ra: #if __WORDSIZE == 64 - getarg %r0 $_l + getarg_l %r0 $_l beqi r_l %r0 -11 calli @abort r_l: @@ -217,27 +217,27 @@ r_c: _puti: prolog tramp 160 - arg $ac - arg $auc - arg $as - arg $aus - arg $ai + arg_c $ac + arg_c $auc + arg_s $as + arg_s $aus + arg_i $ai #if __WORDSIZE == 64 - arg $aui - arg $al + arg_i $aui + arg_l $al #endif arg_f $af arg_d $ad arg $a #if __WORDSIZE == 64 - arg $_l - arg $_ui + arg_l $_l + arg_i $_ui #endif - arg $_i - arg $_us - arg $_s - arg $_uc - arg $_c + arg_i $_i + arg_s $_us + arg_s $_s + arg_c $_uc + arg_c $_c getarg_c %r0 $ac beqi iac %r0 17 calli @abort @@ -281,7 +281,7 @@ iad: calli @abort ia: #if __WORDSIZE == 64 - getarg %r0 $_l + getarg_l %r0 $_l beqi i_l %r0 7 calli @abort i_l: @@ -390,27 +390,27 @@ fd2: main: prolog prepare - pushargi 1 - pushargi 2 - pushargi 3 - pushargi 4 - pushargi 5 + pushargi_c 1 + pushargi_uc 2 + pushargi_s 3 + pushargi_us 4 + pushargi_i 5 #if __WORDSIZE == 64 - pushargi 6 - pushargi 7 + pushargi_ui 6 + pushargi_l 7 #endif - pushargi_f 8 - pushargi_d 9 - pushargi 10 + pushargi_f 8 + pushargi_d 9 + pushargi 10 #if __WORDSIZE == 64 - pushargi 11 - pushargi 12 + pushargi_l 11 + pushargi_ui 12 #endif - pushargi 13 - pushargi 14 - pushargi 15 - pushargi 16 - pushargi 17 + pushargi_i 13 + pushargi_us 14 + pushargi_s 15 + pushargi_uc 16 + pushargi_c 17 finishi putr prepare pushargi 1 diff --git a/parallel-rsp/lightning/check/riprel.c b/parallel-rsp/lightning/check/riprel.c new file mode 100644 index 0000000..c776e2a --- /dev/null +++ b/parallel-rsp/lightning/check/riprel.c @@ -0,0 +1,173 @@ +/* + * Simple test for x86_64 rip relative access that can also be useful + * on other ports when data is close to instruction pointer. + */ + +#include +#include +#include +#include +#if defined(__sgi) +# include +#endif +#include + +#ifndef MAP_ANON +# define MAP_ANON MAP_ANONYMOUS +# ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS 0 +# endif +#endif + +#if !defined(__sgi) +#define mmap_fd -1 +#endif + +int +main(int argc, char *argv[]) +{ + jit_uint8_t *ptr; + jit_state_t *_jit; + jit_word_t length; +#if defined(__sgi) + int mmap_fd; +#endif + void (*function)(void); + int mmap_prot, mmap_flags, result, pagesize; + int mult; + +#if defined(__ia64__) + mult = 8; +#else + mult = 2; +#endif + pagesize = sysconf(_SC_PAGESIZE); + if (pagesize < 4096) + pagesize = 4096; + +#if defined(__sgi) + mmap_fd = open("/dev/zero", O_RDWR); +#endif + + mmap_prot = PROT_READ | PROT_WRITE; +#if !(__OpenBSD__ || __APPLE__) + mmap_prot |= PROT_EXEC; +#endif +#if __NetBSD__ + mmap_prot = PROT_MPROTECT(mmap_prot); + mmap_flags = 0; +#else + mmap_flags = MAP_PRIVATE; +#endif + mmap_flags |= MAP_ANON; + ptr = mmap(NULL, pagesize * mult, mmap_prot, mmap_flags, mmap_fd, 0); + assert(ptr != MAP_FAILED); +#if defined(__sgi) + close(mmap_fd); +#endif + + init_jit(argv[0]); + _jit = jit_new_state(); + + jit_prolog(); + jit_movi(JIT_R0, 'c'); + jit_sti_c(ptr + 0, JIT_R0); + jit_movi(JIT_R0, 'C'); + jit_sti_c(ptr + 1, JIT_R0); + + jit_movi(JIT_R0, 's'); + jit_sti_s(ptr + 2, JIT_R0); + jit_movi(JIT_R0, 'S'); + jit_sti_s(ptr + 4, JIT_R0); + + jit_movi(JIT_R0, 'i'); + jit_sti_i(ptr + 8, JIT_R0); +#if __WORDSIZE == 64 + jit_movi(JIT_R0, 'I'); + jit_sti_i(ptr + 12, JIT_R0); + + jit_movi(JIT_R0, 'l'); + jit_sti_l(ptr + 16, JIT_R0); +#endif + jit_movi_f(JIT_F0, 1.0); + jit_sti_f(ptr + 24, JIT_F0); + jit_movi_d(JIT_F0, 2.0); + jit_sti_d(ptr + 32, JIT_F0); + + jit_ldi_c(JIT_R0, ptr + 0); + jit_ldi_s(JIT_R1, ptr + 2); + jit_ldi_i(JIT_R2, ptr + 8); +#if __WORDSIZE == 64 + jit_ldi_l(JIT_V0, ptr + 16); +#endif + jit_prepare(); +#if __WORDSIZE == 64 + jit_pushargi((jit_word_t)"%c %c %c %c\n"); +#else + jit_pushargi((jit_word_t)"%c %c %c l\n"); +#endif + jit_ellipsis(); + jit_pushargr(JIT_R0); + jit_pushargr(JIT_R1); + jit_pushargr(JIT_R2); +#if __WORDSIZE == 64 + jit_pushargr(JIT_V0); +#endif + jit_finishi(printf); + + jit_ldi_uc(JIT_R0, ptr + 1); + jit_ldi_us(JIT_R1, ptr + 4); +#if __WORDSIZE == 64 + jit_ldi_ui(JIT_R2, ptr + 12); +#endif + jit_prepare(); +#if __WORDSIZE == 64 + jit_pushargi((jit_word_t)"%c %c %c\n"); +#else + jit_pushargi((jit_word_t)"%c %c I\n"); +#endif + jit_ellipsis(); + jit_pushargr(JIT_R0); + jit_pushargr(JIT_R1); +#if __WORDSIZE == 64 + jit_pushargr(JIT_R2); +#endif + jit_finishi(printf); + + jit_ldi_f(JIT_F0, ptr + 24); + jit_extr_f_d(JIT_F0, JIT_F0); + jit_ldi_d(JIT_F1, ptr + 32); + + jit_prepare(); + jit_pushargi((jit_word_t)"%.1f %.1f\n"); + jit_ellipsis(); + jit_pushargr_d(JIT_F0); + jit_pushargr_d(JIT_F1); + jit_finishi(printf); + + jit_realize(); + + jit_set_code(ptr + pagesize, pagesize * (mult - 1)); + + #if __NetBSD__ || __OpenBSD__ || __APPLE__ + result = mprotect(ptr, pagesize, PROT_READ | PROT_WRITE); + assert(result == 0); +#endif + function = jit_emit(); + if (function == NULL) + abort(); + + //jit_disassemble(); + jit_clear_state(); +#if __NetBSD__ || __OpenBSD__ || __APPLE__ + result = mprotect(ptr + pagesize, pagesize, PROT_READ | PROT_EXEC); + assert(result == 0); +#endif + (*function)(); + jit_destroy_state(); + finish_jit(); + + munmap(ptr, pagesize * mult); + + return (0); +} diff --git a/parallel-rsp/lightning/check/riprel.ok b/parallel-rsp/lightning/check/riprel.ok new file mode 100644 index 0000000..4b90837 --- /dev/null +++ b/parallel-rsp/lightning/check/riprel.ok @@ -0,0 +1,3 @@ +c s i l +C S I +1.0 2.0 diff --git a/parallel-rsp/lightning/check/setcode.c b/parallel-rsp/lightning/check/setcode.c index 0047f34..08611d9 100644 --- a/parallel-rsp/lightning/check/setcode.c +++ b/parallel-rsp/lightning/check/setcode.c @@ -31,14 +31,24 @@ main(int argc, char *argv[]) int mmap_fd; #endif void (*function)(void); + int mmap_prot, mmap_flags, result; #if defined(__sgi) mmap_fd = open("/dev/zero", O_RDWR); #endif - ptr = mmap(NULL, 1024 * 1024, - PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, mmap_fd, 0); + mmap_prot = PROT_READ | PROT_WRITE; +#if !(__OpenBSD__ || __APPLE__) + mmap_prot |= PROT_EXEC; +#endif +#if __NetBSD__ + mmap_prot = PROT_MPROTECT(mmap_prot); + mmap_flags = 0; +#else + mmap_flags = MAP_PRIVATE; +#endif + mmap_flags |= MAP_ANON; + ptr = mmap(NULL, 1024 * 1024, mmap_prot, mmap_flags, mmap_fd, 0); assert(ptr != MAP_FAILED); #if defined(__sgi) close(mmap_fd); @@ -72,6 +82,10 @@ main(int argc, char *argv[]) if (function != NULL) abort(); +#if __NetBSD__ + result = mprotect(ptr, 1024 * 1024, PROT_READ | PROT_WRITE); + assert(result == 0); +#endif /* and calling again with enough space works */ jit_set_code(ptr, 1024 * 1024); function = jit_emit(); @@ -79,6 +93,10 @@ main(int argc, char *argv[]) abort(); jit_clear_state(); +#if __NetBSD__ || __OpenBSD__ || __APPLE__ + result = mprotect(ptr, 1024 * 1024, PROT_READ | PROT_EXEC); + assert(result == 0); +#endif (*function)(); jit_destroy_state(); finish_jit(); diff --git a/parallel-rsp/lightning/check/skip.ok b/parallel-rsp/lightning/check/skip.ok new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/parallel-rsp/lightning/check/skip.ok @@ -0,0 +1 @@ +10 diff --git a/parallel-rsp/lightning/check/skip.tst b/parallel-rsp/lightning/check/skip.tst new file mode 100644 index 0000000..94eec76 --- /dev/null +++ b/parallel-rsp/lightning/check/skip.tst @@ -0,0 +1,13 @@ +.data 32 +fmt: +.c "%d\n" +.code + prolog + skip 4 + prepare + pushargi fmt + ellipsis + pushargi 10 + finishi @printf + ret + epilog diff --git a/parallel-rsp/lightning/check/stack.tst b/parallel-rsp/lightning/check/stack.tst index e699719..1ebe4f5 100644 --- a/parallel-rsp/lightning/check/stack.tst +++ b/parallel-rsp/lightning/check/stack.tst @@ -55,7 +55,7 @@ fill##T##done: \ #define fill_us fill_s #define fill_ui fill_i -#define ARG( T, N) arg $arg##T##N +#define ARG( T, N) arg##T $arg##T##N #define ARGF( T, N) arg##T $arg##T##N #define ARG1( K, T) ARG##K(T, 0) #define ARG2( K, T) ARG1( K, T) ARG##K(T, 1) @@ -74,56 +74,56 @@ fill##T##done: \ #define ARG15(K, T) ARG14(K, T) ARG##K(T, 14) #define ARG16(K, T) ARG15(K, T) ARG##K(T, 15) #define ARG_c(N) ARG##N( , _c) -#define ARG_uc(N) ARG##N( , _uc) +#define ARG_uc(N) ARG##N( , _c) #define ARG_s(N) ARG##N( , _s) -#define ARG_us(N) ARG##N( , _us) +#define ARG_us(N) ARG##N( , _s) #define ARG_i(N) ARG##N( , _i) -#define ARG_ui(N) ARG##N( , _ui) +#define ARG_ui(N) ARG##N( , _i) #define ARG_l(N) ARG##N( , _l) #define ARG_f(N) ARG##N(F, _f) #define ARG_d(N) ARG##N(F, _d) -#define CHK(N, T, V) \ - getarg %r0 $arg##T##V \ +#define CHK(N, T, TT, V) \ + getarg##T %r0 $arg##TT##V \ ldxi##T %r1 %v0 $(V * szof##T) \ beqr N##T##V %r0 %r1 \ calli @abort \ N##T##V: -#define CHKF(N, T, V) \ - getarg##T %f0 $arg##T##V \ +#define CHKF(N, T, TT, V) \ + getarg##T %f0 $arg##TT##V \ ldxi##T %f1 %v0 $(V * szof##T) \ beqr##T N##T##V %f0 %f1 \ calli @abort \ N##T##V: -#define GET1( K, N, T, V) CHK##K(N, T, 0) -#define GET2( K, N, T, V) GET1( K, N, T, V) CHK##K(N, T, 1) -#define GET3( K, N, T, V) GET2( K, N, T, V) CHK##K(N, T, 2) -#define GET4( K, N, T, V) GET3( K, N, T, V) CHK##K(N, T, 3) -#define GET5( K, N, T, V) GET4( K, N, T, V) CHK##K(N, T, 4) -#define GET6( K, N, T, V) GET5( K, N, T, V) CHK##K(N, T, 5) -#define GET7( K, N, T, V) GET6( K, N, T, V) CHK##K(N, T, 6) -#define GET8( K, N, T, V) GET7( K, N, T, V) CHK##K(N, T, 7) -#define GET9( K, N, T, V) GET8( K, N, T, V) CHK##K(N, T, 8) -#define GET10(K, N, T, V) GET9( K, N, T, V) CHK##K(N, T, 9) -#define GET11(K, N, T, V) GET10(K, N, T, V) CHK##K(N, T, 10) -#define GET12(K, N, T, V) GET11(K, N, T, V) CHK##K(N, T, 11) -#define GET13(K, N, T, V) GET12(K, N, T, V) CHK##K(N, T, 12) -#define GET14(K, N, T, V) GET13(K, N, T, V) CHK##K(N, T, 13) -#define GET15(K, N, T, V) GET14(K, N, T, V) CHK##K(N, T, 14) -#define GET16(K, N, T, V) GET15(K, N, T, V) CHK##K(N, T, 15) +#define GET1( K, N, T, TT, V) CHK##K(N, T, TT, 0) +#define GET2( K, N, T, TT, V) GET1( K, N, T, TT, V) CHK##K(N, T, TT, 1) +#define GET3( K, N, T, TT, V) GET2( K, N, T, TT, V) CHK##K(N, T, TT, 2) +#define GET4( K, N, T, TT, V) GET3( K, N, T, TT, V) CHK##K(N, T, TT, 3) +#define GET5( K, N, T, TT, V) GET4( K, N, T, TT, V) CHK##K(N, T, TT, 4) +#define GET6( K, N, T, TT, V) GET5( K, N, T, TT, V) CHK##K(N, T, TT, 5) +#define GET7( K, N, T, TT, V) GET6( K, N, T, TT, V) CHK##K(N, T, TT, 6) +#define GET8( K, N, T, TT, V) GET7( K, N, T, TT, V) CHK##K(N, T, TT, 7) +#define GET9( K, N, T, TT, V) GET8( K, N, T, TT, V) CHK##K(N, T, TT, 8) +#define GET10(K, N, T, TT, V) GET9( K, N, T, TT, V) CHK##K(N, T, TT, 9) +#define GET11(K, N, T, TT, V) GET10(K, N, T, TT, V) CHK##K(N, T, TT, 10) +#define GET12(K, N, T, TT, V) GET11(K, N, T, TT, V) CHK##K(N, T, TT, 11) +#define GET13(K, N, T, TT, V) GET12(K, N, T, TT, V) CHK##K(N, T, TT, 12) +#define GET14(K, N, T, TT, V) GET13(K, N, T, TT, V) CHK##K(N, T, TT, 13) +#define GET15(K, N, T, TT, V) GET14(K, N, T, TT, V) CHK##K(N, T, TT, 14) +#define GET16(K, N, T, TT, V) GET15(K, N, T, TT, V) CHK##K(N, T, TT, 15) -#define GET_c(N, M) GET##N( , c##N, _c, M) -#define GET_uc(N, M) GET##N( , uc##N, _uc, M) -#define GET_s(N, M) GET##N( , s##N, _s, M) -#define GET_us(N, M) GET##N( , us##N, _us, M) -#define GET_i(N, M) GET##N( , i##N, _i, M) -#define GET_ui(N, M) GET##N( , ui##N, _ui, M) -#define GET_l(N, M) GET##N( , l##N, _l, M) -#define GET_f(N, M) GET##N(F, f##N, _f, M) -#define GET_d(N, M) GET##N(F, d##N, _d, M) +#define GET_c(N, M) GET##N( , c##N, _c, _c, M) +#define GET_uc(N, M) GET##N( , uc##N, _uc, _c, M) +#define GET_s(N, M) GET##N( , s##N, _s, _s, M) +#define GET_us(N, M) GET##N( , us##N, _us, _s, M) +#define GET_i(N, M) GET##N( , i##N, _i, _i, M) +#define GET_ui(N, M) GET##N( , ui##N, _ui, _i, M) +#define GET_l(N, M) GET##N( , l##N, _l, _l, M) +#define GET_f(N, M) GET##N(F, f##N, _f, _f, M) +#define GET_d(N, M) GET##N(F, d##N, _d, _d, M) -#define PUSH( T, V) pushargi V +#define PUSH( T, V) pushargi##T V #define PUSHF( T, V) pushargi##T V #define PUSH0( K, T) /**/ #define PUSH1( K, T) PUSH##K(T, 0) @@ -161,14 +161,14 @@ test##T##_0: \ ret \ epilog -#define DEFN(N, M, T) \ +#define DEFN(N, M, T, TT) \ name test##T##_##N \ test##T##_##N: \ prolog \ arg $argp \ /* stack buffer in %v0 */ \ getarg %v0 $argp \ - ARG##T(N) \ + ARG##TT(N) \ /* validate arguments */ \ GET##T(N, M) \ /* heap buffer in %v1 */ \ @@ -258,24 +258,24 @@ test##T##_17_done: \ ret \ epilog -#define DEF( T) \ +#define DEF( T, TT) \ DEF0( T) \ - DEFN( 1, 0, T) \ - DEFN( 2, 1, T) \ - DEFN( 3, 2, T) \ - DEFN( 4, 3, T) \ - DEFN( 5, 4, T) \ - DEFN( 6, 5, T) \ - DEFN( 7, 6, T) \ - DEFN( 8, 7, T) \ - DEFN( 9, 8, T) \ - DEFN(10, 9, T) \ - DEFN(11, 10, T) \ - DEFN(12, 11, T) \ - DEFN(13, 12, T) \ - DEFN(14, 13, T) \ - DEFN(15, 14, T) \ - DEFN(16, 15, T) \ + DEFN( 1, 0, T, TT) \ + DEFN( 2, 1, T, TT) \ + DEFN( 3, 2, T, TT) \ + DEFN( 4, 3, T, TT) \ + DEFN( 5, 4, T, TT) \ + DEFN( 6, 5, T, TT) \ + DEFN( 7, 6, T, TT) \ + DEFN( 8, 7, T, TT) \ + DEFN( 9, 8, T, TT) \ + DEFN(10, 9, T, TT) \ + DEFN(11, 10, T, TT) \ + DEFN(12, 11, T, TT) \ + DEFN(13, 12, T, TT) \ + DEFN(14, 13, T, TT) \ + DEFN(15, 14, T, TT) \ + DEFN(16, 15, T, TT) \ DEFX(T) #define CALL(T) calli test##T##_17 @@ -321,17 +321,17 @@ memcpy_done: FILLF(_f) FILLF(_d) - DEF(_c) - DEF(_uc) - DEF(_s) - DEF(_us) - DEF(_i) + DEF(_c, _c) + DEF(_uc, _c) + DEF(_s, _s) + DEF(_us, _s) + DEF(_i, _i) #if __WORDSIZE == 64 - DEF(_ui) - DEF(_l) + DEF(_ui, _i) + DEF(_l, _l) #endif - DEF(_f) - DEF(_d) + DEF(_f, _f) + DEF(_d, _d) name main main: diff --git a/parallel-rsp/lightning/configure.ac b/parallel-rsp/lightning/configure.ac index 9261255..3fb09e7 100644 --- a/parallel-rsp/lightning/configure.ac +++ b/parallel-rsp/lightning/configure.ac @@ -1,5 +1,5 @@ dnl -dnl Copyright 2000, 2001, 2002, 2012-2019 Free Software Foundation, Inc. +dnl Copyright 2000, 2001, 2002, 2012-2023 Free Software Foundation, Inc. dnl dnl This file is part of GNU lightning. dnl @@ -7,26 +7,29 @@ dnl GNU lightning is free software; you can redistribute it and/or modify it dnl under the terms of the GNU Lesser General Public License as published dnl by the Free Software Foundation; either version 3, or (at your option) dnl any later version. -dnl +dnl dnl GNU lightning is distributed in the hope that it will be useful, but dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public dnl License for more details. dnl -AC_PREREQ(2.57) -AC_INIT([GNU lightning], 2.1.3, pcpa@gnu.org, lightning) +AC_PREREQ([2.71]) +AC_INIT([GNU lightning],[2.2.1],[pcpa@gnu.org],[lightning]) +AC_CONFIG_AUX_DIR([build-aux]) AC_CANONICAL_TARGET AC_CONFIG_SRCDIR([Makefile.am]) AM_INIT_AUTOMAKE([dist-bzip2]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) -AC_CONFIG_MACRO_DIR(m4) +AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_HEADERS(config.h) +AC_CONFIG_HEADERS([config.h]) AC_PROG_CC -AC_PROG_INSTALL -AC_PROG_LIBTOOL +gl_EARLY +AM_PROG_AR +LT_INIT +gl_INIT case "$target_cpu" in ia64) @@ -57,10 +60,16 @@ case "$target_cpu" in fi ;; *) ;; esac ;; + aarch64) + case "$host_os" in + darwin*) + LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DPACKED_STACK=1" ;; + *) ;; + esac ;; *) ;; esac -AC_CHECK_FUNCS(mremap ffsl getopt_long_only isnan isinf,,) +AC_CHECK_FUNCS(mmap mremap ffsl getopt_long_only isnan isinf,,) AC_CHECK_HEADERS([getopt.h stdint.h],,,) @@ -69,7 +78,7 @@ AC_ARG_ENABLE(disassembler, [Enable jit disassembler using binutils]), [DISASSEMBLER=$enableval], [DISASSEMBLER=auto]) if test "x$DISASSEMBLER" != "xno"; then - # FIXME need to check for libiberty first or will fail to link + AC_CHECK_LIB(iberty, htab_try_create, , [HAVE_IBERTY="no"]) AC_CHECK_LIB(bfd, bfd_init, , @@ -78,26 +87,50 @@ if test "x$DISASSEMBLER" != "xno"; then [HAVE_Z="no"]) AC_CHECK_LIB(opcodes, init_disassemble_info, , [HAVE_OPCODES="no"]) - if test "x$HAVE_IBERTY" = "xno" -o \ - "x$HAVE_BFD" = "xno" -o \ - "x$HAVE_Z" = "xno" -o \ - "x$HAVE_OPCODES" = "xno"; then - if test "x$DISASSEMBLER" != "xauto"; then - AC_MSG_ERROR([binutils not found, see http://www.gnu.org/software/binutils/]) - else - AC_MSG_WARN([binutils not found, see http://www.gnu.org/software/binutils/]) - DISASSEMBLER="no" - fi + + if test "x$HAVE_IBERTY" = "xno"; then + if test "x$DISASSEMBLER" = "xyes"; then + AC_MSG_ERROR([libiberty not found]) + else + AC_MSG_WARN([libiberty not found]) + DISASSEMBLER="no" + fi fi + + if test "x$HAVE_BFD" = "xno"; then + if test "x$DISASSEMBLER" = "xyes"; then + AC_MSG_ERROR([binutils BFD not found, see http://www.gnu.org/software/binutils/]) + else + AC_MSG_WARN([binutils BFD not found, see http://www.gnu.org/software/binutils/]) + DISASSEMBLER="no" + fi + fi + + if test "x$HAVE_Z" = "xno"; then + if test "x$DISASSEMBLER" = "xyes"; then + AC_MSG_ERROR([zlib not found, see https://zlib.net/]) + else + AC_MSG_WARN([zlib not found, see https://zlib.net/]) + DISASSEMBLER="no" + fi + fi + + if test "x$HAVE_OPCODES" = "xno"; then + if test "x$DISASSEMBLER" = "xyes"; then + AC_MSG_ERROR([binutils opcodes not found, see https://www.gnu.org/software/binutils/]) + else + AC_MSG_WARN([binutils opcodes not found, see https://www.gnu.org/software/binutils/]) + DISASSEMBLER="no" + fi + fi + fi AM_CONDITIONAL(with_disassembler, [test "x$DISASSEMBLER" != "xno"]) if test "x$DISASSEMBLER" != "xno"; then LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DDISASSEMBLER=1" save_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -I$PWD/include -D_GNU_SOURCE" + CFLAGS="$CFLAGS -D_GNU_SOURCE" AC_COMPILE_IFELSE([AC_LANG_SOURCE( - #include - #include #include int main(int argc, char *argv[]) { @@ -107,10 +140,22 @@ if test "x$DISASSEMBLER" != "xno"; then return 0; } )], [ac_cv_test_new_disassembler=no],,) + AC_COMPILE_IFELSE([AC_LANG_SOURCE( + #include + int main(int argc, char *argv[]) + { + struct disassemble_info dinfo; + INIT_DISASSEMBLE_INFO(dinfo, NULL, NULL, NULL); + return 0; + } + )], [ac_cv_test_new_disassemble_info=yes],[ac_cv_test_new_disassemble_info=no],) CFLAGS="$save_CFLAGS" if test "x$ac_cv_test_new_disassembler" != "xno"; then LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DBINUTILS_2_29=1" fi + if test "x$ac_cv_test_new_disassemble_info" != "xno"; then + LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DBINUTILS_2_38=1" + fi fi AC_ARG_ENABLE(devel-disassembler, @@ -124,17 +169,6 @@ if test "x$DEVEL_DISASSEMBLER" != "xno"; then LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DDEVEL_DISASSEMBLER=1" fi -AC_ARG_ENABLE(assertions, - AS_HELP_STRING([--enable-assertions], - [Enable runtime code generation assertions]), - [DEBUG=$enableval], [DEBUG=auto]) -if test "x$DEBUG" = xyes; then - LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DDEBUG=1" -else - LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DNDEBUG" - DEBUG=no -fi - # This option is only useful during development. AC_ARG_ENABLE(devel-get-jit-size, AS_HELP_STRING([--enable-devel-get-jit-size], @@ -142,11 +176,36 @@ AC_ARG_ENABLE(devel-get-jit-size, [GET_JIT_SIZE=$enableval], [GET_JIT_SIZE=no]) AM_CONDITIONAL(get_jit_size, [test $GET_JIT_SIZE = yes]) -case "$host_os" in - *bsd*|osf*) SHLIB="" ;; - *hpux*) SHLIB="-ldld" ;; - *) SHLIB="-ldl" ;; -esac +AC_ARG_ENABLE(assertions, + AS_HELP_STRING([--enable-assertions], + [Enable runtime code generation assertions]), + [DEBUG=$enableval], [DEBUG=auto]) + +# This option might be made default in the future +# Currently it is only useful to ensure existing code will work +# if PACKED_STACK is also defined. +AC_ARG_ENABLE(devel-strong-type-checking, + AS_HELP_STRING([--enable-devel-strong-type-checking], + [Devel mode for strong type checking]), + [STRONG_TYPE_CHECKING=$enableval], [STRONG_TYPE_CHECKING=no]) +if test "x$DEBUG" = xyes -o x"$STRONG_TYPE_CHECKING" = xyes; then + LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DDEBUG=1" +else + LIGHTNING_CFLAGS="$LIGHTNING_CFLAGS -DNDEBUG" + DEBUG=no +fi +AM_CONDITIONAL(strong_type_checking, [test $STRONG_TYPE_CHECKING = yes]) + +AC_CHECK_LIB(dl, dlopen, [HAVE_LIBDL="yes"]) +AC_CHECK_LIB(dld, dlopen, [HAVE_LIBDLD="yes"]) + +if test "x$HAVE_LIBDL" = xyes; then + SHLIB="-ldl"; +elif test "x$HAVE_LIBDLD" = xyes; then + SHLIB="-ldld"; +else + SHLIB=""; +fi AC_SUBST(SHLIB) cpu= @@ -162,19 +221,21 @@ case "$target_cpu" in s390*) cpu=s390 ;; alpha*) cpu=alpha ;; riscv*) cpu=riscv ;; + loongarch*) cpu=loongarch ;; *) ;; esac -AM_CONDITIONAL(cpu_arm, [test cpu-$cpu = cpu-arm]) -AM_CONDITIONAL(cpu_mips, [test cpu-$cpu = cpu-mips]) -AM_CONDITIONAL(cpu_ppc, [test cpu-$cpu = cpu-ppc]) -AM_CONDITIONAL(cpu_sparc, [test cpu-$cpu = cpu-sparc]) -AM_CONDITIONAL(cpu_x86, [test cpu-$cpu = cpu-x86]) -AM_CONDITIONAL(cpu_ia64, [test cpu-$cpu = cpu-ia64]) -AM_CONDITIONAL(cpu_hppa, [test cpu-$cpu = cpu-hppa]) -AM_CONDITIONAL(cpu_aarch64, [test cpu-$cpu = cpu-aarch64]) -AM_CONDITIONAL(cpu_s390, [test cpu-$cpu = cpu-s390]) -AM_CONDITIONAL(cpu_alpha, [test cpu-$cpu = cpu-alpha]) -AM_CONDITIONAL(cpu_riscv, [test cpu-$cpu = cpu-riscv]) +AM_CONDITIONAL(cpu_arm, [test cpu-$cpu = cpu-arm]) +AM_CONDITIONAL(cpu_mips, [test cpu-$cpu = cpu-mips]) +AM_CONDITIONAL(cpu_ppc, [test cpu-$cpu = cpu-ppc]) +AM_CONDITIONAL(cpu_sparc, [test cpu-$cpu = cpu-sparc]) +AM_CONDITIONAL(cpu_x86, [test cpu-$cpu = cpu-x86]) +AM_CONDITIONAL(cpu_ia64, [test cpu-$cpu = cpu-ia64]) +AM_CONDITIONAL(cpu_hppa, [test cpu-$cpu = cpu-hppa]) +AM_CONDITIONAL(cpu_aarch64, [test cpu-$cpu = cpu-aarch64]) +AM_CONDITIONAL(cpu_s390, [test cpu-$cpu = cpu-s390]) +AM_CONDITIONAL(cpu_alpha, [test cpu-$cpu = cpu-alpha]) +AM_CONDITIONAL(cpu_riscv, [test cpu-$cpu = cpu-riscv]) +AM_CONDITIONAL(cpu_loongarch, [test cpu-$cpu = cpu-loongarch]) # Test x87 if both, x87 and sse2 available ac_cv_test_x86_x87= @@ -184,16 +245,15 @@ ac_cv_test_arm_arm= ac_cv_test_arm_swf= save_CFLAGS=$CFLAGS -CFLAGS="$CFLAGS -I$PWD/include -D_GNU_SOURCE" +CFLAGS="$CFLAGS -D_GNU_SOURCE" if test x$cpu = x; then AC_MSG_ERROR([cpu $target_cpu not supported]) elif test $cpu = x86; then AC_RUN_IFELSE([AC_LANG_SOURCE([[ - #include int main(void) { int ac, flags; unsigned int eax, ebx, ecx, edx; - if (__WORDSIZE == 64) + if (sizeof(long) == 8) return 1; __asm__ volatile ("pushfl;\n\t" "popl %0;\n\t" @@ -289,11 +349,13 @@ if test $ac_cv_header_stdint_h = yes; then AC_SUBST(MAYBE_INCLUDE_STDINT_H, ["#include "]) fi -AC_OUTPUT([Makefile +AC_CONFIG_FILES([Makefile lightning.pc + gnulib-lib/Makefile doc/Makefile include/Makefile include/lightning/Makefile include/lightning.h lib/Makefile check/Makefile]) +AC_OUTPUT diff --git a/parallel-rsp/lightning/doc/.gitignore b/parallel-rsp/lightning/doc/.gitignore index f62c13f..0e0a6da 100644 --- a/parallel-rsp/lightning/doc/.gitignore +++ b/parallel-rsp/lightning/doc/.gitignore @@ -1,2 +1,13 @@ *.info* stamp-* +/version.texi + +texinfo.tex +mdate-sh + +fact +ifib +incr +printf +rfib +rpn diff --git a/parallel-rsp/lightning/doc/Makefile.am b/parallel-rsp/lightning/doc/Makefile.am index 20d4456..4cec67e 100644 --- a/parallel-rsp/lightning/doc/Makefile.am +++ b/parallel-rsp/lightning/doc/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright 2012-2019 Free Software Foundation, Inc. +# Copyright 2012-2023 Free Software Foundation, Inc. # # This file is part of GNU lightning. # @@ -14,7 +14,8 @@ # License for more details. # -AM_CFLAGS = -I$(top_srcdir)/include -D_GNU_SOURCE +AM_CFLAGS = -I $(top_builddir)/include -I$(top_srcdir)/include \ + -D_GNU_SOURCE $(LIGHTNING_CFLAGS) info_TEXINFOS = lightning.texi MOSTLYCLEANFILES = lightning.tmp diff --git a/parallel-rsp/lightning/doc/body.texi b/parallel-rsp/lightning/doc/body.texi index 4aef7a3..1bd3f67 100644 --- a/parallel-rsp/lightning/doc/body.texi +++ b/parallel-rsp/lightning/doc/body.texi @@ -30,6 +30,7 @@ dynamic code generation. * The instruction set:: The RISC instruction set used in GNU lightning * GNU lightning examples:: GNU lightning's examples * Reentrancy:: Re-entrant usage of GNU lightning +* Registers:: Accessing the whole register file * Customizations:: Advanced code generation customizations * Acknowledgements:: Acknowledgements for GNU lightning @end menu @@ -43,8 +44,8 @@ This document describes @value{TOPIC} the @lightning{} library for dynamic code generation. @end iftex -Dynamic code generation is the generation of machine code -at runtime. It is typically used to strip a layer of interpretation +Dynamic code generation is the generation of machine code +at runtime. It is typically used to strip a layer of interpretation by allowing compilation to occur at runtime. One of the most well-known applications of dynamic code generation is perhaps that of interpreters that compile source code to an intermediate bytecode @@ -53,7 +54,7 @@ approach effectively combines the portability of bytecode representations with the speed of machine code. Another common application of dynamic code generation is in the field of hardware simulators and binary emulators, which can use the same techniques -to translate simulated instructions to the instructions of the +to translate simulated instructions to the instructions of the underlying machine. Yet other applications come to mind: for example, windowing @@ -68,7 +69,7 @@ retargeted for each machine; in addition, coding a run-time code generator is a tedious and error-prone task more than a difficult one. @lightning{} provides a portable, fast and easily retargetable dynamic -code generation system. +code generation system. To be portable, @lightning{} abstracts over current architectures' quirks and unorthogonalities. The interface that it exposes to is that @@ -88,7 +89,11 @@ assembles machine instructions without further tests. @node Installation @chapter Configuring and installing @lightning{} -The first thing to do to use @lightning{} is to configure the +Here we will assume that your system already has the dependencies +necessary to build @lightning{}. For more on dependencies, see +@lightning{}'s @file{README-hacking} file. + +The first thing to do to build @lightning{} is to configure the program, picking the set of macros to be used on the host architecture; this configuration is automatically performed by the @file{configure} shell script; to run it, merely type: @@ -96,17 +101,30 @@ the @file{configure} shell script; to run it, merely type: ./configure @end example -@lightning{} supports the @code{--enable-disassembler} option, that -enables linking to GNU binutils and optionally print human readable +The @file{configure} accepts the @code{--enable-disassembler} option, +hat enables linking to GNU binutils and optionally print human readable disassembly of the jit code. This option can be disabled by the @code{--disable-disassembler} option. -Another option that @file{configure} accepts is -@code{--enable-assertions}, which enables several consistency checks in -the run-time assemblers. These are not usually needed, so you can -decide to simply forget about it; also remember that these consistency +@file{configure} also accepts the @code{--enable-devel-disassembler}, +option useful to check exactly hat machine instructions were generated +for a @lightning{} instrction. Basically mixing @code{jit_print} and +@code{jit_disassembly}. + +The @code{--enable-assertions} option, which enables several consistency +hecks in the run-time assemblers. These are not usually needed, so you +can decide to simply forget about it; also remember that these consistency checks tend to slow down your code generator. +The @code{--enable-devel-strong-type-checking} option that does extra type +checking using @code{assert}. This option also enables the +@code{--enable-assertions} unless it is explicitly disabled. + +The option @code{--enable-devel-get-jit-size} should only be used +when doing updates or maintenance to lightning. It regenerates the +@code{jit_$ARCH]-sz.c} creating a table or maximum bytes usage when +translating a @lightning{} instruction to machine code. + After you've configured @lightning{}, run @file{make} as usual. @lightning{} has an extensive set of tests to validate it is working @@ -243,6 +261,8 @@ lshr O1 = O2 << O3 lshi O1 = O2 << O3 rshr _u O1 = O2 >> O3@footnote{The sign bit is propagated unless using the @code{_u} modifier.} rshi _u O1 = O2 >> O3@footnote{The sign bit is propagated unless using the @code{_u} modifier.} +movzr O1 = O3 ? O1 : O2 +movnr O1 = O3 ? O2 : O1 @end example @item Four operand binary ALU operations @@ -271,12 +291,27 @@ These accept two operands, both of which must be registers. @example negr _f _d O1 = -O2 comr O1 = ~O2 +clor O1 = number of leading one bits +clzr O1 = number of leading zero bits +ctor O1 = number of trailing one bits +ctzr O1 = number of trailing zero bits @end example +Note that @code{ctzr} is basically equivalent of a @code{C} call +@code{ffs} but indexed at bit zero, not one. + +Contrary to @code{__builtin_ctz} and @code{__builtin_clz}, an input +value of zero is not an error, it just returns the number of bits +in a word, 64 if @lightning{} generates 64 bit instructions, otherwise +it returns 32. + +The @code{clor} and @code{ctor} are just counterparts of the versions +that search for zero bits. + These unary ALU operations are only defined for float operands. @example absr _f _d O1 = fabs(O2) -sqrtr O1 = sqrt(O2) +sqrtr _f _d O1 = sqrt(O2) @end example Besides requiring the @code{r} modifier, there are no unary operations @@ -365,6 +400,14 @@ htonr _us _ui _ul @r{Host-to-network (big endian) order} ntohr _us _ui _ul @r{Network-to-host order } @end example +@code{bswapr} can be used to unconditionally byte-swap an operand. +On little-endian architectures, @code{htonr} and @code{ntohr} resolve +to this. +The @code{_ul} variant is only available in 64-bit architectures. +@example +bswapr _us _ui _ul 01 = byte_swap(02) +@end example + @item Load operations @code{ld} accepts two operands while @code{ldx} accepts three; in both cases, the last can be either a register or an immediate @@ -386,31 +429,33 @@ ldxi _c _uc _s _us _i _ui _l _f _d O1 = *(O2+O3) both cases, the first can be either a register or an immediate value. Values are sign-extended to fit a whole register. @example -str _c _uc _s _us _i _ui _l _f _d *O1 = O2 -sti _c _uc _s _us _i _ui _l _f _d *O1 = O2 -stxr _c _uc _s _us _i _ui _l _f _d *(O1+O2) = O3 -stxi _c _uc _s _us _i _ui _l _f _d *(O1+O2) = O3 +str _c _s _i _l _f _d *O1 = O2 +sti _c _s _i _l _f _d *O1 = O2 +stxr _c _s _i _l _f _d *(O1+O2) = O3 +stxi _c _s _i _l _f _d *(O1+O2) = O3 @end example -As for the load operations, the @code{_ui} and @code{_l} types are -only available in 64-bit architectures, and for convenience, there -is a version without a type modifier for integer or pointer operands -that uses the appropriate wordsize call. +Note that the unsigned type modifier is not available, as the store +only writes to the 1, 2, 4 or 8 sized memory address. +The @code{_l} type is only available in 64-bit architectures, and for +convenience, there is a version without a type modifier for integer or +pointer operands that uses the appropriate wordsize call. @item Argument management These are: @example prepare (not specified) va_start (not specified) -pushargr _f _d -pushargi _f _d +pushargr _c _uc _s _us _i _ui _l _f _d +pushargi _c _uc _s _us _i _ui _l _f _d va_push (not specified) -arg _f _d +arg _c _uc _s _us _i _ui _l _f _d getarg _c _uc _s _us _i _ui _l _f _d va_arg _d -putargr _f _d -putargi _f _d +putargr _c _uc _s _us _i _ui _l _f _d +putargi _c _uc _s _us _i _ui _l _f _d ret (not specified) -retr _f _d +retr _c _uc _s _us _i _ui _l _f _d +reti _c _uc _s _us _i _ui _l _f _d reti _f _d va_end (not specified) retval _c _uc _s _us _i _ui _l _f _d @@ -429,6 +474,15 @@ the @code{pushargr} or @code{pushargi} to push the arguments @strong{in left to right order}; and use @code{finish} or @code{call} (explained below) to perform the actual call. +Note that @code{arg}, @code{pusharg}, @code{putarg} and @code{ret} when +handling integer types can be used without a type modifier. +It is suggested to use matching type modifiers to @code{arg}, @code{putarg} +and @code{getarg} otherwise problems will happen if generating jit for +environments that require arguments to be truncated and zero or sign +extended by the caller and/or excess arguments might be passed packed +in the stack. Currently only Apple systems with @code{aarch64} cpus are +known to have this restriction. + @code{va_start} returns a @code{C} compatible @code{va_list}. To fetch arguments, use @code{va_arg} for integers and @code{va_arg_d} for doubles. @code{va_push} is required when passing a @code{va_list} to another function, @@ -550,6 +604,10 @@ bxsubr _u O2 -= O3@r{, goto }O1@r{ if no overflow} bxsubi _u O2 -= O3@r{, goto }O1@r{ if no overflow} @end example +Note that the @code{C} code does not have an @code{O1} argument. It is +required to always use the return value as an argument to @code{patch}, +@code{patch_at} or @code{patch_abs}. + @item Jump and return operations These accept one argument except @code{ret} and @code{jmpi} which have none; the difference between @code{finishi} and @code{calli} @@ -582,6 +640,20 @@ forward (not specified) @r{forward label} indirect (not specified) @r{special simple label} @end example +The following instruction is used to specify a minimal alignment for +the next instruction, usually with a label: +@example +align (not specified) @r{align code} +@end example + +Similar to @code{align} is the next instruction, also usually used with +a label: +@example +skip (not specified) @r{skip code} +@end example +It is used to specify a minimal number of bytes of nops to be inserted +before the next instruction. + @code{label} is normally used as @code{patch_at} argument for backward jumps. @@ -634,6 +706,42 @@ that automatically binds the implicit label added by @code{patch} with the @code{movi}, but on some special conditions it is required to create an "unbound" label. +@code{align} is useful for creating multiple entry points to a +(trampoline) function that are all accessible through a single +function pointer. @code{align} receives an integer argument that +defines the minimal alignment of the address of a label directly +following the @code{align} instruction. The integer argument must be +a power of two and the effective alignment will be a power of two no +less than the argument to @code{align}. If the argument to +@code{align} is 16 or more, the effective alignment will match the +specified minimal alignment exactly. + +@example + jit_node_t *forward, *label1, *label2, *jump; + unsigned char *addr1, *addr2; +forward = jit_forward(); + jit_align(16); +label1 = jit_indirect(); @rem{/* first entry point */} +jump = jit_jmpi(); @rem{/* jump to first handler */} + jit_patch_at(jump, forward); + jit_align(16); +label2 = jit_indirect(); @rem{/* second entry point */} + ... @rem{/* second handler */} + jit_jmpr(...); + jit_link(forward); + ... @rem{/* first handler /*} + jit_jmpr(...); + ... + jit_emit(); + addr1 = jit_address(label1); + addr2 = jit_address(label2); + assert(addr2 - addr1 == 16); @rem{/* only one of the addresses needs to be remembered */} +@end example + +@code{skip} is useful for reserving space in the code buffer that can +later be filled (possibly with the help of the pair of functions +@code{jit_unprotect} and @code{jit_protect}). + @item Function prolog These macros are used to set up a function prolog. The @code{allocai} @@ -695,6 +803,51 @@ in = arg @rem{! Same as above} ret @rem{! Return to caller} @end example +@item Register liveness + +During code generation, @lightning{} occasionally needs scratch registers +or needs to use architecture-defined registers. For that, @lightning{} +internally maintains register liveness information. + +In the following example, @code{qdivr} will need special registers like +@code{R0} on some architectures. As @lightning{} understands that +@code{R0} is used in the subsequent instruction, it will create +save/restore code for @code{R0} in case. + +@example +... +qdivr V0, V1, V2, V3 +movr V3, R0 +... +@end example + +The same is not true in the example that follows. Here, @code{R0} is +not alive after the division operation because @code{R0} is neither an +argument register nor a callee-save register. Thus, no save/restore +code for @code{R0} will be created in case. + +@example +... +qdivr V0, V1, V2, V3 +jmpr R1 +... +@end example + +The @code{live} instruction can be used to mark a register as live after +it as in the following example. Here, @code{R0} will be preserved +across the division. + +@example +... +qdivr V0, V1, V2, V3 +live R0 +jmpr R1 +... +@end example + +The @code{live} instruction is useful at code entry and exit points, +like after and before a @code{callr} instruction. + @item Trampolines, continuations and tail call optimization Frequently it is required to generate jit code that must jump to @@ -821,7 +974,7 @@ will return non zero if the argument lives in a register. This call is useful to know the live range of register arguments, as those are very fast to read and write, but have volatile values. -@code{callee_save_p} exects a valid @code{JIT_Rn}, @code{JIT_Vn}, or +@code{callee_save_p} expects a valid @code{JIT_Rn}, @code{JIT_Vn}, or @code{JIT_Fn}, and will return non zero if the register is callee save. This call is useful because on several ports, the @code{JIT_Rn} and @code{JIT_Fn} registers are actually callee save; no need @@ -830,6 +983,34 @@ to save and load the values when making function calls. @code{pointer_p} expects a pointer argument, and will return non zero if the pointer is inside the generated jit code. Must be called after @code{jit_emit} and before @code{jit_destroy_state}. + +@item Atomic operations +Only compare-and-swap is implemented. It accepts four operands; +the second can be an immediate. + +The first argument is set with a boolean value telling if the operation +did succeed. + +Arguments must be different, cannot use the result register to also pass +an argument. + +The second argument is the address of a machine word. + +The third argument is the old value. + +The fourth argument is the new value. + +@example +casr 01 = (*O2 == O3) ? (*O2 = O4, 1) : 0 +casi 01 = (*O2 == O3) ? (*O2 = O4, 1) : 0 +@end example + +If value at the address in the second argument is equal to the third +argument, the address value is atomically modified to the value of the +fourth argument and the first argument is set to a non zero value. + +If the value at the address in the second argument is not equal to the +third argument nothing is done and the first argument is set to zero. @end table @node GNU lightning examples @@ -1005,9 +1186,9 @@ programmer would write): mov %i0, %g2 retl inc %g2 inc %o0 mov %g2, %i0 - restore - retl - nop + restore + retl + nop @end example In this case, @lightning{} introduces overhead to create a register window (not knowing that the procedure is a leaf procedure) and to @@ -1018,26 +1199,13 @@ maps to @code{%g2} on the SPARC). @table @b @item x86_64 @example - sub $0x30,%rsp - mov %rbp,(%rsp) - mov %rsp,%rbp - sub $0x18,%rsp - mov %rdi,%rax mov %rdi, %rax - add $0x1,%rax inc %rax - mov %rbp,%rsp - mov (%rsp),%rbp - add $0x30,%rsp - retq retq + mov %rdi,%rax + add $0x1,%rax + ret @end example -In this case, the main overhead is due to the function's prolog and -epilog, and stack alignment after reserving stack space for word -to/from float conversions or moving data from/to x87 to/from SSE. -Note that besides allocating space to save callee saved registers, -no registers are saved/restored because @lightning{} notices those -registers are not modified. There is currently no logic to detect -if it needs to allocate stack space for type conversions neither -proper leaf function detection, but these are subject to change -(FIXME). +In this case, for the x86 port, @lightning{} has simple optimizations +to understand it is a leaf function, and that it is not required to +create a stack frame nor update the stack pointer. @end table @node printf @@ -1201,7 +1369,7 @@ jit_node_t *compile_rpn(char *expr) in = jit_arg(); stack_ptr = stack_base = jit_allocai (32 * sizeof (int)); - jit_getarg_i(JIT_R2, in); + jit_getarg(JIT_R2, in); while (*expr) @{ char buf[32]; @@ -1480,7 +1648,7 @@ implementation and to avoid needing the user to keep adding an extra argument to every call, as multiple jit states generating code in paralell should be very uncommon. -@section Registers +@node Registers @chapter Accessing the whole register file As mentioned earlier in this chapter, all @lightning{} back-ends are @@ -1496,6 +1664,33 @@ constant. Of course, expressions like @code{JIT_R0} and @code{JIT_R(0)} denote the same register, and likewise for integer callee-saved, or floating-point, registers. +@section Scratch registers + +For operations, @lightning{} does not support directly, like storing +a literal in memory, @code{jit_get_reg} and @code{jit_unget_reg} can be used to +acquire and release a scratch register as in the following pattern: + +@example + jit_int32_t reg = jit_get_reg (jit_class_gpr); + jit_movi (reg, immediate); + jit_stxi (offsetof (some_struct, some_field), JIT_V0, reg); + jit_unget_reg (reg); +@end example + +As @code{jit_get_reg} and @code{jit_unget_reg} may generate spills and +reloads but don't follow branches, the code between both must be in +the same basic block and must not contain any branches as in the +following (bad) example. + +@example + jit_int32_t reg = jit_get_reg (jit_class_gpr); + jit_ldxi (reg, JIT_V0, offset); + jump = jit_bnei (reg, V0); + jit_movr (JIT_V1, reg); + jit_patch (jump); + jit_unget_reg (reg); +@end example + @node Customizations @chapter Customizations @@ -1527,6 +1722,28 @@ Get the current memory allocation function. Also, unlike the GNU GMP counterpart, it is an error to pass @code{NULL} pointers as arguments. @end deftypefun +@section Protection +Unless an alternate code buffer is used (see below), @code{jit_emit} +set the access protections that the code buffer's memory can be read and +executed, but not modified. One can use the following functions after +@code{jit_emit} but before @code{jit_clear} to temporarily lift the +protection: + +@deftypefun void jit_unprotect () +Changes the access protection that the code buffer's memory can be read and +modified. Before the emitted code can be invoked, @code{jit_protect} +has to be called to reset the change. + +This procedure has no effect when an alternate code buffer (see below) is used. +@end deftypefun + +@deftypefun void jit_protect () +Changes the access protection that the code buffer's memory can be read and +executed. + +This procedure has no effect when an alternate code buffer (see below) is used. +@end deftypefun + @section Alternate code buffer To instruct @lightning{} to use an alternate code buffer it is required to call @code{jit_realize} before @code{jit_emit}, and then query states diff --git a/parallel-rsp/lightning/doc/rpn.c b/parallel-rsp/lightning/doc/rpn.c index 8131484..edb3d8e 100644 --- a/parallel-rsp/lightning/doc/rpn.c +++ b/parallel-rsp/lightning/doc/rpn.c @@ -24,7 +24,7 @@ jit_node_t *compile_rpn(char *expr) fn = jit_note(NULL, 0); jit_prolog(); - in = jit_arg(); + in = jit_arg_i(); stack_ptr = stack_base = jit_allocai (32 * sizeof (int)); jit_getarg_i(JIT_R2, in); diff --git a/parallel-rsp/lightning/doc/version.texi b/parallel-rsp/lightning/doc/version.texi deleted file mode 100644 index b4a0c22..0000000 --- a/parallel-rsp/lightning/doc/version.texi +++ /dev/null @@ -1,4 +0,0 @@ -@set UPDATED 3 October 2017 -@set UPDATED-MONTH October 2017 -@set EDITION 2.1.3 -@set VERSION 2.1.3 diff --git a/parallel-rsp/lightning/gnulib b/parallel-rsp/lightning/gnulib new file mode 160000 index 0000000..e54b645 --- /dev/null +++ b/parallel-rsp/lightning/gnulib @@ -0,0 +1 @@ +Subproject commit e54b645fc6b8422562327443bda575c65d931fbd diff --git a/parallel-rsp/lightning/gnulib-lib/.gitignore b/parallel-rsp/lightning/gnulib-lib/.gitignore new file mode 100644 index 0000000..d9f5394 --- /dev/null +++ b/parallel-rsp/lightning/gnulib-lib/.gitignore @@ -0,0 +1,2 @@ +/Makefile.am +/dummy.c diff --git a/parallel-rsp/lightning/include/Makefile.am b/parallel-rsp/lightning/include/Makefile.am index 8f91594..bd48777 100644 --- a/parallel-rsp/lightning/include/Makefile.am +++ b/parallel-rsp/lightning/include/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright 2000, 2001, 2002, 2012-2019 Free Software Foundation, Inc. +# Copyright 2000, 2001, 2002, 2012-2023 Free Software Foundation, Inc. # # This file is part of GNU lightning. # diff --git a/parallel-rsp/lightning/include/lightning.h b/parallel-rsp/lightning/include/lightning.h index 5b74b09..9c99ad9 100644 --- a/parallel-rsp/lightning/include/lightning.h +++ b/parallel-rsp/lightning/include/lightning.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -24,6 +24,7 @@ #include #include #include +#include #if defined(__hpux) && defined(__hppa__) # include @@ -123,6 +124,11 @@ typedef jit_int32_t jit_bool_t; typedef jit_int32_t jit_gpr_t; typedef jit_int32_t jit_fpr_t; +#if !defined(__powerpc__) && \ + (defined(__POWERPC__) || defined(__ppc__) || defined(__PPC__)) +#define __powerpc__ 1 +#endif + #if defined(__i386__) || defined(__x86_64__) # include #elif defined(__mips__) @@ -145,6 +151,8 @@ typedef jit_int32_t jit_fpr_t; # include #elif defined(__riscv) # include +#elif defined(__loongarch__) +# include #endif #define jit_flag_node 0x0001 /* patch node not absolute */ @@ -182,6 +190,8 @@ typedef enum { #define jit_align(u) jit_new_node_w(jit_code_align, u) jit_code_live, jit_code_align, jit_code_save, jit_code_load, +#define jit_skip(u) jit_new_node_w(jit_code_skip, u) + jit_code_skip, #define jit_name(u) _jit_name(_jit,u) jit_code_name, #define jit_note(u, v) _jit_note(_jit, u, v) @@ -202,27 +212,80 @@ typedef enum { #define jit_allocar(u, v) _jit_allocar(_jit,u,v) jit_code_allocai, jit_code_allocar, -#define jit_arg() _jit_arg(_jit) - jit_code_arg, +#define jit_arg_c() _jit_arg(_jit, jit_code_arg_c) +#define jit_arg_s() _jit_arg(_jit, jit_code_arg_s) +#define jit_arg_i() _jit_arg(_jit, jit_code_arg_i) +# if __WORDSIZE == 32 +# define jit_arg() jit_arg_i() +#else +# define jit_arg_l() _jit_arg(_jit, jit_code_arg_l) +# define jit_arg() jit_arg_l() +#endif + jit_code_arg_c, jit_code_arg_s, + jit_code_arg_i, jit_code_arg_l, +#if __WORDSIZE == 32 +# define jit_code_arg jit_code_arg_i +#else +# define jit_code_arg jit_code_arg_l +#endif + #define jit_getarg_c(u,v) _jit_getarg_c(_jit,u,v) #define jit_getarg_uc(u,v) _jit_getarg_uc(_jit,u,v) - jit_code_getarg_c, jit_code_getarg_uc, #define jit_getarg_s(u,v) _jit_getarg_s(_jit,u,v) #define jit_getarg_us(u,v) _jit_getarg_us(_jit,u,v) - jit_code_getarg_s, jit_code_getarg_us, #define jit_getarg_i(u,v) _jit_getarg_i(_jit,u,v) #if __WORDSIZE == 32 # define jit_getarg(u,v) jit_getarg_i(u,v) #else -# define jit_getarg(u,v) jit_getarg_l(u,v) # define jit_getarg_ui(u,v) _jit_getarg_ui(_jit,u,v) # define jit_getarg_l(u,v) _jit_getarg_l(_jit,u,v) +# define jit_getarg(u,v) jit_getarg_l(u,v) #endif + jit_code_getarg_c, jit_code_getarg_uc, + jit_code_getarg_s, jit_code_getarg_us, jit_code_getarg_i, jit_code_getarg_ui, jit_code_getarg_l, -# define jit_putargr(u,v) _jit_putargr(_jit,u,v) -# define jit_putargi(u,v) _jit_putargi(_jit,u,v) - jit_code_putargr, jit_code_putargi, +#if __WORDSIZE == 32 +# define jit_code_getarg jit_code_getarg_i +#else +# define jit_code_getarg jit_code_getarg_l +#endif + +#define jit_putargr_c(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_c) +#define jit_putargi_c(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_c) +#define jit_putargr_uc(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_uc) +#define jit_putargi_uc(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_uc) +#define jit_putargr_s(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_s) +#define jit_putargi_s(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_s) +#define jit_putargr_us(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_us) +#define jit_putargi_us(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_us) +#define jit_putargr_i(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_i) +#define jit_putargi_i(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_i) +#if __WORDSIZE == 32 +# define jit_putargr(u,v) jit_putargr_i(u,v) +# define jit_putargi(u,v) jit_putargi_i(u,v) +#else +# define jit_putargr_ui(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_ui) +# define jit_putargi_ui(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_ui) +# define jit_putargr_l(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_l) +# define jit_putargi_l(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_l) +# define jit_putargr(u,v) jit_putargr_l(u,v) +# define jit_putargi(u,v) jit_putargi_l(u,v) +#endif + jit_code_putargr_c, jit_code_putargi_c, + jit_code_putargr_uc, jit_code_putargi_uc, + jit_code_putargr_s, jit_code_putargi_s, + jit_code_putargr_us, jit_code_putargi_us, + jit_code_putargr_i, jit_code_putargi_i, + jit_code_putargr_ui, jit_code_putargi_ui, + jit_code_putargr_l, jit_code_putargi_l, +#if __WORDSIZE == 32 +# define jit_code_putargr jit_code_putargr_i +# define jit_code_putargi jit_code_putargi_i +#else +# define jit_code_putargr jit_code_putargr_l +# define jit_code_putargi jit_code_putargi_l +#endif #define jit_va_start(u) jit_new_node_w(jit_code_va_start, u) jit_code_va_start, @@ -339,6 +402,15 @@ typedef enum { #define jit_movr(u,v) jit_new_node_ww(jit_code_movr,u,v) #define jit_movi(u,v) jit_new_node_ww(jit_code_movi,u,v) jit_code_movr, jit_code_movi, + +#define jit_movnr(u,v,w) jit_new_node_www(jit_code_movnr,u,v,w) +#define jit_movzr(u,v,w) jit_new_node_www(jit_code_movzr,u,v,w) + jit_code_movnr, jit_code_movzr, + + jit_code_casr, jit_code_casi, +#define jit_casr(u, v, w, x) jit_new_node_wwq(jit_code_casr, u, v, w, x) +#define jit_casi(u, v, w, x) jit_new_node_wwq(jit_code_casi, u, v, w, x) + #define jit_extr_c(u,v) jit_new_node_ww(jit_code_extr_c,u,v) #define jit_extr_uc(u,v) jit_new_node_ww(jit_code_extr_uc,u,v) jit_code_extr_c, jit_code_extr_uc, @@ -351,6 +423,18 @@ typedef enum { #endif jit_code_extr_i, jit_code_extr_ui, +#define jit_bswapr_us(u,v) jit_new_node_ww(jit_code_bswapr_us,u,v) + jit_code_bswapr_us, +#define jit_bswapr_ui(u,v) jit_new_node_ww(jit_code_bswapr_ui,u,v) + jit_code_bswapr_ui, +#define jit_bswapr_ul(u,v) jit_new_node_ww(jit_code_bswapr_ul,u,v) + jit_code_bswapr_ul, +#if __WORDSIZE == 32 +#define jit_bswapr(u,v) jit_new_node_ww(jit_code_bswapr_ui,u,v) +#else +#define jit_bswapr(u,v) jit_new_node_ww(jit_code_bswapr_ul,u,v) +#endif + #define jit_htonr_us(u,v) jit_new_node_ww(jit_code_htonr_us,u,v) #define jit_ntohr_us(u,v) jit_new_node_ww(jit_code_htonr_us,u,v) jit_code_htonr_us, @@ -537,33 +621,106 @@ typedef enum { #define jit_prepare() _jit_prepare(_jit) jit_code_prepare, -#define jit_pushargr(u) _jit_pushargr(_jit,u) -#define jit_pushargi(u) _jit_pushargi(_jit,u) - jit_code_pushargr, jit_code_pushargi, + +#define jit_pushargr_c(u) _jit_pushargr(_jit,u,jit_code_pushargr_c) +#define jit_pushargi_c(u) _jit_pushargi(_jit,u,jit_code_pushargi_c) +#define jit_pushargr_uc(u) _jit_pushargr(_jit,u,jit_code_pushargr_uc) +#define jit_pushargi_uc(u) _jit_pushargi(_jit,u,jit_code_pushargi_uc) +#define jit_pushargr_s(u) _jit_pushargr(_jit,u,jit_code_pushargr_s) +#define jit_pushargi_s(u) _jit_pushargi(_jit,u,jit_code_pushargi_s) +#define jit_pushargr_us(u) _jit_pushargr(_jit,u,jit_code_pushargr_us) +#define jit_pushargi_us(u) _jit_pushargi(_jit,u,jit_code_pushargi_us) +#define jit_pushargr_i(u) _jit_pushargr(_jit,u,jit_code_pushargr_i) +#define jit_pushargi_i(u) _jit_pushargi(_jit,u,jit_code_pushargi_i) +#if __WORDSIZE == 32 +# define jit_pushargr(u) jit_pushargr_i(u) +# define jit_pushargi(u) jit_pushargi_i(u) +#else +# define jit_pushargr_ui(u) _jit_pushargr(_jit,u,jit_code_pushargr_ui) +# define jit_pushargi_ui(u) _jit_pushargi(_jit,u,jit_code_pushargi_ui) +# define jit_pushargr_l(u) _jit_pushargr(_jit,u,jit_code_pushargr_l) +# define jit_pushargi_l(u) _jit_pushargi(_jit,u,jit_code_pushargi_l) +# define jit_pushargr(u) jit_pushargr_l(u) +# define jit_pushargi(u) jit_pushargi_l(u) +#endif + jit_code_pushargr_c, jit_code_pushargi_c, + jit_code_pushargr_uc, jit_code_pushargi_uc, + jit_code_pushargr_s, jit_code_pushargi_s, + jit_code_pushargr_us, jit_code_pushargi_us, + jit_code_pushargr_i, jit_code_pushargi_i, + jit_code_pushargr_ui, jit_code_pushargi_ui, + jit_code_pushargr_l, jit_code_pushargi_l, +#if __WORDSIZE == 32 +# define jit_code_pushargr jit_code_pushargr_i +# define jit_code_pushargi jit_code_pushargi_i +#else +# define jit_code_pushargr jit_code_pushargr_l +# define jit_code_pushargi jit_code_pushargi_l +#endif + #define jit_finishr(u) _jit_finishr(_jit,u) #define jit_finishi(u) _jit_finishi(_jit,u) jit_code_finishr, jit_code_finishi, #define jit_ret() _jit_ret(_jit) jit_code_ret, -#define jit_retr(u) _jit_retr(_jit,u) -#define jit_reti(u) _jit_reti(_jit,u) - jit_code_retr, jit_code_reti, + +#define jit_retr_c(u) _jit_retr(_jit,u,jit_code_retr_c) +#define jit_reti_c(u) _jit_reti(_jit,u,jit_code_reti_c) +#define jit_retr_uc(u) _jit_retr(_jit,u,jit_code_retr_uc) +#define jit_reti_uc(u) _jit_reti(_jit,u,jit_code_reti_uc) +#define jit_retr_s(u) _jit_retr(_jit,u,jit_code_retr_s) +#define jit_reti_s(u) _jit_reti(_jit,u,jit_code_reti_s) +#define jit_retr_us(u) _jit_retr(_jit,u,jit_code_retr_us) +#define jit_reti_us(u) _jit_reti(_jit,u,jit_code_reti_us) +#define jit_retr_i(u) _jit_retr(_jit,u,jit_code_retr_i) +#define jit_reti_i(u) _jit_reti(_jit,u,jit_code_reti_i) +#if __WORDSIZE == 32 +# define jit_retr(u) jit_retr_i(u) +# define jit_reti(u) jit_reti_i(u) +#else +# define jit_retr_ui(u) _jit_retr(_jit,u,jit_code_retr_ui) +# define jit_reti_ui(u) _jit_reti(_jit,u,jit_code_reti_ui) +# define jit_retr_l(u) _jit_retr(_jit,u,jit_code_retr_l) +# define jit_reti_l(u) _jit_reti(_jit,u,jit_code_reti_l) +# define jit_retr(u) jit_retr_l(u) +# define jit_reti(u) jit_reti_l(u) +#endif + jit_code_retr_c, jit_code_reti_c, + jit_code_retr_uc, jit_code_reti_uc, + jit_code_retr_s, jit_code_reti_s, + jit_code_retr_us, jit_code_reti_us, + jit_code_retr_i, jit_code_reti_i, + jit_code_retr_ui, jit_code_reti_ui, + jit_code_retr_l, jit_code_reti_l, +#if __WORDSIZE == 32 +# define jit_code_retr jit_code_retr_i +# define jit_code_reti jit_code_reti_i +#else +# define jit_code_retr jit_code_retr_l +# define jit_code_reti jit_code_reti_l +#endif + #define jit_retval_c(u) _jit_retval_c(_jit,u) #define jit_retval_uc(u) _jit_retval_uc(_jit,u) - jit_code_retval_c, jit_code_retval_uc, #define jit_retval_s(u) _jit_retval_s(_jit,u) #define jit_retval_us(u) _jit_retval_us(_jit,u) - jit_code_retval_s, jit_code_retval_us, #define jit_retval_i(u) _jit_retval_i(_jit,u) #if __WORDSIZE == 32 # define jit_retval(u) jit_retval_i(u) #else -# define jit_retval(u) jit_retval_l(u) # define jit_retval_ui(u) _jit_retval_ui(_jit,u) # define jit_retval_l(u) _jit_retval_l(_jit,u) +# define jit_retval(u) jit_retval_l(u) #endif + jit_code_retval_c, jit_code_retval_uc, + jit_code_retval_s, jit_code_retval_us, jit_code_retval_i, jit_code_retval_ui, jit_code_retval_l, +#if __WORDSIZE == 32 +# define jit_code_retval jit_code_retval_i +#else +# define jit_code_retval jit_code_retval_l +#endif #define jit_epilog() _jit_epilog(_jit) jit_code_epilog, @@ -891,6 +1048,14 @@ typedef enum { #define jit_movr_d_w(u, v) jit_new_node_ww(jit_code_movr_d_w, u, v) #define jit_movi_d_w(u, v) jit_new_node_wd(jit_code_movi_d_w, u, v) +#define jit_clor(u,v) jit_new_node_ww(jit_code_clor,u,v) +#define jit_clzr(u,v) jit_new_node_ww(jit_code_clzr,u,v) + jit_code_clor, jit_code_clzr, + +#define jit_ctor(u,v) jit_new_node_ww(jit_code_ctor,u,v) +#define jit_ctzr(u,v) jit_new_node_ww(jit_code_ctzr,u,v) + jit_code_ctor, jit_code_ctzr, + jit_code_last_code } jit_code_t; @@ -931,7 +1096,8 @@ extern jit_int32_t _jit_allocai(jit_state_t*, jit_int32_t); extern void _jit_allocar(jit_state_t*, jit_int32_t, jit_int32_t); extern void _jit_ellipsis(jit_state_t*); -extern jit_node_t *_jit_arg(jit_state_t*); +extern jit_node_t *_jit_arg(jit_state_t*, jit_code_t); + extern void _jit_getarg_c(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_uc(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_s(jit_state_t*, jit_gpr_t, jit_node_t*); @@ -941,19 +1107,24 @@ extern void _jit_getarg_i(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_ui(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_l(jit_state_t*, jit_gpr_t, jit_node_t*); #endif -extern void _jit_putargr(jit_state_t*, jit_gpr_t, jit_node_t*); -extern void _jit_putargi(jit_state_t*, jit_word_t, jit_node_t*); + +extern void _jit_putargr(jit_state_t*, jit_gpr_t, jit_node_t*, jit_code_t); +extern void _jit_putargi(jit_state_t*, jit_word_t, jit_node_t*, jit_code_t); extern void _jit_prepare(jit_state_t*); extern void _jit_ellipsis(jit_state_t*); extern void _jit_va_push(jit_state_t*, jit_gpr_t); -extern void _jit_pushargr(jit_state_t*, jit_gpr_t); -extern void _jit_pushargi(jit_state_t*, jit_word_t); + +extern void _jit_pushargr(jit_state_t*, jit_gpr_t, jit_code_t); +extern void _jit_pushargi(jit_state_t*, jit_word_t, jit_code_t); + extern void _jit_finishr(jit_state_t*, jit_gpr_t); extern jit_node_t *_jit_finishi(jit_state_t*, jit_pointer_t); extern void _jit_ret(jit_state_t*); -extern void _jit_retr(jit_state_t*, jit_gpr_t); -extern void _jit_reti(jit_state_t*, jit_word_t); + +extern void _jit_retr(jit_state_t*, jit_gpr_t, jit_code_t); +extern void _jit_reti(jit_state_t*, jit_word_t, jit_code_t); + extern void _jit_retval_c(jit_state_t*, jit_gpr_t); extern void _jit_retval_uc(jit_state_t*, jit_gpr_t); extern void _jit_retval_s(jit_state_t*, jit_gpr_t); @@ -963,6 +1134,7 @@ extern void _jit_retval_i(jit_state_t*, jit_gpr_t); extern void _jit_retval_ui(jit_state_t*, jit_gpr_t); extern void _jit_retval_l(jit_state_t*, jit_gpr_t); #endif + extern void _jit_epilog(jit_state_t*); #define jit_patch(u) _jit_patch(_jit,u) @@ -987,6 +1159,10 @@ extern void _jit_frame(jit_state_t*, jit_int32_t); extern void _jit_tramp(jit_state_t*, jit_int32_t); #define jit_emit() _jit_emit(_jit) extern jit_pointer_t _jit_emit(jit_state_t*); +#define jit_unprotect() _jit_unprotect(_jit) +extern void _jit_unprotect(jit_state_t*); +#define jit_protect() _jit_protect(_jit) +extern void _jit_protect(jit_state_t*); #define jit_print() _jit_print(_jit) extern void _jit_print(jit_state_t*); @@ -1011,6 +1187,12 @@ extern void _jit_retr_d(jit_state_t*, jit_fpr_t); extern void _jit_reti_d(jit_state_t*, jit_float64_t); extern void _jit_retval_d(jit_state_t*, jit_fpr_t); +#define jit_get_reg(s) _jit_get_reg(_jit,s) +extern jit_int32_t _jit_get_reg(jit_state_t*, jit_int32_t); + +#define jit_unget_reg(r) _jit_unget_reg(_jit,r) +extern void _jit_unget_reg(jit_state_t*, jit_int32_t); + #define jit_new_node(c) _jit_new_node(_jit,c) extern jit_node_t *_jit_new_node(jit_state_t*, jit_code_t); #define jit_new_node_w(c,u) _jit_new_node_w(_jit,c,u) @@ -1053,6 +1235,10 @@ extern jit_node_t *_jit_new_node_www(jit_state_t*, jit_code_t, extern jit_node_t *_jit_new_node_qww(jit_state_t*, jit_code_t, jit_int32_t, jit_int32_t, jit_word_t, jit_word_t); +#define jit_new_node_wwq(c,u,v,l,h) _jit_new_node_wwq(_jit,c,u,v,l,h) +extern jit_node_t *_jit_new_node_wwq(jit_state_t*, jit_code_t, + jit_word_t, jit_word_t, + jit_int32_t, jit_int32_t); #define jit_new_node_wwf(c,u,v,w) _jit_new_node_wwf(_jit,c,u,v,w) extern jit_node_t *_jit_new_node_wwf(jit_state_t*, jit_code_t, jit_word_t, jit_word_t, jit_float32_t); diff --git a/parallel-rsp/lightning/include/lightning.h.in b/parallel-rsp/lightning/include/lightning.h.in index 70560c9..7aa654c 100644 --- a/parallel-rsp/lightning/include/lightning.h.in +++ b/parallel-rsp/lightning/include/lightning.h.in @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -24,6 +24,7 @@ #include @MAYBE_INCLUDE_STDINT_H@ #include +#include #if defined(__hpux) && defined(__hppa__) # include @@ -123,6 +124,11 @@ typedef jit_int32_t jit_bool_t; typedef jit_int32_t jit_gpr_t; typedef jit_int32_t jit_fpr_t; +#if !defined(__powerpc__) && \ + (defined(__POWERPC__) || defined(__ppc__) || defined(__PPC__)) +#define __powerpc__ 1 +#endif + #if defined(__i386__) || defined(__x86_64__) # include #elif defined(__mips__) @@ -145,6 +151,8 @@ typedef jit_int32_t jit_fpr_t; # include #elif defined(__riscv) # include +#elif defined(__loongarch__) +# include #endif #define jit_flag_node 0x0001 /* patch node not absolute */ @@ -182,6 +190,8 @@ typedef enum { #define jit_align(u) jit_new_node_w(jit_code_align, u) jit_code_live, jit_code_align, jit_code_save, jit_code_load, +#define jit_skip(u) jit_new_node_w(jit_code_skip, u) + jit_code_skip, #define jit_name(u) _jit_name(_jit,u) jit_code_name, #define jit_note(u, v) _jit_note(_jit, u, v) @@ -202,27 +212,80 @@ typedef enum { #define jit_allocar(u, v) _jit_allocar(_jit,u,v) jit_code_allocai, jit_code_allocar, -#define jit_arg() _jit_arg(_jit) - jit_code_arg, +#define jit_arg_c() _jit_arg(_jit, jit_code_arg_c) +#define jit_arg_s() _jit_arg(_jit, jit_code_arg_s) +#define jit_arg_i() _jit_arg(_jit, jit_code_arg_i) +# if __WORDSIZE == 32 +# define jit_arg() jit_arg_i() +#else +# define jit_arg_l() _jit_arg(_jit, jit_code_arg_l) +# define jit_arg() jit_arg_l() +#endif + jit_code_arg_c, jit_code_arg_s, + jit_code_arg_i, jit_code_arg_l, +#if __WORDSIZE == 32 +# define jit_code_arg jit_code_arg_i +#else +# define jit_code_arg jit_code_arg_l +#endif + #define jit_getarg_c(u,v) _jit_getarg_c(_jit,u,v) #define jit_getarg_uc(u,v) _jit_getarg_uc(_jit,u,v) - jit_code_getarg_c, jit_code_getarg_uc, #define jit_getarg_s(u,v) _jit_getarg_s(_jit,u,v) #define jit_getarg_us(u,v) _jit_getarg_us(_jit,u,v) - jit_code_getarg_s, jit_code_getarg_us, #define jit_getarg_i(u,v) _jit_getarg_i(_jit,u,v) #if __WORDSIZE == 32 # define jit_getarg(u,v) jit_getarg_i(u,v) #else -# define jit_getarg(u,v) jit_getarg_l(u,v) # define jit_getarg_ui(u,v) _jit_getarg_ui(_jit,u,v) # define jit_getarg_l(u,v) _jit_getarg_l(_jit,u,v) +# define jit_getarg(u,v) jit_getarg_l(u,v) #endif + jit_code_getarg_c, jit_code_getarg_uc, + jit_code_getarg_s, jit_code_getarg_us, jit_code_getarg_i, jit_code_getarg_ui, jit_code_getarg_l, -# define jit_putargr(u,v) _jit_putargr(_jit,u,v) -# define jit_putargi(u,v) _jit_putargi(_jit,u,v) - jit_code_putargr, jit_code_putargi, +#if __WORDSIZE == 32 +# define jit_code_getarg jit_code_getarg_i +#else +# define jit_code_getarg jit_code_getarg_l +#endif + +#define jit_putargr_c(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_c) +#define jit_putargi_c(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_c) +#define jit_putargr_uc(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_uc) +#define jit_putargi_uc(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_uc) +#define jit_putargr_s(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_s) +#define jit_putargi_s(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_s) +#define jit_putargr_us(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_us) +#define jit_putargi_us(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_us) +#define jit_putargr_i(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_i) +#define jit_putargi_i(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_i) +#if __WORDSIZE == 32 +# define jit_putargr(u,v) jit_putargr_i(u,v) +# define jit_putargi(u,v) jit_putargi_i(u,v) +#else +# define jit_putargr_ui(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_ui) +# define jit_putargi_ui(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_ui) +# define jit_putargr_l(u,v) _jit_putargr(_jit,u,v,jit_code_putargr_l) +# define jit_putargi_l(u,v) _jit_putargi(_jit,u,v,jit_code_putargi_l) +# define jit_putargr(u,v) jit_putargr_l(u,v) +# define jit_putargi(u,v) jit_putargi_l(u,v) +#endif + jit_code_putargr_c, jit_code_putargi_c, + jit_code_putargr_uc, jit_code_putargi_uc, + jit_code_putargr_s, jit_code_putargi_s, + jit_code_putargr_us, jit_code_putargi_us, + jit_code_putargr_i, jit_code_putargi_i, + jit_code_putargr_ui, jit_code_putargi_ui, + jit_code_putargr_l, jit_code_putargi_l, +#if __WORDSIZE == 32 +# define jit_code_putargr jit_code_putargr_i +# define jit_code_putargi jit_code_putargi_i +#else +# define jit_code_putargr jit_code_putargr_l +# define jit_code_putargi jit_code_putargi_l +#endif #define jit_va_start(u) jit_new_node_w(jit_code_va_start, u) jit_code_va_start, @@ -339,6 +402,15 @@ typedef enum { #define jit_movr(u,v) jit_new_node_ww(jit_code_movr,u,v) #define jit_movi(u,v) jit_new_node_ww(jit_code_movi,u,v) jit_code_movr, jit_code_movi, + +#define jit_movnr(u,v,w) jit_new_node_www(jit_code_movnr,u,v,w) +#define jit_movzr(u,v,w) jit_new_node_www(jit_code_movzr,u,v,w) + jit_code_movnr, jit_code_movzr, + + jit_code_casr, jit_code_casi, +#define jit_casr(u, v, w, x) jit_new_node_wwq(jit_code_casr, u, v, w, x) +#define jit_casi(u, v, w, x) jit_new_node_wwq(jit_code_casi, u, v, w, x) + #define jit_extr_c(u,v) jit_new_node_ww(jit_code_extr_c,u,v) #define jit_extr_uc(u,v) jit_new_node_ww(jit_code_extr_uc,u,v) jit_code_extr_c, jit_code_extr_uc, @@ -351,6 +423,18 @@ typedef enum { #endif jit_code_extr_i, jit_code_extr_ui, +#define jit_bswapr_us(u,v) jit_new_node_ww(jit_code_bswapr_us,u,v) + jit_code_bswapr_us, +#define jit_bswapr_ui(u,v) jit_new_node_ww(jit_code_bswapr_ui,u,v) + jit_code_bswapr_ui, +#define jit_bswapr_ul(u,v) jit_new_node_ww(jit_code_bswapr_ul,u,v) + jit_code_bswapr_ul, +#if __WORDSIZE == 32 +#define jit_bswapr(u,v) jit_new_node_ww(jit_code_bswapr_ui,u,v) +#else +#define jit_bswapr(u,v) jit_new_node_ww(jit_code_bswapr_ul,u,v) +#endif + #define jit_htonr_us(u,v) jit_new_node_ww(jit_code_htonr_us,u,v) #define jit_ntohr_us(u,v) jit_new_node_ww(jit_code_htonr_us,u,v) jit_code_htonr_us, @@ -537,33 +621,106 @@ typedef enum { #define jit_prepare() _jit_prepare(_jit) jit_code_prepare, -#define jit_pushargr(u) _jit_pushargr(_jit,u) -#define jit_pushargi(u) _jit_pushargi(_jit,u) - jit_code_pushargr, jit_code_pushargi, + +#define jit_pushargr_c(u) _jit_pushargr(_jit,u,jit_code_pushargr_c) +#define jit_pushargi_c(u) _jit_pushargi(_jit,u,jit_code_pushargi_c) +#define jit_pushargr_uc(u) _jit_pushargr(_jit,u,jit_code_pushargr_uc) +#define jit_pushargi_uc(u) _jit_pushargi(_jit,u,jit_code_pushargi_uc) +#define jit_pushargr_s(u) _jit_pushargr(_jit,u,jit_code_pushargr_s) +#define jit_pushargi_s(u) _jit_pushargi(_jit,u,jit_code_pushargi_s) +#define jit_pushargr_us(u) _jit_pushargr(_jit,u,jit_code_pushargr_us) +#define jit_pushargi_us(u) _jit_pushargi(_jit,u,jit_code_pushargi_us) +#define jit_pushargr_i(u) _jit_pushargr(_jit,u,jit_code_pushargr_i) +#define jit_pushargi_i(u) _jit_pushargi(_jit,u,jit_code_pushargi_i) +#if __WORDSIZE == 32 +# define jit_pushargr(u) jit_pushargr_i(u) +# define jit_pushargi(u) jit_pushargi_i(u) +#else +# define jit_pushargr_ui(u) _jit_pushargr(_jit,u,jit_code_pushargr_ui) +# define jit_pushargi_ui(u) _jit_pushargi(_jit,u,jit_code_pushargi_ui) +# define jit_pushargr_l(u) _jit_pushargr(_jit,u,jit_code_pushargr_l) +# define jit_pushargi_l(u) _jit_pushargi(_jit,u,jit_code_pushargi_l) +# define jit_pushargr(u) jit_pushargr_l(u) +# define jit_pushargi(u) jit_pushargi_l(u) +#endif + jit_code_pushargr_c, jit_code_pushargi_c, + jit_code_pushargr_uc, jit_code_pushargi_uc, + jit_code_pushargr_s, jit_code_pushargi_s, + jit_code_pushargr_us, jit_code_pushargi_us, + jit_code_pushargr_i, jit_code_pushargi_i, + jit_code_pushargr_ui, jit_code_pushargi_ui, + jit_code_pushargr_l, jit_code_pushargi_l, +#if __WORDSIZE == 32 +# define jit_code_pushargr jit_code_pushargr_i +# define jit_code_pushargi jit_code_pushargi_i +#else +# define jit_code_pushargr jit_code_pushargr_l +# define jit_code_pushargi jit_code_pushargi_l +#endif + #define jit_finishr(u) _jit_finishr(_jit,u) #define jit_finishi(u) _jit_finishi(_jit,u) jit_code_finishr, jit_code_finishi, #define jit_ret() _jit_ret(_jit) jit_code_ret, -#define jit_retr(u) _jit_retr(_jit,u) -#define jit_reti(u) _jit_reti(_jit,u) - jit_code_retr, jit_code_reti, + +#define jit_retr_c(u) _jit_retr(_jit,u,jit_code_retr_c) +#define jit_reti_c(u) _jit_reti(_jit,u,jit_code_reti_c) +#define jit_retr_uc(u) _jit_retr(_jit,u,jit_code_retr_uc) +#define jit_reti_uc(u) _jit_reti(_jit,u,jit_code_reti_uc) +#define jit_retr_s(u) _jit_retr(_jit,u,jit_code_retr_s) +#define jit_reti_s(u) _jit_reti(_jit,u,jit_code_reti_s) +#define jit_retr_us(u) _jit_retr(_jit,u,jit_code_retr_us) +#define jit_reti_us(u) _jit_reti(_jit,u,jit_code_reti_us) +#define jit_retr_i(u) _jit_retr(_jit,u,jit_code_retr_i) +#define jit_reti_i(u) _jit_reti(_jit,u,jit_code_reti_i) +#if __WORDSIZE == 32 +# define jit_retr(u) jit_retr_i(u) +# define jit_reti(u) jit_reti_i(u) +#else +# define jit_retr_ui(u) _jit_retr(_jit,u,jit_code_retr_ui) +# define jit_reti_ui(u) _jit_reti(_jit,u,jit_code_reti_ui) +# define jit_retr_l(u) _jit_retr(_jit,u,jit_code_retr_l) +# define jit_reti_l(u) _jit_reti(_jit,u,jit_code_reti_l) +# define jit_retr(u) jit_retr_l(u) +# define jit_reti(u) jit_reti_l(u) +#endif + jit_code_retr_c, jit_code_reti_c, + jit_code_retr_uc, jit_code_reti_uc, + jit_code_retr_s, jit_code_reti_s, + jit_code_retr_us, jit_code_reti_us, + jit_code_retr_i, jit_code_reti_i, + jit_code_retr_ui, jit_code_reti_ui, + jit_code_retr_l, jit_code_reti_l, +#if __WORDSIZE == 32 +# define jit_code_retr jit_code_retr_i +# define jit_code_reti jit_code_reti_i +#else +# define jit_code_retr jit_code_retr_l +# define jit_code_reti jit_code_reti_l +#endif + #define jit_retval_c(u) _jit_retval_c(_jit,u) #define jit_retval_uc(u) _jit_retval_uc(_jit,u) - jit_code_retval_c, jit_code_retval_uc, #define jit_retval_s(u) _jit_retval_s(_jit,u) #define jit_retval_us(u) _jit_retval_us(_jit,u) - jit_code_retval_s, jit_code_retval_us, #define jit_retval_i(u) _jit_retval_i(_jit,u) #if __WORDSIZE == 32 # define jit_retval(u) jit_retval_i(u) #else -# define jit_retval(u) jit_retval_l(u) # define jit_retval_ui(u) _jit_retval_ui(_jit,u) # define jit_retval_l(u) _jit_retval_l(_jit,u) +# define jit_retval(u) jit_retval_l(u) #endif + jit_code_retval_c, jit_code_retval_uc, + jit_code_retval_s, jit_code_retval_us, jit_code_retval_i, jit_code_retval_ui, jit_code_retval_l, +#if __WORDSIZE == 32 +# define jit_code_retval jit_code_retval_i +#else +# define jit_code_retval jit_code_retval_l +#endif #define jit_epilog() _jit_epilog(_jit) jit_code_epilog, @@ -891,6 +1048,14 @@ typedef enum { #define jit_movr_d_w(u, v) jit_new_node_ww(jit_code_movr_d_w, u, v) #define jit_movi_d_w(u, v) jit_new_node_wd(jit_code_movi_d_w, u, v) +#define jit_clor(u,v) jit_new_node_ww(jit_code_clor,u,v) +#define jit_clzr(u,v) jit_new_node_ww(jit_code_clzr,u,v) + jit_code_clor, jit_code_clzr, + +#define jit_ctor(u,v) jit_new_node_ww(jit_code_ctor,u,v) +#define jit_ctzr(u,v) jit_new_node_ww(jit_code_ctzr,u,v) + jit_code_ctor, jit_code_ctzr, + jit_code_last_code } jit_code_t; @@ -931,7 +1096,8 @@ extern jit_int32_t _jit_allocai(jit_state_t*, jit_int32_t); extern void _jit_allocar(jit_state_t*, jit_int32_t, jit_int32_t); extern void _jit_ellipsis(jit_state_t*); -extern jit_node_t *_jit_arg(jit_state_t*); +extern jit_node_t *_jit_arg(jit_state_t*, jit_code_t); + extern void _jit_getarg_c(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_uc(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_s(jit_state_t*, jit_gpr_t, jit_node_t*); @@ -941,19 +1107,24 @@ extern void _jit_getarg_i(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_ui(jit_state_t*, jit_gpr_t, jit_node_t*); extern void _jit_getarg_l(jit_state_t*, jit_gpr_t, jit_node_t*); #endif -extern void _jit_putargr(jit_state_t*, jit_gpr_t, jit_node_t*); -extern void _jit_putargi(jit_state_t*, jit_word_t, jit_node_t*); + +extern void _jit_putargr(jit_state_t*, jit_gpr_t, jit_node_t*, jit_code_t); +extern void _jit_putargi(jit_state_t*, jit_word_t, jit_node_t*, jit_code_t); extern void _jit_prepare(jit_state_t*); extern void _jit_ellipsis(jit_state_t*); extern void _jit_va_push(jit_state_t*, jit_gpr_t); -extern void _jit_pushargr(jit_state_t*, jit_gpr_t); -extern void _jit_pushargi(jit_state_t*, jit_word_t); + +extern void _jit_pushargr(jit_state_t*, jit_gpr_t, jit_code_t); +extern void _jit_pushargi(jit_state_t*, jit_word_t, jit_code_t); + extern void _jit_finishr(jit_state_t*, jit_gpr_t); extern jit_node_t *_jit_finishi(jit_state_t*, jit_pointer_t); extern void _jit_ret(jit_state_t*); -extern void _jit_retr(jit_state_t*, jit_gpr_t); -extern void _jit_reti(jit_state_t*, jit_word_t); + +extern void _jit_retr(jit_state_t*, jit_gpr_t, jit_code_t); +extern void _jit_reti(jit_state_t*, jit_word_t, jit_code_t); + extern void _jit_retval_c(jit_state_t*, jit_gpr_t); extern void _jit_retval_uc(jit_state_t*, jit_gpr_t); extern void _jit_retval_s(jit_state_t*, jit_gpr_t); @@ -963,6 +1134,7 @@ extern void _jit_retval_i(jit_state_t*, jit_gpr_t); extern void _jit_retval_ui(jit_state_t*, jit_gpr_t); extern void _jit_retval_l(jit_state_t*, jit_gpr_t); #endif + extern void _jit_epilog(jit_state_t*); #define jit_patch(u) _jit_patch(_jit,u) @@ -987,6 +1159,10 @@ extern void _jit_frame(jit_state_t*, jit_int32_t); extern void _jit_tramp(jit_state_t*, jit_int32_t); #define jit_emit() _jit_emit(_jit) extern jit_pointer_t _jit_emit(jit_state_t*); +#define jit_unprotect() _jit_unprotect(_jit) +extern void _jit_unprotect(jit_state_t*); +#define jit_protect() _jit_protect(_jit) +extern void _jit_protect(jit_state_t*); #define jit_print() _jit_print(_jit) extern void _jit_print(jit_state_t*); @@ -1011,6 +1187,12 @@ extern void _jit_retr_d(jit_state_t*, jit_fpr_t); extern void _jit_reti_d(jit_state_t*, jit_float64_t); extern void _jit_retval_d(jit_state_t*, jit_fpr_t); +#define jit_get_reg(s) _jit_get_reg(_jit,s) +extern jit_int32_t _jit_get_reg(jit_state_t*, jit_int32_t); + +#define jit_unget_reg(r) _jit_unget_reg(_jit,r) +extern void _jit_unget_reg(jit_state_t*, jit_int32_t); + #define jit_new_node(c) _jit_new_node(_jit,c) extern jit_node_t *_jit_new_node(jit_state_t*, jit_code_t); #define jit_new_node_w(c,u) _jit_new_node_w(_jit,c,u) @@ -1053,6 +1235,10 @@ extern jit_node_t *_jit_new_node_www(jit_state_t*, jit_code_t, extern jit_node_t *_jit_new_node_qww(jit_state_t*, jit_code_t, jit_int32_t, jit_int32_t, jit_word_t, jit_word_t); +#define jit_new_node_wwq(c,u,v,l,h) _jit_new_node_wwq(_jit,c,u,v,l,h) +extern jit_node_t *_jit_new_node_wwq(jit_state_t*, jit_code_t, + jit_word_t, jit_word_t, + jit_int32_t, jit_int32_t); #define jit_new_node_wwf(c,u,v,w) _jit_new_node_wwf(_jit,c,u,v,w) extern jit_node_t *_jit_new_node_wwf(jit_state_t*, jit_code_t, jit_word_t, jit_word_t, jit_float32_t); diff --git a/parallel-rsp/lightning/include/lightning/Makefile.am b/parallel-rsp/lightning/include/lightning/Makefile.am index 9b1b3e6..e21bbaa 100644 --- a/parallel-rsp/lightning/include/lightning/Makefile.am +++ b/parallel-rsp/lightning/include/lightning/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright 2000, 2001, 2002, 2012-2019 Free Software Foundation, Inc. +# Copyright 2000, 2001, 2002, 2012-2022 Free Software Foundation, Inc. # # This file is part of GNU lightning. # @@ -63,3 +63,7 @@ if cpu_riscv lightning_include_HEADERS = \ jit_riscv.h endif +if cpu_loongarch +lightning_include_HEADERS = \ + jit_loongarch.h +endif diff --git a/parallel-rsp/lightning/include/lightning/jit_aarch64.h b/parallel-rsp/lightning/include/lightning/jit_aarch64.h index 6e7d8be..3086499 100644 --- a/parallel-rsp/lightning/include/lightning/jit_aarch64.h +++ b/parallel-rsp/lightning/include/lightning/jit_aarch64.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -23,6 +23,10 @@ #define JIT_HASH_CONSTS 0 #define JIT_NUM_OPERANDS 3 +#if __APPLE__ +# define PACKED_STACK 1 +#endif + /* * Types */ diff --git a/parallel-rsp/lightning/include/lightning/jit_alpha.h b/parallel-rsp/lightning/include/lightning/jit_alpha.h index 9bae343..7986c34 100644 --- a/parallel-rsp/lightning/include/lightning/jit_alpha.h +++ b/parallel-rsp/lightning/include/lightning/jit_alpha.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2019 Free Software Foundation, Inc. + * Copyright (C) 2014-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/include/lightning/jit_arm.h b/parallel-rsp/lightning/include/lightning/jit_arm.h index 81451f1..0ed9535 100644 --- a/parallel-rsp/lightning/include/lightning/jit_arm.h +++ b/parallel-rsp/lightning/include/lightning/jit_arm.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -106,6 +106,9 @@ typedef enum { typedef struct { jit_uint32_t version : 4; + /* this field originally was only used for the 'e' in armv5te. + * it can also be used to force hardware division, if setting + * version to 7, telling it is armv7r or better. */ jit_uint32_t extend : 1; /* only generate thumb instructions for thumb2 */ jit_uint32_t thumb : 1; @@ -117,6 +120,12 @@ typedef struct { * due to some memory ordering constraint not being respected, so, * disable by default */ jit_uint32_t ldrt_strt : 1; + /* assume functions called never match jit instruction set? + * that is libc, gmp, mpfr, etc functions are in thumb mode and jit + * is in arm mode, or the reverse, what may cause a crash upon return + * of that function if generating jit for a relative jump. + */ + jit_uint32_t exchange : 1; } jit_cpu_t; /* diff --git a/parallel-rsp/lightning/include/lightning/jit_hppa.h b/parallel-rsp/lightning/include/lightning/jit_hppa.h index ddc3950..df361ba 100644 --- a/parallel-rsp/lightning/include/lightning/jit_hppa.h +++ b/parallel-rsp/lightning/include/lightning/jit_hppa.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/include/lightning/jit_ia64.h b/parallel-rsp/lightning/include/lightning/jit_ia64.h index 718f191..e45818a 100644 --- a/parallel-rsp/lightning/include/lightning/jit_ia64.h +++ b/parallel-rsp/lightning/include/lightning/jit_ia64.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -117,4 +117,13 @@ typedef enum { _NOREG, } jit_reg_t; +typedef struct { + jit_uint32_t clz : 1; +} jit_cpu_t; + +/* + * Initialization + */ +extern jit_cpu_t jit_cpu; + #endif /* _jit_ia64_h */ diff --git a/parallel-rsp/lightning/include/lightning/jit_loongarch.h b/parallel-rsp/lightning/include/lightning/jit_loongarch.h new file mode 100644 index 0000000..89b1a86 --- /dev/null +++ b/parallel-rsp/lightning/include/lightning/jit_loongarch.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2023 Free Software Foundation, Inc. + * + * This file is part of GNU lightning. + * + * GNU lightning is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 3, or (at your option) + * any later version. + * + * GNU lightning is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#ifndef _jit_loongarch_h +#define _jit_loongarch_h + +/* + * Types + */ +#define JIT_FP _FP +typedef enum { +#define jit_r_num() 9 +#define jit_r(i) (_T0 - (i)) +#define JIT_R0 _T0 +#define JIT_R1 _T1 +#define JIT_R2 _T2 +#define JIT_R3 _T3 +#define JIT_R4 _T4 +#define JIT_R5 _T5 +#define JIT_R6 _T6 +#define JIT_R7 _T7 +#define JIT_R8 _T8 + _T8, _T7, _T6, _T5, _T4, _T3, _T2, _T1, _T0, +#define jit_v_num() 9 +#define jit_v(i) (_S0 - (i)) +#define JIT_V0 _S0 +#define JIT_V1 _S1 +#define JIT_V2 _S2 +#define JIT_V3 _S3 +#define JIT_V4 _S4 +#define JIT_V5 _S5 +#define JIT_V6 _S6 +#define JIT_V7 _S7 +#define JIT_V8 _S8 + _S8, _S7, _S6, _S5, _S4, _S3, _S2, _S1, _S0, + _A7, _A6, _A5, _A4, _A3, _A2, _A1, _A0, + _FP, + _R21, + _ZERO, + _RA, + _TP, + _SP, + _FT0, _FT1, _FT2, _FT3, _FT4, _FT5, _FT6, _FT7, + _FT8, _FT9, _FT10, _FT11, _FT12, _FT13, _FT14, _FT15, + _FA7, _FA6, _FA5, _FA4, FA3, _FA2, _FA1, _FA0, +#define jit_f_num() 8 +#define jit_f(i) (_FS0 - (i)) +#define JIT_F0 _FS0 +#define JIT_F1 _FS1 +#define JIT_F2 _FS2 +#define JIT_F3 _FS3 +#define JIT_F4 _FS4 +#define JIT_F5 _FS5 +#define JIT_F6 _FS6 +#define JIT_F7 _FS7 + _FS7, _FS6, _FS5, _FS4, _FS3, _FS2, _FS1, _FS0, +#define JIT_NOREG _NOREG + _NOREG, +} jit_reg_t; + +#endif /* _jit_loongarch_h */ diff --git a/parallel-rsp/lightning/include/lightning/jit_mips.h b/parallel-rsp/lightning/include/lightning/jit_mips.h index eb7d783..b5c7be5 100644 --- a/parallel-rsp/lightning/include/lightning/jit_mips.h +++ b/parallel-rsp/lightning/include/lightning/jit_mips.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -34,9 +34,9 @@ typedef enum { #define jit_r(i) (_V0 + (i)) #if NEW_ABI -# define jit_r_num() 7 +# define jit_r_num() 8 #else -# define jit_r_num() 11 +# define jit_r_num() 12 #endif #define jit_v(i) (_S0 + (i)) #define jit_v_num() 8 @@ -55,6 +55,7 @@ typedef enum { # define JIT_R4 _T6 # define JIT_R5 _T7 # define JIT_R6 _T8 +# define JIT_R7 _T9 #else # define JIT_R2 _T0 # define JIT_R3 _T1 @@ -65,6 +66,7 @@ typedef enum { # define JIT_R8 _T6 # define JIT_R9 _T7 # define JIT_R10 _T8 +# define JIT_R11 _T9 #endif _V0, _V1, #if !NEW_ABI @@ -112,4 +114,13 @@ typedef enum { _NOREG, } jit_reg_t; +typedef struct { + jit_uint32_t release : 4; +} jit_cpu_t; + +/* + * Initialization + */ +extern jit_cpu_t jit_cpu; + #endif /* _jit_mips_h */ diff --git a/parallel-rsp/lightning/include/lightning/jit_ppc.h b/parallel-rsp/lightning/include/lightning/jit_ppc.h index f1bdbcb..460c491 100644 --- a/parallel-rsp/lightning/include/lightning/jit_ppc.h +++ b/parallel-rsp/lightning/include/lightning/jit_ppc.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -22,6 +22,9 @@ #define JIT_HASH_CONSTS 1 #define JIT_NUM_OPERANDS 3 +#if defined(_AIX) && !defined(_CALL_AIX) && !defined(_CALL_LINUX) +# define _CALL_AIXDESC 1 +#endif /* * Types diff --git a/parallel-rsp/lightning/include/lightning/jit_private.h b/parallel-rsp/lightning/include/lightning/jit_private.h index 8c05853..3da27dd 100644 --- a/parallel-rsp/lightning/include/lightning/jit_private.h +++ b/parallel-rsp/lightning/include/lightning/jit_private.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -55,6 +55,26 @@ # define HIDDEN /**/ #endif +#if PACKED_STACK || STRONG_TYPE_CHECKING +# define assert_arg_type(code, expect) \ + do assert((code) == (expect)); while (0) +# define assert_putarg_type(code, expect) \ + do \ + assert((((code) - jit_code_putargr_c) >> 2) == \ + ((expect) - jit_code_arg_c)); \ + while (0) +#else +# define assert_arg_type(code, expect) \ + do assert((int)(code) == (int)(expect) || \ + (code) == jit_code_arg); while (0) +# define assert_putarg_type(code, expect) \ + do \ + assert(((((code) - jit_code_putargr_c) >> 2) == \ + ((expect) - jit_code_arg_c)) || \ + ((code) == jit_code_arg)); \ + while (0) +#endif + #define rc(value) jit_class_##value #define rn(reg) (jit_regno(_rvs[jit_regno(reg)].spec)) @@ -150,6 +170,13 @@ typedef jit_uint64_t jit_regset_t; # define JIT_RET _A0 # define JIT_FRET _FA0 typedef jit_uint64_t jit_regset_t; +#elif defined(__loongarch__) +# define JIT_RA0 _A0 +# define JIT_FA0 _FA0 +# define JIT_SP _SP +# define JIT_RET _A0 +# define JIT_FRET _FA0 +typedef jit_uint64_t jit_regset_t; #endif #define jit_data(u,v,w) _jit_data(_jit,u,v,w) @@ -167,48 +194,80 @@ extern jit_node_t *_jit_data(jit_state_t*, const void*, (!jit_regset_tstbit(&_jitc->regarg, regno) && \ !jit_regset_tstbit(&_jitc->regsav, regno)) -#define jit_inc_synth(code) \ +#define jit_code_inc_synth(code) \ do { \ - (void)jit_new_node(jit_code_##code); \ + (void)jit_new_node(code); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_w(code, u) \ +#define jit_inc_synth(name) \ + jit_code_inc_synth(jit_code_##name) +#define jit_code_inc_synth_w(code, u) \ do { \ - (void)jit_new_node_w(jit_code_##code, u); \ + (void)jit_new_node_w(code, u); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_f(code, u) \ +#define jit_inc_synth_w(name, u) \ + jit_code_inc_synth_w(jit_code_##name, u) +#define jit_code_inc_synth_f(code, u) \ do { \ - (void)jit_new_node_f(jit_code_##code, u); \ + (void)jit_new_node_f(code, u); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_d(code, u) \ +#define jit_inc_synth_f(name, u) \ + jit_code_inc_synth_f(jit_code_##name, u) +#define jit_code_inc_synth_d(code, u) \ do { \ - (void)jit_new_node_d(jit_code_##code, u); \ + (void)jit_new_node_d(code, u); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_ww(code, u, v) \ +#define jit_inc_synth_d(name, u) \ + jit_code_inc_synth_d(jit_code_##name, u) +#define jit_code_inc_synth_ww(code, u, v) \ do { \ - (void)jit_new_node_ww(jit_code_##code, u, v); \ + (void)jit_new_node_ww(code, u, v); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_wp(code, u, v) \ +#define jit_inc_synth_ww(name, u, v) \ + jit_code_inc_synth_ww(jit_code_##name, u, v) +#define jit_code_inc_synth_wp(code, u, v) \ do { \ - (void)jit_new_node_wp(jit_code_##code, u, v); \ + (void)jit_new_node_wp(code, u, v); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_fp(code, u, v) \ +#define jit_inc_synth_wp(name, u, v) \ + jit_code_inc_synth_wp(jit_code_##name, u, v) +#define jit_code_inc_synth_fp(code, u, v) \ do { \ - (void)jit_new_node_fp(jit_code_##code, u, v); \ + (void)jit_new_node_fp(code, u, v); \ jit_synth_inc(); \ } while (0) -#define jit_inc_synth_dp(code, u, v) \ +#define jit_inc_synth_fp(name, u, v) \ + jit_code_inc_synth_fp(jit_code_##name, u, v) +#define jit_code_inc_synth_dp(code, u, v) \ do { \ - (void)jit_new_node_dp(jit_code_##code, u, v); \ + (void)jit_new_node_dp(code, u, v); \ jit_synth_inc(); \ } while (0) +#define jit_inc_synth_dp(name, u, v) \ + jit_code_inc_synth_dp(jit_code_##name, u, v) #define jit_dec_synth() jit_synth_dec() +#define jit_link_alist(node) \ + do { \ + node->link = _jitc->function->alist; \ + _jitc->function->alist = node; \ + } while (0) +#define jit_check_frame() \ + do { \ + if (!_jitc->function->need_frame) { \ + _jitc->again = 1; \ + _jitc->function->need_frame = 1; \ + } \ + } while (0) +#define jit_diffsize() (stack_framesize - _jitc->framesize) +#define jit_framesize() (stack_framesize - jit_diffsize()) +#define jit_selfsize() (_jitc->function->self.size - jit_diffsize()) + #define jit_link_prolog() \ do { \ _jitc->tail->link = _jitc->function->prolog->link; \ @@ -241,8 +300,8 @@ extern jit_node_t *_jit_data(jit_state_t*, const void*, #define jit_class_xpr 0x80000000 /* float / vector */ /* Used on sparc64 where %f0-%f31 can be encode for single float * but %f32 to %f62 only as double precision */ -#define jit_class_sng 0x10000000 /* Single precision float */ -#define jit_class_dbl 0x20000000 /* Only double precision float */ +#define jit_class_sng 0x00010000 /* Single precision float */ +#define jit_class_dbl 0x00020000 /* Only double precision float */ #define jit_regno_patch 0x00008000 /* this is a register * returned by a "user" call * to jit_get_reg() */ @@ -264,8 +323,9 @@ extern jit_node_t *_jit_data(jit_state_t*, const void*, #define jit_cc_a0_flt 0x00000020 /* arg0 is immediate float */ #define jit_cc_a0_dbl 0x00000040 /* arg0 is immediate double */ #define jit_cc_a0_arg 0x00000080 /* arg1 is an argument int id */ -#define jit_cc_a1_reg 0x00000100 /* arg1 is a register */ -#define jit_cc_a1_chg 0x00000200 /* arg1 is modified */ +#define jit_cc_a0_cnd 0x00000100 /* arg1 is a conditinally set register */ +#define jit_cc_a1_reg 0x00000200 /* arg1 is a register */ +#define jit_cc_a1_chg 0x00000400 /* arg1 is modified */ #define jit_cc_a1_int 0x00001000 /* arg1 is immediate word */ #define jit_cc_a1_flt 0x00002000 /* arg1 is immediate float */ #define jit_cc_a1_dbl 0x00004000 /* arg1 is immediate double */ @@ -275,6 +335,7 @@ extern jit_node_t *_jit_data(jit_state_t*, const void*, #define jit_cc_a2_int 0x00100000 /* arg2 is immediate word */ #define jit_cc_a2_flt 0x00200000 /* arg2 is immediate float */ #define jit_cc_a2_dbl 0x00400000 /* arg2 is immediate double */ +#define jit_cc_a2_rlh 0x00800000 /* arg2 is a register pair */ #if __ia64__ || (__sparc__ && __WORDSIZE == 64) extern void @@ -363,6 +424,8 @@ typedef struct jit_register jit_register_t; # if DISASSEMBLER typedef struct jit_data_info jit_data_info_t; # endif +#elif __riscv +typedef struct jit_const jit_const_t; #endif union jit_data { @@ -412,6 +475,9 @@ struct jit_block { jit_node_t *label; jit_regset_t reglive; jit_regset_t regmask; + jit_bool_t again; /* Flag need to rebuild regset masks + * due to changes in live and unknown + * state. */ }; struct jit_value { @@ -434,6 +500,12 @@ struct jit_data_info { jit_uword_t code; /* pointer in code buffer */ jit_word_t length; /* length of constant vector */ }; +#elif __riscv && __WORDSIZE == 64 +struct jit_const { + jit_word_t value; + jit_word_t address; + jit_const_t *next; +}; #endif struct jit_function { @@ -454,9 +526,14 @@ struct jit_function { } call; jit_node_t *prolog; jit_node_t *epilog; + jit_node_t *alist; jit_int32_t *regoff; jit_regset_t regset; jit_int32_t stack; +#if defined(__i386__) || defined(__x86_64__) + jit_int32_t cvt_offset; /* allocai'd offset for x87<->xmm or + * fpr<->gpr transfer using the stack */ +#endif /* Helper for common jit generation pattern, used in GNU Smalltalk * and possibly others, where a static frame layout is required or @@ -465,11 +542,25 @@ struct jit_function { jit_uint32_t define_frame : 1; jit_uint32_t assume_frame : 1; + jit_uint32_t need_frame : 1; /* need frame pointer? */ + jit_uint32_t need_stack : 1; /* need stack pointer? */ + jit_uint32_t need_return : 1; /* not a leaf function */ + /* alloca offset offset */ jit_int32_t aoffoff; /* uses allocar flag */ jit_uint32_t allocar : 1; +#if __arm__ + /* If will, or might use float registers and vfp is not available. + * Use the first 64 bytes always, as the access to the virtual float + * registers use hardcoded instructions that can only reach 64 byte + * displacements, and to keep code simpler, do not use temporaries. */ + jit_uint32_t swf_offset : 1; + /* If need to call C functions for some operation, or variadic function */ + jit_uint32_t save_reg_args : 1; +#endif + /* varargs state offsets */ jit_int32_t vaoff; /* offset of jit_va_list */ jit_int32_t vagp; /* first gp va argument */ @@ -490,7 +581,7 @@ struct jit_compiler { jit_int32_t breg; /* base register for prolog/epilog */ #endif #if __mips__ || __ia64__ || __alpha__ || \ - (__sparc__ && __WORDSIZE == 64) || __riscv + (__sparc__ && __WORDSIZE == 64) || __riscv || __loongarch__ jit_int32_t carry; #define jit_carry _jitc->carry #endif @@ -508,11 +599,14 @@ struct jit_compiler { #endif jit_uint32_t no_data : 1; jit_uint32_t no_note : 1; + jit_int32_t framesize; /* space for callee save registers, + * frame pointer and return address */ jit_int32_t reglen; /* number of registers */ jit_regset_t regarg; /* cannot allocate */ jit_regset_t regsav; /* automatic spill only once */ jit_regset_t reglive; /* known live registers at some point */ jit_regset_t regmask; /* register mask to update reglive */ + jit_regset_t explive; /* explicitly marked as live */ struct { jit_uint8_t *end; } code; @@ -593,6 +687,27 @@ struct jit_compiler { jit_word_t length; } prolog; jit_bool_t jump; +#elif __riscv && __WORDSIZE == 64 + struct { + /* Hash table for constants to be resolved and patched */ + struct { + jit_const_t **table; /* very simple hash table */ + jit_word_t size; /* number of vectors in table */ + jit_word_t count; /* number of distinct entries */ + } hash; + struct { + jit_const_t **ptr; /* keep a single pointer */ + jit_const_t *list; /* free list */ + jit_word_t length; /* length of pool */ + } pool; + /* Linear list for constants that cannot be encoded easily */ + struct { + jit_word_t *instrs; /* list of direct movi instructions */ + jit_word_t *values; /* list of direct movi constants */ + jit_word_t offset; /* offset in instrs/values vector */ + jit_word_t length; /* length of instrs/values vector */ + } vector; + } consts; #endif #if GET_JIT_SIZE /* Temporary storage to calculate instructions length */ @@ -616,6 +731,8 @@ struct jit_state { struct { jit_uint8_t *ptr; jit_word_t length; + /* PROTECTED bytes starting at PTR are mprotect'd. */ + jit_word_t protected; } code; struct { jit_uint8_t *ptr; @@ -672,14 +789,6 @@ _jit_regarg_set(jit_state_t*, jit_node_t*, jit_int32_t); extern void _jit_regarg_clr(jit_state_t*, jit_node_t*, jit_int32_t); -#define jit_get_reg(s) _jit_get_reg(_jit,s) -extern jit_int32_t -_jit_get_reg(jit_state_t*, jit_int32_t); - -#define jit_unget_reg(r) _jit_unget_reg(_jit,r) -extern void -_jit_unget_reg(jit_state_t*, jit_int32_t); - #define jit_save(reg) _jit_save(_jit, reg) extern void _jit_save(jit_state_t*, jit_int32_t); @@ -726,6 +835,7 @@ _emit_ldxi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); extern void _emit_stxi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +extern void jit_init_print(void); extern void jit_init_debug(const char*); extern void jit_finish_debug(void); diff --git a/parallel-rsp/lightning/include/lightning/jit_riscv.h b/parallel-rsp/lightning/include/lightning/jit_riscv.h index 1b4f93d..bf59c5b 100644 --- a/parallel-rsp/lightning/include/lightning/jit_riscv.h +++ b/parallel-rsp/lightning/include/lightning/jit_riscv.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Free Software Foundation, Inc. + * Copyright (C) 2019-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/include/lightning/jit_s390.h b/parallel-rsp/lightning/include/lightning/jit_s390.h index 6ab196b..d51cfec 100644 --- a/parallel-rsp/lightning/include/lightning/jit_s390.h +++ b/parallel-rsp/lightning/include/lightning/jit_s390.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -65,4 +65,13 @@ typedef enum { #define JIT_NOREG _NOREG } jit_reg_t; +typedef struct { + jit_uint32_t flogr : 1; +} jit_cpu_t; + +/* + * Initialization + */ +extern jit_cpu_t jit_cpu; + #endif /* _jit_s390_h */ diff --git a/parallel-rsp/lightning/include/lightning/jit_sparc.h b/parallel-rsp/lightning/include/lightning/jit_sparc.h index bee440b..ec21be9 100644 --- a/parallel-rsp/lightning/include/lightning/jit_sparc.h +++ b/parallel-rsp/lightning/include/lightning/jit_sparc.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -99,4 +99,13 @@ typedef enum { _NOREG, } jit_reg_t; +typedef struct { + jit_uint32_t lzcnt : 1; +} jit_cpu_t; + +/* + * Initialization + */ +extern jit_cpu_t jit_cpu; + #endif /* _jit_sparc_h */ diff --git a/parallel-rsp/lightning/include/lightning/jit_x86.h b/parallel-rsp/lightning/include/lightning/jit_x86.h index a278d06..4c48013 100644 --- a/parallel-rsp/lightning/include/lightning/jit_x86.h +++ b/parallel-rsp/lightning/include/lightning/jit_x86.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -189,6 +189,10 @@ typedef struct { jit_uint32_t avx : 1; /* lahf/sahf available in 64 bits mode */ jit_uint32_t lahf : 1; + /* lzcnt and tzcnt? */ + jit_uint32_t abm : 1; + /* adcx and adox instructions available? */ + jit_uint32_t adx : 1; } jit_cpu_t; /* diff --git a/parallel-rsp/lightning/lib/Makefile.am b/parallel-rsp/lightning/lib/Makefile.am index f2ac2ba..44ac4f2 100644 --- a/parallel-rsp/lightning/lib/Makefile.am +++ b/parallel-rsp/lightning/lib/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright 2000, 2001, 2002, 2012-2019 Free Software Foundation, Inc. +# Copyright 2000, 2001, 2002, 2012-2022 Free Software Foundation, Inc. # # This file is part of GNU lightning. # @@ -14,27 +14,34 @@ # License for more details. # -AM_CFLAGS = -I$(top_srcdir)/include -D_GNU_SOURCE $(LIGHTNING_CFLAGS) +AM_CFLAGS = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -D_GNU_SOURCE $(LIGHTNING_CFLAGS) liblightning_LTLIBRARIES = liblightning.la -liblightning_la_LDFLAGS = -version-info 1:0:0 +liblightning_la_LDFLAGS = -version-info 2:0:0 +AM_CPPFLAGS = if get_jit_size JIT_SIZE_PATH = "$(top_builddir)/jit_$(cpu)-sz.c" -AM_CPPFLAGS=-DGET_JIT_SIZE=1 -DJIT_SIZE_PATH='$(JIT_SIZE_PATH)' +AM_CPPFLAGS += -DGET_JIT_SIZE=1 -DJIT_SIZE_PATH='$(JIT_SIZE_PATH)' +endif +if strong_type_checking +AM_CPPFLAGS += -DSTRONG_TYPE_CHECKING=1 endif liblightningdir = $(libdir) liblightning_la_SOURCES = \ jit_disasm.c \ jit_memory.c \ - jit_names.c \ jit_note.c \ jit_print.c \ jit_size.c \ lightning.c EXTRA_DIST = \ + jit_names.c \ + jit_fallback.c \ jit_rewind.c \ + aarch64-logical-immediates.c \ jit_aarch64.c \ jit_aarch64-cpu.c \ jit_aarch64-fpu.c \ @@ -56,6 +63,10 @@ EXTRA_DIST = \ jit_ia64-cpu.c \ jit_ia64-fpu.c \ jit_ia64-sz.c \ + jit_loongarch.c \ + jit_loongarch-cpu.c \ + jit_loongarch-fpu.c \ + jit_loongarch-sz.c \ jit_mips.c \ jit_mips-cpu.c \ jit_mips-fpu.c \ diff --git a/parallel-rsp/lightning/lib/aarch64-logical-immediates.c b/parallel-rsp/lightning/lib/aarch64-logical-immediates.c new file mode 100644 index 0000000..c1e1ab0 --- /dev/null +++ b/parallel-rsp/lightning/lib/aarch64-logical-immediates.c @@ -0,0 +1,161 @@ +// AArch64 Logical Immediate Encoding and Decoding +// +// I hereby place this code in the public domain, as per the terms of the +// CC0 license: https://creativecommons.org/publicdomain/zero/1.0/ + +#include +#include + +static inline int nonzeroCountTrailingZeros64(uint64_t n) { + return __builtin_ctzll(n); +} + +static inline int countTrailingZeros64(uint64_t n) { + return n ? nonzeroCountTrailingZeros64(n) : 64; +} + +static inline int nonzeroCountLeadingZeros64(uint64_t n) { + return __builtin_clzll(n); +} + +static inline int nonzeroCountLeadingZeros32(uint32_t n) { + return __builtin_clz(n); +} + +static inline uint64_t rotateRight64(uint64_t v, int n) { + // return __builtin_rotateright64(v, n); + return (v >> (n & 63)) | (v << (-n & 63)); +} + +static inline uint64_t clearTrailingOnes64(uint64_t n) { + return n & (n + 1); +} + +#define ENCODE_FAILED (-1) + +int encodeLogicalImmediate64(uint64_t val) { + // Consider an ARM64 logical immediate as a pattern of "o" ones preceded + // by "z" more-significant zeroes, repeated to fill a 64-bit integer. + // o > 0, z > 0, and the size (o + z) is a power of two in [2,64]. This + // part of the pattern is encoded in the fields "imms" and "N". + // + // "immr" encodes a further right rotate of the repeated pattern, allowing + // a wide range of useful bitwise constants to be represented. + // + // (The spec describes the "immr" rotate as rotating the "o + z" bit + // pattern before repeating it to fill 64-bits, but, as it's a repeating + // pattern, rotating afterwards is equivalent.) + + // This encoding is not allowed to represent all-zero or all-one values. + if (val == 0 || ~val == 0) + return ENCODE_FAILED; + + // To detect an immediate that may be encoded in this scheme, we first + // remove the right-rotate, by rotating such that the least significant + // bit is a one and the most significant bit is a zero. + // + // We do this by clearing any trailing one bits, then counting the + // trailing zeroes. This finds an "edge", where zero goes to one. + // We then rotate the original value right by that amount, moving + // the first one to the least significant bit. + + int rotation = countTrailingZeros64(clearTrailingOnes64(val)); + uint64_t normalized = rotateRight64(val, rotation & 63); + + // Now we have normalized the value, and determined the rotation, we can + // determine "z" by counting the leading zeroes, and "o" by counting the + // trailing ones. (These will both be positive, as we already rejected 0 + // and ~0, and rotated the value to start with a zero and end with a one.) + + int zeroes = nonzeroCountLeadingZeros64(normalized); + int ones = nonzeroCountTrailingZeros64(~normalized); + int size = zeroes + ones; + + // Detect the repeating pattern (by comparing every repetition to the + // one next to it, using rotate). + + if (rotateRight64(val, size & 63) != val) + return ENCODE_FAILED; + + // We do not need to further validate size to ensure it is a power of two + // between 2 and 64. The only "minimal" patterns that can repeat to fill a + // 64-bit value must have a length that is a factor of 64 (i.e. it is a + // power of two in the range [1,64]). And our pattern cannot be of length + // one (as we already rejected 0 and ~0). + // + // By "minimal" patterns I refer to patterns which do not themselves + // contain repetitions. For example, '010101' is a non-minimal pattern of + // a non-power-of-two length that can pass the above rotational test. It + // consists of the minimal pattern '01'. All our patterns are minimal, as + // they contain only one contiguous run of ones separated by at least one + // zero. + + // Finally, we encode the values. "rotation" is the amount we rotated + // right by to "undo" the right-rotate encoded in immr, so must be + // negated. + + // size 2: N=0 immr=00000r imms=11110s + // size 4: N=0 immr=0000rr imms=1110ss + // size 8: N=0 immr=000rrr imms=110sss + // size 16: N=0 immr=00rrrr imms=10ssss + // size 32: N=0 immr=0rrrrr imms=0sssss + // size 64: N=1 immr=rrrrrr imms=ssssss + int immr = -rotation & (size - 1); + int imms = -(size << 1) | (ones - 1); + int N = (size >> 6); + + return (N << 12) | (immr << 6) | (imms & 0x3f); +} + +int encodeLogicalImmediate32(uint32_t val) { + return encodeLogicalImmediate64(((uint64_t)val << 32) | val); +} + +// Decoding! + +bool isValidLogicalImmediate64(unsigned val) { + unsigned N = (val >> 12) & 1; + unsigned imms = val & 0x3f; + unsigned pattern = (N << 6) | (~imms & 0x3f); + return (pattern & (pattern - 1)) != 0; +} + +bool isValidLogicalImmediate32(unsigned val) { + unsigned N = (val >> 12) & 1; + return N == 0 && isValidLogicalImmediate64(val); +} + +#define DECODE_FAILED 0 + +// returns DECODE_FAILED (zero) if the encoding is invalid +uint64_t decodeLogicalImmediate64(unsigned val) { + // Fun way to generate the immediates with mask ^ (mask << S) + static const uint64_t mask_lookup[] = { + 0xffffffffffffffff, // size = 64 + 0x00000000ffffffff, // size = 32 + 0x0000ffff0000ffff, // size = 16 + 0x00ff00ff00ff00ff, // size = 8 + 0x0f0f0f0f0f0f0f0f, // size = 4 + 0x3333333333333333, // size = 2 + }; + + unsigned N = (val >> 12) & 1; + int immr = (val >> 6) & 0x3f; + unsigned imms = val & 0x3f; + + unsigned pattern = (N << 6) | (~imms & 0x3f); + + if (!(pattern & (pattern - 1))) return DECODE_FAILED; + + int leading_zeroes = nonzeroCountLeadingZeros32(pattern); + unsigned imms_mask = 0x7fffffff >> leading_zeroes; + uint64_t mask = mask_lookup[leading_zeroes - 25]; + unsigned S = (imms + 1) & imms_mask; + return rotateRight64(mask ^ (mask << S), immr); +} + +uint32_t decodeLogicalImmediate32(unsigned val) { + unsigned N = (val >> 12) & 1; + if (N) return DECODE_FAILED; + return (uint32_t)decodeLogicalImmediate64(val); +} diff --git a/parallel-rsp/lightning/lib/jit_aarch64-cpu.c b/parallel-rsp/lightning/lib/jit_aarch64-cpu.c index 8e8a9a0..d5e64ad 100644 --- a/parallel-rsp/lightning/lib/jit_aarch64-cpu.c +++ b/parallel-rsp/lightning/lib/jit_aarch64-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -210,7 +210,7 @@ typedef union { jit_int32_t w; # undef ui } instr_t; -# define stack_framesize 160 +# define s26_p(d) ((d) >= -33554432 && (d) <= 33554431) # define ii(i) *_jit->pc.ui++ = i # define ldr(r0,r1) ldr_l(r0,r1) # define ldxr(r0,r1,r2) ldxr_l(r0,r1,r2) @@ -290,6 +290,7 @@ typedef union { # define A64_CBNZ 0x35000000 # define A64_B_C 0x54000000 # define A64_CSINC 0x1a800400 +# define A64_CSSEL 0x1a800000 # define A64_REV 0xdac00c00 # define A64_UDIV 0x1ac00800 # define A64_SDIV 0x1ac00c00 @@ -317,6 +318,8 @@ typedef union { # define A64_LDRSB 0x38e06800 # define A64_STR 0xf8206800 # define A64_LDR 0xf8606800 +# define A64_LDAXR 0xc85ffc00 +# define A64_STLXR 0xc800fc00 # define A64_STRH 0x78206800 # define A64_LDRH 0x78606800 # define A64_LDRSH 0x78a06800 @@ -346,6 +349,9 @@ typedef union { # define A64_ORR 0x2a000000 # define A64_MOV 0x2a0003e0 /* AKA orr Rd,xzr,Rm */ # define A64_MVN 0x2a2003e0 +# define A64_CLS 0x5ac01400 +# define A64_CLZ 0x5ac01000 +# define A64_RBIT 0x5ac00000 # define A64_UXTW 0x2a0003e0 /* AKA MOV */ # define A64_EOR 0x4a000000 # define A64_ANDS 0x6a000000 @@ -367,6 +373,9 @@ typedef union { # define MOV(Rd,Rm) ox_x(A64_MOV|XS,Rd,Rm) # define MVN(Rd,Rm) ox_x(A64_MVN|XS,Rd,Rm) # define NEG(Rd,Rm) ox_x(A64_NEG|XS,Rd,Rm) +# define CLS(Rd,Rm) o_xx(A64_CLS|XS,Rd,Rm) +# define CLZ(Rd,Rm) o_xx(A64_CLZ|XS,Rd,Rm) +# define RBIT(Rd,Rm) o_xx(A64_RBIT|XS,Rd,Rm) # define MOVN(Rd,Imm16) ox_h(A64_MOVN|XS,Rd,Imm16) # define MOVN_16(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_16,Rd,Imm16) # define MOVN_32(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_32,Rd,Imm16) @@ -444,6 +453,8 @@ typedef union { # define LDR(Rt,Rn,Rm) oxxx(A64_LDR,Rt,Rn,Rm) # define LDRI(Rt,Rn,Imm12) oxxi(A64_LDRI,Rt,Rn,Imm12) # define LDUR(Rt,Rn,Imm9) oxx9(A64_LDUR,Rt,Rn,Imm9) +# define LDAXR(Rt,Rn) o_xx(A64_LDAXR,Rt,Rn) +# define STLXR(Rs,Rt,Rn) oxxx(A64_STLXR,Rs,Rn,Rt) # define STRB(Rt,Rn,Rm) oxxx(A64_STRB,Rt,Rn,Rm) # define STRBI(Rt,Rn,Imm12) oxxi(A64_STRBI,Rt,Rn,Imm12) # define STURB(Rt,Rn,Imm9) oxx9(A64_STURB,Rt,Rn,Imm9) @@ -461,6 +472,7 @@ typedef union { # define LDPI_PRE(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP_PRE|XS,Rt,Rt2,Rn,Simm7) # define STPI_POS(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP_POS|XS,Rt,Rt2,Rn,Simm7) # define CSET(Rd,Cc) CSINC(Rd,XZR_REGNO,XZR_REGNO,Cc) +# define CSEL(Rd,Rn,Rm,Cc) oxxxc(A64_CSSEL|XS,Rd,Rn,Rm,Cc) # define B(Simm26) o26(A64_B,Simm26) # define BL(Simm26) o26(A64_BL,Simm26) # define BR(Rn) o_x_(A64_BR,Rn) @@ -572,8 +584,20 @@ static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); # define rshr_u(r0,r1,r2) LSR(r0,r1,r2) # define rshi_u(r0,r1,i0) _rshi_u(_jit,r0,r1,i0) static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define negr(r0,r1) NEG(r0,r1) # define comr(r0,r1) MVN(r0,r1) +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) CLZ(r0,r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # define andr(r0,r1,r2) AND(r0,r1,r2) # define andi(r0,r1,i0) _andi(_jit,r0,r1,i0) static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); @@ -657,23 +681,22 @@ static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); # define stxr_l(r0,r1,r2) STR(r2,r1,r0) # define stxi_l(i0,r0,r1) _stxi_l(_jit,i0,r0,r1) static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -# if __BYTE_ORDER == __LITTLE_ENDIAN -# define htonr_us(r0,r1) _htonr_us(_jit,r0,r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0,r1) _htonr_ui(_jit,r0,r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ul(r0,r1) REV(r0,r1) -# else -# define htonr_us(r0,r1) extr_us(r0,r1) -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) -# endif +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ul(r0,r1) REV(r0,r1) # define extr_c(r0,r1) SXTB(r0,r1) # define extr_uc(r0,r1) UXTB(r0,r1) # define extr_s(r0,r1) SXTH(r0,r1) # define extr_us(r0,r1) UXTH(r0,r1) # define extr_i(r0,r1) SXTW(r0,r1) # define extr_ui(r0,r1) UXTW(r0,r1) +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define movr(r0,r1) _movr(_jit,r0,r1) static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); # define movi(r0,i0) _movi(_jit,r0,i0) @@ -772,12 +795,12 @@ _bmxi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t); # define bmci(i0,r0,i1) bmxi(BCC_EQ,i0,r0,i1) # define jmpr(r0) BR(r0) # define jmpi(i0) _jmpi(_jit,i0) -static void _jmpi(jit_state_t*,jit_word_t); +static jit_word_t _jmpi(jit_state_t*,jit_word_t); # define jmpi_p(i0) _jmpi_p(_jit,i0) static jit_word_t _jmpi_p(jit_state_t*,jit_word_t); # define callr(r0) BLR(r0) # define calli(i0) _calli(_jit,i0) -static void _calli(jit_state_t*,jit_word_t); +static jit_word_t _calli(jit_state_t*,jit_word_t); # define calli_p(i0) _calli_p(_jit,i0) static jit_word_t _calli_p(jit_state_t*,jit_word_t); # define prolog(i0) _prolog(_jit,i0) @@ -793,36 +816,17 @@ static void _patch_at(jit_state_t*,jit_word_t,jit_word_t); #endif #if CODE +/* https://dougallj.wordpress.com/2021/10/30/bit-twiddling-optimising-aarch64-logical-immediate-encoding-and-decoding/ */ +#include "aarch64-logical-immediates.c" static jit_int32_t logical_immediate(jit_word_t imm) { - /* There are 5334 possible immediate values, but to avoid the - * need of either too complex code or large lookup tables, - * only check for (simply) encodable common/small values */ - switch (imm) { - case -16: return (0xf3b); - case -15: return (0xf3c); - case -13: return (0xf3d); - case -9: return (0xf3e); - case -8: return (0xf7c); - case -7: return (0xf7d); - case -5: return (0xf7e); - case -4: return (0xfbd); - case -3: return (0xfbe); - case -2: return (0xffe); - case 1: return (0x000); - case 2: return (0xfc0); - case 3: return (0x001); - case 4: return (0xf80); - case 6: return (0xfc1); - case 7: return (0x002); - case 8: return (0xf40); - case 12: return (0xf81); - case 14: return (0xfc2); - case 15: return (0x003); - case 16: return (0xf00); - default: return (-1); + jit_int32_t result = encodeLogicalImmediate64(imm); + if (result != ENCODE_FAILED) { + assert(isValidLogicalImmediate64(result)); + return (result & 0xfff); } + return (-1); } static void @@ -903,7 +907,7 @@ static void _o26(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Simm26) { instr_t i; - assert(Simm26 >= -33554432 && Simm26 <= 33554431); + assert(s26_p(Simm26)); assert(!(Op & ~0xfc000000)); i.w = Op; i.imm26.b = Simm26; @@ -1375,6 +1379,41 @@ _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) } } +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + CMPI(r2, 0); + CSEL(r0, r0, r1, CC_NE); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + CMPI(r2, 0); + CSEL(r0, r0, r1, CC_EQ); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + comr(r0, r1); + clzr(r0, r0); +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + RBIT(r0, r1); + clor(r0, r0); +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + RBIT(r0, r1); + clzr(r0, r0); +} + static void _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -1441,21 +1480,19 @@ _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) } } -#if __BYTE_ORDER == __LITTLE_ENDIAN static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - htonr_ul(r0, r1); + bswapr_ul(r0, r1); rshi_u(r0, r0, 48); } static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - htonr_ul(r0, r1); + bswapr_ul(r0, r1); rshi_u(r0, r0, 32); } -#endif static void _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) @@ -1610,8 +1647,7 @@ static void _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - assert(!(i0 & 1)); - if (i0 >= 0 && i0 <= 8191) + if (i0 >= 0 && i0 <= 8191 && !(i0 & 1)) LDRSHI(r0, r1, i0 >> 1); else if (i0 > -256 && i0 < 0) LDURSH(r0, r1, i0 & 0x1ff); @@ -1636,8 +1672,7 @@ static void _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - assert(!(i0 & 1)); - if (i0 >= 0 && i0 <= 8191) + if (i0 >= 0 && i0 <= 8191 && !(i0 & 1)) LDRHI(r0, r1, i0 >> 1); else if (i0 > -256 && i0 < 0) LDURH(r0, r1, i0 & 0x1ff); @@ -1656,8 +1691,7 @@ static void _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - assert(!(i0 & 3)); - if (i0 >= 0 && i0 <= 16383) + if (i0 >= 0 && i0 <= 16383 && !(i0 & 3)) LDRSWI(r0, r1, i0 >> 2); else if (i0 > -256 && i0 < 0) LDURSW(r0, r1, i0 & 0x1ff); @@ -1682,8 +1716,7 @@ static void _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - assert(!(i0 & 3)); - if (i0 >= 0 && i0 <= 16383) + if (i0 >= 0 && i0 <= 16383 && !(i0 & 3)) LDRWI(r0, r1, i0 >> 2); else if (i0 > -256 && i0 < 0) LDURW(r0, r1, i0 & 0x1ff); @@ -1702,8 +1735,7 @@ static void _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - assert(!(i0 & 7)); - if (i0 >= 0 && i0 <= 32767) + if (i0 >= 0 && i0 <= 32767 && !(i0 & 7)) LDRI(r0, r1, i0 >> 3); else if (i0 > -256 && i0 < 0) LDUR(r0, r1, i0 & 0x1ff); @@ -1775,8 +1807,7 @@ static void _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; - assert(!(i0 & 1)); - if (i0 >= 0 && i0 <= 8191) + if (i0 >= 0 && i0 <= 8191 && !(i0 & 1)) STRHI(r1, r0, i0 >> 1); else if (i0 > -256 && i0 < 0) STURH(r1, r0, i0 & 0x1ff); @@ -1792,8 +1823,7 @@ static void _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; - assert(!(i0 & 3)); - if (i0 >= 0 && i0 <= 16383) + if (i0 >= 0 && i0 <= 16383 && !(i0 & 3)) STRWI(r1, r0, i0 >> 2); else if (i0 > -256 && i0 < 0) STURW(r1, r0, i0 & 0x1ff); @@ -1809,8 +1839,7 @@ static void _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; - assert(!(i0 & 7)); - if (i0 >= 0 && i0 <= 32767) + if (i0 >= 0 && i0 <= 32767 && !(i0 & 7)) STRI(r1, r0, i0 >> 3); else if (i0 > -256 && i0 < 0) STUR(r1, r0, i0 & 0x1ff); @@ -1822,6 +1851,33 @@ _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) } } +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t r1_reg, iscasi; + jit_word_t retry, done, jump0, jump1; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + /* retry: */ + retry = _jit->pc.w; + LDAXR(r0, r1); + eqr(r0, r0, r2); + jump0 = beqi(_jit->pc.w, r0, 0); /* beqi done r0 0 */ + STLXR(r3, r0, r1); + jump1 = bnei(_jit->pc.w, r0, 0); /* bnei retry r0 0 */ + /* done: */ + CSET(r0, CC_EQ); + done = _jit->pc.w; + patch_at(jump0, done); + patch_at(jump1, retry); + if (iscasi) + jit_unget_reg(r1_reg); +} + static void _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { @@ -2126,20 +2182,22 @@ _bmxi(jit_state_t *_jit, jit_int32_t cc, return (w); } -static void +static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { - jit_word_t w; jit_int32_t reg; - w = (i0 - _jit->pc.w) >> 2; - if (w >= -33554432 && w <= 33554431) - B(w); + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 2; + if (s26_p(d)) + B(d); else { reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i0); jmpr(rn(reg)); jit_unget_reg(reg); } + return (w); } static jit_word_t @@ -2154,20 +2212,22 @@ _jmpi_p(jit_state_t *_jit, jit_word_t i0) return (w); } -static void +static jit_word_t _calli(jit_state_t *_jit, jit_word_t i0) { - jit_word_t w; jit_int32_t reg; - w = (i0 - _jit->pc.w) >> 2; - if (w >= -33554432 && w <= 33554431) - BL(w); + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 2; + if (s26_p(d)) + BL(d); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); callr(rn(reg)); jit_unget_reg(reg); } + return (w); } static jit_word_t @@ -2182,20 +2242,13 @@ _calli_p(jit_state_t *_jit, jit_word_t i0) return (w); } -/* - * prolog and epilog not as "optimized" as one would like, but the - * problem of overallocating stack space to save callee save registers - * exists on all ports, and is still a todo to use a variable - * stack_framesize - * value, what would cause needing to patch some calls, most likely - * the offset of jit_arg* of stack arguments. - */ static void _prolog(jit_state_t *_jit, jit_node_t *node) { - jit_int32_t reg; + jit_int32_t reg, rreg, offs; if (_jitc->function->define_frame || _jitc->function->assume_frame) { jit_int32_t frame = -_jitc->function->frame; + jit_check_frame(); assert(_jitc->function->self.aoff >= frame); if (_jitc->function->assume_frame) return; @@ -2206,40 +2259,51 @@ _prolog(jit_state_t *_jit, jit_node_t *node) _jitc->function->stack = ((_jitc->function->self.alen - /* align stack at 16 bytes */ _jitc->function->self.aoff) + 15) & -16; - STPI_POS(FP_REGNO, LR_REGNO, SP_REGNO, -(stack_framesize >> 3)); - MOV_XSP(FP_REGNO, SP_REGNO); -#define SPILL(L, R, O) \ - do { \ - if (jit_regset_tstbit(&_jitc->function->regset, _R##L)) { \ - if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \ - STPI(L, R, SP_REGNO, O); \ - else \ - STRI(L, SP_REGNO, O); \ - } \ - else if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \ - STRI(R, SP_REGNO, O + 1); \ - } while (0) - SPILL(19, 20, 2); - SPILL(21, 22, 4); - SPILL(23, 24, 6); - SPILL(25, 26, 8); - SPILL(27, 28, 10); -#undef SPILL -#define SPILL(R, O) \ - do { \ - if (jit_regset_tstbit(&_jitc->function->regset, _V##R)) \ - stxi_d(O, SP_REGNO, R); \ - } while (0) - SPILL( 8, 96); - SPILL( 9, 104); - SPILL(10, 112); - SPILL(11, 120); - SPILL(12, 128); - SPILL(13, 136); - SPILL(14, 144); - SPILL(15, 152); -#undef SPILL - if (_jitc->function->stack) + + if (!_jitc->function->need_frame) { + /* check if any callee save register needs to be saved */ + for (reg = 0; reg < _jitc->reglen; ++reg) + if (jit_regset_tstbit(&_jitc->function->regset, reg) && + (_rvs[reg].spec & jit_class_sav)) { + jit_check_frame(); + break; + } + } + + if (_jitc->function->need_frame) { + STPI_POS(FP_REGNO, LR_REGNO, SP_REGNO, -(jit_framesize() >> 3)); + MOV_XSP(FP_REGNO, SP_REGNO); + } + /* callee save registers */ + for (reg = 0, offs = 2; reg < jit_size(iregs);) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + for (rreg = reg + 1; rreg < jit_size(iregs); rreg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[rreg])) + break; + } + if (rreg < jit_size(iregs)) { + STPI(rn(iregs[reg]), rn(iregs[rreg]), SP_REGNO, offs); + offs += 2; + reg = rreg + 1; + } + else { + STRI(rn(iregs[reg]), SP_REGNO, offs); + ++offs; + /* No pair found */ + break; + } + } + else + ++reg; + } + for (reg = 0, offs <<= 3; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + stxi_d(offs, SP_REGNO, rn(fregs[reg])); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->stack) subi(SP_REGNO, SP_REGNO, _jitc->function->stack); if (_jitc->function->allocar) { reg = jit_get_reg(jit_class_gpr); @@ -2248,6 +2312,7 @@ _prolog(jit_state_t *_jit, jit_node_t *node) jit_unget_reg(reg); } +#if !__APPLE__ if (_jitc->function->self.call & jit_call_varargs) { /* Save gp registers in the save area, if any is a vararg */ for (reg = 8 - _jitc->function->vagp / -8; @@ -2265,53 +2330,55 @@ _prolog(jit_state_t *_jit, jit_node_t *node) stxi_d(_jitc->function->vaoff + offsetof(jit_va_list_t, q0) + reg * 16 + offsetof(jit_qreg_t, l), FP_REGNO, rn(_V0 - reg)); } +#endif } static void _epilog(jit_state_t *_jit, jit_node_t *node) { + jit_int32_t reg, rreg, offs; if (_jitc->function->assume_frame) return; if (_jitc->function->stack) MOV_XSP(SP_REGNO, FP_REGNO); -#define LOAD(L, R, O) \ - do { \ - if (jit_regset_tstbit(&_jitc->function->regset, _R##L)) { \ - if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \ - LDPI(L, R, SP_REGNO, O); \ - else \ - LDRI(L, SP_REGNO, O); \ - } \ - else if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \ - LDRI(R, SP_REGNO, O + 1); \ - } while (0) - LOAD(19, 20, 2); - LOAD(21, 22, 4); - LOAD(23, 24, 6); - LOAD(25, 26, 8); - LOAD(27, 28, 10); -#undef LOAD -#define LOAD(R, O) \ - do { \ - if (jit_regset_tstbit(&_jitc->function->regset, _V##R)) \ - ldxi_d(R, SP_REGNO, O); \ - } while (0) - LOAD( 8, 96); - LOAD( 9, 104); - LOAD(10, 112); - LOAD(11, 120); - LOAD(12, 128); - LOAD(13, 136); - LOAD(14, 144); - LOAD(15, 152); -#undef LOAD - LDPI_PRE(FP_REGNO, LR_REGNO, SP_REGNO, stack_framesize >> 3); + /* callee save registers */ + for (reg = 0, offs = 2; reg < jit_size(iregs);) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + for (rreg = reg + 1; rreg < jit_size(iregs); rreg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[rreg])) + break; + } + if (rreg < jit_size(iregs)) { + LDPI(rn(iregs[reg]), rn(iregs[rreg]), SP_REGNO, offs); + offs += 2; + reg = rreg + 1; + } + else { + LDRI(rn(iregs[reg]), SP_REGNO, offs); + ++offs; + /* No pair found */ + break; + } + } + else + ++reg; + } + for (reg = 0, offs <<= 3; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + ldxi_d(rn(fregs[reg]), SP_REGNO, offs); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->need_frame) + LDPI_PRE(FP_REGNO, LR_REGNO, SP_REGNO, jit_framesize() >> 3); RET(); } static void _vastart(jit_state_t *_jit, jit_int32_t r0) { +#if !__APPLE__ jit_int32_t reg; assert(_jitc->function->self.call & jit_call_varargs); @@ -2322,7 +2389,7 @@ _vastart(jit_state_t *_jit, jit_int32_t r0) reg = jit_get_reg(jit_class_gpr); /* Initialize stack pointer to the first stack argument. */ - addi(rn(reg), FP_REGNO, _jitc->function->self.size); + addi(rn(reg), FP_REGNO, jit_selfsize()); stxi(offsetof(jit_va_list_t, stack), r0, rn(reg)); /* Initialize gp top pointer to the first stack argument. */ @@ -2342,11 +2409,16 @@ _vastart(jit_state_t *_jit, jit_int32_t r0) stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg)); jit_unget_reg(reg); +#else + assert(_jitc->function->self.call & jit_call_varargs); + addi(r0, FP_REGNO, jit_selfsize()); +#endif } static void _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { +#if !__APPLE__ jit_word_t ge_code; jit_word_t lt_code; jit_int32_t rg0, rg1; @@ -2376,7 +2448,7 @@ _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) jit_unget_reg(rg1); /* Jump over overflow code. */ - lt_code = jmpi_p(_jit->pc.w); + lt_code = jmpi(_jit->pc.w); /* Where to land if argument is in overflow area. */ patch_at(ge_code, _jit->pc.w); @@ -2395,6 +2467,11 @@ _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) patch_at(lt_code, _jit->pc.w); jit_unget_reg(rg0); +#else + assert(_jitc->function->self.call & jit_call_varargs); + ldr(r0, r1); + addi(r1, r1, sizeof(jit_word_t)); +#endif } static void @@ -2414,7 +2491,7 @@ _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) ffc = i.w & 0xffc00000; if (fc == A64_B || fc == A64_BL) { d = (label - instr) >> 2; - assert(d >= -33554432 && d <= 33554431); + assert(s26_p(d)); i.imm26.b = d; u.i[0] = i.w; } diff --git a/parallel-rsp/lightning/lib/jit_aarch64-fpu.c b/parallel-rsp/lightning/lib/jit_aarch64-fpu.c index 871ba7e..3d17e32 100644 --- a/parallel-rsp/lightning/lib/jit_aarch64-fpu.c +++ b/parallel-rsp/lightning/lib/jit_aarch64-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -862,6 +862,7 @@ dbopi(ltgt) static void _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { +#if !__APPLE__ jit_word_t ge_code; jit_word_t lt_code; jit_int32_t rg0, rg1; @@ -891,7 +892,7 @@ _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) jit_unget_reg(rg1); /* Jump over overflow code. */ - lt_code = jmpi_p(_jit->pc.w); + lt_code = jmpi(_jit->pc.w); /* Where to land if argument is in overflow area. */ patch_at(ge_code, _jit->pc.w); @@ -910,5 +911,10 @@ _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) patch_at(lt_code, _jit->pc.w); jit_unget_reg(rg0); +#else + assert(_jitc->function->self.call & jit_call_varargs); + ldr_d(r0, r1); + addi(r1, r1, sizeof(jit_float64_t)); +#endif } #endif diff --git a/parallel-rsp/lightning/lib/jit_aarch64-sz.c b/parallel-rsp/lightning/lib/jit_aarch64-sz.c index 7e22e0e..3d1ea99 100644 --- a/parallel-rsp/lightning/lib/jit_aarch64-sz.c +++ b/parallel-rsp/lightning/lib/jit_aarch64-sz.c @@ -1,20 +1,25 @@ #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 120 +# if PACKED_STACK +#define JIT_INSTR_MAX 96 0, /* data */ 0, /* live */ - 4, /* align */ + 12, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ - 120, /* prolog */ + 96, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -22,11 +27,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 44, /* va_start */ - 64, /* va_arg */ - 72, /* va_arg_d */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 4, /* va_start */ + 8, /* va_arg */ + 12, /* va_arg_d */ 0, /* va_end */ 4, /* addr */ 20, /* addi */ @@ -95,29 +112,36 @@ 8, /* nei */ 4, /* movr */ 16, /* movi */ + 8, /* movnr */ + 8, /* movzr */ + 28, /* casr */ + 36, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 4, /* extr_i */ 4, /* extr_ui */ + 8, /* bswapr_us */ + 8, /* bswapr_ui */ + 4, /* bswapr_ul */ 8, /* htonr_us */ 8, /* htonr_ui */ 4, /* htonr_ul */ 4, /* ldr_c */ - 12, /* ldi_c */ + 16, /* ldi_c */ 4, /* ldr_uc */ - 12, /* ldi_uc */ + 16, /* ldi_uc */ 4, /* ldr_s */ - 12, /* ldi_s */ + 16, /* ldi_s */ 4, /* ldr_us */ - 12, /* ldi_us */ + 16, /* ldi_us */ 4, /* ldr_i */ - 12, /* ldi_i */ + 16, /* ldi_i */ 4, /* ldr_ui */ - 12, /* ldi_ui */ + 16, /* ldi_ui */ 4, /* ldr_l */ - 12, /* ldi_l */ + 16, /* ldi_l */ 8, /* ldxr_c */ 20, /* ldxi_c */ 4, /* ldxr_uc */ @@ -133,13 +157,13 @@ 4, /* ldxr_l */ 20, /* ldxi_l */ 4, /* str_c */ - 12, /* sti_c */ + 16, /* sti_c */ 4, /* str_s */ - 12, /* sti_s */ + 16, /* sti_s */ 4, /* str_i */ - 12, /* sti_i */ + 16, /* sti_i */ 4, /* str_l */ - 12, /* sti_l */ + 16, /* sti_l */ 4, /* stxr_c */ 20, /* stxi_c */ 4, /* stxr_s */ @@ -189,17 +213,41 @@ 8, /* bxsubr_u */ 8, /* bxsubi_u */ 4, /* jmpr */ - 20, /* jmpi */ + 4, /* jmpi */ 4, /* callr */ - 20, /* calli */ + 16, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -259,11 +307,11 @@ 4, /* movr_f */ 8, /* movi_f */ 8, /* ldr_f */ - 16, /* ldi_f */ + 20, /* ldi_f */ 8, /* ldxr_f */ 24, /* ldxi_f */ 8, /* str_f */ - 16, /* sti_f */ + 20, /* sti_f */ 8, /* stxr_f */ 24, /* stxi_f */ 8, /* bltr_f */ @@ -350,11 +398,11 @@ 4, /* movr_d */ 12, /* movi_d */ 8, /* ldr_d */ - 16, /* ldi_d */ + 20, /* ldi_d */ 8, /* ldxr_d */ 24, /* ldxi_d */ 8, /* str_d */ - 16, /* sti_d */ + 20, /* sti_d */ 8, /* stxr_d */ 24, /* stxi_d */ 8, /* bltr_d */ @@ -399,4 +447,461 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 12, /* cto */ + 8, /* ctz */ + +# else /* PACKED_STACK */ +#define JIT_INSTR_MAX 120 + 0, /* data */ + 0, /* live */ + 12, /* align */ + 0, /* save */ + 0, /* load */ + 4, /* skip */ + 0, /* #name */ + 0, /* #note */ + 0, /* label */ + 120, /* prolog */ + 0, /* ellipsis */ + 0, /* va_push */ + 0, /* allocai */ + 0, /* allocar */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ + 0, /* getarg_c */ + 0, /* getarg_uc */ + 0, /* getarg_s */ + 0, /* getarg_us */ + 0, /* getarg_i */ + 0, /* getarg_ui */ + 0, /* getarg_l */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 4, /* va_start */ + 8, /* va_arg */ + 12, /* va_arg_d */ + 0, /* va_end */ + 4, /* addr */ + 20, /* addi */ + 4, /* addcr */ + 12, /* addci */ + 4, /* addxr */ + 8, /* addxi */ + 4, /* subr */ + 20, /* subi */ + 4, /* subcr */ + 12, /* subci */ + 4, /* subxr */ + 8, /* subxi */ + 24, /* rsbi */ + 4, /* mulr */ + 20, /* muli */ + 12, /* qmulr */ + 20, /* qmuli */ + 12, /* qmulr_u */ + 20, /* qmuli_u */ + 4, /* divr */ + 20, /* divi */ + 4, /* divr_u */ + 12, /* divi_u */ + 20, /* qdivr */ + 16, /* qdivi */ + 20, /* qdivr_u */ + 16, /* qdivi_u */ + 12, /* remr */ + 28, /* remi */ + 12, /* remr_u */ + 20, /* remi_u */ + 4, /* andr */ + 20, /* andi */ + 4, /* orr */ + 20, /* ori */ + 4, /* xorr */ + 20, /* xori */ + 4, /* lshr */ + 4, /* lshi */ + 4, /* rshr */ + 4, /* rshi */ + 4, /* rshr_u */ + 4, /* rshi_u */ + 4, /* negr */ + 4, /* comr */ + 8, /* ltr */ + 8, /* lti */ + 8, /* ltr_u */ + 8, /* lti_u */ + 8, /* ler */ + 8, /* lei */ + 8, /* ler_u */ + 8, /* lei_u */ + 8, /* eqr */ + 8, /* eqi */ + 8, /* ger */ + 8, /* gei */ + 8, /* ger_u */ + 8, /* gei_u */ + 8, /* gtr */ + 8, /* gti */ + 8, /* gtr_u */ + 8, /* gti_u */ + 8, /* ner */ + 8, /* nei */ + 4, /* movr */ + 16, /* movi */ + 8, /* movnr */ + 8, /* movzr */ + 28, /* casr */ + 36, /* casi */ + 4, /* extr_c */ + 4, /* extr_uc */ + 4, /* extr_s */ + 4, /* extr_us */ + 4, /* extr_i */ + 4, /* extr_ui */ + 8, /* bswapr_us */ + 8, /* bswapr_ui */ + 4, /* bswapr_ul */ + 8, /* htonr_us */ + 8, /* htonr_ui */ + 4, /* htonr_ul */ + 4, /* ldr_c */ + 16, /* ldi_c */ + 4, /* ldr_uc */ + 16, /* ldi_uc */ + 4, /* ldr_s */ + 16, /* ldi_s */ + 4, /* ldr_us */ + 16, /* ldi_us */ + 4, /* ldr_i */ + 16, /* ldi_i */ + 4, /* ldr_ui */ + 16, /* ldi_ui */ + 4, /* ldr_l */ + 16, /* ldi_l */ + 8, /* ldxr_c */ + 20, /* ldxi_c */ + 4, /* ldxr_uc */ + 20, /* ldxi_uc */ + 4, /* ldxr_s */ + 16, /* ldxi_s */ + 4, /* ldxr_us */ + 16, /* ldxi_us */ + 4, /* ldxr_i */ + 20, /* ldxi_i */ + 4, /* ldxr_ui */ + 16, /* ldxi_ui */ + 4, /* ldxr_l */ + 20, /* ldxi_l */ + 4, /* str_c */ + 16, /* sti_c */ + 4, /* str_s */ + 16, /* sti_s */ + 4, /* str_i */ + 16, /* sti_i */ + 4, /* str_l */ + 16, /* sti_l */ + 4, /* stxr_c */ + 20, /* stxi_c */ + 4, /* stxr_s */ + 20, /* stxi_s */ + 4, /* stxr_i */ + 20, /* stxi_i */ + 4, /* stxr_l */ + 20, /* stxi_l */ + 8, /* bltr */ + 8, /* blti */ + 8, /* bltr_u */ + 8, /* blti_u */ + 8, /* bler */ + 8, /* blei */ + 8, /* bler_u */ + 8, /* blei_u */ + 8, /* beqr */ + 24, /* beqi */ + 8, /* bger */ + 8, /* bgei */ + 8, /* bger_u */ + 8, /* bgei_u */ + 8, /* bgtr */ + 8, /* bgti */ + 8, /* bgtr_u */ + 8, /* bgti_u */ + 8, /* bner */ + 24, /* bnei */ + 8, /* bmsr */ + 8, /* bmsi */ + 8, /* bmcr */ + 8, /* bmci */ + 8, /* boaddr */ + 8, /* boaddi */ + 8, /* boaddr_u */ + 8, /* boaddi_u */ + 8, /* bxaddr */ + 8, /* bxaddi */ + 8, /* bxaddr_u */ + 8, /* bxaddi_u */ + 8, /* bosubr */ + 8, /* bosubi */ + 8, /* bosubr_u */ + 8, /* bosubi_u */ + 8, /* bxsubr */ + 8, /* bxsubi */ + 8, /* bxsubr_u */ + 8, /* bxsubi_u */ + 4, /* jmpr */ + 4, /* jmpi */ + 4, /* callr */ + 16, /* calli */ + 0, /* prepare */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ + 0, /* finishr */ + 0, /* finishi */ + 0, /* ret */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ + 0, /* retval_c */ + 0, /* retval_uc */ + 0, /* retval_s */ + 0, /* retval_us */ + 0, /* retval_i */ + 0, /* retval_ui */ + 0, /* retval_l */ + 96, /* epilog */ + 0, /* arg_f */ + 0, /* getarg_f */ + 0, /* putargr_f */ + 0, /* putargi_f */ + 4, /* addr_f */ + 12, /* addi_f */ + 4, /* subr_f */ + 12, /* subi_f */ + 12, /* rsbi_f */ + 4, /* mulr_f */ + 12, /* muli_f */ + 4, /* divr_f */ + 12, /* divi_f */ + 4, /* negr_f */ + 4, /* absr_f */ + 4, /* sqrtr_f */ + 8, /* ltr_f */ + 16, /* lti_f */ + 8, /* ler_f */ + 16, /* lei_f */ + 8, /* eqr_f */ + 16, /* eqi_f */ + 8, /* ger_f */ + 16, /* gei_f */ + 8, /* gtr_f */ + 16, /* gti_f */ + 8, /* ner_f */ + 16, /* nei_f */ + 8, /* unltr_f */ + 16, /* unlti_f */ + 8, /* unler_f */ + 16, /* unlei_f */ + 16, /* uneqr_f */ + 24, /* uneqi_f */ + 8, /* unger_f */ + 16, /* ungei_f */ + 8, /* ungtr_f */ + 16, /* ungti_f */ + 16, /* ltgtr_f */ + 24, /* ltgti_f */ + 8, /* ordr_f */ + 16, /* ordi_f */ + 8, /* unordr_f */ + 16, /* unordi_f */ + 8, /* truncr_f_i */ + 4, /* truncr_f_l */ + 4, /* extr_f */ + 4, /* extr_d_f */ + 4, /* movr_f */ + 8, /* movi_f */ + 8, /* ldr_f */ + 20, /* ldi_f */ + 8, /* ldxr_f */ + 24, /* ldxi_f */ + 8, /* str_f */ + 20, /* sti_f */ + 8, /* stxr_f */ + 24, /* stxi_f */ + 8, /* bltr_f */ + 16, /* blti_f */ + 8, /* bler_f */ + 16, /* blei_f */ + 8, /* beqr_f */ + 16, /* beqi_f */ + 8, /* bger_f */ + 16, /* bgei_f */ + 8, /* bgtr_f */ + 16, /* bgti_f */ + 8, /* bner_f */ + 16, /* bnei_f */ + 8, /* bunltr_f */ + 16, /* bunlti_f */ + 8, /* bunler_f */ + 16, /* bunlei_f */ + 16, /* buneqr_f */ + 24, /* buneqi_f */ + 8, /* bunger_f */ + 16, /* bungei_f */ + 8, /* bungtr_f */ + 16, /* bungti_f */ + 16, /* bltgtr_f */ + 24, /* bltgti_f */ + 8, /* bordr_f */ + 16, /* bordi_f */ + 8, /* bunordr_f */ + 16, /* bunordi_f */ + 0, /* pushargr_f */ + 0, /* pushargi_f */ + 0, /* retr_f */ + 0, /* reti_f */ + 0, /* retval_f */ + 0, /* arg_d */ + 0, /* getarg_d */ + 0, /* putargr_d */ + 0, /* putargi_d */ + 4, /* addr_d */ + 12, /* addi_d */ + 4, /* subr_d */ + 12, /* subi_d */ + 12, /* rsbi_d */ + 4, /* mulr_d */ + 12, /* muli_d */ + 4, /* divr_d */ + 12, /* divi_d */ + 4, /* negr_d */ + 4, /* absr_d */ + 4, /* sqrtr_d */ + 8, /* ltr_d */ + 16, /* lti_d */ + 8, /* ler_d */ + 16, /* lei_d */ + 8, /* eqr_d */ + 16, /* eqi_d */ + 8, /* ger_d */ + 16, /* gei_d */ + 8, /* gtr_d */ + 16, /* gti_d */ + 8, /* ner_d */ + 16, /* nei_d */ + 8, /* unltr_d */ + 16, /* unlti_d */ + 8, /* unler_d */ + 16, /* unlei_d */ + 16, /* uneqr_d */ + 24, /* uneqi_d */ + 8, /* unger_d */ + 16, /* ungei_d */ + 8, /* ungtr_d */ + 16, /* ungti_d */ + 16, /* ltgtr_d */ + 24, /* ltgti_d */ + 8, /* ordr_d */ + 16, /* ordi_d */ + 8, /* unordr_d */ + 16, /* unordi_d */ + 8, /* truncr_d_i */ + 4, /* truncr_d_l */ + 4, /* extr_d */ + 4, /* extr_f_d */ + 4, /* movr_d */ + 12, /* movi_d */ + 8, /* ldr_d */ + 20, /* ldi_d */ + 8, /* ldxr_d */ + 24, /* ldxi_d */ + 8, /* str_d */ + 20, /* sti_d */ + 8, /* stxr_d */ + 24, /* stxi_d */ + 8, /* bltr_d */ + 16, /* blti_d */ + 8, /* bler_d */ + 16, /* blei_d */ + 8, /* beqr_d */ + 20, /* beqi_d */ + 8, /* bger_d */ + 16, /* bgei_d */ + 8, /* bgtr_d */ + 16, /* bgti_d */ + 8, /* bner_d */ + 16, /* bnei_d */ + 8, /* bunltr_d */ + 16, /* bunlti_d */ + 8, /* bunler_d */ + 16, /* bunlei_d */ + 16, /* buneqr_d */ + 24, /* buneqi_d */ + 8, /* bunger_d */ + 16, /* bungei_d */ + 8, /* bungtr_d */ + 16, /* bungti_d */ + 16, /* bltgtr_d */ + 24, /* bltgti_d */ + 8, /* bordr_d */ + 16, /* bordi_d */ + 8, /* bunordr_d */ + 16, /* bunordi_d */ + 0, /* pushargr_d */ + 0, /* pushargi_d */ + 0, /* retr_d */ + 0, /* reti_d */ + 0, /* retval_d */ + 0, /* movr_w_f */ + 0, /* movr_ww_d */ + 0, /* movr_w_d */ + 0, /* movr_f_w */ + 0, /* movi_f_w */ + 0, /* movr_d_ww */ + 0, /* movi_d_ww */ + 0, /* movr_d_w */ + 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 12, /* cto */ + 8, /* ctz */ +# endif #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_aarch64.c b/parallel-rsp/lightning/lib/jit_aarch64.c index 5b2ff49..243e677 100644 --- a/parallel-rsp/lightning/lib/jit_aarch64.c +++ b/parallel-rsp/lightning/lib/jit_aarch64.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -17,9 +17,16 @@ * Paulo Cesar Pereira de Andrade */ +/* callee save + * align16(lr+fp+x19+x2[0-8]+v8+v9+v1[0-15]) */ +#define stack_framesize 160 + #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8) #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) +#if __APPLE__ +typedef jit_pointer_t jit_va_list_t; +#else typedef struct jit_qreg { jit_float64_t l; jit_float64_t h; @@ -52,10 +59,13 @@ typedef struct jit_va_list { jit_qreg_t q6; jit_qreg_t q7; } jit_va_list_t; +#endif /* * Prototypes */ +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); #define patch(instr, node) _patch(_jit, instr, node) static void _patch(jit_state_t*,jit_word_t,jit_node_t*); @@ -72,7 +82,11 @@ extern void __clear_cache(void *, void *); */ jit_register_t _rvs[] = { { rc(gpr) | 0x08, "x8" }, +#if __APPLE__ + { 0x12, "x18" }, +#else { rc(gpr) | 0x12, "x18" }, +#endif { rc(gpr) | 0x11, "x17" }, { rc(gpr) | 0x10, "x16" }, { rc(gpr) | 0x09, "x9" }, @@ -138,6 +152,14 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { + _R19, _R20, _R21, _R22, _R23, _R24, _R25, _R26, _R27, _R28 +}; + +static jit_int32_t fregs[] = { + _V8, _V9, _V10, _V11, _V12, _V13, _V14, _V15 +}; + /* * Implementation */ @@ -198,6 +220,7 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + jit_check_frame(); switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -258,20 +281,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -331,7 +352,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_f_reg_p(u->u.w)); @@ -341,6 +362,7 @@ void _jit_ellipsis(jit_state_t *_jit) { jit_inc_synth(ellipsis); + jit_check_frame(); if (_jitc->prepare) { jit_link_prepare(); assert(!(_jitc->function->call.call & jit_call_varargs)); @@ -351,6 +373,7 @@ _jit_ellipsis(jit_state_t *_jit) assert(!(_jitc->function->self.call & jit_call_varargs)); _jitc->function->self.call |= jit_call_varargs; +#if !__APPLE_ /* Allocate va_list like object in the stack, * with enough space to save all argument * registers, and use fixed offsets for them. */ @@ -367,6 +390,7 @@ _jit_ellipsis(jit_state_t *_jit) _jitc->function->vafp = (8 - _jitc->function->self.argf) * -16; else _jitc->function->vafp = 0; +#endif } jit_dec_synth(); } @@ -380,7 +404,7 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; @@ -389,10 +413,22 @@ _jit_arg(jit_state_t *_jit) if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { +#if PACKED_STACK || STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif +#if PACKED_STACK + _jitc->function->self.size += + _jitc->function->self.size & ((1 << (code - jit_code_arg_c)) - 1); +#endif offset = _jitc->function->self.size; +#if PACKED_STACK + _jitc->function->self.size += 1 << (code - jit_code_arg_c); +#else _jitc->function->self.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -408,8 +444,17 @@ _jit_arg_f(jit_state_t *_jit) if (jit_arg_f_reg_p(_jitc->function->self.argf)) offset = _jitc->function->self.argf++; else { +#if PACKED_STACK + _jitc->function->self.size += + _jitc->function->self.size & (sizeof(jit_float32_t) - 1); +#endif offset = _jitc->function->self.size; +#if PACKED_STACK + _jitc->function->self.size += sizeof(jit_float32_t); +#else _jitc->function->self.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_f, offset, ++_jitc->function->self.argn); @@ -427,8 +472,13 @@ _jit_arg_d(jit_state_t *_jit) if (jit_arg_f_reg_p(_jitc->function->self.argf)) offset = _jitc->function->self.argf++; else { +#if PACKED_STACK + _jitc->function->self.size += + _jitc->function->self.size & (sizeof(jit_float64_t) - 1); +#endif offset = _jitc->function->self.size; - _jitc->function->self.size += sizeof(jit_word_t); + _jitc->function->self.size += sizeof(jit_float64_t); + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_d, offset, ++_jitc->function->self.argn); @@ -439,111 +489,235 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_c(u, JIT_RA0 - v->u.w); - else - jit_ldxi_c(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_c(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_uc(u, JIT_RA0 - v->u.w); - else - jit_ldxi_uc(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_uc(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_s(u, JIT_RA0 - v->u.w); - else - jit_ldxi_s(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_s(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_us(u, JIT_RA0 - v->u.w); - else - jit_ldxi_us(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_us(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK || __WORDSIZE == 32 + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_i(u, JIT_RA0 - v->u.w); - else - jit_ldxi_i(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_i(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } +#if __WORDSIZE == 64 void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_ui(u, JIT_RA0 - v->u.w); - else - jit_ldxi_ui(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_ui(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); - else - jit_ldxi_l(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_l(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } +#endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); - if (jit_arg_reg_p(v->u.w)) - jit_movr(JIT_RA0 - v->u.w, u); - else - jit_stxi(v->u.w, JIT_FP, u); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); + if (jit_arg_reg_p(v->u.w)) { + jit_int32_t regno = JIT_RA0 - v->u.w; +#if PACKED_STACK + switch (code) { + case jit_code_putargr_c: jit_extr_c(regno, u); break; + case jit_code_putargr_uc: jit_extr_uc(regno, u); break; + case jit_code_putargr_s: jit_extr_s(regno, u); break; + case jit_code_putargr_us: jit_extr_us(regno, u); break; +# if __WORDISZE == 32 + case jit_code_putargr_i: jit_movr(regno, u); break; +# else + case jit_code_putargr_i: jit_extr_i(regno, u); break; + case jit_code_putargr_ui: jit_extr_ui(regno, u); break; + case jit_code_putargr_l: jit_movr(regno, u); break; +# endif + default: abort(); break; + } +#else + jit_movr(regno, u); +#endif + } + else { + jit_node_t *node; +#if PACKED_STACK + switch (code) { + case jit_code_putargr_c: case jit_code_putargr_uc: + node = jit_stxi_c(v->u.w, JIT_FP, u); break; + case jit_code_putargr_s: case jit_code_putargr_us: + node = jit_stxi_s(v->u.w, JIT_FP, u); break; +# if __WORDSIZE == 32 + case jit_code_putargr_i: + node = jit_stxi(v->u.w, JIT_FP, u); break; +# else + case jit_code_putargr_i: case jit_code_putargr_ui: + node = jit_stxi_i(v->u.w, JIT_FP, u); break; + case jit_code_putargr_l: + node = jit_stxi(v->u.w, JIT_FP, u); break; +# endif + default: abort(); break; + } +#else + node = jit_stxi(v->u.w, JIT_FP, u); +#endif + jit_link_alist(node); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); +#if PACKED_STACK + switch (code) { + case jit_code_putargi_c: u = (jit_int8_t)u; break; + case jit_code_putargi_uc: u = (jit_uint8_t)u; break; + case jit_code_putargi_s: u = (jit_int16_t)u; break; + case jit_code_putargi_us: u = (jit_uint16_t)u; break; +# if __WORDSIZE == 32 + case jit_code_putargi_i: break; +# else + case jit_code_putargi_i: u = (jit_int32_t)u; break; + case jit_code_putargi_ui: u = (jit_uint32_t)u; break; + case jit_code_putargi_l: break; +# endif + default: abort(); break; + } +#endif if (jit_arg_reg_p(v->u.w)) jit_movi(JIT_RA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w, JIT_FP, regno); +#if PACKED_STACK + switch (code) { + case jit_code_putargi_c: case jit_code_putargi_uc: + node = jit_stxi_c(v->u.w, JIT_FP, regno); break; + case jit_code_putargi_s: case jit_code_putargi_us: + node = jit_stxi_s(v->u.w, JIT_FP, regno); break; +# if __WORDSIZE == 32 + case jit_code_putargi_i: + node = jit_stxi(v->u.w, JIT_FP, regno); break; +# else + case jit_code_putargi_i: case jit_code_putargi_ui: + node = jit_stxi_i(v->u.w, JIT_FP, regno); break; + case jit_code_putargi_l: + node = jit_stxi(v->u.w, JIT_FP, regno); break; +# endif + default: abort(); break; + } +#else + node = jit_stxi(v->u.w, JIT_FP, regno); +#endif + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -556,8 +730,10 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(getarg_f, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr_f(u, JIT_FA0 - v->u.w); - else - jit_ldxi_f(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_f(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -568,8 +744,10 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(putargr_f, u, v); if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(JIT_FA0 - v->u.w, u); - else - jit_stxi_f(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi_f(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -582,9 +760,11 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) if (jit_arg_f_reg_p(v->u.w)) jit_movi_f(JIT_FA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -597,8 +777,10 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(getarg_d, u, v); if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(u, JIT_FA0 - v->u.w); - else - jit_ldxi_d(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_d(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -609,8 +791,10 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(putargr_d, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr_d(JIT_FA0 - v->u.w, u); - else - jit_stxi_d(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi_d(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -623,48 +807,161 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movi_d(JIT_FA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { - jit_movr(JIT_RA0 - _jitc->function->call.argi, u); + jit_int32_t regno = JIT_RA0 - _jitc->function->call.argi; +#if PACKED_STACK + switch (code) { + case jit_code_pushargr_c: jit_extr_c(regno, u); break; + case jit_code_pushargr_uc: jit_extr_uc(regno, u); break; + case jit_code_pushargr_s: jit_extr_s(regno, u); break; + case jit_code_pushargr_us: jit_extr_us(regno, u); break; +# if __WORDISZE == 32 + case jit_code_pushargr_i: jit_movr(regno, u); break; +# else + case jit_code_pushargr_i: jit_extr_i(regno, u); break; + case jit_code_pushargr_ui: jit_extr_ui(regno, u); break; + case jit_code_pushargr_l: jit_movr(regno, u); break; +# endif + default: abort(); break; + } +#else + jit_movr(regno, u); +#endif +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(code == jit_code_pushargr); + jit_stxi(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argi; } else { +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & + ((1 << ((code - jit_code_pushargr_c) >> 2)) - 1); + switch (code) { + case jit_code_pushargr_c: case jit_code_pushargr_uc: + jit_stxi_c(_jitc->function->call.size, JIT_SP, u); + break; + case jit_code_pushargr_s: case jit_code_pushargr_us: + jit_stxi_s(_jitc->function->call.size, JIT_SP, u); + break; +# if __WORDSIZE == 32 + case jit_code_pushargr_i: + jit_stxi(_jitc->function->call.size, JIT_SP, u); + break; +# else + case jit_code_pushargr_i: case jit_code_pushargr_ui: + jit_stxi_i(_jitc->function->call.size, JIT_SP, u); + break; + case jit_code_pushargr_l: + jit_stxi(_jitc->function->call.size, JIT_SP, u); + break; +# endif + default: + abort(); + break; + } + _jitc->function->call.size += 1 << ((code - jit_code_pushargr_c) >> 2); +#else jit_stxi(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } jit_dec_synth(); } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); +#if PACKED_STACK + switch (code) { + case jit_code_pushargi_c: u = (jit_int8_t)u; break; + case jit_code_pushargi_uc: u = (jit_uint8_t)u; break; + case jit_code_pushargi_s: u = (jit_int16_t)u; break; + case jit_code_pushargi_us: u = (jit_uint16_t)u; break; +# if __WORDSIZE == 32 + case jit_code_pushargi_i: break; +# else + case jit_code_pushargi_i: u = (jit_int32_t)u; break; + case jit_code_pushargi_ui: u = (jit_uint32_t)u; break; + case jit_code_pushargi_l: break; +# endif + default: abort(); break; + } +#endif if (jit_arg_reg_p(_jitc->function->call.argi)) { - jit_movi(JIT_RA0 - _jitc->function->call.argi, u); + regno = JIT_RA0 - _jitc->function->call.argi; + jit_movi(regno, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(code == jit_code_pushargi); + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argi; } else { regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & + ((1 << ((code - jit_code_pushargr_c) >> 2)) - 1); + switch (code) { + case jit_code_pushargi_c: case jit_code_pushargi_uc: + jit_stxi_c(_jitc->function->call.size, JIT_SP, regno); + break; + case jit_code_pushargi_s: case jit_code_pushargi_us: + jit_stxi_s(_jitc->function->call.size, JIT_SP, regno); + break; +# if __WORDSIZE == 32 + case jit_code_pushargi_i: + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + break; +# else + case jit_code_pushargi_i: case jit_code_pushargi_ui: + jit_stxi_i(_jitc->function->call.size, JIT_SP, regno); + break; + case jit_code_pushargi_l: + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + break; +# endif + default: + abort(); + break; + } + _jitc->function->call.size += 1 << ((code - jit_code_pushargr_c) >> 2); +#else jit_stxi(_jitc->function->call.size, JIT_SP, regno); - jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -677,11 +974,27 @@ _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float32_t) == sizeof(jit_word_t)); + jit_stxi_f(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argf; } else { +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float32_t) - 1); + jit_stxi_f(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_float32_t); +#else jit_stxi_f(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } jit_dec_synth(); } @@ -695,14 +1008,30 @@ _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float32_t) == sizeof(jit_word_t)); + jit_stxi_f(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argf; } else { regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float32_t) - 1); + jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); + _jitc->function->call.size += sizeof(jit_float32_t); +#else jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); - jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -715,11 +1044,24 @@ _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float64_t) == sizeof(jit_word_t)); + jit_stxi_d(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_float64_t); + } +#endif ++_jitc->function->call.argf; } else { +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float64_t) - 1); +#endif jit_stxi_d(_jitc->function->call.size, JIT_SP, u); - _jitc->function->call.size += sizeof(jit_word_t); + _jitc->function->call.size += sizeof(jit_float64_t); + jit_check_frame(); } jit_dec_synth(); } @@ -733,14 +1075,27 @@ _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float64_t) == sizeof(jit_word_t)); + jit_stxi_d(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_float64_t); + } +#endif ++_jitc->function->call.argf; } else { regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float64_t) - 1); +#endif jit_stxi_d(_jitc->function->call.size, JIT_SP, regno); jit_unget_reg(regno); - _jitc->function->call.size += sizeof(jit_word_t); + _jitc->function->call.size += sizeof(jit_float64_t); + jit_check_frame(); } jit_dec_synth(); } @@ -769,7 +1124,12 @@ _jit_finishr(jit_state_t *_jit, jit_int32_t r0) { jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishr, r0); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_word_t) - 1); +#endif if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; node = jit_callr(r0); @@ -786,7 +1146,12 @@ _jit_finishi(jit_state_t *_jit, jit_pointer_t i0) { jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishi, (jit_word_t)i0); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_word_t) - 1); +#endif if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; node = jit_calli(i0); @@ -835,10 +1200,15 @@ void _jit_retval_i(jit_state_t *_jit, jit_int32_t r0) { jit_inc_synth_w(retval_i, r0); +#if __WORDSIZE == 32 + jit_movr(r0, JIT_RET); +#else jit_extr_i(r0, JIT_RET); +#endif jit_dec_synth(); } +#if __WORDSIZE == 64 void _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0) { @@ -851,10 +1221,10 @@ void _jit_retval_l(jit_state_t *_jit, jit_int32_t r0) { jit_inc_synth_w(retval_l, r0); - if (r0 != JIT_RET) - jit_movr(r0, JIT_RET); + jit_movr(r0, JIT_RET); jit_dec_synth(); } +#endif void _jit_retval_f(jit_state_t *_jit, jit_int32_t r0) @@ -886,6 +1256,7 @@ _emit_code(jit_state_t *_jit) jit_node_t *node; jit_uint8_t *data; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1018,11 +1389,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1071,6 +1444,10 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh, _u); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(and,); case_rrw(and,); case_rrr(or,); @@ -1128,13 +1505,26 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _us); case_rr(hton, _ui); case_rr(hton, _ul); + case_rr(bswap, _us); + case_rr(bswap, _ui); + case_rr(bswap, _ul); case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); case_rr(ext, _us); case_rr(ext, _i); case_rr(ext, _ui); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; case_rr(mov,); + case_rrr(movn,); + case_rrr(movz,); case jit_code_movi: if (node->flag & jit_flag_node) { temp = node->v.n; @@ -1379,6 +1769,7 @@ _emit_code(jit_state_t *_jit) case_brr(bunord, _d); case_brd(bunord); case jit_code_jmpr: + jit_check_frame(); jmpr(rn(node->u.w)); break; case jit_code_jmpi: @@ -1389,17 +1780,26 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s26_p(word)) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } - else + else { + jit_check_frame(); jmpi(node->u.w); + } break; case jit_code_callr: + jit_check_frame(); callr(rn(node->u.w)); break; case jit_code_calli: + jit_check_frame(); if (node->flag & jit_flag_node) { temp = node->u.n; assert(temp->code == jit_code_label || @@ -1407,7 +1807,12 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) calli(temp->u.w); else { - word = calli_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s26_p(word)) + word = calli(_jit->pc.w); + else + word = calli_p(_jit->pc.w); patch(word, node); } } @@ -1418,11 +1823,14 @@ _emit_code(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif undo.patch_offset = _jitc->patches.offset; restart_function: + compute_framesize(); + patch_alist(0); _jitc->again = 0; prolog(node); break; @@ -1438,10 +1846,22 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif _jitc->patches.offset = undo.patch_offset; + patch_alist(1); goto restart_function; } /* remember label is defined */ @@ -1462,11 +1882,23 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +# if __WORDSIZE == 64 + case jit_code_arg_l: +# endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1474,16 +1906,34 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_i: case jit_code_getarg_ui: case jit_code_getarg_l: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: case jit_code_retval_s: case jit_code_retval_us: case jit_code_retval_i: +#if __WORDSIZE == 64 case jit_code_retval_ui: case jit_code_retval_l: +#endif case jit_code_retval_f: case jit_code_retval_d: case jit_code_prepare: case jit_code_finishr: case jit_code_finishi: @@ -1559,6 +2009,23 @@ _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) stxi_d(i0, rn(r0), rn(r1)); } +static void +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = 16; /* ra+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += sizeof(jit_word_t); + + for (reg = 0; reg < jit_size(fregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) + _jitc->framesize += sizeof(jit_float64_t); + + /* Make sure functions called have a 16 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 15) & -16; +} + static void _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) { diff --git a/parallel-rsp/lightning/lib/jit_alpha-cpu.c b/parallel-rsp/lightning/lib/jit_alpha-cpu.c index 8bfef9c..91d15c8 100644 --- a/parallel-rsp/lightning/lib/jit_alpha-cpu.c +++ b/parallel-rsp/lightning/lib/jit_alpha-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2019 Free Software Foundation, Inc. + * Copyright (C) 2014-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -311,8 +311,21 @@ static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); static void _movi(jit_state_t*,jit_int32_t,jit_word_t); # define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define movnr(r0,r1,r2) CMOVNE(r2, r1, r0) +# define movzr(r0,r1,r2) CMOVEQ(r2, r1, r0) +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define negr(r0,r1) NEGQ(r1,r0) # define comr(r0,r1) NOT(r1,r0) +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) CTLZ(r1, r0) +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) CTTZ(r1, r0) # define addr(r0,r1,r2) ADDQ(r1,r2,r0) # define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); @@ -622,21 +635,15 @@ static void _extr_us(jit_state_t*,jit_int32_t,jit_int32_t); static void _extr_i(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_ui(r0,r1) _extr_ui(_jit,r0,r1) static void _extr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# if __BYTE_ORDER == __LITTLE_ENDIAN -# define htonr_us(r0,r1) _htonr_us(_jit,r0,r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0,r1) _htonr_ui(_jit,r0,r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ul(r0,r1) _htonr_ul(_jit,r0,r1) -static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t); -# else -# define htonr_us(r0,r1) extr_us(r0,r1) -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) -# endif +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ul(r0,r1) _bswapr_ul(_jit,r0,r1) +static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t); # define jmpr(r0) JMP(_R31_REGNO,r0,0) # define jmpi(i0) _jmpi(_jit,i0) -static void _jmpi(jit_state_t*, jit_word_t); +static jit_word_t _jmpi(jit_state_t*, jit_word_t); # define jmpi_p(i0) _jmpi_p(_jit,i0) static jit_word_t _jmpi_p(jit_state_t*, jit_word_t); #define callr(r0) _callr(_jit,r0) @@ -811,6 +818,48 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_word_t jump0, jump1, again, done; + jit_int32_t iscasi, r1_reg; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + again = _jit->pc.w; /* AGAIN */ + LDQ_L(r0, r1, 0); /* Load r0 locked */ + jump0 = bner(_jit->pc.w, r0, r2); /* bne FAIL r0 r2 */ + movr(r0, r3); /* Move to r0 to attempt to store */ + STQ_C(r0, r1, 0); /* r0 is an in/out argument */ + jump1 = _jit->pc.w; + BEQ(r0, 0); /* beqi AGAIN r0 0 */ + patch_at(jump1, again); + jump1 = _jit->pc.w; + BR(_R31_REGNO, 0); /* r0 set to 1 if store succeeded */ + patch_at(jump0, _jit->pc.w); /* FAIL: */ + movi(r0, 0); /* Already locked */ + patch_at(jump1, _jit->pc.w); + if (iscasi) + jit_unget_reg(r1_reg); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + comr(r0, r1); + clzr(r0, r0); +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + comr(r0, r1); + ctzr(r0, r0); +} + static void _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -2453,7 +2502,7 @@ _extr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t t0; t0 = jit_get_reg(jit_class_gpr); @@ -2465,7 +2514,7 @@ _htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t t0; jit_int32_t t1; @@ -2491,7 +2540,7 @@ _htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } static void -_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t t0; jit_int32_t t1; @@ -2514,7 +2563,7 @@ _htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) jit_unget_reg(t0); } -static void +static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { jit_word_t w; @@ -2524,7 +2573,8 @@ _jmpi(jit_state_t *_jit, jit_word_t i0) if (_s21_p(d)) BR(_R31_REGNO, d); else - (void)jmpi_p(i0); + w = jmpi_p(i0); + return (w); } static jit_word_t diff --git a/parallel-rsp/lightning/lib/jit_alpha-fpu.c b/parallel-rsp/lightning/lib/jit_alpha-fpu.c index ea5c746..83736b7 100644 --- a/parallel-rsp/lightning/lib/jit_alpha-fpu.c +++ b/parallel-rsp/lightning/lib/jit_alpha-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2019 Free Software Foundation, Inc. + * Copyright (C) 2014-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/lib/jit_alpha-sz.c b/parallel-rsp/lightning/lib/jit_alpha-sz.c index e1a572a..14949b2 100644 --- a/parallel-rsp/lightning/lib/jit_alpha-sz.c +++ b/parallel-rsp/lightning/lib/jit_alpha-sz.c @@ -1,20 +1,23 @@ - #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 76 +#define JIT_INSTR_MAX 88 0, /* data */ 0, /* live */ - 4, /* align */ + 12, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ - 76, /* prolog */ + 88, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -22,11 +25,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 0, /* va_start */ - 0, /* va_arg */ - 0, /* va_arg_d */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 20, /* va_start */ + 24, /* va_arg */ + 44, /* va_arg_d */ 0, /* va_end */ 4, /* addr */ 32, /* addi */ @@ -47,18 +62,18 @@ 56, /* qmuli */ 12, /* qmulr_u */ 32, /* qmuli_u */ - 48, /* divr */ - 72, /* divi */ - 48, /* divr_u */ - 72, /* divi_u */ - 56, /* qdivr */ - 56, /* qdivi */ - 56, /* qdivr_u */ - 56, /* qdivi_u */ - 48, /* remr */ - 72, /* remi */ - 48, /* remr_u */ - 72, /* remi_u */ + 44, /* divr */ + 68, /* divi */ + 44, /* divr_u */ + 68, /* divi_u */ + 52, /* qdivr */ + 52, /* qdivi */ + 52, /* qdivr_u */ + 52, /* qdivi_u */ + 44, /* remr */ + 68, /* remi */ + 44, /* remr_u */ + 68, /* remi_u */ 4, /* andr */ 32, /* andi */ 4, /* orr */ @@ -95,12 +110,19 @@ 12, /* nei */ 4, /* movr */ 32, /* movi */ + 4, /* movnr */ + 4, /* movzr */ + 32, /* casr */ + 60, /* casi */ 8, /* extr_c */ 8, /* extr_uc */ 8, /* extr_s */ 8, /* extr_us */ 8, /* extr_i */ 8, /* extr_ui */ + 16, /* bswapr_us */ + 36, /* bswapr_ui */ + 36, /* bswapr_ul */ 16, /* htonr_us */ 36, /* htonr_ui */ 36, /* htonr_ul */ @@ -119,19 +141,19 @@ 4, /* ldr_l */ 32, /* ldi_l */ 16, /* ldxr_c */ - 12, /* ldxi_c */ + 44, /* ldxi_c */ 8, /* ldxr_uc */ - 4, /* ldxi_uc */ + 36, /* ldxi_uc */ 16, /* ldxr_s */ - 12, /* ldxi_s */ + 44, /* ldxi_s */ 8, /* ldxr_us */ - 4, /* ldxi_us */ + 36, /* ldxi_us */ 8, /* ldxr_i */ - 4, /* ldxi_i */ + 36, /* ldxi_i */ 16, /* ldxr_ui */ - 12, /* ldxi_ui */ + 44, /* ldxi_ui */ 8, /* ldxr_l */ - 4, /* ldxi_l */ + 36, /* ldxi_l */ 4, /* str_c */ 32, /* sti_c */ 4, /* str_s */ @@ -141,13 +163,13 @@ 4, /* str_l */ 32, /* sti_l */ 8, /* stxr_c */ - 4, /* stxi_c */ + 36, /* stxi_c */ 8, /* stxr_s */ - 4, /* stxi_s */ + 36, /* stxi_s */ 8, /* stxr_i */ - 4, /* stxi_i */ + 36, /* stxi_i */ 8, /* stxr_l */ - 4, /* stxi_l */ + 36, /* stxi_l */ 8, /* bltr */ 8, /* blti */ 8, /* bltr_u */ @@ -188,18 +210,42 @@ 32, /* bxsubi */ 16, /* bxsubr_u */ 16, /* bxsubi_u */ - 0, /* jmpr */ - 36, /* jmpi */ + 4, /* jmpr */ + 4, /* jmpi */ 8, /* callr */ 36, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -207,93 +253,93 @@ 0, /* retval_i */ 0, /* retval_ui */ 0, /* retval_l */ - 68, /* epilog */ + 76, /* epilog */ 0, /* arg_f */ 0, /* getarg_f */ 0, /* putargr_f */ 0, /* putargi_f */ 8, /* addr_f */ - 32, /* addi_f */ + 40, /* addi_f */ 8, /* subr_f */ - 32, /* subi_f */ - 32, /* rsbi_f */ + 40, /* subi_f */ + 40, /* rsbi_f */ 8, /* mulr_f */ - 32, /* muli_f */ + 40, /* muli_f */ 8, /* divr_f */ - 32, /* divi_f */ + 40, /* divi_f */ 4, /* negr_f */ 4, /* absr_f */ 8, /* sqrtr_f */ 32, /* ltr_f */ - 56, /* lti_f */ + 64, /* lti_f */ 32, /* ler_f */ - 56, /* lei_f */ + 64, /* lei_f */ 32, /* eqr_f */ - 56, /* eqi_f */ + 64, /* eqi_f */ 32, /* ger_f */ - 56, /* gei_f */ + 64, /* gei_f */ 32, /* gtr_f */ - 56, /* gti_f */ + 64, /* gti_f */ 32, /* ner_f */ - 56, /* nei_f */ + 64, /* nei_f */ 32, /* unltr_f */ - 56, /* unlti_f */ + 64, /* unlti_f */ 32, /* unler_f */ - 56, /* unlei_f */ + 64, /* unlei_f */ 32, /* uneqr_f */ - 56, /* uneqi_f */ + 64, /* uneqi_f */ 32, /* unger_f */ - 56, /* ungei_f */ + 64, /* ungei_f */ 32, /* ungtr_f */ - 56, /* ungti_f */ + 64, /* ungti_f */ 32, /* ltgtr_f */ - 56, /* ltgti_f */ + 64, /* ltgti_f */ 20, /* ordr_f */ - 44, /* ordi_f */ + 52, /* ordi_f */ 20, /* unordr_f */ - 44, /* unordi_f */ + 52, /* unordi_f */ 16, /* truncr_f_i */ 16, /* truncr_f_l */ 12, /* extr_f */ 4, /* extr_d_f */ 4, /* movr_f */ - 24, /* movi_f */ + 32, /* movi_f */ 4, /* ldr_f */ 32, /* ldi_f */ 8, /* ldxr_f */ - 4, /* ldxi_f */ + 36, /* ldxi_f */ 4, /* str_f */ 32, /* sti_f */ 8, /* stxr_f */ - 4, /* stxi_f */ + 36, /* stxi_f */ 24, /* bltr_f */ - 48, /* blti_f */ + 56, /* blti_f */ 24, /* bler_f */ - 48, /* blei_f */ + 56, /* blei_f */ 24, /* beqr_f */ - 48, /* beqi_f */ + 56, /* beqi_f */ 24, /* bger_f */ - 48, /* bgei_f */ + 56, /* bgei_f */ 24, /* bgtr_f */ - 48, /* bgti_f */ + 56, /* bgti_f */ 28, /* bner_f */ - 52, /* bnei_f */ + 60, /* bnei_f */ 28, /* bunltr_f */ - 52, /* bunlti_f */ + 60, /* bunlti_f */ 28, /* bunler_f */ - 52, /* bunlei_f */ + 60, /* bunlei_f */ 28, /* buneqr_f */ - 52, /* buneqi_f */ + 60, /* buneqi_f */ 28, /* bunger_f */ - 52, /* bungei_f */ + 60, /* bungei_f */ 28, /* bungtr_f */ - 52, /* bungti_f */ + 60, /* bungti_f */ 28, /* bltgtr_f */ - 52, /* bltgti_f */ + 60, /* bltgti_f */ 12, /* bordr_f */ - 36, /* bordi_f */ + 44, /* bordi_f */ 12, /* bunordr_f */ - 36, /* bunordi_f */ + 44, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -304,87 +350,87 @@ 0, /* putargr_d */ 0, /* putargi_d */ 8, /* addr_d */ - 28, /* addi_d */ + 40, /* addi_d */ 8, /* subr_d */ - 28, /* subi_d */ - 28, /* rsbi_d */ + 40, /* subi_d */ + 40, /* rsbi_d */ 8, /* mulr_d */ - 28, /* muli_d */ + 40, /* muli_d */ 8, /* divr_d */ - 28, /* divi_d */ + 40, /* divi_d */ 4, /* negr_d */ 4, /* absr_d */ 8, /* sqrtr_d */ 32, /* ltr_d */ - 52, /* lti_d */ + 64, /* lti_d */ 32, /* ler_d */ - 52, /* lei_d */ + 64, /* lei_d */ 32, /* eqr_d */ - 52, /* eqi_d */ + 64, /* eqi_d */ 32, /* ger_d */ - 52, /* gei_d */ + 64, /* gei_d */ 32, /* gtr_d */ - 52, /* gti_d */ + 64, /* gti_d */ 32, /* ner_d */ - 52, /* nei_d */ + 64, /* nei_d */ 32, /* unltr_d */ - 52, /* unlti_d */ + 64, /* unlti_d */ 32, /* unler_d */ - 52, /* unlei_d */ + 64, /* unlei_d */ 32, /* uneqr_d */ - 52, /* uneqi_d */ + 64, /* uneqi_d */ 32, /* unger_d */ - 52, /* ungei_d */ + 64, /* ungei_d */ 32, /* ungtr_d */ - 52, /* ungti_d */ + 64, /* ungti_d */ 32, /* ltgtr_d */ - 52, /* ltgti_d */ + 64, /* ltgti_d */ 20, /* ordr_d */ - 40, /* ordi_d */ + 52, /* ordi_d */ 20, /* unordr_d */ - 40, /* unordi_d */ + 52, /* unordi_d */ 16, /* truncr_d_i */ 16, /* truncr_d_l */ 12, /* extr_d */ 4, /* extr_f_d */ 4, /* movr_d */ - 20, /* movi_d */ + 32, /* movi_d */ 4, /* ldr_d */ 32, /* ldi_d */ 8, /* ldxr_d */ - 4, /* ldxi_d */ + 36, /* ldxi_d */ 4, /* str_d */ 32, /* sti_d */ 8, /* stxr_d */ - 4, /* stxi_d */ + 36, /* stxi_d */ 24, /* bltr_d */ - 44, /* blti_d */ + 56, /* blti_d */ 24, /* bler_d */ - 44, /* blei_d */ + 56, /* blei_d */ 24, /* beqr_d */ - 44, /* beqi_d */ + 56, /* beqi_d */ 24, /* bger_d */ - 44, /* bgei_d */ + 56, /* bgei_d */ 24, /* bgtr_d */ - 44, /* bgti_d */ + 56, /* bgti_d */ 28, /* bner_d */ - 48, /* bnei_d */ + 60, /* bnei_d */ 28, /* bunltr_d */ - 48, /* bunlti_d */ + 60, /* bunlti_d */ 28, /* bunler_d */ - 48, /* bunlei_d */ + 60, /* bunlei_d */ 28, /* buneqr_d */ - 48, /* buneqi_d */ + 60, /* buneqi_d */ 28, /* bunger_d */ - 48, /* bungei_d */ + 60, /* bungei_d */ 28, /* bungtr_d */ - 48, /* bungti_d */ + 60, /* bungti_d */ 28, /* bltgtr_d */ - 48, /* bltgti_d */ + 60, /* bltgti_d */ 12, /* bordr_d */ - 32, /* bordi_d */ + 44, /* bordi_d */ 12, /* bunordr_d */ - 32, /* bunordi_d */ + 44, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -399,4 +445,8 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 8, /* cto */ + 4, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_alpha.c b/parallel-rsp/lightning/lib/jit_alpha.c index 9a067aa..25566f4 100644 --- a/parallel-rsp/lightning/lib/jit_alpha.c +++ b/parallel-rsp/lightning/lib/jit_alpha.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2019 Free Software Foundation, Inc. + * Copyright (C) 2014-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -246,20 +246,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -319,7 +317,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_f_reg_p(u->u.w)); @@ -361,18 +359,22 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function != NULL); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { offset = _jitc->function->self.size; _jitc->function->self.size += 8; } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -417,7 +419,7 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, _A0 - v->u.w); @@ -429,7 +431,7 @@ _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, _A0 - v->u.w); @@ -441,7 +443,7 @@ _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, _A0 - v->u.w); @@ -453,7 +455,7 @@ _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, _A0 - v->u.w); @@ -465,7 +467,7 @@ _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_i(u, _A0 - v->u.w); @@ -477,7 +479,7 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, _A0 - v->u.w); @@ -489,7 +491,7 @@ _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, _A0 - v->u.w); @@ -499,10 +501,10 @@ _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(_A0 - v->u.w, u); else @@ -511,11 +513,11 @@ _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(_A0 - v->u.w, u); else { @@ -610,10 +612,10 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function != NULL); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(_A0 - _jitc->function->call.argi, u); @@ -627,11 +629,11 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_int64_t u) +_jit_pushargi(jit_state_t *_jit, jit_int64_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function != NULL); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(_A0 - _jitc->function->call.argi, u); @@ -863,6 +865,7 @@ _emit_code(jit_state_t *_jit) jit_node_t *node; jit_uint8_t *data; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -978,11 +981,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1086,12 +1091,25 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _us); case_rr(hton, _ui); case_rr(hton, _ul); + case_rr(bswap, _us); + case_rr(bswap, _ui); + case_rr(bswap, _ul); case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); case_rr(ext, _us); case_rr(ext, _i); case_rr(ext, _ui); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1112,6 +1130,10 @@ _emit_code(jit_state_t *_jit) break; case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(lt,); case_rrw(lt,); case_rrr(lt, _u); @@ -1349,7 +1371,12 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (_s21_p(word)) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } @@ -1378,6 +1405,7 @@ _emit_code(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1398,6 +1426,16 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1422,11 +1460,18 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_l: case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1434,10 +1479,22 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_i: case jit_code_getarg_ui: case jit_code_getarg_l: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: diff --git a/parallel-rsp/lightning/lib/jit_arm-cpu.c b/parallel-rsp/lightning/lib/jit_arm-cpu.c index b6ee260..a0852a2 100644 --- a/parallel-rsp/lightning/lib/jit_arm-cpu.c +++ b/parallel-rsp/lightning/lib/jit_arm-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -36,8 +36,8 @@ # define jit_armv5_p() (jit_cpu.version >= 5) # define jit_armv5e_p() (jit_cpu.version > 5 || (jit_cpu.version == 5 && jit_cpu.extend)) # define jit_armv6_p() (jit_cpu.version >= 6) -# define jit_armv7r_p() 0 -# define stack_framesize 48 +# define jit_armv7_p() (jit_cpu.version >= 7) +# define jit_armv7r_p() (jit_cpu.version > 7 || (jit_cpu.version == 7 && jit_cpu.extend)) extern int __aeabi_idivmod(int, int); extern unsigned __aeabi_uidivmod(unsigned, unsigned); # define _R0_REGNO 0x00 @@ -140,8 +140,12 @@ extern unsigned __aeabi_uidivmod(unsigned, unsigned); # define THUMB2_UMULL 0xfba00000 # define ARM_SMULL 0x00c00090 # define THUMB2_SMULL 0xfb800000 +/* >> ARMv7r */ +# define ARM_SDIV 0x07100010 +# define ARM_UDIV 0x07300010 # define THUMB2_SDIV 0xfb90f0f0 # define THUMB2_UDIV 0xfbb0f0f0 +/* << ARMv7r */ # define ARM_AND 0x00000000 # define THUMB_AND 0x4000 # define THUMB2_AND 0xea000000 @@ -179,7 +183,29 @@ extern unsigned __aeabi_uidivmod(unsigned, unsigned); # define ARM_XTR8 0x00000400 /* ?xt? rotate 8 bits */ # define ARM_XTR16 0x00000800 /* ?xt? rotate 16 bits */ # define ARM_XTR24 0x00000c00 /* ?xt? rotate 24 bits */ +# define ARM_LDREX 0x01900090 +# define THUMB2_LDREX 0xe8500000 +# define ARM_STREX 0x01800090 +# define THUMB2_STREX 0xe8400000 /* << ARMv6* */ +/* >> ARMv6t2 */ +# define THUMB2_CLZ 0xfab0f080 +# define THUMB2_RBIT 0xfa90f0a0 +# define ARM_RBIT 0x06f00030 +/* << ARMv6t2 */ +# define ARM_CLZ 0x01600010 +/* >> ARMv7 */ +# define ARM_DMB 0xf57ff050 +# define THUMB2_DMB 0xf3bf8f50 +# define DMB_SY 0xf +# define DMB_ST 0xe +# define DMB_ISH 0xb +# define DMB_ISHST 0xa +# define DMB_NSH 0x7 +# define DMB_NSHT 0x6 +# define DMB_OSH 0x3 +# define DMB_OSHST 0x2 +/* << ARMv7 */ # define ARM_SHIFT 0x01a00000 # define ARM_R 0x00000010 /* register shift */ # define ARM_LSL 0x00000000 @@ -399,6 +425,12 @@ static void _tcit(jit_state_t*,unsigned int,int); static void _tpp(jit_state_t*,int,int); # define torl(o,rn,im) _torl(_jit,o,rn,im) static void _torl(jit_state_t*,int,int,int) maybe_unused; +# define DMB(im) dmb(im) +# define T2_DMB(im) tdmb(im) +# define dmb(im) _dmb(_jit, im) +static void _dmb(jit_state_t *_jit, int im); +# define tdmb(im) _tdmb(_jit, im) +static void _tdmb(jit_state_t *_jit, int im); # define CC_MOV(cc,rd,rm) corrr(cc,ARM_MOV,0,rd,rm) # define MOV(rd,rm) CC_MOV(ARM_CC_AL,rd,rm) # define T1_MOV(rd,rm) is(THUMB_MOV|((_u4(rd)&8)<<4)|(_u4(rm)<<3)|(rd&7)) @@ -424,6 +456,12 @@ static void _torl(jit_state_t*,int,int,int) maybe_unused; # define NOT(rd,rm) CC_NOT(ARM_CC_AL,rd,rm) # define T1_NOT(rd,rm) T1_MVN(rd,rm) # define T2_NOT(rd,rm) T2_MVN(rd,rm) +# define T2_CLZ(rd,rm) torrr(THUMB2_CLZ,rm,rd,rm) +# define CC_CLZ(cc,rd,rm) corrrr(cc,ARM_CLZ,_R15_REGNO,rd,_R15_REGNO,rm) +# define CLZ(rd,rm) CC_CLZ(ARM_CC_AL,rd,rm) +# define T2_RBIT(rd,rm) torrr(THUMB2_RBIT,rm,rd,rm) +# define CC_RBIT(cc,rd,rm) corrrr(cc,ARM_RBIT,_R15_REGNO,rd,_R15_REGNO,rm) +# define RBIT(rd,rm) CC_RBIT(ARM_CC_AL,rd,rm) # define NOP() MOV(_R0_REGNO, _R0_REGNO) # define T1_NOP() is(0xbf00) # define CC_ADD(cc,rd,rn,rm) corrr(cc,ARM_ADD,rn,rd,rm) @@ -501,6 +539,10 @@ static void _torl(jit_state_t*,int,int,int) maybe_unused; # define CC_UMULL(cc,rl,rh,rn,rm) corrrr(cc,ARM_UMULL,rh,rl,rm,rn) # define UMULL(rl,rh,rn,rm) CC_UMULL(ARM_CC_AL,rl,rh,rn,rm) # define T2_UMULL(rl,rh,rn,rm) torrrr(THUMB2_UMULL,rn,rl,rh,rm) +# define CC_SDIV(cc,rd,rn,rm) corrrr(cc,ARM_SDIV,rd,15,rn,rm) +# define SDIV(rd,rn,rm) CC_SDIV(ARM_CC_AL,rd,rm,rn) +# define CC_UDIV(cc,rd,rn,rm) corrrr(cc,ARM_UDIV,rd,15,rn,rm) +# define UDIV(rd,rn,rm) CC_UDIV(ARM_CC_AL,rd,rm,rn) # define T2_SDIV(rd,rn,rm) torrr(THUMB2_SDIV,rn,rd,rm) # define T2_UDIV(rd,rn,rm) torrr(THUMB2_UDIV,rn,rd,rm) # define CC_AND(cc,rd,rn,rm) corrr(cc,ARM_AND,rn,rd,rm) @@ -612,7 +654,7 @@ static void _torl(jit_state_t*,int,int,int) maybe_unused; # define CMNI(rn,im) CC_CMNI(ARM_CC_AL,rn,im) # define T2_CMNI(rn,im) torri(THUMB2_CMNI,rn,_R15_REGNO,im) # define CC_TST(cc,rn,rm) corrr(cc,ARM_TST,rn,r0,rm) -# define TST(rn,rm) CC_TST(ARM_CC_AL,rn,rm) +# define TST(rn,rm) corrr(ARM_CC_AL,ARM_TST,rn,0,rm) # define T1_TST(rn,rm) is(THUMB_TST|(_u3(rm)<<3)|_u3(rn)) # define T2_TST(rn,rm) torrr(THUMB2_TST,rn,_R15_REGNO,rm) # define CC_TSTI(cc,rn,im) corri(cc,ARM_TST|ARM_I,rn,0,im) @@ -718,6 +760,9 @@ static void _torl(jit_state_t*,int,int,int) maybe_unused; # define CC_LDRDIN(cc,rt,rn,im) corri8(cc,ARM_LDRDI,rn,rt,im) # define LDRDIN(rt,rn,im) CC_LDRDIN(ARM_CC_AL,rt,rn,im) # define T2_LDRDIN(rt,rt2,rn,im) torrri8(THUMB2_LDRDI,rn,rt,rt2,im) +# define CC_LDREX(cc,rt,rn) corrrr(cc,ARM_LDREX,rn,rt,0xf,0xf) +# define LDREX(rt,rn) CC_LDREX(ARM_CC_AL,rt,rn) +# define T2_LDREX(rt,rn,im) torrri8(THUMB2_LDREX,rn,rt,0xf,im) # define CC_STRB(cc,rt,rn,rm) corrr(cc,ARM_STRB|ARM_P,rn,rt,rm) # define STRB(rt,rn,rm) CC_STRB(ARM_CC_AL,rt,rn,rm) # define T1_STRB(rt,rn,rm) is(THUMB_STRB|(_u3(rm)<<6)|(_u3(rn)<<3)|_u3(rt)) @@ -771,6 +816,9 @@ static void _torl(jit_state_t*,int,int,int) maybe_unused; # define CC_STRDIN(cc,rt,rn,im) corri8(cc,ARM_STRDI,rn,rt,im) # define STRDIN(rt,rn,im) CC_STRDIN(ARM_CC_AL,rt,rn,im) # define T2_STRDIN(rt,rt2,rn,im) torrri8(THUMB2_STRDI,rn,rt,rt2,im) +# define CC_STREX(cc,rd,rt,rn) corrrr(cc,ARM_STREX,rn,rd,0xf,rt) +# define STREX(rd,rt,rn) CC_STREX(ARM_CC_AL,rd,rt,rn) +# define T2_STREX(rd,rt,rn,im) torrri8(THUMB2_STREX,rn,rt,rd,im) # define CC_LDMIA(cc,rn,im) corl(cc,ARM_M|ARM_M_L|ARM_M_I,rn,im) # define LDMIA(rn,im) CC_LDMIA(ARM_CC_AL,rn,im) # define CC_LDM(cc,rn,im) CC_LDMIA(cc,rn,im) @@ -823,6 +871,8 @@ static void _torl(jit_state_t*,int,int,int) maybe_unused; # define T2_POP(im) tpp(THUMB2_POP,im) # define jit_get_reg_args() \ do { \ + CHECK_REG_ARGS(); \ + jit_check_frame(); \ (void)jit_get_reg(_R0|jit_class_named|jit_class_gpr); \ (void)jit_get_reg(_R1|jit_class_named|jit_class_gpr); \ (void)jit_get_reg(_R2|jit_class_named|jit_class_gpr); \ @@ -843,10 +893,27 @@ static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); static void _movi(jit_state_t*,jit_int32_t,jit_word_t); # define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define comr(r0,r1) _comr(_jit,r0,r1) static void _comr(jit_state_t*,jit_int32_t,jit_int32_t); # define negr(r0,r1) _negr(_jit,r0,r1) static void _negr(jit_state_t*,jit_int32_t,jit_int32_t); +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) _clzr(_jit, r0, r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # define addr(r0,r1,r2) _addr(_jit,r0,r1,r2) static void _addr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) @@ -1091,15 +1158,10 @@ static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t); static void _stxr_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); # define stxi_i(r0,r1,i0) _stxi_i(_jit,r0,r1,i0) static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -# if __BYTE_ORDER == __LITTLE_ENDIAN -# define htonr_us(r0,r1) _htonr_us(_jit,r0,r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0,r1) _htonr_ui(_jit,r0,r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# else -# define htonr_us(r0,r1) extr_us(r0,r1) -# define htonr(r0,r1) movr(r0,r1) -# endif +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_c(r0,r1) _extr_c(_jit,r0,r1) static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_uc(r0,r1) _extr_uc(_jit,r0,r1) @@ -1114,10 +1176,10 @@ static void _prolog(jit_state_t*,jit_node_t*); static void _epilog(jit_state_t*,jit_node_t*); # define callr(r0) _callr(_jit,r0) static void _callr(jit_state_t*,jit_int32_t); -# define calli(i0) _calli(_jit,i0) -static void _calli(jit_state_t*,jit_word_t); -# define calli_p(i0) _calli_p(_jit,i0) -static jit_word_t _calli_p(jit_state_t*,jit_word_t); +# define calli(i0,i1) _calli(_jit,i0,i1) +static void _calli(jit_state_t*,jit_word_t,jit_bool_t); +# define calli_p(i0,i1) _calli_p(_jit,i0,i1) +static jit_word_t _calli_p(jit_state_t*,jit_word_t,jit_bool_t); # define vastart(r0) _vastart(_jit, r0) static void _vastart(jit_state_t*, jit_int32_t); # define vaarg(r0, r1) _vaarg(_jit, r0, r1) @@ -1493,7 +1555,7 @@ _tpp(jit_state_t *_jit, int o, int im) assert(!(o & 0x0000ffff)); if (o == THUMB2_PUSH) assert(!(im & 0x8000)); - assert(__builtin_popcount(im & 0x1fff) > 1); + assert(__builtin_popcount(im & 0x7fff) > 1); thumb.i = o|im; iss(thumb.s[0], thumb.s[1]); } @@ -1509,6 +1571,22 @@ _torl(jit_state_t *_jit, int o, int rn, int im) iss(thumb.s[0], thumb.s[1]); } +static void +_dmb(jit_state_t *_jit, int im) +{ + assert(!(im & 0xfffffff0)); + ii(ARM_DMB|im); +} + +static void +_tdmb(jit_state_t *_jit, int im) +{ + jit_thumb_t thumb; + assert(!(im & 0xfffffff0)); + thumb.i = THUMB2_DMB | im; + iss(thumb.s[0], thumb.s[1]); +} + static void _nop(jit_state_t *_jit, jit_int32_t i0) { @@ -1582,6 +1660,86 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_movznr(jit_state_t *_jit, int ct, jit_int32_t r0, + jit_int32_t r1, jit_int32_t r2) +{ + if (jit_thumb_p()) { + if (r2 < 7) + T1_CMPI(r2, 0); + else + T2_CMPI(r2, 0); + IT(ct); + T1_MOV(r0, r1); + } else { + CMPI(r2, 0); + CC_MOV(ct, r0, r1); + } +} + +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + _movznr(_jit, ARM_CC_NE, r0, r1, r2); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + _movznr(_jit, ARM_CC_EQ, r0, r1, r2); +} + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t r1_reg, iscasi; + jit_word_t retry, done, jump0, jump1; + if (!jit_armv7_p()) + fallback_casx(r0, r1, r2, r3, i0); + else { + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + if (jit_thumb_p()) { + T2_DMB(DMB_ISH); + /* retry: */ + retry = _jit->pc.w; + T2_LDREX(r0, r1, 0); + eqr(r0, r0, r2); + jump0 = beqi(_jit->pc.w, r0, 0); /* beqi done r0 0 */ + T2_STREX(r0, r3, r1, 0); + jump1 = bnei(_jit->pc.w, r0, 0); /* bnei retry r0 0 */ + /* r0 = 0 if memory updated, 1 otherwise */ + xori(r0, r0, 1); + /* done: */ + done = _jit->pc.w; + T2_DMB(DMB_ISH); + } + else { + DMB(DMB_ISH); + /* retry: */ + retry = _jit->pc.w; + LDREX(r0, r1); + eqr(r0, r0, r2); + jump0 = beqi(_jit->pc.w, r0, 0); /* beqi done r0 0 */ + STREX(r0, r3, r1); + jump1 = bnei(_jit->pc.w, r0, 0); /* bnei retry r0 0 */ + /* r0 = 0 if memory updated, 1 otherwise */ + xori(r0, r0, 1); + /* done: */ + done = _jit->pc.w; + DMB(DMB_ISH); + } + patch_at(arm_patch_jump, jump0, done); + patch_at(arm_patch_jump, jump1, retry); + if (iscasi) + jit_unget_reg(r1_reg); + } +} + static void _comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { @@ -1608,6 +1766,53 @@ _negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) RSBI(r0, r1, 0); } +static void +_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (!jit_thumb_p() && jit_armv5e_p()) + CLZ(r0, r1); + else if (jit_thumb_p() && jit_armv7_p()) { /* armv6t2 actually */ + T2_CLZ(r0, r1); + } + else + fallback_clz(r0, r0); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + comr(r0, r1); + clzr(r0, r0); +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_armv7_p()) { /* armv6t2 actually */ + if (jit_thumb_p()) + T2_RBIT(r0, r1); + else + RBIT(r0, r1); + clor(r0, r0); + } + else + fallback_cto(r0, r1); +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_armv7_p()) { /* armv6t2 actually */ + if (jit_thumb_p()) + T2_RBIT(r0, r1); + else + RBIT(r0, r1); + clzr(r0, r0); + } + else + fallback_ctz(r0, r1); +} + static void _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { @@ -2103,8 +2308,12 @@ _divrem(jit_state_t *_jit, int div, int sign, static void _divr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - if (jit_armv7r_p() && jit_thumb_p()) - T2_SDIV(r0, r1, r2); + if (jit_armv7r_p()) { + if (jit_thumb_p()) + T2_SDIV(r0, r1, r2); + else + SDIV(r0, r1, r2); + } else divrem(1, 1, r0, r1, r2); } @@ -2122,8 +2331,12 @@ _divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _divr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - if (jit_armv7r_p() && jit_thumb_p()) - T2_UDIV(r0, r1, r2); + if (jit_armv7r_p()) { + if (jit_thumb_p()) + T2_UDIV(r0, r1, r2); + else + UDIV(r0, r1, r2); + } else divrem(1, 0, r0, r1, r2); } @@ -2183,7 +2396,23 @@ _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, static void _remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - divrem(0, 1, r0, r1, r2); + if (jit_armv7r_p()) { + jit_int32_t reg; + if (r0 == r1 || r0 == r2) { + reg = jit_get_reg(jit_class_gpr); + divr(rn(reg), r1, r2); + mulr(rn(reg), r2, rn(reg)); + subr(r0, r1, rn(reg)); + jit_unget_reg(reg); + } + else { + divr(r0, r1, r2); + mulr(r0, r2, r0); + subr(r0, r1, r0); + } + } + else + divrem(0, 1, r0, r1, r2); } static void @@ -2199,7 +2428,23 @@ _remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - divrem(0, 0, r0, r1, r2); + if (jit_armv7r_p()) { + jit_int32_t reg; + if (r0 == r1 || r0 == r2) { + reg = jit_get_reg(jit_class_gpr); + divr_u(rn(reg), r1, r2); + mulr(rn(reg), r2, rn(reg)); + subr(r0, r1, rn(reg)); + jit_unget_reg(reg); + } + else { + divr_u(r0, r1, r2); + mulr(r0, r2, r0); + subr(r0, r1, r0); + } + } + else + divrem(0, 0, r0, r1, r2); } static void @@ -2612,8 +2857,8 @@ _jmpi_p(jit_state_t *_jit, jit_word_t i0, jit_bool_t i1) jit_word_t w; jit_word_t d; jit_int32_t reg; + /* i1 means jump is reachable in signed 24 bits */ if (i1) { - /* Assume jump is not longer than 23 bits if inside jit */ w = _jit->pc.w; /* if thumb and in thumb mode */ if (jit_thumb_p() && _jitc->thumb) { @@ -3576,11 +3821,9 @@ _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) } } -# if __BYTE_ORDER == __LITTLE_ENDIAN static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - jit_int32_t t0; if (jit_thumb_p()) { if ((r0|r1) < 8) T1_REV(r0, r1); @@ -3594,20 +3837,14 @@ _htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) rshi_u(r0, r0, 16); } else { - t0 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 8); - andi(r0, r1, 0xff); - andi(rn(t0), rn(t0), 0xff); - lshi(r0, r0, 8); - orr(r0, r0, rn(t0)); - jit_unget_reg(t0); + generic_bswapr_us(_jit, r0, r1); } } } /* inline glibc htonl (without register clobber) */ static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (jit_thumb_p()) { @@ -3629,7 +3866,6 @@ _htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } } } -#endif static void _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) @@ -3715,14 +3951,29 @@ _callr(jit_state_t *_jit, jit_int32_t r0) } static void -_calli(jit_state_t *_jit, jit_word_t i0) +_calli(jit_state_t *_jit, jit_word_t i0, jit_bool_t exchange_p) { jit_word_t d; jit_int32_t reg; - d = ((i0 - _jit->pc.w) >> 2) - 2; - if (!jit_exchange_p() && !jit_thumb_p() && _s24P(d)) - BLI(d & 0x00ffffff); + if (!exchange_p) { + if (jit_thumb_p()) { + if (jit_exchange_p()) + /* skip switch from arm to thumb + * exchange_p set to zero means a jit function + * call in the same jit code buffer */ + d = ((i0 + 8 - _jit->pc.w) >> 1) - 2; + else + d = ((i0 - _jit->pc.w) >> 1) - 2; + } + else d = ((i0 - _jit->pc.w) >> 2) - 2; + if (_s24P(d)) { + if (jit_thumb_p()) T2_BLI(encode_thumb_jump(d)); + else BLI(d & 0x00ffffff); + } + else goto fallback; + } else { + fallback: reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); if (jit_thumb_p()) @@ -3734,28 +3985,44 @@ _calli(jit_state_t *_jit, jit_word_t i0) } static jit_word_t -_calli_p(jit_state_t *_jit, jit_word_t i0) +_calli_p(jit_state_t *_jit, jit_word_t i0, jit_bool_t i1) { jit_word_t w; + jit_word_t d; jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - w = _jit->pc.w; - movi_p(rn(reg), i0); - if (jit_thumb_p()) - T1_BLX(rn(reg)); - else - BLX(rn(reg)); - jit_unget_reg(reg); + /* i1 means call is reachable in signed 24 bits */ + if (i1) { + w = _jit->pc.w; + if (jit_thumb_p()) d = ((i0 - _jit->pc.w) >> 1) - 2; + else d = ((i0 - _jit->pc.w) >> 2) - 2; + assert(_s24P(d)); + if (jit_thumb_p()) T2_BLI(encode_thumb_jump(d)); + else BLI(d & 0x00ffffff); + } + else { + reg = jit_get_reg(jit_class_gpr); + w = _jit->pc.w; + movi_p(rn(reg), i0); + if (jit_thumb_p()) + T1_BLX(rn(reg)); + else + BLX(rn(reg)); + jit_unget_reg(reg); + } return (w); } static void _prolog(jit_state_t *_jit, jit_node_t *node) { - jit_int32_t reg; + jit_int32_t reg, mask, count; if (_jitc->function->define_frame || _jitc->function->assume_frame) { jit_int32_t frame = -_jitc->function->frame; + jit_check_frame(); assert(_jitc->function->self.aoff >= frame); + if (jit_swf_p()) + CHECK_SWF_OFFSET(); + CHECK_REG_ARGS(); if (_jitc->function->assume_frame) { if (jit_thumb_p() && !_jitc->thumb) _jitc->thumb = _jit->pc.w; @@ -3768,38 +4035,66 @@ _prolog(jit_state_t *_jit, jit_node_t *node) _jitc->function->stack = ((_jitc->function->self.alen - /* align stack at 8 bytes */ _jitc->function->self.aoff) + 7) & -8; + /* If this jit_check_frame() succeeds, it actually is just a need_stack, + * usually for arguments, so, allocai was not called, but pusharg* + * was called increasing stack size, for negative access offsets. + * This can be optimized for one less prolog instruction, that is, + * do not create the frame pointer, and only add _jitc->function->stack + * to sp, and on epilog, instead of moving fp to sp, just add negative + * value of _jitc->function->stack. Since this condition requires a + * large function body for excess arguments to called function, keep + * things a bit simpler for now, as this is the only place need_stack + * would be useful. */ + if (_jitc->function->stack) + jit_check_frame(); + + for (reg = mask = count = 0; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + mask |= 1 << rn(iregs[reg]); + ++count; + } + } + /* One extra register to keep stack 8 bytes aligned */ + if (count & 1) { + for (reg = 4; reg < 10; reg++) { + if (!(mask & (1 << reg))) { + mask |= 1 << reg; + break; + } + } + } + if (_jitc->function->need_frame || _jitc->function->need_return) + mask |= (1 << _FP_REGNO) | (1 << _LR_REGNO); + if (!jit_swf_p() && _jitc->function->save_reg_args && + !(_jitc->function->self.call & jit_call_varargs)) + mask |= 0xf; if (jit_thumb_p()) { /* switch to thumb mode (better approach would be to * ORR 1 address being called, but no clear distinction * of what is a pointer to a jit function, or if patching * a pointer to a jit function) */ - ADDI(_R12_REGNO, _R15_REGNO, 1); - BX(_R12_REGNO); + if (jit_exchange_p()) { + ADDI(_R12_REGNO, _R15_REGNO, 1); + BX(_R12_REGNO); + } if (!_jitc->thumb) _jitc->thumb = _jit->pc.w; - if (jit_cpu.abi) { + if (jit_swf_p() || (_jitc->function->save_reg_args && + (_jitc->function->self.call & jit_call_varargs))) T2_PUSH(0xf); - T2_PUSH(0x3f0|(1<<_FP_REGNO)|(1<<_LR_REGNO)); - VPUSH_F64(_D8_REGNO, 8); - } - else { - T2_PUSH(0xf); - T2_PUSH(0x3f0|(1<<_FP_REGNO)|(1<<_LR_REGNO)); - } + if (mask) + T2_PUSH(mask); } else { - if (jit_cpu.abi) { + if (jit_swf_p() || (_jitc->function->save_reg_args && + (_jitc->function->self.call & jit_call_varargs))) PUSH(0xf); - PUSH(0x3f0|(1<<_FP_REGNO)|(1<<_LR_REGNO)); - VPUSH_F64(_D8_REGNO, 8); - } - else { - PUSH(0xf); - PUSH(0x3f0|(1<<_FP_REGNO)|(1<<_LR_REGNO)); - } + if (mask) + PUSH(mask); } - movr(_FP_REGNO, _SP_REGNO); + if (_jitc->function->need_frame) + movr(_FP_REGNO, _SP_REGNO); if (_jitc->function->stack) subi(_SP_REGNO, _SP_REGNO, _jitc->function->stack); if (_jitc->function->allocar) { @@ -3813,17 +4108,41 @@ _prolog(jit_state_t *_jit, jit_node_t *node) static void _epilog(jit_state_t *_jit, jit_node_t *node) { + jit_int32_t reg, mask, count; if (_jitc->function->assume_frame) return; - movr(_SP_REGNO, _FP_REGNO); - if (jit_cpu.abi) - VPOP_F64(_D8_REGNO, 8); - if (jit_thumb_p()) - T2_POP(0x3f0|(1<<_FP_REGNO)|(1<<_LR_REGNO)); - else - POP(0x3f0|(1<<_FP_REGNO)|(1<<_LR_REGNO)); - addi(_SP_REGNO, _SP_REGNO, 16); + for (reg = mask = count = 0; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + mask |= 1 << rn(iregs[reg]); + ++count; + } + } + /* One extra register to keep stack 8 bytes aligned */ + if (count & 1) { + for (reg = 4; reg < 10; reg++) { + if (!(mask & (1 << reg))) { + mask |= 1 << reg; + break; + } + } + } + if (_jitc->function->need_frame || _jitc->function->need_return) + mask |= (1 << _FP_REGNO) | (1 << _LR_REGNO); + if (_jitc->function->need_frame) + movr(_SP_REGNO, _FP_REGNO); + if (!jit_swf_p() && _jitc->function->save_reg_args && + !(_jitc->function->self.call & jit_call_varargs)) + addi(_SP_REGNO, _SP_REGNO, 16); + if (mask) { + if (jit_thumb_p()) + T2_POP(mask); + else + POP(mask); + } + if (jit_swf_p() || (_jitc->function->save_reg_args && + (_jitc->function->self.call & jit_call_varargs))) + addi(_SP_REGNO, _SP_REGNO, 16); if (jit_thumb_p()) T1_BX(_LR_REGNO); else @@ -3841,8 +4160,7 @@ _vastart(jit_state_t *_jit, jit_int32_t r0) * The -16 is to account for the 4 argument registers * always saved, and _jitc->function->vagp is to account * for declared arguments. */ - addi(r0, _FP_REGNO, _jitc->function->self.size - - 16 + _jitc->function->vagp); + addi(r0, _FP_REGNO, jit_selfsize() - 16 + _jitc->function->vagp); } static void @@ -3869,7 +4187,28 @@ _patch_at(jit_state_t *_jit, jit_word_t w; } u; u.w = instr; - if (kind == arm_patch_jump) { + if (kind == arm_patch_call) { + if (jit_thumb_p() && (jit_uword_t)instr >= _jitc->thumb) { + code2thumb(thumb.s[0], thumb.s[1], u.s[0], u.s[1]); + assert((thumb.i & THUMB2_BLI) == THUMB2_BLI); + /* skip code to switch from arm to thumb mode */ + if (jit_exchange_p()) + d = ((label + 8 - instr) >> 1) - 2; + else + d = ((label - instr) >> 1) - 2; + assert(_s24P(d)); + thumb.i = THUMB2_BLI | encode_thumb_jump(d); + thumb2code(thumb.s[0], thumb.s[1], u.s[0], u.s[1]); + } + else { + thumb.i = u.i[0]; + assert((thumb.i & 0x0f000000) == ARM_BLI); + d = ((label - instr) >> 2) - 2; + assert(_s24P(d)); + u.i[0] = (thumb.i & 0xff000000) | (d & 0x00ffffff); + } + } + else if (kind == arm_patch_jump) { if (jit_thumb_p() && (jit_uword_t)instr >= _jitc->thumb) { code2thumb(thumb.s[0], thumb.s[1], u.s[0], u.s[1]); if ((thumb.i & THUMB2_B) == THUMB2_B) { diff --git a/parallel-rsp/lightning/lib/jit_arm-swf.c b/parallel-rsp/lightning/lib/jit_arm-swf.c index bf86ca1..2aa6a12 100644 --- a/parallel-rsp/lightning/lib/jit_arm-swf.c +++ b/parallel-rsp/lightning/lib/jit_arm-swf.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -564,6 +564,8 @@ _swf_ff(jit_state_t *_jit, float(*i0)(float), jit_int32_t r0, jit_int32_t r1) { jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); else @@ -581,6 +583,8 @@ _swf_dd(jit_state_t *_jit, double (*i0)(double), jit_int32_t r0, jit_int32_t r1) { jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -614,6 +618,8 @@ _swf_fff(jit_state_t *_jit, float (*i0)(float, float), jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1) || jit_fpr_p(r2)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); else @@ -635,6 +641,8 @@ _swf_ddd(jit_state_t *_jit, double (*i0)(double, double), jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1) || jit_fpr_p(r2)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -684,6 +692,8 @@ _swf_fff_(jit_state_t *_jit, float (*i0)(float, float), jit_float32_t f; } data; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.f = i1; if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -706,6 +716,8 @@ _swf_rsbi_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float32_t i0) jit_float32_t f; } data; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.f = i0; movi(_R0_REGNO, data.i); if (jit_fpr_p(r1)) @@ -729,7 +741,8 @@ _swf_ddd_(jit_state_t *_jit, double (*i0)(double, double), jit_float64_t d; } data; jit_get_reg_args(); - + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.d = i1; if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) @@ -769,6 +782,8 @@ _swf_rsbi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float64_t i0) jit_float64_t d; } data; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.d = i0; movi(_R0_REGNO, data.i[0]); movi(_R1_REGNO, data.i[1]); @@ -805,6 +820,8 @@ _swf_iff(jit_state_t *_jit, int (*i0)(float, float), jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_get_reg_args(); + if (jit_fpr_p(r1) || jit_fpr_p(r2)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); else @@ -823,6 +840,8 @@ _swf_idd(jit_state_t *_jit, int (*i0)(double, double), jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_get_reg_args(); + if (jit_fpr_p(r1) || jit_fpr_p(r2)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -861,6 +880,8 @@ _swf_iff_(jit_state_t *_jit, int (*i0)(float, float), jit_float32_t f; } data; jit_get_reg_args(); + if (jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.f = i1; if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -881,6 +902,8 @@ _swf_idd_(jit_state_t *_jit, int (*i0)(double, double), jit_float64_t d; } data; jit_get_reg_args(); + if (jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.d = i1; if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) @@ -907,6 +930,8 @@ _swf_iunff(jit_state_t *_jit, int (*i0)(float, float), { jit_word_t instr; jit_get_reg_args(); + if (jit_fpr_p(r1) || jit_fpr_p(r2)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); else @@ -952,6 +977,8 @@ _swf_iundd(jit_state_t *_jit, int (*i0)(double, double), { jit_word_t instr; jit_get_reg_args(); + if (jit_fpr_p(r1) || jit_fpr_p(r2)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -1033,6 +1060,8 @@ _swf_iunff_(jit_state_t *_jit, int (*i0)(float, float), jit_float32_t f; } data; jit_get_reg_args(); + if (jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.f = i1; if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -1077,6 +1106,8 @@ _swf_iundd_(jit_state_t *_jit, int (*i0)(double, double), jit_float64_t d; } data; jit_get_reg_args(); + if (jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); data.d = i1; if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) @@ -1135,6 +1166,8 @@ _swf_bff(jit_state_t *_jit, int (*i0)(float, float), int cc, { jit_word_t w, d; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r0) + 8); else @@ -1168,6 +1201,8 @@ _swf_bdd(jit_state_t *_jit, int (*i0)(double, double), int cc, { jit_word_t w, d; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r0) + 8); @@ -1221,6 +1256,8 @@ _swf_bff_(jit_state_t *_jit, int (*i0)(float, float), int cc, } data; jit_word_t w, d; jit_get_reg_args(); + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); data.f = i2; if (jit_fpr_p(r0)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r0) + 8); @@ -1256,6 +1293,8 @@ _swf_bdd_(jit_state_t *_jit, int (*i0)(double, double), int cc, jit_float64_t d; } data; jit_get_reg_args(); + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); data.d = i2; if (jit_fpr_p(r0)) { if (!jit_thumb_p() && jit_armv5e_p()) @@ -1296,6 +1335,8 @@ _swf_bunff(jit_state_t *_jit, int eq, { jit_word_t w, d, j0, j1; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r0) + 8); else @@ -1366,6 +1407,8 @@ _swf_bundd(jit_state_t *_jit, int eq, { jit_word_t w, d, j0, j1; jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r0) + 8); @@ -1473,6 +1516,8 @@ _swf_bunff_(jit_state_t *_jit, int eq, jit_word_t w, d, j0, j1; data.f = i1; jit_get_reg_args(); + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r0) + 8); else @@ -1541,6 +1586,8 @@ _swf_bundd_(jit_state_t *_jit, int eq, jit_float64_t d; } data; jit_get_reg_args(); + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); data.d = i1; if (jit_fpr_p(r0)) { if (!jit_thumb_p() && jit_armv5e_p()) @@ -1622,6 +1669,8 @@ static void _swf_extr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_get_reg_args(); + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); movr(_R0_REGNO, r1); swf_call(__aeabi_i2f, i2f, _R1_REGNO); if (jit_fpr_p(r0)) @@ -1635,6 +1684,8 @@ static void _swf_extr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_get_reg_args(); + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); movr(_R0_REGNO, r1); swf_call(__aeabi_i2d, i2d, _R2_REGNO); if (jit_fpr_p(r0)) { @@ -1656,6 +1707,8 @@ static void _swf_extr_d_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -1680,6 +1733,8 @@ static void _swf_extr_f_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_get_reg_args(); + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); else @@ -1709,6 +1764,8 @@ _swf_truncr_f_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) jit_word_t slow_not_nan; #endif jit_get_reg_args(); + if (jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) swf_ldrin(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); else @@ -1763,6 +1820,8 @@ _swf_truncr_d_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) jit_word_t slow_not_nan; #endif jit_get_reg_args(); + if (jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p()) LDRDIN(_R0_REGNO, _FP_REGNO, swf_off(r1) + 8); @@ -1823,6 +1882,8 @@ _swf_movr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (r0 != r1) { + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r1) + 8); @@ -1844,6 +1905,8 @@ _swf_movr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (r0 != r1) { + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (!jit_thumb_p() && jit_armv5e_p() && (reg = jit_get_reg_pair()) != JIT_NOREG) { @@ -1894,6 +1957,8 @@ _swf_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0) jit_float32_t f; } data; jit_int32_t reg; + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); data.f = i0; if (jit_fpr_p(r0)) { reg = jit_get_reg(jit_class_gpr); @@ -1913,6 +1978,8 @@ _swf_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0) jit_int32_t i[2]; jit_float64_t d; } data; + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); data.d = i0; if (jit_fpr_p(r0)) { if (!jit_thumb_p() && jit_armv5e_p() && @@ -1941,6 +2008,8 @@ static void _swf_absr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r1) + 8); @@ -1966,6 +2035,8 @@ static void _swf_absr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (jit_fpr_p(r0) && !jit_thumb_p() && jit_armv5e_p() && r0 != r1 && (reg = jit_get_reg_pair()) != JIT_NOREG) { @@ -2013,6 +2084,8 @@ static void _swf_negr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r1) + 8); @@ -2038,6 +2111,8 @@ static void _swf_negr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; + if (jit_fpr_p(r0) || jit_fpr_p(r1)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r1)) { if (jit_fpr_p(r0) && !jit_thumb_p() && jit_armv5e_p() && r0 != r1 && (reg = jit_get_reg_pair()) != JIT_NOREG) { @@ -2170,6 +2245,7 @@ _swf_ldr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); ldxi_i(rn(reg), r1, 0); swf_strin(rn(reg), _FP_REGNO, swf_off(r0) + 8); @@ -2184,6 +2260,7 @@ _swf_ldr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && (reg = jit_get_reg_pair()) != JIT_NOREG) { LDRDI(rn(reg), r1, 0); @@ -2212,6 +2289,7 @@ _swf_ldi_f(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); ldi_i(rn(reg), i0); swf_strin(rn(reg), _FP_REGNO, swf_off(r0) + 8); @@ -2225,6 +2303,8 @@ static void _swf_ldi_d(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t rg0, rg1; + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0) && !jit_thumb_p() && jit_armv5e_p() && (rg0 = jit_get_reg_pair()) != JIT_NOREG) { movi(rn(rg0), i0); @@ -2258,6 +2338,7 @@ _swf_ldxr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_int32_t reg; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); ldxr_i(rn(reg), r1, r2); swf_strin(rn(reg), _FP_REGNO, swf_off(r0) + 8); @@ -2272,6 +2353,7 @@ _swf_ldxr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_int32_t rg0, rg1; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && (rg0 = jit_get_reg_pair()) != JIT_NOREG) { LDRD(rn(rg0), r1, r2); @@ -2307,6 +2389,8 @@ static void _swf_ldxi_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; + if (jit_fpr_p(r0)) + CHECK_SWF_OFFSET(); if (jit_fpr_p(r0)) { reg = jit_get_reg(jit_class_gpr); ldxi_i(rn(reg), r1, i0); @@ -2322,6 +2406,7 @@ _swf_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t rg0, rg1; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && ((i0 >= 0 && i0 <= 255) || (i0 < 0 && i0 >= -255)) && (rg0 = jit_get_reg_pair()) != JIT_NOREG) { @@ -2391,6 +2476,7 @@ _swf_str_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (jit_fpr_p(r1)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r1) + 8); stxi_i(0, r0, rn(reg)); @@ -2405,6 +2491,7 @@ _swf_str_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (jit_fpr_p(r1)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && (reg = jit_get_reg_pair()) != JIT_NOREG) { LDRDIN(rn(reg), _FP_REGNO, swf_off(r1) + 8); @@ -2435,6 +2522,7 @@ _swf_sti_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r0) + 8); sti_i(i0, rn(reg)); @@ -2449,6 +2537,7 @@ _swf_sti_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t rg0, rg1; if (jit_fpr_p(r0)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && (rg0 = jit_get_reg_pair()) != JIT_NOREG) { rg1 = jit_get_reg(jit_class_gpr); @@ -2488,6 +2577,7 @@ _swf_stxr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_int32_t reg; if (jit_fpr_p(r2)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r2) + 8); stxr_i(r1, r0, rn(reg)); @@ -2502,6 +2592,7 @@ _swf_stxr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { jit_int32_t rg0, rg1; if (jit_fpr_p(r2)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && (rg0 = jit_get_reg_pair()) != JIT_NOREG) { LDRDIN(rn(rg0), _FP_REGNO, swf_off(r2) + 8); @@ -2538,6 +2629,7 @@ _swf_stxi_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_int32_t reg; if (jit_fpr_p(r1)) { + CHECK_SWF_OFFSET(); reg = jit_get_reg(jit_class_gpr); swf_ldrin(rn(reg), _FP_REGNO, swf_off(r1) + 8); stxi_i(i0, r0, rn(reg)); @@ -2552,6 +2644,7 @@ _swf_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_int32_t rg0, rg1; if (jit_fpr_p(r1)) { + CHECK_SWF_OFFSET(); if (!jit_thumb_p() && jit_armv5e_p() && ((i0 >= 0 && i0 <= 255) || (i0 < 0 && i0 >= -255)) && (rg0 = jit_get_reg_pair()) != JIT_NOREG) { diff --git a/parallel-rsp/lightning/lib/jit_arm-sz.c b/parallel-rsp/lightning/lib/jit_arm-sz.c index 9f0d012..faba5a8 100644 --- a/parallel-rsp/lightning/lib/jit_arm-sz.c +++ b/parallel-rsp/lightning/lib/jit_arm-sz.c @@ -1,12 +1,13 @@ #if __WORDSIZE == 32 #if defined(__ARM_PCS_VFP) -#define JIT_INSTR_MAX 48 +#define JIT_INSTR_MAX 50 0, /* data */ 0, /* live */ - 2, /* align */ + 14, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 2, /* #name */ 0, /* #note */ 0, /* label */ @@ -15,7 +16,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -23,8 +27,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 16, /* va_arg_d */ @@ -48,18 +64,18 @@ 12, /* qmuli */ 4, /* qmulr_u */ 8, /* qmuli_u */ - 40, /* divr */ - 48, /* divi */ - 40, /* divr_u */ - 44, /* divi_u */ - 34, /* qdivr */ - 38, /* qdivi */ - 34, /* qdivr_u */ - 38, /* qdivi_u */ - 40, /* remr */ - 48, /* remi */ - 40, /* remr_u */ - 44, /* remi_u */ + 32, /* divr */ + 36, /* divi */ + 24, /* divr_u */ + 28, /* divi_u */ + 18, /* qdivr */ + 22, /* qdivi */ + 18, /* qdivr_u */ + 22, /* qdivi_u */ + 24, /* remr */ + 32, /* remi */ + 24, /* remr_u */ + 28, /* remi_u */ 4, /* andr */ 12, /* andi */ 4, /* orr */ @@ -96,12 +112,19 @@ 14, /* nei */ 4, /* movr */ 8, /* movi */ + 8, /* movnr */ + 8, /* movzr */ + 42, /* casr */ + 50, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 8, /* bswapr_us */ + 4, /* bswapr_ui */ + 0, /* bswapr_ul */ 8, /* htonr_us */ 4, /* htonr_ui */ 0, /* htonr_ul */ @@ -194,13 +217,37 @@ 4, /* callr */ 20, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -349,7 +396,7 @@ 8, /* extr_d */ 4, /* extr_f_d */ 4, /* movr_d */ - 16, /* movi_d */ + 32, /* movi_d */ 4, /* ldr_d */ 12, /* ldi_d */ 8, /* ldxr_d */ @@ -363,7 +410,7 @@ 12, /* bler_d */ 28, /* blei_d */ 12, /* beqr_d */ - 28, /* beqi_d */ + 36, /* beqi_d */ 12, /* bger_d */ 28, /* bgei_d */ 12, /* bgtr_d */ @@ -400,17 +447,22 @@ 12, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 12, /* cto */ + 8, /* ctz */ #endif /* __ARM_PCS_VFP */ #endif /* __WORDSIZE */ #if __WORDSIZE == 32 #if !defined(__ARM_PCS_VFP) -#define JIT_INSTR_MAX 160 +#define JIT_INSTR_MAX 50 0, /* data */ 0, /* live */ - 2, /* align */ + 18, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 2, /* #name */ 0, /* #note */ 0, /* label */ @@ -419,7 +471,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -427,8 +482,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 28, /* va_arg_d */ @@ -452,18 +519,18 @@ 12, /* qmuli */ 4, /* qmulr_u */ 8, /* qmuli_u */ - 40, /* divr */ - 48, /* divi */ - 40, /* divr_u */ - 44, /* divi_u */ - 34, /* qdivr */ - 38, /* qdivi */ - 34, /* qdivr_u */ - 38, /* qdivi_u */ - 40, /* remr */ - 48, /* remi */ - 40, /* remr_u */ - 44, /* remi_u */ + 32, /* divr */ + 36, /* divi */ + 24, /* divr_u */ + 28, /* divi_u */ + 18, /* qdivr */ + 22, /* qdivi */ + 18, /* qdivr_u */ + 22, /* qdivi_u */ + 24, /* remr */ + 32, /* remi */ + 24, /* remr_u */ + 28, /* remi_u */ 4, /* andr */ 12, /* andi */ 4, /* orr */ @@ -500,12 +567,19 @@ 14, /* nei */ 4, /* movr */ 8, /* movi */ + 8, /* movnr */ + 8, /* movzr */ + 42, /* casr */ + 46, /* casi */ 8, /* extr_c */ 4, /* extr_uc */ 8, /* extr_s */ 8, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 20, /* bswapr_us */ + 16, /* bswapr_ui */ + 0, /* bswapr_ul */ 20, /* htonr_us */ 16, /* htonr_ui */ 0, /* htonr_ul */ @@ -598,13 +672,37 @@ 4, /* callr */ 20, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -612,57 +710,57 @@ 0, /* retval_i */ 0, /* retval_ui */ 0, /* retval_l */ - 160, /* epilog */ + 30, /* epilog */ 0, /* arg_f */ 0, /* getarg_f */ 0, /* putargr_f */ 0, /* putargi_f */ - 40, /* addr_f */ - 40, /* addi_f */ - 40, /* subr_f */ - 40, /* subi_f */ - 40, /* rsbi_f */ - 40, /* mulr_f */ - 40, /* muli_f */ - 40, /* divr_f */ - 40, /* divi_f */ + 24, /* addr_f */ + 24, /* addi_f */ + 24, /* subr_f */ + 24, /* subi_f */ + 24, /* rsbi_f */ + 24, /* mulr_f */ + 24, /* muli_f */ + 24, /* divr_f */ + 24, /* divi_f */ 12, /* negr_f */ 12, /* absr_f */ - 36, /* sqrtr_f */ - 40, /* ltr_f */ - 44, /* lti_f */ - 40, /* ler_f */ - 44, /* lei_f */ - 40, /* eqr_f */ - 44, /* eqi_f */ - 40, /* ger_f */ - 44, /* gei_f */ - 40, /* gtr_f */ - 44, /* gti_f */ - 44, /* ner_f */ - 48, /* nei_f */ - 72, /* unltr_f */ - 80, /* unlti_f */ - 72, /* unler_f */ - 80, /* unlei_f */ - 72, /* uneqr_f */ - 80, /* uneqi_f */ - 72, /* unger_f */ - 80, /* ungei_f */ - 72, /* ungtr_f */ - 80, /* ungti_f */ - 76, /* ltgtr_f */ - 84, /* ltgti_f */ - 44, /* ordr_f */ - 48, /* ordi_f */ - 72, /* unordr_f */ - 80, /* unordi_f */ - 36, /* truncr_f_i */ + 20, /* sqrtr_f */ + 24, /* ltr_f */ + 30, /* lti_f */ + 24, /* ler_f */ + 32, /* lei_f */ + 24, /* eqr_f */ + 30, /* eqi_f */ + 24, /* ger_f */ + 30, /* gei_f */ + 24, /* gtr_f */ + 30, /* gti_f */ + 28, /* ner_f */ + 32, /* nei_f */ + 56, /* unltr_f */ + 64, /* unlti_f */ + 56, /* unler_f */ + 64, /* unlei_f */ + 56, /* uneqr_f */ + 64, /* uneqi_f */ + 56, /* unger_f */ + 64, /* ungei_f */ + 56, /* ungtr_f */ + 64, /* ungti_f */ + 60, /* ltgtr_f */ + 68, /* ltgti_f */ + 28, /* ordr_f */ + 32, /* ordi_f */ + 56, /* unordr_f */ + 64, /* unordi_f */ + 20, /* truncr_f_i */ 0, /* truncr_f_l */ - 36, /* extr_f */ - 38, /* extr_d_f */ + 28, /* extr_f */ + 22, /* extr_d_f */ 8, /* movr_f */ - 12, /* movi_f */ + 16, /* movi_f */ 8, /* ldr_f */ 16, /* ldi_f */ 8, /* ldxr_f */ @@ -671,34 +769,34 @@ 16, /* sti_f */ 8, /* stxr_f */ 16, /* stxi_f */ - 44, /* bltr_f */ - 48, /* blti_f */ - 44, /* bler_f */ - 48, /* blei_f */ - 44, /* beqr_f */ - 52, /* beqi_f */ - 44, /* bger_f */ - 48, /* bgei_f */ - 44, /* bgtr_f */ - 48, /* bgti_f */ - 44, /* bner_f */ - 48, /* bnei_f */ - 44, /* bunltr_f */ - 48, /* bunlti_f */ - 44, /* bunler_f */ - 48, /* bunlei_f */ - 76, /* buneqr_f */ - 84, /* buneqi_f */ - 44, /* bunger_f */ - 48, /* bungei_f */ - 44, /* bungtr_f */ - 48, /* bungti_f */ - 76, /* bltgtr_f */ - 84, /* bltgti_f */ - 44, /* bordr_f */ - 48, /* bordi_f */ - 44, /* bunordr_f */ - 48, /* bunordi_f */ + 28, /* bltr_f */ + 32, /* blti_f */ + 28, /* bler_f */ + 32, /* blei_f */ + 28, /* beqr_f */ + 48, /* beqi_f */ + 28, /* bger_f */ + 32, /* bgei_f */ + 28, /* bgtr_f */ + 32, /* bgti_f */ + 28, /* bner_f */ + 32, /* bnei_f */ + 28, /* bunltr_f */ + 32, /* bunlti_f */ + 28, /* bunler_f */ + 32, /* bunlei_f */ + 60, /* buneqr_f */ + 68, /* buneqi_f */ + 28, /* bunger_f */ + 32, /* bungei_f */ + 28, /* bungtr_f */ + 32, /* bungti_f */ + 60, /* bltgtr_f */ + 68, /* bltgti_f */ + 28, /* bordr_f */ + 32, /* bordi_f */ + 28, /* bunordr_f */ + 32, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -708,52 +806,52 @@ 0, /* getarg_d */ 0, /* putargr_d */ 0, /* putargi_d */ - 50, /* addr_d */ - 52, /* addi_d */ - 50, /* subr_d */ - 52, /* subi_d */ - 52, /* rsbi_d */ - 50, /* mulr_d */ - 52, /* muli_d */ - 50, /* divr_d */ - 52, /* divi_d */ + 34, /* addr_d */ + 36, /* addi_d */ + 34, /* subr_d */ + 36, /* subi_d */ + 36, /* rsbi_d */ + 34, /* mulr_d */ + 36, /* muli_d */ + 34, /* divr_d */ + 36, /* divi_d */ 20, /* negr_d */ 20, /* absr_d */ - 42, /* sqrtr_d */ - 44, /* ltr_d */ - 48, /* lti_d */ - 44, /* ler_d */ - 48, /* lei_d */ - 44, /* eqr_d */ - 48, /* eqi_d */ - 44, /* ger_d */ - 48, /* gei_d */ - 44, /* gtr_d */ - 48, /* gti_d */ - 48, /* ner_d */ - 52, /* nei_d */ - 82, /* unltr_d */ - 88, /* unlti_d */ - 82, /* unler_d */ - 88, /* unlei_d */ - 82, /* uneqr_d */ - 88, /* uneqi_d */ - 82, /* unger_d */ - 88, /* ungei_d */ - 82, /* ungtr_d */ - 88, /* ungti_d */ - 86, /* ltgtr_d */ - 92, /* ltgti_d */ - 48, /* ordr_d */ - 52, /* ordi_d */ - 82, /* unordr_d */ - 88, /* unordi_d */ - 36, /* truncr_d_i */ + 26, /* sqrtr_d */ + 28, /* ltr_d */ + 34, /* lti_d */ + 28, /* ler_d */ + 36, /* lei_d */ + 28, /* eqr_d */ + 34, /* eqi_d */ + 28, /* ger_d */ + 34, /* gei_d */ + 28, /* gtr_d */ + 34, /* gti_d */ + 32, /* ner_d */ + 36, /* nei_d */ + 66, /* unltr_d */ + 72, /* unlti_d */ + 66, /* unler_d */ + 72, /* unlei_d */ + 66, /* uneqr_d */ + 72, /* uneqi_d */ + 66, /* unger_d */ + 72, /* ungei_d */ + 66, /* ungtr_d */ + 72, /* ungti_d */ + 70, /* ltgtr_d */ + 76, /* ltgti_d */ + 32, /* ordr_d */ + 36, /* ordi_d */ + 66, /* unordr_d */ + 72, /* unordi_d */ + 20, /* truncr_d_i */ 0, /* truncr_d_l */ 36, /* extr_d */ - 38, /* extr_f_d */ + 22, /* extr_f_d */ 16, /* movr_d */ - 20, /* movi_d */ + 32, /* movi_d */ 16, /* ldr_d */ 24, /* ldi_d */ 20, /* ldxr_d */ @@ -762,34 +860,34 @@ 24, /* sti_d */ 20, /* stxr_d */ 28, /* stxi_d */ - 48, /* bltr_d */ - 52, /* blti_d */ - 48, /* bler_d */ - 52, /* blei_d */ - 48, /* beqr_d */ - 60, /* beqi_d */ - 48, /* bger_d */ - 52, /* bgei_d */ - 48, /* bgtr_d */ - 52, /* bgti_d */ - 48, /* bner_d */ - 52, /* bnei_d */ - 48, /* bunltr_d */ - 52, /* bunlti_d */ - 48, /* bunler_d */ - 52, /* bunlei_d */ - 84, /* buneqr_d */ - 92, /* buneqi_d */ - 48, /* bunger_d */ - 52, /* bungei_d */ - 48, /* bungtr_d */ - 52, /* bungti_d */ - 84, /* bltgtr_d */ - 92, /* bltgti_d */ - 48, /* bordr_d */ - 52, /* bordi_d */ - 48, /* bunordr_d */ - 52, /* bunordi_d */ + 32, /* bltr_d */ + 36, /* blti_d */ + 32, /* bler_d */ + 36, /* blei_d */ + 32, /* beqr_d */ + 52, /* beqi_d */ + 32, /* bger_d */ + 36, /* bgei_d */ + 32, /* bgtr_d */ + 36, /* bgti_d */ + 32, /* bner_d */ + 36, /* bnei_d */ + 32, /* bunltr_d */ + 36, /* bunlti_d */ + 32, /* bunler_d */ + 36, /* bunlei_d */ + 68, /* buneqr_d */ + 76, /* buneqi_d */ + 32, /* bunger_d */ + 36, /* bungei_d */ + 32, /* bungtr_d */ + 36, /* bungti_d */ + 68, /* bltgtr_d */ + 76, /* bltgti_d */ + 32, /* bordr_d */ + 36, /* bordi_d */ + 32, /* bunordr_d */ + 36, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -804,5 +902,9 @@ 12, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 12, /* cto */ + 8, /* ctz */ #endif /* __ARM_PCS_VFP */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_arm-vfp.c b/parallel-rsp/lightning/lib/jit_arm-vfp.c index 743a3ef..20f80a2 100644 --- a/parallel-rsp/lightning/lib/jit_arm-vfp.c +++ b/parallel-rsp/lightning/lib/jit_arm-vfp.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -1255,7 +1255,7 @@ _vfp_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0) if (jit_fpr_p(r0)) { /* float arguments are packed, for others, * lightning only address even registers */ - if (!(r0 & 1) && (r0 - 16) >= 0 && + if (!(r0 & 1) && (r0 - 32) >= 0 && ((code = encode_vfp_double(1, 0, u.i, u.i)) != -1 || (code = encode_vfp_double(1, 1, ~u.i, ~u.i)) != -1)) VIMM(code, r0); diff --git a/parallel-rsp/lightning/lib/jit_arm.c b/parallel-rsp/lightning/lib/jit_arm.c index 783fa90..64a70f9 100644 --- a/parallel-rsp/lightning/lib/jit_arm.c +++ b/parallel-rsp/lightning/lib/jit_arm.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -21,6 +21,8 @@ # include #endif +#define stack_framesize 48 + #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4) #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 16) #define jit_arg_d_reg_p(i) ((i) >= 0 && (i) < 15) @@ -28,12 +30,12 @@ #define arm_patch_node 0x80000000 #define arm_patch_word 0x40000000 #define arm_patch_jump 0x20000000 -#define arm_patch_load 0x00000000 +#define arm_patch_load 0x10000000 +#define arm_patch_call 0x08000000 #define jit_fpr_p(rn) ((rn) > 15) -#define arg_base() \ - (stack_framesize - 16 + (jit_cpu.abi ? 64 : 0)) +#define arg_base() (stack_framesize - 16) #define arg_offset(n) \ ((n) < 4 ? arg_base() + ((n) << 2) : (n)) @@ -42,10 +44,32 @@ * arm mode, what may cause a crash upon return of that function * if generating jit for a relative jump. */ -#define jit_exchange_p() 1 +#define jit_exchange_p() jit_cpu.exchange /* FIXME is it really required to not touch _R10? */ +#define CHECK_REG_ARGS() \ + do { \ + if (!_jitc->function->save_reg_args) \ + _jitc->again = _jitc->function->save_reg_args = 1; \ + } while (0) + +#define CHECK_SWF_OFFSET() \ + do { \ + if (!_jitc->function->swf_offset) { \ + _jitc->again = _jitc->function->save_reg_args = \ + _jitc->function->swf_offset = 1; \ + _jitc->function->self.aoff = -64; \ + } \ + } while (0) + +#define CHECK_RETURN() \ + do { \ + if (!_jitc->function->need_frame && \ + !_jitc->function->need_return) \ + _jitc->again = _jitc->function->need_return = 1; \ + } while (0) + /* * Types */ @@ -59,8 +83,8 @@ typedef jit_pointer_t jit_va_list; /* * Prototypes */ -#define jit_make_arg(node) _jit_make_arg(_jit,node) -static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*); +#define jit_make_arg(node,code) _jit_make_arg(_jit,node,code) +static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*,jit_code_t); #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node) static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*); #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node) @@ -77,8 +101,10 @@ static void _load_const(jit_state_t*,jit_bool_t,jit_int32_t,jit_word_t); static void _flush_consts(jit_state_t*); #define invalidate_consts() _invalidate_consts(_jit) static void _invalidate_consts(jit_state_t*); -#define patch(instr, node) _patch(_jit, instr, node) -static void _patch(jit_state_t*,jit_word_t,jit_node_t*); +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); +#define patch(instr, node, kind) _patch(_jit, instr, node, kind) +static void _patch(jit_state_t*,jit_word_t,jit_node_t*,jit_int32_t); #if defined(__GNUC__) /* libgcc */ @@ -90,6 +116,7 @@ extern void __clear_cache(void *, void *); # include "jit_arm-cpu.c" # include "jit_arm-swf.c" # include "jit_arm-vfp.c" +# include "jit_fallback.c" #undef PROTO /* @@ -148,6 +175,10 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { + _R4, _R5, _R6, _R7, _R8, _R9, +}; + /* * Implementation */ @@ -201,6 +232,14 @@ jit_get_cpu(void) /* armv6t2 todo (software float and thumb2) */ if (!jit_cpu.vfp && jit_cpu.thumb) jit_cpu.thumb = 0; + /* FIXME need test environments for the below. For the moment just + * be very conservative */ + /* force generation of code assuming jit and function libraries called + * instruction set do not match */ + jit_cpu.exchange = 1; + /* do not generate hardware integer division by default */ + if (jit_cpu.version == 7) + jit_cpu.extend = 0; } void @@ -244,15 +283,10 @@ _jit_prolog(jit_state_t *_jit) } _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++; _jitc->function->self.size = stack_framesize; - if (jit_cpu.abi) - _jitc->function->self.size += 64; _jitc->function->self.argi = _jitc->function->self.argf = - _jitc->function->self.alen = 0; - if (jit_swf_p()) - /* 8 soft float registers */ - _jitc->function->self.aoff = -64; - else - _jitc->function->self.aoff = 0; + _jitc->function->self.alen = _jitc->function->self.aoff = 0; + _jitc->function->swf_offset = _jitc->function->save_reg_args = + _jitc->function->need_return = 0; _jitc->function->self.call = jit_call_default; jit_alloc((jit_pointer_t *)&_jitc->function->regoff, _jitc->reglen * sizeof(jit_int32_t)); @@ -278,6 +312,9 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + if (jit_swf_p()) + CHECK_SWF_OFFSET(); + jit_check_frame(); switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -326,20 +363,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -421,7 +456,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code != jit_code_arg) { + if (!(u->code >= jit_code_arg_c && u->code <= jit_code_arg)) { if (u->code == jit_code_arg_f) { if (jit_cpu.abi) return (jit_arg_f_reg_p(u->u.w)); @@ -436,7 +471,7 @@ _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) } static jit_node_t * -_jit_make_arg(jit_state_t *_jit, jit_node_t *node) +_jit_make_arg(jit_state_t *_jit, jit_node_t *node, jit_code_t code) { jit_int32_t offset; if (jit_arg_reg_p(_jitc->function->self.argi)) @@ -446,7 +481,7 @@ _jit_make_arg(jit_state_t *_jit, jit_node_t *node) _jitc->function->self.size += sizeof(jit_word_t); } if (node == (jit_node_t *)0) - node = jit_new_node(jit_code_arg); + node = jit_new_node(code); else link_node(node); node->u.w = offset; @@ -533,9 +568,10 @@ _jit_ellipsis(jit_state_t *_jit) else { assert(!(_jitc->function->self.call & jit_call_varargs)); _jitc->function->self.call |= jit_call_varargs; + CHECK_REG_ARGS(); if (jit_cpu.abi && _jitc->function->self.argf) rewind_prolog(); - /* First 4 stack addresses are always spilled r0-r3 */ + /* First 4 stack addresses need to be spilled r0-r3 */ if (jit_arg_reg_p(_jitc->function->self.argi)) _jitc->function->vagp = _jitc->function->self.argi * 4; else @@ -558,16 +594,21 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { assert(_jitc->function); - return (jit_make_arg((jit_node_t*)0)); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif + return (jit_make_arg((jit_node_t*)0, code)); } jit_node_t * _jit_arg_f(jit_state_t *_jit) { assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); return (jit_make_arg_f((jit_node_t*)0)); } @@ -575,103 +616,141 @@ jit_node_t * _jit_arg_d(jit_state_t *_jit) { assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); return (jit_make_arg_d((jit_node_t*)0)); } void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_swf_p()) - jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, JIT_RA0 - v->u.w); else - jit_ldxi_c(u, JIT_FP, v->u.w); + node = jit_ldxi_c(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_swf_p()) - jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, JIT_RA0 - v->u.w); else - jit_ldxi_uc(u, JIT_FP, v->u.w); + node = jit_ldxi_uc(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_swf_p()) - jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, JIT_RA0 - v->u.w); else - jit_ldxi_s(u, JIT_FP, v->u.w); + node = jit_ldxi_s(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_swf_p()) - jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, JIT_RA0 - v->u.w); else - jit_ldxi_us(u, JIT_FP, v->u.w); + node = jit_ldxi_us(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_swf_p()) - jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); else - jit_ldxi_i(u, JIT_FP, v->u.w); + node = jit_ldxi_i(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + jit_node_t *node = NULL; + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_swf_p()) - jit_stxi(arg_offset(v->u.w), JIT_FP, u); + node = jit_stxi(arg_offset(v->u.w), JIT_FP, u); else if (jit_arg_reg_p(v->u.w)) jit_movr(JIT_RA0 - v->u.w, u); else - jit_stxi(v->u.w, JIT_FP, u); + node = jit_stxi(v->u.w, JIT_FP, u); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { - jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + jit_int32_t regno; + jit_node_t *node = NULL; + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_swf_p()) { regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(arg_offset(v->u.w), JIT_FP, regno); + node = jit_stxi(arg_offset(v->u.w), JIT_FP, regno); jit_unget_reg(regno); } else if (jit_arg_reg_p(v->u.w)) @@ -679,30 +758,41 @@ _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) else { regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w, JIT_FP, regno); + node = jit_stxi(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_f); jit_inc_synth_wp(getarg_f, u, v); if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(u, JIT_FA0 - v->u.w); else - jit_ldxi_f(u, JIT_FP, v->u.w); + node = jit_ldxi_f(u, JIT_FP, v->u.w); } else if (jit_swf_p()) - jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w)); else { if (jit_arg_reg_p(v->u.w)) jit_movr_w_f(u, JIT_RA0 - v->u.w); else - jit_ldxi_f(u, JIT_FP, v->u.w); + node = jit_ldxi_f(u, JIT_FP, v->u.w); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -710,21 +800,27 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_f); jit_inc_synth_wp(putargr_f, u, v); if (jit_cpu.abi) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(JIT_FA0 - v->u.w, u); else - jit_stxi_f(v->u.w, JIT_FP, u); + node = jit_stxi_f(v->u.w, JIT_FP, u); } else if (jit_swf_p()) - jit_stxi_f(arg_offset(v->u.w), JIT_FP, u); + node = jit_stxi_f(arg_offset(v->u.w), JIT_FP, u); else { if (jit_arg_reg_p(v->u.w)) jit_movr_f_w(JIT_RA0 - v->u.w, u); else - jit_stxi_f(v->u.w, JIT_FP, u); + node = jit_stxi_f(v->u.w, JIT_FP, u); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -732,7 +828,8 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) { - jit_int32_t regno; + jit_int32_t regno; + jit_node_t *node = NULL; assert(v->code == jit_code_arg_f); jit_inc_synth_fp(putargi_f, u, v); if (jit_cpu.abi) { @@ -741,14 +838,14 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) else { regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } } else if (jit_swf_p()) { regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno); + node = jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno); jit_unget_reg(regno); } else { @@ -757,30 +854,41 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_f_w(JIT_RA0 - v->u.w, regno); else - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_d); jit_inc_synth_wp(getarg_d, u, v); if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(u, JIT_FA0 - v->u.w); else - jit_ldxi_d(u, JIT_FP, v->u.w); + node = jit_ldxi_d(u, JIT_FP, v->u.w); } else if (jit_swf_p()) - jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w)); else { if (jit_arg_reg_p(v->u.w)) jit_movr_ww_d(u, JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1)); else - jit_ldxi_d(u, JIT_FP, v->u.w); + node = jit_ldxi_d(u, JIT_FP, v->u.w); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -788,21 +896,27 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_d); jit_inc_synth_wp(putargr_d, u, v); if (jit_cpu.abi) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(JIT_FA0 - v->u.w, u); else - jit_stxi_d(v->u.w, JIT_FP, u); + node = jit_stxi_d(v->u.w, JIT_FP, u); } else if (jit_swf_p()) - jit_stxi_d(arg_offset(v->u.w), JIT_FP, u); + node = jit_stxi_d(arg_offset(v->u.w), JIT_FP, u); else { if (jit_arg_reg_p(v->u.w)) jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), u); else - jit_stxi_d(v->u.w, JIT_FP, u); + node = jit_stxi_d(v->u.w, JIT_FP, u); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -810,7 +924,8 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) { - jit_int32_t regno; + jit_int32_t regno; + jit_node_t *node = NULL; assert(v->code == jit_code_arg_d); jit_inc_synth_dp(putargi_d, u, v); if (jit_cpu.abi) { @@ -819,14 +934,14 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) else { regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } } else if (jit_swf_p()) { regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno); + node = jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno); jit_unget_reg(regno); } else { @@ -835,17 +950,22 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), regno); else - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(JIT_RA0 - _jitc->function->call.argi, u); @@ -859,11 +979,11 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(JIT_RA0 - _jitc->function->call.argi, u); @@ -1147,6 +1267,7 @@ _emit_code(jit_state_t *_jit) jit_node_t *node; jit_uint8_t *data; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1292,7 +1413,7 @@ _emit_code(jit_state_t *_jit) else { \ word = name##r##type(_jit->pc.w, \ rn(node->v.w), rn(node->w.w)); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #define case_bvv(name, type) \ @@ -1317,7 +1438,7 @@ _emit_code(jit_state_t *_jit) word = vfp_##name##r##type(_jit->pc.w, \ rn(node->v.w), \ rn(node->w.w)); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #define case_brw(name, type) \ @@ -1331,7 +1452,7 @@ _emit_code(jit_state_t *_jit) else { \ word = name##i##type(_jit->pc.w, \ rn(node->v.w), node->w.w); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break; #define case_bvf(name) \ @@ -1356,7 +1477,7 @@ _emit_code(jit_state_t *_jit) word = vfp_##name##i_f(_jit->pc.w, \ rn(node->v.w), \ node->w.f); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #define case_bvd(name) \ @@ -1381,7 +1502,7 @@ _emit_code(jit_state_t *_jit) word = vfp_##name##i_d(_jit->pc.w, \ rn(node->v.w), \ node->w.d); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #if DEVEL_DISASSEMBLER @@ -1399,11 +1520,16 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + if (jit_thumb_p()) + nop((node->u.w + 1) & ~1); + else + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: if (must_align_p(node->next)) @@ -1456,6 +1582,10 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh, _u); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(and,); case_rrw(and,); case_rrr(or,); @@ -1498,11 +1628,23 @@ _emit_code(jit_state_t *_jit) case_wrr(stx, _i); case_rr(hton, _us); case_rr(hton, _ui); + case_rr(bswap, _us); + case_rr(bswap, _ui); case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); case_rr(ext, _us); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; case_rr(mov,); + case_rrr(movn,); + case_rrr(movz,); case jit_code_movi: if (node->flag & jit_flag_node) { temp = node->v.n; @@ -1514,7 +1656,7 @@ _emit_code(jit_state_t *_jit) assert(temp->code == jit_code_label || temp->code == jit_code_epilog); word = movi_p(rn(node->u.w), temp->u.w); - patch(word, node); + patch(word, node, arm_patch_word); } } else @@ -1753,6 +1895,7 @@ _emit_code(jit_state_t *_jit) case_bvv(bunord, _d); case_bvd(bunord); case jit_code_jmpr: + jit_check_frame(); jmpr(rn(node->u.w)); flush_consts(); break; @@ -1764,36 +1907,59 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w, 1); - patch(word, node); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (jit_thumb_p()) word >>= 1; + else word >>= 2; + word -= 2; + value = _s24P(word); + word = jmpi_p(_jit->pc.w, value); + patch(word, node, value ? + arm_patch_jump : arm_patch_word); } } - else + else { + jit_check_frame(); jmpi(node->u.w); + } flush_consts(); break; case jit_code_callr: + jit_check_frame(); callr(rn(node->u.w)); break; case jit_code_calli: if (node->flag & jit_flag_node) { + CHECK_RETURN(); temp = node->u.n; assert(temp->code == jit_code_label || temp->code == jit_code_epilog); if (temp->flag & jit_flag_patch) - calli(temp->u.w); + calli(temp->u.w, 0); else { - word = calli_p(_jit->pc.w); - patch(word, node); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (jit_exchange_p()) + word -= 8; + if (jit_thumb_p()) word >>= 1; + else word >>= 2; + word -= 2; + value = _s24P(word); + word = calli_p(_jit->pc.w, value); + patch(word, node, value ? + arm_patch_call : arm_patch_word); } } - else - calli(node->u.w); + else { + jit_check_frame(); + calli(node->u.w, jit_exchange_p()); + } break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1807,6 +1973,8 @@ _emit_code(jit_state_t *_jit) #endif restart_function: _jitc->again = 0; + compute_framesize(); + patch_alist(0); prolog(node); break; case jit_code_epilog: @@ -1821,6 +1989,21 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + undo.func.need_return = _jitc->function->need_return; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + /* swf_offset and check_reg_args must also not be undone */ + undo.func.swf_offset = _jitc->function->swf_offset; + undo.func.save_reg_args = _jitc->function->save_reg_args; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1833,6 +2016,7 @@ _emit_code(jit_state_t *_jit) if (_jitc->data_info.ptr) _jitc->data_info.offset = undo.info_offset; #endif + patch_alist(1); goto restart_function; } /* remember label is defined */ @@ -1895,21 +2079,34 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: case jit_code_getarg_s: case jit_code_getarg_us: case jit_code_getarg_i: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1972,7 +2169,10 @@ _emit_code(jit_state_t *_jit) node = _jitc->patches.ptr[offset].node; word = _jitc->patches.ptr[offset].inst; if (!jit_thumb_p() && - (node->code == jit_code_movi || node->code == jit_code_calli)) { + (node->code == jit_code_movi || + (node->code == jit_code_calli && + (_jitc->patches.ptr[offset].kind & ~arm_patch_node) == + arm_patch_word))) { /* calculate where to patch word */ value = *(jit_int32_t *)word; assert((value & 0x0f700000) == ARM_LDRI); @@ -1999,6 +2199,7 @@ _emit_code(jit_state_t *_jit) # include "jit_arm-cpu.c" # include "jit_arm-swf.c" # include "jit_arm-vfp.c" +# include "jit_fallback.c" #undef CODE void @@ -2241,24 +2442,31 @@ _invalidate_consts(jit_state_t *_jit) } static void -_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = sizeof(jit_word_t) * 2; /* lr+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += sizeof(jit_word_t); + + if (_jitc->function->save_reg_args) + _jitc->framesize += 16; + + /* Make sure functions called have a 8 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 7) & -8; +} + +static void +_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node, jit_int32_t kind) { jit_int32_t flag; - jit_int32_t kind; assert(node->flag & jit_flag_node); - if (node->code == jit_code_movi) { + if (node->code == jit_code_movi) flag = node->v.n->flag; - kind = arm_patch_word; - } - else { + else flag = node->u.n->flag; - if (node->code == jit_code_calli || - (node->code == jit_code_jmpi && !(node->flag & jit_flag_node))) - kind = arm_patch_word; - else - kind = arm_patch_jump; - } assert(!(flag & jit_flag_patch)); kind |= arm_patch_node; if (_jitc->patches.offset >= _jitc->patches.length) { diff --git a/parallel-rsp/lightning/lib/jit_disasm.c b/parallel-rsp/lightning/lib/jit_disasm.c index 15b91b9..456b4f6 100644 --- a/parallel-rsp/lightning/lib/jit_disasm.c +++ b/parallel-rsp/lightning/lib/jit_disasm.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -50,7 +50,21 @@ static asymbol *disasm_synthetic; static long disasm_num_symbols; static long disasm_num_synthetic; static jit_state_t *disasm_jit; -#define disasm_stream stdout +static FILE *disasm_stream; +#endif + +#if BINUTILS_2_38 +static int fprintf_styled(void * stream, enum disassembler_style style, const char* fmt, ...) +{ + va_list args; + int r; + + va_start(args, fmt); + r = vfprintf(disasm_stream, fmt, args); + va_end(args); + + return r; +} #endif /* @@ -59,6 +73,7 @@ static jit_state_t *disasm_jit; void jit_init_debug(const char *progname) { + jit_init_print(); #if DISASSEMBLER bfd_init(); @@ -73,64 +88,29 @@ jit_init_debug(const char *progname) } bfd_check_format(disasm_bfd, bfd_object); bfd_check_format(disasm_bfd, bfd_archive); + if (!disasm_stream) + disasm_stream = stderr; + +#if BINUTILS_2_38 + INIT_DISASSEMBLE_INFO(disasm_info, disasm_stream, fprintf, fprintf_styled); +#else INIT_DISASSEMBLE_INFO(disasm_info, disasm_stream, fprintf); -# if defined(__i386__) || defined(__x86_64__) - disasm_info.arch = bfd_arch_i386; -# if defined(__x86_64__) -# if __WORDSIZE == 32 - disasm_info.mach = bfd_mach_x64_32; -# else - disasm_info.mach = bfd_mach_x86_64; -# endif -# else - disasm_info.mach = bfd_mach_i386_i386; -# endif -# endif -# if defined(__powerpc__) - disasm_info.arch = bfd_arch_powerpc; - disasm_info.mach = bfd_mach_ppc64; -# if HAVE_DISASSEMBLE_INIT_FOR_TARGET +#endif + disasm_info.arch = bfd_get_arch(disasm_bfd); + disasm_info.mach = bfd_get_mach(disasm_bfd); + +# if HAVE_DISASSEMBLE_INIT_FOR_TARGET disassemble_init_for_target(&disasm_info); -# elif HAVE_DISASSEMBLE_INIT_POWERPC - disassemble_init_powerpc(&disasm_info); -# endif -# if defined(__powerpc64__) +# endif + +# if defined(__powerpc64__) disasm_info.disassembler_options = "64"; -# endif -# if HAVE_DISASSEMBLE_INIT_FOR_TARGET - disassemble_init_for_target(&disasm_info); -# elif HAVE_DISASSEMBLE_INIT_POWERPC - disassemble_init_powerpc(&disasm_info); -# endif # endif -# if defined(__sparc__) +# if defined(__sparc__) || defined(__s390__) || defined(__s390x__) disasm_info.endian = disasm_info.display_endian = BFD_ENDIAN_BIG; # endif # if defined(__s390__) || defined(__s390x__) - disasm_info.arch = bfd_arch_s390; -# if __WORDSIZE == 32 - disasm_info.mach = bfd_mach_s390_31; -# else - disasm_info.mach = bfd_mach_s390_64; -# endif - disasm_info.endian = disasm_info.display_endian = BFD_ENDIAN_BIG; disasm_info.disassembler_options = "zarch"; -# endif -# if defined(__alpha__) - disasm_info.arch = bfd_arch_alpha; - disasm_info.mach = bfd_mach_alpha_ev6; -# endif -# if defined(__hppa__) - disasm_info.arch = bfd_arch_hppa; - disasm_info.mach = bfd_mach_hppa10; -# endif -# if defined(__riscv) - disasm_info.arch = bfd_arch_riscv; -# if __WORDSIZE == 32 - disasm_info.mach = bfd_mach_riscv32; -# else - disasm_info.mach = bfd_mach_riscv64; -# endif # endif disasm_info.print_address_func = disasm_print_address; @@ -276,7 +256,7 @@ disasm_print_address(bfd_vma addr, struct disassemble_info *info) int line; char buffer[address_buffer_length]; - sprintf(buffer, address_buffer_format, (long long)addr); + sprintf(buffer, address_buffer_format, addr); (*info->fprintf_func)(info->stream, "0x%s", buffer); # define _jit disasm_jit @@ -339,7 +319,10 @@ _disassemble(jit_state_t *_jit, jit_pointer_t code, jit_int32_t length) char *name, *old_name; char *file, *old_file; int line, old_line; -#if __arm__ +#if __riscv && __WORDSIZE == 64 + jit_word_t *vector; + jit_int32_t offset; +#elif __arm__ jit_int32_t offset; jit_bool_t data_info; jit_int32_t data_offset; @@ -352,6 +335,10 @@ _disassemble(jit_state_t *_jit, jit_pointer_t code, jit_int32_t length) jit_uword_t prevw; #endif +#if __riscv && __WORDSIZE == 64 + end -= _jitc->consts.hash.count * 8; +#endif + #if __arm__ data_info = _jitc && _jitc->data_info.ptr; data_offset = 0; @@ -374,7 +361,7 @@ _disassemble(jit_state_t *_jit, jit_pointer_t code, jit_int32_t length) } while (node && (jit_uword_t)(prevw + node->offset) == (jit_uword_t)pc) { jit_print_node(node); - fputc('\n', stdout); + fputc('\n', disasm_stream); prevw += node->offset; node = node->next; } @@ -419,11 +406,22 @@ _disassemble(jit_state_t *_jit, jit_pointer_t code, jit_int32_t length) old_line = line; } - bytes = sprintf(buffer, address_buffer_format, (long long)pc); + bytes = sprintf(buffer, address_buffer_format, pc); (*disasm_info.fprintf_func)(disasm_stream, "%*c0x%s\t", 16 - bytes, ' ', buffer); pc += (*disasm_print)(pc, &disasm_info); putc('\n', disasm_stream); } +#if __riscv && __WORDSIZE == 64 + for (vector = (jit_word_t *)end, offset = 0; + offset < _jitc->consts.hash.count; offset++) { + bytes = sprintf(buffer, address_buffer_format, + (long long)end + offset * sizeof(jit_word_t)); + (*disasm_info.fprintf_func)(disasm_stream, + "%*c0x%s\t.quad\t0x%016lx\t# (%ld)\n", + 16 - bytes, ' ', buffer, + vector[offset], vector[offset]); + } +#endif } #endif diff --git a/parallel-rsp/lightning/lib/jit_fallback.c b/parallel-rsp/lightning/lib/jit_fallback.c new file mode 100644 index 0000000..beed2bf --- /dev/null +++ b/parallel-rsp/lightning/lib/jit_fallback.c @@ -0,0 +1,332 @@ +#if PROTO +#define fallback_save(r0) _fallback_save(_jit, r0) +static void _fallback_save(jit_state_t*, jit_int32_t); +#define fallback_load(r0) _fallback_load(_jit, r0) +static void _fallback_load(jit_state_t*, jit_int32_t); +#define fallback_save_regs(r0) _fallback_save_regs(_jit, r0) +static void _fallback_save_regs(jit_state_t*, jit_int32_t); +#define fallback_load_regs(r0) _fallback_load_regs(_jit, r0) +static void _fallback_load_regs(jit_state_t*, jit_int32_t); +#define fallback_calli(i0, i1) _fallback_calli(_jit, i0, i1) +static void _fallback_calli(jit_state_t*, jit_word_t, jit_word_t); +#define fallback_casx(r0,r1,r2,r3,im) _fallback_casx(_jit,r0,r1,r2,r3,im) +static void _fallback_casx(jit_state_t *, jit_int32_t, jit_int32_t, + jit_int32_t, jit_int32_t, jit_word_t); +#define fallback_clo(r0,r1) _fallback_clo(_jit,r0,r1) +static void _fallback_clo(jit_state_t*, jit_int32_t, jit_int32_t); +#define fallback_clz(r0,r1) _fallback_clz(_jit,r0,r1) +static void _fallback_clz(jit_state_t*, jit_int32_t, jit_int32_t); +#define fallback_cto(r0,r1) _fallback_cto(_jit,r0,r1) +static void _fallback_cto(jit_state_t*, jit_int32_t, jit_int32_t); +#define fallback_ctz(r0,r1) _fallback_ctz(_jit,r0,r1) +static void _fallback_ctz(jit_state_t*, jit_int32_t, jit_int32_t); +# if defined(__ia64__) +# define fallback_patch_jmpi(inst,lbl) \ + do { \ + sync(); \ + patch_at(jit_code_jmpi, inst, lbl); \ + } while (0) +# else +# define fallback_patch_jmpi(inst,lbl) fallback_patch_at(inst,lbl) +# endif +# if defined(__arm__) +# define fallback_patch_at(inst,lbl) patch_at(arm_patch_jump,inst,lbl) +# elif defined(__ia64__) +# define fallback_patch_at(inst,lbl) \ + do { \ + sync(); \ + patch_at(jit_code_bnei, inst, lbl); \ + } while (0); +# else +# define fallback_patch_at(inst,lbl) patch_at(inst,lbl) +# endif +# if defined(__mips__) +# define fallback_jmpi(i0) jmpi(i0,NULL,1) +# elif defined(__arm__) +# define fallback_jmpi(i0) jmpi_p(i0,1) +# elif defined(__s390__) || defined(__s390x__) +# define fallback_jmpi(i0) jmpi(i0,1) +# else +# define fallback_jmpi(i0) jmpi(i0) +# endif +# if defined(__mips__) +# define fallback_bnei(i0,r0,i1) bnei(i0,r0,i1,NULL) +# elif defined(__s390__) || defined(__s390x__) +# define fallback_bnei(i0,r0,i1) bnei_p(i0,r0,i1) +# else +# define fallback_bnei(i0,r0,i1) bnei(i0,r0,i1) +# endif +# if defined(__s390__) || defined(__s390x__) +# define fallback_bmsr(i0,r0,r1) bmsr_p(i0,r0,r1) +# else +# define fallback_bmsr(i0,r0,r1) bmsr(i0,r0,r1) +# endif +#endif + +#if CODE +static void +_fallback_save(jit_state_t *_jit, jit_int32_t r0) +{ + jit_int32_t offset, regno, spec; + for (offset = 0; offset < JIT_R_NUM; offset++) { + spec = _rvs[offset].spec; + regno = jit_regno(spec); + if (regno == r0) { + if (!(spec & jit_class_sav)) + stxi(_jitc->function->regoff[JIT_R(offset)], rn(JIT_FP), regno); + break; + } + } +} + +static void +_fallback_load(jit_state_t *_jit, jit_int32_t r0) +{ + jit_int32_t offset, regno, spec; + for (offset = 0; offset < JIT_R_NUM; offset++) { + spec = _rvs[offset].spec; + regno = jit_regno(spec); + if (regno == r0) { + if (!(spec & jit_class_sav)) + ldxi(regno, rn(JIT_FP), _jitc->function->regoff[JIT_R(offset)]); + break; + } + } +} + +static void +_fallback_save_regs(jit_state_t *_jit, jit_int32_t r0) +{ + jit_int32_t regno, spec; + for (regno = 0; regno < _jitc->reglen; regno++) { + spec = _rvs[regno].spec; + if ((jit_regset_tstbit(&_jitc->regarg, regno) || + jit_regset_tstbit(&_jitc->reglive, regno)) && + !(spec & jit_class_sav)) { + if (!_jitc->function->regoff[regno]) { + _jitc->function->regoff[regno] = + jit_allocai(spec & jit_class_gpr ? + sizeof(jit_word_t) : sizeof(jit_float64_t)); + _jitc->again = 1; + } + if ((spec & jit_class_gpr) && rn(regno) == r0) + continue; + jit_regset_setbit(&_jitc->regsav, regno); + if (spec & jit_class_gpr) + emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno); + else + emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno); + } + } +} + +static void +_fallback_load_regs(jit_state_t *_jit, jit_int32_t r0) +{ + jit_int32_t regno, spec; + for (regno = 0; regno < _jitc->reglen; regno++) { + spec = _rvs[regno].spec; + if ((jit_regset_tstbit(&_jitc->regarg, regno) || + jit_regset_tstbit(&_jitc->reglive, regno)) && + !(spec & jit_class_sav)) { + if ((spec & jit_class_gpr) && rn(regno) == r0) + continue; + jit_regset_setbit(&_jitc->regsav, regno); + if (spec & jit_class_gpr) + emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]); + else + emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]); + } + } +} + +static void +_fallback_calli(jit_state_t *_jit, jit_word_t i0, jit_word_t i1) +{ +# if defined(__arm__) + movi(rn(_R0), i1); +# elif defined(__hppa__) + movi(_R26_REGNO, i1); +# endif +# if defined(__arm__) + calli(i0, jit_exchange_p()); +# elif defined(__mips__) + calli(i0, NULL, 0); +# elif defined(__powerpc__) && _CALL_SYSV + calli(i0, 0); +# elif defined(__s390__) || defined(__s390x__) + calli(i0, 0); +# else + calli(i0); +# endif +} + +static void +_fallback_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t r1_reg, iscasi; + jit_word_t jump, done; + /* XXX only attempts to fallback cas for lightning jit code */ + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + if ((iscasi = r1 == _NOREG)) { + r1_reg = jit_get_reg(jit_class_gpr|jit_class_sav); + r1 = rn(r1_reg); + movi(r1, i0); + } + fallback_save_regs(r0); + fallback_calli((jit_word_t)pthread_mutex_lock, (jit_word_t)&mutex); + fallback_load(r1); + ldr(r0, r1); + fallback_load(r2); + eqr(r0, r0, r2); + fallback_save(r0); + jump = fallback_bnei(_jit->pc.w, r0, 1); + fallback_load(r3); +# if __WORDSIZE == 32 + str_i(r1, r3); +# else + str_l(r1, r3); +# endif + /* done: */ + done = _jit->pc.w; + fallback_calli((jit_word_t)pthread_mutex_unlock, (jit_word_t)&mutex); + fallback_load(r0); + fallback_patch_at(jump, done); + fallback_load_regs(r0); + if (iscasi) + jit_unget_reg(r1_reg); +} + +static void +_fallback_clo(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t clz, done; + comr(r0, r1); + clz = fallback_bnei(_jit->pc.w, r0, 0); + movi(r0, __WORDSIZE); + done = fallback_jmpi(_jit->pc.w); + fallback_patch_at(clz, _jit->pc.w); + fallback_clz(r0, r0); + fallback_patch_jmpi(done, _jit->pc.w); +} + +static void +_fallback_clz(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t r1_reg, r2, r2_reg; + jit_word_t clz, l32, l16, l8, l4, l2, l1; + l32 = fallback_bnei(_jit->pc.w, r1, 0); + movi(r0, __WORDSIZE); + clz = fallback_jmpi(_jit->pc.w); + fallback_patch_at(l32, _jit->pc.w); + r2_reg = jit_get_reg(jit_class_gpr); + r2 = rn(r2_reg); + r1_reg = jit_get_reg(jit_class_gpr); + movr(rn(r1_reg), r1); + r1 = rn(r1_reg); + movi(r0, 0); +# if __WORDSIZE == 64 + movi(r2, 0xffffffff00000000UL); + l32 = fallback_bmsr(_jit->pc.w, r1, r2); + lshi(r1, r1, 32); + addi(r0, r0, 32); + fallback_patch_at(l32, _jit->pc.w); + lshi(r2, r2, 16); +# else + movi(r2, 0xffff0000UL); +# endif + l16 = fallback_bmsr(_jit->pc.w, r1, r2); + lshi(r1, r1, 16); + addi(r0, r0, 16); + fallback_patch_at(l16, _jit->pc.w); + lshi(r2, r2, 8); + l8 = fallback_bmsr(_jit->pc.w, r1, r2); + lshi(r1, r1, 8); + addi(r0, r0, 8); + fallback_patch_at(l8, _jit->pc.w); + lshi(r2, r2, 4); + l4 = fallback_bmsr(_jit->pc.w, r1, r2); + lshi(r1, r1, 4); + addi(r0, r0, 4); + fallback_patch_at(l4, _jit->pc.w); + lshi(r2, r2, 2); + l2 = fallback_bmsr(_jit->pc.w, r1, r2); + lshi(r1, r1, 2); + addi(r0, r0, 2); + fallback_patch_at(l2, _jit->pc.w); + lshi(r2, r2, 1); + l1 = fallback_bmsr(_jit->pc.w, r1, r2); + addi(r0, r0, 1); + fallback_patch_at(l1, _jit->pc.w); + fallback_patch_jmpi(clz, _jit->pc.w); + jit_unget_reg(r2_reg); + jit_unget_reg(r1_reg); +} + +static void +_fallback_cto(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t ctz, done; + comr(r0, r1); + ctz = fallback_bnei(_jit->pc.w, r0, 0); + movi(r0, __WORDSIZE); + done = fallback_jmpi(_jit->pc.w); + fallback_patch_at(ctz, _jit->pc.w); + fallback_ctz(r0, r0); + fallback_patch_jmpi(done, _jit->pc.w); +} + +static void +_fallback_ctz(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t r1_reg, r2, r2_reg; + jit_word_t ctz, l32, l16, l8, l4, l2, l1; + l32 = fallback_bnei(_jit->pc.w, r1, 0); + movi(r0, __WORDSIZE); + ctz = fallback_jmpi(_jit->pc.w); + fallback_patch_at(l32, _jit->pc.w); + r2_reg = jit_get_reg(jit_class_gpr); + r2 = rn(r2_reg); + r1_reg = jit_get_reg(jit_class_gpr); + movr(rn(r1_reg), r1); + r1 = rn(r1_reg); + movi(r0, 0); +# if __WORDSIZE == 64 + movi(r2, 0xffffffffUL); + l32 = fallback_bmsr(_jit->pc.w, r1, r2); + rshi_u(r1, r1, 32); + addi(r0, r0, 32); + fallback_patch_at(l32, _jit->pc.w); + rshi(r2, r2, 16); +# else + movi(r2, 0xffffUL); +# endif + l16 = fallback_bmsr(_jit->pc.w, r1, r2); + rshi_u(r1, r1, 16); + addi(r0, r0, 16); + fallback_patch_at(l16, _jit->pc.w); + rshi(r2, r2, 8); + l8 = fallback_bmsr(_jit->pc.w, r1, r2); + rshi_u(r1, r1, 8); + addi(r0, r0, 8); + fallback_patch_at(l8, _jit->pc.w); + rshi(r2, r2, 4); + l4 = fallback_bmsr(_jit->pc.w, r1, r2); + rshi_u(r1, r1, 4); + addi(r0, r0, 4); + fallback_patch_at(l4, _jit->pc.w); + rshi(r2, r2, 2); + l2 = fallback_bmsr(_jit->pc.w, r1, r2); + rshi_u(r1, r1, 2); + addi(r0, r0, 2); + fallback_patch_at(l2, _jit->pc.w); + rshi(r2, r2, 1); + l1 = fallback_bmsr(_jit->pc.w, r1, r2); + addi(r0, r0, 1); + fallback_patch_at(l1, _jit->pc.w); + fallback_patch_jmpi(ctz, _jit->pc.w); + jit_unget_reg(r2_reg); + jit_unget_reg(r1_reg); +} +#endif diff --git a/parallel-rsp/lightning/lib/jit_hppa-cpu.c b/parallel-rsp/lightning/lib/jit_hppa-cpu.c index db5a36a..ebb01fd 100644 --- a/parallel-rsp/lightning/lib/jit_hppa-cpu.c +++ b/parallel-rsp/lightning/lib/jit_hppa-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -648,18 +648,25 @@ static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); static void _movi(jit_state_t*,jit_int32_t,jit_word_t); #define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) #define comr(r0,r1) UADDCM(_R0_REGNO,r1,r0) #define negr(r0,r1) SUB(_R0_REGNO,r1,r0) #define extr_c(r0,r1) EXTRWR(r1,31,8,r0) #define extr_uc(r0,r1) EXTRWR_U(r1,31,8,r0) #define extr_s(r0,r1) EXTRWR(r1,31,16,r0) #define extr_us(r0,r1) EXTRWR_U(r1,31,16,r0) -#if __BYTE_ORDER == __BIG_ENDIAN -# define htonr_us(r0,r1) extr_us(r0,r1) -# define htonr_ui(r0,r1) movr(r0,r1) -#else -# error need htonr implementation -#endif #define addr(r0,r1,r2) ADD(r1,r2,r0) #define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); @@ -907,7 +914,7 @@ static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); #define jmpr(r0) _jmpr(_jit,r0) static void _jmpr(jit_state_t*,jit_int32_t); #define jmpi(i0) _jmpi(_jit,i0) -static void _jmpi(jit_state_t*,jit_word_t); +static jit_word_t _jmpi(jit_state_t*,jit_word_t); #define jmpi_p(i0) _jmpi_p(_jit,i0) static jit_word_t _jmpi_p(jit_state_t*,jit_word_t); #define callr(r0) _callr(_jit,r0) @@ -1633,6 +1640,108 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (r0 == r1) { + reg = jit_get_reg(jit_class_gpr); + movr(rn(reg), r1); + EXTRWR_U(rn(reg), 23, 8, r0); + DEPWR(rn(reg), 23, 8, r0); + jit_unget_reg(reg); + } + else { + EXTRWR_U(r1, 23, 8, r0); + DEPWR(r1, 23, 8, r0); + } +} + +static void +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (r0 == r1) { + reg = jit_get_reg(jit_class_gpr); + movr(rn(reg), r1); + SHRPWI(rn(reg), rn(reg), 16, r0); + DEPWR(r0, 15, 8, r0); + SHRPWI(rn(reg), r0, 8, r0); + jit_unget_reg(reg); + } + else { + SHRPWI(r1, r1, 16, r0); + DEPWR(r0, 15, 8, r0); + SHRPWI(r1, r0, 8, r0); + } +} + +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = beqi(_jit->pc.w, r2, 0); + COPY(r1, r0); + patch_at(w, _jit->pc.w); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = bnei(_jit->pc.w, r2, 0); + COPY(r1, r0); + patch_at(w, _jit->pc.w); +} + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ +#if defined(__linux__) && defined(SYS_atomic_cmpxchg_32) && __WORDSIZE == 32 + /* Not defined, and unlikely to ever be defined, but could be a way to do it */ + movi(_R26_REGNO, SYS_atomic_cmpxchg_32); + if (r1 == _NOREG) + movi(_R25_REGNO, i0); + else + movr(_R25_REGNO, r1); + movr(_R24_REGNO, r2); + movr(_R23_REGNO, r3); + /* Should only fail for an invalid or unaligned address. + * Do not handle this condition. */ + calli(syscall); + movr(r0, _R28_REGNO); +#else + /* + * The only atomic operations are LDCW and LDCD, that load a value, + * and store zero at the address atomically. The (semaphore) address + * must be 16 byte aligned. + */ + fallback_casx(r0, r1, r2, r3, i0); + /* + * It is important to be aware of the delayed nature of cache flush and + * purge operations, and to use SYNC instructions to force completion + * where necessary. The following example illustrates this. + * Consider two processes sharing a memory location x which is protected + * by a semaphore s. + * + * process A on Processor 1 | process B on Processor 2 | note + * -------------------------+---------------------------+------------ + * LDCW s | | A acquires semaphore + * PDC x | | A executes purge + * SYNC | | Force completion of purge + * STW s | | A releases semaphore + * | LDCW s | B acquires semaphore + * | STW x + * + * In the absence of the SYNC instruction, it would be possible for + * process B's store to x to complete before the purge of x is completed + * (since the purge may have been delayed). The purge of x could then + * destroy the new value. + */ +#endif +} + static void _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -2561,17 +2670,19 @@ _jmpr(jit_state_t *_jit, jit_int32_t r0) BV_N(_R0_REGNO, r0); } -static void +static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { - jit_word_t w; - w = ((i0 - _jit->pc.w) >> 2) - 2; - if (w >= -32768 && w <= 32767) - B_N(w, _R0_REGNO); + jit_word_t d, w; + w = _jit->pc.w; + d = ((i0 - w) >> 2) - 2; + if (d >= -32768 && d <= 32767) + B_N(d, _R0_REGNO); else { - movi(_R1_REGNO, w); + movi(_R1_REGNO, d); BV_N(_R0_REGNO, _R1_REGNO); } + return (w); } static jit_word_t diff --git a/parallel-rsp/lightning/lib/jit_hppa-fpu.c b/parallel-rsp/lightning/lib/jit_hppa-fpu.c index 5fa6856..ed141a7 100644 --- a/parallel-rsp/lightning/lib/jit_hppa-fpu.c +++ b/parallel-rsp/lightning/lib/jit_hppa-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/lib/jit_hppa-sz.c b/parallel-rsp/lightning/lib/jit_hppa-sz.c index 3c04f63..e41f89c 100644 --- a/parallel-rsp/lightning/lib/jit_hppa-sz.c +++ b/parallel-rsp/lightning/lib/jit_hppa-sz.c @@ -1,20 +1,24 @@ #if __WORDSIZE == 32 -#define JIT_INSTR_MAX 64 +#define JIT_INSTR_MAX 196 0, /* data */ 0, /* live */ - 0, /* align */ + 28, /* align */ 0, /* save */ 0, /* load */ + 0, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ - 64, /* prolog */ + 156, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -22,11 +26,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 0, /* va_start */ - 0, /* va_arg */ - 0, /* va_arg_d */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 4, /* va_start */ + 8, /* va_arg */ + 20, /* va_arg_d */ 0, /* va_end */ 4, /* addr */ 12, /* addi */ @@ -40,13 +56,13 @@ 12, /* subci */ 4, /* subxr */ 8, /* subxi */ - 16, /* rsbi */ - 28, /* mulr */ - 36, /* muli */ + 12, /* rsbi */ + 48, /* mulr */ + 56, /* muli */ 40, /* qmulr */ 44, /* qmuli */ - 32, /* qmulr_u */ - 40, /* qmuli_u */ + 52, /* qmulr_u */ + 60, /* qmuli_u */ 36, /* divr */ 40, /* divi */ 36, /* divr_u */ @@ -95,15 +111,22 @@ 8, /* nei */ 4, /* movr */ 8, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 88, /* casr */ + 96, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 12, /* bswapr_us */ + 16, /* bswapr_ui */ + 0, /* bswapr_ul */ 4, /* htonr_us */ 4, /* htonr_ui */ - 0, /* htonr_l */ + 0, /* htonr_ul */ 8, /* ldr_c */ 12, /* ldi_c */ 4, /* ldr_uc */ @@ -119,15 +142,15 @@ 0, /* ldr_l */ 0, /* ldi_l */ 8, /* ldxr_c */ - 8, /* ldxi_c */ + 12, /* ldxi_c */ 4, /* ldxr_uc */ - 4, /* ldxi_uc */ + 8, /* ldxi_uc */ 8, /* ldxr_s */ - 8, /* ldxi_s */ + 12, /* ldxi_s */ 4, /* ldxr_us */ - 4, /* ldxi_us */ + 8, /* ldxi_us */ 4, /* ldxr_i */ - 4, /* ldxi_i */ + 8, /* ldxi_i */ 0, /* ldxr_ui */ 0, /* ldxi_ui */ 0, /* ldxr_l */ @@ -141,11 +164,11 @@ 0, /* str_l */ 0, /* sti_l */ 8, /* stxr_c */ - 4, /* stxi_c */ + 12, /* stxi_c */ 8, /* stxr_s */ - 4, /* stxi_s */ + 12, /* stxi_s */ 8, /* stxr_i */ - 4, /* stxi_i */ + 12, /* stxi_i */ 0, /* stxr_l */ 0, /* stxi_l */ 8, /* bltr */ @@ -188,18 +211,42 @@ 16, /* bxsubi */ 16, /* bxsubr_u */ 20, /* bxsubi_u */ - 0, /* jmpr */ + 4, /* jmpr */ 12, /* jmpi */ 40, /* callr */ 44, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -207,93 +254,93 @@ 0, /* retval_i */ 0, /* retval_ui */ 0, /* retval_l */ - 64, /* epilog */ + 196, /* epilog */ 0, /* arg_f */ 0, /* getarg_f */ 0, /* putargr_f */ 0, /* putargi_f */ 4, /* addr_f */ - 16, /* addi_f */ + 24, /* addi_f */ 4, /* subr_f */ - 16, /* subi_f */ - 16, /* rsbi_f */ + 24, /* subi_f */ + 24, /* rsbi_f */ 4, /* mulr_f */ - 16, /* muli_f */ + 24, /* muli_f */ 4, /* divr_f */ - 16, /* divi_f */ + 24, /* divi_f */ 4, /* negr_f */ 4, /* absr_f */ 4, /* sqrtr_f */ 16, /* ltr_f */ - 28, /* lti_f */ + 36, /* lti_f */ 16, /* ler_f */ - 28, /* lei_f */ + 36, /* lei_f */ 16, /* eqr_f */ - 28, /* eqi_f */ + 36, /* eqi_f */ 16, /* ger_f */ - 28, /* gei_f */ + 36, /* gei_f */ 16, /* gtr_f */ - 28, /* gti_f */ + 36, /* gti_f */ 16, /* ner_f */ - 28, /* nei_f */ + 36, /* nei_f */ 16, /* unltr_f */ - 28, /* unlti_f */ + 36, /* unlti_f */ 16, /* unler_f */ - 28, /* unlei_f */ + 36, /* unlei_f */ 16, /* uneqr_f */ - 28, /* uneqi_f */ + 36, /* uneqi_f */ 16, /* unger_f */ - 28, /* ungei_f */ + 36, /* ungei_f */ 16, /* ungtr_f */ - 28, /* ungti_f */ + 36, /* ungti_f */ 16, /* ltgtr_f */ - 28, /* ltgti_f */ + 36, /* ltgti_f */ 16, /* ordr_f */ - 28, /* ordi_f */ + 36, /* ordi_f */ 16, /* unordr_f */ - 28, /* unordi_f */ - 12, /* truncr_f_i */ + 36, /* unordi_f */ + 16, /* truncr_f_i */ 0, /* truncr_f_l */ - 12, /* extr_f */ + 20, /* extr_f */ 4, /* extr_d_f */ 4, /* movr_f */ - 12, /* movi_f */ + 20, /* movi_f */ 4, /* ldr_f */ 12, /* ldi_f */ - 4, /* ldxr_f */ - 4, /* ldxi_f */ + 8, /* ldxr_f */ + 12, /* ldxi_f */ 4, /* str_f */ 12, /* sti_f */ 8, /* stxr_f */ - 4, /* stxi_f */ + 12, /* stxi_f */ 16, /* bltr_f */ - 28, /* blti_f */ + 36, /* blti_f */ 16, /* bler_f */ - 28, /* blei_f */ + 36, /* blei_f */ 16, /* beqr_f */ - 28, /* beqi_f */ + 36, /* beqi_f */ 16, /* bger_f */ - 28, /* bgei_f */ + 36, /* bgei_f */ 16, /* bgtr_f */ - 28, /* bgti_f */ + 36, /* bgti_f */ 16, /* bner_f */ - 28, /* bnei_f */ + 36, /* bnei_f */ 16, /* bunltr_f */ - 28, /* bunlti_f */ + 36, /* bunlti_f */ 16, /* bunler_f */ - 28, /* bunlei_f */ + 36, /* bunlei_f */ 16, /* buneqr_f */ - 28, /* buneqi_f */ + 36, /* buneqi_f */ 16, /* bunger_f */ - 28, /* bungei_f */ + 36, /* bungei_f */ 16, /* bungtr_f */ - 28, /* bungti_f */ + 36, /* bungti_f */ 16, /* bltgtr_f */ - 28, /* bltgti_f */ + 36, /* bltgti_f */ 16, /* bordr_f */ - 28, /* bordi_f */ + 36, /* bordi_f */ 16, /* bunordr_f */ - 28, /* bunordi_f */ + 36, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -304,87 +351,87 @@ 0, /* putargr_d */ 0, /* putargi_d */ 4, /* addr_d */ - 24, /* addi_d */ + 32, /* addi_d */ 4, /* subr_d */ - 24, /* subi_d */ - 24, /* rsbi_d */ + 32, /* subi_d */ + 32, /* rsbi_d */ 4, /* mulr_d */ - 24, /* muli_d */ + 32, /* muli_d */ 4, /* divr_d */ - 24, /* divi_d */ + 32, /* divi_d */ 4, /* negr_d */ 4, /* absr_d */ 4, /* sqrtr_d */ 16, /* ltr_d */ - 36, /* lti_d */ + 44, /* lti_d */ 16, /* ler_d */ - 36, /* lei_d */ + 44, /* lei_d */ 16, /* eqr_d */ - 36, /* eqi_d */ + 44, /* eqi_d */ 16, /* ger_d */ - 36, /* gei_d */ + 44, /* gei_d */ 16, /* gtr_d */ - 36, /* gti_d */ + 44, /* gti_d */ 16, /* ner_d */ - 36, /* nei_d */ + 44, /* nei_d */ 16, /* unltr_d */ - 36, /* unlti_d */ + 44, /* unlti_d */ 16, /* unler_d */ - 36, /* unlei_d */ + 44, /* unlei_d */ 16, /* uneqr_d */ - 36, /* uneqi_d */ + 44, /* uneqi_d */ 16, /* unger_d */ - 36, /* ungei_d */ + 44, /* ungei_d */ 16, /* ungtr_d */ - 36, /* ungti_d */ + 44, /* ungti_d */ 16, /* ltgtr_d */ - 36, /* ltgti_d */ + 44, /* ltgti_d */ 16, /* ordr_d */ - 36, /* ordi_d */ + 44, /* ordi_d */ 16, /* unordr_d */ - 36, /* unordi_d */ - 12, /* truncr_d_i */ + 44, /* unordi_d */ + 16, /* truncr_d_i */ 0, /* truncr_d_l */ - 12, /* extr_d */ + 20, /* extr_d */ 4, /* extr_f_d */ 4, /* movr_d */ - 20, /* movi_d */ + 28, /* movi_d */ 4, /* ldr_d */ 12, /* ldi_d */ - 4, /* ldxr_d */ - 4, /* ldxi_d */ + 8, /* ldxr_d */ + 12, /* ldxi_d */ 4, /* str_d */ 12, /* sti_d */ 8, /* stxr_d */ - 4, /* stxi_d */ + 12, /* stxi_d */ 16, /* bltr_d */ - 36, /* blti_d */ + 44, /* blti_d */ 16, /* bler_d */ - 36, /* blei_d */ + 44, /* blei_d */ 16, /* beqr_d */ - 36, /* beqi_d */ + 44, /* beqi_d */ 16, /* bger_d */ - 36, /* bgei_d */ + 44, /* bgei_d */ 16, /* bgtr_d */ - 36, /* bgti_d */ + 44, /* bgti_d */ 16, /* bner_d */ - 36, /* bnei_d */ + 44, /* bnei_d */ 16, /* bunltr_d */ - 36, /* bunlti_d */ + 44, /* bunlti_d */ 16, /* bunler_d */ - 36, /* bunlei_d */ + 44, /* bunlei_d */ 16, /* buneqr_d */ - 36, /* buneqi_d */ + 44, /* buneqi_d */ 16, /* bunger_d */ - 36, /* bungei_d */ + 44, /* bungei_d */ 16, /* bungtr_d */ - 36, /* bungti_d */ + 44, /* bungti_d */ 16, /* bltgtr_d */ - 36, /* bltgti_d */ + 44, /* bltgti_d */ 16, /* bordr_d */ - 36, /* bordi_d */ + 44, /* bordi_d */ 16, /* bunordr_d */ - 36, /* bunordi_d */ + 44, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -399,4 +446,8 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 160, /* clo */ + 140, /* clz */ + 164, /* cto */ + 144, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_hppa.c b/parallel-rsp/lightning/lib/jit_hppa.c index 21fe20c..d3c5ef7 100644 --- a/parallel-rsp/lightning/lib/jit_hppa.c +++ b/parallel-rsp/lightning/lib/jit_hppa.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -19,12 +19,17 @@ #include #include +#if defined(__linux__) +# include +# include +#endif #define jit_arg_reg_p(i) (i >= 0 && i < 4) #define PROTO 1 # include "jit_hppa-cpu.c" # include "jit_hppa-fpu.c" +# include "jit_fallback.c" #undef PROTO /* @@ -240,18 +245,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); + jit_code_inc_synth_w(code, u); jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -305,7 +310,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - assert(u->code == jit_code_arg || + assert((u->code >= jit_code_arg_c && u->code <= jit_code_arg) || u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_reg_p(u->u.w)); } @@ -338,17 +343,21 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif _jitc->function->self.size -= sizeof(jit_word_t); if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else offset = _jitc->function->self.size; - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -401,7 +410,7 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (v->u.w >= 0) jit_extr_c(u, _R26 - v->u.w); @@ -413,7 +422,7 @@ _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (v->u.w >= 0) jit_extr_uc(u, _R26 - v->u.w); @@ -425,7 +434,7 @@ _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (v->u.w >= 0) jit_extr_s(u, _R26 - v->u.w); @@ -437,7 +446,7 @@ _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (v->u.w >= 0) jit_extr_us(u, _R26 - v->u.w); @@ -449,7 +458,7 @@ _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (v->u.w >= 0) jit_movr(u, _R26 - v->u.w); @@ -459,10 +468,10 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (v->u.w >= 0) jit_movr(_R26 - v->u.w, u); else @@ -471,11 +480,11 @@ _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (v->u.w >= 0) jit_movi(_R26 - v->u.w, u); else { @@ -570,10 +579,10 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); _jitc->function->call.size -= sizeof(jit_word_t); if (jit_arg_reg_p(_jitc->function->call.argi)) { @@ -586,11 +595,11 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); _jitc->function->call.size -= sizeof(jit_word_t); if (jit_arg_reg_p(_jitc->function->call.argi)) { @@ -854,6 +863,7 @@ _emit_code(jit_state_t *_jit) struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -969,11 +979,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1026,6 +1038,16 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh,); case_rrr(rsh, _u); case_rrw(rsh, _u); + case_rrr(movn,); + case_rrr(movz,); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1046,12 +1068,22 @@ _emit_code(jit_state_t *_jit) break; case_rr(neg,); case_rr(com,); +#define clor(r0, r1) fallback_clo(r0, r1) +#define clzr(r0, r1) fallback_clz(r0, r1) +#define ctor(r0, r1) fallback_cto(r0, r1) +#define ctzr(r0, r1) fallback_ctz(r0, r1) + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); case_rr(ext, _us); case_rr(hton, _us); case_rr(hton, _ui); + case_rr(bswap, _us); + case_rr(bswap, _ui); case_rrr(lt,); case_rrw(lt,); case_rrr(lt, _u); @@ -1323,7 +1355,12 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (word >= -32768 && word <= 32767) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } @@ -1352,6 +1389,7 @@ _emit_code(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1372,6 +1410,18 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. + * Note that for hppa use '-' instead of '+' as hppa + * stack grows up */ + undo.func.self.aoff = _jitc->function->frame - + _jitc->function->self.aoff; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1393,24 +1443,37 @@ _emit_code(jit_state_t *_jit) case jit_code_va_arg_d: vaarg_d(rn(node->u.w), rn(node->v.w)); break; - case jit_code_live: - case jit_code_arg: case jit_code_ellipsis: + case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: case jit_code_getarg_s: case jit_code_getarg_us: case jit_code_getarg_i: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1455,6 +1518,7 @@ _emit_code(jit_state_t *_jit) #define CODE 1 # include "jit_hppa-cpu.c" # include "jit_hppa-fpu.c" +# include "jit_fallback.c" #undef CODE void diff --git a/parallel-rsp/lightning/lib/jit_ia64-cpu.c b/parallel-rsp/lightning/lib/jit_ia64-cpu.c index dec1465..98a10c3 100644 --- a/parallel-rsp/lightning/lib/jit_ia64-cpu.c +++ b/parallel-rsp/lightning/lib/jit_ia64-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -1175,6 +1175,8 @@ static void _X5(jit_state_t*,jit_word_t, #define ZXT2(r1,r3) I29(0x11,r3,r1) #define ZXT4(r1,r3) I29(0x12,r3,r1) +# define nop(i0) _nop(_jit,i0) +static void _nop(jit_state_t*, jit_int32_t); #define addr(r0,r1,r2) ADD(r0,r1,r2) #define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); @@ -1299,6 +1301,16 @@ static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); #define nei(r0,r1,i0) _nei(_jit,r0,r1,i0) static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); +#define bitswap(r0, r1) _bitswap(_jit, r0, r1) +static void _bitswap(jit_state_t*, jit_int32_t, jit_int32_t); +#define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +#define clzr(r0, r1) _clzr(_jit, r0, r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +#define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +#define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); #define negr(r0,r1) subr(r0,0,r1) #define comr(r0,r1) ANDCMI(r0,-1,r1) #define movr(r0,r1) _movr(_jit,r0,r1) @@ -1307,17 +1319,20 @@ static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); static void _movi(jit_state_t*,jit_int32_t,jit_word_t); #define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); -#if __BYTE_ORDER == __LITTLE_ENDIAN -# define htonr_us(r0,r1) _htonr_us(_jit,r0,r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0,r1) _htonr_ui(_jit,r0,r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ul(r0,r1) MUX1(r0,r1,MUX_REV) -#else -# define htonr_us(r0,r1) extr_us(r0,r1) -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) -#endif +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ul(r0,r1) MUX1(r0,r1,MUX_REV) #define extr_c(r0,r1) SXT1(r0,r1) #define extr_uc(r0,r1) ZXT1(r0,r1) #define extr_s(r0,r1) SXT2(r0,r1) @@ -1495,7 +1510,7 @@ static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); #define jmpr(r0) _jmpr(_jit,r0) static void _jmpr(jit_state_t*,jit_int32_t); #define jmpi(i0) _jmpi(_jit,i0) -static void _jmpi(jit_state_t*,jit_word_t); +static jit_word_t _jmpi(jit_state_t*,jit_word_t); #define jmpi_p(i0) _jmpi_p(_jit,i0) static jit_word_t _jmpi_p(jit_state_t*,jit_word_t); #define callr(r0) _callr(_jit,r0) @@ -1512,7 +1527,7 @@ static void _epilog(jit_state_t*,jit_node_t*); static void _vastart(jit_state_t*, jit_int32_t); # define vaarg(r0, r1) _vaarg(_jit, r0, r1) static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t); -#define patch_at(node,instr,label) _patch_at(_jit,node,instr,label) +#define patch_at(code,instr,label) _patch_at(_jit,code,instr,label) static void _patch_at(jit_state_t*,jit_code_t,jit_word_t,jit_word_t); #endif @@ -2451,7 +2466,7 @@ _I9(jit_state_t *_jit, jit_word_t _p, TSTREG1(r3); TSTPRED(_p); TSTREG1(r1); - inst((7L<<37)|(1L<<34)|(1L<<34)|(1L<<33)| + inst((7L<<37)|(1L<<34)|(1L<<33)| (x2<<30)|(1L<<28)|(r3<<20)|(r1<<6)|_p, INST_I); SETREG(r1); } @@ -3026,7 +3041,7 @@ _M29(jit_state_t *_jit, jit_word_t _p, jit_word_t ar, jit_word_t r2) { assert(!(_p & ~0x3fL)); - assert(!(ar & ~0x7L)); + assert(!(ar & ~0x7fL)); assert(!(r2 & ~0x7fL)); TSTREG1(r2); TSTPRED(_p); @@ -3450,6 +3465,104 @@ _X5(jit_state_t *_jit, jit_word_t _p, inst((i1<<36)|(1L<<27)|(y<<26)|(i20<<6)|_p, INST_X); } +static void +_nop(jit_state_t *_jit, jit_int32_t i0) +{ + for (; i0 > 0; i0 -= 8) { + NOP_M(0); + sync(); + } + assert(i0 == 0); +} + +static void +_bitswap(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t t0, t1, t2, t3, t4; + movr(r0, r1); + t0 = jit_get_reg(jit_class_gpr); + t1 = jit_get_reg(jit_class_gpr); + t2 = jit_get_reg(jit_class_gpr); + movi(rn(t0), __WORDSIZE == 32 ? 0x55555555L : 0x5555555555555555L); + rshi_u(rn(t1), r0, 1); /* t1 = v >> 1 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 1); /* t2 <<= 1 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x33333333L : 0x3333333333333333L); + rshi_u(rn(t1), r0, 2); /* t1 = v >> 2 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 2); /* t2 <<= 2 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x0f0f0f0fL : 0x0f0f0f0f0f0f0f0fL); + rshi_u(rn(t1), r0, 4); /* t1 = v >> 4 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 4); /* t2 <<= 4 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x00ff00ffL : 0x00ff00ff00ff00ffL); + rshi_u(rn(t1), r0, 8); /* t1 = v >> 8 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 8); /* t2 <<= 8 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), 0x0000ffff0000ffffL); + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 16); /* t2 <<= 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + rshi_u(rn(t1), r0, 32); /* t1 = v >> 32 */ + lshi(rn(t2), r0, 32); /* t2 = v << 32 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + jit_unget_reg(t2); + jit_unget_reg(t1); + jit_unget_reg(t0); +} + +static void +_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.clz) + CLZ(r0, r1); + else + fallback_clz(r0, r1); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.clz) { + comr(r0, r1); + clzr(r0, r0); + } + else + fallback_clo(r0, r1); +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.clz) { + bitswap(r0, r1); + clor(r0, r0); + } + else + fallback_cto(r0, r1); +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.clz) { + bitswap(r0, r1); + clzr(r0, r0); + } + else + fallback_ctz(r0, r1); +} + static void _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { @@ -3483,6 +3596,53 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + CMP_EQ(PR_6, PR_7, r2, GR_0); + MOV_p(r0, r1, PR_7); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + CMP_EQ(PR_6, PR_7, r2, GR_0); + MOV_p(r0, r1, PR_6); +} + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t r1_reg, iscasi; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + sync(); + MOV_M_ar_rn(AR_CCV, r2); + CMPXCHG8_ACQ(r0, r1, r3); + eqr(r0, r0, r2); + if (iscasi) + jit_unget_reg(r1_reg); +} + + +static void +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + bswapr_ul(r0, r1); + rshi_u(r0, r0, 48); +} + +static void +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + bswapr_ul(r0, r1); + rshi_u(r0, r0, 32); +} + static void _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -3949,48 +4109,6 @@ _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) } } -#if __BYTE_ORDER == __LITTLE_ENDIAN -static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t t0; - t0 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 8); - andi(r0, r1, 0xff); - andi(rn(t0), rn(t0), 0xff); - lshi(r0, r0, 8); - orr(r0, r0, rn(t0)); - jit_unget_reg(t0); -} - -static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t t0; - jit_int32_t t1; - jit_int32_t t2; - t0 = jit_get_reg(jit_class_gpr); - t1 = jit_get_reg(jit_class_gpr); - t2 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 24); - rshi(rn(t1), r1, 16); - rshi(rn(t2), r1, 8); - andi(rn(t0), rn(t0), 0xff); - andi(rn(t1), rn(t1), 0xff); - andi(rn(t2), rn(t2), 0xff); - andi(r0, r1, 0xff); - lshi(r0, r0, 24); - lshi(rn(t1), rn(t1), 8); - orr(r0, r0, rn(t0)); - lshi(rn(t2), rn(t2), 16); - orr(r0, r0, rn(t1)); - orr(r0, r0, rn(t2)); - jit_unget_reg(t2); - jit_unget_reg(t1); - jit_unget_reg(t0); -} -#endif - static void _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -5125,16 +5243,18 @@ _jmpr(jit_state_t *_jit, jit_int32_t r0) BR(BR_6); } -static void +static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { - jit_word_t d; + jit_word_t d, w; sync(); - d = ((jit_word_t)i0 - _jit->pc.w) >> 4; + w = _jit->pc.w; + d = ((jit_word_t)i0 - w) >> 4; if (d >= -16777216 && d <= 16777215) BRI(d); else BRL(d); + return (w); } static jit_word_t @@ -5380,14 +5500,16 @@ _patch_at(jit_state_t *_jit, jit_code_t code, i1 = (ic >> 61) & 0x1L; i41 = (ic >> 22) & 0x1ffffffffffL; i20 = ic & 0xfffffL; - assert((tm & ~1) == TM_M_L_X_ && + if (!((tm & ~1) == TM_M_L_X_ && (s2 & 0xfL<<37) == (0xcL<<37) && - s0 == nop_m); + s0 == nop_m)) + goto short_jump; s1 = i41; s2 &= (0xcL<<37)|(0x7L<<33)|(1L<<12); s2 |= (i1<<36)|(i20<<13); break; default: + short_jump: /* Only B1 in slot 0 expected due to need to either * a stop to update predicates, or a sync before * unconditional short branch */ diff --git a/parallel-rsp/lightning/lib/jit_ia64-fpu.c b/parallel-rsp/lightning/lib/jit_ia64-fpu.c index 19cc381..f0fb32c 100644 --- a/parallel-rsp/lightning/lib/jit_ia64-fpu.c +++ b/parallel-rsp/lightning/lib/jit_ia64-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/lib/jit_ia64-sz.c b/parallel-rsp/lightning/lib/jit_ia64-sz.c index 59826d9..e1d973c 100644 --- a/parallel-rsp/lightning/lib/jit_ia64-sz.c +++ b/parallel-rsp/lightning/lib/jit_ia64-sz.c @@ -1,11 +1,11 @@ - #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 224 +#define JIT_INSTR_MAX 608 0, /* data */ 0, /* live */ - 0, /* align */ + 48, /* align */ 0, /* save */ 0, /* load */ + 16, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -14,7 +14,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -22,8 +25,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 16, /* va_start */ 32, /* va_arg */ 32, /* va_arg_d */ @@ -95,14 +110,21 @@ 32, /* nei */ 16, /* movr */ 16, /* movi */ + 16, /* movnr */ + 16, /* movzr */ + 48, /* casr */ + 64, /* casi */ 16, /* extr_c */ 16, /* extr_uc */ 16, /* extr_s */ 16, /* extr_us */ 16, /* extr_i */ 16, /* extr_ui */ - 64, /* htonr_us */ - 160, /* htonr_ui */ + 32, /* bswapr_us */ + 32, /* bswapr_ui */ + 16, /* bswapr_ul */ + 32, /* htonr_us */ + 32, /* htonr_ui */ 16, /* htonr_ul */ 16, /* ldr_c */ 32, /* ldi_c */ @@ -193,13 +215,37 @@ 32, /* callr */ 48, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -399,4 +445,8 @@ 0, /* movi_d_ww */ 16, /* movr_d_w */ 32, /* movi_d_w */ + 608, /* clo */ + 544, /* clz */ + 608, /* cto */ + 544, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_ia64.c b/parallel-rsp/lightning/lib/jit_ia64.c index 9207d81..2968278 100644 --- a/parallel-rsp/lightning/lib/jit_ia64.c +++ b/parallel-rsp/lightning/lib/jit_ia64.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -52,11 +52,13 @@ extern void __clear_cache(void *, void *); #define PROTO 1 # include "jit_ia64-cpu.c" # include "jit_ia64-fpu.c" +# include "jit_fallback.c" #undef PROTO /* * Initialization */ +jit_cpu_t jit_cpu; jit_register_t _rvs[] = { /* Always 0 */ { 0, "r0" }, @@ -238,6 +240,11 @@ jit_register_t _rvs[] = { void jit_get_cpu(void) { + jit_word_t clz = -1; + __asm__ volatile("tf.nz.unc p6,p7=32;(p6)mov %0=1;(p7)mov %0=0" + : "=r" (clz)); + assert(clz == 0 || clz == 1); + jit_cpu.clz = clz; } void @@ -344,18 +351,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); + jit_code_inc_synth_w(code, u); jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -409,9 +416,10 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - assert(u->code == jit_code_arg || - u->code == jit_code_arg_f || u->code == jit_code_arg_d); - return (jit_arg_reg_p(u->u.w)); + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) + return (jit_arg_reg_p(u->u.w)); + assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); + return (jit_arg_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8)); } void @@ -441,18 +449,22 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { offset = _jitc->function->self.size; _jitc->function->self.size += sizeof(jit_word_t); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -507,7 +519,7 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, _R32 + v->u.w); @@ -519,7 +531,7 @@ _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, _R32 + v->u.w); @@ -531,7 +543,7 @@ _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, _R32 + v->u.w); @@ -543,7 +555,7 @@ _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, _R32 + v->u.w); @@ -555,7 +567,7 @@ _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_i(u, _R32 + v->u.w); @@ -567,7 +579,7 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, _R32 + v->u.w); @@ -579,7 +591,7 @@ _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, _R32 + v->u.w); @@ -589,10 +601,10 @@ _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(_R32 + v->u.w, u); else @@ -601,11 +613,11 @@ _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(_R32 + v->u.w, u); else { @@ -712,10 +724,10 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(_OUT0 + _jitc->function->call.argi, u); @@ -729,11 +741,11 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(_OUT0 + _jitc->function->call.argi, u); @@ -972,6 +984,7 @@ _emit_code(jit_state_t *_jit) struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1116,9 +1129,14 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - /* nothing done */ + assert(!(node->u.w & (node->u.w - 1))); + sync(); + if (node->u.w > 8) + nop(node->u.w - 8); + break; + case jit_code_skip: + sync(); + nop((node->u.w + 7) & ~7); break; case jit_code_note: case jit_code_name: sync(); @@ -1175,6 +1193,20 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh, _u); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1196,6 +1228,9 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _us); case_rr(hton, _ui); case_rr(hton, _ul); + case_rr(bswap, _us); + case_rr(bswap, _ui); + case_rr(bswap, _ul); case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); @@ -1489,7 +1524,12 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (word >= -16777216 && word <= 16777215) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } @@ -1518,6 +1558,7 @@ _emit_code(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1556,6 +1597,16 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1584,14 +1635,21 @@ _emit_code(jit_state_t *_jit) case jit_code_va_arg_d: vaarg_d(rn(node->u.w), rn(node->v.w)); break; - case jit_code_live: - case jit_code_arg: case jit_code_ellipsis: + case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_l: case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1599,10 +1657,22 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_i: case jit_code_getarg_ui: case jit_code_getarg_l: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1688,6 +1758,7 @@ _emit_code(jit_state_t *_jit) #define CODE 1 # include "jit_ia64-cpu.c" # include "jit_ia64-fpu.c" +# include "jit_fallback.c" #undef CODE void diff --git a/parallel-rsp/lightning/lib/jit_loongarch-cpu.c b/parallel-rsp/lightning/lib/jit_loongarch-cpu.c new file mode 100644 index 0000000..ab05852 --- /dev/null +++ b/parallel-rsp/lightning/lib/jit_loongarch-cpu.c @@ -0,0 +1,2706 @@ +/* + * Copyright (C) 2022-2023 Free Software Foundation, Inc. + * + * This file is part of GNU lightning. + * + * GNU lightning is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 3, or (at your option) + * any later version. + * + * GNU lightning is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#if PROTO +# define ii(i) *_jit->pc.ui++ = (i) +# define can_sign_extend_si12_p(s12) ((s12) <= 2047 && (s12) >= -2048) +# define can_zero_extend_u12_p(u12) ((u12) <= 4095 && (u12) >= 0) +# define can_sign_extend_si16_p(s16) ((s16) <= 32767 && (s16) >= -32768) +# define can_sign_extend_si21_p(s21) ((s21) <= 1048575 && (s21) >= -1048576) +# define can_sign_extend_si26_p(s26) \ + ((s26) <= 33554431 && (s26) >= -33554432) +# define can_sign_extend_si32_p(s32) \ + ((s32) <= 2147483647LL && (s32) >= -2147483648LL) +# define _ZERO_REGNO 0 +# define _RA_REGNO 1 +# define _SP_REGNO 3 +# define _FP_REGNO 22 +# define ldr(u, v) ldr_l(u, v) +# define ldi(u, v) ldi_l(u, v) +# define ldxi(u, v, w) ldxi_l(u, v, w) +# define sti(u, v) sti_l(u, v) +# define stxi(u, v, w) stxi_l(u, v, w) +# define orrr(op, rk, rj, rd) _orrr(_jit, op, rk, rj, rd) +# define ou5rr(op, i5, rj, rd) _orrr(_jit, op, i5, rj, rd) +# define orru5(op, rk, rj, i5) _orrr(_jit, op, rk, rj, i5) +static void _orrr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou2rrr(op, i2, rk, rj, rd) _ou2rrr(_jit, op, i2, rk, rj, rd) +static void _ou2rrr(jit_state_t*,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou3rrr(op, u3, rk, rj, rd) _ou3rrr(_jit, op, u3, rk, rj, rd) +static void _ou3rrr(jit_state_t*,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou6rr(op, u6, rj, rd) _ou6rr(_jit, op, u6, rj, rd) +static void _ou6rr(jit_state_t*,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t); +# define ou5u1u5rr(op,m5,u1,l5,rj,rd) _ou5u1u5rr(_jit,op,m5,u1,l5,rj,rd) +static void _ou5u1u5rr(jit_state_t*,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou6u6rr(op, m6, l6, rj, rd) _ou6u6rr(_jit, op, m6, l6, rj, rd) +static void _ou6u6rr(jit_state_t*,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define o5r23(op, i5, rj, i2, rd) _o5r23(_jit, op, i5, rj, i2, rd) +static void _o5r23(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define o523r(op, i5, i2, rj, i3) _o523r(_jit, op, i5, i2, rj, i3) +static void _o523r(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define os12rr(op, i12, rj, rd) _os12rr(_jit, op, i12, rj, rd) +# define os12ru5(op, i12, rj, u5) _os12rr(_jit, op, i12, rj, u5) +static void _os12rr(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou12rr(op, u12, rj, rd) _ou12rr(_jit, op, u12, rj, rd) +static void _ou12rr(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou14u5r(op, u14, u5, rd) _ou14u5r(_jit, op, u14, u5, rd) +static void _osu14u5r(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define os14rr(op, s14, rj, rd) _os14rr(_jit, op, s14, rj, rd) +static void _os14rr(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou8rr(op, u8, rd, rj) _ou8rr(_jit, op, u8, rd, rj) +# define ou8u5r(op, u8, u5, rj) _ou8rr(_jit, op, u8, u5, rj) +static void _ou8rr(jit_state_t*,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t); +# define ou15(op, u15) _ou15(_jit, op, u15) +static void _ou15(jit_state_t*, jit_int32_t,jit_int32_t); +# define orrrr(op, ra, rk, rj, rd) _orrrr(_jit, op, ra, rk, rj, rd) +static void _orrrr(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou5rru2u3(op,u5,rj,rk,u2,u3) _ou5rru2u3(_jit, op, u5, rj, rk, u2, u3) +static void _ou5rru2u3(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t); +# define os16rr(op, s16, rj, rd) _os16rr(_jit, op, s16, rj, rd) +static void _os16rr(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define os20r(op, s20, rd) _os20r(_jit, op, s20, rd) +static void _os20r(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define orj21(op, rj, j21) _orj21(_jit, op, rj, j21) +static void _orj21(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define ou2u3j21(op, u2, u3, j21) _ou2u3j21(_jit, op, u2, u3, j21) +static void _o2cj21(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define oj16rr(op, j16, rj, rd) _oj16rr(_jit, op, j16, rj, rd) +static void _oj16rr(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define oj26(op, j26) _oj26(_jit, op, j26) +static void _oj26(jit_state_t*, jit_int32_t,jit_int32_t); +# define CLO_W(rd, rj) ou5rr(0x000, 0x04, rj, rd) +# define CLZ_W(rd, rj) ou5rr(0x000, 0x05, rj, rd) +# define CTO_W(rd, rj) ou5rr(0x000, 0x06, rj, rd) +# define CTZ_W(rd, rj) ou5rr(0x000, 0x07, rj, rd) +# define CLO_D(rd, rj) ou5rr(0x000, 0x08, rj, rd) +# define CLZ_D(rd, rj) ou5rr(0x000, 0x09, rj, rd) +# define CTO_D(rd, rj) ou5rr(0x000, 0x0a, rj, rd) +# define CTZ_D(rd, rj) ou5rr(0x000, 0x0b, rj, rd) +# define REVB_2H(rd, rj) ou5rr(0x000, 0x0c, rj, rd) +# define REVB_4H(rd, rj) ou5rr(0x000, 0x0d, rj, rd) +# define REVB_2W(rd, rj) ou5rr(0x000, 0x0e, rj, rd) +# define REVB_D(rd, rj) ou5rr(0x000, 0x0f, rj, rd) +# define REVH_2W(rd, rj) ou5rr(0x000, 0x10, rj, rd) +# define REVH_D(rd, rj) ou5rr(0x000, 0x11, rj, rd) +# define BITREV_4B(rd, rj) ou5rr(0x000, 0x12, rj, rd) +# define BITREV_8B(rd, rj) ou5rr(0x000, 0x13, rj, rd) +# define BITREV_W(rd, rj) ou5rr(0x000, 0x14, rj, rd) +# define BITREV_D(rd, rj) ou5rr(0x000, 0x15, rj, rd) +# define EXT_W_H(rd, rj) ou5rr(0x000, 0x16, rj, rd) +# define EXT_W_B(rd, rj) ou5rr(0x000, 0x17, rj, rd) +# define RDTIMEL_W(rd, rj) ou5rr(0x000, 0x18, rj, rd) +# define RDTIMEH_W(rd, rj) ou5rr(0x000, 0x19, rj, rd) +# define RDTIME_D(rd, rj) ou5rr(0x000, 0x20, rj, rd) +# define CPUCFG(rd, rj) ou5rr(0x000, 0x21, rj, rd) +# define ASRTLE_D( rj, rk) orru5(0x002, rk, rj, 0x00) +# define ASRTGT_D( rj, rk) orru5(0x003, rk, rj, 0x00) +# define ALSL_W(rd, rj, rk, sa2) ou2rrr(0x002, sa2, rk, rj, rd) +# define ALSL_WU(rd, rj, rk, sa2) ou2rrr(0x003, sa2, rk, rj, rd) +# define BYTEPICK_W(rd, rj, rk, sa2) ou2rrr(0x004, sa2, rk, rj, rd) +# define BYTEPICK_D(rd, rj, rk, sa3) ou3rrr(0x003, sa3, rk, rj, rd) +# define ADD_W(rd, rj, rk) orrr(0x020, rk, rj, rd) +# define ADD_D(rd, rj, rk) orrr(0x021, rk, rj, rd) +# define SUB_W(rd, rj, rk) orrr(0x022, rk, rj, rd) +# define SUB_D(rd, rj, rk) orrr(0x023, rk, rj, rd) +# define SLT(rd, rj, rk) orrr(0x024, rk, rj, rd) +# define SLTU(rd, rj, rk) orrr(0x025, rk, rj, rd) +# define MASKEQZ(rd, rj, rk) orrr(0x026, rk, rj, rd) +# define MASKNEZ(rd, rj, rk) orrr(0x027, rk, rj, rd) +# define NOR(rd, rj, rk) orrr(0x028, rk, rj, rd) +# define AND(rd, rj, rk) orrr(0x029, rk, rj, rd) +# define OR(rd, rj, rk) orrr(0x02a, rk, rj, rd) +# define XOR(rd, rj, rk) orrr(0x02b, rk, rj, rd) +# define ORN(rd, rj, rk) orrr(0x02c, rk, rj, rd) +# define ANDN(rd, rj, rk) orrr(0x02d, rk, rj, rd) +# define SLL_W(rd, rj, rk) orrr(0x02e, rk, rj, rd) +# define SRL_W(rd, rj, rk) orrr(0x02f, rk, rj, rd) +# define SRA_W(rd, rj, rk) orrr(0x030, rk, rj, rd) +# define SLL_D(rd, rj, rk) orrr(0x031, rk, rj, rd) +# define SRL_D(rd, rj, rk) orrr(0x032, rk, rj, rd) +# define SRA_D(rd, rj, rk) orrr(0x033, rk, rj, rd) +# define ROTR_W(rd, rj, rk) orrr(0x036, rk, rj, rd) +# define ROTR_D(rd, rj, rk) orrr(0x037, rk, rj, rd) +# define MUL_W(rd, rj, rk) orrr(0x038, rk, rj, rd) +# define MULH_W(rd, rj, rk) orrr(0x039, rk, rj, rd) +# define MULH_WU(rd, rj, rk) orrr(0x03a, rk, rj, rd) +# define MUL_D(rd, rj, rk) orrr(0x03b, rk, rj, rd) +# define MULH_D(rd, rj, rk) orrr(0x03c, rk, rj, rd) +# define MULH_DU(rd, rj, rk) orrr(0x03d, rk, rj, rd) +# define MULW_D_W(rd, rj, rk) orrr(0x03e, rk, rj, rd) +# define MULW_D_WU(rd, rj, rk) orrr(0x03f, rk, rj, rd) +# define DIV_W(rd, rj, rk) orrr(0x040, rk, rj, rd) +# define MOD_W(rd, rj, rk) orrr(0x041, rk, rj, rd) +# define DIV_WU(rd, rj, rk) orrr(0x042, rk, rj, rd) +# define MOD_WU(rd, rj, rk) orrr(0x043, rk, rj, rd) +# define DIV_D(rd, rj, rk) orrr(0x044, rk, rj, rd) +# define MOD_D(rd, rj, rk) orrr(0x045, rk, rj, rd) +# define DIV_DU(rd, rj, rk) orrr(0x046, rk, rj, rd) +# define MOD_DU(rd, rj, rk) orrr(0x047, rk, rj, rd) +# define CRC_W_B_W(rd, rj, rk) orrr(0x048, rk, rj, rd) +# define CRC_W_H_W(rd, rj, rk) orrr(0x049, rk, rj, rd) +# define CRC_W_W_W(rd, rj, rk) orrr(0x04a, rk, rj, rd) +# define CRC_W_D_W(rd, rj, rk) orrr(0x04b, rk, rj, rd) +# define CRCC_W_B_W(rd, rj, rk) orrr(0x04c, rk, rj, rd) +# define CRCC_W_H_W(rd, rj, rk) orrr(0x04d, rk, rj, rd) +# define CCRC_W_W_W(rd, rj, rk) orrr(0x04e, rk, rj, rd) +# define CCRC_W_D_W(rd, rj, rk) orrr(0x04f, rk, rj, rd) +# define BREAK(code) ou15(0x054, code) +# define DBCL(code) ou15(0x055, code) +# define SYSCALL(code) ou15(0x056, code) +# define ALSL_D(rd, rj, rk, sa2) ou2rrr(0x016, sa2, rk, rj, rd) +# define SLLI_W(rd, rj, ui5) ou5rr(0x081, ui5, rj, rd) +# define SLLI_D(rd, rj, ui6) ou6rr(0x041, ui6, rj, rd) +# define SRLI_W(rd, rj, ui5) ou5rr(0x089, ui5, rj, rd) +# define SRLI_D(rd, rj, ui6) ou6rr(0x045, ui6, rj, rd) +# define SRAI_W(rd, rj, ui5) ou5rr(0x091, ui5, rj, rd) +# define SRAI_D(rd, rj, ui6) ou6rr(0x049, ui6, rj, rd) +# define ROTRI_W(rd, rj, ui5) ou5rr(0x099, ui5, rj, rd) +# define ROTRI_D(rd, rj, ui6) ou6rr(0x04d, ui6, rj, rd) +# define BSTRINS_W(rd, rj, m5, l5) ou5u1u5rr(0x003, m5, 0x0, l5, rj, rd) +# define BSTRPICK_W(rd, rj, m5, l5) ou5u1u5rr(0x003, m5, 0x1, l5, rj, rd) +# define BSTRINS_D(rd, rj, m6, l6) ou6u6rr(0x002, m6, l6, rj, rd) +# define BSTRPICK_D(rd, rj, m6, l6) ou6u6rr(0x003, m6, l6, rj, rd) +# define SLTI(rd, rj, i12) os12rr(0x008, i12, rj, rd) +# define SLTUI(rd, rj, i12) os12rr(0x009, i12, rj, rd) +# define ADDI_W(rd, rj, si12) os12rr(0x00a, si12, rj, rd) +# define ADDI_D(rd, rj, si12) os12rr(0x00b, si12, rj, rd) +# define LU52I_D(rd, rj, i12) os12rr(0x00c, i12, rj, rd) +# define ANDI(rd, rj, i12) ou12rr(0x00d, i12, rj, rd) +# define ORI(rd, rj, i12) ou12rr(0x00e, i12, rj, rd) +# define XORI(rd, rj, i12) ou12rr(0x00f, i12, rj, rd) +# define CSRRD(rd, csr) ou14u5r(0x004, csr, 0x00, rd) +# define CSRWR(rd, csr) ou14u5r(0x004, csr, 0x01, rd) +# define CSRXCHG(rd, rj, csr) ou14u5r(0x004, csr, rj, rd) +# define CACOP(i5, rj, si12) os12ru5(0x018, si12, rj, i5) +# define LDDIR(rd, rj, level) ou8rr(0x190, level, rj, rd) +# define LDPTE( rj, level) ou8u5r(0x191, level, rj, 0x00) +# define IOCSRRD_B(rd, rj) ou5rr(0xc90, 0x00, rj, rd) +# define IOCSRRD_H(rd, rj) ou5rr(0xc90, 0x01, rj, rd) +# define IOCSRRD_W(rd, rj) ou5rr(0xc90, 0x02, rj, rd) +# define IOCSRRD_D(rd, rj) ou5rr(0xc90, 0x03, rj, rd) +# define IOCSRWR_B(rd, rj) ou5rr(0xc90, 0x04, rj, rd) +# define IOCSRWR_H(rd, rj) ou5rr(0xc90, 0x05, rj, rd) +# define IOCSRWR_W(rd, rj) ou5rr(0xc90, 0x06, rj, rd) +# define IOCSRWR_D(rd, rj) ou5rr(0xc90, 0x07, rj, rd) +# define TLBCLR() ii( 0x6482000) +# define TLBFLUSH() ii( 0x6482400) +# define TLBSRCH() ii( 0x6482800) +# define TLBRD() ii( 0x6482c00) +# define TLBWR() ii( 0x6483000) +# define TLBFILL() ii( 0x6483400) +# define ERTN() ii( 0x6483800) +# define IDLE(level) ou15(0xc91, level) +# define INVTLB(op, rj, rk) orru5(0xc93, rk, rj, op) +# define ADDU16I_D(rd, rj, si16) os16rr(0x004, si16, rj, rd) +# define LU12I_W(rd, si20) os20r(0x00a, si20, rd) +# define LU32I_D(rd, si20) os20r(0x00b, si20, rd) +# define PCADDI(rd, si20) os20r(0x00c, si20, rd) +# define PCALAU12I(rd, si20) os20r(0x00d, si20, rd) +# define PCADDU12I(rd, si20) os20r(0x00e, si20, rd) +# define PCADDU18I(rd, si20) os20r(0x00f, si20, rd) +# define LL_W(rd, rj, si14) os14rr(0x020, si14, rj, rd) +# define SC_W(rd, rj, si14) os14rr(0x021, si14, rj, rd) +# define LL_D(rd, rj, si14) os14rr(0x022, si14, rj, rd) +# define SC_D(rd, rj, si14) os14rr(0x023, si14, rj, rd) +# define LDPTR_W(rd, rj, si14) os14rr(0x024, si14, rj, rd) +# define SDPTR_W(rd, rj, si14) os14rr(0x025, si14, rj, rd) +# define LDPTR_D(rd, rj, si14) os14rr(0x026, si14, rj, rd) +# define SDPTR_D(rd, rj, si14) os14rr(0x027, si14, rj, rd) +# define LD_B(rd, rj, si12) os12rr(0x0a0, si12, rj, rd) +# define LD_H(rd, rj, si12) os12rr(0x0a1, si12, rj, rd) +# define LD_W(rd, rj, si12) os12rr(0x0a2, si12, rj, rd) +# define LD_D(rd, rj, si12) os12rr(0x0a3, si12, rj, rd) +# define ST_B(rd, rj, si12) os12rr(0x0a4, si12, rj, rd) +# define ST_H(rd, rj, si12) os12rr(0x0a5, si12, rj, rd) +# define ST_W(rd, rj, si12) os12rr(0x0a6, si12, rj, rd) +# define ST_D(rd, rj, si12) os12rr(0x0a7, si12, rj, rd) +# define LD_BU(rd, rj, si12) os12rr(0x0a8, si12, rj, rd) +# define LD_HU(rd, rj, si12) os12rr(0x0a9, si12, rj, rd) +# define LD_WU(rd, rj, si12) os12rr(0x0aa, si12, rj, rd) +# define PRELD(hint, rj, si12) os12ru5(0x0ab, si12, rj , hint) +# define LDX_B(rd, rj, rk) orrr(0x7000, rk, rj, rd) +# define LDX_H(rd, rj, rk) orrr(0x7008, rk, rj, rd) +# define LDX_W(rd, rj, rk) orrr(0x7010, rk, rj, rd) +# define LDX_D(rd, rj, rk) orrr(0x7018, rk, rj, rd) +# define STX_B(rd, rj, rk) orrr(0x7020, rk, rj, rd) +# define STX_H(rd, rj, rk) orrr(0x7028, rk, rj, rd) +# define STX_W(rd, rj, rk) orrr(0x7030, rk, rj, rd) +# define STX_D(rd, rj, rk) orrr(0x7038, rk, rj, rd) +# define LDX_BU(rd, rj, rk) orrr(0x7040, rk, rj, rd) +# define LDX_HU(rd, rj, rk) orrr(0x7048, rk, rj, rd) +# define LDX_WU(rd, rj, rk) orrr(0x7050, rk, rj, rd) +# define PRELDX(hint, rj, rk) orru5(0x7058, rk, rj, hint) +# define AMSWAP_W(rd, rj, rk) orrr(0x70c0, rk, rj, rd) +# define AMSWAP_D(rd, rj, rk) orrr(0x70c1, rk, rj, rd) +# define AMADD_W(rd, rj, rk) orrr(0x70c2, rk, rj, rd) +# define AMADD_D(rd, rj, rk) orrr(0x70c3, rk, rj, rd) +# define AMAND_W(rd, rj, rk) orrr(0x70c4, rk, rj, rd) +# define AMAND_D(rd, rj, rk) orrr(0x70c5, rk, rj, rd) +# define AMOR_W(rd, rj, rk) orrr(0x70c6, rk, rj, rd) +# define AMOR_D(rd, rj, rk) orrr(0x70c7, rk, rj, rd) +# define AMXOR_W(rd, rj, rk) orrr(0x70c8, rk, rj, rd) +# define AMXOR_D(rd, rj, rk) orrr(0x70c9, rk, rj, rd) +# define AMMAX_W(rd, rj, rk) orrr(0x70ca, rk, rj, rd) +# define AMMAX_D(rd, rj, rk) orrr(0x70cb, rk, rj, rd) +# define AMMIN_W(rd, rj, rk) orrr(0x70cc, rk, rj, rd) +# define AMMIN_D(rd, rj, rk) orrr(0x70cd, rk, rj, rd) +# define AMMAX_WU(rd, rj, rk) orrr(0x70ce, rk, rj, rd) +# define AMMAX_DU(rd, rj, rk) orrr(0x70cf, rk, rj, rd) +# define AMMIN_WU(rd, rj, rk) orrr(0x70d0, rk, rj, rd) +# define AMMIN_DU(rd, rj, rk) orrr(0x70d1, rk, rj, rd) +# define AMSWAP_DB_W(rd, rj, rk) orrr(0x70d2, rk, rj, rd) +# define AMSWAP_DB_D(rd, rj, rk) orrr(0x70d3, rk, rj, rd) +# define AMADD_DB_W(rd, rj, rk) orrr(0x70d4, rk, rj, rd) +# define AMADD_DB_D(rd, rj, rk) orrr(0x70d5, rk, rj, rd) +# define AMAND_DB_W(rd, rj, rk) orrr(0x70d6, rk, rj, rd) +# define AMAND_DB_D(rd, rj, rk) orrr(0x70d7, rk, rj, rd) +# define AMOR_DB_W(rd, rj, rk) orrr(0x70d8, rk, rj, rd) +# define AMOR_DB_D(rd, rj, rk) orrr(0x70d9, rk, rj, rd) +# define AMXOR_DB_W(rd, rj, rk) orrr(0x70da, rk, rj, rd) +# define AMXOR_DB_D(rd, rj, rk) orrr(0x70db, rk, rj, rd) +# define AMMAX_DB_W(rd, rj, rk) orrr(0x70dc, rk, rj, rd) +# define AMMAX_DB_D(rd, rj, rk) orrr(0x70dd, rk, rj, rd) +# define AMMIN_DB_W(rd, rj, rk) orrr(0x70de, rk, rj, rd) +# define AMMIN_DB_D(rd, rj, rk) orrr(0x70df, rk, rj, rd) +# define AMMAX_DB_WU(rd, rj, rk) orrr(0x70e0, rk, rj, rd) +# define AMMAX_DB_DU(rd, rj, rk) orrr(0x70e1, rk, rj, rd) +# define AMMIN_DB_WU(rd, rj, rk) orrr(0x70e2, rk, rj, rd) +# define AMMIN_DB_DU(rd, rj, rk) orrr(0x70e3, rk, rj, rd) +# define DBAR(hint) ou15(0x70e4, hint) +# define IBAR(hint) ou15(0x70e5, hint) +# define LDGT_B(rd, rj, rk) orrr(0x70f0, rk, rj, rd) +# define LDGT_H(rd, rj, rk) orrr(0x70f1, rk, rj, rd) +# define LDGT_W(rd, rj, rk) orrr(0x70f2, rk, rj, rd) +# define LDGT_D(rd, rj, rk) orrr(0x70f3, rk, rj, rd) +# define LDLE_B(rd, rj, rk) orrr(0x70f4, rk, rj, rd) +# define LDLE_H(rd, rj, rk) orrr(0x70f5, rk, rj, rd) +# define LDLE_W(rd, rj, rk) orrr(0x70f6, rk, rj, rd) +# define LDLE_D(rd, rj, rk) orrr(0x70f7, rk, rj, rd) +# define STGT_B(rd, rj, rk) orrr(0x70f8, rk, rj, rd) +# define STGT_H(rd, rj, rk) orrr(0x70f9, rk, rj, rd) +# define STGT_W(rd, rj, rk) orrr(0x70fa, rk, rj, rd) +# define STGT_D(rd, rj, rk) orrr(0x70fb, rk, rj, rd) +# define STLE_B(rd, rj, rk) orrr(0x70fc, rk, rj, rd) +# define STLE_H(rd, rj, rk) orrr(0x70rd, rk, rj, rd) +# define STLE_W(rd, rj, rk) orrr(0x70fe, rk, rj, rd) +# define STLE_D(rd, rj, rk) orrr(0x70ff, rk, rj, rd) +# define BEQZ( rj, offs) orj21(0x010, rj, offs) +# define BNEZ( rj, offs) orj21(0x011, rj, offs) +# define BCEQZ( cj, offs) ou2u3j21(0x012, 0x0, cj, offs) +# define BCNEZ( cj, offs) ou2u3j21(0x012, 0x1, cj, offs) +# define JIRL(rd, rj, offs) oj16rr(0x013, offs, rj, rd) +# define B(offs) oj26(0x014, offs) +# define BL(offs) oj26(0x015, offs) +# define BEQ(rj, rd, offs) oj16rr(0x016, offs, rj, rd) +# define BNE(rj, rd, offs) oj16rr(0x017, offs, rj, rd) +# define BLT(rj, rd, offs) oj16rr(0x018, offs, rj, rd) +# define BGE(rj, rd, offs) oj16rr(0x019, offs, rj, rd) +# define BLTU(rj, rd, offs) oj16rr(0x01a, offs, rj, rd) +# define BGEU(rj, rd, offs) oj16rr(0x01b, offs, rj, rd) +# define NOP() ANDI(_ZERO_REGNO, _ZERO_REGNO, 0) +# define nop(i0) _nop(_jit, i0) +# define comr(r0, r1) NOR(r0, r1, r1) +# define negr(r0, r1) subr(r0, _ZERO_REGNO, r1) +# define clor(r0, r1) CLO_D(r0, r1) +# define clzr(r0, r1) CLZ_D(r0, r1) +# define ctor(r0, r1) CTO_D(r0, r1) +# define ctzr(r0, r1) CTZ_D(r0, r1) +static void _nop(jit_state_t*,jit_int32_t); +# define movr(r0, r1) _movr(_jit, r0, r1) +static void _movr(jit_state_t*, jit_int32_t, jit_int32_t); +# define movi(r0, i0) _movi(_jit, r0, i0) +static void _movi(jit_state_t*, jit_int32_t, jit_word_t); +# define movi_p(r0, i0) _movi_p(_jit, r0, i0) +static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t); +# define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2) +static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2) +static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) +# define addr(r0, r1, r2) ADD_D(r0, r1, r2) +# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0) +static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2) +static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define addci(r0, r1, i0) _addci(_jit, r0, r1, i0) +static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2) +static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0) +static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define subr(r0, r1, r2) SUB_D(r0, r1, r2) +# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0) +static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2) +static void _subcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0) +static void _subci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2) +static void _subxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0) +static void _subxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0) +static void _rsbi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define mulr(r0, r1, r2) MUL_D(r0, r1, r2) +# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0) +static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define qmulr(r0, r1, r2, r3) iqmulr(r0, r1, r2, r3, 1) +# define qmulr_u(r0, r1, r2, r3) iqmulr(r0, r1, r2, r3, 0) +# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign) +static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t, + jit_int32_t, jit_int32_t, jit_bool_t); +# define qmuli(r0, r1, r2, i0) iqmuli(r0, r1, r2, i0, 1) +# define qmuli_u(r0, r1, r2, i0) iqmuli(r0, r1, r2, i0, 0) +# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign) +static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t, + jit_int32_t, jit_word_t, jit_bool_t); +# define divr(r0, r1, r2) DIV_D(r0, r1, r2) +# define divi(r0, r1, i0) _divi(_jit, r0, r1, i0) +static void _divi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define divr_u(r0, r1, r2) DIV_DU(r0, r1, r2) +# define divi_u(r0, r1, i0) _divi_u(_jit, r0, r1, i0) +static void _divi_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define qdivr(r0, r1, r2, r3) iqdivr(r0, r1, r2, r3, 1) +# define qdivr_u(r0, r1, r2, r3) iqdivr(r0, r1, r2, r3, 0) +# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign) +static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t, + jit_int32_t, jit_int32_t, jit_bool_t); +# define qdivi(r0, r1, r2, i0) iqdivi(r0, r1, r2, i0, 1) +# define qdivi_u(r0, r1, r2, i0) iqdivi(r0, r1, r2, i0, 0) +# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign) +static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t, + jit_int32_t, jit_word_t, jit_bool_t); +# define remr(r0, r1, r2) MOD_D(r0, r1, r2) +# define remi(r0, r1, i0) _remi(_jit, r0, r1, i0) +static void _remi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define remr_u(r0, r1, r2) MOD_DU(r0, r1, r2) +# define remi_u(r0, r1, i0) _remi_u(_jit, r0, r1, i0) +static void _remi_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define lshr(r0, r1, r2) SLL_D(r0, r1, r2) +# define lshi(r0, r1, i0) SLLI_D(r0, r1, i0) +# define rshr(r0, r1, r2) SRA_D(r0, r1, r2) +# define rshi(r0, r1, i0) SRAI_D(r0, r1, i0) +# define rshr_u(r0, r1, r2) SRL_D(r0, r1, r2) +# define rshi_u(r0, r1, i0) SRLI_D(r0, r1, i0) +# define andr(r0, r1, r2) AND(r0, r1, r2) +# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0) +static void _andi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define orr(r0, r1, r2) OR(r0, r1, r2) +# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0) +static void _ori(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define xorr(r0, r1, r2) XOR(r0, r1, r2) +# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0) +static void _xori(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldr_c(r0, r1) LD_B(r0, r1, 0) +# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0) +static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_uc( r0, r1) LD_BU(r0, r1, 0) +# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0) +static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_s(r0, r1) LD_H(r0, r1, 0) +# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0) +static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_us(r0, r1) LD_HU(r0, r1, 0) +# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0) +static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_i(r0, r1) LD_W(r0, r1, 0) +# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0) +static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_ui(r0, r1) LD_WU(r0, r1, 0) +# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0) +static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t); +# define ldr_l(r0, r1) LD_D(r0, r1, 0) +# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0) +static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t); +# define ldxr_c(r0, r1, r2) LDX_B(r0, r1, r2) +# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0) +static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_uc(r0, r1, r2) LDX_BU(r0, r1, r2) +# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit,r0, r1, i0) +static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_s(r0, r1, r2) LDX_H(r0, r1, r2) +# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0) +static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_us(r0, r1, r2) LDX_HU(r0, r1, r2) +# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0) +static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_i(r0, r1, r2) LDX_W(r0, r1, r2) +# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0) +static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_ui(r0, r1, r2) LDX_WU(r0, r1, r2) +# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0) +static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ldxr_l(r0, r1, r2) LDX_D(r0, r1, r2) +# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0) +static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define str_c(r0, r1) ST_B(r1, r0, 0) +# define sti_c(i0, r0) _sti_c(_jit, i0, r0) +static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t); +# define str_s(r0, r1) ST_H(r1, r0, 0) +# define sti_s(i0, r0) _sti_s(_jit, i0, r0) +static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t); +# define str_i(r0, r1) ST_W(r1, r0, 0) +# define sti_i(i0, r0) _sti_i(_jit, i0, r0) +static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t); +# define str_l(r0, r1) ST_D(r1, r0, 0) +# define sti_l(i0, r0) _sti_l(_jit, i0, r0) +static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t); +# define stxr_c(r0, r1, r2) STX_B(r2, r1, r0) +# define stxi_c(i0, r0, r1) _stxi_c(_jit,i0, r0, r1) +static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define stxr_s(r0, r1, r2) STX_H(r2, r1, r0) +# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1) +static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define stxr_i(r0, r1, r2) STX_W(r2, r1, r0) +# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1) +static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define stxr_l(r0, r1, r2) STX_D(r2, r1, r0) +# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1) +static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1) +static void _bswapr_us(jit_state_t*, jit_int32_t, jit_int32_t); +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1) +static void _bswapr_ui(jit_state_t*, jit_int32_t, jit_int32_t); +# define bswapr_ul(r0, r1) REVB_D(r0, r1) +# define extr_c(r0, r1) EXT_W_B(r0, r1) +# define extr_uc(r0, r1) BSTRPICK_D(r0, r1, 7, 0) +# define extr_s(r0, r1) EXT_W_H(r0, r1) +# define extr_us(r0, r1) BSTRPICK_D(r0, r1, 15, 0) +# define extr_i(r0, r1) SLLI_W(r0, r1, 0) +# define extr_ui(r0, r1) BSTRPICK_D(r0, r1, 31, 0) +# define ltr(r0, r1, r2) SLT(r0, r1, r2) +# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0) +static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ltr_u(r0, r1, r2) SLTU(r0, r1, r2) +# define lti_u(r0, r1, i0) _lti_u(_jit, r0, r1, i0) +static void _lti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2) +static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lei(r0, r1, i0) _lei(_jit, r0, r1, i0) +static void _lei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2) +static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0) +static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2) +static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0) +static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2) +static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0) +static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2) +static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0) +static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define gtr(r0, r1, r2) SLT(r0, r2, r1) +# define gti(r0, r1, i0) _gti(_jit, r0, r1, i0) +static void _gti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define gtr_u(r0, r1, r2) SLTU(r0, r2, r1) +# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0) +static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2) +static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0) +static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1) +static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1) +static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1) +static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1) +static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1) +static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1) +static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1) +static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1) +static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1) +static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1) +static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1) +static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1) +static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1) +static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1) +static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1) +static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1) +static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1) +static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1) +static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1) +static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1) +static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define jmpr(r0) JIRL(_ZERO_REGNO, r0, 0) +# define jmpi(i0) _jmpi(_jit, i0) +static jit_word_t _jmpi(jit_state_t*, jit_word_t); +# define jmpi_p(i0) _jmpi_p(_jit, i0) +static jit_word_t _jmpi_p(jit_state_t*, jit_word_t); +# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1) +static jit_word_t _boaddr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1) +static jit_word_t _boaddi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1) +static jit_word_t _boaddr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1) +static jit_word_t _boaddi_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1) +static jit_word_t _bxaddr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1) +static jit_word_t _bxaddi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1) +static jit_word_t _bxaddr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1) +static jit_word_t _bxaddi_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1) +static jit_word_t _bosubr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1) +static jit_word_t _bosubi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1) +static jit_word_t _bosubr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1) +static jit_word_t _bosubi_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1) +static jit_word_t _bxsubr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1) +static jit_word_t _bxsubi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1) +static jit_word_t _bxsubr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1) +static jit_word_t _bxsubi_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t); +# define bmsr(br, r0, r1) _bmsr(_jit, br, r0, r1) +static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bmsi(br, r0, i0) _bmsi(_jit, br, r0, i0) +static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define bmcr(br, r0, r1) _bmcr(_jit, br, r0, r1) +static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); +# define bmci(br, r0, i0) _bmci(_jit, br, r0, i0) +static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); +# define callr(r0) JIRL(_RA_REGNO, r0, 0) +# define calli(i0) _calli(_jit, i0) +static jit_word_t _calli(jit_state_t*, jit_word_t); +# define calli_p(i0) _calli_p(_jit, i0) +static jit_word_t _calli_p(jit_state_t*, jit_word_t); +# define prolog(i0) _prolog(_jit, i0) +static void _prolog(jit_state_t*, jit_node_t*); +# define epilog(i0) _epilog(_jit, i0) +static void _epilog(jit_state_t*, jit_node_t*); +# define vastart(r0) _vastart(_jit, r0) +static void _vastart(jit_state_t*, jit_int32_t); +# define vaarg(r0, r1) _vaarg(_jit, r0, r1) +static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t); +#define patch_at(jump, label) _patch_at(_jit, jump, label) +static void _patch_at(jit_state_t*,jit_word_t,jit_word_t); +#endif + +#if CODE +static void +_orrr(jit_state_t *_jit, + jit_int32_t op, jit_int32_t rk, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0xffff)); + assert(!(rk & ~0x1f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 15) | (rk << 10) | (rj << 5) | rd); +} + +static void +_ou2rrr(jit_state_t *_jit, jit_int32_t op, + jit_int32_t u2, jit_int32_t rk, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x7ff)); + assert(!(u2 & ~3)); + assert(!(rk & ~0x1f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 17) | (u2 << 15) | (rk << 10) | (rj << 5) | rd); +} + +static void +_ou3rrr(jit_state_t *_jit, jit_int32_t op, + jit_int32_t u3, jit_int32_t rk, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3ff)); + assert(!(u3 & ~7)); + assert(!(rk & ~0x1f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 18) | (u3 << 15) | (rk << 10) | (rj << 5) | rd); +} + +static void +_ou6rr(jit_state_t *_jit, jit_int32_t op, + jit_int32_t u6, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3ff)); + assert(!(u6 & ~0x3f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 16) | (u6 << 10) | (rj << 5) | rd); +} + +static void +_ou5u1u5rr(jit_state_t *_jit, jit_int32_t op, jit_int32_t m5, + jit_int32_t u1, jit_int32_t l5, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x7ff)); + assert(!(m5 & ~0x1f)); + assert(!(u1 & ~1)); + assert(!(l5 & ~0x1f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 21) | (m5 << 16) | (u1 << 15) | (l5 << 10) | (rj << 5) | rd); +} + +static void +_ou6u6rr(jit_state_t *_jit, jit_int32_t op, + jit_int32_t m6, jit_int32_t l6, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3ff)); + assert(!(m6 & ~0x3f)); + assert(!(l6 & ~0x3f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 22) | (m6 << 16) | (l6 << 10) | (rj << 5) | rd); +} + +static void +_o5r23(jit_state_t *_jit, jit_int32_t op, + jit_int32_t i5, jit_int32_t rj, jit_int32_t i2, jit_int32_t i3) +{ + assert(!(op & ~0xffff)); + assert(!(i5 & ~0x3f)); + assert(!(rj & ~0x3f)); + assert(!(i2 & ~0x3)); + assert(!(i3 & ~0x1f)); + ii((op << 15) | (i5 << 10) | (rj << 5) | (i2 << 3) | i3); +} + +static void +_o523r(jit_state_t *_jit, jit_int32_t op, + jit_int32_t i5, jit_int32_t i2, jit_int32_t i3, jit_int32_t rd) +{ + assert(!(op & ~0xffff)); + assert(!(i5 & ~0x3f)); + assert(!(i2 & ~0x3)); + assert(!(i3 & ~0x7)); + assert(!(rd & ~0x3f)); + ii((op << 15) | (i5 << 10) | (i2 << 8) | (i3 << 5) | rd); +} + +static void +_os12rr(jit_state_t *_jit, + jit_int32_t op, jit_int32_t s12, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3ff)); + assert(s12 <= 2047 && s12 >= -2048); s12 &= 0xfff; + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 22) | (s12 << 10) | (rj << 5) | rd); +} + +static void +_ou12rr(jit_state_t *_jit, + jit_int32_t op, jit_int32_t u12, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3ff)); + assert(!(u12 & ~0xfff)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 22) | (u12 << 10) | (rj << 5) | rd); +} + +static void +_ou14u5r(jit_state_t *_jit, + jit_int32_t op, jit_int32_t u14, jit_int32_t u5, jit_int32_t rd) +{ + assert(!(op & ~0xff)); + assert(!(u14 & ~0x3fff)); + assert(!(u5 & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 24) | (u14 << 10) | (u5 << 5) | rd); +} + +static void +_os14rr(jit_state_t *_jit, + jit_int32_t op, jit_int32_t s14, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0xff)); + assert(s14 <= 8191 && s14 >= -8192); s14 &= 0x3fff; + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 24) | (s14 << 10) | (rj << 5) | rd); +} + +static void +_ou8rr(jit_state_t *_jit, jit_int32_t op, + jit_int32_t u8, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3fff)); + assert(!(u8 & ~0xff)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 18) | (u8 << 10) | (rj << 5) | rd); +} + +static void +_ou15(jit_state_t *_jit, jit_int32_t op, jit_int32_t u15) +{ + assert(!(op & ~0x1ffff)); + assert(!(u15 & ~0x7fff)); + ii((op << 15) | u15); +} + +static void +_orrrr(jit_state_t *_jit, jit_int32_t op, + jit_int32_t ra, jit_int32_t rk, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0xfff)); + assert(!(ra & ~0x1f)); + assert(!(rk & ~0x1f)); + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 20) | (ra << 15) | (rk << 10) | (rj << 5) | rd); +} + +static void +_ou5rru2u3(jit_state_t *_jit,jit_int32_t op, jit_int32_t u5, + jit_int32_t rk, jit_int32_t rj, jit_int32_t u2, jit_int32_t u3) +{ + assert(!(op & ~0xfff)); + assert(!(u5 & ~0x1f)); + assert(!(rk & ~0x1f)); + assert(!(rj & ~0x1f)); + assert(!(u2 & ~3)); + assert(!(u3 & ~7)); + ii((op << 20) | (u5 << 15) | (rk << 10) | (rj << 5) | (u2 << 3) | u3); +} + +static void +_os16rr(jit_state_t *_jit, + jit_int32_t op, jit_int32_t s16, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x3f)); + assert(s16 <= 32767 && s16 >= -32768); s16 &= 0xffff; + assert(!(rj & ~0x1f)); + assert(!(rd & ~0x1f)); + ii((op << 26) | (s16 << 10) | (rj << 5) | rd); +} + +static void +_os20r(jit_state_t *_jit, jit_int32_t op, jit_int32_t s20, jit_int32_t rd) +{ + assert(!(op & ~0x7f)); + assert(s20 <= 524287 && s20 >= -524288); s20 &= 0xfffff; + assert(!(rd & ~0x1f)); + ii((op << 25) | (s20 << 5) | rd); +} + +static void +_orj21(jit_state_t *_jit, jit_int32_t op, jit_int32_t rj ,jit_int32_t j21) +{ + assert(!(op & ~0x7f)); + assert(j21 <= 1048575 && j21 >= -1048576); j21 &= 0x1fffff; + assert(!(rj & ~0x1f)); + ii((op << 26) | ((j21 & 0xffff) << 10) | (rj << 5) | (j21 >> 16)); +} + +static void +_ou2u3j21(jit_state_t *_jit, + jit_int32_t op, jit_int32_t u2, jit_int32_t u3, jit_int32_t j21) +{ + assert(!(op & ~0x7f)); + assert(j21 <= 1048575 && j21 >= -1048576); j21 &= 0x1fffff; + assert(!(u2 & ~3)); + assert(!(u3 & ~7)); + ii((op << 26) | ((j21 & 0xffff) << 10) | (u2 << 8) | (u3 << 5) | (j21 >> 16)); +} + +static void +_oj16rr(jit_state_t *_jit, + jit_int32_t op, jit_int32_t j16, jit_int32_t rj, jit_int32_t rd) +{ + assert(!(op & ~0x7f)); + assert(j16 <= 32767 && j16 >= -32768); j16 &= 0xffff; + assert(!(rd & ~0x1f)); + assert(!(rj & ~0x1f)); + ii((op << 26) | (j16 << 10) | (rj << 5) | rd); +} + +static void +_oj26(jit_state_t *_jit, jit_int32_t op, jit_int32_t j26) +{ + assert(!(op & ~0x7f)); + assert(j26 <= 33554431 && j26 >= -33554432); j26 &= 0x3ffffff; + ii((op << 26) | ((j26 & 0x1ffffff) << 10) | (j26 >> 16)); +} + +static void +_nop(jit_state_t *_jit, jit_int32_t i0) +{ + for (; i0 > 0; i0 -= 4) + NOP(); + assert(i0 == 0); +} + +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + MASKEQZ(rn(reg), r1, r2); + MASKNEZ(r0, r0, r2); + OR(r0, r0, rn(reg)); + jit_unget_reg(reg); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + MASKNEZ(rn(reg), r1, r2); + MASKEQZ(r0, r0, r2); + OR(r0, r0, rn(reg)); + jit_unget_reg(reg); +} + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t t0, r1_reg, iscasi; + jit_word_t retry, done, jump0, jump1; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + t0 = jit_get_reg(jit_class_gpr); + movi(r0, 0); /* Assume will fail */ + DBAR(0); /* Barrier */ + retry = _jit->pc.w; + LL_D(rn(t0), r1, 0); /* Load current value */ + jump0 = _jit->pc.w; + BNE(rn(t0), r2, 0); /* If not equal, already done and r0 is zero */ + movr(r0, r3); /* Make r0 an inout argument */ + SC_D(r0, r1, 0); /* r0 is 0 if failed, 1 if succeed */ + jump1 = _jit->pc.w; + BEQ(r0, _ZERO_REGNO, 0); + /* FIXME Not certain what 0x700 hint means. Copied from C generated code */ + DBAR(0x700); + done = _jit->pc.w; + patch_at(jump0, done); + patch_at(jump1, retry); + jit_unget_reg(t0); + if (iscasi) + jit_unget_reg(r1_reg); +} + +static void +_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 != r1) + OR(r0, r1, _ZERO_REGNO); +} + +static void +_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + if (i0 == 0) + OR(r0, _ZERO_REGNO, _ZERO_REGNO); + else if (can_sign_extend_si12_p(i0)) + ADDI_D(r0, _ZERO_REGNO, i0); + else if (!(i0 & 0xffff) && can_sign_extend_si16_p(i0 >> 16)) + ADDU16I_D(r0, _ZERO_REGNO, i0 >> 16); + else { + jit_word_t w = i0 - _jit->pc.w; + /* If loading some constant reachable address */ + if (can_sign_extend_si32_p(w)) { + jit_int32_t lo = (jit_int32_t)w << 20 >> 20; + jit_int32_t hi = w - lo; + PCADDU12I(r0, hi >> 12); + if (lo) + ADDI_D(r0, r0, lo); + } + else { + jit_int32_t _00_11, _12_31, _32_51, _52_63; + _00_11 = i0 & 0xfff; + _12_31 = (i0 >> 12) & 0xfffff; + _32_51 = (i0 >> 32) & 0xfffff; + _52_63 = (i0 >> 52) & 0xfff; + if (_12_31) { + LU12I_W(r0, _12_31 << 12 >> 12); + if (_00_11) + ORI(r0, r0, _00_11); + } + else + ORI(r0, _ZERO_REGNO, _00_11); + if (_32_51 || (_12_31 & 0x80000)) + LU32I_D(r0, _32_51 << 12 >> 12); + if (_52_63 || (_12_31 & 0x80000) || (_32_51 & 0x80000)) + LU52I_D(r0, r0, _52_63 << 20 >> 20); + } + } +} + +static jit_word_t +_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_word_t w; + jit_int32_t _00_11, _12_31, _32_51, _52_63; + _00_11 = i0 & 0xfff; + _12_31 = (i0 >> 12) & 0xfffff; + _32_51 = (i0 >> 32) & 0xfffff; + _52_63 = (i0 >> 52) & 0xfff; + w = _jit->pc.w; + LU12I_W(r0, _12_31 << 12 >> 12); + ORI(r0, r0, _00_11); + LU32I_D(r0, _32_51 << 12 >> 12); + LU52I_D(r0, r0, _52_63 << 20 >> 20); + return (w); +} + +static void +_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + movr(r0, r1); + else if (can_sign_extend_si12_p(i0)) + ADDI_D(r0, r1, i0); + else if (!(i0 & 0xffff) && can_sign_extend_si16_p(i0 >> 16)) + ADDU16I_D(r0, r1, i0 >> 16); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + addr(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t t0; + if (jit_carry == _NOREG) + jit_carry = jit_get_reg(jit_class_gpr); + if (r0 == r1) { + t0 = jit_get_reg(jit_class_gpr); + addr(rn(t0), r1, r2); + SLTU(rn(jit_carry), rn(t0), r1); + movr(r0, rn(t0)); + jit_unget_reg(t0); + } + else { + addr(r0, r1, r2); + SLTU(rn(jit_carry), r0, r1); + } +} + +static void +_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t t0; + if (jit_carry == _NOREG) + jit_carry = jit_get_reg(jit_class_gpr); + t0 = jit_get_reg(jit_class_gpr); + if (r0 == r1) { + if (can_sign_extend_si12_p(i0)) + ADDI_D(rn(t0), r1, i0); + else if (!(i0 & 0xffff) && can_sign_extend_si16_p(i0 >> 16)) + ADDU16I_D(rn(t0), r1, i0 >> 16); + else { + movi(rn(t0), i0); + addr(rn(t0), r1, rn(t0)); + } + SLTU(rn(jit_carry), rn(t0), r1); + movr(r0, rn(t0)); + } + else { + if (can_sign_extend_si12_p(i0)) + ADDI_D(r0, r1, i0); + else if (!(i0 & 0xffff) && can_sign_extend_si16_p(i0 >> 16)) + ADDU16I_D(r0, r1, i0 >> 16); + else { + movi(rn(t0), i0); + addr(r0, r1, rn(t0)); + } + SLTU(rn(jit_carry), r0, r1); + } + jit_unget_reg(t0); +} + +static void +_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t t0; + assert(jit_carry != _NOREG); + t0 = jit_get_reg(jit_class_gpr); + movr(rn(t0), rn(jit_carry)); + addcr(r0, r1, r2); + addcr(r0, r0, rn(t0)); + jit_unget_reg(t0); +} + +static void +_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t t0; + assert(jit_carry != _NOREG); + t0 = jit_get_reg(jit_class_gpr); + movr(rn(t0), rn(jit_carry)); + addci(r0, r1, i0); + addcr(r0, r0, rn(t0)); + jit_unget_reg(t0); +} + +static void +_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + movr(r0, r1); + else if (can_sign_extend_si12_p(-i0)) + ADDI_D(r0, r1, -i0); + else if (!(-i0 & 0xffff) && can_sign_extend_si16_p(-i0 >> 16)) + ADDU16I_D(r0, r1, -i0 >> 16); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + subr(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t t0; + if (jit_carry == _NOREG) + jit_carry = jit_get_reg(jit_class_gpr); + if (r0 == r1) { + t0 = jit_get_reg(jit_class_gpr); + subr(rn(t0), r1, r2); + SLTU(rn(jit_carry), r1, rn(t0)); + movr(r0, rn(t0)); + jit_unget_reg(t0); + } + else { + subr(r0, r1, r2); + SLTU(rn(jit_carry), r1, r0); + } +} + +static void +_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t t0; + if (jit_carry == _NOREG) + jit_carry = jit_get_reg(jit_class_gpr); + t0 = jit_get_reg(jit_class_gpr); + if (r0 == r1) { + if (can_sign_extend_si12_p(-i0)) + ADDI_D(rn(t0), r1, -i0); + else if (!(-i0 & 0xffff) && can_sign_extend_si16_p(-i0 >> 16)) + ADDU16I_D(rn(t0), r1, -i0 >> 16); + else { + movi(rn(t0), i0); + subr(rn(t0), r1, rn(t0)); + } + SLTU(rn(jit_carry), r1, rn(t0)); + movr(r0, rn(t0)); + } + else { + if (can_sign_extend_si12_p(-i0)) + ADDI_D(r0, r1, -i0); + else if (!(-i0 & 0xffff) && can_sign_extend_si16_p(-i0 >> 16)) + ADDU16I_D(r0, r1, -i0 >> 16); + else { + movi(rn(t0), i0); + subr(r0, r1, rn(t0)); + } + SLTU(rn(jit_carry), r1, r0); + } + jit_unget_reg(t0); +} + +static void +_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_int32_t t0; + assert(jit_carry != _NOREG); + t0 = jit_get_reg(jit_class_gpr); + movr(rn(t0), rn(jit_carry)); + subcr(r0, r1, r2); + subcr(r0, r0, rn(t0)); + jit_unget_reg(t0); +} + +static void +_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t t0; + assert(jit_carry != _NOREG); + t0 = jit_get_reg(jit_class_gpr); + movr(rn(t0), rn(jit_carry)); + subci(r0, r1, i0); + subcr(r0, r0, rn(t0)); + jit_unget_reg(t0); +} + +static void +_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + subi(r0, r1, i0); + negr(r0, r0); +} + +static void +_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + mulr(r0, r1, rn(reg)); + jit_unget_reg(reg); +} + +static void +_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_bool_t sign) +{ + jit_int32_t t0; + if (r0 == r2 || r0 == r3) { + t0 = jit_get_reg(jit_class_gpr); + mulr(rn(t0), r2, r3); + } + else + mulr(r0, r2, r3); + if (sign) + MULH_D(r1, r2, r3); + else + MULH_DU(r1, r2, r3); + if (r0 == r2 || r0 == r3) { + movr(r0, rn(t0)); + jit_unget_reg(t0); + } +} + +static void +_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_word_t i0, jit_bool_t sign) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iqmulr(r0, r1, r2, rn(reg), sign); + jit_unget_reg(reg); +} + +static void +_divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + divr(r0, r1, rn(reg)); + jit_unget_reg(reg); +} + +static void +_divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + divr_u(r0, r1, rn(reg)); + jit_unget_reg(reg); +} + +static void +_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_bool_t sign) +{ + jit_int32_t t0; + if (r0 == r2 || r0 == r3) + t0 = jit_get_reg(jit_class_gpr); + else + t0 = _NOREG; + if (sign) { + if (t0 == _NOREG) + DIV_D(r0, r2, r3); + else + DIV_D(rn(t0), r2, r3); + MOD_D(r1, r2, r3); + } + else { + if (t0 == _NOREG) + DIV_DU(r0, r2, r3); + else + DIV_DU(rn(t0), r2, r3); + MOD_DU(r1, r2, r3); + } + if (t0 != _NOREG) { + movr(r0, rn(t0)); + jit_unget_reg(t0); + } +} + +static void +_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_word_t i0, jit_bool_t sign) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + iqdivr(r0, r1, r2, rn(reg), sign); + jit_unget_reg(reg); +} + +static void +_remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + remr(r0, r1, rn(reg)); + jit_unget_reg(reg); +} + +static void +_remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + remr_u(r0, r1, rn(reg)); + jit_unget_reg(reg); +} + +static void +_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_zero_extend_u12_p(i0)) + ANDI(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + AND(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_zero_extend_u12_p(i0)) + ORI(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + OR(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_zero_extend_u12_p(i0)) + XORI(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + XOR(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_B(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_B(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_BU(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_BU(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_H(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_H(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_HU(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_HU(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_W(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_W(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_WU(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_WU(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_D(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LD_D(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_B(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_B(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_BU(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_BU(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_H(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_H(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_HU(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_HU(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_W(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_W(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_WU(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_WU(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + LD_D(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + LDX_D(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_B(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ST_B(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_H(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ST_H(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_W(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ST_W(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_D(r0, _ZERO_REGNO, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ST_D(r0, rn(reg), 0); + jit_unget_reg(reg); + } +} + +static void +_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_B(r1, r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + STX_B(r1, r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_H(r1, r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + STX_H(r1, r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_W(r1, r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + STX_W(r1, r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + ST_D(r1, r0, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + STX_D(r1, r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + REVB_2H(r0, r1); + extr_us(r0, r0); +} + +static void +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + REVB_2W(r0, r1); + extr_ui(r0, r0); +} + +static void +_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + SLTI(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ltr(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) + SLTUI(r0, r1, i0); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ltr_u(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + SLT(r0, r2, r1); + XORI(r0, r0, 1); +} + +static void +_lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0 + 1)) + SLTI(r0, r1, i0 + 1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ler(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + SLTU(r0, r2, r1); + XORI(r0, r0, 1); +} + +static void +_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0 + 1)) + SLTUI(r0, r1, i0 + 1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ler_u(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + subr(r0, r1, r2); + SLTU(r0, _ZERO_REGNO, r0); + XORI(r0, r0, 1); +} + +static void +_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) { + subi(r0, r1, i0); + SLTU(r0, _ZERO_REGNO, r0); + } + else + SLTU(r0, _ZERO_REGNO, r1); + XORI(r0, r0, 1); +} + +static void +_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + SLT(r0, r1, r2); + XORI(r0, r0, 1); +} + +static void +_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) { + SLTI(r0, r1, i0); + XORI(r0, r0, 1); + } else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ger(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + SLTU(r0, r1, r2); + XORI(r0, r0, 1); +} + +static void +_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (can_sign_extend_si12_p(i0)) { + SLTUI(r0, r1, i0); + XORI(r0, r0, 1); + } else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ger_u(r0, r1, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + SLT(r0, _ZERO_REGNO, r1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + SLT(r0, rn(reg), r1); + jit_unget_reg(reg); + } +} + +static void +_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t reg; + if (i0 == 0) + SLTU(r0, _ZERO_REGNO, r1); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + SLTU(r0, rn(reg), r1); + jit_unget_reg(reg); + } +} + +static void +_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + subr(r0, r1, r2); + SLTU(r0, _ZERO_REGNO, r0); +} + +static void +_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + if (i0) { + subi(r0, r1, i0); + SLTU(r0, _ZERO_REGNO, r0); + } + else + SLTU(r0, _ZERO_REGNO, r1); +} + +static jit_word_t +_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + w = _jit->pc.w; + BLT(r0, r1, (i0 - w) >> 2); + return (w); +} + +static jit_word_t +_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) { + w = _jit->pc.w; + BLT(r0, _ZERO_REGNO, (i0 - w) >> 2); + } + if (i1) { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = _jit->pc.w; + BLT(r0, rn(reg), (i0 - w) >> 2); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + w = _jit->pc.w; + BLTU(r0, r1, (i0 - w) >> 2); + return (w); +} + +static jit_word_t +_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) + w = bltr_u(i0, r0, _ZERO_REGNO); + else { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = _jit->pc.w; + BLTU(r0, rn(reg), (i0 - w) >> 2); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + SLT(rn(reg), r1, r0); + w = _jit->pc.w; + BEQZ(rn(reg), (i0 - w) >> 2); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bler(i0, r0, rn(reg)); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + SLTU(rn(reg), r1, r0); + w = _jit->pc.w; + BEQZ(rn(reg), (i0 - w) >> 2); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) + w = bler_u(i0, r0, _ZERO_REGNO); + else { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bler_u(i0, r0, rn(reg)); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + w = _jit->pc.w; + BEQ(r0, r1, (i0 - w) >> 2); + return (w); +} + +static jit_word_t +_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) { + w = _jit->pc.w; + BEQZ(r0, (i0 - w) >> 2); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = beqr(i0, r0, rn(reg)); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + w = _jit->pc.w; + BGE(r0, r1, (i0 - w) >> 2); + return (w); +} + +static jit_word_t +_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) + w = bger(i0, r0, _ZERO_REGNO); + else { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bger(i0, r0, rn(reg)); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + w = _jit->pc.w; + BGEU(r0, r1, (i0 - w) >> 2); + return (w); +} + +static jit_word_t +_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) { + w = _jit->pc.w; + B((i0 - w) >> 2); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bger_u(i0, r0, rn(reg)); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + SLT(rn(reg), r1, r0); + w = _jit->pc.w; + BNEZ(rn(reg), (i0 - w) >> 2); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bgtr(i0, r0, rn(reg)); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + SLTU(rn(reg), r1, r0); + w = _jit->pc.w; + BNEZ(rn(reg), (i0 - w) >> 2); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bgtr_u(i0, r0, rn(reg)); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + w = _jit->pc.w; + BNE(r0, r1, (i0 - w) >> 2); + return (w); +} + +static jit_word_t +_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t reg; + if (i1 == 0) { + w = _jit->pc.w; + BNEZ(r0, (i0 - w) >> 2); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(reg), i1); + w = bner(i0, r0, rn(reg)); + jit_unget_reg(reg); + } + return (w); +} + +static jit_word_t +_jmpi(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 2; + if (can_sign_extend_si26_p(i0)) + B(d); + else + w = jmpi_p(i0); + return (w); +} + +static jit_word_t +_jmpi_p(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + w = movi_p(rn(reg), i0); + jmpr(rn(reg)); + jit_unget_reg(reg); + return (w); +} + +static jit_word_t +_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w, b; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + /* t0 = r1 < 0; */ + SLT(rn(t0), r1, _ZERO_REGNO); + /* t1 = r0 */ + movr(rn(t1), r0); + /* r0 = r0 + r1 */ + addr(r0, r0, r1); + /* overflow = r1 < 0 ? t1 < r0 : r0 < t1 */ + w = _jit->pc.w; + BNEZ(rn(t0), 0); + /* r1 >= 0 */ + SLT(rn(t1), r0, rn(t1)); + b = _jit->pc.w; + B(0); + /* r1 < 0 */ + patch_at(w, _jit->pc.w); + SLT(rn(t1), rn(t1), r0); + /**/ + patch_at(b, _jit->pc.w); + w = _jit->pc.w; + BNEZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = boaddr(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + addr(rn(t0), r0, r1); + SLTU(rn(t1), rn(t0), r0); + movr(r0, rn(t0)); + w = _jit->pc.w; + BNEZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = boaddr_u(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w, b; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + /* t0 = r1 < 0; */ + SLT(rn(t0), r1, _ZERO_REGNO); + /* t1 = r0 */ + movr(rn(t1), r0); + /* r0 = r0 + r1 */ + addr(r0, r0, r1); + /* overflow = r1 < 0 ? t1 < r0 : r0 < t1 */ + w = _jit->pc.w; + BNEZ(rn(t0), 0); + /* r1 >= 0 */ + SLT(rn(t1), r0, rn(t1)); + b = _jit->pc.w; + B(0); + /* r1 < 0 */ + patch_at(w, _jit->pc.w); + SLT(rn(t1), rn(t1), r0); + /**/ + patch_at(b, _jit->pc.w); + w = _jit->pc.w; + BEQZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = bxaddr(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + addr(rn(t0), r0, r1); + SLTU(rn(t1), rn(t0), r0); + movr(r0, rn(t0)); + w = _jit->pc.w; + BEQZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = bxaddr_u(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w, b; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + /* t0 = 0 < r1; */ + SLT(rn(t0), _ZERO_REGNO, r1); + /* t1 = r0 */ + movr(rn(t1), r0); + /* r0 = r0 - r1 */ + subr(r0, r0, r1); + /* overflow = r1 < 0 ? t1 < r0 : r0 < t1 */ + w = _jit->pc.w; + BNE(rn(t0), _ZERO_REGNO, 0); + /* r1 >= 0 */ + SLT(rn(t1), r0, rn(t1)); + b = _jit->pc.w; + B(0); + /* r1 < 0 */ + patch_at(w, _jit->pc.w); + SLT(rn(t1), rn(t1), r0); + /**/ + patch_at(b, _jit->pc.w); + w = _jit->pc.w; + BNEZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = bosubr(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + subr(rn(t0), r0, r1); + SLTU(rn(t1), r0, rn(t0)); + movr(r0, rn(t0)); + w = _jit->pc.w; + BNEZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bosubi_u(jit_state_t *_jit, jit_word_t br, jit_int32_t r0, jit_word_t i0) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i0); + w = bosubr_u(br, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w, b; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + /* t0 = r1 < 0; */ + SLT(rn(t0), _ZERO_REGNO, r1); + /* t1 = r0 */ + movr(rn(t1), r0); + /* r0 = r0 - r1 */ + subr(r0, r0, r1); + /* overflow = r1 < 0 ? t1 < r0 : r0 < t1 */ + w = _jit->pc.w; + BNEZ(rn(t0), 0); + /* r1 >= 0 */ + SLT(rn(t1), r0, rn(t1)); + b = _jit->pc.w; + B(0); + /* r1 < 0 */ + patch_at(w, _jit->pc.w); + SLT(rn(t1), rn(t1), r0); + /**/ + patch_at(b, _jit->pc.w); + w = _jit->pc.w; + BEQZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = bxsubr(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0, t1; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + t1 = jit_get_reg(jit_class_gpr|jit_class_nospill); + subr(rn(t0), r0, r1); + SLTU(rn(t1), r0, rn(t0)); + movr(r0, rn(t0)); + w = _jit->pc.w; + BEQZ(rn(t1), (i0 - w) >> 2); + jit_unget_reg(t1); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + movi(rn(t0), i1); + w = bxsubr_u(i0, r0, rn(t0)); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + AND(rn(t0), r0, r1); + w = _jit->pc.w; + BNEZ(rn(t0), (i0 - w) >> 2); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + andi(rn(t0), r0, i1); + w = _jit->pc.w; + BNEZ(rn(t0), (i0 - w) >> 2); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + AND(rn(t0), r0, r1); + w = _jit->pc.w; + BEQZ(rn(t0), (i0 - w) >> 2); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +{ + jit_word_t w; + jit_int32_t t0; + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); + andi(rn(t0), r0, i1); + w = _jit->pc.w; + BEQZ(rn(t0), (i0 - w) >> 2); + jit_unget_reg(t0); + return (w); +} + +static jit_word_t +_calli(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 2; + if (can_sign_extend_si26_p(i0)) + BL(d); + else + w = calli_p(i0); + return (w); +} + +static jit_word_t +_calli_p(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t w; + jit_word_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + w = movi_p(rn(reg), i0); + callr(rn(reg)); + jit_unget_reg(reg); + return (w); +} + +static void +_prolog(jit_state_t *_jit, jit_node_t *node) +{ + jit_int32_t reg, offs; + if (_jitc->function->define_frame || _jitc->function->assume_frame) { + jit_int32_t frame = -_jitc->function->frame; + jit_check_frame(); + assert(_jitc->function->self.aoff >= frame); + if (_jitc->function->assume_frame) + return; + _jitc->function->self.aoff = frame; + } + if (_jitc->function->allocar) + _jitc->function->self.aoff &= -16; + _jitc->function->stack = ((_jitc->function->self.alen - + /* align stack at 16 bytes */ + _jitc->function->self.aoff) + 15) & -16; + + if (_jitc->function->stack) + _jitc->function->need_stack = 1; + if (!_jitc->function->need_frame && !_jitc->function->need_stack) { + /* check if any callee save register needs to be saved */ + for (reg = 0; reg < _jitc->reglen; ++reg) + if (jit_regset_tstbit(&_jitc->function->regset, reg) && + (_rvs[reg].spec & jit_class_sav)) { + _jitc->function->need_stack = 1; + break; + } + } + + if (_jitc->function->need_frame || _jitc->function->need_stack) + subi(_SP_REGNO, _SP_REGNO, jit_framesize()); + if (_jitc->function->need_frame) { + stxi(0, _SP_REGNO, _RA_REGNO); + stxi(8, _SP_REGNO, _FP_REGNO); + } + /* callee save registers */ + for (reg = 0, offs = 16; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + stxi(offs, _SP_REGNO, rn(iregs[reg])); + offs += sizeof(jit_word_t); + } + } + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + stxi_d(offs, _SP_REGNO, rn(fregs[reg])); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->need_frame) + movr(_FP_REGNO, _SP_REGNO); + if (_jitc->function->stack) + subi(_SP_REGNO, _SP_REGNO, _jitc->function->stack); + if (_jitc->function->allocar) { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), _jitc->function->self.aoff); + stxi_i(_jitc->function->aoffoff, _FP_REGNO, rn(reg)); + jit_unget_reg(reg); + } + if (_jitc->function->self.call & jit_call_varargs) { + for (reg = _jitc->function->vagp; jit_arg_reg_p(reg); ++reg) + stxi(jit_framesize() - ((8 - reg) * 8), + _FP_REGNO, rn(JIT_RA0 - reg)); + } +} + +static void +_epilog(jit_state_t *_jit, jit_node_t *node) +{ + jit_int32_t reg, offs; + if (_jitc->function->assume_frame) + return; + if (_jitc->function->need_frame) { + movr(_SP_REGNO, _FP_REGNO); + ldxi(_RA_REGNO, _SP_REGNO, 0); + ldxi(_FP_REGNO, _SP_REGNO, 8); + } + + /* callee save registers */ + for (reg = 0, offs = 16; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + ldxi(rn(iregs[reg]), _SP_REGNO, offs); + offs += sizeof(jit_word_t); + } + } + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + ldxi_d(rn(fregs[reg]), _SP_REGNO, offs); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->need_frame || _jitc->function->need_stack) + addi(_SP_REGNO, _SP_REGNO, jit_framesize()); + JIRL(_ZERO_REGNO, _RA_REGNO, 0); +} + +static void +_vastart(jit_state_t *_jit, jit_int32_t r0) +{ + assert(_jitc->function->self.call & jit_call_varargs); + /* Initialize va_list to the first stack argument. */ + if (jit_arg_reg_p(_jitc->function->vagp)) + addi(r0, _FP_REGNO, jit_framesize() - ((8 - _jitc->function->vagp) * 8)); + else + addi(r0, _FP_REGNO, jit_selfsize()); +} + +static void +_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + assert(_jitc->function->self.call & jit_call_varargs); + /* Load argument. */ + ldr(r0, r1); + /* Update va_list. */ + addi(r1, r1, sizeof(jit_word_t)); +} + +static void +_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) +{ + jit_uint32_t c; + union { + jit_uint32_t *i; + jit_word_t w; + } u; + u.w = instr; + c = u.i[0]; + /* movi_p? */ + if ((c >> 25) == 10) { /* LU12I_W */ + jit_int32_t _00_11, _12_31, _32_51, _52_63; + _00_11 = label & 0xfff; + _12_31 = (label >> 12) & 0xfffff; + _32_51 = (label >> 32) & 0xfffff; + _52_63 = (label >> 52) & 0xfff; + u.i[0] &= ~(0xfffff << 5); + u.i[0] |= _12_31 << 5; + assert((u.i[1] >> 22) == 14); /* ORI */ + u.i[1] &= ~(0xfff << 10); + u.i[1] |= _00_11 << 10; + assert((u.i[2] >> 25) == 11); /* LU32I_D */ + u.i[2] &= ~(0xfffff << 5); + u.i[2] |= _32_51 << 5; + assert((u.i[3] >> 22) == 12); /* LU52I_D */ + u.i[3] &= ~(0xfff << 10); + u.i[3] |= _52_63 << 0; + } + else if ((c >> 26) >= 22 && (c >> 26) <= 27) { + /* B{EQ,NE,LT,GE,LTU,GEU} */ + jit_word_t disp = (label - instr) >> 2; + assert(can_sign_extend_si16_p(disp)); + u.i[0] &= ~(0xffff << 10); + u.i[0] |= (disp & 0xffff) << 10; + } + else if ((c >> 26) == 20 || (c >> 26) == 21) { /* B or BL */ + jit_word_t disp = (label - instr) >> 2; + assert(can_sign_extend_si26_p(disp)); + disp &= 0x3ffffff; + u.i[0] &= ~0x3ffffff; + u.i[0] |= ((disp & 0xffff) << 10) | (disp >> 16); + } + else if ((c >> 26) >= 16 && (c >> 26) <= 18) { /* B{,C}{EQ,NE}Z */ + jit_word_t disp = (label - instr) >> 2; + assert(can_sign_extend_si21_p(disp)); + disp &= 0x1fffff; + u.i[0] &= ~((0xffff << 10) | 0x1f); + u.i[0] |= ((disp & 0xffff) << 10) | (disp >> 16); + } + else + abort(); +} +#endif diff --git a/parallel-rsp/lightning/lib/jit_loongarch-fpu.c b/parallel-rsp/lightning/lib/jit_loongarch-fpu.c new file mode 100644 index 0000000..2871de3 --- /dev/null +++ b/parallel-rsp/lightning/lib/jit_loongarch-fpu.c @@ -0,0 +1,1318 @@ +/* + * Copyright (C) 2022-2023 Free Software Foundation, Inc. + * + * This file is part of GNU lightning. + * + * GNU lightning is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 3, or (at your option) + * any later version. + * + * GNU lightning is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +#if PROTO +# define FADD_S(fd, fj, fk) orrr(0x201, fk, fj, fd) +# define FADD_D(fd, fj, fk) orrr(0x202, fk, fj, fd) +# define FSUB_S(fd, fj, fk) orrr(0x205, fk, fj, fd) +# define FSUB_D(fd, fj, fk) orrr(0x206, fk, fj, fd) +# define FMUL_S(fd, fj, fk) orrr(0x209, fk, fj, fd) +# define FMUL_D(fd, fj, fk) orrr(0x20a, fk, fj, fd) +# define FDIV_S(fd, fj, fk) orrr(0x20d, fk, fj, fd) +# define FDIV_D(fd, fj, fk) orrr(0x20e, fk, fj, fd) +# define FMAX_S(fd, fj, fk) orrr(0x211, fk, fj, fd) +# define FMAX_D(fd, fj, fk) orrr(0x212, fk, fj, fd) +# define FMIN_S(fd, fj, fk) orrr(0x215, fk, fj, fd) +# define FMIN_D(fd, fj, fk) orrr(0x216, fk, fj, fd) +# define FMAXA_S(fd, fj, fk) orrr(0x219, fk, fj, fd) +# define FMAXA_D(fd, fj, fk) orrr(0x21a, fk, fj, fd) +# define FMINA_S(fd, fj, fk) orrr(0x21d, fk, fj, fd) +# define FMINA_D(fd, fj, fk) orrr(0x21e, fk, fj, fd) +# define FSCALEB_S(fd, fj, fk) orrr(0x221, fk, fj, fd) +# define FSCALEB_D(fd, fj, fk) orrr(0x222, fk, fj, fd) +# define FCOPYSIGN_S(fd, fj, fk) orrr(0x225, fk, fj, fd) +# define FCOPYSIGN_D(fd, fj, fk) orrr(0x226, fk, fj, fd) +# define FABS_S(fd, fj) ou5rr(0x228, 0x01, fj, fd) +# define FABS_D(fd, fj) ou5rr(0x228, 0x02, fj, fd) +# define FNEG_S(fd, fj) ou5rr(0x228, 0x05, fj, fd) +# define FNEG_D(fd, fj) ou5rr(0x228, 0x06, fj, fd) +# define FGLOB_S(fd, fj) ou5rr(0x228, 0x09, fj, fd) +# define FGLOB_D(fd, fj) ou5rr(0x228, 0x0a, fj, fd) +# define FCLASS_S(fd, fj) ou5rr(0x228, 0x0d, fj, fd) +# define FCLASS_D(fd, fj) ou5rr(0x228, 0x0e, fj, fd) +# define FSQRT_S(fd, fj) ou5rr(0x228, 0x11, fj, fd) +# define FSQRT_D(fd, fj) ou5rr(0x228, 0x12, fj, fd) +# define FRECIP_S(fd, fj) ou5rr(0x228, 0x15, fj, fd) +# define FRECIP_D(fd, fj) ou5rr(0x228, 0x16, fj, fd) +# define FRSQRT_S(fd, fj) ou5rr(0x228, 0x19, fj, fd) +# define FRSQRT_D(fd, fj) ou5rr(0x228, 0x1a, fj, fd) +# define FMOV_S(fd, fj) ou5rr(0x229, 0x05, fj, fd) +# define FMOV_D(fd, fj) ou5rr(0x229, 0x06, fj, fd) +# define MOVGR2FR_W(fd, rj) ou5rr(0x229, 0x09, rj, fd) +# define MOVGR2FR_D(fd, rj) ou5rr(0x229, 0x0a, rj, fd) +# define MOVGR2FRH_W(fd, rj) ou5rr(0x229, 0x0b, rj, fd) +# define MOVFR2GR_S(rd, fj) ou5rr(0x229, 0x0d, fj, rd) +# define MOVFR2GR_D(rd, fj) ou5rr(0x229, 0x0e, fj, rd) +# define MOVFRH2GR_S(rd, fj) ou5rr(0x229, 0x0f, fj, rd) +# define MOVGR2FCSR(fc, rj) ou5rr(0x229, 0x10, rj, fc) +# define MOVFCSR2GR(rd, fc) ou5rr(0x229, 0x12, fc, rd) +# define MOVFR2CF(cd, fj) o5r23(0x229, 0x14, fj, 0, cd) +# define MOVCF2FR(fd, cj) o523r(0x229, 0x15, 0, cj, fd) +# define MOVGR2CF(cd, fj) o5r23(0x229, 0x16, fj, 0, cd) +# define MOVCF2GR(fd, cj) o523r(0x229, 0x17, 0, cj, fd) +# define FCVT_S_D(fd, fj) ou5rr(0x232, 0x06, fj, fd) +# define FCVT_D_S(fd, fj) ou5rr(0x232, 0x09, fj, fd) +# define FTINTRM_W_S(fd, fj) ou5rr(0x234, 0x01, fj, fd) +# define FTINTRM_W_D(fd, fj) ou5rr(0x234, 0x02, fj, fd) +# define FTINTRM_L_S(fd, fj) ou5rr(0x234, 0x09, fj, fd) +# define FTINTRM_L_D(fd, fj) ou5rr(0x234, 0x0a, fj, fd) +# define FTINTRP_W_S(fd, fj) ou5rr(0x234, 0x11, fj, fd) +# define FTINTRP_W_D(fd, fj) ou5rr(0x234, 0x12, fj, fd) +# define FTINTRP_L_S(fd, fj) ou5rr(0x234, 0x19, fj, fd) +# define FTINTRP_L_D(fd, fj) ou5rr(0x234, 0x1a, fj, fd) +# define FTINTRZ_W_S(fd, fj) ou5rr(0x235, 0x01, fj, fd) +# define FTINTRZ_W_D(fd, fj) ou5rr(0x235, 0x02, fj, fd) +# define FTINTRZ_L_S(fd, fj) ou5rr(0x235, 0x09, fj, fd) +# define FTINTRZ_L_D(fd, fj) ou5rr(0x235, 0x0a, fj, fd) +# define FTINTRNE_W_S(fd, fj) ou5rr(0x235, 0x11, fj, fd) +# define FTINTRNE_W_D(fd, fj) ou5rr(0x235, 0x12, fj, fd) +# define FTINTRNE_L_S(fd, fj) ou5rr(0x235, 0x19, fj, fd) +# define FTINTRNE_L_D(fd, fj) ou5rr(0x235, 0x1a, fj, fd) +# define FTINT_W_S(fd, fj) ou5rr(0x236, 0x01, fj, fd) +# define FTINT_W_D(fd, fj) ou5rr(0x236, 0x02, fj, fd) +# define FTINT_L_S(fd, fj) ou5rr(0x236, 0x09, fj, fd) +# define FTINT_L_D(fd, fj) ou5rr(0x236, 0x0a, fj, fd) +# define FFINT_S_W(fd, fj) ou5rr(0x23a, 0x04, fj, fd) +# define FFINT_S_L(fd, fj) ou5rr(0x23a, 0x06, fj, fd) +# define FFINT_D_W(fd, fj) ou5rr(0x23a, 0x08, fj, fd) +# define FFINT_D_L(fd, fj) ou5rr(0x23a, 0x0a, fj, fd) +# define FRINT_S(fd, fj) ou5rr(0x23c, 0x11, fj, fd) +# define FRINT_D(fd, fj) ou5rr(0x23c, 0x12, fj, fd) +# define FMADD_S(fd, fj, fk, fa) orrrr(0x081, fa, fk, fj, fd) +# define FMADD_D(fd, fj, fk, fa) orrrr(0x082, fa, fk, fj, fd) +# define FMSUB_S(fd, fj, fk, fa) orrrr(0x085, fa, fk, fj, fd) +# define FMSUB_D(fd, fj, fk, fa) orrrr(0x086, fa, fk, fj, fd) +# define FNMADD_S(fd, fj, fk, fa) orrrr(0x089, fa, fk, fj, fd) +# define FNMADD_D(fd, fj, fk, fa) orrrr(0x08a, fa, fk, fj, fd) +# define FNMSUB_S(fd, fj, fk, fa) orrrr(0x08d, fa, fk, fj, fd) +# define FNMSUB_D(fd, fj, fk, fa) orrrr(0x08e, fa, fk, fj, fd) +/* No QNaN exception */ +# define _CAF 0x00 +# define _CUN 0x08 +# define _CEQ 0x04 +# define _CUEQ 0x0c +# define _CLT 0x02 +# define _CULT 0x0a +# define _CLE 0x06 +# define _CULE 0x0e +# define _CNE 0x10 +# define _COR 0x14 +# define _CUNE 0x18 +/* QNaN exception */ +# define _SAF 0x01 +# define _SUN 0x09 +# define _SEQ 0x05 +# define _SUEQ 0x0D +# define _SLT 0x03 +# define _SULT 0x0B +# define _SLE 0x07 +# define _SULE 0x0F +# define _SNE 0x11 +# define _SOR 0x15 +# define _SUNE 0x19 +# define FCMP_cond_S(cc, cd, fj, fk) ou5rru2u3(0x0c1, cc, fk, fj, 0, cd) +# define FCMP_CAF_S(cd, fj, fk) FCMP_cond_S( _CAF, cd, fj, fk) +# define FCMP_CUN_S(cd, fj, fk) FCMP_cond_S( _CUN, cd, fj, fk) +# define FCMP_CEQ_S(cd, fj, fk) FCMP_cond_S( _CEQ, cd, fj, fk) +# define FCMP_CUEQ_S(cd, fj, fk) FCMP_cond_S(_CUEQ, cd, fj, fk) +# define FCMP_CLT_S(cd, fj, fk) FCMP_cond_S( _CLT, cd, fj, fk) +# define FCMP_CULT_S(cd, fj, fk) FCMP_cond_S(_CULT, cd, fj, fk) +# define FCMP_CLE_S(cd, fj, fk) FCMP_cond_S( _CLE, cd, fj, fk) +# define FCMP_CULE_S(cd, fj, fk) FCMP_cond_S(_CULE, cd, fj, fk) +# define FCMP_CNE_S(cd, fj, fk) FCMP_cond_S( _CNE, cd, fj, fk) +# define FCMP_COR_S(cd, fj, fk) FCMP_cond_S( _COR, cd, fj, fk) +# define FCMP_CUNE_S(cd, fj, fk) FCMP_cond_S(_CUNE, cd, fj, fk) +# define FCMP_SAF_S(cd, fj, fk) FCMP_cond_S( _CAF, cd, fj, fk) +# define FCMP_SUN_S(cd, fj, fk) FCMP_cond_S( _CUN, cd, fj, fk) +# define FCMP_SEQ_S(cd, fj, fk) FCMP_cond_S( _CEQ, cd, fj, fk) +# define FCMP_SUEQ_S(cd, fj, fk) FCMP_cond_S(_CUEQ, cd, fj, fk) +# define FCMP_SLT_S(cd, fj, fk) FCMP_cond_S( _CLT, cd, fj, fk) +# define FCMP_SULT_S(cd, fj, fk) FCMP_cond_S(_CULT, cd, fj, fk) +# define FCMP_SLE_S(cd, fj, fk) FCMP_cond_S( _CLE, cd, fj, fk) +# define FCMP_SULE_S(cd, fj, fk) FCMP_cond_S(_CULE, cd, fj, fk) +# define FCMP_SNE_S(cd, fj, fk) FCMP_cond_S( _CNE, cd, fj, fk) +# define FCMP_SOR_S(cd, fj, fk) FCMP_cond_S( _COR, cd, fj, fk) +# define FCMP_SUNE_S(cd, fj, fk) FCMP_cond_S(_CUNE, cd, fj, fk) +# define FCMP_cond_D(cc, cd, fj, fk) ou5rru2u3(0x0c2, cc, fk, fj, 0, cd) +# define FCMP_CAF_D(cd, fj, fk) FCMP_cond_D( _CAF, cd, fj, fk) +# define FCMP_CUN_D(cd, fj, fk) FCMP_cond_D( _CUN, cd, fj, fk) +# define FCMP_CEQ_D(cd, fj, fk) FCMP_cond_D( _CEQ, cd, fj, fk) +# define FCMP_CUEQ_D(cd, fj, fk) FCMP_cond_D(_CUEQ, cd, fj, fk) +# define FCMP_CLT_D(cd, fj, fk) FCMP_cond_D( _CLT, cd, fj, fk) +# define FCMP_CULT_D(cd, fj, fk) FCMP_cond_D(_CULT, cd, fj, fk) +# define FCMP_CLE_D(cd, fj, fk) FCMP_cond_D( _CLE, cd, fj, fk) +# define FCMP_CULE_D(cd, fj, fk) FCMP_cond_D(_CULE, cd, fj, fk) +# define FCMP_CNE_D(cd, fj, fk) FCMP_cond_D( _CNE, cd, fj, fk) +# define FCMP_COR_D(cd, fj, fk) FCMP_cond_D( _COR, cd, fj, fk) +# define FCMP_CUNE_D(cd, fj, fk) FCMP_cond_D(_CUNE, cd, fj, fk) +# define FCMP_SAF_D(cd, fj, fk) FCMP_cond_D( _CAF, cd, fj, fk) +# define FCMP_SUN_D(cd, fj, fk) FCMP_cond_D( _CUN, cd, fj, fk) +# define FCMP_SEQ_D(cd, fj, fk) FCMP_cond_D( _CEQ, cd, fj, fk) +# define FCMP_SUEQ_D(cd, fj, fk) FCMP_cond_D(_CUEQ, cd, fj, fk) +# define FCMP_SLT_D(cd, fj, fk) FCMP_cond_D( _CLT, cd, fj, fk) +# define FCMP_SULT_D(cd, fj, fk) FCMP_cond_D(_CULT, cd, fj, fk) +# define FCMP_SLE_D(cd, fj, fk) FCMP_cond_D( _CLE, cd, fj, fk) +# define FCMP_SULE_D(cd, fj, fk) FCMP_cond_D(_CULE, cd, fj, fk) +# define FCMP_SNE_D(cd, fj, fk) FCMP_cond_D( _CNE, cd, fj, fk) +# define FCMP_SOR_D(cd, fj, fk) FCMP_cond_D( _COR, cd, fj, fk) +# define FCMP_SUNE_D(cd, fj, fk) FCMP_cond_D(_CUNE, cd, fj, fk) +# define FSEL(fd, fj, fk, u3) ou3rrr(0x340, u3, fk, fj, fd) +# define FLD_S(fd, rj, si12) os12rr(0x0ac, si12, rj, fd) +# define FST_S(fd, rj, si12) os12rr(0x0ad, si12, rj, fd) +# define FLD_D(fd, rj, si12) os12rr(0x0ae, si12, rj, fd) +# define FST_D(fd, rj, si12) os12rr(0x0af, si12, rj, fd) +# define FLDX_S(fd, rj, rk) orrr(0x7060, rk, rj, fd) +# define FLDX_D(fd, rj, rk) orrr(0x7068, rk, rj, fd) +# define FSTX_S(fd, rj, rk) orrr(0x7070, rk, rj, fd) +# define FSTX_D(fd, rj, rk) orrr(0x7078, rk, rj, fd) +# define FLDGT_S(fd, rj, rk) orrr(0x70e8, rk, rj, fd) +# define FLDGT_D(fd, rj, rk) orrr(0x70e9, rk, rj, fd) +# define FLDLE_S(fd, rj, rk) orrr(0x70ea, rk, rj, fd) +# define FLDLE_D(fd, rj, rk) orrr(0x70eb, rk, rj, fd) +# define FSTGT_S(fd, rj, rk) orrr(0x70ec, rk, rj, fd) +# define FSTGT_D(fd, rj, rk) orrr(0x70ed, rk, rj, fd) +# define FSTLE_S(fd, rj, rk) orrr(0x70ee, rk, rj, fd) +# define FSTLE_D(fd, rj, rk) orrr(0x70ef, rk, rj, fd) +# define truncr_f_i(r0, r1) _truncr_f_i(_jit, r0, r1) +static void _truncr_f_i(jit_state_t*, jit_int32_t, jit_int32_t); +# define truncr_d_i(r0, r1) _truncr_d_i(_jit, r0, r1) +static void _truncr_d_i(jit_state_t*, jit_int32_t, jit_int32_t); +# define truncr_f_l(r0, r1) _truncr_f_l(_jit, r0, r1) +static void _truncr_f_l(jit_state_t*, jit_int32_t, jit_int32_t); +# define truncr_d_l(r0, r1) _truncr_d_l(_jit, r0, r1) +static void _truncr_d_l(jit_state_t*, jit_int32_t, jit_int32_t); +# define addr_f(r0, r1, r2) FADD_S(r0, r1, r2) +# define addi_f(r0, r1, i0) _addi_f(_jit, r0, r1, i0) +static void _addi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define subr_f(r0, r1, r2) FSUB_S(r0, r1, r2) +# define subi_f(r0, r1, i0) _subi_f(_jit, r0, r1, i0) +static void _subi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define rsbr_f(r0, r1, r2) FSUB_S(r0, r2, r1) +# define rsbi_f(r0, r1, i0) _rsbi_f(_jit, r0, r1, i0) +static void _rsbi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define mulr_f(r0, r1, r2) FMUL_S(r0, r1, r2) +# define muli_f(r0, r1, i0) _muli_f(_jit, r0, r1, i0) +static void _muli_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define divr_f(r0, r1, r2) FDIV_S(r0, r1, r2) +# define divi_f(r0, r1, i0) _divi_f(_jit, r0, r1, i0) +static void _divi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define absr_f(r0, r1) FABS_S(r0, r1) +# define negr_f(r0, r1) FNEG_S(r0, r1) +# define sqrtr_f(r0, r1) FSQRT_S(r0, r1) +# define extr_f(r0, r1) _extr_f(_jit, r0, r1) +static void _extr_f(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldr_f(r0, r1) FLD_S(r0, r1, 0) +# define ldi_f(r0, i0) _ldi_f(_jit, r0, i0) +static void _ldi_f(jit_state_t*, jit_int32_t, jit_word_t); +# define ldxr_f(r0, r1, r2) FLDX_S(r0, r1, r2) +# define ldxi_f(r0, r1, i0) _ldxi_f(_jit, r0, r1, i0) +static void _ldxi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define str_f(r0, r1) FST_S(r1, r0, 0) +# define sti_f(i0, r0) _sti_f(_jit, i0, r0) +static void _sti_f(jit_state_t*, jit_word_t, jit_int32_t); +# define stxr_f(r0, r1, r2) FSTX_S(r2, r1, r0) +# define stxi_f(i0, r0, r1) _stxi_f(_jit, i0, r0, r1) +static void _stxi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define movr_f(r0, r1) FMOV_S(r0, r1) +# define movi_f(r0, i0) _movi_f(_jit, r0, i0) +static void _movi_f(jit_state_t*, jit_int32_t, jit_float32_t); +# define movr_f_w(r0, r1) MOVFR2GR_S(r0, r1) +# define movi_f_w(r0, im) _movi_f_w(_jit, r0, im) +static void _movi_f_w(jit_state_t*, jit_int32_t, jit_float32_t); +# define movr_w_f(r0, r1) MOVGR2FR_W(r0, r1) +# define extr_d_f(r0, r1) FCVT_S_D(r0, r1) +# define ltr_f(r0, r1, r2) _ltr_f(_jit, r0, r1, r2) +static void _ltr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lti_f(r0, r1, i0) _lti_f(_jit, r0, r1, i0) +static void _lti_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define ler_f(r0, r1, r2) _ler_f(_jit, r0, r1, r2) +static void _ler_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lei_f(r0, r1, i0) _lei_f(_jit, r0, r1, i0) +static void _lei_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define eqr_f(r0, r1, r2) _eqr_f(_jit, r0, r1, r2) +static void _eqr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define eqi_f(r0, r1, i0) _eqi_f(_jit, r0, r1, i0) +static void _eqi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define ger_f(r0, r1, r2) _ger_f(_jit, r0, r1, r2) +static void _ger_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gei_f(r0, r1, i0) _gei_f(_jit, r0, r1, i0) +static void _gei_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define gtr_f(r0, r1, r2) _gtr_f(_jit, r0, r1, r2) +static void _gtr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gti_f(r0, r1, i0) _gti_f(_jit, r0, r1, i0) +static void _gti_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define ner_f(r0, r1, r2) _ner_f(_jit, r0, r1, r2) +static void _ner_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define nei_f(r0, r1, i0) _nei_f(_jit, r0, r1, i0) +static void _nei_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define unltr_f(r0, r1, r2) _unltr_f(_jit, r0, r1, r2) +static void _unltr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define unlti_f(r0, r1, i0) _unlti_f(_jit, r0, r1, i0) +static void _unlti_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define unler_f(r0, r1, r2) _unler_f(_jit, r0, r1, r2) +static void _unler_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define unlei_f(r0, r1, i1) _unlei_f(_jit, r0, r1, i1) +static void _unlei_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define uneqr_f(r0, r1, r2) _uneqr_f(_jit, r0, r1, r2) +static void _uneqr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define uneqi_f(r0, r1, i0) _uneqi_f(_jit, r0, r1, i0) +static void _uneqi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define unger_f(r0, r1, r2) _unger_f(_jit, r0, r1, r2) +static void _unger_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ungei_f(r0, r1, i0) _ungei_f(_jit, r0, r1, i0) +static void _ungei_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define ungtr_f(r0, r1, r2) _ungtr_f(_jit, r0, r1, r2) +static void _ungtr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ungti_f(r0, r1, i0) _ungti_f(_jit, r0, r1, i0) +static void _ungti_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define ltgtr_f(r0, r1, r2) _ltgtr_f(_jit, r0, r1, r2) +static void _ltgtr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ltgti_f(r0, r1, i0) _ltgti_f(_jit, r0, r1, i0) +static void _ltgti_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define ordr_f(r0, r1, r2) _ordr_f(_jit, r0, r1, r2) +static void _ordr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ordi_f(r0, r1, i0) _ordi_f(_jit, r0, r1, i0) +static void _ordi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define unordr_f(r0, r1, r2) _unordr_f(_jit, r0, r1, r2) +static void _unordr_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define unordi_f(r0, r1, i0) _unordi_f(_jit, r0, r1, i0) +static void _unordi_f(jit_state_t*, jit_int32_t, jit_int32_t, jit_float32_t); +# define bltr_f(i0, r0, r1) _bltr_f(_jit, i0, r0, r1) +static jit_word_t _bltr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blti_f(i0, r0, i1) _blti_f(_jit, i0, r0, i1) +static jit_word_t _blti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t); +# define bler_f(i0, r0, r1) _bler_f(_jit, i0, r0, r1) +static jit_word_t _bler_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blei_f(i0, r0, i1) _blei_f(_jit, i0, r0, i1) +static jit_word_t _blei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t); +# define beqr_f(i0, r0, r1) _beqr_f(_jit, i0, r0, r1) +static jit_word_t _beqr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define beqi_f(i0, r0, i1) _beqi_f(_jit, i0, r0, i1) +static jit_word_t _beqi_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t); +# define bger_f(i0, r0, r1) _bger_f(_jit, i0, r0, r1) +static jit_word_t _bger_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgei_f(i0, r0, i1) _bgei_f(_jit, i0, r0, i1) +static jit_word_t _bgei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t); +# define bgtr_f(i0, r0, r1) _bgtr_f(_jit,i0, r0, r1) +static jit_word_t _bgtr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgti_f(i0, r0, i1) _bgti_f(_jit, i0, r0, i1) +static jit_word_t _bgti_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t); +# define bner_f(i0, r0, r1) _bner_f(_jit, i0, r0, r1) +static jit_word_t _bner_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bnei_f(i0, r0, i1) _bnei_f(_jit, i0, r0, i1) +static jit_word_t _bnei_f(jit_state_t*, jit_word_t, jit_int32_t, jit_float32_t); +# define bunltr_f(i0, r0, r1) _bunltr_f(_jit, i0, r0, r1) +static jit_word_t _bunltr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bunlti_f(i0, r0, i1) _bunlti_f(_jit, i0, r0, i1) +static jit_word_t _bunlti_f(jit_state_t*, jit_word_t,jit_int32_t,jit_float32_t); +# define bunler_f(i0, r0, r1) _bunler_f(_jit, i0, r0, r1) +static jit_word_t _bunler_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bunlei_f(i0, r0, i1) _bunlei_f(_jit, i0, r0, i1) +static jit_word_t _bunlei_f(jit_state_t*, jit_word_t,jit_int32_t,jit_float32_t); +# define buneqr_f(i0, r0, r1) _buneqr_f(_jit, i0, r0, r1) +static jit_word_t _buneqr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define buneqi_f(i0, r0, i1) _buneqi_f(_jit, i0, r0, i1) +static jit_word_t _buneqi_f(jit_state_t*, jit_word_t,jit_int32_t,jit_float32_t); +# define bunger_f(i0, r0, r1) _bunger_f(_jit, i0, r0, r1) +static jit_word_t _bunger_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bungei_f(i0, r0, i1) _bungei_f(_jit, i0, r0, i1) +static jit_word_t _bungei_f(jit_state_t*, jit_word_t,jit_int32_t,jit_float32_t); +# define bungtr_f(i0, r0, r1) _bungtr_f(_jit, i0, r0, r1) +static jit_word_t _bungtr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bungti_f(i0, r0, i1) _bungti_f(_jit, i0, r0, i1) +static jit_word_t _bungti_f(jit_state_t*, jit_word_t,jit_int32_t,jit_float32_t); +# define bltgtr_f(i0, r0, r1) _bltgtr_f(_jit, i0, r0, r1) +static jit_word_t _bltgtr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bltgti_f(i0, r0, i1) _bltgti_f(_jit, i0, r0, i1) +static jit_word_t _bltgti_f(jit_state_t*, jit_word_t,jit_int32_t,jit_float32_t); +# define bordr_f(i0, r0, r1) _bordr_f(_jit, i0, r0, r1) +static jit_word_t _bordr_f(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bordi_f(i0, r0, i1) _bordi_f(_jit, i0, r0, i1) +static jit_word_t _bordi_f(jit_state_t*, jit_word_t, jit_int32_t,jit_float32_t); +# define bunordr_f(i0, r0, r1) _bunordr_f(_jit, i0, r0, r1) +static jit_word_t _bunordr_f(jit_state_t*, jit_word_t, jit_int32_t,jit_int32_t); +# define bunordi_f(i0, r0, i1) _bunordi_f(_jit, i0,r0, i1) +static jit_word_t _bunordi_f(jit_state_t*,jit_word_t,jit_int32_t,jit_float32_t); +# define addr_d(r0, r1, r2) FADD_D(r0, r1, r2) +# define addi_d(r0, r1, i0) _addi_d(_jit, r0, r1, i0) +static void _addi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define subr_d(r0, r1, r2) FSUB_D(r0, r1, r2) +# define subi_d(r0, r1, i0) _subi_d(_jit, r0, r1, i0) +static void _subi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define rsbr_d(r0, r1, r2) FSUB_D(r0, r2, r1) +# define rsbi_d(r0, r1, i0) _rsbi_d(_jit, r0, r1, i0) +static void _rsbi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define mulr_d(r0, r1, r2) FMUL_D(r0, r1, r2) +# define muli_d(r0, r1, i0) _muli_d(_jit, r0, r1, i0) +static void _muli_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define divr_d(r0, r1, r2) FDIV_D(r0, r1, r2) +# define divi_d(r0, r1, i0) _divi_d(_jit, r0, r1, i0) +static void _divi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define absr_d(r0, r1) FABS_D(r0, r1) +# define negr_d(r0, r1) FNEG_D(r0, r1) +# define sqrtr_d(r0, r1) FSQRT_D(r0, r1) +# define extr_d(r0, r1) _extr_d(_jit, r0, r1) +static void _extr_d(jit_state_t*, jit_int32_t, jit_int32_t); +# define ldr_d(r0, r1) FLD_D(r0, r1, 0) +# define ldi_d(r0, i0) _ldi_d(_jit, r0, i0) +static void _ldi_d(jit_state_t*, jit_int32_t, jit_word_t); +# define ldxr_d(r0, r1, r2) FLDX_D(r0, r1, r2) +# define ldxi_d(r0, r1, i0) _ldxi_d(_jit, r0, r1, i0) +static void _ldxi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); +# define str_d(r0, r1) FST_D(r1, r0, 0) +# define sti_d(i0, r0) _sti_d(_jit, i0, r0) +static void _sti_d(jit_state_t*, jit_word_t, jit_int32_t); +# define stxr_d(r0, r1, r2) FSTX_D(r2, r1, r0) +# define stxi_d(i0, r0, r1) _stxi_d(_jit, i0, r0, r1) +static void _stxi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define movr_d(r0, r1) FMOV_D(r0, r1) +# define movi_d(r0, i0) _movi_d(_jit, r0, i0) +static void _movi_d(jit_state_t*, jit_int32_t, jit_float64_t); +# define movr_d_w(r0, r1) MOVFR2GR_D(r0, r1) +# define movi_d_w(r0, im) _movi_d_w(_jit, r0, im) +static void _movi_d_w(jit_state_t*, jit_int32_t, jit_float64_t); +# define movr_w_d(r0, r1) MOVGR2FR_D(r0, r1) +# define extr_f_d(r0, r1) FCVT_D_S(r0, r1) +# define ltr_d(r0, r1, r2) _ltr_d(_jit, r0, r1, r2) +static void _ltr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lti_d(r0, r1, i0) _lti_d(_jit, r0, r1, i0) +static void _lti_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define ler_d(r0, r1, r2) _ler_d(_jit, r0, r1, r2) +static void _ler_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define lei_d(r0, r1, i0) _lei_d(_jit, r0, r1, i0) +static void _lei_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define eqr_d(r0, r1, r2) _eqr_d(_jit, r0, r1, r2) +static void _eqr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define eqi_d(r0, r1, i0) _eqi_d(_jit, r0, r1, i0) +static void _eqi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define ger_d(r0, r1, r2) _ger_d(_jit, r0, r1, r2) +static void _ger_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gei_d(r0, r1, i0) _gei_d(_jit, r0, r1, i0) +static void _gei_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define gtr_d(r0, r1, r2) _gtr_d(_jit, r0, r1, r2) +static void _gtr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define gti_d(r0, r1, i0) _gti_d(_jit, r0, r1, i0) +static void _gti_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define ner_d(r0, r1, r2) _ner_d(_jit, r0, r1, r2) +static void _ner_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define nei_d(r0, r1, i0) _nei_d(_jit, r0, r1, i0) +static void _nei_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define unltr_d(r0, r1, r2) _unltr_d(_jit, r0, r1, r2) +static void _unltr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define unlti_d(r0, r1, i0) _unlti_d(_jit, r0, r1, i0) +static void _unlti_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define unler_d(r0, r1, r2) _unler_d(_jit, r0, r1, r2) +static void _unler_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define unlei_d(r0, r1, i0) _unlei_d(_jit, r0, r1, i0) +static void _unlei_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define uneqr_d(r0, r1, r2) _uneqr_d(_jit, r0, r1, r2) +static void _uneqr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define uneqi_d(r0, r1, i0) _uneqi_d(_jit, r0, r1, i0) +static void _uneqi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define unger_d(r0, r1, r2) _unger_d(_jit, r0, r1, r2) +static void _unger_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ungei_d(r0, r1, i0) _ungei_d(_jit, r0, r1, i0) +static void _ungei_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define ungtr_d(r0, r1, r2) _ungtr_d(_jit, r0, r1, r2) +static void _ungtr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ungti_d(r0, r1, i0) _ungti_d(_jit, r0, r1, i0) +static void _ungti_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define ltgtr_d(r0, r1, r2) _ltgtr_d(_jit, r0, r1, r2) +static void _ltgtr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ltgti_d(r0, r1, i0) _ltgti_d(_jit, r0, r1, i0) +static void _ltgti_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define ordr_d(r0, r1, r2) _ordr_d(_jit, r0, r1, r2) +static void _ordr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define ordi_d(r0, r1, i0) _ordi_d(_jit, r0, r1, i0) +static void _ordi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define unordr_d(r0, r1, r2) _unordr_d(_jit, r0, r1, r2) +static void _unordr_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +# define unordi_d(r0, r1, i0) _unordi_d(_jit, r0, r1, i0) +static void _unordi_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t); +# define bltr_d(i0, r0, r1) _bltr_d(_jit, i0, r0, r1) +static jit_word_t _bltr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blti_d(i0, r0, i1) _blti_d(_jit, i0, r0, i1) +static jit_word_t _blti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t); +# define bler_d(i0, r0, r1) _bler_d(_jit, i0, r0, r1) +static jit_word_t _bler_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define blei_d(i0, r0, i1) _blei_d(_jit, i0,r0, i1) +static jit_word_t _blei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t); +# define beqr_d(i0, r0, r1) _beqr_d(_jit, i0, r0, r1) +static jit_word_t _beqr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define beqi_d(i0, r0, i1) _beqi_d(_jit, i0, r0, i1) +static jit_word_t _beqi_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t); +# define bger_d(i0, r0, r1) _bger_d(_jit, i0, r0, r1) +static jit_word_t _bger_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgei_d(i0, r0, i1) _bgei_d(_jit, i0, r0, i1) +static jit_word_t _bgei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t); +# define bgtr_d(i0, r0, r1) _bgtr_d(_jit, i0, r0, r1) +static jit_word_t _bgtr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bgti_d(i0, r0, i1) _bgti_d(_jit, i0, r0, i1) +static jit_word_t _bgti_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t); +# define bner_d(i0, r0, r1) _bner_d(_jit, i0, r0, r1) +static jit_word_t _bner_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bnei_d(i0, r0, i1) _bnei_d(_jit, i0, r0, i1) +static jit_word_t _bnei_d(jit_state_t*, jit_word_t, jit_int32_t, jit_float64_t); +# define bunltr_d(i0, r0, r1) _bunltr_d(_jit, i0, r0, r1) +static jit_word_t _bunltr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bunlti_d(i0, r0, i1) _bunlti_d(_jit, i0, r0, i1) +static jit_word_t _bunlti_d(jit_state_t*, jit_word_t,jit_int32_t,jit_float64_t); +# define bunler_d(i0, r0, r1) _bunler_d(_jit, i0, r0, r1) +static jit_word_t _bunler_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bunlei_d(i0, r0, i1) _bunlei_d(_jit, i0, r0, i1) +static jit_word_t _bunlei_d(jit_state_t*, jit_word_t,jit_int32_t,jit_float64_t); +# define buneqr_d(i0, r0, r1) _buneqr_d(_jit, i0, r0, r1) +static jit_word_t _buneqr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define buneqi_d(i0, r0, i1) _buneqi_d(_jit, i0,r0, i1) +static jit_word_t _buneqi_d(jit_state_t*, jit_word_t,jit_int32_t,jit_float64_t); +# define bunger_d(i0, r0, r1) _bunger_d(_jit, i0, r0, r1) +static jit_word_t _bunger_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bungei_d(i0, r0, i1) _bungei_d(_jit, i0, r0, i1) +static jit_word_t _bungei_d(jit_state_t*, jit_word_t,jit_int32_t,jit_float64_t); +# define bungtr_d(i0, r0, r1) _bungtr_d(_jit, i0, r0, r1) +static jit_word_t _bungtr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bungti_d(i0, r0, i1) _bungti_d(_jit, i0, r0, i1) +static jit_word_t _bungti_d(jit_state_t*, jit_word_t,jit_int32_t,jit_float64_t); +# define bltgtr_d(i0, r0, r1) _bltgtr_d(_jit, i0, r0, r1) +static jit_word_t _bltgtr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bltgti_d(i0, r0, i1) _bltgti_d(_jit, i0, r0, i1) +static jit_word_t _bltgti_d(jit_state_t*, jit_word_t,jit_int32_t,jit_float64_t); +# define bordr_d(i0, r0, r1) _bordr_d(_jit, i0, r0, r1) +static jit_word_t _bordr_d(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); +# define bordi_d(i0, r0, i1) _bordi_d(_jit, i0, r0, i1) +static jit_word_t _bordi_d(jit_state_t*, jit_word_t, jit_int32_t,jit_float64_t); +# define bunordr_d(i0, r0, r1) _bunordr_d(_jit, i0, r0, r1) +static jit_word_t _bunordr_d(jit_state_t*, jit_word_t, jit_int32_t,jit_int32_t); +# define bunordi_d(i0, r0, i1) _bunordi_d(_jit, i0, r0, i1) +static jit_word_t _bunordi_d(jit_state_t*,jit_word_t,jit_int32_t,jit_float64_t); +# define vaarg_d(r0, r1) _vaarg_d(_jit, r0, r1) +static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t); +#endif /* PROTO */ + +#if CODE +static void +_truncr_f_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_fpr); + FTINTRZ_W_S(rn(reg), r1); + MOVFR2GR_S(r0, rn(reg)); + jit_unget_reg(reg); +} + +static void +_truncr_d_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_fpr); + FTINTRZ_W_D(rn(reg), r1); + MOVFR2GR_S(r0, rn(reg)); + jit_unget_reg(reg); +} + +static void +_truncr_f_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_fpr); + FTINTRZ_L_S(rn(reg), r1); + MOVFR2GR_D(r0, rn(reg)); + jit_unget_reg(reg); +} + +static void +_truncr_d_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_fpr); + FTINTRZ_L_D(rn(reg), r1); + MOVFR2GR_D(r0, rn(reg)); + jit_unget_reg(reg); +} + +# define fpr_opi(name, type, size) \ +static void \ +_##name##i_##type(jit_state_t *_jit, \ + jit_int32_t r0, jit_int32_t r1, \ + jit_float##size##_t i0) \ +{ \ + jit_int32_t reg = jit_get_reg(jit_class_fpr); \ + movi_##type(rn(reg), i0); \ + name##r_##type(r0, r1, rn(reg)); \ + jit_unget_reg(reg); \ +} +# define fopi(name) fpr_opi(name, f, 32) +# define dopi(name) fpr_opi(name, d, 64) + +# define fpr_bopi(name, type, size) \ +static jit_word_t \ +_b##name##i_##type(jit_state_t *_jit, \ + jit_word_t i0, jit_int32_t r0, \ + jit_float##size##_t i1) \ +{ \ + jit_word_t word; \ + jit_int32_t reg = jit_get_reg(jit_class_fpr| \ + jit_class_nospill); \ + movi_##type(rn(reg), i1); \ + word = b##name##r_##type(i0, r0, rn(reg)); \ + jit_unget_reg(reg); \ + return (word); \ +} +# define fbopi(name) fpr_bopi(name, f, 32) +# define dbopi(name) fpr_bopi(name, d, 64) + +fopi(add) +fopi(sub) +fopi(rsb) +fopi(mul) +fopi(div) + +static void +_extr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + MOVGR2FR_D(r0, r1); + FFINT_S_L(r0, r0); +} + +static void +_ldi_f(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FLD_S(r0, _ZERO_REGNO, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + ldr_f(r0, rn(t0)); + jit_unget_reg(t0); + } +} + +static void +_ldxi_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FLD_S(r0, r1, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + ldxr_f(r0, r1, rn(t0)); + jit_unget_reg(t0); + } +} + +static void +_sti_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FST_S(r0, _ZERO_REGNO, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + str_f(rn(t0), r0); + jit_unget_reg(t0); + } +} + +static void +_stxi_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FST_S(r1, r0, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + stxr_f(rn(t0), r0, r1); + jit_unget_reg(t0); + } +} + +static void +_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0) +{ + union { + jit_int32_t i; + jit_float32_t f; + } data; + jit_int32_t reg; + data.f = i0; + if (data.i == 0) + movr_w_f(r0, _ZERO_REGNO); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), data.i); + movr_w_f(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_movi_f_w(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0) +{ + union { + jit_int32_t i; + jit_float32_t f; + } data; + data.f = i0; + movi(r0, data.i); +} + +static void +_ltr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLT_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(lt) + +static void +_ler_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLE_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(le) + +static void +_eqr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CEQ_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(eq) + +static void +_ger_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLE_S(0, r2, r1); + MOVCF2GR(r0, 0); +} +fopi(ge) + +static void +_gtr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLT_S(0, r2, r1); + MOVCF2GR(r0, 0); +} +fopi(gt) + +static void +_ner_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CUNE_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(ne) + +static void +_unltr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULT_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(unlt) + +static void +_unler_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULE_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(unle) + +static void +_uneqr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CUEQ_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(uneq) + +static void +_unger_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULE_S(0, r2, r1); + MOVCF2GR(r0, 0); +} +fopi(unge) + +static void +_ungtr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULT_S(0, r2, r1); + MOVCF2GR(r0, 0); +} +fopi(ungt) + +static void +_ltgtr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CNE_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(ltgt) + +static void +_ordr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_COR_S(0, r1, r2); + MOVCF2GR(r0, 0); +} +fopi(ord) + +static void +_unordr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_COR_S(0, r1, r2); + MOVCF2GR(r0, 0); + XORI(r0, r0, 1); +} +fopi(unord) + +static jit_word_t +_bltr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLT_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(lt) + +static jit_word_t +_bler_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLE_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(le) + +static jit_word_t +_beqr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CEQ_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(eq) + +static jit_word_t +_bger_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLE_S(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(ge) + +static jit_word_t +_bgtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLT_S(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(gt) + +static jit_word_t +_bner_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CUNE_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(ne) + +static jit_word_t +_bunltr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULT_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(unlt) + +static jit_word_t +_bunler_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULE_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(unle) + +static jit_word_t +_buneqr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CUEQ_S(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(uneq) + +static jit_word_t +_bunger_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULE_S(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(unge) + +static jit_word_t +_bungtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULT_S(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(ungt) + +static jit_word_t +_bltgtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CNE_S(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(ltgt) + +static jit_word_t +_bordr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_COR_S(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(ord) + +static jit_word_t +_bunordr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_COR_S(0, r2, r1); + w = _jit->pc.w; + BCEQZ(0, (i0 - w) >> 2); + return (w); +} +fbopi(unord) + +dopi(add) +dopi(sub) +dopi(rsb) +dopi(mul) +dopi(div) + +static void +_extr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + MOVGR2FR_D(r0, r1); + FFINT_D_L(r0, r0); +} + +static void +_ldi_d(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FLD_D(r0, _ZERO_REGNO, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + ldr_d(r0, rn(t0)); + jit_unget_reg(t0); + } +} + +static void +_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FLD_D(r0, r1, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + ldxr_d(r0, r1, rn(t0)); + jit_unget_reg(t0); + } +} + +static void +_sti_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FST_D(r0, _ZERO_REGNO, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + str_d(rn(t0), r0); + jit_unget_reg(t0); + } +} + +static void +_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t t0; + if (can_sign_extend_si12_p(i0)) + FST_D(r1, r0, i0); + else { + t0 = jit_get_reg(jit_class_gpr); + movi(rn(t0), i0); + stxr_d(rn(t0), r0, r1); + jit_unget_reg(t0); + } +} + +static void +_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0) +{ + union { + jit_word_t w; + jit_float64_t d; + } data; + jit_int32_t reg; + data.d = i0; + if (data.w == 0) + movr_w_d(r0, _ZERO_REGNO); + else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), data.w); + movr_w_d(r0, rn(reg)); + jit_unget_reg(reg); + } +} + +static void +_movi_d_w(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0) +{ + union { + jit_int64_t l; + jit_float64_t d; + } data; + data.d = i0; + movi(r0, data.l); +} + +static void +_ltr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLT_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(lt) + +static void +_ler_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLE_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(le) + +static void +_eqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CEQ_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(eq) + +static void +_ger_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLE_D(0, r2, r1); + MOVCF2GR(r0, 0); +} +dopi(ge) + +static void +_gtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_SLT_D(0, r2, r1); + MOVCF2GR(r0, 0); +} +dopi(gt) + +static void +_ner_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CUNE_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(ne) + +static void +_unltr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULT_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(unlt) + +static void +_unler_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULE_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(unle) + +static void +_uneqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CUEQ_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(uneq) + +static void +_unger_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULE_D(0, r2, r1); + MOVCF2GR(r0, 0); +} +dopi(unge) + +static void +_ungtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CULT_D(0, r2, r1); + MOVCF2GR(r0, 0); +} +dopi(ungt) + +static void +_ltgtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_CNE_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(ltgt) + +static void +_ordr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_COR_D(0, r1, r2); + MOVCF2GR(r0, 0); +} +dopi(ord) + +static void +_unordr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + FCMP_COR_D(0, r1, r2); + MOVCF2GR(r0, 0); + XORI(r0, r0, 1); +} +dopi(unord) + +static jit_word_t +_bltr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLT_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(lt) + +static jit_word_t +_bler_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLE_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(le) + +static jit_word_t +_beqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CEQ_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(eq) + +static jit_word_t +_bger_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLE_D(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(ge) + +static jit_word_t +_bgtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_SLT_D(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(gt) + +static jit_word_t +_bner_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CUNE_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(ne) + +static jit_word_t +_bunltr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULT_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(unlt) + +static jit_word_t +_bunler_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULE_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(unle) + +static jit_word_t +_buneqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CUEQ_D(0, r1, r2); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(uneq) + +static jit_word_t +_bunger_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULE_D(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(unge) + +static jit_word_t +_bungtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CULT_D(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(ungt) + +static jit_word_t +_bltgtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_CNE_D(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(ltgt) + +static jit_word_t +_bordr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_COR_D(0, r2, r1); + w = _jit->pc.w; + BCNEZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(ord) + +static jit_word_t +_bunordr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + FCMP_COR_D(0, r2, r1); + w = _jit->pc.w; + BCEQZ(0, (i0 - w) >> 2); + return (w); +} +dbopi(unord) + +static void +_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + assert(_jitc->function->self.call & jit_call_varargs); + /* Load argument. */ + ldr_d(r0, r1); + /* Update va_list. */ + addi(r1, r1, sizeof(jit_float64_t)); +} + +#endif /* CODE */ diff --git a/parallel-rsp/lightning/lib/jit_loongarch-sz.c b/parallel-rsp/lightning/lib/jit_loongarch-sz.c new file mode 100644 index 0000000..4b95047 --- /dev/null +++ b/parallel-rsp/lightning/lib/jit_loongarch-sz.c @@ -0,0 +1,452 @@ +#if __WORDSIZE == 64 +#define JIT_INSTR_MAX 88 + 0, /* data */ + 0, /* live */ + 28, /* align */ + 0, /* save */ + 0, /* load */ + 4, /* skip */ + 0, /* #name */ + 0, /* #note */ + 0, /* label */ + 88, /* prolog */ + 0, /* ellipsis */ + 0, /* va_push */ + 0, /* allocai */ + 0, /* allocar */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ + 0, /* getarg_c */ + 0, /* getarg_uc */ + 0, /* getarg_s */ + 0, /* getarg_us */ + 0, /* getarg_i */ + 0, /* getarg_ui */ + 0, /* getarg_l */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 4, /* va_start */ + 8, /* va_arg */ + 8, /* va_arg_d */ + 0, /* va_end */ + 4, /* addr */ + 20, /* addi */ + 12, /* addcr */ + 28, /* addci */ + 28, /* addxr */ + 28, /* addxi */ + 4, /* subr */ + 20, /* subi */ + 12, /* subcr */ + 28, /* subci */ + 28, /* subxr */ + 28, /* subxi */ + 24, /* rsbi */ + 4, /* mulr */ + 20, /* muli */ + 12, /* qmulr */ + 24, /* qmuli */ + 12, /* qmulr_u */ + 24, /* qmuli_u */ + 4, /* divr */ + 20, /* divi */ + 4, /* divr_u */ + 20, /* divi_u */ + 12, /* qdivr */ + 12, /* qdivi */ + 12, /* qdivr_u */ + 12, /* qdivi_u */ + 4, /* remr */ + 20, /* remi */ + 4, /* remr_u */ + 20, /* remi_u */ + 4, /* andr */ + 20, /* andi */ + 4, /* orr */ + 20, /* ori */ + 4, /* xorr */ + 20, /* xori */ + 4, /* lshr */ + 4, /* lshi */ + 4, /* rshr */ + 4, /* rshi */ + 4, /* rshr_u */ + 4, /* rshi_u */ + 4, /* negr */ + 4, /* comr */ + 4, /* ltr */ + 4, /* lti */ + 4, /* ltr_u */ + 4, /* lti_u */ + 8, /* ler */ + 4, /* lei */ + 8, /* ler_u */ + 4, /* lei_u */ + 12, /* eqr */ + 12, /* eqi */ + 8, /* ger */ + 8, /* gei */ + 8, /* ger_u */ + 8, /* gei_u */ + 4, /* gtr */ + 8, /* gti */ + 4, /* gtr_u */ + 8, /* gti_u */ + 8, /* ner */ + 8, /* nei */ + 4, /* movr */ + 16, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 32, /* casr */ + 44, /* casi */ + 4, /* extr_c */ + 4, /* extr_uc */ + 4, /* extr_s */ + 4, /* extr_us */ + 4, /* extr_i */ + 4, /* extr_ui */ + 8, /* bswapr_us */ + 8, /* bswapr_ui */ + 4, /* bswapr_ul */ + 8, /* htonr_us */ + 8, /* htonr_ui */ + 4, /* htonr_ul */ + 4, /* ldr_c */ + 20, /* ldi_c */ + 4, /* ldr_uc */ + 20, /* ldi_uc */ + 4, /* ldr_s */ + 20, /* ldi_s */ + 4, /* ldr_us */ + 20, /* ldi_us */ + 4, /* ldr_i */ + 20, /* ldi_i */ + 4, /* ldr_ui */ + 20, /* ldi_ui */ + 4, /* ldr_l */ + 20, /* ldi_l */ + 4, /* ldxr_c */ + 16, /* ldxi_c */ + 4, /* ldxr_uc */ + 16, /* ldxi_uc */ + 4, /* ldxr_s */ + 16, /* ldxi_s */ + 4, /* ldxr_us */ + 16, /* ldxi_us */ + 4, /* ldxr_i */ + 16, /* ldxi_i */ + 4, /* ldxr_ui */ + 16, /* ldxi_ui */ + 4, /* ldxr_l */ + 16, /* ldxi_l */ + 4, /* str_c */ + 20, /* sti_c */ + 4, /* str_s */ + 20, /* sti_s */ + 4, /* str_i */ + 20, /* sti_i */ + 4, /* str_l */ + 20, /* sti_l */ + 4, /* stxr_c */ + 16, /* stxi_c */ + 4, /* stxr_s */ + 16, /* stxi_s */ + 4, /* stxr_i */ + 16, /* stxi_i */ + 4, /* stxr_l */ + 16, /* stxi_l */ + 4, /* bltr */ + 8, /* blti */ + 4, /* bltr_u */ + 8, /* blti_u */ + 8, /* bler */ + 12, /* blei */ + 8, /* bler_u */ + 12, /* blei_u */ + 4, /* beqr */ + 20, /* beqi */ + 4, /* bger */ + 8, /* bgei */ + 4, /* bger_u */ + 8, /* bgei_u */ + 8, /* bgtr */ + 12, /* bgti */ + 8, /* bgtr_u */ + 12, /* bgti_u */ + 4, /* bner */ + 20, /* bnei */ + 8, /* bmsr */ + 8, /* bmsi */ + 8, /* bmcr */ + 8, /* bmci */ + 32, /* boaddr */ + 36, /* boaddi */ + 16, /* boaddr_u */ + 20, /* boaddi_u */ + 32, /* bxaddr */ + 36, /* bxaddi */ + 16, /* bxaddr_u */ + 20, /* bxaddi_u */ + 32, /* bosubr */ + 36, /* bosubi */ + 16, /* bosubr_u */ + 20, /* bosubi_u */ + 32, /* bxsubr */ + 36, /* bxsubi */ + 16, /* bxsubr_u */ + 20, /* bxsubi_u */ + 4, /* jmpr */ + 20, /* jmpi */ + 4, /* callr */ + 20, /* calli */ + 0, /* prepare */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ + 0, /* finishr */ + 0, /* finishi */ + 0, /* ret */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ + 0, /* retval_c */ + 0, /* retval_uc */ + 0, /* retval_s */ + 0, /* retval_us */ + 0, /* retval_i */ + 0, /* retval_ui */ + 0, /* retval_l */ + 88, /* epilog */ + 0, /* arg_f */ + 0, /* getarg_f */ + 0, /* putargr_f */ + 0, /* putargi_f */ + 4, /* addr_f */ + 12, /* addi_f */ + 4, /* subr_f */ + 12, /* subi_f */ + 12, /* rsbi_f */ + 4, /* mulr_f */ + 12, /* muli_f */ + 4, /* divr_f */ + 12, /* divi_f */ + 4, /* negr_f */ + 4, /* absr_f */ + 4, /* sqrtr_f */ + 8, /* ltr_f */ + 16, /* lti_f */ + 8, /* ler_f */ + 16, /* lei_f */ + 8, /* eqr_f */ + 16, /* eqi_f */ + 8, /* ger_f */ + 16, /* gei_f */ + 8, /* gtr_f */ + 16, /* gti_f */ + 8, /* ner_f */ + 16, /* nei_f */ + 8, /* unltr_f */ + 16, /* unlti_f */ + 8, /* unler_f */ + 16, /* unlei_f */ + 8, /* uneqr_f */ + 16, /* uneqi_f */ + 8, /* unger_f */ + 16, /* ungei_f */ + 8, /* ungtr_f */ + 16, /* ungti_f */ + 8, /* ltgtr_f */ + 16, /* ltgti_f */ + 8, /* ordr_f */ + 16, /* ordi_f */ + 12, /* unordr_f */ + 20, /* unordi_f */ + 8, /* truncr_f_i */ + 8, /* truncr_f_l */ + 8, /* extr_f */ + 4, /* extr_d_f */ + 4, /* movr_f */ + 8, /* movi_f */ + 4, /* ldr_f */ + 20, /* ldi_f */ + 4, /* ldxr_f */ + 16, /* ldxi_f */ + 4, /* str_f */ + 20, /* sti_f */ + 4, /* stxr_f */ + 16, /* stxi_f */ + 8, /* bltr_f */ + 16, /* blti_f */ + 8, /* bler_f */ + 16, /* blei_f */ + 8, /* beqr_f */ + 16, /* beqi_f */ + 8, /* bger_f */ + 16, /* bgei_f */ + 8, /* bgtr_f */ + 16, /* bgti_f */ + 8, /* bner_f */ + 16, /* bnei_f */ + 8, /* bunltr_f */ + 16, /* bunlti_f */ + 8, /* bunler_f */ + 16, /* bunlei_f */ + 8, /* buneqr_f */ + 16, /* buneqi_f */ + 8, /* bunger_f */ + 16, /* bungei_f */ + 8, /* bungtr_f */ + 16, /* bungti_f */ + 8, /* bltgtr_f */ + 16, /* bltgti_f */ + 8, /* bordr_f */ + 16, /* bordi_f */ + 8, /* bunordr_f */ + 16, /* bunordi_f */ + 0, /* pushargr_f */ + 0, /* pushargi_f */ + 0, /* retr_f */ + 0, /* reti_f */ + 0, /* retval_f */ + 0, /* arg_d */ + 0, /* getarg_d */ + 0, /* putargr_d */ + 0, /* putargi_d */ + 4, /* addr_d */ + 20, /* addi_d */ + 4, /* subr_d */ + 20, /* subi_d */ + 16, /* rsbi_d */ + 4, /* mulr_d */ + 20, /* muli_d */ + 4, /* divr_d */ + 20, /* divi_d */ + 4, /* negr_d */ + 4, /* absr_d */ + 4, /* sqrtr_d */ + 8, /* ltr_d */ + 24, /* lti_d */ + 8, /* ler_d */ + 24, /* lei_d */ + 8, /* eqr_d */ + 24, /* eqi_d */ + 8, /* ger_d */ + 24, /* gei_d */ + 8, /* gtr_d */ + 24, /* gti_d */ + 8, /* ner_d */ + 24, /* nei_d */ + 8, /* unltr_d */ + 24, /* unlti_d */ + 8, /* unler_d */ + 24, /* unlei_d */ + 8, /* uneqr_d */ + 24, /* uneqi_d */ + 8, /* unger_d */ + 24, /* ungei_d */ + 8, /* ungtr_d */ + 24, /* ungti_d */ + 8, /* ltgtr_d */ + 24, /* ltgti_d */ + 8, /* ordr_d */ + 24, /* ordi_d */ + 12, /* unordr_d */ + 28, /* unordi_d */ + 8, /* truncr_d_i */ + 8, /* truncr_d_l */ + 8, /* extr_d */ + 4, /* extr_f_d */ + 4, /* movr_d */ + 16, /* movi_d */ + 4, /* ldr_d */ + 20, /* ldi_d */ + 4, /* ldxr_d */ + 16, /* ldxi_d */ + 4, /* str_d */ + 20, /* sti_d */ + 4, /* stxr_d */ + 16, /* stxi_d */ + 8, /* bltr_d */ + 20, /* blti_d */ + 8, /* bler_d */ + 20, /* blei_d */ + 8, /* beqr_d */ + 24, /* beqi_d */ + 8, /* bger_d */ + 24, /* bgei_d */ + 8, /* bgtr_d */ + 24, /* bgti_d */ + 8, /* bner_d */ + 24, /* bnei_d */ + 8, /* bunltr_d */ + 24, /* bunlti_d */ + 8, /* bunler_d */ + 24, /* bunlei_d */ + 8, /* buneqr_d */ + 24, /* buneqi_d */ + 8, /* bunger_d */ + 24, /* bungei_d */ + 8, /* bungtr_d */ + 24, /* bungti_d */ + 8, /* bltgtr_d */ + 24, /* bltgti_d */ + 8, /* bordr_d */ + 20, /* bordi_d */ + 8, /* bunordr_d */ + 24, /* bunordi_d */ + 0, /* pushargr_d */ + 0, /* pushargi_d */ + 0, /* retr_d */ + 0, /* reti_d */ + 0, /* retval_d */ + 4, /* movr_w_f */ + 0, /* movr_ww_d */ + 4, /* movr_w_d */ + 0, /* movr_f_w */ + 4, /* movi_f_w */ + 0, /* movr_d_ww */ + 0, /* movi_d_ww */ + 4, /* movr_d_w */ + 12, /* movi_d_w */ + 4, /* clo */ + 4, /* clz */ + 4, /* cto */ + 4, /* ctz */ +#endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_loongarch.c b/parallel-rsp/lightning/lib/jit_loongarch.c new file mode 100644 index 0000000..c9b5b8c --- /dev/null +++ b/parallel-rsp/lightning/lib/jit_loongarch.c @@ -0,0 +1,1751 @@ +/* + * Copyright (C) 2022-2023 Free Software Foundation, Inc. + * + * This file is part of GNU lightning. + * + * GNU lightning is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 3, or (at your option) + * any later version. + * + * GNU lightning is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * Authors: + * Paulo Cesar Pereira de Andrade + */ + +/* callee save + variadic arguments + * align16(ra+fp+s[0-8]+fs[0-7]) + align16(a[0-7]) */ +#define stack_framesize (144 + 64) + +#define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8) +#define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) + +/* + * Types + */ +typedef struct jit_pointer_t jit_va_list_t; + +/* + * Prototypes + */ +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); +#define patch(instr, node) _patch(_jit, instr, node) +static void _patch(jit_state_t*,jit_word_t,jit_node_t*); + +#define PROTO 1 +# include "jit_loongarch-cpu.c" +# include "jit_loongarch-fpu.c" +#undef PROTO + +/* + * Initialization + */ +jit_register_t _rvs[] = { + { rc(gpr) | 0x14, "$t8" }, + { rc(gpr) | 0x13, "$t7" }, + { rc(gpr) | 0x12, "$t6" }, + { rc(gpr) | 0x11, "$t5" }, + { rc(gpr) | 0x10, "$t4" }, + { rc(gpr) | 0x0f, "$t3" }, + { rc(gpr) | 0x0e, "$t2" }, + { rc(gpr) | 0x0d, "$t1" }, + { rc(gpr) | 0x0c, "$t0" }, + { rc(sav) | rc(gpr) | 0x1f, "$s8" }, + { rc(sav) | rc(gpr) | 0x1e, "$s7" }, + { rc(sav) | rc(gpr) | 0x1d, "$s6" }, + { rc(sav) | rc(gpr) | 0x1c, "$s5" }, + { rc(sav) | rc(gpr) | 0x1b, "$s4" }, + { rc(sav) | rc(gpr) | 0x1a, "$s3" }, + { rc(sav) | rc(gpr) | 0x19, "$s2" }, + { rc(sav) | rc(gpr) | 0x18, "$s1" }, + { rc(sav) | rc(gpr) | 0x17, "$s0" }, + { rc(arg) | rc(gpr) | 0x0b, "$a7" }, + { rc(arg) | rc(gpr) | 0x0a, "$a6" }, + { rc(arg) | rc(gpr) | 0x09, "$a5" }, + { rc(arg) | rc(gpr) | 0x08, "$a4" }, + { rc(arg) | rc(gpr) | 0x07, "$a3" }, + { rc(arg) | rc(gpr) | 0x06, "$a2" }, + { rc(arg) | rc(gpr) | 0x05, "$a1" }, + { rc(arg) | rc(gpr) | 0x04, "$a0" }, + { 0x16, "$fp" }, + { 0x15, "" }, + { 0x00, "$zero" }, + { 0x01, "$ra" }, + { 0x02, "$tp" }, + { 0x03, "$sp" }, + { rc(fpr) | 0x08, "$ft0" }, + { rc(fpr) | 0x09, "$ft1" }, + { rc(fpr) | 0x0a, "$ft2" }, + { rc(fpr) | 0x0b, "$ft3" }, + { rc(fpr) | 0x0c, "$ft4" }, + { rc(fpr) | 0x0d, "$ft5" }, + { rc(fpr) | 0x0e, "$ft6" }, + { rc(fpr) | 0x0f, "$ft7" }, + { rc(fpr) | 0x10, "$ft8" }, + { rc(fpr) | 0x11, "$ft9" }, + { rc(fpr) | 0x12, "$ft10" }, + { rc(fpr) | 0x13, "$ft11" }, + { rc(fpr) | 0x14, "$ft12" }, + { rc(fpr) | 0x15, "$ft13" }, + { rc(fpr) | 0x16, "$ft14" }, + { rc(fpr) | 0x17, "$ft15" }, + { rc(arg) | rc(fpr) | 0x07, "$fa7" }, + { rc(arg) | rc(fpr) | 0x06, "$fa6" }, + { rc(arg) | rc(fpr) | 0x05, "$fa5" }, + { rc(arg) | rc(fpr) | 0x04, "$fa4" }, + { rc(arg) | rc(fpr) | 0x03, "$fa3" }, + { rc(arg) | rc(fpr) | 0x02, "$fa2" }, + { rc(arg) | rc(fpr) | 0x01, "$fa1" }, + { rc(arg) | rc(fpr) | 0x00, "$fa0" }, + { rc(sav) | rc(fpr) | 0x1f, "$fs7" }, + { rc(sav) | rc(fpr) | 0x1e, "$fs6" }, + { rc(sav) | rc(fpr) | 0x1d, "$fs5" }, + { rc(sav) | rc(fpr) | 0x1c, "$fs4" }, + { rc(sav) | rc(fpr) | 0x1b, "$fs3" }, + { rc(sav) | rc(fpr) | 0x1a, "$fs2" }, + { rc(sav) | rc(fpr) | 0x19, "$fs1" }, + { rc(sav) | rc(fpr) | 0x18, "$fs0" }, + { _NOREG, "" }, +}; + +static jit_int32_t iregs[] = { + _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7, _S8 +}; + +static jit_int32_t fregs[] = { + _FS0, _FS1, _FS2, _FS3, _FS4, _FS5, _FS6, _FS7 +}; + +/* + * Implementation + */ +void +jit_get_cpu(void) +{ +} + +void +_jit_init(jit_state_t *_jit) +{ + _jitc->reglen = jit_size(_rvs) - 1; + jit_carry = _NOREG; +} + +void +_jit_prolog(jit_state_t *_jit) +{ + jit_int32_t offset; + + if (_jitc->function) + jit_epilog(); + assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0); + jit_regset_set_ui(&_jitc->regsav, 0); + offset = _jitc->functions.offset; + if (offset >= _jitc->functions.length) { + jit_realloc((jit_pointer_t *)&_jitc->functions.ptr, + _jitc->functions.length * sizeof(jit_function_t), + (_jitc->functions.length + 16) * sizeof(jit_function_t)); + _jitc->functions.length += 16; + } + _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++; + _jitc->function->self.size = stack_framesize; + _jitc->function->self.argi = _jitc->function->self.argf = + _jitc->function->self.aoff = _jitc->function->self.alen = 0; + _jitc->function->self.call = jit_call_default; + jit_alloc((jit_pointer_t *)&_jitc->function->regoff, + _jitc->reglen * sizeof(jit_int32_t)); + + /* _no_link here does not mean the jit_link() call can be removed + * by rewriting as: + * _jitc->function->prolog = jit_new_node(jit_code_prolog); + */ + _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog); + jit_link(_jitc->function->prolog); + _jitc->function->prolog->w.w = offset; + _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog); + /* u: label value + * v: offset in blocks vector + * w: offset in functions vector + */ + _jitc->function->epilog->w.w = offset; + + jit_regset_new(&_jitc->function->regset); +} + +jit_int32_t +_jit_allocai(jit_state_t *_jit, jit_int32_t length) +{ + assert(_jitc->function); + jit_check_frame(); + switch (length) { + case 0: case 1: break; + case 2: _jitc->function->self.aoff &= -2; break; + case 3: case 4: _jitc->function->self.aoff &= -4; break; + default: _jitc->function->self.aoff &= -8; break; + } + _jitc->function->self.aoff -= length; + if (!_jitc->realize) { + jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length); + jit_dec_synth(); + } + return (_jitc->function->self.aoff); +} + +void +_jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v) +{ + jit_int32_t reg; + assert(_jitc->function); + jit_inc_synth_ww(allocar, u, v); + if (!_jitc->function->allocar) { + _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t)); + _jitc->function->allocar = 1; + } + reg = jit_get_reg(jit_class_gpr); + jit_negr(reg, v); + jit_andi(reg, reg, -16); + jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff); + jit_addr(u, u, reg); + jit_addr(JIT_SP, JIT_SP, reg); + jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u); + jit_unget_reg(reg); + jit_dec_synth(); +} + +void +_jit_ret(jit_state_t *_jit) +{ + jit_node_t *instr; + assert(_jitc->function); + jit_inc_synth(ret); + /* jump to epilog */ + instr = jit_jmpi(); + jit_patch_at(instr, _jitc->function->epilog); + jit_dec_synth(); +} + +void +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) +{ + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); + jit_ret(); + jit_dec_synth(); +} + +void +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) +{ + jit_code_inc_synth_w(code, u); + jit_movi(JIT_RET, u); + jit_ret(); + jit_dec_synth(); +} + +void +_jit_retr_f(jit_state_t *_jit, jit_int32_t u) +{ + jit_inc_synth_w(retr_f, u); + if (JIT_FRET != u) + jit_movr_f(JIT_FRET, u); + else + jit_live(JIT_FRET); + jit_ret(); + jit_dec_synth(); +} + +void +_jit_reti_f(jit_state_t *_jit, jit_float32_t u) +{ + jit_inc_synth_f(reti_f, u); + jit_movi_f(JIT_FRET, u); + jit_ret(); + jit_dec_synth(); +} + +void +_jit_retr_d(jit_state_t *_jit, jit_int32_t u) +{ + jit_inc_synth_w(retr_d, u); + if (JIT_FRET != u) + jit_movr_d(JIT_FRET, u); + else + jit_live(JIT_FRET); + jit_ret(); + jit_dec_synth(); +} + +void +_jit_reti_d(jit_state_t *_jit, jit_float64_t u) +{ + jit_inc_synth_d(reti_d, u); + jit_movi_d(JIT_FRET, u); + jit_ret(); + jit_dec_synth(); +} + +void +_jit_epilog(jit_state_t *_jit) +{ + assert(_jitc->function); + assert(_jitc->function->epilog->next == NULL); + jit_link(_jitc->function->epilog); + _jitc->function = NULL; +} + +jit_bool_t +_jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) +{ + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) + return (jit_arg_reg_p(u->u.w)); + assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); + return (jit_arg_f_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8)); +} + +void +_jit_ellipsis(jit_state_t *_jit) +{ + jit_inc_synth(ellipsis); + jit_check_frame(); + if (_jitc->prepare) { + jit_link_prepare(); + assert(!(_jitc->function->call.call & jit_call_varargs)); + _jitc->function->call.call |= jit_call_varargs; + } + else { + jit_link_prolog(); + assert(!(_jitc->function->self.call & jit_call_varargs)); + _jitc->function->self.call |= jit_call_varargs; + _jitc->function->vagp = _jitc->function->self.argi; + } + jit_dec_synth(); +} + +void +_jit_va_push(jit_state_t *_jit, jit_int32_t u) +{ + jit_inc_synth_w(va_push, u); + jit_pushargr(u); + jit_dec_synth(); +} + +jit_node_t * +_jit_arg(jit_state_t *_jit, jit_code_t code) +{ + jit_node_t *node; + jit_int32_t offset; + assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif + if (jit_arg_reg_p(_jitc->function->self.argi)) + offset = _jitc->function->self.argi++; + else { + offset = _jitc->function->self.size; + _jitc->function->self.size += sizeof(jit_word_t); + jit_check_frame(); + } + node = jit_new_node_ww(code, offset, + ++_jitc->function->self.argn); + jit_link_prolog(); + return (node); +} + +jit_node_t * +_jit_arg_f(jit_state_t *_jit) +{ + jit_node_t *node; + jit_int32_t offset; + assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); + if (jit_arg_f_reg_p(_jitc->function->self.argf)) + offset = _jitc->function->self.argf++; + else if (jit_arg_reg_p(_jitc->function->self.argi)) { + offset = _jitc->function->self.argi++; + offset += 8; + } + else { + offset = _jitc->function->self.size; + _jitc->function->self.size += sizeof(jit_word_t); + jit_check_frame(); + } + node = jit_new_node_ww(jit_code_arg_f, offset, + ++_jitc->function->self.argn); + jit_link_prolog(); + return (node); +} + +jit_node_t * +_jit_arg_d(jit_state_t *_jit) +{ + jit_node_t *node; + jit_int32_t offset; + assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); + if (jit_arg_f_reg_p(_jitc->function->self.argf)) + offset = _jitc->function->self.argf++; + else if (jit_arg_reg_p(_jitc->function->self.argi)) { + offset = _jitc->function->self.argi++; + offset += 8; + } + else { + offset = _jitc->function->self.size; + _jitc->function->self.size += sizeof(jit_word_t); + jit_check_frame(); + } + node = jit_new_node_ww(jit_code_arg_d, offset, + ++_jitc->function->self.argn); + jit_link_prolog(); + return (node); +} + +void +_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_c); + jit_inc_synth_wp(getarg_c, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_extr_c(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_c(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_c); + jit_inc_synth_wp(getarg_uc, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_extr_uc(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_uc(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_s); + jit_inc_synth_wp(getarg_s, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_extr_s(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_s(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_s); + jit_inc_synth_wp(getarg_us, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_extr_us(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_us(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_i); + jit_inc_synth_wp(getarg_i, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_extr_i(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_i(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_i); + jit_inc_synth_wp(getarg_ui, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_extr_ui(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_ui(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert_arg_type(v->code, jit_code_arg_l); + jit_inc_synth_wp(getarg_l, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_movr(u, _A0 - v->u.w); + else { + jit_node_t *node = jit_ldxi_l(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) +{ + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_movr(_A0 - v->u.w, u); + else { + jit_node_t *node = jit_stxi(v->u.w, JIT_FP, u); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) +{ + jit_int32_t regno; + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); + if (jit_arg_reg_p(v->u.w)) + jit_movi(_A0 - v->u.w, u); + else { + jit_node_t *node; + regno = jit_get_reg(jit_class_gpr); + jit_movi(regno, u); + node = jit_stxi(v->u.w, JIT_FP, regno); + jit_link_alist(node); + jit_unget_reg(regno); + } + jit_dec_synth(); +} + +void +_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert(v->code == jit_code_arg_f); + jit_inc_synth_wp(getarg_f, u, v); + if (jit_arg_f_reg_p(v->u.w)) + jit_movr_f(u, _FA0 - v->u.w); + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8)); + else { + jit_node_t *node = jit_ldxi_f(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert(v->code == jit_code_arg_f); + jit_inc_synth_wp(putargr_f, u, v); + if (jit_arg_f_reg_p(v->u.w)) + jit_movr_f(_FA0 - v->u.w, u); + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u); + else { + jit_node_t *node = jit_stxi_f(v->u.w, JIT_FP, u); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) +{ + jit_int32_t regno; + assert(v->code == jit_code_arg_f); + jit_inc_synth_fp(putargi_f, u, v); + if (jit_arg_f_reg_p(v->u.w)) + jit_movi_f(_FA0 - v->u.w, u); + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movi_f_w(JIT_RA0 - (v->u.w - 8), u); + else { + jit_node_t *node; + regno = jit_get_reg(jit_class_fpr); + jit_movi_f(regno, u); + node = jit_stxi_f(v->u.w, JIT_FP, regno); + jit_link_alist(node); + jit_unget_reg(regno); + } + jit_dec_synth(); +} + +void +_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert(v->code == jit_code_arg_d); + jit_inc_synth_wp(getarg_d, u, v); + if (jit_arg_f_reg_p(v->u.w)) + jit_movr_d(u, _FA0 - v->u.w); + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8)); + else { + jit_node_t *node = jit_ldxi_d(u, JIT_FP, v->u.w); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +{ + assert(v->code == jit_code_arg_d); + jit_inc_synth_wp(putargr_d, u, v); + if (jit_arg_f_reg_p(v->u.w)) + jit_movr_d(_FA0 - v->u.w, u); + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u); + else { + jit_node_t *node = jit_stxi_d(v->u.w, JIT_FP, u); + jit_link_alist(node); + } + jit_dec_synth(); +} + +void +_jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) +{ + jit_int32_t regno; + assert(v->code == jit_code_arg_d); + jit_inc_synth_dp(putargi_d, u, v); + if (jit_arg_f_reg_p(v->u.w)) + jit_movi_d(_FA0 - v->u.w, u); + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movi_d_w(JIT_RA0 - (v->u.w - 8), u); + else { + jit_node_t *node; + regno = jit_get_reg(jit_class_fpr); + jit_movi_d(regno, u); + node = jit_stxi_d(v->u.w, JIT_FP, regno); + jit_link_alist(node); + jit_unget_reg(regno); + } + jit_dec_synth(); +} + +void +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) +{ + assert(_jitc->function); + jit_code_inc_synth_w(code, u); + jit_link_prepare(); + if (jit_arg_reg_p(_jitc->function->call.argi)) { + jit_movr(_A0 - _jitc->function->call.argi, u); + ++_jitc->function->call.argi; + } + else { + jit_stxi(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); + } + jit_dec_synth(); +} + +void +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) +{ + jit_int32_t regno; + assert(_jitc->function); + jit_code_inc_synth_w(code, u); + jit_link_prepare(); + if (jit_arg_reg_p(_jitc->function->call.argi)) { + jit_movi(_A0 - _jitc->function->call.argi, u); + ++_jitc->function->call.argi; + } + else { + regno = jit_get_reg(jit_class_gpr); + jit_movi(regno, u); + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + jit_unget_reg(regno); + _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); + } + jit_dec_synth(); +} + +void +_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) +{ + assert(_jitc->function); + jit_inc_synth_w(pushargr_f, u); + jit_link_prepare(); + if (jit_arg_f_reg_p(_jitc->function->call.argf) && + !(_jitc->function->call.call & jit_call_varargs)) { + jit_movr_f(_FA0 - _jitc->function->call.argf, u); + ++_jitc->function->call.argf; + } + else if (jit_arg_reg_p(_jitc->function->call.argi)) { + jit_movr_f_w(_A0 - _jitc->function->call.argi, u); + ++_jitc->function->call.argi; + } + else { + jit_stxi_f(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); + } + jit_dec_synth(); +} + +void +_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) +{ + jit_int32_t regno; + assert(_jitc->function); + jit_inc_synth_f(pushargi_f, u); + jit_link_prepare(); + if (jit_arg_f_reg_p(_jitc->function->call.argf) && + !(_jitc->function->call.call & jit_call_varargs)) { + jit_movi_f(_FA0 - _jitc->function->call.argf, u); + ++_jitc->function->call.argf; + } + else if (jit_arg_reg_p(_jitc->function->call.argi)) { + jit_movi_f_w(_A0 - _jitc->function->call.argi, u); + ++_jitc->function->call.argi; + } + else { + regno = jit_get_reg(jit_class_fpr); + jit_movi_f(regno, u); + jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); + jit_unget_reg(regno); + _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); + } + jit_dec_synth(); +} + +void +_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) +{ + assert(_jitc->function); + jit_inc_synth_w(pushargr_d, u); + jit_link_prepare(); + if (jit_arg_f_reg_p(_jitc->function->call.argf) && + !(_jitc->function->call.call & jit_call_varargs)) { + jit_movr_d(_FA0 - _jitc->function->call.argf, u); + ++_jitc->function->call.argf; + } + else if (jit_arg_reg_p(_jitc->function->call.argi)) { + jit_movr_d_w(_A0 - _jitc->function->call.argi, u); + ++_jitc->function->call.argi; + } + else { + jit_stxi_d(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); + } + jit_dec_synth(); +} + +void +_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) +{ + jit_int32_t regno; + assert(_jitc->function); + jit_inc_synth_d(pushargi_d, u); + jit_link_prepare(); + if (jit_arg_f_reg_p(_jitc->function->call.argf) && + !(_jitc->function->call.call & jit_call_varargs)) { + jit_movi_d(_FA0 - _jitc->function->call.argf, u); + ++_jitc->function->call.argf; + } + else if (jit_arg_reg_p(_jitc->function->call.argi)) { + jit_movi_d_w(_A0 - _jitc->function->call.argi, u); + ++_jitc->function->call.argi; + } + else { + regno = jit_get_reg(jit_class_fpr); + jit_movi_d(regno, u); + jit_stxi_d(_jitc->function->call.size, JIT_SP, regno); + jit_unget_reg(regno); + _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); + } + jit_dec_synth(); +} + +jit_bool_t +_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno) +{ + jit_int32_t spec; + spec = jit_class(_rvs[regno].spec); + if (spec & jit_class_arg) { + regno = _A0 - regno; + if (regno >= 0 && regno < node->v.w) + return (1); + if (spec & jit_class_fpr) { + regno = _FA0 - regno; + if (regno >= 0 && regno < node->w.w) + return (1); + } + } + return (0); +} + +void +_jit_finishr(jit_state_t *_jit, jit_int32_t r0) +{ + jit_node_t *node; + assert(_jitc->function); + jit_check_frame(); + jit_inc_synth_w(finishr, r0); + if (_jitc->function->self.alen < _jitc->function->call.size) + _jitc->function->self.alen = _jitc->function->call.size; + node = jit_callr(r0); + node->v.w = _jitc->function->call.argi; + node->w.w = _jitc->function->call.argf; + _jitc->function->call.argi = _jitc->function->call.argf = + _jitc->function->call.size = 0; + _jitc->prepare = 0; + jit_dec_synth(); +} + +jit_node_t * +_jit_finishi(jit_state_t *_jit, jit_pointer_t i0) +{ + jit_node_t *node; + assert(_jitc->function); + jit_check_frame(); + jit_inc_synth_w(finishi, (jit_word_t)i0); + if (_jitc->function->self.alen < _jitc->function->call.size) + _jitc->function->self.alen = _jitc->function->call.size; + node = jit_calli(i0); + node->v.w = _jitc->function->call.argi; + node->w.w = _jitc->function->call.argf; + _jitc->function->call.argi = _jitc->function->call.argf = + _jitc->function->call.size = 0; + _jitc->prepare = 0; + jit_dec_synth(); + return (node); +} + +void +_jit_retval_c(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_c, r0); + jit_extr_c(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_uc, r0); + jit_extr_uc(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_s(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_s, r0); + jit_extr_s(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_us(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_us, r0); + jit_extr_us(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_i(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_i, r0); + jit_extr_i(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_ui(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_ui, r0); + jit_extr_ui(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_l(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_l, r0); + if (r0 != JIT_RET) + jit_movr(r0, JIT_RET); + jit_dec_synth(); +} + +void +_jit_retval_f(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_f, r0); + if (r0 != JIT_FRET) + jit_movr_f(r0, JIT_FRET); + jit_dec_synth(); +} + +void +_jit_retval_d(jit_state_t *_jit, jit_int32_t r0) +{ + jit_inc_synth_w(retval_d, r0); + if (r0 != JIT_FRET) + jit_movr_d(r0, JIT_FRET); + jit_dec_synth(); +} + +jit_pointer_t +_emit_code(jit_state_t *_jit) +{ + jit_node_t *node; + jit_node_t *temp; + jit_word_t word; + jit_int32_t value; + jit_int32_t offset; + struct { + jit_node_t *node; + jit_word_t word; + jit_function_t func; +#if DEVEL_DISASSEMBLER + jit_word_t prevw; +#endif + jit_int32_t patch_offset; + } undo; +#if DEVEL_DISASSEMBLER + jit_word_t prevw; +#endif + + _jitc->function = NULL; + + jit_reglive_setup(); + + undo.word = 0; + undo.node = NULL; + undo.patch_offset = 0; + +#define assert_data(node) /**/ +#define case_rr(name, type) \ + case jit_code_##name##r##type: \ + name##r##type(rn(node->u.w), rn(node->v.w)); \ + break +#define case_rw(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(rn(node->u.w), node->v.w); \ + break +#define case_wr(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(node->u.w, rn(node->v.w)); \ + break +#define case_rrr(name, type) \ + case jit_code_##name##r##type: \ + name##r##type(rn(node->u.w), \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_rrrr(name, type) \ + case jit_code_##name##r##type: \ + name##r##type(rn(node->u.q.l), rn(node->u.q.h), \ + rn(node->v.w), rn(node->w.w)); \ + break +#define case_rrw(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \ + break +#define case_rrrw(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(rn(node->u.q.l), rn(node->u.q.h), \ + rn(node->v.w), node->w.w); \ + break +#define case_rrf(name) \ + case jit_code_##name##i_f: \ + assert_data(node); \ + name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \ + break +#define case_rrd(name) \ + case jit_code_##name##i_d: \ + assert_data(node); \ + name##i_d(rn(node->u.w), rn(node->v.w),node->w.d); \ + break +#define case_wrr(name, type) \ + case jit_code_##name##i##type: \ + name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \ + break +#define case_brr(name, type) \ + case jit_code_##name##r##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##r##type(temp->u.w, rn(node->v.w), \ + rn(node->w.w)); \ + else { \ + word = name##r##type(_jit->pc.w, \ + rn(node->v.w), \ + rn(node->w.w)); \ + patch(word, node); \ + } \ + break +#define case_brw(name, type) \ + case jit_code_##name##i##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##i##type(temp->u.w, \ + rn(node->v.w), node->w.w); \ + else { \ + word = name##i##type(_jit->pc.w, \ + rn(node->v.w), node->w.w); \ + patch(word, node); \ + } \ + break; +#define case_brf(name) \ + case jit_code_##name##i_f: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##i_f(temp->u.w, rn(node->v.w), node->w.f); \ + else { \ + word = name##i_f(_jit->pc.w, rn(node->v.w), \ + node->w.f); \ + patch(word, node); \ + } \ + break +#define case_brd(name) \ + case jit_code_##name##i_d: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##i_d(temp->u.w, rn(node->v.w), node->w.d); \ + else { \ + word = name##i_d(_jit->pc.w, rn(node->v.w), \ + node->w.d); \ + patch(word, node); \ + } \ + break +#if DEVEL_DISASSEMBLER + prevw = _jit->pc.w; +#endif + for (node = _jitc->head; node; node = node->next) { + if (_jit->pc.uc >= _jitc->code.end) + return (NULL); + +#if DEVEL_DISASSEMBLER + node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw; + prevw = _jit->pc.w; +#endif + value = jit_classify(node->code); + jit_regarg_set(node, value); + switch (node->code) { + case jit_code_align: + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); + break; + case jit_code_note: case jit_code_name: + node->u.w = _jit->pc.w; + break; + case jit_code_label: + /* remember label is defined */ + node->flag |= jit_flag_patch; + node->u.w = _jit->pc.w; + break; + case_rrr(add,); + case_rrw(add,); + case_rrr(addc,); + case_rrw(addc,); + case_rrr(addx,); + case_rrw(addx,); + case_rrr(sub,); + case_rrw(sub,); + case_rrr(subc,); + case_rrw(subc,); + case_rrr(subx,); + case_rrw(subx,); + case_rrw(rsb,); + case_rrr(mul,); + case_rrw(mul,); + case_rrrr(qmul,); + case_rrrw(qmul,); + case_rrrr(qmul, _u); + case_rrrw(qmul, _u); + case_rrr(div,); + case_rrw(div,); + case_rrr(div, _u); + case_rrw(div, _u); + case_rrr(rem,); + case_rrw(rem,); + case_rrr(rem, _u); + case_rrw(rem, _u); + case_rrrr(qdiv,); + case_rrrw(qdiv,); + case_rrrr(qdiv, _u); + case_rrrw(qdiv, _u); + case_rrr(lsh,); + case_rrw(lsh,); + case_rrr(rsh,); + case_rrw(rsh,); + case_rrr(rsh, _u); + case_rrw(rsh, _u); + case_rr(neg,); + case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); + case_rrr(and,); + case_rrw(and,); + case_rrr(or,); + case_rrw(or,); + case_rrr(xor,); + case_rrw(xor,); + case_rr(trunc, _f_i); + case_rr(trunc, _d_i); + case_rr(trunc, _f_l); + case_rr(trunc, _d_l); + case_rr(ld, _c); + case_rw(ld, _c); + case_rr(ld, _uc); + case_rw(ld, _uc); + case_rr(ld, _s); + case_rw(ld, _s); + case_rr(ld, _us); + case_rw(ld, _us); + case_rr(ld, _i); + case_rw(ld, _i); + case_rr(ld, _ui); + case_rw(ld, _ui); + case_rr(ld, _l); + case_rw(ld, _l); + case_rrr(ldx, _c); + case_rrw(ldx, _c); + case_rrr(ldx, _uc); + case_rrw(ldx, _uc); + case_rrr(ldx, _s); + case_rrw(ldx, _s); + case_rrr(ldx, _us); + case_rrw(ldx, _us); + case_rrr(ldx, _i); + case_rrw(ldx, _i); + case_rrr(ldx, _ui); + case_rrw(ldx, _ui); + case_rrr(ldx, _l); + case_rrw(ldx, _l); + case_rr(st, _c); + case_wr(st, _c); + case_rr(st, _s); + case_wr(st, _s); + case_rr(st, _i); + case_wr(st, _i); + case_rr(st, _l); + case_wr(st, _l); + case_rrr(stx, _c); + case_wrr(stx, _c); + case_rrr(stx, _s); + case_wrr(stx, _s); + case_rrr(stx, _i); + case_wrr(stx, _i); + case_rrr(stx, _l); + case_wrr(stx, _l); + case_rr(hton, _us); + case_rr(hton, _ui); + case_rr(hton, _ul); + case_rr(bswap, _us); + case_rr(bswap, _ui); + case_rr(bswap, _ul); + case_rr(ext, _c); + case_rr(ext, _uc); + case_rr(ext, _s); + case_rr(ext, _us); + case_rr(ext, _i); + case_rr(ext, _ui); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); + case_rr(mov,); + case jit_code_movi: + if (node->flag & jit_flag_node) { + temp = node->v.n; + if (temp->code == jit_code_data || + (temp->code == jit_code_label && + (temp->flag & jit_flag_patch))) + movi(rn(node->u.w), temp->u.w); + else { + assert(temp->code == jit_code_label || + temp->code == jit_code_epilog); + word = movi_p(rn(node->u.w), temp->u.w); + patch(word, node); + } + } + else + movi(rn(node->u.w), node->v.w); + break; + case_rrr(lt,); + case_rrw(lt,); + case_rrr(lt, _u); + case_rrw(lt, _u); + case_rrr(le,); + case_rrw(le,); + case_rrr(le, _u); + case_rrw(le, _u); + case_rrr(eq,); + case_rrw(eq,); + case_rrr(ge,); + case_rrw(ge,); + case_rrr(ge, _u); + case_rrw(ge, _u); + case_rrr(gt,); + case_rrw(gt,); + case_rrr(gt, _u); + case_rrw(gt, _u); + case_rrr(ne,); + case_rrw(ne,); + case_brr(blt,); + case_brw(blt,); + case_brr(blt, _u); + case_brw(blt, _u); + case_brr(ble,); + case_brw(ble,); + case_brr(ble, _u); + case_brw(ble, _u); + case_brr(beq,); + case_brw(beq,); + case_brr(bge,); + case_brw(bge,); + case_brr(bge, _u); + case_brw(bge, _u); + case_brr(bgt,); + case_brw(bgt,); + case_brr(bgt, _u); + case_brw(bgt, _u); + case_brr(bne,); + case_brw(bne,); + case_brr(boadd,); + case_brw(boadd,); + case_brr(boadd, _u); + case_brw(boadd, _u); + case_brr(bxadd,); + case_brw(bxadd,); + case_brr(bxadd, _u); + case_brw(bxadd, _u); + case_brr(bosub,); + case_brw(bosub,); + case_brr(bosub, _u); + case_brw(bosub, _u); + case_brr(bxsub,); + case_brw(bxsub,); + case_brr(bxsub, _u); + case_brw(bxsub, _u); + case_brr(bms,); + case_brw(bms,); + case_brr(bmc,); + case_brw(bmc,); + case_rrr(add, _f); + case_rrf(add); + case_rrr(sub, _f); + case_rrf(sub); + case_rrf(rsb); + case_rrr(mul, _f); + case_rrf(mul); + case_rrr(div, _f); + case_rrf(div); + case_rr(abs, _f); + case_rr(neg, _f); + case_rr(sqrt, _f); + case_rr(ext, _f); + case_rr(ld, _f); + case_rw(ld, _f); + case_rrr(ldx, _f); + case_rrw(ldx, _f); + case_rr(st, _f); + case_wr(st, _f); + case_rrr(stx, _f); + case_wrr(stx, _f); + case_rr(mov, _f); + case jit_code_movi_f: + assert_data(node); + movi_f(rn(node->u.w), node->v.f); + break; + case_rr(ext, _d_f); + case_rrr(lt, _f); + case_rrf(lt); + case_rrr(le, _f); + case_rrf(le); + case_rrr(eq, _f); + case_rrf(eq); + case_rrr(ge, _f); + case_rrf(ge); + case_rrr(gt, _f); + case_rrf(gt); + case_rrr(ne, _f); + case_rrf(ne); + case_rrr(unlt, _f); + case_rrf(unlt); + case_rrr(unle, _f); + case_rrf(unle); + case_rrr(uneq, _f); + case_rrf(uneq); + case_rrr(unge, _f); + case_rrf(unge); + case_rrr(ungt, _f); + case_rrf(ungt); + case_rrr(ltgt, _f); + case_rrf(ltgt); + case_rrr(ord, _f); + case_rrf(ord); + case_rrr(unord, _f); + case_rrf(unord); + case_brr(blt, _f); + case_brf(blt); + case_brr(ble, _f); + case_brf(ble); + case_brr(beq, _f); + case_brf(beq); + case_brr(bge, _f); + case_brf(bge); + case_brr(bgt, _f); + case_brf(bgt); + case_brr(bne, _f); + case_brf(bne); + case_brr(bunlt, _f); + case_brf(bunlt); + case_brr(bunle, _f); + case_brf(bunle); + case_brr(buneq, _f); + case_brf(buneq); + case_brr(bunge, _f); + case_brf(bunge); + case_brr(bungt, _f); + case_brf(bungt); + case_brr(bltgt, _f); + case_brf(bltgt); + case_brr(bord, _f); + case_brf(bord); + case_brr(bunord, _f); + case_brf(bunord); + case_rrr(add, _d); + case_rrd(add); + case_rrr(sub, _d); + case_rrd(sub); + case_rrd(rsb); + case_rrr(mul, _d); + case_rrd(mul); + case_rrr(div, _d); + case_rrd(div); + case_rr(abs, _d); + case_rr(neg, _d); + case_rr(sqrt, _d); + case_rr(ext, _d); + case_rr(ld, _d); + case_rw(ld, _d); + case_rrr(ldx, _d); + case_rrw(ldx, _d); + case_rr(st, _d); + case_wr(st, _d); + case_rrr(stx, _d); + case_wrr(stx, _d); + case_rr(mov, _d); + case jit_code_movi_d: + assert_data(node); + movi_d(rn(node->u.w), node->v.d); + break; + case_rr(ext, _f_d); + case_rrr(lt, _d); + case_rrd(lt); + case_rrr(le, _d); + case_rrd(le); + case_rrr(eq, _d); + case_rrd(eq); + case_rrr(ge, _d); + case_rrd(ge); + case_rrr(gt, _d); + case_rrd(gt); + case_rrr(ne, _d); + case_rrd(ne); + case_rrr(unlt, _d); + case_rrd(unlt); + case_rrr(unle, _d); + case_rrd(unle); + case_rrr(uneq, _d); + case_rrd(uneq); + case_rrr(unge, _d); + case_rrd(unge); + case_rrr(ungt, _d); + case_rrd(ungt); + case_rrr(ltgt, _d); + case_rrd(ltgt); + case_rrr(ord, _d); + case_rrd(ord); + case_rrr(unord, _d); + case_rrd(unord); + case_brr(blt, _d); + case_brd(blt); + case_brr(ble, _d); + case_brd(ble); + case_brr(beq, _d); + case_brd(beq); + case_brr(bge, _d); + case_brd(bge); + case_brr(bgt, _d); + case_brd(bgt); + case_brr(bne, _d); + case_brd(bne); + case_brr(bunlt, _d); + case_brd(bunlt); + case_brr(bunle, _d); + case_brd(bunle); + case_brr(buneq, _d); + case_brd(buneq); + case_brr(bunge, _d); + case_brd(bunge); + case_brr(bungt, _d); + case_brd(bungt); + case_brr(bltgt, _d); + case_brd(bltgt); + case_brr(bord, _d); + case_brd(bord); + case_brr(bunord, _d); + case_brd(bunord); + case jit_code_jmpr: + jit_check_frame(); + jmpr(rn(node->u.w)); + break; + case jit_code_jmpi: + if (node->flag & jit_flag_node) { + temp = node->u.n; + assert(temp->code == jit_code_label || + temp->code == jit_code_epilog); + if (temp->flag & jit_flag_patch) + jmpi(temp->u.w); + else { + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (can_sign_extend_si26_p(word)) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); + patch(word, node); + } + } + else { + jit_check_frame(); + jmpi(node->u.w); + } + break; + case jit_code_callr: + jit_check_frame(); + callr(rn(node->u.w)); + break; + case jit_code_calli: + if (node->flag & jit_flag_node) { + temp = node->u.n; + assert(temp->code == jit_code_label || + temp->code == jit_code_epilog); + if (temp->flag & jit_flag_patch) + calli(temp->u.w); + else { + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (can_sign_extend_si26_p(word)) + word = calli(_jit->pc.w); + else + word = calli_p(_jit->pc.w); + patch(word, node); + } + } + else { + jit_check_frame(); + calli(node->u.w); + } + break; + case jit_code_prolog: + _jitc->function = _jitc->functions.ptr + node->w.w; + undo.node = node; + undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); +#if DEVEL_DISASSEMBLER + undo.prevw = prevw; +#endif + undo.patch_offset = _jitc->patches.offset; + restart_function: + compute_framesize(); + patch_alist(0); + _jitc->again = 0; + prolog(node); + break; + case jit_code_epilog: + assert(_jitc->function == _jitc->functions.ptr + node->w.w); + if (_jitc->again) { + for (temp = undo.node->next; + temp != node; temp = temp->next) { + if (temp->code == jit_code_label || + temp->code == jit_code_epilog) + temp->flag &= ~jit_flag_patch; + } + temp->flag &= ~jit_flag_patch; + node = undo.node; + _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + /* this will be recomputed but undo anyway to have it + * better self documented.*/ + undo.func.need_stack = _jitc->function->need_stack; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); +#if DEVEL_DISASSEMBLER + prevw = undo.prevw; +#endif + _jitc->patches.offset = undo.patch_offset; + patch_alist(1); + goto restart_function; + } + if (node->link && (word = _jit->pc.w & 3)) + nop(4 - word); + /* remember label is defined */ + node->flag |= jit_flag_patch; + node->u.w = _jit->pc.w; + epilog(node); + _jitc->function = NULL; + break; + case jit_code_movr_w_f: + movr_w_f(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movr_f_w: + movr_f_w(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movi_f_w: + assert_data(node); + movi_f_w(rn(node->u.w), node->v.f); + break; + case jit_code_movr_w_d: + movr_w_d(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movr_d_w: + movr_d_w(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movi_d_w: + assert_data(node); + movi_d_w(rn(node->u.w), node->v.d); + break; + case jit_code_va_start: + vastart(rn(node->u.w)); + break; + case jit_code_va_arg: + vaarg(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_va_arg_d: + vaarg_d(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_live: case jit_code_ellipsis: + case jit_code_va_push: + case jit_code_allocai: case jit_code_allocar: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_l: + case jit_code_arg_f: case jit_code_arg_d: + case jit_code_va_end: + case jit_code_ret: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: + case jit_code_retr_f: case jit_code_reti_f: + case jit_code_retr_d: case jit_code_reti_d: + case jit_code_getarg_c: case jit_code_getarg_uc: + case jit_code_getarg_s: case jit_code_getarg_us: + case jit_code_getarg_i: + case jit_code_getarg_ui: case jit_code_getarg_l: + case jit_code_getarg_f: case jit_code_getarg_d: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: + case jit_code_putargr_f: case jit_code_putargi_f: + case jit_code_putargr_d: case jit_code_putargi_d: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: + case jit_code_pushargr_f: case jit_code_pushargi_f: + case jit_code_pushargr_d: case jit_code_pushargi_d: + case jit_code_retval_c: case jit_code_retval_uc: + case jit_code_retval_s: case jit_code_retval_us: + case jit_code_retval_i: + case jit_code_retval_ui: case jit_code_retval_l: + case jit_code_retval_f: case jit_code_retval_d: + case jit_code_prepare: + case jit_code_finishr: case jit_code_finishi: + break; + default: + abort(); + } + if (jit_carry != _NOREG) { + switch (node->code) { + case jit_code_note: + case jit_code_addcr: case jit_code_addci: + case jit_code_addxr: case jit_code_addxi: + case jit_code_subcr: case jit_code_subci: + case jit_code_subxr: case jit_code_subxi: + break; + default: + jit_unget_reg(jit_carry); + jit_carry = _NOREG; + break; + } + } + jit_regarg_clr(node, value); + assert(_jitc->regarg == 0 || + (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry))); + assert(_jitc->synth == 0); + /* update register live state */ + jit_reglive(node); + } +#undef case_brw +#undef case_brr +#undef case_wrr +#undef case_rrw +#undef case_rrr +#undef case_wr +#undef case_rw +#undef case_rr + + for (offset = 0; offset < _jitc->patches.offset; offset++) { + node = _jitc->patches.ptr[offset].node; + word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w; + patch_at(_jitc->patches.ptr[offset].inst, word); + } + + jit_flush(_jit->code.ptr, _jit->pc.uc); + + return (_jit->code.ptr); +} + +#define CODE 1 +# include "jit_loongarch-cpu.c" +# include "jit_loongarch-fpu.c" +#undef CODE + +void +jit_flush(void *fptr, void *tptr) +{ +#if defined(__GNUC__) + jit_word_t f, t, s; + + s = sysconf(_SC_PAGE_SIZE); + f = (jit_word_t)fptr & -s; + t = (((jit_word_t)tptr) + s - 1) & -s; + __clear_cache((void *)f, (void *)t); +#endif +} + +void +_emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + ldxi(rn(r0), rn(r1), i0); +} + +void +_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + stxi(i0, rn(r0), rn(r1)); +} + +void +_emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) +{ + ldxi_d(rn(r0), rn(r1), i0); +} + +void +_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +{ + stxi_d(i0, rn(r0), rn(r1)); +} + +static void +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = 16; /* ra+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += sizeof(jit_word_t); + + for (reg = 0; reg < jit_size(fregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) + _jitc->framesize += sizeof(jit_float64_t); + + /* Space to store variadic arguments */ + if (_jitc->function->self.call & jit_call_varargs) + _jitc->framesize += (8 - _jitc->function->vagp) * 8; + + /* Make sure functions called have a 16 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 15) & -16; +} + +static void +_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) +{ + jit_int32_t flag; + + assert(node->flag & jit_flag_node); + if (node->code == jit_code_movi) + flag = node->v.n->flag; + else + flag = node->u.n->flag; + assert(!(flag & jit_flag_patch)); + if (_jitc->patches.offset >= _jitc->patches.length) { + jit_realloc((jit_pointer_t *)&_jitc->patches.ptr, + _jitc->patches.length * sizeof(jit_patch_t), + (_jitc->patches.length + 1024) * sizeof(jit_patch_t)); + _jitc->patches.length += 1024; + } + _jitc->patches.ptr[_jitc->patches.offset].inst = instr; + _jitc->patches.ptr[_jitc->patches.offset].node = node; + ++_jitc->patches.offset; +} diff --git a/parallel-rsp/lightning/lib/jit_memory.c b/parallel-rsp/lightning/lib/jit_memory.c index 4d7f92d..8e736da 100644 --- a/parallel-rsp/lightning/lib/jit_memory.c +++ b/parallel-rsp/lightning/lib/jit_memory.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -19,7 +19,6 @@ #include #include -#include /* * Prototypes diff --git a/parallel-rsp/lightning/lib/jit_mips-cpu.c b/parallel-rsp/lightning/lib/jit_mips-cpu.c index 8fb7fa1..7d0fa92 100644 --- a/parallel-rsp/lightning/lib/jit_mips-cpu.c +++ b/parallel-rsp/lightning/lib/jit_mips-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -52,10 +52,11 @@ typedef union { struct { jit_uint32_t _:16; jit_uint32_t b : 16; } is; struct { jit_uint32_t _: 6; jit_uint32_t b : 26; } ii; #endif + jit_uint32_t op_u32; int op; } jit_instr_t; -/* FIXME */ -# define jit_mips2_p() 0 +#define jit_mips2_p() (jit_cpu.release >= 2) +#define jit_mips6_p() (jit_cpu.release >= 6) # define _ZERO_REGNO 0 # define _T0_REGNO 0x08 # define _T1_REGNO 0x09 @@ -87,26 +88,27 @@ typedef union { # define _F28_REGNO 28 # define _F30_REGNO 30 # if __WORDSIZE == 32 -# if NEW_ABI -# define stack_framesize 144 -# else -# define stack_framesize 112 -# endif # define ldr(u,v) ldr_i(u,v) # define ldi(u,v) ldi_i(u,v) # define ldxi(u,v,w) ldxi_i(u,v,w) # define sti(u,v) sti_i(u,v) # define stxi(u,v,w) stxi_i(u,v,w) # else -# define stack_framesize 144 # define ldr(u,v) ldr_l(u,v) # define ldi(u,v) ldi_l(u,v) # define ldxi(u,v,w) ldxi_l(u,v,w) # define sti(u,v) sti_l(u,v) # define stxi(u,v,w) stxi_l(u,v,w) # endif +/* can_relative_jump_p(im) => can_sign_extend_short_p(im << 2) */ +# define can_relative_jump_p(im) ((im) >= -130712 && (im) <= 131068) # define can_sign_extend_short_p(im) ((im) >= -32678 && (im) <= 32767) # define can_zero_extend_short_p(im) ((im) >= 0 && (im) <= 65535) +# define is_low_mask(im) (((im) & 1) ? (__builtin_popcountl((im) + 1) <= 1) : 0) +# define is_middle_mask(im) ((im) ? (__builtin_popcountl((im) + (1 << __builtin_ctzl(im))) <= 1) : 0) +# define is_high_mask(im) ((im) ? (__builtin_popcountl((im) + (1 << __builtin_ctzl(im))) == 0) : 0) +# define masked_bits_count(im) __builtin_popcountl(im) +# define unmasked_bits_count(im) (__WORDSIZE - masked_bits_count(im)) # if __WORDSIZE == 32 # define can_sign_extend_int_p(im) 1 # define can_zero_extend_int_p(im) 1 @@ -317,6 +319,7 @@ static void _nop(jit_state_t*,jit_int32_t); # define DADDIU(rt,rs,im) hrri(MIPS_DADDIU,rs,rt,im) # define SUBU(rd,rs,rt) rrr_t(rs,rt,rd,MIPS_SUBU) # define DSUBU(rd,rs,rt) rrr_t(rs,rt,rd,MIPS_DSUBU) +# define MUL(rd,rs,rt) hrrr_t(MIPS_SPECIAL2,rs,rt,rd,MIPS_MUL) # define MULT(rs,rt) rrr_t(rs,rt,_ZERO_REGNO,MIPS_MULT) # define MULTU(rs,rt) rrr_t(rs,rt,_ZERO_REGNO,MIPS_MULTU) # define DMULT(rs,rt) rrr_t(rs,rt,_ZERO_REGNO,MIPS_DMULT) @@ -340,10 +343,17 @@ static void _nop(jit_state_t*,jit_int32_t); # define DSRLV(rd,rt,rs) rrr_t(rs,rt,rd,MIPS_DSRLV) # define DSRL(rd,rt,sa) rrit(rt,rd,sa,MIPS_DSRL) # define DSRL32(rd,rt,sa) rrit(rt,rd,sa,MIPS_DSRL32) -# define INS(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,pos,pos+size-1,MIPS_INS) -# define DINS(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,pos,pos+size-1,MIPS_DINS) +# define INS(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,pos+size-1,pos,MIPS_INS) +# define DINS(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,pos+size-1,pos,MIPS_DINS) +# define DINSU(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,pos+size-32-1,pos-32,MIPS_DINSU) +# define DINSM(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,pos+size-32-1,pos,MIPS_DINSM) +# define EXT(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,size-1,pos,MIPS_EXT) +# define DEXT(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,size-1,pos,MIPS_DEXT) +# define DEXTU(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,size-1,pos-32,MIPS_DEXTU) +# define DEXTM(rt,rs,pos,size) hrrrit(MIPS_SPECIAL3,rs,rt,size-32-1,pos,MIPS_DEXTM) # define ROTR(rd,rt,sa) hrrrit(MIPS_SPECIAL,1,rt,rd,sa,MIPS_SRL) # define DROTR(rd,rt,sa) hrrrit(MIPS_SPECIAL,1,rt,rd,sa,MIPS_DSRL) +# define SYNC() rrr_t(_ZERO_REGNO,_ZERO_REGNO,_ZERO_REGNO,MIPS_SYNC) # define MFHI(rd) rrr_t(_ZERO_REGNO,_ZERO_REGNO,rd,MIPS_MFHI) # define MFLO(rd) rrr_t(_ZERO_REGNO,_ZERO_REGNO,rd,MIPS_MFLO) # define MTHI(rs) rrr_t(rs,_ZERO_REGNO,_ZERO_REGNO,MIPS_MTHI) @@ -361,10 +371,14 @@ static void _nop(jit_state_t*,jit_int32_t); # define LW(rt,of,rb) hrri(MIPS_LW,rb,rt,of) # define LWU(rt,of,rb) hrri(MIPS_LWU,rb,rt,of) # define LD(rt,of,rb) hrri(MIPS_LD,rb,rt,of) +# define LL(rt,of,rb) hrri(MIPS_LL,rb,rt,of) +# define LLD(rt,of,rb) hrri(MIPS_LLD,rb,rt,of) # define SB(rt,of,rb) hrri(MIPS_SB,rb,rt,of) # define SH(rt,of,rb) hrri(MIPS_SH,rb,rt,of) # define SW(rt,of,rb) hrri(MIPS_SW,rb,rt,of) # define SD(rt,of,rb) hrri(MIPS_SD,rb,rt,of) +# define SC(rt,of,rb) hrri(MIPS_SC,rb,rt,of) +# define SCD(rt,of,rb) hrri(MIPS_SCD,rb,rt,of) # define WSBH(rd,rt) hrrrit(MIPS_SPECIAL3,0,rt,rd,MIPS_WSBH,MIPS_BSHFL) # define SEB(rd,rt) hrrrit(MIPS_SPECIAL3,0,rt,rd,MIPS_SEB,MIPS_BSHFL) # define SEH(rd,rt) hrrrit(MIPS_SPECIAL3,0,rt,rd,MIPS_SEH,MIPS_BSHFL) @@ -378,16 +392,39 @@ static void _nop(jit_state_t*,jit_int32_t); # define BGEZ(rs,im) hrri(MIPS_REGIMM,rs,MIPS_BGEZ,im) # define BGTZ(rs,im) hrri(MIPS_BGTZ,rs,_ZERO_REGNO,im) # define BNE(rs,rt,im) hrri(MIPS_BNE,rs,rt,im) +# define BGEZAL(rs,im) hrri(MIPS_REGIMM,rs,MIPS_BGEZAL,im) # define JALR(r0) hrrrit(MIPS_SPECIAL,r0,0,_RA_REGNO,0,MIPS_JALR) -# if 1 /* supports MIPS32 R6 */ -# define JR(r0) hrrrit(MIPS_SPECIAL,r0,0,0,0,MIPS_JALR) -# else /* does not support MIPS32 R6 */ -# define JR(r0) hrrrit(MIPS_SPECIAL,r0,0,0,0,MIPS_JR) +# if 1 /* This should work for mips r6 or older */ +# define JR(r0) hrrrit(MIPS_SPECIAL,r0,0,0,0,MIPS_JALR) +# else /* This should generate an illegal instruction in mips r6 */ +# define JR(r0) hrrrit(MIPS_SPECIAL,r0,0,0,0,MIPS_JR) # endif +# define CLO_R6(rd,rs) hrrrit(MIPS_SPECIAL,rs,0,rd,1,0x11) +# define DCLO_R6(rd,rs) hrrrit(MIPS_SPECIAL,rs,0,rd,1,0x13) +# define CLZ_R6(rd,rs) hrrrit(MIPS_SPECIAL,rs,0,rd,1,0x10) +# define DCLZ_R6(rd,rs) hrrrit(MIPS_SPECIAL,rs,0,rd,1,0x12) +# define BITSWAP(rd,rt) hrrrit(MIPS_SPECIAL3,0,rt,rd,0,0x20) +# define DBITSWAP(rd,rt) hrrrit(MIPS_SPECIAL3,0,rt,rd,0,0x24) +# define CLO(rd,rs) hrrrit(MIPS_SPECIAL2,rs,rd,rd,0,MIPS_CLO) +# define DCLO(rd,rs) hrrrit(MIPS_SPECIAL2,rs,rd,rd,0,MIPS_DCLO) +# define CLZ(rd,rs) hrrrit(MIPS_SPECIAL2,rs,rd,rd,0,MIPS_CLZ) +# define DCLZ(rd,rs) hrrrit(MIPS_SPECIAL2,rs,rd,rd,0,MIPS_DCLZ) # define J(i0) hi(MIPS_J,i0) +# define JAL(i0) hi(MIPS_JAL,i0) +# define MOVN(rd,rs,rt) hrrrit(0,rs,rt,rd,0,MIPS_MOVN) # define MOVZ(rd,rs,rt) hrrrit(0,rs,rt,rd,0,MIPS_MOVZ) # define comr(r0,r1) xori(r0,r1,-1) # define negr(r0,r1) subr(r0,_ZERO_REGNO,r1) +# define bitswap(r0,r1) _bitswap(_jit, r0, r1); +static void _bitswap(jit_state_t*,jit_int32_t,jit_int32_t); +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) _clzr(_jit, r0, r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # if __WORDSIZE == 32 # define addr(rd,rs,rt) ADDU(rd,rs,rt) # define addiu(r0,r1,i0) ADDIU(r0,r1,i0) @@ -405,6 +442,10 @@ static void _nop(jit_state_t*,jit_int32_t); # define div(rs,rt) DDIV(rs,rt) # define divu(rs,rt) DDIVU(rs,rt) # endif +# define extr(rd,rt,lsb,nb) _extr(_jit,rd,rt,lsb,nb) +static void _extr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); +# define insr(rd,rt,lsb,nb) _insr(_jit,rd,rt,lsb,nb) +static void _insr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); # define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); #define addcr(r0,r1,r2) _addcr(_jit,r0,r1,r2) @@ -494,11 +535,19 @@ static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); # define xorr(r0,r1,r2) XOR(r0,r1,r2) # define xori(r0,r1,i0) _xori(_jit,r0,r1,i0) static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); -# define movr(r0,r1) orr(r0,r1,_ZERO_REGNO) +# define movr(r0,r1) _movr(_jit,r0,r1) +static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); # define movi(r0,i0) _movi(_jit,r0,i0) static void _movi(jit_state_t*,jit_int32_t,jit_word_t); # define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define movnr(r0,r1,r2) MOVN(r0, r1, r2) +# define movzr(r0,r1,r2) MOVZ(r0, r1, r2) +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define ldr_c(r0,r1) LB(r0,0,r1) # define ldi_c(r0,i0) _ldi_c(_jit,r0,i0) static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t); @@ -584,23 +633,12 @@ static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define stxi_l(i0,r0,r1) _stxi_l(_jit,i0,r0,r1) static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); # endif -# if __BYTE_ORDER == __LITTLE_ENDIAN -# define htonr_us(r0,r1) _htonr_us(_jit,r0,r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0,r1) _htonr_ui(_jit,r0,r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# if __WORDSIZE == 64 -# define htonr_ul(r0,r1) _htonr_ul(_jit,r0,r1) -static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t); -# endif -# else -# define htonr_us(r0,r1) extr_us(r0,r1) -# if __WORDSIZE == 32 -# define htonr_ui(r0,r1) movr(r0,r1) -# else -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) -# endif +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# if __WORDSIZE == 64 +# define bswapr_ul(r0,r1) generic_bswapr_ul(_jit,r0,r1) # endif # define extr_c(r0,r1) _extr_c(_jit,r0,r1) static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); @@ -649,50 +687,43 @@ static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); #define nei(r0,r1,i0) _nei(_jit,r0,r1,i0) static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); -#define bltr(i0,r0,r1) _bltr(_jit,i0,r0,r1) -static jit_word_t _bltr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bltr_u(i0,r0,r1) _bltr_u(_jit,i0,r0,r1) -static jit_word_t _bltr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define blti(i0,r0,i1) _blti(_jit,i0,r0,i1) -static jit_word_t _blti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define blti_u(i0,r0,i1) _blti_u(_jit,i0,r0,i1) -static jit_word_t _blti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define bler(i0,r0,r1) _bler(_jit,i0,r0,r1) -static jit_word_t _bler(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bler_u(i0,r0,r1) _bler_u(_jit,i0,r0,r1) -static jit_word_t _bler_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define blei(i0,r0,i1) _blei(_jit,i0,r0,i1) -static jit_word_t _blei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define blei_u(i0,r0,i1) _blei_u(_jit,i0,r0,i1) -static jit_word_t _blei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define beqr(i0,r0,r1) _beqr(_jit,i0,r0,r1) -static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define beqi(i0,r0,i1) _beqi(_jit,i0,r0,i1) -static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define bger(i0,r0,r1) _bger(_jit,i0,r0,r1) -static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bger_u(i0,r0,r1) _bger_u(_jit,i0,r0,r1) -static jit_word_t _bger_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bgei(i0,r0,i1) _bgei(_jit,i0,r0,i1) -static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define bgei_u(i0,r0,i1) _bgei_u(_jit,i0,r0,i1) -static jit_word_t _bgei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define bgtr(i0,r0,r1) _bgtr(_jit,i0,r0,r1) -static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bgtr_u(i0,r0,r1) _bgtr_u(_jit,i0,r0,r1) -static jit_word_t _bgtr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bgti(i0,r0,i1) _bgti(_jit,i0,r0,i1) -static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define bgti_u(i0,r0,i1) _bgti_u(_jit,i0,r0,i1) -static jit_word_t _bgti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -#define bner(i0,r0,r1) _bner(_jit,i0,r0,r1) -static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -#define bnei(i0,r0,i1) _bnei(_jit,i0,r0,i1) -static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -# define jmpr(r0) _jmpr(_jit,r0) -static void _jmpr(jit_state_t*,jit_int32_t); -# define jmpi(i0) _jmpi(_jit,i0) -static jit_word_t _jmpi(jit_state_t*,jit_word_t); +#define bltr(i0,r0,r1,prev) bger(i0,r1,r0,prev) +#define bltr_u(i0,r0,r1,prev) bger_u(i0,r1,r0,prev) +#define blti(i0,r0,i1,prev) _bgei(_jit,i0,r0,i1,0,1,prev) +#define blti_u(i0,r0,i1,prev) _bgei(_jit,i0,r0,i1,1,1,prev) +#define bler(i0,r0,r1,prev) _bgtr(_jit,i0,r1,r0,0,1,prev) +#define bler_u(i0,r0,r1,prev) _bgtr(_jit,i0,r1,r0,1,1,prev) +#define blei(i0,r0,i1,prev) _bgti(_jit,i0,r0,i1,0,1,prev) +#define blei_u(i0,r0,i1,prev) _bgti(_jit,i0,r0,i1,1,1,prev) +#define beqr(i0,r0,r1,prev) _beqr(_jit,i0,r0,r1,prev) +static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t,jit_node_t*); +#define beqi(i0,r0,i1,prev) _beqi(_jit,i0,r0,i1,prev) +static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t,jit_node_t*); +#define bger(i0,r0,r1,prev) _bger(_jit,i0,r0,r1,0,prev) +#define bger_u(i0,r0,r1,prev) _bger(_jit,i0,r0,r1,1,prev) +static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t,jit_bool_t,jit_node_t*); +#define bgei(i0,r0,i1,prev) _bgei(_jit,i0,r0,i1,0,0,prev) +#define bgei_u(i0,r0,i1,prev) _bgei(_jit,i0,r0,i1,1,0,prev) +static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t, + jit_bool_t,jit_bool_t,jit_node_t*); +#define bgtr(i0,r0,r1,prev) _bgtr(_jit,i0,r0,r1,0,0,prev) +#define bgtr_u(i0,r0,r1,prev) _bgtr(_jit,i0,r0,r1,1,0,prev) +static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t, + jit_bool_t,jit_bool_t,jit_node_t*); +#define bgti(i0,r0,i1,prev) _bgti(_jit,i0,r0,i1,0,0,prev) +#define bgti_u(i0,r0,i1,prev) _bgti(_jit,i0,r0,i1,1,0,prev) +static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t, + jit_bool_t,jit_bool_t,jit_node_t*); +#define bner(i0,r0,r1,prev) _bner(_jit,i0,r0,r1,prev) +static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t,jit_node_t*); +#define bnei(i0,r0,i1,prev) _bnei(_jit,i0,r0,i1,prev) +static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t,jit_node_t*); +# define jmpr(r0,prev) _jmpr(_jit,r0,prev) +static void _jmpr(jit_state_t*,jit_int32_t,jit_node_t*); +# define jmpi(i0,prev,patch) _jmpi(_jit,i0,prev,patch) +static jit_word_t _jmpi(jit_state_t*,jit_word_t,jit_node_t*,jit_bool_t); +# define jmpi_p(i0,prev) _jmpi_p(_jit,i0,prev) +static jit_word_t _jmpi_p(jit_state_t*,jit_word_t,jit_node_t*); # define boaddr(i0,r0,r1) _boaddr(_jit,i0,r0,r1) static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); # define boaddi(i0,r0,i1) _boaddi(_jit,i0,r0,i1) @@ -733,10 +764,10 @@ static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); # define bmci(i0,r0,i1) _bmci(_jit,i0,r0,i1) static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); -# define callr(r0) _callr(_jit,r0) -static void _callr(jit_state_t*,jit_int32_t); -# define calli(i0) _calli(_jit,i0) -static void _calli(jit_state_t*,jit_word_t); +# define callr(r0,prev) _callr(_jit,r0,prev) +static void _callr(jit_state_t*,jit_int32_t,jit_node_t*); +# define calli(i0,prev,patch) _calli(_jit,i0,prev,patch) +static jit_word_t _calli(jit_state_t*,jit_word_t,jit_node_t*,jit_bool_t); # define calli_p(i0) _calli_p(_jit,i0) static jit_word_t _calli_p(jit_state_t*,jit_word_t); # define prolog(node) _prolog(_jit,node) @@ -751,9 +782,192 @@ static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t); static void _patch_abs(jit_state_t*,jit_word_t,jit_word_t); #define patch_at(jump,label) _patch_at(_jit,jump,label) static void _patch_at(jit_state_t*,jit_word_t,jit_word_t); +# define can_swap_ds(node,r0,r1) _can_swap_ds(_jit,node,r0,r1) +static jit_bool_t _can_swap_ds(jit_state_t *_jit, jit_node_t *prev, + jit_uint32_t r0, jit_uint32_t r1); +#define get_reg_can_swap(swap_ds) _get_reg_can_swap(_jit, swap_ds) +static jit_int32_t _get_reg_can_swap(jit_state_t *_jit, jit_bool_t swap_ds); #endif #if CODE +static jit_bool_t has_delay_slot(jit_instr_t instr) +{ + switch (instr.hc.b) { + case MIPS_SPECIAL: + switch (instr.tc.b) { + case MIPS_JR: + case MIPS_JALR: + return 1; + default: + return 0; + } + case MIPS_J: case MIPS_JAL: + case MIPS_BLEZ: case MIPS_BLEZL: + case MIPS_BEQ: case MIPS_BEQL: + case MIPS_BGTZ: case MIPS_BGTZL: + case MIPS_BNE: case MIPS_BNEL: + case MIPS_REGIMM: + return 1; + default: + return 0; + } +} + +static jit_bool_t op_reads_register(jit_instr_t instr, jit_int32_t r0) +{ + switch (instr.hc.b) { + case MIPS_SPECIAL: + switch (instr.tc.b) { + case MIPS_JR: + case MIPS_JALR: + case MIPS_MTHI: + case MIPS_MTLO: + return r0 == instr.rs.b; + case MIPS_SLL: + case MIPS_SRL: + case MIPS_SRA: + case MIPS_DSLL: + case MIPS_DSRL: + case MIPS_DSRA: + return r0 == instr.rt.b; + case MIPS_SYNC: + case MIPS_MFHI: + case MIPS_MFLO: + return 0; + case MIPS_MOVN: + case MIPS_MOVZ: + return r0 == instr.rt.b || r0 == instr.rs.b || r0 == instr.rd.b; + default: + return r0 == instr.rt.b || r0 == instr.rs.b; + } + case MIPS_SPECIAL3: + switch (instr.tc.b) { + case MIPS_INS: + case MIPS_DINS: + case MIPS_DINSU: + case MIPS_DINSM: + return r0 == instr.rt.b || r0 == instr.rs.b; + case MIPS_BSHFL: + return r0 == instr.rt.b; + default: + return r0 == instr.rs.b; + } + case MIPS_COP1: + if (instr.tc.b) + return 0; + + switch (instr.rs.b) { + case MIPS_MT: + case MIPS_DMT: + return r0 == instr.rt.b; + default: + return 0; + } + case MIPS_J: + case MIPS_JAL: + case MIPS_LUI: + return 0; + case MIPS_BEQ: + if (instr.rs.b == instr.rt.b) + return 0; + /* fallthrough */ + case MIPS_SPECIAL2: + case MIPS_BNE: + case MIPS_LWL: + case MIPS_LWR: + case MIPS_SB: + case MIPS_SH: + case MIPS_SWL: + case MIPS_SW: + case MIPS_SWR: + case MIPS_SC: + case MIPS_SCD: + case MIPS_SD: + return r0 == instr.rt.b || r0 == instr.rs.b; + default: + return r0 == instr.rs.b; + } +} + +static jit_bool_t op_writes_register(jit_instr_t instr, jit_int32_t r0) +{ + if (r0 == _ZERO_REGNO) + return 0; + + switch (instr.hc.b) { + case MIPS_SPECIAL: + switch (instr.tc.b) { + case MIPS_JR: + case MIPS_SYSCALL: + case MIPS_BREAK: + case MIPS_MULT: + case MIPS_MULTU: + case MIPS_DIV: + case MIPS_DIVU: + case MIPS_MTHI: + case MIPS_MTLO: + return 0; + default: + return r0 == instr.rd.b; + } + case MIPS_SPECIAL2: + return r0 == instr.rd.b; + case MIPS_SPECIAL3: + switch(instr.tc.b) { + case MIPS_BSHFL: + return r0 == instr.rd.b; + default: + return r0 == instr.rt.b; + } + case MIPS_COP1: + if (instr.tc.b) + return 0; + + switch (instr.rs.b) { + case MIPS_MF: + case MIPS_DMF: + return r0 == instr.rt.b; + default: + return 0; + } + case MIPS_ADDI: + case MIPS_ADDIU: + case MIPS_DADDI: + case MIPS_DADDIU: + case MIPS_SLTI: + case MIPS_SLTIU: + case MIPS_ANDI: + case MIPS_ORI: + case MIPS_XORI: + case MIPS_LUI: + case MIPS_LB: + case MIPS_LH: + case MIPS_LWL: + case MIPS_LW: + case MIPS_LWU: + case MIPS_LBU: + case MIPS_LHU: + case MIPS_LWR: + case MIPS_LL: + case MIPS_LLD: + case MIPS_LD: + case MIPS_LDC1: + return r0 == instr.rt.b; + case MIPS_JAL: + return r0 == 31; + case MIPS_REGIMM: + switch (instr.rt.b) { + case MIPS_BLTZAL: case MIPS_BLTZALL: + case MIPS_BGEZAL: case MIPS_BGEZALL: + return r0 == 31; + default: + return 0; + } + default: + return 0; + } +} + static void _hrrrit(jit_state_t *_jit,jit_int32_t hc, jit_int32_t rs, jit_int32_t rt, jit_int32_t rd, @@ -799,6 +1013,157 @@ _nop(jit_state_t *_jit, jit_int32_t i0) assert(i0 == 0); } +static void +_extr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t pos, jit_int32_t size) +{ + assert(size > 0); + + if (__WORDSIZE == 32) + EXT(r0, r1, pos, size); + else if (pos >= 32) + DEXTU(r0, r1, pos, size); + else if (size > 32) + DEXTM(r0, r1, pos, size); + else + DEXT(r0, r1, pos, size); +} + +static void +_insr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t pos, jit_int32_t size) +{ + assert(size > 0); + + if (__WORDSIZE == 32) + INS(r0, r1, pos, size); + else if (pos >= 32) + DINSU(r0, r1, pos, size); + else if (size > 32) + DINSM(r0, r1, pos, size); + else + DINS(r0, r1, pos, size); +} + +/* http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */ +/* +unsigned int s = sizeof(v) * CHAR_BIT; // bit size; must be power of 2 +unsigned int mask = ~0; +while ((s >>= 1) > 0) +{ + mask ^= (mask << s); + v = ((v >> s) & mask) | ((v << s) & ~mask); +} +*/ +static void +_bitswap(jit_state_t *_jit, jit_int32_t v, jit_int32_t r1) +{ + jit_int32_t s, mask; + jit_word_t loop, done, t0, t1; + movr(v, r1); + s = jit_get_reg(jit_class_gpr); + movi(rn(s), __WORDSIZE); /* s = sizeof(v) * CHAR_BIT; */ + mask = jit_get_reg(jit_class_gpr); + movi(rn(mask), ~0L); /* mask = ~0; */ + loop = _jit->pc.w; /* while ((s >>= 1) > 0) */ + rshi(rn(s), rn(s), 1); /* (s >>= 1) */ + done = blei(_jit->pc.w, rn(s), 0, NULL); /* no loop if s <= 0 */ + t0 = jit_get_reg(jit_class_gpr); + lshr(rn(t0), rn(mask), rn(s)); /* t0 = (mask << s) */ + xorr(rn(mask), rn(mask), rn(t0)); /* mask ^= t0 */ + rshr(rn(t0), v, rn(s)); /* t0 = v >> s */ + andr(rn(t0), rn(t0), rn(mask)); /* t0 = t0 & mask */ + t1 = jit_get_reg(jit_class_gpr); + lshr(rn(t1), v, rn(s)); /* t1 = v << s */ + comr(v, rn(mask)); /* v = ~mask */ + andr(rn(t1), v, rn(t1)); /* t1 = t1 & v */ + orr(v, rn(t0), rn(t1)); /* v = t0 | t1 */ + jmpi(loop, NULL, 0); + patch_at(done, _jit->pc.w); + jit_unget_reg(t1); + jit_unget_reg(t0); + jit_unget_reg(mask); + jit_unget_reg(s); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if __WORDSIZE == 32 + /* must sign extend top 32 bits */ + SLL(r0, r1, 0); + if (jit_mips6_p()) + CLO_R6(r0, r0); + else + CLO(r0, r0); +#else + if (jit_mips6_p()) + DCLO_R6(r0, r1); + else + DCLO(r0, r1); +#endif +} + +static void +_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if __WORDSIZE == 32 + /* must sign extend top 32 bits */ + SLL(r0, r1, 0); + if (jit_mips6_p()) + CLZ_R6(r0, r0); + else + CLZ(r0, r0); +#else + if (jit_mips6_p()) + DCLZ_R6(r0, r1); + else + DCLZ(r0, r1); +#endif +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_mips6_p()) { +#if __WORDSIZE == 32 + BITSWAP(r0, r1); + /* must sign extend top 32 bits */ + SLL(r0, r0, 0); + CLO_R6(r0, r0); +#else + DBITSWAP(r0, r1); + DCLO_R6(r0, r0); +#endif + } + else { + //fallback_cto(r0, r1); + bitswap(r0, r1); + clor(r0, r0); + } +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_mips6_p()) { +#if __WORDSIZE == 32 + BITSWAP(r0, r1); + /* must sign extend top 32 bits */ + SLL(r0, r0, 0); + CLZ_R6(r0, r0); +#else + DBITSWAP(r0, r1); + DCLZ_R6(r0, r0); +#endif + } + else { + //fallback_ctz(r0, r1); + bitswap(r0, r1); + clzr(r0, r0); + } +} + static void _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -993,8 +1358,12 @@ _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - multu(r1, r2); - MFLO(r0); + if (jit_mips2_p() && __WORDSIZE == 32) + MUL(r0, r1, r2); + else { + multu(r1, r2); + MFLO(r0); + } } static void @@ -1159,12 +1528,33 @@ _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_zero_extend_short_p(i0)) - ANDI(r0, r1, i0); - else { - reg = jit_get_reg(jit_class_gpr); - movi(rn(reg), i0); - AND(r0, r1, rn(reg)); - jit_unget_reg(reg); + ANDI(r0, r1, i0); + else if (is_low_mask(i0)) { + if (jit_mips2_p()) + extr(r0, r1, 0, masked_bits_count(i0)); + else { + lshi(r0, r1, unmasked_bits_count(i0)); + rshi_u(r0, r0, unmasked_bits_count(i0)); + } + } else if (is_high_mask(i0)) { + if (jit_mips2_p() && r0 == r1) + insr(r0, _ZERO_REGNO, 0, unmasked_bits_count(i0)); + else { + rshi(r0, r1, unmasked_bits_count(i0)); + lshi(r0, r0, unmasked_bits_count(i0)); + } + } else if (jit_mips2_p() && is_middle_mask(i0)) { + extr(r0, r1, __builtin_ctzl(i0), masked_bits_count(i0)); + lshi(r0, r0, __builtin_ctzl(i0)); + } else if (jit_mips2_p() && is_middle_mask(~i0)) { + if (r0 != r1) + movr(r0, r1); + insr(r0, _ZERO_REGNO, __builtin_ctzl(~i0), masked_bits_count(~i0)); + } else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + AND(r0, r1, rn(reg)); + jit_unget_reg(reg); } } @@ -1196,6 +1586,13 @@ _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) } } +static void +_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (r0 != r1) + orr(r0, r1, _ZERO_REGNO); +} + static void _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { @@ -1252,6 +1649,47 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t r1_reg, iscasi; + jit_word_t retry, done, jump0, jump1; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + SYNC(); + /* retry: */ + retry = _jit->pc.w; +# if __WORDSIZE == 32 + LL(r0, 0, r1); +# else + LLD(r0, 0, r1); +# endif + jump0 = _jit->pc.w; + BNE(r0, r2, 1); /* bne done r0 r2 */ + movi(r0, 0); /* set to 0 in delay slot */ + movr(r0, r3); /* after jump and delay slot */ + /* store new value */ +# if __WORDSIZE == 32 + SC(r0, 0, r1); +# else + SCD(r0, 0, r1); +# endif + jump1 = _jit->pc.w; + BEQ(r0, _ZERO_REGNO, 0); /* beqi retry r0 0 */ + movi(r0, 1); /* set to 1 in delay slot */ + SYNC(); + /* done: */ + done = _jit->pc.w; + patch_at(jump0, done); + patch_at(jump1, retry); + if (iscasi) + jit_unget_reg(r1_reg); +} + static void _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { @@ -1355,120 +1793,90 @@ _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) static void _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_c(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_c(r0, r0); } static void _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LB(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_c(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_c(r0, r0); } } static void _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_uc(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_uc(r0, r0); } static void _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LBU(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_uc(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_uc(r0, r0); } } static void _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_s(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_s(r0, r0); } static void _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LH(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_s(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_s(r0, r0); } } static void _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_us(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_us(r0, r0); } static void _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LHU(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_us(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_us(r0, r0); } } static void _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_i(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_i(r0, r0); } static void _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LW(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_i(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_i(r0, r0); } } @@ -1476,48 +1884,36 @@ _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_ui(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_ui(r0, r0); } static void _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LWU(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_ui(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_ui(r0, r0); } } static void _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1 ,jit_int32_t r2) { - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - addr(rn(reg), r1, r2); - ldr_l(r0, rn(reg)); - jit_unget_reg(reg); + addr(r0, r1, r2); + ldr_l(r0, r0); } static void _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; if (can_sign_extend_short_p(i0)) LD(r0, i0, r1); else { - reg = jit_get_reg(jit_class_gpr); - addi(rn(reg), r1, i0); - ldr_l(r0, rn(reg)); - jit_unget_reg(reg); + addi(r0, r1, i0); + ldr_l(r0, r0); } } #endif @@ -1678,61 +2074,35 @@ _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) } #endif -# if __BYTE_ORDER == __LITTLE_ENDIAN static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - jit_int32_t t0; - t0 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 8); - andi(r0, r1, 0xff); - andi(rn(t0), rn(t0), 0xff); - lshi(r0, r0, 8); - orr(r0, r0, rn(t0)); - jit_unget_reg(t0); + if (jit_mips2_p()) { + extr_us(r0, r1); + WSBH(r0, r0); + } else { + generic_bswapr_us(_jit, r0, r1); + } } static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - jit_int32_t t0; - jit_int32_t t1; - jit_int32_t t2; - t0 = jit_get_reg(jit_class_gpr); - t1 = jit_get_reg(jit_class_gpr); - t2 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 24); - rshi(rn(t1), r1, 16); - rshi(rn(t2), r1, 8); - andi(rn(t0), rn(t0), 0xff); - andi(rn(t1), rn(t1), 0xff); - andi(rn(t2), rn(t2), 0xff); - andi(r0, r1, 0xff); - lshi(r0, r0, 24); - lshi(rn(t1), rn(t1), 8); - orr(r0, r0, rn(t0)); - lshi(rn(t2), rn(t2), 16); - orr(r0, r0, rn(t1)); - orr(r0, r0, rn(t2)); - jit_unget_reg(t2); - jit_unget_reg(t1); - jit_unget_reg(t0); + if (jit_mips2_p()) { + if (__WORDSIZE == 64) { + SLL(r0, r1, 0); + WSBH(r0, r0); + ROTR(r0, r0, 16); + extr(r0, r0, 0, 32); + } else { + WSBH(r0, r1); + ROTR(r0, r0, 16); + } + } else { + generic_bswapr_ui(_jit, r0, r1); + } } -static void -_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - rshi_u(rn(reg), r1, 32); - htonr_ui(r0, r1); - htonr_ui(rn(reg), rn(reg)); - lshi(r0, r0, 32); - orr(r0, r0, rn(reg)); - jit_unget_reg(reg); -} -# endif - static void _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { @@ -1759,8 +2129,12 @@ _extr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _extr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - lshi(r0, r1, 32); - rshi_u(r0, r0, 32); + if (jit_mips2_p()) + DEXT(r0, r1, 0, 32); + else { + lshi(r0, r1, 32); + rshi_u(r0, r0, 32); + } } # endif @@ -1806,10 +2180,8 @@ _lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - if (i0 == 0) { - SLT(r0, _ZERO_REGNO, r1); - XORI(r0, r0, 1); - } + if (can_sign_extend_short_p(i0 + 1)) + SLTI(r0, r1, i0 + 1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); @@ -1830,10 +2202,8 @@ _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - if (i0 == 0) { - SLTU(r0, _ZERO_REGNO, r1); - XORI(r0, r0, 1); - } + if (can_sign_extend_short_p(i0 + 1)) + SLTIU(r0, r1, i0 + 1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); @@ -1846,8 +2216,7 @@ static void _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { subr(r0, r1, r2); - SLTU(r0, _ZERO_REGNO, r0); - XORI(r0, r0, 1); + SLTIU(r0, r0, 1); } static void @@ -1855,11 +2224,10 @@ _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { if (i0) { subi(r0, r1, i0); - SLTU(r0, _ZERO_REGNO, r0); + SLTIU(r0, r0, 1); + } else { + SLTIU(r0, r1, 1); } - else - SLTU(r0, _ZERO_REGNO, r1); - XORI(r0, r0, 1); } static void @@ -1874,10 +2242,15 @@ _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - movi(rn(reg), i0); - ger(r0, r1, rn(reg)); - jit_unget_reg(reg); + if (can_sign_extend_short_p(i0)) { + SLTI(r0, r1, i0); + XORI(r0, r0, 1); + } else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ger(r0, r1, rn(reg)); + jit_unget_reg(reg); + } } static void @@ -1892,10 +2265,15 @@ _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - movi(rn(reg), i0); - ger_u(r0, r1, rn(reg)); - jit_unget_reg(reg); + if (can_sign_extend_short_p(i0)) { + SLTIU(r0, r1, i0); + XORI(r0, r0, 1); + } else { + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), i0); + ger_u(r0, r1, rn(reg)); + jit_unget_reg(reg); + } } static void @@ -1946,413 +2324,516 @@ _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) SLTU(r0, _ZERO_REGNO, r1); } -static jit_word_t -_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +static jit_bool_t _can_swap_ds(jit_state_t *_jit, jit_node_t *prev, + jit_uint32_t r0, jit_uint32_t r1) { - jit_word_t w; - jit_int32_t reg; + jit_int32_t offset; - reg = jit_get_reg(jit_class_gpr); - SLT(rn(reg), r0, r1); - w = _jit->pc.w; - BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); + if (!prev) + return 0; - return (w); + switch (prev->code) { + case jit_code_prolog: + if (!_jitc->function->assume_frame) + break; + case jit_code_ltr_f: + case jit_code_ler_f: + case jit_code_eqr_f: + case jit_code_ger_f: + case jit_code_gtr_f: + case jit_code_ner_f: + case jit_code_unltr_f: + case jit_code_unler_f: + case jit_code_uneqr_f: + case jit_code_unger_f: + case jit_code_ungtr_f: + case jit_code_ltgtr_f: + case jit_code_ordr_f: + case jit_code_unordr_f: + case jit_code_lti_f: + case jit_code_lei_f: + case jit_code_eqi_f: + case jit_code_gei_f: + case jit_code_gti_f: + case jit_code_nei_f: + case jit_code_unlti_f: + case jit_code_unlei_f: + case jit_code_uneqi_f: + case jit_code_ungei_f: + case jit_code_ungti_f: + case jit_code_ltgti_f: + case jit_code_ordi_f: + case jit_code_unordi_f: + case jit_code_ltr_d: + case jit_code_ler_d: + case jit_code_eqr_d: + case jit_code_ger_d: + case jit_code_gtr_d: + case jit_code_ner_d: + case jit_code_unltr_d: + case jit_code_unler_d: + case jit_code_uneqr_d: + case jit_code_unger_d: + case jit_code_ungtr_d: + case jit_code_ltgtr_d: + case jit_code_ordr_d: + case jit_code_unordr_d: + case jit_code_lti_d: + case jit_code_lei_d: + case jit_code_eqi_d: + case jit_code_gei_d: + case jit_code_gti_d: + case jit_code_nei_d: + case jit_code_unlti_d: + case jit_code_unlei_d: + case jit_code_uneqi_d: + case jit_code_ungei_d: + case jit_code_ungti_d: + case jit_code_ltgti_d: + case jit_code_ordi_d: + case jit_code_unordi_d: + case jit_code_casr: + case jit_code_casi: + /* These instructions will generate a branch to the next opcode. + * Therefore, we cannot swap the last opcode to the delay slot. */ + return 0; + default: + break; + } + + offset = ((jit_word_t)_jit->pc.ui - (jit_word_t)_jit->code.ptr) / sizeof(jit_instr_t); + + return prev && !(prev->flag & jit_flag_patch) + && (offset < 2 || !has_delay_slot((jit_instr_t)*(_jit->pc.ui - 2))) + && (!r0 || !op_writes_register((jit_instr_t)*(_jit->pc.ui - 1), r0)) + && (!r1 || !op_writes_register((jit_instr_t)*(_jit->pc.ui - 1), r1)); } -static jit_word_t -_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +static jit_int32_t _get_reg_can_swap(jit_state_t *_jit, jit_bool_t swap_ds) { - jit_word_t w; - jit_int32_t reg; + jit_int32_t reg, reg2; reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLTU(rn(reg), r0, r1); - w = _jit->pc.w; - BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); - return (w); + if (swap_ds && reg == _AT + && op_reads_register((jit_instr_t)*(_jit->pc.ui - 1), rn(_AT))) { + reg2 = jit_get_reg(jit_class_gpr); + jit_unget_reg(reg); + + if (reg2 == JIT_NOREG) + return JIT_NOREG; + + reg = reg2; + } + + return reg; } static jit_word_t -_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1, jit_node_t *prev) { jit_word_t w; - jit_word_t d; - jit_int32_t reg; - jit_bool_t zero_p; + jit_int32_t op; + jit_bool_t swap_ds; - if (!(zero_p = i1 == 0)) - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - if (can_sign_extend_short_p(i1)) { - if (!zero_p) - SLTI(rn(reg), r0, i1); - w = _jit->pc.w; - d = ((i0 - w) >> 2) - 1; - if (!zero_p) - BNE(rn(reg), _ZERO_REGNO, d); - else - BLTZ(r0, d); - NOP(1); - } - else { - movi(rn(reg), i1); - w = bltr(i0, r0, rn(reg)); - } - if (!zero_p) - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) -{ - jit_word_t w; - jit_int32_t reg; - - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - if (can_sign_extend_short_p(i1)) { - SLTIU(rn(reg), r0, i1); - w = _jit->pc.w; - BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - } - else { - movi(rn(reg), i1); - w = bltr_u(i0, r0, rn(reg)); - } - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; - jit_int32_t reg; - - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLT(rn(reg), r1, r0); - w = _jit->pc.w; - BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; - jit_int32_t reg; - - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLTU(rn(reg), r1, r0); - w = _jit->pc.w; - BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) -{ - jit_word_t w; - jit_int32_t reg; - - if (i1 == 0) { - w = _jit->pc.w; - BLEZ(r0, ((i0 - w) >> 2) - 1); - NOP(1); - } - else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi(rn(reg), i1); - w = bler(i0, r0, rn(reg)); - jit_unget_reg(reg); - } - - return (w); -} - -static jit_word_t -_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) -{ - jit_word_t w; - jit_int32_t reg; - - if (i1 == 0) { - w = _jit->pc.w; - BEQ(r0, _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - } - else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi(rn(reg), i1); - w = bler_u(i0, r0, rn(reg)); - jit_unget_reg(reg); - } - - return (w); -} - -static jit_word_t -_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; + swap_ds = can_swap_ds(prev, r0, r1); + if (swap_ds) + op = *--_jit->pc.ui; w = _jit->pc.w; BEQ(r0, r1, ((i0 - w) >> 2) - 1); - NOP(1); + if (swap_ds) + ii(op); + else + NOP(1); return (w); } static jit_word_t -_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1, jit_node_t *prev) { jit_word_t w; - jit_int32_t reg; + jit_int32_t reg, op; + jit_bool_t swap_ds; if (i1 == 0) { - w = _jit->pc.w; - BEQ(r0, _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); + w = beqr(i0, r0, _ZERO_REGNO, prev); } else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi(rn(reg), i1); - w = beqr(i0, r0, rn(reg)); - jit_unget_reg(reg); + swap_ds = can_swap_ds(prev, r0, 0); + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + } + + if (swap_ds) + op = *--_jit->pc.ui; + + movi(rn(reg), i1); + w = _jit->pc.w; + BEQ(r0, rn(reg), ((i0 - w) >> 2) - 1); + + if (swap_ds) + ii(op); + else + NOP(1); + + jit_unget_reg(reg); } return (w); } static jit_word_t -_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) +_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1, + jit_bool_t sltu, jit_node_t *prev) { jit_word_t w; - jit_int32_t reg; + jit_int32_t reg, op; + jit_bool_t swap_ds; + + swap_ds = can_swap_ds(prev, r0, r1); + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + } + + if (swap_ds) + op = *--_jit->pc.ui; + + if (sltu) + SLTU(rn(reg), r0, r1); + else + SLT(rn(reg), r0, r1); - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLT(rn(reg), r0, r1); w = _jit->pc.w; BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); + + if (swap_ds) + ii(op); + else + NOP(1); + jit_unget_reg(reg); return (w); } static jit_word_t -_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; - jit_int32_t reg; - - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLTU(rn(reg), r0, r1); - w = _jit->pc.w; - BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1, + jit_bool_t sltiu, jit_bool_t bne, jit_node_t *prev) { jit_word_t w; jit_word_t d; - jit_int32_t reg; - jit_bool_t zero_p; + jit_int32_t reg, op; + jit_bool_t zero_p, swap_ds; + + zero_p = !sltiu && i1 == 0; + + swap_ds = can_swap_ds(prev, r0, 0); + + if (!zero_p) { + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + } + } + if (swap_ds) + op = *--_jit->pc.ui; - if (!(zero_p = i1 == 0)) - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); if (can_sign_extend_short_p(i1)) { - if (!zero_p) - SLTI(rn(reg), r0, i1); - w = _jit->pc.w; - d = ((i0 - w) >> 2) - 1; - if (!zero_p) - BEQ(rn(reg), _ZERO_REGNO, d); - else - BGEZ(r0, d); - NOP(1); + if (!zero_p) { + if (sltiu) + SLTIU(rn(reg), r0, i1); + else + SLTI(rn(reg), r0, i1); + } + + w = _jit->pc.w; + d = ((i0 - w) >> 2) - 1; + if (bne) { + if (!zero_p) + BNE(rn(reg), _ZERO_REGNO, d); + else + BLTZ(r0, d); + } else { + if (!zero_p) + BEQ(rn(reg), _ZERO_REGNO, d); + else + BGEZ(r0, d); + } } else { - movi(rn(reg), i1); - w = bger(i0, r0, rn(reg)); + movi(rn(reg), i1); + if (sltiu) + SLTU(rn(reg), r0, rn(reg)); + else + SLT(rn(reg), r0, rn(reg)); + + w = _jit->pc.w; + if (bne) + BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); + else + BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); } + + if (swap_ds) + ii(op); + else + NOP(1); + if (!zero_p) - jit_unget_reg(reg); + jit_unget_reg(reg); return (w); } static jit_word_t -_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1, + jit_bool_t sltu, jit_bool_t inv, jit_node_t *prev) { jit_word_t w; - jit_int32_t reg; + jit_int32_t reg, op; + jit_bool_t swap_ds; - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - if (can_sign_extend_short_p(i1)) { - SLTIU(rn(reg), r0, i1); - w = _jit->pc.w; - BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); + swap_ds = can_swap_ds(prev, r0, r1); + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); } - else { - movi(rn(reg), i1); - w = bger_u(i0, r0, rn(reg)); + if (swap_ds) + op = *--_jit->pc.ui; + + if (sltu) + SLTU(rn(reg), r1, r0); + else + SLT(rn(reg), r1, r0); + + w = _jit->pc.w; + if (inv) + BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); + else + BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); + + if (swap_ds) + ii(op); + else + NOP(1); + + jit_unget_reg(reg); + + return (w); +} + +static jit_word_t +_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1, + jit_bool_t sltiu, jit_bool_t inv, jit_node_t *prev) +{ + jit_word_t w; + jit_int32_t reg, op; + jit_bool_t swap_ds; + + swap_ds = can_swap_ds(prev, r0, 0); + + if (i1 != 0) { + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + } } - jit_unget_reg(reg); - return (w); -} - -static jit_word_t -_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; - jit_int32_t reg; - - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLT(rn(reg), r1, r0); - w = _jit->pc.w; - BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; - jit_int32_t reg; - - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - SLTU(rn(reg), r1, r0); - w = _jit->pc.w; - BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - jit_unget_reg(reg); - - return (w); -} - -static jit_word_t -_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) -{ - jit_word_t w; - jit_int32_t reg; + if (swap_ds) + op = *--_jit->pc.ui; if (i1 == 0) { - w = _jit->pc.w; - BGTZ(r0, ((i0 - w) >> 2) - 1); - NOP(1); + w = _jit->pc.w; + if (inv) { + if (sltiu) + BEQ(r0, _ZERO_REGNO, ((i0 - w) >> 2) - 1); + else + BLEZ(r0, ((i0 - w) >> 2) - 1); + } else { + if (sltiu) + BNE(r0, _ZERO_REGNO, ((i0 - w) >> 2) - 1); + else + BGTZ(r0, ((i0 - w) >> 2) - 1); + } } else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi(rn(reg), i1); - w = bgtr(i0, r0, rn(reg)); - jit_unget_reg(reg); + movi(rn(reg), i1); + if (sltiu) + SLTU(rn(reg), rn(reg), r0); + else + SLT(rn(reg), rn(reg), r0); + + w = _jit->pc.w; + if (inv) + BEQ(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); + else + BNE(rn(reg), _ZERO_REGNO, ((i0 - w) >> 2) - 1); + + jit_unget_reg(reg); } + if (swap_ds) + ii(op); + else + NOP(1); + return (w); } static jit_word_t -_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1, jit_node_t *prev) { jit_word_t w; - jit_int32_t reg; + jit_int32_t op; + jit_bool_t swap_ds; - if (i1 == 0) { - w = _jit->pc.w; - BNE(r0, _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); - } - else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi(rn(reg), i1); - w = bgtr_u(i0, r0, rn(reg)); - jit_unget_reg(reg); - } - - return (w); -} - -static jit_word_t -_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) -{ - jit_word_t w; + swap_ds = can_swap_ds(prev, r0, r1); + if (swap_ds) + op = *--_jit->pc.ui; w = _jit->pc.w; BNE(r0, r1, ((i0 - w) >> 2) - 1); - NOP(1); + if (swap_ds) + ii(op); + else + NOP(1); return (w); } static jit_word_t -_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) +_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1, jit_node_t *prev) { jit_word_t w; - jit_int32_t reg; + jit_int32_t reg, op; + jit_bool_t swap_ds; if (i1 == 0) { - w = _jit->pc.w; - BNE(r0, _ZERO_REGNO, ((i0 - w) >> 2) - 1); - NOP(1); + w = bner(i0, r0, _ZERO_REGNO, prev); } else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi(rn(reg), i1); - w = bner(i0, r0, rn(reg)); - jit_unget_reg(reg); + swap_ds = can_swap_ds(prev, r0, 0); + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + } + + if (swap_ds) + op = *--_jit->pc.ui; + + movi(rn(reg), i1); + w = _jit->pc.w; + BNE(r0, rn(reg), ((i0 - w) >> 2) - 1); + if (swap_ds) + ii(op); + else + NOP(1); + + jit_unget_reg(reg); } return (w); } static void -_jmpr(jit_state_t *_jit, jit_int32_t r0) +_jmpr(jit_state_t *_jit, jit_int32_t r0, jit_node_t *prev) { + jit_int32_t op, offset; + jit_bool_t swap_ds; + + swap_ds = can_swap_ds(prev, r0, 0); + if (swap_ds) + op = *--_jit->pc.ui; + JR(r0); - NOP(1); + if (swap_ds) + ii(op); + else + NOP(1); } static jit_word_t -_jmpi(jit_state_t *_jit, jit_word_t i0) +_jmpi(jit_state_t *_jit, jit_word_t i0, jit_node_t *prev, jit_bool_t patch) +{ + jit_int32_t op; + jit_bool_t swap_ds; + jit_word_t w, disp; + + if (jit_mips2_p()) { + swap_ds = can_swap_ds(prev, 0, 0); + w = _jit->pc.w; + disp = ((i0 - w) >> 2) - 1; + if (patch && swap_ds) { + op = *--_jit->pc.ui; + w -= sizeof(jit_int32_t); + BEQ(_ZERO_REGNO, _ZERO_REGNO, disp); + ii(op); + goto done; + } + if (patch || can_sign_extend_short_p(disp)) { + BEQ(_ZERO_REGNO, _ZERO_REGNO, disp); + NOP(1); + goto done; + } + } + w = jmpi_p(i0, prev); + +done: + return (w); +} + +static jit_word_t +_jmpi_p(jit_state_t *_jit, jit_word_t i0, jit_node_t *prev) { jit_word_t w; - jit_int32_t reg; + jit_int32_t reg, op; + jit_bool_t swap_ds; + + swap_ds = can_swap_ds(prev, 0, 0); w = _jit->pc.w; if (((w + sizeof(jit_int32_t)) & 0xf0000000) == (i0 & 0xf0000000)) { - J((i0 & ~0xf0000000) >> 2); - NOP(1); + if (swap_ds) { + op = *--_jit->pc.ui; + w -= sizeof(jit_int32_t); + } + + J((i0 & ~0xf0000000) >> 2); + if (swap_ds) + ii(op); + else + NOP(1); } else { - reg = jit_get_reg(jit_class_gpr|jit_class_nospill); - movi_p(rn(reg), i0); - jmpr(rn(reg)); - jit_unget_reg(reg); + reg = get_reg_can_swap(swap_ds); + if (reg == JIT_NOREG) { + swap_ds = 0; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + } + + if (swap_ds) + op = *--_jit->pc.ui; + + movi_p(rn(reg), i0); + + w = _jit->pc.w; + JR(rn(reg)); + if (swap_ds) + ii(op); + else + NOP(1); + + jit_unget_reg(reg); } return (w); @@ -2818,16 +3299,12 @@ _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_word_t w; jit_int32_t t0; t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); - if (can_zero_extend_short_p(i1)) { - ANDI(rn(t0), r0, i1); - w = _jit->pc.w; - BNE(_ZERO_REGNO, rn(t0), ((i0 - w) >> 2) - 1); - NOP(1); - } - else { - movi(rn(t0), i1); - w = bmsr(i0, r0, rn(t0)); - } + + andi(rn(t0), r0, i1); + w = _jit->pc.w; + BNE(_ZERO_REGNO, rn(t0), ((i0 - w) >> 2) - 1); + NOP(1); + jit_unget_reg(t0); return (w); } @@ -2852,35 +3329,93 @@ _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_word_t w; jit_int32_t t0; t0 = jit_get_reg(jit_class_gpr|jit_class_nospill); - if (can_zero_extend_short_p(i1)) { - ANDI(rn(t0), r0, i1); - w = _jit->pc.w; - BEQ(_ZERO_REGNO, rn(t0), ((i0 - w) >> 2) - 1); - NOP(1); - } - else { - movi(rn(t0), i1); - w = bmcr(i0, r0, rn(t0)); - } + + andi(rn(t0), r0, i1); + w = _jit->pc.w; + BEQ(_ZERO_REGNO, rn(t0), ((i0 - w) >> 2) - 1); + NOP(1); + jit_unget_reg(t0); return (w); } static void -_callr(jit_state_t *_jit, jit_int32_t r0) +_callr(jit_state_t *_jit, jit_int32_t r0, jit_node_t *prev) { - if (r0 != _T9_REGNO) - movr(_T9_REGNO, r0); - JALR(r0); - NOP(1); + jit_bool_t swap_ds; + jit_int32_t op; + + if (r0 != _T9_REGNO) { + JALR(r0); + movr(_T9_REGNO, r0); + } else { + swap_ds = can_swap_ds(prev, r0, 0); + if (swap_ds) + op = *--_jit->pc.ui; + + JALR(r0); + if (swap_ds) + ii(op); + else + NOP(1); + } } -static void -_calli(jit_state_t *_jit, jit_word_t i0) +static jit_word_t +_calli(jit_state_t *_jit, jit_word_t i0, jit_node_t *prev, jit_bool_t patch) { - movi(_T9_REGNO, i0); - JALR(_T9_REGNO); - NOP(1); + jit_int32_t op; + jit_bool_t swap_ds; + jit_word_t w, disp; + + w = _jit->pc.w; + if (jit_mips2_p()) { + swap_ds = can_swap_ds(prev, 0, 0); + disp = ((i0 - w) >> 2) - 1; + if (patch && swap_ds) { + op = *--_jit->pc.ui; + w -= sizeof(jit_int32_t); + BGEZAL(_ZERO_REGNO, disp); + ii(op); + goto done; + } + if (patch || can_sign_extend_short_p(disp)) { + BGEZAL(_ZERO_REGNO, disp); + NOP(1); + goto done; + } + } + if (((w + sizeof(jit_int32_t)) & 0xf0000000) == (i0 & 0xf0000000)) { + if (can_sign_extend_short_p(i0)) { + JAL((i0 & ~0xf0000000) >> 2); + addiu(_T9_REGNO, _ZERO_REGNO, i0); + } + else if (can_zero_extend_short_p(i0)) { + JAL((i0 & ~0xf0000000) >> 2); + ORI(_T9_REGNO, _ZERO_REGNO, i0); + } + else if (can_sign_extend_int_p(i0)) { + if (i0 & 0xffff) { + LUI(_T9_REGNO, i0 >> 16); + JAL((i0 & ~0xf0000000) >> 2); + ORI(_T9_REGNO, _T9_REGNO, i0); + } + else { + JAL((i0 & ~0xf0000000) >> 2); + LUI(_T9_REGNO, i0 >> 16); + } + } + else + goto fallback; + } + else { + fallback: + movi(_T9_REGNO, i0); + JALR(_T9_REGNO); + NOP(1); + } + done: + return (w); } static jit_word_t @@ -2896,24 +3431,13 @@ _calli_p(jit_state_t *_jit, jit_word_t i0) return (word); } -static jit_int32_t fregs[] = { - _F30, _F28, _F26, _F24, _F22, _F20, -#if !NEW_ABI - _F18, _F16, -#endif -}; - -static jit_int32_t iregs[] = { - _S7, _S6, _S5, _S4, _S3, _S2, _S1, _S0, -}; - static void _prolog(jit_state_t *_jit, jit_node_t *node) { - jit_int32_t index; - jit_int32_t offset; + jit_int32_t reg, offs; if (_jitc->function->define_frame || _jitc->function->assume_frame) { jit_int32_t frame = -_jitc->function->frame; + jit_check_frame(); assert(_jitc->function->self.aoff >= frame); if (_jitc->function->assume_frame) return; @@ -2932,51 +3456,65 @@ _prolog(jit_state_t *_jit, jit_node_t *node) /* align stack at 8 bytes */ _jitc->function->self.aoff) + 7) & -8; #endif - /* callee save registers */ + #if NEW_ABI - if ((_jitc->function->self.call & jit_call_varargs) && - jit_arg_reg_p(_jitc->function->vagp)) - subi(_SP_REGNO, _SP_REGNO, stack_framesize + 64); - else + if (_jitc->function->stack) + _jitc->function->need_stack = 1; + if (!_jitc->function->need_frame && !_jitc->function->need_stack) { + /* check if any callee save register needs to be saved */ + for (reg = 0; reg < _jitc->reglen; ++reg) + if (jit_regset_tstbit(&_jitc->function->regset, reg) && + (_rvs[reg].spec & jit_class_sav)) { + _jitc->function->need_stack = 1; + break; + } + } +#else + /* Need always a frame due to the need to always allocate 16 bytes */ + jit_check_frame(); #endif - subi(_SP_REGNO, _SP_REGNO, stack_framesize); - offset = stack_framesize - (sizeof(jit_word_t) << 1); - for (index = 0; index < jit_size(fregs); index++, offset -= 8) { - if (jit_regset_tstbit(&_jitc->function->regset, fregs[index])) - stxi_d(offset, _SP_REGNO, rn(fregs[index])); + + if (_jitc->function->need_frame || _jitc->function->need_stack) + subi(_SP_REGNO, _SP_REGNO, jit_framesize()); + if (_jitc->function->need_frame) { + stxi(0, _SP_REGNO, _RA_REGNO); + stxi(STACK_SLOT, _SP_REGNO, _BP_REGNO); } - for (index = 0; index < jit_size(iregs); - index++, offset -= sizeof(jit_word_t)) { - if (jit_regset_tstbit(&_jitc->function->regset, iregs[index])) - stxi(offset, _SP_REGNO, rn(iregs[index])); + /* callee save registers */ + for (reg = 0, offs = STACK_SLOT << 1; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + stxi(offs, _SP_REGNO, rn(iregs[reg])); + offs += STACK_SLOT; + } } - assert(offset >= sizeof(jit_word_t)); - stxi(offset, _SP_REGNO, _RA_REGNO); - stxi(0, _SP_REGNO, _BP_REGNO); - movr(_BP_REGNO, _SP_REGNO); + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + stxi_d(offs, _SP_REGNO, rn(fregs[reg])); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->need_frame) + movr(_BP_REGNO, _SP_REGNO); /* alloca */ if (_jitc->function->stack) subi(_SP_REGNO, _SP_REGNO, _jitc->function->stack); if (_jitc->function->allocar) { - index = jit_get_reg(jit_class_gpr); - movi(rn(index), _jitc->function->self.aoff); - stxi_i(_jitc->function->aoffoff, _BP_REGNO, rn(index)); - jit_unget_reg(index); + reg = jit_get_reg(jit_class_gpr); + movi(rn(reg), _jitc->function->self.aoff); + stxi_i(_jitc->function->aoffoff, _BP_REGNO, rn(reg)); + jit_unget_reg(reg); } if (_jitc->function->self.call & jit_call_varargs) { + for (reg = _jitc->function->vagp; jit_arg_reg_p(reg); ++reg) { + offs = jit_framesize() - ((NUM_WORD_ARGS - reg) * STACK_SLOT); #if NEW_ABI - index = _jitc->function->vagp; + SD(rn(_A0 - reg), offs, _BP_REGNO); #else - index = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT; -#endif - offset = stack_framesize + index * STACK_SLOT; - for (; jit_arg_reg_p(index); ++index, offset += STACK_SLOT) { -#if NEW_ABI - SD(rn(_A0 - index), offset, _BP_REGNO); -#else - stxi(offset + WORD_ADJUST, _BP_REGNO, rn(_A0 - index)); + offs += 16 + WORD_ADJUST; + stxi(offs, _BP_REGNO, rn(_A0 - reg)); #endif } } @@ -2985,48 +3523,50 @@ _prolog(jit_state_t *_jit, jit_node_t *node) static void _epilog(jit_state_t *_jit, jit_node_t *node) { - jit_int32_t index; - jit_int32_t offset; + jit_int32_t reg, offs; if (_jitc->function->assume_frame) return; + + if (_jitc->function->need_frame) { + movr(_SP_REGNO, _BP_REGNO); + ldxi(_RA_REGNO, _SP_REGNO, 0); + ldxi(_BP_REGNO, _SP_REGNO, STACK_SLOT); + } + /* callee save registers */ - movr(_SP_REGNO, _BP_REGNO); - offset = stack_framesize - (sizeof(jit_word_t) << 1); - for (index = 0; index < jit_size(fregs); index++, offset -= 8) { - if (jit_regset_tstbit(&_jitc->function->regset, fregs[index])) - ldxi_d(rn(fregs[index]), _SP_REGNO, offset); + for (reg = 0, offs = STACK_SLOT << 1; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + ldxi(rn(iregs[reg]), _SP_REGNO, offs); + offs += sizeof(jit_word_t); + } } - for (index = 0; index < jit_size(iregs); - index++, offset -= sizeof(jit_word_t)) { - if (jit_regset_tstbit(&_jitc->function->regset, iregs[index])) - ldxi(rn(iregs[index]), _SP_REGNO, offset); + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + ldxi_d(rn(fregs[reg]), _SP_REGNO, offs); + offs += sizeof(jit_float64_t); + } } - assert(offset >= sizeof(jit_word_t)); - ldxi(_RA_REGNO, _SP_REGNO, offset); - ldxi(_BP_REGNO, _SP_REGNO, 0); JR(_RA_REGNO); /* delay slot */ -#if NEW_ABI - if ((_jitc->function->self.call & jit_call_varargs) && - jit_arg_reg_p(_jitc->function->vagp)) - addi(_SP_REGNO, _SP_REGNO, stack_framesize + 64); + if (_jitc->function->need_frame || _jitc->function->need_stack) + addi(_SP_REGNO, _SP_REGNO, jit_framesize()); else -#endif - addi(_SP_REGNO, _SP_REGNO, stack_framesize); + NOP(1); } static void _vastart(jit_state_t *_jit, jit_int32_t r0) { assert(_jitc->function->self.call & jit_call_varargs); - /* Initialize va_list to the first stack argument. */ #if NEW_ABI + /* Initialize va_list to the first stack argument. */ if (jit_arg_reg_p(_jitc->function->vagp)) - addi(r0, _BP_REGNO, stack_framesize + _jitc->function->vagp * - sizeof(jit_int64_t)); + addi(r0, _BP_REGNO, + jit_framesize() - + ((NUM_WORD_ARGS - _jitc->function->vagp) * STACK_SLOT)); else #endif - addi(r0, _BP_REGNO, _jitc->function->self.size); + addi(r0, _BP_REGNO, jit_selfsize()); } static void diff --git a/parallel-rsp/lightning/lib/jit_mips-fpu.c b/parallel-rsp/lightning/lib/jit_mips-fpu.c index 7513219..8bff202 100644 --- a/parallel-rsp/lightning/lib/jit_mips-fpu.c +++ b/parallel-rsp/lightning/lib/jit_mips-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -220,7 +220,7 @@ static void _divi_d(jit_state_t*,jit_int32_t,jit_int32_t,jit_float64_t*); # define sqrtr_f(r0,r1) SQRT_S(r0,r1) # define sqrtr_d(r0,r1) SQRT_D(r0,r1) # define movr_w_f(r0, r1) MTC1(r1, r0) -# define movr_f_w(r0, r1) MFC1(r1, r0) +# define movr_f_w(r0, r1) MFC1(r0, r1) # define movi_f_w(r0, i0) _movi_f_w(_jit, r0, i0) static void _movi_f_w(jit_state_t*,jit_int32_t,jit_float32_t*); # define extr_f(r0, r1) _extr_f(_jit, r0, r1) diff --git a/parallel-rsp/lightning/lib/jit_mips-sz.c b/parallel-rsp/lightning/lib/jit_mips-sz.c index 613aa00..0a7436d 100644 --- a/parallel-rsp/lightning/lib/jit_mips-sz.c +++ b/parallel-rsp/lightning/lib/jit_mips-sz.c @@ -1,416 +1,12 @@ #if __WORDSIZE == 32 -#if NEW_ABI -#define JIT_INSTR_MAX 44 - 0, /* data */ - 0, /* live */ - 0, /* align */ - 0, /* save */ - 0, /* load */ - 0, /* #name */ - 0, /* #note */ - 0, /* label */ - 44, /* prolog */ - 0, /* ellipsis */ - 0, /* va_push */ - 0, /* allocai */ - 0, /* allocar */ - 0, /* arg */ - 0, /* getarg_c */ - 0, /* getarg_uc */ - 0, /* getarg_s */ - 0, /* getarg_us */ - 0, /* getarg_i */ - 0, /* getarg_ui */ - 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 0, /* va_start */ - 0, /* va_arg */ - 0, /* va_arg_d */ - 0, /* va_end */ - 4, /* addr */ - 12, /* addi */ - 12, /* addcr */ - 20, /* addci */ - 28, /* addxr */ - 28, /* addxi */ - 4, /* subr */ - 12, /* subi */ - 12, /* subcr */ - 20, /* subci */ - 28, /* subxr */ - 28, /* subxi */ - 16, /* rsbi */ - 8, /* mulr */ - 16, /* muli */ - 12, /* qmulr */ - 20, /* qmuli */ - 12, /* qmulr_u */ - 20, /* qmuli_u */ - 8, /* divr */ - 16, /* divi */ - 8, /* divr_u */ - 16, /* divi_u */ - 12, /* qdivr */ - 16, /* qdivi */ - 12, /* qdivr_u */ - 16, /* qdivi_u */ - 8, /* remr */ - 16, /* remi */ - 8, /* remr_u */ - 16, /* remi_u */ - 4, /* andr */ - 12, /* andi */ - 4, /* orr */ - 12, /* ori */ - 4, /* xorr */ - 12, /* xori */ - 4, /* lshr */ - 4, /* lshi */ - 4, /* rshr */ - 4, /* rshi */ - 4, /* rshr_u */ - 4, /* rshi_u */ - 4, /* negr */ - 8, /* comr */ - 4, /* ltr */ - 4, /* lti */ - 4, /* ltr_u */ - 4, /* lti_u */ - 8, /* ler */ - 12, /* lei */ - 8, /* ler_u */ - 12, /* lei_u */ - 12, /* eqr */ - 12, /* eqi */ - 8, /* ger */ - 12, /* gei */ - 8, /* ger_u */ - 12, /* gei_u */ - 4, /* gtr */ - 8, /* gti */ - 4, /* gtr_u */ - 8, /* gti_u */ - 8, /* ner */ - 8, /* nei */ - 4, /* movr */ - 8, /* movi */ - 8, /* extr_c */ - 4, /* extr_uc */ - 8, /* extr_s */ - 4, /* extr_us */ - 0, /* extr_i */ - 0, /* extr_ui */ - 4, /* htonr_us */ - 4, /* htonr_ui */ - 0, /* htonr_ul */ - 4, /* ldr_c */ - 12, /* ldi_c */ - 4, /* ldr_uc */ - 12, /* ldi_uc */ - 4, /* ldr_s */ - 12, /* ldi_s */ - 4, /* ldr_us */ - 12, /* ldi_us */ - 4, /* ldr_i */ - 12, /* ldi_i */ - 0, /* ldr_ui */ - 0, /* ldi_ui */ - 0, /* ldr_l */ - 0, /* ldi_l */ - 8, /* ldxr_c */ - 4, /* ldxi_c */ - 8, /* ldxr_uc */ - 4, /* ldxi_uc */ - 8, /* ldxr_s */ - 4, /* ldxi_s */ - 8, /* ldxr_us */ - 4, /* ldxi_us */ - 8, /* ldxr_i */ - 4, /* ldxi_i */ - 0, /* ldxr_ui */ - 0, /* ldxi_ui */ - 0, /* ldxr_l */ - 0, /* ldxi_l */ - 4, /* str_c */ - 12, /* sti_c */ - 4, /* str_s */ - 12, /* sti_s */ - 4, /* str_i */ - 12, /* sti_i */ - 0, /* str_l */ - 0, /* sti_l */ - 8, /* stxr_c */ - 4, /* stxi_c */ - 8, /* stxr_s */ - 4, /* stxi_s */ - 8, /* stxr_i */ - 4, /* stxi_i */ - 0, /* stxr_l */ - 0, /* stxi_l */ - 12, /* bltr */ - 12, /* blti */ - 12, /* bltr_u */ - 12, /* blti_u */ - 12, /* bler */ - 16, /* blei */ - 12, /* bler_u */ - 16, /* blei_u */ - 8, /* beqr */ - 16, /* beqi */ - 12, /* bger */ - 12, /* bgei */ - 12, /* bger_u */ - 12, /* bgei_u */ - 12, /* bgtr */ - 16, /* bgti */ - 12, /* bgtr_u */ - 16, /* bgti_u */ - 8, /* bner */ - 16, /* bnei */ - 12, /* bmsr */ - 12, /* bmsi */ - 12, /* bmcr */ - 12, /* bmci */ - 28, /* boaddr */ - 28, /* boaddi */ - 16, /* boaddr_u */ - 20, /* boaddi_u */ - 28, /* bxaddr */ - 28, /* bxaddi */ - 16, /* bxaddr_u */ - 20, /* bxaddi_u */ - 28, /* bosubr */ - 28, /* bosubi */ - 16, /* bosubr_u */ - 20, /* bosubi_u */ - 28, /* bxsubr */ - 28, /* bxsubi */ - 16, /* bxsubr_u */ - 20, /* bxsubi_u */ - 0, /* jmpr */ - 8, /* jmpi */ - 12, /* callr */ - 16, /* calli */ - 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ - 0, /* finishr */ - 0, /* finishi */ - 0, /* ret */ - 0, /* retr */ - 0, /* reti */ - 0, /* retval_c */ - 0, /* retval_uc */ - 0, /* retval_s */ - 0, /* retval_us */ - 0, /* retval_i */ - 0, /* retval_ui */ - 0, /* retval_l */ - 44, /* epilog */ - 0, /* arg_f */ - 0, /* getarg_f */ - 0, /* putargr_f */ - 0, /* putargi_f */ - 4, /* addr_f */ - 16, /* addi_f */ - 4, /* subr_f */ - 16, /* subi_f */ - 16, /* rsbi_f */ - 4, /* mulr_f */ - 16, /* muli_f */ - 4, /* divr_f */ - 16, /* divi_f */ - 4, /* negr_f */ - 4, /* absr_f */ - 4, /* sqrtr_f */ - 16, /* ltr_f */ - 28, /* lti_f */ - 16, /* ler_f */ - 28, /* lei_f */ - 16, /* eqr_f */ - 28, /* eqi_f */ - 16, /* ger_f */ - 28, /* gei_f */ - 16, /* gtr_f */ - 28, /* gti_f */ - 16, /* ner_f */ - 28, /* nei_f */ - 16, /* unltr_f */ - 28, /* unlti_f */ - 16, /* unler_f */ - 28, /* unlei_f */ - 16, /* uneqr_f */ - 28, /* uneqi_f */ - 16, /* unger_f */ - 28, /* ungei_f */ - 16, /* ungtr_f */ - 28, /* ungti_f */ - 16, /* ltgtr_f */ - 28, /* ltgti_f */ - 16, /* ordr_f */ - 28, /* ordi_f */ - 16, /* unordr_f */ - 28, /* unordi_f */ - 8, /* truncr_f_i */ - 0, /* truncr_f_l */ - 8, /* extr_f */ - 4, /* extr_d_f */ - 4, /* movr_f */ - 12, /* movi_f */ - 4, /* ldr_f */ - 12, /* ldi_f */ - 8, /* ldxr_f */ - 4, /* ldxi_f */ - 4, /* str_f */ - 12, /* sti_f */ - 8, /* stxr_f */ - 4, /* stxi_f */ - 12, /* bltr_f */ - 24, /* blti_f */ - 12, /* bler_f */ - 24, /* blei_f */ - 12, /* beqr_f */ - 24, /* beqi_f */ - 12, /* bger_f */ - 24, /* bgei_f */ - 12, /* bgtr_f */ - 24, /* bgti_f */ - 12, /* bner_f */ - 24, /* bnei_f */ - 12, /* bunltr_f */ - 24, /* bunlti_f */ - 12, /* bunler_f */ - 24, /* bunlei_f */ - 12, /* buneqr_f */ - 24, /* buneqi_f */ - 12, /* bunger_f */ - 24, /* bungei_f */ - 12, /* bungtr_f */ - 24, /* bungti_f */ - 12, /* bltgtr_f */ - 24, /* bltgti_f */ - 12, /* bordr_f */ - 24, /* bordi_f */ - 12, /* bunordr_f */ - 24, /* bunordi_f */ - 0, /* pushargr_f */ - 0, /* pushargi_f */ - 0, /* retr_f */ - 0, /* reti_f */ - 0, /* retval_f */ - 0, /* arg_d */ - 0, /* getarg_d */ - 0, /* putargr_d */ - 0, /* putargi_d */ - 4, /* addr_d */ - 16, /* addi_d */ - 4, /* subr_d */ - 16, /* subi_d */ - 16, /* rsbi_d */ - 4, /* mulr_d */ - 16, /* muli_d */ - 4, /* divr_d */ - 16, /* divi_d */ - 4, /* negr_d */ - 4, /* absr_d */ - 4, /* sqrtr_d */ - 16, /* ltr_d */ - 28, /* lti_d */ - 16, /* ler_d */ - 28, /* lei_d */ - 16, /* eqr_d */ - 28, /* eqi_d */ - 16, /* ger_d */ - 28, /* gei_d */ - 16, /* gtr_d */ - 28, /* gti_d */ - 16, /* ner_d */ - 28, /* nei_d */ - 16, /* unltr_d */ - 28, /* unlti_d */ - 16, /* unler_d */ - 28, /* unlei_d */ - 16, /* uneqr_d */ - 28, /* uneqi_d */ - 16, /* unger_d */ - 28, /* ungei_d */ - 16, /* ungtr_d */ - 28, /* ungti_d */ - 16, /* ltgtr_d */ - 28, /* ltgti_d */ - 16, /* ordr_d */ - 28, /* ordi_d */ - 16, /* unordr_d */ - 28, /* unordi_d */ - 8, /* truncr_d_i */ - 0, /* truncr_d_l */ - 8, /* extr_d */ - 4, /* extr_f_d */ - 4, /* movr_d */ - 12, /* movi_d */ - 4, /* ldr_d */ - 12, /* ldi_d */ - 8, /* ldxr_d */ - 4, /* ldxi_d */ - 4, /* str_d */ - 12, /* sti_d */ - 8, /* stxr_d */ - 4, /* stxi_d */ - 12, /* bltr_d */ - 24, /* blti_d */ - 12, /* bler_d */ - 24, /* blei_d */ - 12, /* beqr_d */ - 24, /* beqi_d */ - 12, /* bger_d */ - 24, /* bgei_d */ - 12, /* bgtr_d */ - 24, /* bgti_d */ - 12, /* bner_d */ - 24, /* bnei_d */ - 12, /* bunltr_d */ - 24, /* bunlti_d */ - 12, /* bunler_d */ - 24, /* bunlei_d */ - 12, /* buneqr_d */ - 24, /* buneqi_d */ - 12, /* bunger_d */ - 24, /* bungei_d */ - 12, /* bungtr_d */ - 24, /* bungti_d */ - 12, /* bltgtr_d */ - 24, /* bltgti_d */ - 12, /* bordr_d */ - 24, /* bordi_d */ - 12, /* bunordr_d */ - 24, /* bunordi_d */ - 0, /* pushargr_d */ - 0, /* pushargi_d */ - 0, /* retr_d */ - 0, /* reti_d */ - 0, /* retval_d */ - 0, /* movr_w_f */ - 0, /* movr_ww_d */ - 0, /* movr_w_d */ - 0, /* movr_f_w */ - 0, /* movi_f_w */ - 0, /* movr_d_ww */ - 0, /* movi_d_ww */ - 4, /* movr_d_w */ - 12, /* movi_d_w */ -#endif /* NEW_ABI */ -#endif /* __WORDSIZE */ - -#if __WORDSIZE == 32 -#if !NEW_ABI #define JIT_INSTR_MAX 116 0, /* data */ 0, /* live */ - 0, /* align */ + 20, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -419,7 +15,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -427,8 +26,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 20, /* va_arg_d */ @@ -446,8 +57,8 @@ 28, /* subxr */ 28, /* subxi */ 16, /* rsbi */ - 8, /* mulr */ - 16, /* muli */ + 4, /* mulr */ + 12, /* muli */ 12, /* qmulr */ 20, /* qmuli */ 12, /* qmulr_u */ @@ -465,7 +76,7 @@ 8, /* remr_u */ 16, /* remi_u */ 4, /* andr */ - 12, /* andi */ + 8, /* andi */ 4, /* orr */ 12, /* ori */ 4, /* xorr */ @@ -483,15 +94,15 @@ 4, /* ltr_u */ 4, /* lti_u */ 8, /* ler */ - 12, /* lei */ + 4, /* lei */ 8, /* ler_u */ - 12, /* lei_u */ - 12, /* eqr */ - 12, /* eqi */ + 4, /* lei_u */ + 8, /* eqr */ + 8, /* eqi */ 8, /* ger */ - 12, /* gei */ + 8, /* gei */ 8, /* ger_u */ - 12, /* gei_u */ + 8, /* gei_u */ 4, /* gtr */ 8, /* gti */ 4, /* gtr_u */ @@ -500,14 +111,21 @@ 8, /* nei */ 4, /* movr */ 8, /* movi */ - 8, /* extr_c */ + 4, /* movnr */ + 4, /* movzr */ + 36, /* casr */ + 44, /* casi */ + 4, /* extr_c */ 4, /* extr_uc */ - 8, /* extr_s */ + 4, /* extr_s */ 4, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ - 20, /* htonr_us */ - 52, /* htonr_ui */ + 8, /* bswapr_us */ + 8, /* bswapr_ui */ + 0, /* bswapr_ul */ + 4, /* htonr_us */ + 4, /* htonr_ui */ 0, /* htonr_ul */ 4, /* ldr_c */ 12, /* ldi_c */ @@ -595,16 +213,40 @@ 20, /* bxsubi_u */ 8, /* jmpr */ 8, /* jmpi */ - 12, /* callr */ + 8, /* callr */ 16, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -804,25 +446,32 @@ 8, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ -#endif /* NEW_ABI */ + 8, /* clo */ + 8, /* clz */ + 76, /* cto */ + 76, /* ctz */ #endif /* __WORDSIZE */ #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 44 +#define JIT_INSTR_MAX 76 0, /* data */ 0, /* live */ - 4, /* align */ + 24, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ - 44, /* prolog */ + 76, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -830,11 +479,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 0, /* va_start */ - 0, /* va_arg */ - 0, /* va_arg_d */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 4, /* va_start */ + 8, /* va_arg */ + 8, /* va_arg_d */ 0, /* va_end */ 4, /* addr */ 28, /* addi */ @@ -848,7 +509,7 @@ 36, /* subci */ 28, /* subxr */ 28, /* subxi */ - 32, /* rsbi */ + 36, /* rsbi */ 8, /* mulr */ 32, /* muli */ 12, /* qmulr */ @@ -868,7 +529,7 @@ 8, /* remr_u */ 32, /* remi_u */ 4, /* andr */ - 28, /* andi */ + 8, /* andi */ 4, /* orr */ 28, /* ori */ 4, /* xorr */ @@ -886,15 +547,15 @@ 4, /* ltr_u */ 4, /* lti_u */ 8, /* ler */ - 12, /* lei */ + 4, /* lei */ 8, /* ler_u */ - 12, /* lei_u */ - 12, /* eqr */ - 12, /* eqi */ + 4, /* lei_u */ + 8, /* eqr */ + 8, /* eqi */ 8, /* ger */ - 12, /* gei */ + 8, /* gei */ 8, /* ger_u */ - 12, /* gei_u */ + 8, /* gei_u */ 4, /* gtr */ 8, /* gti */ 4, /* gtr_u */ @@ -903,59 +564,66 @@ 8, /* nei */ 4, /* movr */ 28, /* movi */ - 8, /* extr_c */ + 4, /* movnr */ + 4, /* movzr */ + 36, /* casr */ + 56, /* casi */ + 4, /* extr_c */ 4, /* extr_uc */ - 8, /* extr_s */ + 4, /* extr_s */ 4, /* extr_us */ 4, /* extr_i */ - 8, /* extr_ui */ + 4, /* extr_ui */ + 8, /* bswapr_us */ + 16, /* bswapr_ui */ + 44, /* bswapr_ul */ 4, /* htonr_us */ 4, /* htonr_ui */ 4, /* htonr_ul */ 4, /* ldr_c */ - 12, /* ldi_c */ + 24, /* ldi_c */ 4, /* ldr_uc */ - 12, /* ldi_uc */ + 24, /* ldi_uc */ 4, /* ldr_s */ - 12, /* ldi_s */ + 24, /* ldi_s */ 4, /* ldr_us */ - 12, /* ldi_us */ + 24, /* ldi_us */ 4, /* ldr_i */ - 12, /* ldi_i */ + 24, /* ldi_i */ 4, /* ldr_ui */ - 12, /* ldi_ui */ + 24, /* ldi_ui */ 4, /* ldr_l */ - 12, /* ldi_l */ + 24, /* ldi_l */ 8, /* ldxr_c */ - 4, /* ldxi_c */ + 16, /* ldxi_c */ 8, /* ldxr_uc */ - 4, /* ldxi_uc */ + 16, /* ldxi_uc */ 8, /* ldxr_s */ - 4, /* ldxi_s */ + 16, /* ldxi_s */ 8, /* ldxr_us */ - 4, /* ldxi_us */ + 16, /* ldxi_us */ 8, /* ldxr_i */ - 4, /* ldxi_i */ + 16, /* ldxi_i */ 8, /* ldxr_ui */ - 4, /* ldxi_ui */ + 16, /* ldxi_ui */ 8, /* ldxr_l */ - 4, /* ldxi_l */ + 16, /* ldxi_l */ 4, /* str_c */ - 12, /* sti_c */ + 24, /* sti_c */ 4, /* str_s */ - 12, /* sti_s */ + 24, /* sti_s */ 4, /* str_i */ - 12, /* sti_i */ + 24, /* sti_i */ 4, /* str_l */ - 12, /* sti_l */ + 24, /* sti_l */ 8, /* stxr_c */ - 4, /* stxi_c */ + 16, /* stxi_c */ 8, /* stxr_s */ - 4, /* stxi_s */ + 16, /* stxi_s */ 8, /* stxr_i */ - 4, /* stxi_i */ + 16, /* stxi_i */ 8, /* stxr_l */ - 4, /* stxi_l */ + 16, /* stxi_l */ 12, /* bltr */ 12, /* blti */ 12, /* bltr_u */ @@ -975,7 +643,7 @@ 12, /* bgtr_u */ 16, /* bgti_u */ 8, /* bner */ - 32, /* bnei */ + 28, /* bnei */ 12, /* bmsr */ 12, /* bmsi */ 12, /* bmcr */ @@ -996,18 +664,42 @@ 28, /* bxsubi */ 16, /* bxsubr_u */ 20, /* bxsubi_u */ - 0, /* jmpr */ + 8, /* jmpr */ 8, /* jmpi */ - 12, /* callr */ + 8, /* callr */ 32, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -1015,93 +707,93 @@ 0, /* retval_i */ 0, /* retval_ui */ 0, /* retval_l */ - 44, /* epilog */ + 76, /* epilog */ 0, /* arg_f */ 0, /* getarg_f */ 0, /* putargr_f */ 0, /* putargi_f */ 4, /* addr_f */ - 16, /* addi_f */ + 28, /* addi_f */ 4, /* subr_f */ - 16, /* subi_f */ - 16, /* rsbi_f */ + 28, /* subi_f */ + 28, /* rsbi_f */ 4, /* mulr_f */ - 16, /* muli_f */ + 28, /* muli_f */ 4, /* divr_f */ - 16, /* divi_f */ + 28, /* divi_f */ 4, /* negr_f */ 4, /* absr_f */ 4, /* sqrtr_f */ 16, /* ltr_f */ - 28, /* lti_f */ + 40, /* lti_f */ 16, /* ler_f */ - 28, /* lei_f */ + 40, /* lei_f */ 16, /* eqr_f */ - 28, /* eqi_f */ + 40, /* eqi_f */ 16, /* ger_f */ - 28, /* gei_f */ + 40, /* gei_f */ 16, /* gtr_f */ - 28, /* gti_f */ + 40, /* gti_f */ 16, /* ner_f */ - 28, /* nei_f */ + 40, /* nei_f */ 16, /* unltr_f */ - 28, /* unlti_f */ + 40, /* unlti_f */ 16, /* unler_f */ - 28, /* unlei_f */ + 40, /* unlei_f */ 16, /* uneqr_f */ - 28, /* uneqi_f */ + 40, /* uneqi_f */ 16, /* unger_f */ - 28, /* ungei_f */ + 40, /* ungei_f */ 16, /* ungtr_f */ - 28, /* ungti_f */ + 40, /* ungti_f */ 16, /* ltgtr_f */ - 28, /* ltgti_f */ + 40, /* ltgti_f */ 16, /* ordr_f */ - 28, /* ordi_f */ + 40, /* ordi_f */ 16, /* unordr_f */ - 28, /* unordi_f */ + 40, /* unordi_f */ 8, /* truncr_f_i */ 8, /* truncr_f_l */ 8, /* extr_f */ 4, /* extr_d_f */ 4, /* movr_f */ - 12, /* movi_f */ + 24, /* movi_f */ 4, /* ldr_f */ - 12, /* ldi_f */ + 24, /* ldi_f */ 8, /* ldxr_f */ - 4, /* ldxi_f */ + 16, /* ldxi_f */ 4, /* str_f */ - 12, /* sti_f */ + 24, /* sti_f */ 8, /* stxr_f */ - 4, /* stxi_f */ + 16, /* stxi_f */ 12, /* bltr_f */ - 24, /* blti_f */ + 36, /* blti_f */ 12, /* bler_f */ - 24, /* blei_f */ + 36, /* blei_f */ 12, /* beqr_f */ - 24, /* beqi_f */ + 36, /* beqi_f */ 12, /* bger_f */ - 24, /* bgei_f */ + 36, /* bgei_f */ 12, /* bgtr_f */ - 24, /* bgti_f */ + 36, /* bgti_f */ 12, /* bner_f */ - 24, /* bnei_f */ + 36, /* bnei_f */ 12, /* bunltr_f */ - 24, /* bunlti_f */ + 36, /* bunlti_f */ 12, /* bunler_f */ - 24, /* bunlei_f */ + 36, /* bunlei_f */ 12, /* buneqr_f */ - 24, /* buneqi_f */ + 36, /* buneqi_f */ 12, /* bunger_f */ - 24, /* bungei_f */ + 36, /* bungei_f */ 12, /* bungtr_f */ - 24, /* bungti_f */ + 36, /* bungti_f */ 12, /* bltgtr_f */ - 24, /* bltgti_f */ + 36, /* bltgti_f */ 12, /* bordr_f */ - 24, /* bordi_f */ + 36, /* bordi_f */ 12, /* bunordr_f */ - 24, /* bunordi_f */ + 36, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -1112,87 +804,87 @@ 0, /* putargr_d */ 0, /* putargi_d */ 4, /* addr_d */ - 16, /* addi_d */ + 28, /* addi_d */ 4, /* subr_d */ - 16, /* subi_d */ - 16, /* rsbi_d */ + 28, /* subi_d */ + 28, /* rsbi_d */ 4, /* mulr_d */ - 16, /* muli_d */ + 28, /* muli_d */ 4, /* divr_d */ - 16, /* divi_d */ + 28, /* divi_d */ 4, /* negr_d */ 4, /* absr_d */ 4, /* sqrtr_d */ 16, /* ltr_d */ - 28, /* lti_d */ + 44, /* lti_d */ 16, /* ler_d */ - 28, /* lei_d */ + 44, /* lei_d */ 16, /* eqr_d */ - 28, /* eqi_d */ + 44, /* eqi_d */ 16, /* ger_d */ - 28, /* gei_d */ + 44, /* gei_d */ 16, /* gtr_d */ - 28, /* gti_d */ + 44, /* gti_d */ 16, /* ner_d */ - 28, /* nei_d */ + 44, /* nei_d */ 16, /* unltr_d */ - 28, /* unlti_d */ + 44, /* unlti_d */ 16, /* unler_d */ - 28, /* unlei_d */ + 44, /* unlei_d */ 16, /* uneqr_d */ - 28, /* uneqi_d */ + 44, /* uneqi_d */ 16, /* unger_d */ - 28, /* ungei_d */ + 44, /* ungei_d */ 16, /* ungtr_d */ - 28, /* ungti_d */ + 44, /* ungti_d */ 16, /* ltgtr_d */ - 28, /* ltgti_d */ + 44, /* ltgti_d */ 16, /* ordr_d */ - 28, /* ordi_d */ + 44, /* ordi_d */ 16, /* unordr_d */ - 28, /* unordi_d */ + 44, /* unordi_d */ 8, /* truncr_d_i */ 8, /* truncr_d_l */ 8, /* extr_d */ 4, /* extr_f_d */ 4, /* movr_d */ - 12, /* movi_d */ + 28, /* movi_d */ 4, /* ldr_d */ - 12, /* ldi_d */ + 24, /* ldi_d */ 8, /* ldxr_d */ - 4, /* ldxi_d */ + 16, /* ldxi_d */ 4, /* str_d */ - 12, /* sti_d */ + 24, /* sti_d */ 8, /* stxr_d */ - 4, /* stxi_d */ + 16, /* stxi_d */ 12, /* bltr_d */ - 24, /* blti_d */ + 36, /* blti_d */ 12, /* bler_d */ - 24, /* blei_d */ + 36, /* blei_d */ 12, /* beqr_d */ - 24, /* beqi_d */ + 36, /* beqi_d */ 12, /* bger_d */ - 24, /* bgei_d */ + 36, /* bgei_d */ 12, /* bgtr_d */ - 24, /* bgti_d */ + 36, /* bgti_d */ 12, /* bner_d */ - 24, /* bnei_d */ + 40, /* bnei_d */ 12, /* bunltr_d */ - 24, /* bunlti_d */ + 40, /* bunlti_d */ 12, /* bunler_d */ - 24, /* bunlei_d */ + 40, /* bunlei_d */ 12, /* buneqr_d */ - 24, /* buneqi_d */ + 40, /* buneqi_d */ 12, /* bunger_d */ - 24, /* bungei_d */ + 40, /* bungei_d */ 12, /* bungtr_d */ - 24, /* bungti_d */ + 40, /* bungti_d */ 12, /* bltgtr_d */ - 24, /* bltgti_d */ + 36, /* bltgti_d */ 12, /* bordr_d */ - 24, /* bordi_d */ + 36, /* bordi_d */ 12, /* bunordr_d */ - 24, /* bunordi_d */ + 40, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -1206,5 +898,9 @@ 0, /* movr_d_ww */ 0, /* movi_d_ww */ 4, /* movr_d_w */ - 12, /* movi_d_w */ + 24, /* movi_d_w */ + 4, /* clo */ + 4, /* clz */ + 72, /* cto */ + 72, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_mips.c b/parallel-rsp/lightning/lib/jit_mips.c index dafade8..0c50882 100644 --- a/parallel-rsp/lightning/lib/jit_mips.c +++ b/parallel-rsp/lightning/lib/jit_mips.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -21,6 +21,16 @@ # include #endif +#if NEW_ABI +/* callee save + variadic arguments + * align16(ra+fp+s[0-7]++f20+f22+f24+f26+f28+f30) + align16(a[0-7]) */ +# define stack_framesize (128 + 64) +#else +/* callee save + * align16(ra+fp+s[0-7]+f16+f18+f20+f22+f24+f26+f28+f30) */ +# define stack_framesize 128 +#endif + #if NEW_ABI # define NUM_WORD_ARGS 8 # define STACK_SLOT 8 @@ -54,12 +64,14 @@ typedef struct jit_pointer_t jit_va_list_t; /* * Prototypes */ -#define jit_make_arg(node) _jit_make_arg(_jit,node) -static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*); +#define jit_make_arg(node,code) _jit_make_arg(_jit,node,code) +static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*,jit_code_t); #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node) static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*); #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node) static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*); +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); #define patch(instr, node) _patch(_jit, instr, node) static void _patch(jit_state_t*,jit_word_t,jit_node_t*); @@ -67,11 +79,13 @@ static void _patch(jit_state_t*,jit_word_t,jit_node_t*); # include "jit_rewind.c" # include "jit_mips-cpu.c" # include "jit_mips-fpu.c" +# include "jit_fallback.c" #undef PROTO /* * Initialization */ +jit_cpu_t jit_cpu; jit_register_t _rvs[] = { { rc(gpr) | 0x01, "at" }, { rc(gpr) | 0x02, "v0" }, @@ -145,12 +159,47 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { + _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7 +}; + +static jit_int32_t fregs[] = { +#if !NEW_ABI + _F16, _F18, +#endif + _F20, _F22, _F24, _F26, _F28, _F30 +}; + /* * Implementation */ void jit_get_cpu(void) { +#if defined(__linux__) + FILE *fp; + char *ptr; + char buf[128]; + + if ((fp = fopen("/proc/cpuinfo", "r")) != NULL) { + while (fgets(buf, sizeof(buf), fp)) { + if (strncmp(buf, "isa : ", 8) == 0) { + if ((ptr = strstr(buf + 9, "mips64r"))) + jit_cpu.release = strtoul(ptr + 7, NULL, 10); + break; + } + } + fclose(fp); + } +#endif + /* This is very fragile -- with clang, __mips is defined as 64 */ +#if defined _MIPS_ARCH /* OpenBSD/clang */ + if (!jit_cpu.release) + jit_cpu.release = strtoul(&_MIPS_ARCH[4], NULL, 10); +#elif defined(__mips) /* Linux/gcc */ + if (!jit_cpu.release) + jit_cpu.release = __mips; +#endif } void @@ -211,6 +260,7 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + jit_check_frame(); switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -259,20 +309,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -332,18 +380,18 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); #if NEW_ABI - return (jit_arg_reg_p(u->u.w)); + return (jit_arg_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8)); #else return (u->u.w < 8); #endif } static jit_node_t * -_jit_make_arg(jit_state_t *_jit, jit_node_t *node) +_jit_make_arg(jit_state_t *_jit, jit_node_t *node, jit_code_t code) { jit_int32_t offset; #if NEW_ABI @@ -355,13 +403,13 @@ _jit_make_arg(jit_state_t *_jit, jit_node_t *node) } #else offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT; - _jitc->function->self.argi = 1; + ++_jitc->function->self.argi; if (offset >= 4) offset = _jitc->function->self.size; _jitc->function->self.size += STACK_SLOT; #endif if (node == (jit_node_t *)0) - node = jit_new_node(jit_code_arg); + node = jit_new_node(code); else link_node(node); node->u.w = offset; @@ -469,7 +517,6 @@ _jit_ellipsis(jit_state_t *_jit) else { assert(!(_jitc->function->self.call & jit_call_varargs)); #if NEW_ABI - /* If varargs start in a register, allocate extra 64 bytes. */ if (jit_arg_reg_p(_jitc->function->self.argi)) rewind_prolog(); /* Do not set during possible rewind. */ @@ -482,6 +529,7 @@ _jit_ellipsis(jit_state_t *_jit) _jitc->function->vagp = _jitc->function->self.argi; } jit_inc_synth(ellipsis); + jit_check_frame(); if (_jitc->prepare) jit_link_prepare(); else @@ -498,10 +546,14 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { assert(_jitc->function); - return (jit_make_arg((jit_node_t*)0)); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif + return (jit_make_arg((jit_node_t*)0, code)); } jit_node_t * @@ -521,55 +573,67 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, _A0 - v->u.w); - else - jit_ldxi_c(u, _FP, v->u.w + C_DISP); + else { + jit_node_t *node = jit_ldxi_c(u, _FP, v->u.w + C_DISP); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, _A0 - v->u.w); - else - jit_ldxi_uc(u, _FP, v->u.w + C_DISP); + else { + jit_node_t *node = jit_ldxi_uc(u, _FP, v->u.w + C_DISP); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, _A0 - v->u.w); - else - jit_ldxi_s(u, _FP, v->u.w + S_DISP); + else { + jit_node_t *node = jit_ldxi_s(u, _FP, v->u.w + S_DISP); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, _A0 - v->u.w); - else - jit_ldxi_us(u, _FP, v->u.w + S_DISP); + else { + jit_node_t *node = jit_ldxi_us(u, _FP, v->u.w + S_DISP); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) { #if __WORDSIZE == 64 @@ -578,8 +642,11 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr(u, _A0 - v->u.w); #endif } - else - jit_ldxi_i(u, _FP, v->u.w + I_DISP); + else { + jit_node_t *node = jit_ldxi_i(u, _FP, v->u.w + I_DISP); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } @@ -587,52 +654,64 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, _A0 - v->u.w); - else - jit_ldxi_ui(u, _FP, v->u.w + I_DISP); + else { + jit_node_t *node = jit_ldxi_ui(u, _FP, v->u.w + I_DISP); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, _A0 - v->u.w); - else - jit_ldxi_l(u, _FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_l(u, _FP, v->u.w); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } #endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - jit_inc_synth_wp(putargr, u, v); - assert(v->code == jit_code_arg); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(_A0 - v->u.w, u); - else - jit_stxi(v->u.w + WORD_ADJUST, _FP, u); + else { + jit_node_t *node = jit_stxi(v->u.w + WORD_ADJUST, _FP, u); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(_A0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w + WORD_ADJUST, _FP, regno); + node = jit_stxi(v->u.w + WORD_ADJUST, _FP, regno); + jit_link_alist(node); + jit_check_frame(); jit_unget_reg(regno); } jit_dec_synth(); @@ -647,15 +726,18 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_f(u, _F12 - v->u.w); else if (jit_arg_reg_p(v->u.w - 8)) - jit_movr_w_f(u, _A0 - v->u.w - 8); + jit_movr_w_f(u, _A0 - (v->u.w - 8)); #else if (v->u.w < 4) jit_movr_w_f(u, _A0 - v->u.w); else if (v->u.w < 8) jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1)); #endif - else - jit_ldxi_f(u, _FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_f(u, _FP, v->u.w); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } @@ -668,15 +750,18 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_f(_F12 - v->u.w, u); else if (jit_arg_reg_p(v->u.w - 8)) - jit_movr_f_w(_A0 - v->u.w - 8, u); + jit_movr_f_w(_A0 - (v->u.w - 8), u); #else if (v->u.w < 4) jit_movr_f_w(_A0 - v->u.w, u); else if (v->u.w < 8) jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u); #endif - else - jit_stxi_f(v->u.w, _FP, u); + else { + jit_node_t *node = jit_stxi_f(v->u.w, _FP, u); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } @@ -689,12 +774,8 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) #if NEW_ABI if (jit_arg_reg_p(v->u.w)) jit_movi_f(_F12 - v->u.w, u); - else if (jit_arg_reg_p(v->u.w - 8)) { - regno = jit_get_reg(jit_class_fpr); - jit_movi_f(regno, u); - jit_movr_f_w(_A0 - v->u.w - 8, u); - jit_unget_reg(regno); - } + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movi_f_w(_A0 - (v->u.w - 8), u); #else if (v->u.w < 4) { regno = jit_get_reg(jit_class_fpr); @@ -706,9 +787,12 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u); #endif else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, _FP, regno); + node = jit_stxi_f(v->u.w, _FP, regno); + jit_link_alist(node); + jit_check_frame(); jit_unget_reg(regno); } jit_dec_synth(); @@ -723,15 +807,18 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_d(u, _F12 - v->u.w); else if (jit_arg_reg_p(v->u.w - 8)) - jit_movr_d_w(_A0 - v->u.w - 8, u); + jit_movr_d_w(_A0 - (v->u.w - 8), u); #else if (v->u.w < 4) jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1)); else if (v->u.w < 8) jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1)); #endif - else - jit_ldxi_d(u, _FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_d(u, _FP, v->u.w); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } @@ -744,15 +831,18 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_d(_F12 - v->u.w, u); else if (jit_arg_reg_p(v->u.w - 8)) - jit_movr_d_w(_A0 - v->u.w - 8, u); + jit_movr_d_w(_A0 - (v->u.w - 8), u); #else if (v->u.w < 4) jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u); else if (v->u.w < 8) jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u); #endif - else - jit_stxi_d(v->u.w, _FP, u); + else { + jit_node_t *node = jit_stxi_d(v->u.w, _FP, u); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } @@ -765,12 +855,8 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) #if NEW_ABI if (jit_arg_reg_p(v->u.w)) jit_movi_d(_F12 - v->u.w, u); - else if (jit_arg_reg_p(v->u.w - 8)) { - regno = jit_get_reg(jit_class_fpr); - jit_movi_d(regno, u); - jit_movr_d_w(_A0 - v->u.w - 8, u); - jit_unget_reg(regno); - } + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movi_d_w(_A0 - (v->u.w - 8), u); #else if (v->u.w < 4) { regno = jit_get_reg(jit_class_fpr); @@ -782,18 +868,21 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u); #endif else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, _FP, regno); + node = jit_stxi_d(v->u.w, _FP, regno); + jit_link_alist(node); + jit_check_frame(); jit_unget_reg(regno); } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); #if NEW_ABI assert(_jitc->function); @@ -802,6 +891,7 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) ++_jitc->function->call.argi; } else { + jit_check_frame(); jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u); _jitc->function->call.size += STACK_SLOT; } @@ -809,25 +899,27 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) jit_word_t offset; assert(_jitc->function); offset = _jitc->function->call.size >> STACK_SHIFT; - _jitc->function->call.argi = 1; + ++_jitc->function->call.argi; if (jit_arg_reg_p(offset)) jit_movr(_A0 - offset, u); - else + else { + jit_check_frame(); jit_stxi(_jitc->function->call.size, JIT_SP, u); + } _jitc->function->call.size += STACK_SLOT; #endif jit_dec_synth(); } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; #if !NEW_ABI jit_word_t offset; #endif assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); #if NEW_ABI if (jit_arg_reg_p(_jitc->function->call.argi)) { @@ -835,6 +927,7 @@ _jit_pushargi(jit_state_t *_jit, jit_word_t u) ++_jitc->function->call.argi; } else { + jit_check_frame(); regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno); @@ -847,6 +940,7 @@ _jit_pushargi(jit_state_t *_jit, jit_word_t u) if (jit_arg_reg_p(offset)) jit_movi(_A0 - offset, u); else { + jit_check_frame(); regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); jit_stxi(_jitc->function->call.size, JIT_SP, regno); @@ -875,6 +969,7 @@ _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) ++_jitc->function->call.argi; } else { + jit_check_frame(); jit_stxi_f(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += STACK_SLOT; } @@ -889,8 +984,10 @@ _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) ++_jitc->function->call.argi; jit_movr_f_w(_A0 - offset, u); } - else + else { + jit_check_frame(); jit_stxi_f(_jitc->function->call.size, JIT_SP, u); + } _jitc->function->call.size += STACK_SLOT; #endif jit_dec_synth(); @@ -915,6 +1012,7 @@ _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) ++_jitc->function->call.argi; } else { + jit_check_frame(); regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); @@ -933,6 +1031,7 @@ _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) jit_movi_f_w(_A0 - offset, u); } else { + jit_check_frame(); regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); @@ -962,6 +1061,7 @@ _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) ++_jitc->function->call.argi; } else { + jit_check_frame(); jit_stxi_d(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += STACK_SLOT; } @@ -982,8 +1082,10 @@ _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) ++_jitc->function->call.argf; } } - else + else { + jit_check_frame(); jit_stxi_d(_jitc->function->call.size, JIT_SP, u); + } _jitc->function->call.size += sizeof(jit_float64_t); #endif jit_dec_synth(); @@ -1009,6 +1111,7 @@ _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) ++_jitc->function->call.argi; } else { + jit_check_frame(); regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); jit_stxi_d(_jitc->function->call.size, JIT_SP, regno); @@ -1033,6 +1136,7 @@ _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) } } else { + jit_check_frame(); regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); jit_stxi_d(_jitc->function->call.size, JIT_SP, regno); @@ -1070,11 +1174,11 @@ _jit_finishr(jit_state_t *_jit, jit_int32_t r0) { jit_node_t *call; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishr, r0); if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; - jit_movr(_T9, r0); - call = jit_callr(_T9); + call = jit_callr(r0); call->v.w = _jitc->function->self.argi; #if NEW_ABI call->w.w = call->v.w; @@ -1091,13 +1195,12 @@ jit_node_t * _jit_finishi(jit_state_t *_jit, jit_pointer_t i0) { jit_node_t *call; - jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishi, (jit_word_t)i0); if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; - node = jit_movi(_T9, (jit_word_t)i0); - call = jit_callr(_T9); + call = jit_calli(i0); call->v.w = _jitc->function->call.argi; #if NEW_ABI call->w.w = call->v.w; @@ -1108,7 +1211,7 @@ _jit_finishi(jit_state_t *_jit, jit_pointer_t i0) _jitc->function->call.size = 0; _jitc->prepare = 0; jit_dec_synth(); - return (node); + return (call); } void @@ -1178,14 +1281,16 @@ _jit_retval_d(jit_state_t *_jit, jit_int32_t r0) jit_pointer_t _emit_code(jit_state_t *_jit) { - jit_node_t *node; + jit_node_t *node, *prev = NULL; jit_node_t *temp; jit_word_t word; jit_int32_t value; jit_int32_t offset; + struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1257,6 +1362,21 @@ _emit_code(jit_state_t *_jit) patch(word, node); \ } \ break +#define case_brrn(name, type, prev) \ + case jit_code_##name##r##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##r##type(temp->u.w, rn(node->v.w), \ + rn(node->w.w), prev); \ + else { \ + word = name##r##type(_jit->pc.w, \ + rn(node->v.w), rn(node->w.w), \ + prev); \ + patch(word, node); \ + } \ + break #define case_brw(name, type) \ case jit_code_##name##i##type: \ temp = node->u.n; \ @@ -1271,6 +1391,21 @@ _emit_code(jit_state_t *_jit) patch(word, node); \ } \ break +#define case_brwn(name, type, prev) \ + case jit_code_##name##i##type: \ + temp = node->u.n; \ + assert(temp->code == jit_code_label || \ + temp->code == jit_code_epilog); \ + if (temp->flag & jit_flag_patch) \ + name##i##type(temp->u.w, \ + rn(node->v.w), node->w.w, prev); \ + else { \ + word = name##i##type(_jit->pc.w, \ + rn(node->v.w), node->w.w, \ + prev); \ + patch(word, node); \ + } \ + break #define case_brf(name, type, size) \ case jit_code_##name##i##type: \ temp = node->u.n; \ @@ -1300,11 +1435,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1419,6 +1556,11 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _ui); #if __WORDSIZE == 64 case_rr(hton, _ul); +#endif + case_rr(bswap, _us); + case_rr(bswap, _ui); +#if __WORDSIZE == 64 + case_rr(bswap, _ul); #endif case_rr(ext, _c); case_rr(ext, _uc); @@ -1428,6 +1570,16 @@ _emit_code(jit_state_t *_jit) case_rr(ext, _i); case_rr(ext, _ui); #endif + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1448,6 +1600,10 @@ _emit_code(jit_state_t *_jit) break; case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(lt,); case_rrw(lt,); case_rrr(lt, _u); @@ -1468,26 +1624,26 @@ _emit_code(jit_state_t *_jit) case_rrw(gt, _u); case_rrr(ne,); case_rrw(ne,); - case_brr(blt,); - case_brw(blt,); - case_brr(blt, _u); - case_brw(blt, _u); - case_brr(ble,); - case_brw(ble,); - case_brr(ble, _u); - case_brw(ble, _u); - case_brr(beq,); - case_brw(beq,); - case_brr(bge,); - case_brw(bge,); - case_brr(bge, _u); - case_brw(bge, _u); - case_brr(bgt,); - case_brw(bgt,); - case_brr(bgt, _u); - case_brw(bgt, _u); - case_brr(bne,); - case_brw(bne,); + case_brrn(blt,, prev); + case_brwn(blt,, prev); + case_brrn(blt, _u, prev); + case_brwn(blt, _u, prev); + case_brrn(ble,, prev); + case_brwn(ble,, prev); + case_brrn(ble, _u, prev); + case_brwn(ble, _u, prev); + case_brrn(beq,, prev); + case_brwn(beq,, prev); + case_brrn(bge,, prev); + case_brwn(bge,, prev); + case_brrn(bge, _u, prev); + case_brwn(bge, _u, prev); + case_brrn(bgt,, prev); + case_brwn(bgt,, prev); + case_brrn(bgt, _u, prev); + case_brwn(bgt, _u, prev); + case_brrn(bne,, prev); + case_brwn(bne,, prev); case_brr(boadd,); case_brw(boadd,); case_brr(boadd, _u); @@ -1675,7 +1831,8 @@ _emit_code(jit_state_t *_jit) case_brr(bunord, _d); case_brf(bunord, _d, 64); case jit_code_jmpr: - jmpr(rn(node->u.w)); + jit_check_frame(); + jmpr(rn(node->u.w), prev); break; case jit_code_jmpi: if (node->flag & jit_flag_node) { @@ -1683,40 +1840,61 @@ _emit_code(jit_state_t *_jit) assert(temp->code == jit_code_label || temp->code == jit_code_epilog); if (temp->flag & jit_flag_patch) - jmpi(temp->u.w); + jmpi(temp->u.w, prev, 0); else { - word = jmpi(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (can_relative_jump_p(word)) + word = jmpi(_jit->pc.w, prev, 1); + else + word = jmpi_p(_jit->pc.w, prev); patch(word, node); } } - else - jmpi(node->u.w); + else { + jit_check_frame(); + jmpi(node->u.w, prev, 0); + } break; case jit_code_callr: - callr(rn(node->u.w)); + jit_check_frame(); + callr(rn(node->u.w), prev); break; case jit_code_calli: if (node->flag & jit_flag_node) { temp = node->u.n; assert(temp->code == jit_code_label || temp->code == jit_code_epilog); - word = calli_p(temp->u.w); - if (!(temp->flag & jit_flag_patch)) + if (temp->flag & jit_flag_patch) + calli(temp->u.w, NULL, 0); + else { + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (jit_mips2_p() && can_relative_jump_p(word)) + word = calli(_jit->pc.w, prev, 1); + else + word = calli_p(_jit->pc.w); patch(word, node); + } + } + else { + jit_check_frame(); + calli(node->u.w, NULL, 0); } - else - calli(node->u.w); break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif undo.patch_offset = _jitc->patches.offset; restart_function: _jitc->again = 0; + compute_framesize(); + patch_alist(0); prolog(node); break; case jit_code_epilog: @@ -1731,10 +1909,25 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + /* this will be recomputed but undo anyway to have it + * better self documented.*/ + undo.func.need_stack = _jitc->function->need_stack; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif _jitc->patches.offset = undo.patch_offset; + patch_alist(1); goto restart_function; } /* remember label is defined */ @@ -1785,14 +1978,26 @@ _emit_code(jit_state_t *_jit) case jit_code_va_arg_d: vaarg_d(rn(node->u.w), rn(node->v.w)); break; - case jit_code_live: - case jit_code_arg: case jit_code_ellipsis: + case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +# if __WORDSIZE == 64 + case jit_code_arg_l: +# endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1802,10 +2007,26 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_ui: case jit_code_getarg_l: #endif case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1841,6 +2062,11 @@ _emit_code(jit_state_t *_jit) assert(_jitc->synth == 0); /* update register live state */ jit_reglive(node); + + if (node->code != jit_code_note + && node->code != jit_code_name + && node->code != jit_code_live) + prev = node; } #undef case_brf #undef case_brw @@ -1868,6 +2094,7 @@ _emit_code(jit_state_t *_jit) # include "jit_rewind.c" # include "jit_mips-cpu.c" # include "jit_mips-fpu.c" +# include "jit_fallback.c" #undef CODE void @@ -1907,6 +2134,29 @@ _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) stxi_d(i0, rn(r0), rn(r1)); } +static void +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = STACK_SLOT << 1; /* ra+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += STACK_SLOT; + + for (reg = 0; reg < jit_size(fregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) + _jitc->framesize += sizeof(jit_float64_t); + +#if NEW_ABI + /* Space to store variadic arguments */ + if (_jitc->function->self.call & jit_call_varargs) + _jitc->framesize += (NUM_WORD_ARGS - _jitc->function->vagp) * STACK_SLOT; +#endif + + /* Make sure functions called have a 16 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 15) & -16; +} + static void _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) { diff --git a/parallel-rsp/lightning/lib/jit_names.c b/parallel-rsp/lightning/lib/jit_names.c index 475bc96..e5985a3 100644 --- a/parallel-rsp/lightning/lib/jit_names.c +++ b/parallel-rsp/lightning/lib/jit_names.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014-2019 Free Software Foundation, Inc. + * Copyright (C) 2014-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -21,17 +21,27 @@ static char *code_name[] = { "data", "live", "align", "save", "load", + "skip", "#name", "#note", "label", "prolog", "ellipsis", "va_push", "allocai", "allocar", - "arg", + "arg_c", + "arg_s", + "arg_i", + "arg_l", "getarg_c", "getarg_uc", "getarg_s", "getarg_us", "getarg_i", "getarg_ui", "getarg_l", - "putargr", "putargi", + "putargr_c", "putargi_c", + "putargr_uc", "putargi_uc", + "putargr_s", "putargi_s", + "putargr_us", "putargi_us", + "putargr_i", "putargi_i", + "putargr_ui", "putargi_ui", + "putargr_l", "putargi_l", "va_start", "va_arg", "va_arg_d", "va_end", @@ -69,9 +79,13 @@ static char *code_name[] = { "gtr_u", "gti_u", "ner", "nei", "movr", "movi", + "movnr", "movzr", + "casr", "casi", "extr_c", "extr_uc", "extr_s", "extr_us", "extr_i", "extr_ui", + "bswapr_us", + "bswapr_ui", "bswapr_ul", "htonr_us", "htonr_ui", "htonr_ul", "ldr_c", "ldi_c", @@ -119,10 +133,22 @@ static char *code_name[] = { "jmpr", "jmpi", "callr", "calli", "prepare", - "pushargr", "pushargi", + "pushargr_c", "pushargi_c", + "pushargr_uc", "pushargi_uc", + "pushargr_s", "pushargi_s", + "pushargr_us", "pushargi_us", + "pushargr_i", "pushargi_i", + "pushargr_ui", "pushargi_ui", + "pushargr_l", "pushargi_l", "finishr", "finishi", "ret", - "retr", "reti", + "retr_c", "reti_c", + "retr_uc", "reti_uc", + "retr_s", "reti_s", + "retr_us", "reti_us", + "retr_i", "reti_i", + "retr_ui", "reti_ui", + "retr_l", "reti_l", "retval_c", "retval_uc", "retval_s", "retval_us", "retval_i", "retval_ui", @@ -227,4 +253,6 @@ static char *code_name[] = { "movr_f_w", "movi_f_w", "movr_d_ww", "movi_d_ww", "movr_d_w", "movi_d_w", + "clo", "clz", + "cto", "ctz", }; diff --git a/parallel-rsp/lightning/lib/jit_note.c b/parallel-rsp/lightning/lib/jit_note.c index c79b818..b055619 100644 --- a/parallel-rsp/lightning/lib/jit_note.c +++ b/parallel-rsp/lightning/lib/jit_note.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/lib/jit_ppc-cpu.c b/parallel-rsp/lightning/lib/jit_ppc-cpu.c index c4397ad..67874c6 100644 --- a/parallel-rsp/lightning/lib/jit_ppc-cpu.c +++ b/parallel-rsp/lightning/lib/jit_ppc-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -202,8 +202,21 @@ static void _FXS(jit_state_t*,int,int,int,int,int,int,int); # define XCMPLI(cr,l,a,u) FCI(10,cr,l,a,u) # define CMPLDI(a,s) XCMPLI(0,1,a,s) # define CMPLWI(a,s) XCMPLI(0,0,a,s) +# if __WORDSIZE == 32 +# define CMPX(a,b) CMPW(a,b) +# define CMPXI(a,s) CMPWI(a,s) +# define CMPLX(a,b) CMPLW(a,b) +# define CMPLXI(a,s) CMPLWI(a,s) +# else +# define CMPX(a,b) CMPD(a,b) +# define CMPXI(a,s) CMPDI(a,s) +# define CMPLX(a,b) CMPLD(a,b) +# define CMPLXI(a,s) CMPLDI(a,s) +# endif # define CNTLZW(a,s) FX(31,s,a,0,26) # define CNTLZW_(a,s) FX_(31,s,a,0,26) +# define CNTLZD(a,s) FX(31,s,a,0,58) +# define CNTLZD_(a,s) FX_(31,s,a,0,58) # define CRAND(d,a,b) FX(19,d,a,b,257) # define CRANDC(d,a,b) FX(19,d,a,b,129) # define CREQV(d,a,b) FX(19,d,a,b,289) @@ -260,7 +273,7 @@ static void _FXS(jit_state_t*,int,int,int,int,int,int,int); # define LHAU(d,a,s) FDs(43,d,a,s) # define LHAUX(d,a,b) FX(31,d,a,b,375) # define LHAX(d,a,b) FX(31,d,a,b,343) -# define LHRBX(d,a,b) FX(31,d,a,b,790) +# define LHBRX(d,a,b) FX(31,d,a,b,790) # define LHZ(d,a,s) FDs(40,d,a,s) # define LHZU(d,a,s) FDs(41,d,a,s) # define LHZUX(d,a,b) FX(31,d,a,b,311) @@ -271,6 +284,7 @@ static void _FXS(jit_state_t*,int,int,int,int,int,int,int); # define LSWI(d,a,n) FX(31,d,a,n,597) # define LSWX(d,a,b) FX(31,d,a,b,533) # define LWARX(d,a,b) FX(31,d,a,b,20) +# define LDARX(d,a,b) FX(31,d,a,b,84) # define LWBRX(d,a,b) FX(31,d,a,b,534) # define LWA(d,a,s) FDs(58,d,a,s|2) # define LWAUX(d,a,b) FX(31,d,a,b,373) @@ -446,6 +460,7 @@ static void _MCRXR(jit_state_t*, jit_int32_t); # define STW(s,a,d) FDs(36,s,a,d) # define STWBRX(s,a,b) FX(31,s,a,b,662) # define STWCX_(s,a,b) FX_(31,s,a,b,150) +# define STDCX_(s,a,b) FX_(31,s,a,b,214) # define STWU(s,a,d) FDs(37,s,a,d) # define STWUX(s,a,b) FX(31,s,a,b,183) # define STWX(s,a,b) FX(31,s,a,b,151) @@ -505,10 +520,32 @@ static void _nop(jit_state_t*,jit_int32_t); static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); # define movi(r0,i0) _movi(_jit,r0,i0) static void _movi(jit_state_t*,jit_int32_t,jit_word_t); +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define negr(r0,r1) NEG(r0,r1) # define comr(r0,r1) NOT(r0,r1) +# define bitswap(r0, r1) _bitswap(_jit, r0, r1) +static void _bitswap(jit_state_t*, jit_int32_t, jit_int32_t); +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# if __WORDSIZE == 32 +# define clzr(r0, r1) CNTLZW(r0, r1) +# else +# define clzr(r0, r1) CNTLZD(r0, r1) +# endif +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # define extr_c(r0,r1) EXTSB(r0,r1) # define extr_uc(r0,r1) ANDI_(r0,r1,0xff) # define extr_s(r0,r1) EXTSH(r0,r1) @@ -517,23 +554,14 @@ static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); # define extr_i(r0,r1) EXTSW(r0,r1) # define extr_ui(r0,r1) CLRLDI(r0,r1,32) # endif -# if __BYTE_ORDER == __BIG_ENDIAN -# define htonr_us(r0,r1) extr_us(r0,r1) -# if __WORDSIZE == 32 -# define htonr_ui(r0,r1) movr(r0,r1) -# else -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) -# endif -# else -# define htonr_us(r0,r1) _htonr_us(_jit,r0,r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0,r1) _htonr_ui(_jit,r0,r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# if __WORDSIZE == 64 -# define htonr_ul(r0,r1) _htonr_ul(_jit,r0,r1) -static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t); -# endif +# define bswapr_us_lh(r0,r1,no_flag) _bswapr_us(_jit,r0,r1,no_flag) +# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1,0) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_bool_t); +# define bswapr_ui_lw(r0,r1,no_flag) _bswapr_ui(_jit,r0,r1,no_flag) +# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1,0) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_bool_t); +# if __WORDSIZE == 64 +# define bswapr_ul(r0,r1) generic_bswapr_ul(_jit,r0,r1) # endif # define addr(r0,r1,r2) ADD(r0,r1,r2) # define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) @@ -856,14 +884,14 @@ static jit_word_t _jmpi_p(jit_state_t*,jit_word_t) maybe_unused; # define callr(r0,i0) _callr(_jit,r0,i0) static void _callr(jit_state_t*,jit_int32_t,jit_int32_t); # define calli(i0,i1) _calli(_jit,i0,i1) -static void _calli(jit_state_t*,jit_word_t,jit_int32_t); +static jit_word_t _calli(jit_state_t*,jit_word_t,jit_int32_t); # define calli_p(i0,i1) _calli_p(_jit,i0,i1) static jit_word_t _calli_p(jit_state_t*,jit_word_t,jit_int32_t); # else # define callr(r0) _callr(_jit,r0) static void _callr(jit_state_t*,jit_int32_t); # define calli(i0) _calli(_jit,i0) -static void _calli(jit_state_t*,jit_word_t); +static jit_word_t _calli(jit_state_t*,jit_word_t); # define calli_p(i0) _calli_p(_jit,i0) static jit_word_t _calli_p(jit_state_t*,jit_word_t); #endif @@ -1120,6 +1148,22 @@ _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) } } +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + CMPXI(r2, 0); + BEQ(8); + MR(r0, r1); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + CMPXI(r2, 0); + BNE(8); + MR(r0, r1); +} + static jit_word_t _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { @@ -1138,47 +1182,210 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (word); } -# if __BYTE_ORDER == __LITTLE_ENDIAN static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) { - jit_int32_t t0; + jit_int32_t r1_reg, iscasi; + jit_word_t retry, done, jump0, jump1; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + SYNC(); + /* retry: */ + retry = _jit->pc.w; +# if __WORDSIZE == 32 + LWARX(r0, _R0_REGNO, r1); +# else + LDARX(r0, _R0_REGNO, r1); +# endif + jump0 = bner(_jit->pc.w, r0, r2); /* bne done r0 r2 */ +# if __WORDSIZE == 32 + STWCX_(r3, _R0_REGNO, r1); +# else + STDCX_(r3, _R0_REGNO, r1); +# endif + jump1 = _jit->pc.w; + BNE(0); /* BNE retry */ + /* done: */ + done = _jit->pc.w; + ISYNC(); + MFCR(r0); + EXTRWI(r0, r0, 1, CR_EQ); + patch_at(jump0, done); + patch_at(jump1, retry); + if (iscasi) + jit_unget_reg(r1_reg); +} + +/* http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */ +/* +unsigned int v; // 32-bit word to reverse bit order + +// swap odd and even bits +v = ((v >> 1) & 0x55555555) | ((v & 0x55555555) << 1); +// swap consecutive pairs +v = ((v >> 2) & 0x33333333) | ((v & 0x33333333) << 2); +// swap nibbles ... +v = ((v >> 4) & 0x0F0F0F0F) | ((v & 0x0F0F0F0F) << 4); +// swap bytes +v = ((v >> 8) & 0x00FF00FF) | ((v & 0x00FF00FF) << 8); +// swap 2-byte long pairs +v = ( v >> 16 ) | ( v << 16); + */ +static void +_bitswap(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t t0, t1, t2, t3, t4; + movr(r0, r1); t0 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 8); - andi(r0, r1, 0xff); - andi(rn(t0), rn(t0), 0xff); - lshi(r0, r0, 8); - orr(r0, r0, rn(t0)); + t1 = jit_get_reg(jit_class_gpr); + t2 = jit_get_reg(jit_class_gpr); + movi(rn(t0), __WORDSIZE == 32 ? 0x55555555L : 0x5555555555555555L); + rshi_u(rn(t1), r0, 1); /* t1 = v >> 1 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 1); /* t2 <<= 1 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x33333333L : 0x3333333333333333L); + rshi_u(rn(t1), r0, 2); /* t1 = v >> 2 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 2); /* t2 <<= 2 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x0f0f0f0fL : 0x0f0f0f0f0f0f0f0fL); + rshi_u(rn(t1), r0, 4); /* t1 = v >> 4 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 4); /* t2 <<= 4 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x00ff00ffL : 0x00ff00ff00ff00ffL); + rshi_u(rn(t1), r0, 8); /* t1 = v >> 8 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 8); /* t2 <<= 8 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# if __WORDSIZE == 32 + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + lshi(rn(t2), r0, 16); /* t2 = v << 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# else + movi(rn(t0), 0x0000ffff0000ffffL); + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 16); /* t2 <<= 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + rshi_u(rn(t1), r0, 32); /* t1 = v >> 32 */ + lshi(rn(t2), r0, 32); /* t2 = v << 32 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# endif + jit_unget_reg(t2); + jit_unget_reg(t1); jit_unget_reg(t0); } static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { - jit_int32_t reg; + comr(r0, r1); + clzr(r0, r0); +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + bitswap(r0, r1); + clor(r0, r0); +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + bitswap(r0, r1); + clzr(r0, r0); +} + +static void +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t no_flag) +{ + jit_int32_t reg, addr_reg; + + /* Convert load followed by bswap to a single instruction */ + /* FIXME r0 and r1 do not need to be the same, only must check if + * r1 was loaded in previous instruction */ + if (no_flag && r0 == r1) { + if ((*(_jit->pc.ui - 1) & 0xffe007ff) == (0x7c00022e | r0 << 21)) { + /* Convert LHZX to LHBRX */ + _jit->pc.ui--; + LHBRX(r0, (*_jit->pc.ui >> 16) & 0x1f, (*_jit->pc.ui >> 11) & 0x1f); + return; + } + + if ((*(_jit->pc.ui - 1) & 0xffe00000) == (0xa0000000 | r0 << 21)) { + /* Convert LHZ to LHBRX */ + _jit->pc.ui--; + addr_reg = (*_jit->pc.ui >> 16) & 0x1f; + + reg = jit_get_reg(jit_class_gpr); + LI(rn(reg), (short)*_jit->pc.ui); + LHBRX(r0, rn(reg), addr_reg); + jit_unget_reg(reg); + return; + } + } + + if (r0 == r1) { + RLWIMI(r0, r0, 16, 8, 15); + RLWINM(r0, r0, 24, 16, 31); + } else { + RLWINM(r0, r1, 8, 16, 23); + RLWIMI(r0, r1, 24, 24, 31); + } +} + +static void +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t no_flag) +{ + jit_int32_t reg, addr_reg; + + /* Convert load followed by bswap to a single instruction */ + /* FIXME r0 and r1 do not need to be the same, only must check if + * r1 was loaded in previous instruction */ + if (no_flag && r0 == r1) { + if ((*(_jit->pc.ui - 1) & 0xffe007ff) == (0x7c00002e | r0 << 21)) { + /* Convert LWZX to LWBRX */ + _jit->pc.ui--; + LWBRX(r0, (*_jit->pc.ui >> 16) & 0x1f, (*_jit->pc.ui >> 11) & 0x1f); + return; + } + + if ((*(_jit->pc.ui - 1) & 0xffe00000) == (0x80000000 | r0 << 21)) { + /* Convert LWZ to LWBRX */ + _jit->pc.ui--; + addr_reg = (*_jit->pc.ui >> 16) & 0x1f; + + reg = jit_get_reg(jit_class_gpr); + LI(rn(reg), (short)*_jit->pc.ui); + LWBRX(r0, rn(reg), addr_reg); + jit_unget_reg(reg); + return; + } + } + reg = jit_get_reg(jit_class_gpr); ROTLWI(rn(reg), r1, 8); RLWIMI(rn(reg), r1, 24, 0, 7); RLWIMI(rn(reg), r1, 24, 16, 23); +# if __WORDSIZE == 64 CLRLDI(r0, rn(reg), 32); - jit_unget_reg(reg); -} - -# if __WORDSIZE == 64 -static void -_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t reg; - reg = jit_get_reg(jit_class_gpr); - rshi_u(rn(reg), r1, 32); - htonr_ui(r0, r1); - htonr_ui(rn(reg), rn(reg)); - lshi(r0, r0, 32); - orr(r0, r0, rn(reg)); - jit_unget_reg(reg); -} -# endif +# else + MR(r0,rn(reg)); # endif + jit_unget_reg(reg); +} static void _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) @@ -1433,15 +1640,23 @@ _remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) jit_unget_reg(reg); } +# define is_mask(im) ((im) ? (__builtin_popcountl((im) + (1 << __builtin_ctzl(im))) <= 1) : 0) + static void _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; + jit_int32_t reg, offt; if (can_zero_extend_short_p(i0)) ANDI_(r0, r1, i0); else if (can_zero_extend_int_p(i0) && !(i0 & 0x0000ffff)) ANDIS_(r0, r1, (jit_uword_t)i0 >> 16); - else { + else if (__WORDSIZE == 32 && is_mask(i0)) { + offt = __builtin_ctzl(i0); + RLWINM(r0, r1, 0, 32 - offt - __builtin_popcountl(i0), 31 - offt); + } else if (__WORDSIZE == 32 && is_mask(~i0)) { + offt = __builtin_ctzl(~i0); + RLWINM(r0, r1, 0, 32 - offt, 31 - offt - __builtin_popcountl(~i0)); + } else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); AND(r0, r1, rn(reg)); @@ -1526,7 +1741,7 @@ _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - CMPW(r1, r2); + CMPX(r1, r2); MFCR(r0); EXTRWI(r0, r0, 1, CR_LT); } @@ -1536,11 +1751,11 @@ _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_sign_extend_short_p(i0)) - CMPWI(r1, i0); + CMPXI(r1, i0); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); - CMPW(r1, rn(reg)); + CMPX(r1, rn(reg)); jit_unget_reg(reg); } MFCR(r0); @@ -1574,7 +1789,7 @@ _lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - CMPW(r1, r2); + CMPX(r1, r2); CRNOT(CR_GT, CR_GT); MFCR(r0); EXTRWI(r0, r0, 1, CR_GT); @@ -1585,11 +1800,11 @@ _lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_sign_extend_short_p(i0)) - CMPWI(r1, i0); + CMPXI(r1, i0); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); - CMPW(r1, rn(reg)); + CMPX(r1, rn(reg)); jit_unget_reg(reg); } CRNOT(CR_GT, CR_GT); @@ -1626,7 +1841,7 @@ _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - CMPW(r1, r2); + CMPX(r1, r2); MFCR(r0); EXTRWI(r0, r0, 1, CR_EQ); } @@ -1636,13 +1851,13 @@ _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_sign_extend_short_p(i0)) - CMPWI(r1, i0); + CMPXI(r1, i0); else if (can_zero_extend_short_p(i0)) CMPLWI(r1, i0); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); - CMPW(r1, rn(reg)); + CMPX(r1, rn(reg)); jit_unget_reg(reg); } MFCR(r0); @@ -1652,7 +1867,7 @@ _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - CMPW(r1, r2); + CMPX(r1, r2); CRNOT(CR_LT, CR_LT); MFCR(r0); EXTRWI(r0, r0, 1, CR_LT); @@ -1663,11 +1878,11 @@ _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_sign_extend_short_p(i0)) - CMPWI(r1, i0); + CMPXI(r1, i0); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); - CMPW(r1, rn(reg)); + CMPX(r1, rn(reg)); jit_unget_reg(reg); } CRNOT(CR_LT, CR_LT); @@ -1704,7 +1919,7 @@ _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - CMPW(r1, r2); + CMPX(r1, r2); MFCR(r0); EXTRWI(r0, r0, 1, CR_GT); } @@ -1714,11 +1929,11 @@ _gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_sign_extend_short_p(i0)) - CMPWI(r1, i0); + CMPXI(r1, i0); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); - CMPW(r1, rn(reg)); + CMPX(r1, rn(reg)); jit_unget_reg(reg); } MFCR(r0); @@ -1752,7 +1967,7 @@ _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) static void _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { - CMPW(r1, r2); + CMPX(r1, r2); CRNOT(CR_EQ, CR_EQ); MFCR(r0); EXTRWI(r0, r0, 1, CR_EQ); @@ -1763,13 +1978,13 @@ _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; if (can_sign_extend_short_p(i0)) - CMPWI(r1, i0); + CMPXI(r1, i0); else if (can_zero_extend_short_p(i0)) CMPLWI(r1, i0); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); - CMPW(r1, rn(reg)); + CMPX(r1, rn(reg)); jit_unget_reg(reg); } CRNOT(CR_EQ, CR_EQ); @@ -1781,7 +1996,7 @@ static jit_word_t _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_word_t d, w; - CMPW(r0, r1); + CMPX(r0, r1); w = _jit->pc.w; d = (i0 - w) & ~3; BLT(d); @@ -1794,11 +2009,11 @@ _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; jit_word_t d, w; if (can_sign_extend_short_p(i1)) - CMPWI(r0, i1); + CMPXI(r0, i1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i1); - CMPW(r0, rn(reg)); + CMPX(r0, rn(reg)); jit_unget_reg(reg); } w = _jit->pc.w; @@ -1841,7 +2056,7 @@ static jit_word_t _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_word_t d, w; - CMPW(r0, r1); + CMPX(r0, r1); w = _jit->pc.w; d = (i0 - w) & ~3; BLE(d); @@ -1854,11 +2069,11 @@ _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; jit_word_t d, w; if (can_sign_extend_short_p(i1)) - CMPWI(r0, i1); + CMPXI(r0, i1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i1); - CMPW(r0, rn(reg)); + CMPX(r0, rn(reg)); jit_unget_reg(reg); } w = _jit->pc.w; @@ -1901,7 +2116,7 @@ static jit_word_t _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_word_t d, w; - CMPW(r0, r1); + CMPX(r0, r1); w = _jit->pc.w; d = (i0 - w) & ~3; BEQ(d); @@ -1914,13 +2129,13 @@ _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; jit_word_t d, w; if (can_sign_extend_short_p(i1)) - CMPWI(r0, i1); + CMPXI(r0, i1); else if (can_zero_extend_short_p(i1)) CMPLWI(r0, i1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i1); - CMPW(r0, rn(reg)); + CMPX(r0, rn(reg)); jit_unget_reg(reg); } w = _jit->pc.w; @@ -1933,7 +2148,7 @@ static jit_word_t _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_word_t d, w; - CMPW(r0, r1); + CMPX(r0, r1); w = _jit->pc.w; d = (i0 - w) & ~3; BGE(d); @@ -1946,11 +2161,11 @@ _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; jit_word_t d, w; if (can_sign_extend_short_p(i1)) - CMPWI(r0, i1); + CMPXI(r0, i1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i1); - CMPW(r0, rn(reg)); + CMPX(r0, rn(reg)); jit_unget_reg(reg); } w = _jit->pc.w; @@ -1993,7 +2208,7 @@ static jit_word_t _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_word_t d, w; - CMPW(r0, r1); + CMPX(r0, r1); w = _jit->pc.w; d = (i0 - w) & ~3; BGT(d); @@ -2006,11 +2221,11 @@ _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; jit_word_t d, w; if (can_sign_extend_short_p(i1)) - CMPWI(r0, i1); + CMPXI(r0, i1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i1); - CMPW(r0, rn(reg)); + CMPX(r0, rn(reg)); jit_unget_reg(reg); } w = _jit->pc.w; @@ -2053,7 +2268,7 @@ static jit_word_t _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { jit_word_t d, w; - CMPW(r0, r1); + CMPX(r0, r1); w = _jit->pc.w; d = (i0 - w) & ~3; BNE(d); @@ -2066,13 +2281,13 @@ _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; jit_word_t d, w; if (can_sign_extend_short_p(i1)) - CMPWI(r0, i1); + CMPXI(r0, i1); else if (can_zero_extend_short_p(i1)) CMPLWI(r0, i1); else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i1); - CMPW(r0, rn(reg)); + CMPX(r0, rn(reg)); jit_unget_reg(reg); } w = _jit->pc.w; @@ -2671,7 +2886,7 @@ _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) jit_int32_t reg; if (r1 == _R0_REGNO) { if (r2 != _R0_REGNO) - LWZX(r0, r2, r1); + LWAX(r0, r2, r1); else { reg = jit_get_reg(jit_class_gpr); movr(rn(reg), r1); @@ -2680,7 +2895,7 @@ _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) } } else - LWZX(r0, r1, r2); + LWAX(r0, r1, r2); } static void @@ -3200,21 +3415,28 @@ _callr(jit_state_t *_jit, jit_int32_t r0 } /* assume fixed address or reachable address */ -static void +static jit_word_t _calli(jit_state_t *_jit, jit_word_t i0 # if _CALL_SYSV , jit_int32_t varargs # endif ) { + jit_word_t w; # if _CALL_SYSV jit_word_t d; - d = (i0 - _jit->pc.w) & ~3; - if (can_sign_extend_jump_p(d)) + d = (i0 - _jit->pc.w - !!varargs * 4) & ~3; + if (can_sign_extend_jump_p(d)) { + /* Tell double arguments were passed in registers. */ + if (varargs) + CREQV(6, 6, 6); + w = _jit->pc.w; BL(d); + } else # endif { + w = _jit->pc.w; movi(_R12_REGNO, i0); callr(_R12_REGNO # if _CALL_SYSV @@ -3222,6 +3444,7 @@ _calli(jit_state_t *_jit, jit_word_t i0 # endif ); } + return (w); } /* absolute jump */ @@ -3545,7 +3768,7 @@ _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) if (!can_sign_extend_short_p(d)) { /* use absolute address */ assert(can_sign_extend_short_p(label)); - d |= 2; + d = label | 2; } u.i[0] = (u.i[0] & ~0xfffd) | (d & 0xfffe); break; @@ -3573,9 +3796,9 @@ _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) if (!can_sign_extend_jump_p(d)) { /* use absolute address */ assert(can_sign_extend_jump_p(label)); - d |= 2; + d = label | 2; } - u.i[0] = (u.i[0] & ~0x3fffffd) | (d & 0x3fffffe); + u.i[0] = (u.i[0] & ~0x3fffffc) | (d & 0x3fffffd); break; case 15: /* LI */ #if __WORDSIZE == 32 diff --git a/parallel-rsp/lightning/lib/jit_ppc-fpu.c b/parallel-rsp/lightning/lib/jit_ppc-fpu.c index 1e84f8e..12631cd 100644 --- a/parallel-rsp/lightning/lib/jit_ppc-fpu.c +++ b/parallel-rsp/lightning/lib/jit_ppc-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -143,8 +143,17 @@ static void _truncr_d_l(jit_state_t*,jit_int32_t,jit_int32_t); # define absr_d(r0,r1) FABS(r0,r1) # define negr_f(r0,r1) negr_d(r0,r1) # define negr_d(r0,r1) FNEG(r0,r1) -# define sqrtr_f(r0,r1) FSQRTS(r0,r1) -# define sqrtr_d(r0,r1) FSQRT(r0,r1) +# ifdef _ARCH_PPCSQ +# define sqrtr_f(r0,r1) FSQRTS(r0,r1) +# define sqrtr_d(r0,r1) FSQRT(r0,r1) +# else +extern float sqrtf(float); +# define sqrtr_f(r0,r1) _sqrtr_f(_jit,r0,r1) +static void _sqrtr_f(jit_state_t*,jit_int32_t,jit_int32_t); +extern double sqrt(double); +# define sqrtr_d(r0,r1) _sqrtr_d(_jit,r0,r1) +static void _sqrtr_d(jit_state_t*,jit_int32_t,jit_int32_t); +# endif # define addr_f(r0,r1,r2) FADDS(r0,r1,r2) # define addr_d(r0,r1,r2) FADD(r0,r1,r2) # define addi_f(r0,r1,i0) _addi_f(_jit,r0,r1,i0) @@ -484,23 +493,40 @@ _movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t *i0) ldi_d(r0, (jit_word_t)i0); } -/* should only work on newer ppc (fcfid is a ppc64 instruction) */ static void _extr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { # if __WORDSIZE == 32 - jit_int32_t reg; + jit_int32_t reg, freg, off1, off2; + +# if __BYTE_ORDER == __BIG_ENDIAN + off1 = alloca_offset - 8; + off2 = alloca_offset - 4; +# else + off1 = alloca_offset - 4; + off2 = alloca_offset - 8; +# endif + reg = jit_get_reg(jit_class_gpr); - rshi(rn(reg), r1, 31); - /* use reserved 8 bytes area */ - stxi(alloca_offset - 4, _FP_REGNO, r1); - stxi(alloca_offset - 8, _FP_REGNO, rn(reg)); + freg = jit_get_reg(jit_class_fpr); + + movi(rn(reg), 0x43300000); + stxi_i(off1, _FP_REGNO, rn(reg)); + movi(rn(reg), 0x80000000); + stxi_i(off2, _FP_REGNO, rn(reg)); + ldxi_d(rn(freg), _FP_REGNO, alloca_offset - 8); + xorr(rn(reg), r1, rn(reg)); + stxi_i(off2, _FP_REGNO, rn(reg)); + ldxi_d(r0, _FP_REGNO, alloca_offset - 8); + subr_d(r0, r0, rn(freg)); + jit_unget_reg(reg); + jit_unget_reg(freg); # else stxi(alloca_offset - 8, _FP_REGNO, r1); -# endif ldxi_d(r0, _FP_REGNO, alloca_offset - 8); FCFID(r0, r0); +# endif } static void @@ -511,7 +537,11 @@ _truncr_d_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) FCTIWZ(rn(reg), r1); /* use reserved 8 bytes area */ stxi_d(alloca_offset - 8, _FP_REGNO, rn(reg)); +# if __BYTE_ORDER == __BIG_ENDIAN ldxi_i(r0, _FP_REGNO, alloca_offset - 4); +# else + ldxi_i(r0, _FP_REGNO, alloca_offset - 8); +# endif jit_unget_reg(reg); } @@ -529,6 +559,32 @@ _truncr_d_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } # endif +# ifndef _ARCH_PPCSQ +static void +_sqrtr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + movr_f(rn(JIT_FA0), r1); + calli((jit_word_t)sqrtf +# if _CALL_SYSV + , 0 +# endif + ); + movr_f(r0, rn(JIT_FRET)); +} + +static void +_sqrtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + movr_d(rn(JIT_FA0), r1); + calli((jit_word_t)sqrt +# if _CALL_SYSV + , 0 +# endif + ); + movr_d(r0, rn(JIT_FRET)); +} +# endif + # define fpr_opi(name, type, size) \ static void \ _##name##i_##type(jit_state_t *_jit, \ diff --git a/parallel-rsp/lightning/lib/jit_ppc-sz.c b/parallel-rsp/lightning/lib/jit_ppc-sz.c index 788ac45..c8a4471 100644 --- a/parallel-rsp/lightning/lib/jit_ppc-sz.c +++ b/parallel-rsp/lightning/lib/jit_ppc-sz.c @@ -1,13 +1,473 @@ +#if __WORDSIZE == 32 +#if defined(__powerpc__) +#if __BYTE_ORDER == __BIG_ENDIAN +#if !_CALL_SYSV +#define JIT_INSTR_MAX 136 + 0, /* data */ + 0, /* live */ + 20, /* align */ + 0, /* save */ + 0, /* load */ + 4, /* skip */ + 0, /* #name */ + 0, /* #note */ + 0, /* label */ + 136, /* prolog */ + 0, /* ellipsis */ + 0, /* va_push */ + 0, /* allocai */ + 0, /* allocar */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ + 0, /* getarg_c */ + 0, /* getarg_uc */ + 0, /* getarg_s */ + 0, /* getarg_us */ + 0, /* getarg_i */ + 0, /* getarg_ui */ + 0, /* getarg_l */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 4, /* va_start */ + 8, /* va_arg */ + 8, /* va_arg_d */ + 0, /* va_end */ + 4, /* addr */ + 12, /* addi */ + 4, /* addcr */ + 12, /* addci */ + 4, /* addxr */ + 8, /* addxi */ + 4, /* subr */ + 12, /* subi */ + 4, /* subcr */ + 12, /* subci */ + 4, /* subxr */ + 8, /* subxi */ + 16, /* rsbi */ + 4, /* mulr */ + 12, /* muli */ + 12, /* qmulr */ + 16, /* qmuli */ + 12, /* qmulr_u */ + 16, /* qmuli_u */ + 4, /* divr */ + 12, /* divi */ + 4, /* divr_u */ + 12, /* divi_u */ + 20, /* qdivr */ + 16, /* qdivi */ + 20, /* qdivr_u */ + 16, /* qdivi_u */ + 12, /* remr */ + 20, /* remi */ + 12, /* remr_u */ + 20, /* remi_u */ + 4, /* andr */ + 4, /* andi */ + 4, /* orr */ + 12, /* ori */ + 4, /* xorr */ + 12, /* xori */ + 4, /* lshr */ + 4, /* lshi */ + 4, /* rshr */ + 4, /* rshi */ + 4, /* rshr_u */ + 4, /* rshi_u */ + 4, /* negr */ + 4, /* comr */ + 12, /* ltr */ + 12, /* lti */ + 12, /* ltr_u */ + 16, /* lti_u */ + 16, /* ler */ + 16, /* lei */ + 16, /* ler_u */ + 16, /* lei_u */ + 12, /* eqr */ + 12, /* eqi */ + 16, /* ger */ + 16, /* gei */ + 16, /* ger_u */ + 16, /* gei_u */ + 12, /* gtr */ + 12, /* gti */ + 12, /* gtr_u */ + 12, /* gti_u */ + 16, /* ner */ + 16, /* nei */ + 4, /* movr */ + 8, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 36, /* casr */ + 44, /* casi */ + 4, /* extr_c */ + 4, /* extr_uc */ + 4, /* extr_s */ + 4, /* extr_us */ + 0, /* extr_i */ + 0, /* extr_ui */ + 8, /* bswapr_us */ + 16, /* bswapr_ui */ + 0, /* bswapr_ul */ + 4, /* htonr_us */ + 4, /* htonr_ui */ + 0, /* htonr_ul */ + 8, /* ldr_c */ + 12, /* ldi_c */ + 4, /* ldr_uc */ + 8, /* ldi_uc */ + 4, /* ldr_s */ + 8, /* ldi_s */ + 4, /* ldr_us */ + 8, /* ldi_us */ + 4, /* ldr_i */ + 8, /* ldi_i */ + 0, /* ldr_ui */ + 0, /* ldi_ui */ + 0, /* ldr_l */ + 0, /* ldi_l */ + 8, /* ldxr_c */ + 16, /* ldxi_c */ + 4, /* ldxr_uc */ + 12, /* ldxi_uc */ + 4, /* ldxr_s */ + 12, /* ldxi_s */ + 4, /* ldxr_us */ + 12, /* ldxi_us */ + 4, /* ldxr_i */ + 12, /* ldxi_i */ + 0, /* ldxr_ui */ + 0, /* ldxi_ui */ + 0, /* ldxr_l */ + 0, /* ldxi_l */ + 4, /* str_c */ + 8, /* sti_c */ + 4, /* str_s */ + 8, /* sti_s */ + 4, /* str_i */ + 8, /* sti_i */ + 0, /* str_l */ + 0, /* sti_l */ + 4, /* stxr_c */ + 12, /* stxi_c */ + 4, /* stxr_s */ + 12, /* stxi_s */ + 4, /* stxr_i */ + 12, /* stxi_i */ + 0, /* stxr_l */ + 0, /* stxi_l */ + 8, /* bltr */ + 8, /* blti */ + 8, /* bltr_u */ + 12, /* blti_u */ + 8, /* bler */ + 8, /* blei */ + 8, /* bler_u */ + 12, /* blei_u */ + 8, /* beqr */ + 16, /* beqi */ + 8, /* bger */ + 8, /* bgei */ + 8, /* bger_u */ + 8, /* bgei_u */ + 8, /* bgtr */ + 8, /* bgti */ + 8, /* bgtr_u */ + 8, /* bgti_u */ + 8, /* bner */ + 16, /* bnei */ + 12, /* bmsr */ + 12, /* bmsi */ + 12, /* bmcr */ + 12, /* bmci */ + 12, /* boaddr */ + 16, /* boaddi */ + 12, /* boaddr_u */ + 12, /* boaddi_u */ + 12, /* bxaddr */ + 16, /* bxaddi */ + 12, /* bxaddr_u */ + 12, /* bxaddi_u */ + 12, /* bosubr */ + 16, /* bosubi */ + 12, /* bosubr_u */ + 16, /* bosubi_u */ + 12, /* bxsubr */ + 16, /* bxsubi */ + 12, /* bxsubr_u */ + 16, /* bxsubi_u */ + 8, /* jmpr */ + 4, /* jmpi */ + 28, /* callr */ + 36, /* calli */ + 0, /* prepare */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ + 0, /* finishr */ + 0, /* finishi */ + 0, /* ret */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ + 0, /* retval_c */ + 0, /* retval_uc */ + 0, /* retval_s */ + 0, /* retval_us */ + 0, /* retval_i */ + 0, /* retval_ui */ + 0, /* retval_l */ + 124, /* epilog */ + 0, /* arg_f */ + 0, /* getarg_f */ + 0, /* putargr_f */ + 0, /* putargi_f */ + 4, /* addr_f */ + 16, /* addi_f */ + 4, /* subr_f */ + 16, /* subi_f */ + 16, /* rsbi_f */ + 4, /* mulr_f */ + 16, /* muli_f */ + 4, /* divr_f */ + 16, /* divi_f */ + 4, /* negr_f */ + 4, /* absr_f */ + 4, /* sqrtr_f */ + 12, /* ltr_f */ + 24, /* lti_f */ + 16, /* ler_f */ + 28, /* lei_f */ + 12, /* eqr_f */ + 24, /* eqi_f */ + 16, /* ger_f */ + 28, /* gei_f */ + 12, /* gtr_f */ + 24, /* gti_f */ + 16, /* ner_f */ + 28, /* nei_f */ + 16, /* unltr_f */ + 28, /* unlti_f */ + 16, /* unler_f */ + 28, /* unlei_f */ + 16, /* uneqr_f */ + 28, /* uneqi_f */ + 16, /* unger_f */ + 28, /* ungei_f */ + 16, /* ungtr_f */ + 28, /* ungti_f */ + 16, /* ltgtr_f */ + 28, /* ltgti_f */ + 16, /* ordr_f */ + 28, /* ordi_f */ + 12, /* unordr_f */ + 24, /* unordi_f */ + 12, /* truncr_f_i */ + 0, /* truncr_f_l */ + 36, /* extr_f */ + 4, /* extr_d_f */ + 4, /* movr_f */ + 12, /* movi_f */ + 4, /* ldr_f */ + 8, /* ldi_f */ + 4, /* ldxr_f */ + 12, /* ldxi_f */ + 4, /* str_f */ + 8, /* sti_f */ + 4, /* stxr_f */ + 12, /* stxi_f */ + 8, /* bltr_f */ + 20, /* blti_f */ + 12, /* bler_f */ + 24, /* blei_f */ + 8, /* beqr_f */ + 20, /* beqi_f */ + 12, /* bger_f */ + 24, /* bgei_f */ + 8, /* bgtr_f */ + 20, /* bgti_f */ + 8, /* bner_f */ + 20, /* bnei_f */ + 12, /* bunltr_f */ + 24, /* bunlti_f */ + 8, /* bunler_f */ + 20, /* bunlei_f */ + 12, /* buneqr_f */ + 24, /* buneqi_f */ + 8, /* bunger_f */ + 20, /* bungei_f */ + 12, /* bungtr_f */ + 24, /* bungti_f */ + 12, /* bltgtr_f */ + 24, /* bltgti_f */ + 8, /* bordr_f */ + 20, /* bordi_f */ + 8, /* bunordr_f */ + 20, /* bunordi_f */ + 0, /* pushargr_f */ + 0, /* pushargi_f */ + 0, /* retr_f */ + 0, /* reti_f */ + 0, /* retval_f */ + 0, /* arg_d */ + 0, /* getarg_d */ + 0, /* putargr_d */ + 0, /* putargi_d */ + 4, /* addr_d */ + 24, /* addi_d */ + 4, /* subr_d */ + 24, /* subi_d */ + 24, /* rsbi_d */ + 4, /* mulr_d */ + 24, /* muli_d */ + 4, /* divr_d */ + 24, /* divi_d */ + 4, /* negr_d */ + 4, /* absr_d */ + 4, /* sqrtr_d */ + 12, /* ltr_d */ + 32, /* lti_d */ + 16, /* ler_d */ + 36, /* lei_d */ + 12, /* eqr_d */ + 32, /* eqi_d */ + 16, /* ger_d */ + 36, /* gei_d */ + 12, /* gtr_d */ + 32, /* gti_d */ + 16, /* ner_d */ + 36, /* nei_d */ + 16, /* unltr_d */ + 36, /* unlti_d */ + 16, /* unler_d */ + 36, /* unlei_d */ + 16, /* uneqr_d */ + 36, /* uneqi_d */ + 16, /* unger_d */ + 36, /* ungei_d */ + 16, /* ungtr_d */ + 36, /* ungti_d */ + 16, /* ltgtr_d */ + 36, /* ltgti_d */ + 16, /* ordr_d */ + 36, /* ordi_d */ + 12, /* unordr_d */ + 32, /* unordi_d */ + 12, /* truncr_d_i */ + 0, /* truncr_d_l */ + 36, /* extr_d */ + 4, /* extr_f_d */ + 4, /* movr_d */ + 28, /* movi_d */ + 4, /* ldr_d */ + 8, /* ldi_d */ + 4, /* ldxr_d */ + 12, /* ldxi_d */ + 4, /* str_d */ + 8, /* sti_d */ + 4, /* stxr_d */ + 12, /* stxi_d */ + 8, /* bltr_d */ + 28, /* blti_d */ + 12, /* bler_d */ + 32, /* blei_d */ + 8, /* beqr_d */ + 32, /* beqi_d */ + 12, /* bger_d */ + 32, /* bgei_d */ + 8, /* bgtr_d */ + 28, /* bgti_d */ + 8, /* bner_d */ + 28, /* bnei_d */ + 12, /* bunltr_d */ + 32, /* bunlti_d */ + 8, /* bunler_d */ + 28, /* bunlei_d */ + 12, /* buneqr_d */ + 32, /* buneqi_d */ + 8, /* bunger_d */ + 28, /* bungei_d */ + 12, /* bungtr_d */ + 32, /* bungti_d */ + 12, /* bltgtr_d */ + 32, /* bltgti_d */ + 8, /* bordr_d */ + 28, /* bordi_d */ + 8, /* bunordr_d */ + 28, /* bunordi_d */ + 0, /* pushargr_d */ + 0, /* pushargi_d */ + 0, /* retr_d */ + 0, /* reti_d */ + 0, /* retval_d */ + 0, /* movr_w_f */ + 0, /* movr_ww_d */ + 0, /* movr_w_d */ + 0, /* movr_f_w */ + 0, /* movi_f_w */ + 0, /* movr_d_ww */ + 0, /* movi_d_ww */ + 0, /* movr_d_w */ + 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 136, /* cto */ + 132, /* ctz */ +#endif /* !_CALL_SYSV */ +#endif /* __BYTE_ORDER */ +#endif /* __powerpc__ */ +#endif /* __WORDSIZE */ + #if __WORDSIZE == 32 #if defined(__powerpc__) #if __BYTE_ORDER == __BIG_ENDIAN #if _CALL_SYSV -#define JIT_INSTR_MAX 124 +#define JIT_INSTR_MAX 136 0, /* data */ 0, /* live */ - 0, /* align */ + 28, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -16,7 +476,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -24,8 +487,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 36, /* va_start */ 52, /* va_arg */ 64, /* va_arg_d */ @@ -62,7 +537,7 @@ 12, /* remr_u */ 20, /* remi_u */ 4, /* andr */ - 12, /* andi */ + 4, /* andi */ 4, /* orr */ 12, /* ori */ 4, /* xorr */ @@ -97,12 +572,19 @@ 16, /* nei */ 4, /* movr */ 8, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 36, /* casr */ + 44, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 8, /* bswapr_us */ + 16, /* bswapr_ui */ + 0, /* bswapr_ul */ 4, /* htonr_us */ 4, /* htonr_ui */ 0, /* htonr_ul */ @@ -195,13 +677,37 @@ 12, /* callr */ 20, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -256,7 +762,7 @@ 24, /* unordi_f */ 12, /* truncr_f_i */ 0, /* truncr_f_l */ - 20, /* extr_f */ + 36, /* extr_f */ 4, /* extr_d_f */ 4, /* movr_f */ 12, /* movi_f */ @@ -347,10 +853,10 @@ 32, /* unordi_d */ 12, /* truncr_d_i */ 0, /* truncr_d_l */ - 20, /* extr_d */ + 36, /* extr_d */ 4, /* extr_f_d */ 4, /* movr_d */ - 24, /* movi_d */ + 28, /* movi_d */ 4, /* ldr_d */ 8, /* ldi_d */ 4, /* ldxr_d */ @@ -401,428 +907,25 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ -#endif /* _CALL_SYV */ + 8, /* clo */ + 4, /* clz */ + 136, /* cto */ + 132, /* ctz */ +#endif /* _CALL_SYSV */ #endif /* __BYTE_ORDER */ #endif /* __powerpc__ */ #endif /* __WORDSIZE */ -#if __WORDSIZE == 32 -#if defined(__powerpc__) -#if __BYTE_ORDER == __BIG_ENDIAN -#if !_CALL_SYSV -#define JIT_INSTR_MAX 136 - 0, /* data */ - 0, /* live */ - 0, /* align */ - 0, /* save */ - 0, /* load */ - 0, /* #name */ - 0, /* #note */ - 0, /* label */ - 136, /* prolog */ - 0, /* ellipsis */ - 0, /* va_push */ - 0, /* allocai */ - 0, /* allocar */ - 0, /* arg */ - 0, /* getarg_c */ - 0, /* getarg_uc */ - 0, /* getarg_s */ - 0, /* getarg_us */ - 0, /* getarg_i */ - 0, /* getarg_ui */ - 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 4, /* va_start */ - 8, /* va_arg */ - 8, /* va_arg_d */ - 0, /* va_end */ - 4, /* addr */ - 12, /* addi */ - 4, /* addcr */ - 12, /* addci */ - 4, /* addxr */ - 8, /* addxi */ - 4, /* subr */ - 12, /* subi */ - 4, /* subcr */ - 12, /* subci */ - 4, /* subxr */ - 8, /* subxi */ - 16, /* rsbi */ - 4, /* mulr */ - 12, /* muli */ - 12, /* qmulr */ - 16, /* qmuli */ - 12, /* qmulr_u */ - 16, /* qmuli_u */ - 4, /* divr */ - 12, /* divi */ - 4, /* divr_u */ - 12, /* divi_u */ - 20, /* qdivr */ - 16, /* qdivi */ - 20, /* qdivr_u */ - 16, /* qdivi_u */ - 12, /* remr */ - 20, /* remi */ - 12, /* remr_u */ - 20, /* remi_u */ - 4, /* andr */ - 12, /* andi */ - 4, /* orr */ - 12, /* ori */ - 4, /* xorr */ - 12, /* xori */ - 4, /* lshr */ - 4, /* lshi */ - 4, /* rshr */ - 4, /* rshi */ - 4, /* rshr_u */ - 4, /* rshi_u */ - 4, /* negr */ - 4, /* comr */ - 12, /* ltr */ - 12, /* lti */ - 12, /* ltr_u */ - 16, /* lti_u */ - 16, /* ler */ - 16, /* lei */ - 16, /* ler_u */ - 16, /* lei_u */ - 12, /* eqr */ - 12, /* eqi */ - 16, /* ger */ - 16, /* gei */ - 16, /* ger_u */ - 16, /* gei_u */ - 12, /* gtr */ - 12, /* gti */ - 12, /* gtr_u */ - 12, /* gti_u */ - 16, /* ner */ - 16, /* nei */ - 4, /* movr */ - 8, /* movi */ - 4, /* extr_c */ - 4, /* extr_uc */ - 4, /* extr_s */ - 4, /* extr_us */ - 0, /* extr_i */ - 0, /* extr_ui */ - 4, /* htonr_us */ - 4, /* htonr_ui */ - 0, /* htonr_ul */ - 8, /* ldr_c */ - 12, /* ldi_c */ - 4, /* ldr_uc */ - 8, /* ldi_uc */ - 4, /* ldr_s */ - 8, /* ldi_s */ - 4, /* ldr_us */ - 8, /* ldi_us */ - 4, /* ldr_i */ - 8, /* ldi_i */ - 0, /* ldr_ui */ - 0, /* ldi_ui */ - 0, /* ldr_l */ - 0, /* ldi_l */ - 8, /* ldxr_c */ - 16, /* ldxi_c */ - 4, /* ldxr_uc */ - 12, /* ldxi_uc */ - 4, /* ldxr_s */ - 12, /* ldxi_s */ - 4, /* ldxr_us */ - 12, /* ldxi_us */ - 4, /* ldxr_i */ - 12, /* ldxi_i */ - 0, /* ldxr_ui */ - 0, /* ldxi_ui */ - 0, /* ldxr_l */ - 0, /* ldxi_l */ - 4, /* str_c */ - 8, /* sti_c */ - 4, /* str_s */ - 8, /* sti_s */ - 4, /* str_i */ - 8, /* sti_i */ - 0, /* str_l */ - 0, /* sti_l */ - 4, /* stxr_c */ - 12, /* stxi_c */ - 4, /* stxr_s */ - 12, /* stxi_s */ - 4, /* stxr_i */ - 12, /* stxi_i */ - 0, /* stxr_l */ - 0, /* stxi_l */ - 8, /* bltr */ - 8, /* blti */ - 8, /* bltr_u */ - 12, /* blti_u */ - 8, /* bler */ - 8, /* blei */ - 8, /* bler_u */ - 12, /* blei_u */ - 8, /* beqr */ - 16, /* beqi */ - 8, /* bger */ - 8, /* bgei */ - 8, /* bger_u */ - 8, /* bgei_u */ - 8, /* bgtr */ - 8, /* bgti */ - 8, /* bgtr_u */ - 8, /* bgti_u */ - 8, /* bner */ - 16, /* bnei */ - 12, /* bmsr */ - 12, /* bmsi */ - 12, /* bmcr */ - 12, /* bmci */ - 12, /* boaddr */ - 16, /* boaddi */ - 12, /* boaddr_u */ - 12, /* boaddi_u */ - 12, /* bxaddr */ - 16, /* bxaddi */ - 12, /* bxaddr_u */ - 12, /* bxaddi_u */ - 12, /* bosubr */ - 16, /* bosubi */ - 12, /* bosubr_u */ - 16, /* bosubi_u */ - 12, /* bxsubr */ - 16, /* bxsubi */ - 12, /* bxsubr_u */ - 16, /* bxsubi_u */ - 8, /* jmpr */ - 4, /* jmpi */ - 28, /* callr */ - 40, /* calli */ - 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ - 0, /* finishr */ - 0, /* finishi */ - 0, /* ret */ - 0, /* retr */ - 0, /* reti */ - 0, /* retval_c */ - 0, /* retval_uc */ - 0, /* retval_s */ - 0, /* retval_us */ - 0, /* retval_i */ - 0, /* retval_ui */ - 0, /* retval_l */ - 124, /* epilog */ - 0, /* arg_f */ - 0, /* getarg_f */ - 0, /* putargr_f */ - 0, /* putargi_f */ - 4, /* addr_f */ - 16, /* addi_f */ - 4, /* subr_f */ - 16, /* subi_f */ - 16, /* rsbi_f */ - 4, /* mulr_f */ - 16, /* muli_f */ - 4, /* divr_f */ - 16, /* divi_f */ - 4, /* negr_f */ - 4, /* absr_f */ - 4, /* sqrtr_f */ - 12, /* ltr_f */ - 24, /* lti_f */ - 16, /* ler_f */ - 28, /* lei_f */ - 12, /* eqr_f */ - 24, /* eqi_f */ - 16, /* ger_f */ - 28, /* gei_f */ - 12, /* gtr_f */ - 24, /* gti_f */ - 16, /* ner_f */ - 28, /* nei_f */ - 16, /* unltr_f */ - 28, /* unlti_f */ - 16, /* unler_f */ - 28, /* unlei_f */ - 16, /* uneqr_f */ - 28, /* uneqi_f */ - 16, /* unger_f */ - 28, /* ungei_f */ - 16, /* ungtr_f */ - 28, /* ungti_f */ - 16, /* ltgtr_f */ - 28, /* ltgti_f */ - 16, /* ordr_f */ - 28, /* ordi_f */ - 12, /* unordr_f */ - 24, /* unordi_f */ - 12, /* truncr_f_i */ - 0, /* truncr_f_l */ - 20, /* extr_f */ - 4, /* extr_d_f */ - 4, /* movr_f */ - 12, /* movi_f */ - 4, /* ldr_f */ - 8, /* ldi_f */ - 4, /* ldxr_f */ - 12, /* ldxi_f */ - 4, /* str_f */ - 8, /* sti_f */ - 4, /* stxr_f */ - 12, /* stxi_f */ - 8, /* bltr_f */ - 20, /* blti_f */ - 12, /* bler_f */ - 24, /* blei_f */ - 8, /* beqr_f */ - 20, /* beqi_f */ - 12, /* bger_f */ - 24, /* bgei_f */ - 8, /* bgtr_f */ - 20, /* bgti_f */ - 8, /* bner_f */ - 20, /* bnei_f */ - 12, /* bunltr_f */ - 24, /* bunlti_f */ - 8, /* bunler_f */ - 20, /* bunlei_f */ - 12, /* buneqr_f */ - 24, /* buneqi_f */ - 8, /* bunger_f */ - 20, /* bungei_f */ - 12, /* bungtr_f */ - 24, /* bungti_f */ - 12, /* bltgtr_f */ - 24, /* bltgti_f */ - 8, /* bordr_f */ - 20, /* bordi_f */ - 8, /* bunordr_f */ - 20, /* bunordi_f */ - 0, /* pushargr_f */ - 0, /* pushargi_f */ - 0, /* retr_f */ - 0, /* reti_f */ - 0, /* retval_f */ - 0, /* arg_d */ - 0, /* getarg_d */ - 0, /* putargr_d */ - 0, /* putargi_d */ - 4, /* addr_d */ - 24, /* addi_d */ - 4, /* subr_d */ - 24, /* subi_d */ - 24, /* rsbi_d */ - 4, /* mulr_d */ - 24, /* muli_d */ - 4, /* divr_d */ - 24, /* divi_d */ - 4, /* negr_d */ - 4, /* absr_d */ - 4, /* sqrtr_d */ - 12, /* ltr_d */ - 32, /* lti_d */ - 16, /* ler_d */ - 36, /* lei_d */ - 12, /* eqr_d */ - 32, /* eqi_d */ - 16, /* ger_d */ - 36, /* gei_d */ - 12, /* gtr_d */ - 32, /* gti_d */ - 16, /* ner_d */ - 36, /* nei_d */ - 16, /* unltr_d */ - 36, /* unlti_d */ - 16, /* unler_d */ - 36, /* unlei_d */ - 16, /* uneqr_d */ - 36, /* uneqi_d */ - 16, /* unger_d */ - 36, /* ungei_d */ - 16, /* ungtr_d */ - 36, /* ungti_d */ - 16, /* ltgtr_d */ - 36, /* ltgti_d */ - 16, /* ordr_d */ - 36, /* ordi_d */ - 12, /* unordr_d */ - 32, /* unordi_d */ - 12, /* truncr_d_i */ - 0, /* truncr_d_l */ - 20, /* extr_d */ - 4, /* extr_f_d */ - 4, /* movr_d */ - 24, /* movi_d */ - 4, /* ldr_d */ - 8, /* ldi_d */ - 4, /* ldxr_d */ - 12, /* ldxi_d */ - 4, /* str_d */ - 8, /* sti_d */ - 4, /* stxr_d */ - 12, /* stxi_d */ - 8, /* bltr_d */ - 28, /* blti_d */ - 12, /* bler_d */ - 32, /* blei_d */ - 8, /* beqr_d */ - 32, /* beqi_d */ - 12, /* bger_d */ - 32, /* bgei_d */ - 8, /* bgtr_d */ - 28, /* bgti_d */ - 8, /* bner_d */ - 28, /* bnei_d */ - 12, /* bunltr_d */ - 32, /* bunlti_d */ - 8, /* bunler_d */ - 28, /* bunlei_d */ - 12, /* buneqr_d */ - 32, /* buneqi_d */ - 8, /* bunger_d */ - 28, /* bungei_d */ - 12, /* bungtr_d */ - 32, /* bungti_d */ - 12, /* bltgtr_d */ - 32, /* bltgti_d */ - 8, /* bordr_d */ - 28, /* bordi_d */ - 8, /* bunordr_d */ - 28, /* bunordi_d */ - 0, /* pushargr_d */ - 0, /* pushargi_d */ - 0, /* retr_d */ - 0, /* reti_d */ - 0, /* retval_d */ - 0, /* movr_w_f */ - 0, /* movr_ww_d */ - 0, /* movr_w_d */ - 0, /* movr_f_w */ - 0, /* movi_f_w */ - 0, /* movr_d_ww */ - 0, /* movi_d_ww */ - 0, /* movr_d_w */ - 0, /* movi_d_w */ -#endif /* _CALL_AIX */ -#endif /* __BYTEORDER */ -#endif /* __powerpc__ */ -#endif /* __WORDSIZE */ - #if __WORDSIZE == 64 #if defined(__powerpc__) #if __BYTE_ORDER == __BIG_ENDIAN -#define JIT_INSTR_MAX 148 +#define JIT_INSTR_MAX 236 0, /* data */ 0, /* live */ - 4, /* align */ + 28, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -831,7 +934,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -839,8 +945,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 8, /* va_arg_d */ @@ -912,12 +1030,19 @@ 16, /* nei */ 4, /* movr */ 36, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 36, /* casr */ + 44, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 4, /* extr_i */ 4, /* extr_ui */ + 8, /* bswapr_us */ + 16, /* bswapr_ui */ + 44, /* bswapr_ul */ 4, /* htonr_us */ 4, /* htonr_ui */ 4, /* htonr_ul */ @@ -1008,15 +1133,39 @@ 8, /* jmpr */ 4, /* jmpi */ 28, /* callr */ - 56, /* calli */ + 52, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -1216,19 +1365,24 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ -#endif /* __BYTEORDER */ + 8, /* clo */ + 4, /* clz */ + 236, /* cto */ + 232, /* ctz */ +#endif /* __BYTE_ORDER */ #endif /* __powerpc__ */ #endif /* __WORDSIZE */ #if __WORDSIZE == 64 #if defined(__powerpc__) #if __BYTE_ORDER == __LITTLE_ENDIAN -#define JIT_INSTR_MAX 124 +#define JIT_INSTR_MAX 236 0, /* data */ 0, /* live */ - 4, /* align */ + 20, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -1237,7 +1391,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -1245,8 +1402,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 8, /* va_arg_d */ @@ -1318,13 +1487,20 @@ 16, /* nei */ 4, /* movr */ 36, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 36, /* casr */ + 44, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 4, /* extr_i */ 4, /* extr_ui */ - 20, /* htonr_us */ + 8, /* bswapr_us */ + 16, /* bswapr_ui */ + 44, /* bswapr_ul */ + 8, /* htonr_us */ 16, /* htonr_ui */ 44, /* htonr_ul */ 8, /* ldr_c */ @@ -1414,15 +1590,39 @@ 8, /* jmpr */ 4, /* jmpi */ 12, /* callr */ - 36, /* calli */ + 32, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -1622,6 +1822,10 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 8, /* clo */ + 4, /* clz */ + 236, /* cto */ + 232, /* ctz */ #endif /* __BYTE_ORDER */ #endif /* __powerpc__ */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_ppc.c b/parallel-rsp/lightning/lib/jit_ppc.c index 0826f4e..869e876 100644 --- a/parallel-rsp/lightning/lib/jit_ppc.c +++ b/parallel-rsp/lightning/lib/jit_ppc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -291,20 +291,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -364,7 +362,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_f_reg_p(u->u.w)); @@ -404,12 +402,16 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; jit_bool_t incr = 1; assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif if (jit_arg_reg_p(_jitc->function->self.argi)) { offset = _jitc->function->self.argi++; #if _CALL_SYSV @@ -420,7 +422,7 @@ _jit_arg(jit_state_t *_jit) offset = _jitc->function->self.size; if (incr) _jitc->function->self.size += sizeof(jit_word_t); - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -498,7 +500,7 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, JIT_RA0 - v->u.w); @@ -510,7 +512,7 @@ _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, JIT_RA0 - v->u.w); @@ -522,7 +524,7 @@ _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, JIT_RA0 - v->u.w); @@ -534,7 +536,7 @@ _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, JIT_RA0 - v->u.w); @@ -546,7 +548,7 @@ _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) { #if __WORDSIZE == 32 @@ -564,7 +566,7 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, JIT_RA0 - v->u.w); @@ -576,7 +578,7 @@ _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); @@ -587,10 +589,10 @@ _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) #endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(JIT_RA0 - v->u.w, u); else @@ -599,11 +601,11 @@ _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - jit_inc_synth_wp(putargi, u, v); - assert(v->code == jit_code_arg); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(JIT_RA0 - v->u.w, u); else { @@ -698,11 +700,11 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { jit_bool_t incr = 1; assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(JIT_RA0 - _jitc->function->call.argi, u); @@ -719,12 +721,12 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; jit_bool_t incr = 1; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(JIT_RA0 - _jitc->function->call.argi, u); @@ -1148,9 +1150,12 @@ _emit_code(jit_state_t *_jit) jit_word_t word; jit_int32_t value; jit_int32_t offset; + jit_bool_t no_flag = 0; /* Set if previous instruction is + * *not* a jump target. */ struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1286,11 +1291,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1355,9 +1362,32 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _ui); # if __WORDSIZE == 64 case_rr(hton, _ul); +# endif + case jit_code_bswapr_us: + bswapr_us_lh(rn(node->u.w), rn(node->v.w), no_flag); + break; + case jit_code_bswapr_ui: + bswapr_ui_lw(rn(node->u.w), rn(node->v.w), no_flag); + break; +# if __WORDSIZE == 64 + case_rr(bswap, _ul); # endif case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1671,44 +1701,58 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (can_sign_extend_jump_p(word)) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } else - (void)jmpi_p(node->u.w); + jmpi(node->u.w); break; case jit_code_callr: - callr(rn(node->u.w) #if _CALL_SYSV - , !!(node->flag & jit_flag_varargs) +# define xcallr(u, v) callr(u, v) +# define xcalli_p(u, v) calli_p(u, v) +# define xcalli(u, v) calli(u, v) +#else +# define xcallr(u, v) callr(u) +# define xcalli_p(u, v) calli_p(u) +# define xcalli(u, v) calli(u) #endif - ); + xcallr(rn(node->u.w), !!(node->flag & jit_flag_varargs)); break; case jit_code_calli: + value = !!(node->flag & jit_flag_varargs); if (node->flag & jit_flag_node) { temp = node->u.n; assert(temp->code == jit_code_label || temp->code == jit_code_epilog); - word = calli_p(temp->u.w + if (temp->flag & jit_flag_patch) + xcalli(temp->u.w, value); + else { + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); #if _CALL_SYSV - , !!(node->flag & jit_flag_varargs) + if (can_sign_extend_jump_p(word + value * 4)) + word = xcalli(_jit->pc.w, value); + else #endif - ); - if (!(temp->flag & jit_flag_patch)) + word = xcalli_p(_jit->pc.w, value); patch(word, node); + } } else - calli(node->u.w -#if _CALL_SYSV - , !!(node->flag & jit_flag_varargs) -#endif - ); + xcalli(node->u.w, value); break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1752,6 +1796,16 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1776,14 +1830,26 @@ _emit_code(jit_state_t *_jit) case jit_code_va_arg_d: vaarg_d(rn(node->u.w), rn(node->v.w)); break; - case jit_code_live: - case jit_code_arg: case jit_code_ellipsis: + case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +# if __WORDSIZE == 64 + case jit_code_arg_l: +# endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1793,10 +1859,26 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_ui: case jit_code_getarg_l: #endif case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1816,6 +1898,8 @@ _emit_code(jit_state_t *_jit) assert(_jitc->regarg == 0 && _jitc->synth == 0); /* update register live state */ jit_reglive(node); + + no_flag = !(node->flag & jit_flag_patch); } #undef case_brf #undef case_brw diff --git a/parallel-rsp/lightning/lib/jit_print.c b/parallel-rsp/lightning/lib/jit_print.c index 3d7c0ac..f2cfacd 100644 --- a/parallel-rsp/lightning/lib/jit_print.c +++ b/parallel-rsp/lightning/lib/jit_print.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -20,12 +20,29 @@ #include #include -#define print_chr(value) fputc(value, stdout) -#define print_hex(value) fprintf(stdout, "0x%lx", value) -#define print_dec(value) fprintf(stdout, "%ld", value) -#define print_flt(value) fprintf(stdout, "%g", value) -#define print_str(value) fprintf(stdout, "%s", value) -#define print_ptr(value) fprintf(stdout, "%p", value) +#if __WORDSIZE == 32 +# define MININT 0x80000000 +# define DEC_FMT "%d" +# define HEX_FMT "0x%x" +#else +# define MININT 0x8000000000000000 +# define DEC_FMT "%ld" +# define HEX_FMT "0x%lx" +#endif + + +#define print_chr(value) fputc(value, print_stream) +#define print_hex(value) \ + do { \ + if (value < 0 && value != MININT) \ + fprintf(print_stream, "-" HEX_FMT, (jit_uword_t)-value); \ + else \ + fprintf(print_stream, HEX_FMT, (jit_uword_t)value); \ + } while (0) +#define print_dec(value) fprintf(print_stream, DEC_FMT, value) +#define print_flt(value) fprintf(print_stream, "%g", value) +#define print_str(value) fprintf(print_stream, "%s", value) +#define print_ptr(value) fprintf(print_stream, "%p", value) #define print_reg(value) \ do { \ if ((value) & jit_regno_patch) \ @@ -45,10 +62,22 @@ * Initialization */ #include "jit_names.c" +/* + * Initialization + */ +static FILE *print_stream; + /* * Implementation */ +void +jit_init_print(void) +{ + if (!print_stream) + print_stream = stderr; +} + void _jit_print(jit_state_t *_jit) { @@ -95,7 +124,7 @@ _jit_print_node(jit_state_t *_jit, jit_node_t *node) (jit_cc_a0_int|jit_cc_a0_flt|jit_cc_a0_dbl|jit_cc_a0_jmp| jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_arg| jit_cc_a1_reg|jit_cc_a1_int|jit_cc_a1_flt|jit_cc_a1_dbl|jit_cc_a1_arg| - jit_cc_a2_reg|jit_cc_a2_int|jit_cc_a2_flt|jit_cc_a2_dbl); + jit_cc_a2_reg|jit_cc_a2_int|jit_cc_a2_flt|jit_cc_a2_dbl|jit_cc_a2_rlh); if (!(node->flag & jit_flag_synth) && ((value & jit_cc_a0_jmp) || node->code == jit_code_finishr || node->code == jit_code_finishi)) @@ -205,6 +234,18 @@ _jit_print_node(jit_state_t *_jit, jit_node_t *node) print_chr(' '); print_reg(node->u.q.h); print_str(") "); print_reg(node->v.w); print_chr(' '); print_hex(node->w.w); return; + r_r_q: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_reg(node->v.w); + print_str(" ("); print_reg(node->w.q.l); + print_chr(' '); print_reg(node->w.q.h); + print_str(") "); return; + r_w_q: + print_chr(' '); print_reg(node->u.w); + print_chr(' '); print_hex(node->v.w); + print_str(" ("); print_reg(node->w.q.l); + print_chr(' '); print_reg(node->w.q.h); + print_str(") "); return; r_r_f: print_chr(' '); print_reg(node->u.w); print_chr(' '); print_reg(node->v.w); @@ -280,12 +321,12 @@ _jit_print_node(jit_state_t *_jit, jit_node_t *node) case jit_code_name: print_chr(' '); if (node->v.p && _jitc->emit) - print_str(node->v.n->u.p); + print_str((char *)node->v.n->u.p); break; case jit_code_note: print_chr(' '); if (node->v.p && _jitc->emit) - print_str(node->v.n->u.p); + print_str((char *)node->v.n->u.p); if (node->v.p && _jitc->emit && node->w.w) print_chr(':'); if (node->w.w) @@ -345,6 +386,12 @@ _jit_print_node(jit_state_t *_jit, jit_node_t *node) case jit_cc_a0_reg|jit_cc_a0_rlh| jit_cc_a1_reg|jit_cc_a2_int: goto q_r_w; + case jit_cc_a0_reg|jit_cc_a1_reg| + jit_cc_a2_reg|jit_cc_a2_rlh: + goto r_r_q; + case jit_cc_a0_reg|jit_cc_a1_int| + jit_cc_a2_reg|jit_cc_a2_rlh: + goto r_w_q; case jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_flt: goto r_r_f; case jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_dbl: diff --git a/parallel-rsp/lightning/lib/jit_rewind.c b/parallel-rsp/lightning/lib/jit_rewind.c index 5ef1be5..8da8021 100644 --- a/parallel-rsp/lightning/lib/jit_rewind.c +++ b/parallel-rsp/lightning/lib/jit_rewind.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015-2019 Free Software Foundation, Inc. + * Copyright (C) 2015-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -54,13 +54,9 @@ _rewind_prolog(jit_state_t *_jit) _jitc->function->self.size = stack_framesize; #if __arm__ assert(jit_cpu.abi); - _jitc->function->self.size += 64; -#endif -#if __mips__ && NEW_ABI - /* Only add extra stack space if there are varargs - * arguments in registers. */ - assert(jit_arg_reg_p(_jitc->function->self.argi)); - _jitc->function->self.size += 64; + _jitc->function->alist = NULL; +#elif __mips__ + _jitc->function->alist = NULL; #endif _jitc->function->self.argi = _jitc->function->self.argf = _jitc->function->self.argn = 0; @@ -71,9 +67,10 @@ _rewind_prolog(jit_state_t *_jit) for (; node; node = next) { next = node->next; switch (node->code) { - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_l: node->next = (jit_node_t *)0; - jit_make_arg(node); + jit_make_arg(node, node->code); break; case jit_code_arg_f: node->next = (jit_node_t *)0; diff --git a/parallel-rsp/lightning/lib/jit_riscv-cpu.c b/parallel-rsp/lightning/lib/jit_riscv-cpu.c index 388489f..4fd35a8 100644 --- a/parallel-rsp/lightning/lib/jit_riscv-cpu.c +++ b/parallel-rsp/lightning/lib/jit_riscv-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Free Software Foundation, Inc. + * Copyright (C) 2019-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -93,10 +93,6 @@ typedef union { # undef ui } instr_t; # define ii(i) *_jit->pc.ui++ = i -/* FIXME could jit_rewind_prolog() to only use extra 64 bytes - * if a variadic jit function that have variadic arguments in - * registers */ -# define stack_framesize (200 + 64) # define ldr(r0, r1) ldr_l(r0, r1) # define ldi(r0, im) ldi_l(r0, im) # define ldxr(r0, r1, r2) ldxr_l(r0, r1, r2) @@ -434,12 +430,9 @@ static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1) static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); -# define htonr_us(r0, r1) _htonr_us(_jit, r0, r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0, r1) _htonr_ui(_jit, r0, r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ul(r0, r1) _htonr_ul(_jit, r0, r1) -static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_us(r0, r1) generic_bswapr_us(_jit, r0, r1) +# define bswapr_ui(r0, r1) generic_bswapr_ui(_jit, r0, r1) +# define bswapr_ul(r0, r1) generic_bswapr_ul(_jit, r0, r1) # define extr_c(r0, r1) _extr_c(_jit, r0, r1) static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_uc(r0, r1) andi(r0, r1, 0xff) @@ -455,6 +448,15 @@ static void _extr_ui(jit_state_t*,jit_int32_t,jit_int32_t); static void _movi(jit_state_t*,jit_int32_t,jit_word_t); # define movi_p(r0, im) _movi_p(_jit, r0, im) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define ltr(r0, r1, r2) SLT(r0, r1, r2) # define lti(r0, r1, im) _lti(_jit, r0, r1, im) static void _lti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); @@ -573,12 +575,12 @@ static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); # define jmpr(r0) JALR(_ZERO_REGNO, r0, 0) # define jmpi(im) _jmpi(_jit, im) -static void _jmpi(jit_state_t*,jit_word_t); +static jit_word_t _jmpi(jit_state_t*,jit_word_t); # define jmpi_p(im) _jmpi_p(_jit, im) static jit_word_t _jmpi_p(jit_state_t*,jit_word_t); # define callr(r0) JALR(_RA_REGNO, r0, 0) # define calli(im) _calli(_jit, im) -static void _calli(jit_state_t*,jit_word_t); +static jit_word_t _calli(jit_state_t*,jit_word_t); # define calli_p(im) _calli_p(_jit, im) static jit_word_t _calli_p(jit_state_t*,jit_word_t); # define prolog(i0) _prolog(_jit,i0) @@ -1243,59 +1245,6 @@ DEFST(s, H) DEFST(i, W) DEFST(l, D) -static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t t0; - t0 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 8); - andi(r0, r1, 0xff); - andi(rn(t0), rn(t0), 0xff); - lshi(r0, r0, 8); - orr(r0, r0, rn(t0)); - jit_unget_reg(t0); -} - -static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t t0; - jit_int32_t t1; - jit_int32_t t2; - t0 = jit_get_reg(jit_class_gpr); - t1 = jit_get_reg(jit_class_gpr); - t2 = jit_get_reg(jit_class_gpr); - rshi(rn(t0), r1, 24); - rshi(rn(t1), r1, 16); - rshi(rn(t2), r1, 8); - andi(rn(t0), rn(t0), 0xff); - andi(rn(t1), rn(t1), 0xff); - andi(rn(t2), rn(t2), 0xff); - andi(r0, r1, 0xff); - lshi(r0, r0, 24); - lshi(rn(t1), rn(t1), 8); - orr(r0, r0, rn(t0)); - lshi(rn(t2), rn(t2), 16); - orr(r0, r0, rn(t1)); - orr(r0, r0, rn(t2)); - jit_unget_reg(t2); - jit_unget_reg(t1); - jit_unget_reg(t0); -} - -static void -_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) -{ - jit_int32_t t0; - t0 = jit_get_reg(jit_class_gpr); - rshi_u(rn(t0), r1, 32); - htonr_ui(r0, r1); - htonr_ui(rn(t0), rn(t0)); - lshi(r0, r0, 32); - orr(r0, r0, rn(t0)); - jit_unget_reg(t0); -} - static void _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { @@ -1327,7 +1276,9 @@ _extr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { +# if __WORDSIZE == 64 if (simm32_p(i0)) { +# endif jit_int32_t lo = (jit_int32_t)i0 << 20 >> 20; jit_int32_t hi = i0 - lo; if (hi) { @@ -1337,42 +1288,83 @@ _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) } else ADDIW(r0, _ZERO_REGNO, lo); +# if __WORDSIZE == 64 } - else { - jit_int32_t lo = i0 << 32 >> 32; - jit_word_t hi = i0 - lo; - jit_int32_t t0 = jit_get_reg(jit_class_gpr); - movi(rn(t0), (jit_int32_t)(hi >> 32)); - movi(r0, lo); - lshi(rn(t0), rn(t0), 32); - addr(r0, r0, rn(t0)); - jit_unget_reg(t0); - } + else + load_const(r0, i0); +# endif } static jit_word_t _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_word_t w; - jit_int32_t t0; - jit_int32_t ww = i0 << 32 >> 32; - jit_int32_t lo = ww << 20 >> 20; - jit_int32_t hi = ww - lo; w = _jit->pc.w; - t0 = jit_get_reg(jit_class_gpr); - LUI(r0, hi >> 12); - ADDIW(r0, r0, lo); - ww = i0 >> 32; - lo = ww << 20 >> 20; - hi = ww - lo; - LUI(rn(t0), hi >> 12); - ADDIW(rn(t0), rn(t0), lo); - SLLI(rn(t0), rn(t0), 32); - ADD(r0, r0, rn(t0)); - jit_unget_reg(t0); +# if __WORDSIZE == 64 + AUIPC(r0, 0); + ADDI(r0, r0, 0); + LD(r0, r0, 0); +# else + LUI(r0, 0); + ADDIW(r0, r0, 0); +# endif return (w); } +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = beqi(_jit->pc.w, r2, 0); + movr(r0, r1); + patch_at(w, _jit->pc.w); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = bnei(_jit->pc.w, r2, 0); + movr(r0, r1); + patch_at(w, _jit->pc.w); +} + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t t0, r1_reg, iscasi; + jit_word_t retry, done, jump0, jump1; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + t0 = jit_get_reg(jit_class_gpr); + retry = _jit->pc.w; +# if __WORDSIZE == 32 + LR_W(r0, r1); +# else + LR_D(r0, r1); +# endif + jump0 = _jit->pc.w; + BNE(r0, r2, 0); +# if __WORDSIZE == 32 + SC_W(rn(t0), r1, r3); +# else + SC_D(rn(t0), r1, r3); +# endif + jump1 = _jit->pc.w; + BNE(rn(t0), _ZERO_REGNO, 0); + done = _jit->pc.w; + eqr(r0, r0, r2); + patch_at(jump0, done); + patch_at(jump1, retry); + jit_unget_reg(t0); + if (iscasi) + jit_unget_reg(r1_reg); +} + static void _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -2091,12 +2083,13 @@ _bmci(jit_state_t *_jit, jit_word_t br, jit_int32_t r0, jit_word_t i0) return (w); } -static void +static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { jit_int32_t t0; - jit_word_t dsp; - dsp = i0 - _jit->pc.w; + jit_word_t dsp, w; + w = _jit->pc.w; + dsp = i0 - w; if (simm20_p(dsp)) JAL(_ZERO_REGNO, dsp); else { @@ -2105,6 +2098,7 @@ _jmpi(jit_state_t *_jit, jit_word_t i0) jmpr(rn(t0)); jit_unget_reg(t0); } + return (w); } static jit_word_t @@ -2119,12 +2113,13 @@ _jmpi_p(jit_state_t *_jit, jit_word_t i0) return (w); } -static void +static jit_word_t _calli(jit_state_t *_jit, jit_word_t i0) { jit_int32_t t0; - jit_word_t dsp; - dsp = i0 - _jit->pc.w; + jit_word_t dsp, w; + w = _jit->pc.w; + dsp = i0 - w; if (simm20_p(dsp)) JAL(_RA_REGNO, dsp); else { @@ -2133,6 +2128,7 @@ _calli(jit_state_t *_jit, jit_word_t i0) callr(rn(t0)); jit_unget_reg(t0); } + return (w); } static jit_word_t @@ -2150,9 +2146,10 @@ _calli_p(jit_state_t *_jit, jit_word_t i0) static void _prolog(jit_state_t *_jit, jit_node_t *node) { - jit_int32_t reg; + jit_int32_t reg, offs; if (_jitc->function->define_frame || _jitc->function->assume_frame) { jit_int32_t frame = -_jitc->function->frame; + jit_check_frame(); assert(_jitc->function->self.aoff >= frame); if (_jitc->function->assume_frame) return; @@ -2163,56 +2160,41 @@ _prolog(jit_state_t *_jit, jit_node_t *node) _jitc->function->stack = ((_jitc->function->self.alen - /* align stack at 16 bytes */ _jitc->function->self.aoff) + 15) & -16; - subi(_SP_REGNO, _SP_REGNO, stack_framesize); - stxi(0, _SP_REGNO, _RA_REGNO); - stxi(8, _SP_REGNO, _FP_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _S1)) - stxi(16, _SP_REGNO, 9); - if (jit_regset_tstbit(&_jitc->function->regset, _S2)) - stxi(24, _SP_REGNO, 18); - if (jit_regset_tstbit(&_jitc->function->regset, _S3)) - stxi(32, _SP_REGNO, 19); - if (jit_regset_tstbit(&_jitc->function->regset, _S4)) - stxi(40, _SP_REGNO, 20); - if (jit_regset_tstbit(&_jitc->function->regset, _S5)) - stxi(48, _SP_REGNO, 21); - if (jit_regset_tstbit(&_jitc->function->regset, _S6)) - stxi(56, _SP_REGNO, 22); - if (jit_regset_tstbit(&_jitc->function->regset, _S7)) - stxi(64, _SP_REGNO, 23); - if (jit_regset_tstbit(&_jitc->function->regset, _S8)) - stxi(72, _SP_REGNO, 24); - if (jit_regset_tstbit(&_jitc->function->regset, _S9)) - stxi(80, _SP_REGNO, 25); - if (jit_regset_tstbit(&_jitc->function->regset, _S10)) - stxi(88, _SP_REGNO, 26); - if (jit_regset_tstbit(&_jitc->function->regset, _S11)) - stxi(96, _SP_REGNO, 27); - if (jit_regset_tstbit(&_jitc->function->regset, _FS0)) - stxi_d(104, _SP_REGNO, 8); - if (jit_regset_tstbit(&_jitc->function->regset, _FS1)) - stxi_d(112, _SP_REGNO, 9); - if (jit_regset_tstbit(&_jitc->function->regset, _FS2)) - stxi_d(120, _SP_REGNO, 18); - if (jit_regset_tstbit(&_jitc->function->regset, _FS3)) - stxi_d(128, _SP_REGNO, 19); - if (jit_regset_tstbit(&_jitc->function->regset, _FS4)) - stxi_d(136, _SP_REGNO, 20); - if (jit_regset_tstbit(&_jitc->function->regset, _FS5)) - stxi_d(144, _SP_REGNO, 21); - if (jit_regset_tstbit(&_jitc->function->regset, _FS6)) - stxi_d(152, _SP_REGNO, 22); - if (jit_regset_tstbit(&_jitc->function->regset, _FS7)) - stxi_d(160, _SP_REGNO, 23); - if (jit_regset_tstbit(&_jitc->function->regset, _FS8)) - stxi_d(168, _SP_REGNO, 24); - if (jit_regset_tstbit(&_jitc->function->regset, _FS9)) - stxi_d(176, _SP_REGNO, 25); - if (jit_regset_tstbit(&_jitc->function->regset, _FS10)) - stxi_d(184, _SP_REGNO, 26); - if (jit_regset_tstbit(&_jitc->function->regset, _FS11)) - stxi_d(192, _SP_REGNO, 27); - movr(_FP_REGNO, _SP_REGNO); + + if (_jitc->function->stack) + _jitc->function->need_stack = 1; + if (!_jitc->function->need_frame && !_jitc->function->need_stack) { + /* check if any callee save register needs to be saved */ + for (reg = 0; reg < _jitc->reglen; ++reg) + if (jit_regset_tstbit(&_jitc->function->regset, reg) && + (_rvs[reg].spec & jit_class_sav)) { + _jitc->function->need_stack = 1; + break; + } + } + + if (_jitc->function->need_frame || _jitc->function->need_stack) + subi(_SP_REGNO, _SP_REGNO, jit_framesize()); + if (_jitc->function->need_frame) { + stxi(0, _SP_REGNO, _RA_REGNO); + stxi(8, _SP_REGNO, _FP_REGNO); + } + /* callee save registers */ + for (reg = 0, offs = 16; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + stxi(offs, _SP_REGNO, rn(iregs[reg])); + offs += sizeof(jit_word_t); + } + } + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + stxi_d(offs, _SP_REGNO, rn(fregs[reg])); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->need_frame) + movr(_FP_REGNO, _SP_REGNO); if (_jitc->function->stack) subi(_SP_REGNO, _SP_REGNO, _jitc->function->stack); if (_jitc->function->allocar) { @@ -2223,7 +2205,7 @@ _prolog(jit_state_t *_jit, jit_node_t *node) } if (_jitc->function->self.call & jit_call_varargs) { for (reg = _jitc->function->vagp; jit_arg_reg_p(reg); ++reg) - stxi(stack_framesize - ((8 - reg) * 8), + stxi(jit_framesize() - ((8 - reg) * 8), _FP_REGNO, rn(JIT_RA0 - reg)); } } @@ -2231,58 +2213,31 @@ _prolog(jit_state_t *_jit, jit_node_t *node) static void _epilog(jit_state_t *_jit, jit_node_t *node) { + jit_int32_t reg, offs; if (_jitc->function->assume_frame) return; - movr(_SP_REGNO, _FP_REGNO); - ldxi(_RA_REGNO, _SP_REGNO, 0); - ldxi(_FP_REGNO, _SP_REGNO, 8); - if (jit_regset_tstbit(&_jitc->function->regset, _S1)) - ldxi(9, _SP_REGNO, 16); - if (jit_regset_tstbit(&_jitc->function->regset, _S2)) - ldxi(18, _SP_REGNO, 24); - if (jit_regset_tstbit(&_jitc->function->regset, _S3)) - ldxi(19, _SP_REGNO, 32); - if (jit_regset_tstbit(&_jitc->function->regset, _S4)) - ldxi(20, _SP_REGNO, 40); - if (jit_regset_tstbit(&_jitc->function->regset, _S5)) - ldxi(21, _SP_REGNO, 48); - if (jit_regset_tstbit(&_jitc->function->regset, _S6)) - ldxi(22, _SP_REGNO, 56); - if (jit_regset_tstbit(&_jitc->function->regset, _S7)) - ldxi(23, _SP_REGNO, 64); - if (jit_regset_tstbit(&_jitc->function->regset, _S8)) - ldxi(24, _SP_REGNO, 72); - if (jit_regset_tstbit(&_jitc->function->regset, _S9)) - ldxi(25, _SP_REGNO, 80); - if (jit_regset_tstbit(&_jitc->function->regset, _S10)) - ldxi(26, _SP_REGNO, 88); - if (jit_regset_tstbit(&_jitc->function->regset, _S11)) - ldxi(27, _SP_REGNO, 96); - if (jit_regset_tstbit(&_jitc->function->regset, _FS0)) - ldxi_d(8, _SP_REGNO, 104); - if (jit_regset_tstbit(&_jitc->function->regset, _FS1)) - ldxi_d(9, _SP_REGNO, 112); - if (jit_regset_tstbit(&_jitc->function->regset, _FS2)) - ldxi_d(18, _SP_REGNO, 120); - if (jit_regset_tstbit(&_jitc->function->regset, _FS3)) - ldxi_d(19, _SP_REGNO, 128); - if (jit_regset_tstbit(&_jitc->function->regset, _FS4)) - ldxi_d(20, _SP_REGNO, 136); - if (jit_regset_tstbit(&_jitc->function->regset, _FS5)) - ldxi_d(21, _SP_REGNO, 144); - if (jit_regset_tstbit(&_jitc->function->regset, _FS6)) - ldxi_d(22, _SP_REGNO, 152); - if (jit_regset_tstbit(&_jitc->function->regset, _FS7)) - ldxi_d(23, _SP_REGNO, 160); - if (jit_regset_tstbit(&_jitc->function->regset, _FS8)) - ldxi_d(24, _SP_REGNO, 168); - if (jit_regset_tstbit(&_jitc->function->regset, _FS9)) - ldxi_d(25, _SP_REGNO, 176); - if (jit_regset_tstbit(&_jitc->function->regset, _FS10)) - ldxi_d(26, _SP_REGNO, 184); - if (jit_regset_tstbit(&_jitc->function->regset, _FS11)) - ldxi_d(27, _SP_REGNO, 192); - addi(_SP_REGNO, _SP_REGNO, stack_framesize); + if (_jitc->function->need_frame) { + movr(_SP_REGNO, _FP_REGNO); + ldxi(_RA_REGNO, _SP_REGNO, 0); + ldxi(_FP_REGNO, _SP_REGNO, 8); + } + + /* callee save registers */ + for (reg = 0, offs = 16; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + ldxi(rn(iregs[reg]), _SP_REGNO, offs); + offs += sizeof(jit_word_t); + } + } + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + ldxi_d(rn(fregs[reg]), _SP_REGNO, offs); + offs += sizeof(jit_float64_t); + } + } + + if (_jitc->function->need_frame || _jitc->function->need_stack) + addi(_SP_REGNO, _SP_REGNO, jit_framesize()); RET(); } @@ -2292,9 +2247,9 @@ _vastart(jit_state_t *_jit, jit_int32_t r0) assert(_jitc->function->self.call & jit_call_varargs); /* Initialize va_list to the first stack argument. */ if (jit_arg_reg_p(_jitc->function->vagp)) - addi(r0, _FP_REGNO, stack_framesize - ((8 - _jitc->function->vagp) * 8)); + addi(r0, _FP_REGNO, jit_framesize() - ((8 - _jitc->function->vagp) * 8)); else - addi(r0, _FP_REGNO, _jitc->function->self.size); + addi(r0, _FP_REGNO, jit_selfsize()); } static void @@ -2318,42 +2273,43 @@ _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) u.w = instr; i.w = u.i[0]; /* movi_p? */ +# if __WORDSIZE == 64 + if (i.U.opcode == 23) { /* AUIPC */ + jit_int32_t lo, hi; + jit_word_t address, relative; + address = get_const(label); + relative = address - instr; + assert(simm32_p(relative)); + lo = (jit_int32_t)relative << 20 >> 20; + hi = relative - lo; + i.U.imm12_31 = hi >> 12; + u.i[0] = i.w; + i.w = u.i[1]; + if (i.I.opcode == 19 && i.I.funct3 == 0) { /* ADDI */ + i.I.imm11_0 = lo; + u.i[1] = i.w; + i.w = u.i[2]; + } + else + abort(); + assert(i.I.opcode == 3 && i.I.funct3 == 3); /* LD */ + } +# else if (i.U.opcode == 55) { /* LUI */ - jit_int32_t ww = label << 32 >> 32; - jit_int32_t lo = ww << 20 >> 20; - jit_int32_t hi = ww - lo; + jit_int32_t lo = (jit_int32_t)label << 20 >> 20; + jit_int32_t hi = label - lo; i.U.imm12_31 = hi >> 12; u.i[0] = i.w; i.w = u.i[1]; if (i.I.opcode == 27 && i.I.funct3 == 0) { /* ADDIW */ - i.I.imm11_0 = lo & 0xfff; + i.I.imm11_0 = lo; u.i[1] = i.w; i.w = u.i[2]; - if (i.U.opcode == 55) { /* LUI */ - ww = label >> 32; - lo = ww << 20 >> 20; - hi = ww - lo; - i.U.imm12_31 = hi >> 12; - u.i[2] = i.w; - i.w = u.i[3]; - if (i.I.opcode == 27 && i.I.funct3 == 0) { /* ADDIW */ - i.I.imm11_0 = lo & 0xfff; - u.i[3] = i.w; - i.w = u.i[4]; - assert(i.IS.opcode == 19); /* SLLI */ - assert(i.IS.shamt == 32); - i.w = u.i[5]; - assert(i.R.opcode == 51); /* ADD */ - } - else - abort(); - } - else - abort(); } else abort(); } +# endif /* b{lt,le,eq,ge,gt,ne}{,_u}? */ else if (i.B.opcode == 99) { /* B{EQ,NE,LT,GE,LTU,GEU} */ jit_word_t jmp = label - instr; diff --git a/parallel-rsp/lightning/lib/jit_riscv-fpu.c b/parallel-rsp/lightning/lib/jit_riscv-fpu.c index 367975e..89346e0 100644 --- a/parallel-rsp/lightning/lib/jit_riscv-fpu.c +++ b/parallel-rsp/lightning/lib/jit_riscv-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Free Software Foundation, Inc. + * Copyright (C) 2019-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/lib/jit_riscv-sz.c b/parallel-rsp/lightning/lib/jit_riscv-sz.c index 2f1d725..335d3cf 100644 --- a/parallel-rsp/lightning/lib/jit_riscv-sz.c +++ b/parallel-rsp/lightning/lib/jit_riscv-sz.c @@ -1,10 +1,11 @@ #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 116 +#define JIT_INSTR_MAX 168 0, /* data */ 0, /* live */ 4, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -13,7 +14,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -21,49 +25,61 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 8, /* va_arg_d */ 0, /* va_end */ 4, /* addr */ - 20, /* addi */ + 16, /* addi */ 12, /* addcr */ - 28, /* addci */ + 24, /* addci */ 28, /* addxr */ 28, /* addxi */ 4, /* subr */ - 20, /* subi */ + 16, /* subi */ 12, /* subcr */ - 28, /* subci */ + 24, /* subci */ 28, /* subxr */ 28, /* subxi */ - 28, /* rsbi */ + 20, /* rsbi */ 4, /* mulr */ - 20, /* muli */ + 16, /* muli */ 12, /* qmulr */ - 24, /* qmuli */ + 20, /* qmuli */ 12, /* qmulr_u */ - 24, /* qmuli_u */ + 20, /* qmuli_u */ 4, /* divr */ - 20, /* divi */ + 16, /* divi */ 4, /* divr_u */ - 20, /* divi_u */ + 16, /* divi_u */ 20, /* qdivr */ 16, /* qdivi */ 20, /* qdivr_u */ 16, /* qdivi_u */ 4, /* remr */ - 20, /* remi */ + 16, /* remi */ 4, /* remr_u */ - 20, /* remi_u */ + 16, /* remi_u */ 4, /* andr */ - 20, /* andi */ + 16, /* andi */ 4, /* orr */ - 20, /* ori */ + 16, /* ori */ 4, /* xorr */ - 20, /* xori */ + 16, /* xori */ 4, /* lshr */ 4, /* lshi */ 4, /* rshr */ @@ -93,30 +109,37 @@ 8, /* ner */ 8, /* nei */ 4, /* movr */ - 24, /* movi */ + 12, /* movi */ + 12, /* movnr */ + 12, /* movzr */ + 28, /* casr */ + 40, /* casi */ 8, /* extr_c */ 4, /* extr_uc */ 8, /* extr_s */ 8, /* extr_us */ 4, /* extr_i */ 8, /* extr_ui */ + 20, /* bswapr_us */ + 52, /* bswapr_ui */ + 116, /* bswapr_ul */ 20, /* htonr_us */ 52, /* htonr_ui */ 116, /* htonr_ul */ 4, /* ldr_c */ - 12, /* ldi_c */ + 16, /* ldi_c */ 4, /* ldr_uc */ - 12, /* ldi_uc */ + 16, /* ldi_uc */ 4, /* ldr_s */ - 12, /* ldi_s */ + 16, /* ldi_s */ 4, /* ldr_us */ - 12, /* ldi_us */ + 16, /* ldi_us */ 4, /* ldr_i */ - 12, /* ldi_i */ + 16, /* ldi_i */ 4, /* ldr_ui */ - 12, /* ldi_ui */ + 16, /* ldi_ui */ 4, /* ldr_l */ - 12, /* ldi_l */ + 16, /* ldi_l */ 8, /* ldxr_c */ 16, /* ldxi_c */ 8, /* ldxr_uc */ @@ -132,13 +155,13 @@ 8, /* ldxr_l */ 16, /* ldxi_l */ 4, /* str_c */ - 12, /* sti_c */ + 16, /* sti_c */ 4, /* str_s */ - 12, /* sti_s */ + 16, /* sti_s */ 4, /* str_i */ - 12, /* sti_i */ + 16, /* sti_i */ 4, /* str_l */ - 12, /* sti_l */ + 16, /* sti_l */ 8, /* stxr_c */ 16, /* stxi_c */ 8, /* stxr_s */ @@ -156,7 +179,7 @@ 4, /* bler_u */ 8, /* blei_u */ 4, /* beqr */ - 28, /* beqi */ + 16, /* beqi */ 4, /* bger */ 8, /* bgei */ 4, /* bger_u */ @@ -166,7 +189,7 @@ 4, /* bgtr_u */ 8, /* bgti_u */ 4, /* bner */ - 20, /* bnei */ + 16, /* bnei */ 8, /* bmsr */ 12, /* bmsi */ 8, /* bmcr */ @@ -188,17 +211,41 @@ 16, /* bxsubr_u */ 20, /* bxsubi_u */ 4, /* jmpr */ - 28, /* jmpi */ + 16, /* jmpi */ 4, /* callr */ - 28, /* calli */ + 16, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -258,11 +305,11 @@ 4, /* movr_f */ 8, /* movi_f */ 4, /* ldr_f */ - 12, /* ldi_f */ + 16, /* ldi_f */ 8, /* ldxr_f */ 16, /* ldxi_f */ 4, /* str_f */ - 12, /* sti_f */ + 16, /* sti_f */ 8, /* stxr_f */ 16, /* stxi_f */ 8, /* bltr_f */ @@ -303,87 +350,87 @@ 0, /* putargr_d */ 0, /* putargi_d */ 4, /* addr_d */ - 24, /* addi_d */ + 20, /* addi_d */ 4, /* subr_d */ - 24, /* subi_d */ - 24, /* rsbi_d */ + 20, /* subi_d */ + 20, /* rsbi_d */ 4, /* mulr_d */ - 24, /* muli_d */ + 20, /* muli_d */ 4, /* divr_d */ - 24, /* divi_d */ + 20, /* divi_d */ 4, /* negr_d */ 4, /* absr_d */ 4, /* sqrtr_d */ 4, /* ltr_d */ - 24, /* lti_d */ + 20, /* lti_d */ 4, /* ler_d */ - 24, /* lei_d */ + 20, /* lei_d */ 4, /* eqr_d */ - 24, /* eqi_d */ + 20, /* eqi_d */ 4, /* ger_d */ - 24, /* gei_d */ + 20, /* gei_d */ 4, /* gtr_d */ - 24, /* gti_d */ + 20, /* gti_d */ 8, /* ner_d */ - 28, /* nei_d */ + 24, /* nei_d */ 28, /* unltr_d */ - 48, /* unlti_d */ + 44, /* unlti_d */ 28, /* unler_d */ - 48, /* unlei_d */ + 44, /* unlei_d */ 28, /* uneqr_d */ - 48, /* uneqi_d */ + 44, /* uneqi_d */ 28, /* unger_d */ - 48, /* ungei_d */ + 44, /* ungei_d */ 28, /* ungtr_d */ - 48, /* ungti_d */ + 44, /* ungti_d */ 40, /* ltgtr_d */ - 60, /* ltgti_d */ + 56, /* ltgti_d */ 28, /* ordr_d */ - 48, /* ordi_d */ + 44, /* ordi_d */ 20, /* unordr_d */ - 40, /* unordi_d */ + 36, /* unordi_d */ 4, /* truncr_d_i */ 4, /* truncr_d_l */ 4, /* extr_d */ 4, /* extr_f_d */ 4, /* movr_d */ - 20, /* movi_d */ + 16, /* movi_d */ 4, /* ldr_d */ - 12, /* ldi_d */ + 16, /* ldi_d */ 8, /* ldxr_d */ 16, /* ldxi_d */ 4, /* str_d */ - 12, /* sti_d */ + 16, /* sti_d */ 8, /* stxr_d */ 16, /* stxi_d */ 8, /* bltr_d */ - 28, /* blti_d */ + 24, /* blti_d */ 8, /* bler_d */ - 28, /* blei_d */ + 24, /* blei_d */ 8, /* beqr_d */ - 28, /* beqi_d */ + 24, /* beqi_d */ 8, /* bger_d */ - 28, /* bgei_d */ + 24, /* bgei_d */ 8, /* bgtr_d */ - 28, /* bgti_d */ + 24, /* bgti_d */ 8, /* bner_d */ - 28, /* bnei_d */ + 24, /* bnei_d */ 32, /* bunltr_d */ - 52, /* bunlti_d */ + 48, /* bunlti_d */ 32, /* bunler_d */ - 52, /* bunlei_d */ + 48, /* bunlei_d */ 32, /* buneqr_d */ - 52, /* buneqi_d */ + 48, /* buneqi_d */ 32, /* bunger_d */ - 52, /* bungei_d */ + 48, /* bungei_d */ 32, /* bungtr_d */ - 52, /* bungti_d */ + 48, /* bungti_d */ 44, /* bltgtr_d */ - 64, /* bltgti_d */ + 60, /* bltgti_d */ 32, /* bordr_d */ - 52, /* bordi_d */ + 48, /* bordi_d */ 24, /* bunordr_d */ - 44, /* bunordi_d */ + 40, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -392,10 +439,14 @@ 4, /* movr_w_f */ 0, /* movr_ww_d */ 4, /* movr_w_d */ - 0, /* movr_f_w */ + 4, /* movr_f_w */ 4, /* movi_f_w */ 0, /* movr_d_ww */ 0, /* movi_d_ww */ 4, /* movr_d_w */ - 16, /* movi_d_w */ + 12, /* movi_d_w */ + 168, /* clo */ + 148, /* clz */ + 168, /* cto */ + 148, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_riscv.c b/parallel-rsp/lightning/lib/jit_riscv.c index 55b2391..63a5cd9 100644 --- a/parallel-rsp/lightning/lib/jit_riscv.c +++ b/parallel-rsp/lightning/lib/jit_riscv.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Free Software Foundation, Inc. + * Copyright (C) 2019-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -17,6 +17,10 @@ * Paulo Cesar Pereira de Andrade */ +/* callee save + variadic arguments + * align16(ra+fp+s[1-9]+s10+s11+fs[0-9]+fs10+fs11)+align16(a[0-7]) */ +#define stack_framesize (208 + 64) + #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8) #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) @@ -28,12 +32,24 @@ typedef jit_pointer_t jit_va_list_t; /* * Prototypes */ +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); +#if __WORDSIZE == 64 +# define load_const(r0, i0) _load_const(_jit, r0, i0) +static void _load_const(jit_state_t*, jit_int32_t, jit_word_t); +static jit_word_t hash_const(jit_word_t); +# define put_const(i0) _put_const(_jit, i0) +static void _put_const(jit_state_t*, jit_word_t); +# define get_const(i0) _get_const(_jit, i0) +static jit_word_t _get_const(jit_state_t*, jit_word_t); +#endif #define patch(instr, node) _patch(_jit, instr, node) static void _patch(jit_state_t*,jit_word_t,jit_node_t*); #define PROTO 1 # include "jit_riscv-cpu.c" # include "jit_riscv-fpu.c" +# include "jit_fallback.c" #undef PROTO /* @@ -110,6 +126,14 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { + _S1, _S2, _S3, _S4, _S5, _S6, _S7, _S8, _S9, _S10, _S11 +}; + +static jit_int32_t fregs[] = { + _FS0, _FS1, _FS2, _FS3, _FS4, _FS5, _FS6, _FS7, _FS8, _FS9, _FS10, _FS11 +}; + /* * Implementation */ @@ -171,6 +195,7 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + jit_check_frame(); switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -219,20 +244,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -292,16 +315,17 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); - return (jit_arg_f_reg_p(u->u.w)); + return (jit_arg_f_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8)); } void _jit_ellipsis(jit_state_t *_jit) { jit_inc_synth(ellipsis); + jit_check_frame(); if (_jitc->prepare) { jit_link_prepare(); assert(!(_jitc->function->call.call & jit_call_varargs)); @@ -325,19 +349,23 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function); assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { offset = _jitc->function->self.size; _jitc->function->self.size += sizeof(jit_word_t); + jit_check_frame(); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -359,6 +387,7 @@ _jit_arg_f(jit_state_t *_jit) else { offset = _jitc->function->self.size; _jitc->function->self.size += sizeof(jit_word_t); + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_f, offset, ++_jitc->function->self.argn); @@ -382,6 +411,7 @@ _jit_arg_d(jit_state_t *_jit) else { offset = _jitc->function->self.size; _jitc->function->self.size += sizeof(jit_word_t); + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_d, offset, ++_jitc->function->self.argn); @@ -392,111 +422,129 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, JIT_RA0 - v->u.w); - else - jit_ldxi_c(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_c(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, JIT_RA0 - v->u.w); - else - jit_ldxi_uc(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_uc(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, JIT_RA0 - v->u.w); - else - jit_ldxi_s(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_s(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, JIT_RA0 - v->u.w); - else - jit_ldxi_us(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_us(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_i(u, JIT_RA0 - v->u.w); - else - jit_ldxi_i(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_i(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, JIT_RA0 - v->u.w); - else - jit_ldxi_ui(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_ui(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); - else - jit_ldxi_l(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_l(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(JIT_RA0 - v->u.w, u); - else - jit_stxi(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(JIT_RA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w, JIT_FP, regno); + node = jit_stxi(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -511,8 +559,10 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr_f(u, JIT_FA0 - v->u.w); else if (jit_arg_reg_p(v->u.w - 8)) jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8)); - else - jit_ldxi_f(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_f(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -525,8 +575,10 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr_f(JIT_FA0 - v->u.w, u); else if (jit_arg_reg_p(v->u.w - 8)) jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u); - else - jit_stxi_f(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi_f(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -538,18 +590,14 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) jit_inc_synth_fp(putargi_f, u, v); if (jit_arg_f_reg_p(v->u.w)) jit_movi_f(JIT_FA0 - v->u.w, u); - else if (jit_arg_reg_p(v->u.w - 8)) { - union { - jit_float32_t f; - jit_int32_t i; - } uu; - uu.f = u; - jit_movi(JIT_RA0 - (v->u.w - 8), uu.i); - } + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movi_f_w(JIT_RA0 - (v->u.w - 8), u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -564,8 +612,10 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr_d(u, JIT_FA0 - v->u.w); else if (jit_arg_reg_p(v->u.w - 8)) jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8)); - else - jit_ldxi_d(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_d(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -578,8 +628,10 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr_d(JIT_FA0 - v->u.w, u); else if (jit_arg_reg_p(v->u.w - 8)) jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u); - else - jit_stxi_d(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi_d(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -591,28 +643,24 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) jit_inc_synth_dp(putargi_d, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi_d(JIT_FA0 - v->u.w, u); - else if (jit_arg_reg_p(v->u.w - 8)) { - union { - jit_float64_t d; - jit_int64_t w; - } uu; - uu.d = u; - jit_movi(JIT_RA0 - (v->u.w - 8), uu.w); - } + else if (jit_arg_reg_p(v->u.w - 8)) + jit_movi_d_w(JIT_RA0 - (v->u.w - 8), u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(JIT_RA0 - _jitc->function->call.argi, u); @@ -621,16 +669,17 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) else { jit_stxi(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); } jit_dec_synth(); } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(JIT_RA0 - _jitc->function->call.argi, u); @@ -642,6 +691,7 @@ _jit_pushargi(jit_state_t *_jit, jit_word_t u) jit_stxi(_jitc->function->call.size, JIT_SP, regno); jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); } jit_dec_synth(); } @@ -664,6 +714,7 @@ _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) else { jit_stxi_f(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); } jit_dec_synth(); } @@ -690,6 +741,7 @@ _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); } jit_dec_synth(); } @@ -712,6 +764,7 @@ _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) else { jit_stxi_d(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); } jit_dec_synth(); } @@ -738,6 +791,7 @@ _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) jit_stxi_d(_jitc->function->call.size, JIT_SP, regno); jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); + jit_check_frame(); } jit_dec_synth(); } @@ -766,6 +820,7 @@ _jit_finishr(jit_state_t *_jit, jit_int32_t r0) { jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishr, r0); if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; @@ -783,6 +838,7 @@ _jit_finishi(jit_state_t *_jit, jit_pointer_t i0) { jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishi, (jit_word_t)i0); if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; @@ -883,6 +939,7 @@ _emit_code(jit_state_t *_jit) jit_node_t *node; jit_uint8_t *data; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -893,6 +950,43 @@ _emit_code(jit_state_t *_jit) jit_word_t prevw; #endif +#if __WORDSIZE == 64 + if (!_jitc->consts.hash.table) { + jit_alloc((jit_pointer_t *)&_jitc->consts.hash.table, + 16 * sizeof(jit_const_t *)); + _jitc->consts.hash.size = 16; + jit_alloc((jit_pointer_t *)&_jitc->consts.pool.ptr, + sizeof(jit_const_t *)); + jit_alloc((jit_pointer_t *)_jitc->consts.pool.ptr, + 1024 * sizeof(jit_const_t)); + _jitc->consts.pool.length = 1; + } + /* Reset table if starting over jit generation */ + else + memset(_jitc->consts.hash.table, 0, + _jitc->consts.hash.size * sizeof(jit_word_t)); + for (offset = 0; offset < _jitc->consts.pool.length; offset++) { + jit_int32_t i; + jit_const_t *list = _jitc->consts.pool.ptr[offset]; + for (i = 0; i < 1023; ++i, ++list) + list->next = list + 1; + if (offset + 1 < _jitc->consts.pool.length) + list->next = _jitc->consts.pool.ptr[offset + 1]; + else + list->next = NULL; + } + _jitc->consts.pool.list = _jitc->consts.pool.ptr[0]; + _jitc->consts.hash.count = 0; + if (!_jitc->consts.vector.instrs) { + jit_alloc((jit_pointer_t *)&_jitc->consts.vector.instrs, + 16 * sizeof(jit_word_t)); + jit_alloc((jit_pointer_t *)&_jitc->consts.vector.values, + 16 * sizeof(jit_word_t)); + _jitc->consts.vector.length = 16; + } + _jitc->consts.vector.offset = 0; +#endif + _jitc->function = NULL; jit_reglive_setup(); @@ -1015,11 +1109,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1068,6 +1164,14 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh, _u); case_rr(neg,); case_rr(com,); +#define clor(r0, r1) fallback_clo(r0, r1) +#define clzr(r0, r1) fallback_clz(r0, r1) +#define ctor(r0, r1) fallback_cto(r0, r1) +#define ctzr(r0, r1) fallback_ctz(r0, r1) + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(and,); case_rrw(and,); case_rrr(or,); @@ -1125,12 +1229,25 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _us); case_rr(hton, _ui); case_rr(hton, _ul); + case_rr(bswap, _us); + case_rr(bswap, _ui); + case_rr(bswap, _ul); case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); case_rr(ext, _us); case_rr(ext, _i); case_rr(ext, _ui); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1376,6 +1493,7 @@ _emit_code(jit_state_t *_jit) case_brr(bunord, _d); case_brd(bunord); case jit_code_jmpr: + jit_check_frame(); jmpr(rn(node->u.w)); break; case jit_code_jmpi: @@ -1386,14 +1504,22 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (simm20_p(word)) + word = jmpi(_jit->pc.w); + else word = jmpi_p(_jit->pc.w); patch(word, node); } } - else + else { + jit_check_frame(); jmpi(node->u.w); + } break; case jit_code_callr: + jit_check_frame(); callr(rn(node->u.w)); break; case jit_code_calli: @@ -1404,22 +1530,32 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) calli(temp->u.w); else { - word = calli_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (simm20_p(word)) + word = calli(_jit->pc.w); + else + word = calli_p(_jit->pc.w); patch(word, node); } } - else + else { + jit_check_frame(); calli(node->u.w); + } break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif undo.patch_offset = _jitc->patches.offset; restart_function: + compute_framesize(); + patch_alist(0); _jitc->again = 0; prolog(node); break; @@ -1435,10 +1571,25 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + /* this will be recomputed but undo anyway to have it + * better self documented.*/ + undo.func.need_stack = _jitc->function->need_stack; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif _jitc->patches.offset = undo.patch_offset; + patch_alist(1); goto restart_function; } /* remember label is defined */ @@ -1479,11 +1630,19 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: + case jit_code_arg_l: case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1491,10 +1650,22 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_i: case jit_code_getarg_ui: case jit_code_getarg_l: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1538,6 +1709,35 @@ _emit_code(jit_state_t *_jit) #undef case_rw #undef case_rr +#if __WORDSIZE == 64 + /* Record all constants to be patched */ + for (offset = 0; offset < _jitc->patches.offset; offset++) { + node = _jitc->patches.ptr[offset].node; + value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w; + put_const(value); + } + /* Record all direct constants */ + for (offset = 0; offset < _jitc->consts.vector.offset; offset++) + put_const(_jitc->consts.vector.values[offset]); + /* Now actually inject constants at the end of code buffer */ + if (_jitc->consts.hash.count) { + jit_const_t *entry; + /* Insert nop if aligned at 4 bytes */ + if (_jit->pc.w % sizeof(jit_word_t)) + nop(_jit->pc.w % sizeof(jit_word_t)); + for (offset = 0; offset < _jitc->consts.hash.size; offset++) { + entry = _jitc->consts.hash.table[offset]; + for (; entry; entry = entry->next) { + /* Make sure to not write out of bounds */ + if (_jit->pc.uc >= _jitc->code.end) + return (NULL); + entry->address = _jit->pc.w; + *_jit->pc.ul++ = entry->value; + } + } + } +#endif + for (offset = 0; offset < _jitc->patches.offset; offset++) { node = _jitc->patches.ptr[offset].node; word = _jitc->patches.ptr[offset].inst; @@ -1545,6 +1745,25 @@ _emit_code(jit_state_t *_jit) patch_at(word, value); } +#if __WORDSIZE == 64 + /* Patch direct complex constants */ + if (_jitc->consts.vector.instrs) { + for (offset = 0; offset < _jitc->consts.vector.offset; offset++) + patch_at(_jitc->consts.vector.instrs[offset], + _jitc->consts.vector.values[offset]); + jit_free((jit_pointer_t *)&_jitc->consts.vector.instrs); + jit_free((jit_pointer_t *)&_jitc->consts.vector.values); + } + + /* Hash table no longer need */ + if (_jitc->consts.hash.table) { + jit_free((jit_pointer_t *)&_jitc->consts.hash.table); + for (offset = 0; offset < _jitc->consts.pool.length; offset++) + jit_free((jit_pointer_t *)_jitc->consts.pool.ptr + offset); + jit_free((jit_pointer_t *)&_jitc->consts.pool.ptr); + } +#endif + jit_flush(_jit->code.ptr, _jit->pc.uc); return (_jit->code.ptr); @@ -1553,8 +1772,117 @@ _emit_code(jit_state_t *_jit) #define CODE 1 # include "jit_riscv-cpu.c" # include "jit_riscv-fpu.c" +# include "jit_fallback.c" #undef CODE +static void +_load_const(jit_state_t *_jit, jit_int32_t reg, jit_word_t value) +{ + if (_jitc->consts.vector.offset >= _jitc->consts.vector.length) { + jit_word_t new_size = _jitc->consts.vector.length * + 2 * sizeof(jit_word_t); + jit_realloc((jit_pointer_t *)&_jitc->consts.vector.instrs, + _jitc->consts.vector.length * sizeof(jit_word_t), new_size); + jit_realloc((jit_pointer_t *)&_jitc->consts.vector.values, + _jitc->consts.vector.length * sizeof(jit_word_t), new_size); + _jitc->consts.vector.length *= 2; + } + _jitc->consts.vector.instrs[_jitc->consts.vector.offset] = _jit->pc.w; + _jitc->consts.vector.values[_jitc->consts.vector.offset] = value; + ++_jitc->consts.vector.offset; + /* Resolve later the pc relative address */ + put_const(value); + AUIPC(reg, 0); + ADDI(reg, reg, 0); + LD(reg, reg, 0); +} + +static jit_word_t +hash_const(jit_word_t value) +{ + const jit_uint8_t *ptr; + jit_word_t i, key; + for (i = key = 0, ptr = (jit_uint8_t *)&value; i < 4; ++i) + key = (key << (key & 1)) ^ ptr[i]; + return (key); + +} + +static void +_put_const(jit_state_t *_jit, jit_word_t value) +{ + jit_word_t key; + jit_const_t *entry; + + /* Check if already inserted in table */ + key = hash_const(value) % _jitc->consts.hash.size; + for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) { + if (entry->value == value) + return; + } + + /* Check if need to increase pool size */ + if (_jitc->consts.pool.list->next == NULL) { + jit_const_t *list; + jit_word_t offset; + jit_word_t new_size = (_jitc->consts.pool.length + 1) * + sizeof(jit_const_t*); + jit_realloc((jit_pointer_t *)&_jitc->consts.pool.ptr, + _jitc->consts.pool.length * sizeof(jit_const_t*), new_size); + jit_alloc((jit_pointer_t *) + _jitc->consts.pool.ptr + _jitc->consts.pool.length, + 1024 * sizeof(jit_const_t)); + list = _jitc->consts.pool.ptr[_jitc->consts.pool.length]; + _jitc->consts.pool.list->next = list; + for (offset = 0; offset < 1023; ++offset, ++list) + list->next = list + 1; + list->next = NULL; + ++_jitc->consts.pool.length; + } + + /* Rehash if more than 75% used table */ + if (_jitc->consts.hash.count > (_jitc->consts.hash.size / 4) * 3) { + jit_word_t i, k; + jit_const_t *next; + jit_const_t **table; + jit_alloc((jit_pointer_t *)&table, + _jitc->consts.hash.size * 2 * sizeof(jit_const_t *)); + for (i = 0; i < _jitc->consts.hash.size; ++i) { + for (entry = _jitc->consts.hash.table[i]; entry; entry = next) { + next = entry->next; + k = hash_const(entry->value) % (_jitc->consts.hash.size * 2); + entry->next = table[k]; + table[k] = entry; + } + } + jit_free((jit_pointer_t *)&_jitc->consts.hash.table); + _jitc->consts.hash.size *= 2; + _jitc->consts.hash.table = table; + } + + /* Insert in hash */ + entry = _jitc->consts.pool.list; + _jitc->consts.pool.list = entry->next; + ++_jitc->consts.hash.count; + entry->value = value; + entry->next = _jitc->consts.hash.table[key]; + _jitc->consts.hash.table[key] = entry; +} + +static jit_word_t +_get_const(jit_state_t *_jit, jit_word_t value) +{ + jit_word_t key; + jit_const_t *entry; + key = hash_const(value) % _jitc->consts.hash.size; + for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) { + if (entry->value == value) + return (entry->address); + } + /* Only the final patch should call get_const() */ + abort(); +} + void jit_flush(void *fptr, void *tptr) { @@ -1592,6 +1920,30 @@ _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) stxi_d(i0, rn(r0), rn(r1)); } +#if __WORDSIZE != 64 +# error "only 64 bit ports tested" +#endif +static void +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = 16; /* ra+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += sizeof(jit_word_t); + + for (reg = 0; reg < jit_size(fregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) + _jitc->framesize += sizeof(jit_float64_t); + + /* Space to store variadic arguments */ + if (_jitc->function->self.call & jit_call_varargs) + _jitc->framesize += (8 - _jitc->function->vagp) * 8; + + /* Make sure functions called have a 16 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 15) & -16; +} + static void _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) { diff --git a/parallel-rsp/lightning/lib/jit_s390-cpu.c b/parallel-rsp/lightning/lib/jit_s390-cpu.c index 02cac60..2e9e074 100644 --- a/parallel-rsp/lightning/lib/jit_s390-cpu.c +++ b/parallel-rsp/lightning/lib/jit_s390-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -394,6 +394,8 @@ static void _nop(jit_state_t*,jit_int32_t); # define EAR(R1,R2) RRE_(0xB24F,R1,R2) /* EXTRACT PSW */ # define EPSW(R1,R2) RRE_(0xB98D,R1,R2) +/* FIND LEFTMOST ONE */ +# define FLOGR(R1,R2) RRE_(0xB983,R1,R2) /* INSERT CHARACTER */ # define IC(R1,D2,X2,B2) RX_(0x43,R1,X2,B2,D2) # define ICY(R1,D2,X2,B2) RXY_(0xE3,R1,X2,B2,D2,0x73) @@ -966,6 +968,23 @@ static void _movr(jit_state_t*,jit_int32_t,jit_int32_t); static void _movi(jit_state_t*,jit_int32_t,jit_word_t); # define movi_p(r0,i0) _movi_p(_jit,r0,i0) static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t); +# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# if __WORDSIZE == 64 +#define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1) +static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t); +#endif +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define addr(r0,r1,r2) _addr(_jit,r0,r1,r2) static void _addr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define addi(r0,r1,i0) _addi(_jit,r0,r1,i0) @@ -1039,32 +1058,39 @@ static void _qdivi_u(jit_state_t*,jit_int32_t, # if __WORDSIZE == 32 # define lshr(r0,r1,r2) _lshr(_jit,r0,r1,r2) static void _lshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); -# else -# define lshr(r0,r1,r2) SLLG(r0,r1,0,r2) -# endif -# define lshi(r0,r1,i0) _lshi(_jit,r0,r1,i0) +# define lshi(r0,r1,i0) _lshi(_jit,r0,r1,i0) static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); -# if __WORDSIZE == 32 # define rshr(r0,r1,r2) _rshr(_jit,r0,r1,r2) static void _rshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); -# else -# define rshr(r0,r1,r2) SRAG(r0,r1,0,r2) -# endif -# define rshi(r0,r1,i0) _rshi(_jit,r0,r1,i0) +# define rshi(r0,r1,i0) _rshi(_jit,r0,r1,i0); static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); -# if __WORDSIZE == 32 # define rshr_u(r0,r1,r2) _rshr_u(_jit,r0,r1,r2) static void _rshr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); -# else -# define rshr_u(r0,r1,r2) SRLG(r0,r1,0,r2) -# endif # define rshi_u(r0,r1,i0) _rshi_u(_jit,r0,r1,i0) static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); +# else +# define lshr(r0,r1,r2) SLLG(r0,r1,0,r2) +# define lshi(r0,r1,i0) SLLG(r0,r1,i0,0) +# define rshr(r0,r1,r2) SRAG(r0,r1,0,r2) +# define rshi(r0,r1,i0) SRAG(r0,r1,i0,0) +# define rshr_u(r0,r1,r2) SRLG(r0,r1,0,r2) +# define rshi_u(r0,r1,i0) SRLG(r0,r1,i0,0) +# endif # if __WORDSIZE == 32 # define negr(r0,r1) LCR(r0,r1) # else # define negr(r0,r1) LCGR(r0,r1) # endif +# define bitswap(r0, r1) _bitswap(_jit, r0, r1) +static void _bitswap(jit_state_t*, jit_int32_t, jit_int32_t); +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) _clzr(_jit, r0, r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # define comr(r0,r1) _comr(_jit,r0,r1) static void _comr(jit_state_t*,jit_int32_t,jit_int32_t); # define andr(r0,r1,r2) _andr(_jit,r0,r1,r2) @@ -1079,13 +1105,6 @@ static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); static void _xorr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); # define xori(r0,r1,i0) _xori(_jit,r0,r1,i0) static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t); -# define htonr_us(r0,r1) extr_us(r0,r1) -# if __WORDSIZE == 32 -# define htonr_ui(r0,r1) movr(r0,r1) -# else -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) -# endif # define extr_c(r0,r1) LGBR(r0,r1) # define extr_uc(r0,r1) LLGCR(r0,r1) # define extr_s(r0,r1) LGHR(r0,r1) @@ -1284,13 +1303,13 @@ static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t); # define bmci(i0,r0,i1) bmxi(CC_E,i0,r0,i1) # define bmci_p(i0,r0,i1) bmxi_p(CC_E,i0,r0,i1) # define jmpr(r0) BR(r0) -# define jmpi(i0) _jmpi(_jit,i0) -static void _jmpi(jit_state_t*,jit_word_t); +# define jmpi(i0,i1) _jmpi(_jit,i0,i1) +static jit_word_t _jmpi(jit_state_t*,jit_word_t, jit_bool_t); # define jmpi_p(i0) _jmpi_p(_jit,i0) static jit_word_t _jmpi_p(jit_state_t*,jit_word_t); # define callr(r0) BALR(_R14_REGNO,r0) -# define calli(i0) _calli(_jit,i0) -static void _calli(jit_state_t*,jit_word_t); +# define calli(i0,i1) _calli(_jit,i0,i1) +static jit_word_t _calli(jit_state_t*,jit_word_t, jit_bool_t); # define calli_p(i0) _calli_p(_jit,i0) static jit_word_t _calli_p(jit_state_t*,jit_word_t); # define prolog(i0) _prolog(_jit,i0) @@ -2442,6 +2461,90 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = beqi_p(_jit->pc.w, r2, 0); +#if __WORDSIZE == 32 + LR(r0, r1); +#else + LGR(r0, r1); +#endif + patch_at(w, _jit->pc.w); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = bnei_p(_jit->pc.w, r2, 0); +#if __WORDSIZE == 32 + LR(r0, r1); +#else + LGR(r0, r1); +#endif + patch_at(w, _jit->pc.w); +} + +static void +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + LRVR(r0, r1); + SRL(r0, 16, 0); + LLGHR(r0, r0); +} + +static void +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + LRVR(r0, r1); +# if __WORDSIZE == 64 + LLGFR(r0, r0); +# endif +} + +#if __WORDSIZE == 64 +static void +_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + LRVGR(r0, r1); +} +#endif + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t iscasi, r1_reg; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg_but_zero(0); + r1 = rn(r1_reg); + movi(r1, i0); + } + /* Do not clobber r2 */ + movr(r0, r2); + /* The CS and CSG instructions below effectively do atomically: + * if (*r1 == r0) + * *r1 = r3; + * else + * r0 = *r1 + * So, we do not need to check cpu flags to know if it did work, + * just compare if values are different. + * Obviously it is somewhat of undefined behavior if old_value (r2) + * and new_value (r3) have the same value, but should still work + * as expected as a noop. + */ +# if __WORDSIZE == 32 + CS(r0, r3, 0, r1); +# else + CSG(r0, r3, 0, r1); +# endif + eqr(r0, r0, r2); + if (iscasi) + jit_unget_reg(r1_reg); +} + static void _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { @@ -2833,19 +2936,14 @@ _lshr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) SLL(r0, 0, r2); } } -#endif static void _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; - reg = jit_get_reg_but_zero(0); - movi(rn(reg), i0); - lshr(r0, r1, rn(reg)); - jit_unget_reg_but_zero(reg); + movr(r0, r1); + SLL(r0, i0, 0); } -# if __WORDSIZE == 32 static void _rshr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { @@ -2862,19 +2960,14 @@ _rshr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) SRA(r0, 0, r2); } } -#endif static void _rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; - reg = jit_get_reg_but_zero(0); - movi(rn(reg), i0); - rshr(r0, r1, rn(reg)); - jit_unget_reg_but_zero(reg); + movr(r0, r1); + SRA(r0, i0, 0); } -# if __WORDSIZE == 32 static void _rshr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { @@ -2891,16 +2984,141 @@ _rshr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) SRL(r0, 0, r2); } } -#endif static void _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { - jit_int32_t reg; - reg = jit_get_reg_but_zero(0); - movi(rn(reg), i0); - rshr_u(r0, r1, rn(reg)); - jit_unget_reg_but_zero(reg); + movr(r0, r1); + SRL(r0, i0, 0); +} +#endif + +static void +_bitswap(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t t0, t1, t2, t3, t4; + movr(r0, r1); + t0 = jit_get_reg(jit_class_gpr); + t1 = jit_get_reg(jit_class_gpr); + t2 = jit_get_reg(jit_class_gpr); + movi(rn(t0), __WORDSIZE == 32 ? 0x55555555L : 0x5555555555555555L); + rshi_u(rn(t1), r0, 1); /* t1 = v >> 1 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 1); /* t2 <<= 1 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x33333333L : 0x3333333333333333L); + rshi_u(rn(t1), r0, 2); /* t1 = v >> 2 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 2); /* t2 <<= 2 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x0f0f0f0fL : 0x0f0f0f0f0f0f0f0fL); + rshi_u(rn(t1), r0, 4); /* t1 = v >> 4 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 4); /* t2 <<= 4 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x00ff00ffL : 0x00ff00ff00ff00ffL); + rshi_u(rn(t1), r0, 8); /* t1 = v >> 8 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 8); /* t2 <<= 8 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# if __WORDSIZE == 32 + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + lshi(rn(t2), r0, 16); /* t2 = v << 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# else + movi(rn(t0), 0x0000ffff0000ffffL); + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 16); /* t2 <<= 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + rshi_u(rn(t1), r0, 32); /* t1 = v >> 32 */ + lshi(rn(t2), r0, 32); /* t2 = v << 32 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# endif + jit_unget_reg(t2); + jit_unget_reg(t1); + jit_unget_reg(t0); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if CHECK_FLOGR + if (jit_cpu.flogr) { +#endif + comr(r0, r1); + clzr(r0, r0); +#if CHECK_FLOGR + } + else + fallback_clo(r0, r1); +#endif +} + +static void +_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if CHECK_FLOGR + if (jit_cpu.flogr) { +#endif +#if __WORDSIZE == 32 + jit_word_t w; +#endif + jit_int32_t regno; + regno = jit_get_reg_pair(); +#if __WORDSIZE == 32 + SLLG(rn(regno), r1, 32, 0); +#else + movr(rn(regno), r1); +#endif + FLOGR(rn(regno), rn(regno)); + movr(r0, rn(regno)); +#if __WORDSIZE == 32 + w = blei_p(_jit->pc.w, r0, 31); + rshi(r0, r0, 1); /* r0 is 64 */ + patch_at(w, _jit->pc.w); +#endif + jit_unget_reg_pair(regno); +#if CHECK_FLOGR + } + else + fallback_clz(r0, r1); +#endif +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if CHECK_FLOGR + if (jit_cpu.flogr) { +#endif + bitswap(r0, r1); + clor(r0, r0); +#if CHECK_FLOGR + } + else + fallback_cto(r0, r1); +#endif +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ +#if CHECK_FLOGR + if (jit_cpu.flogr) { +#endif + bitswap(r0, r1); + clzr(r0, r0); +#if CHECK_FLOGR + } + else + fallback_ctz(r0, r1); +#endif } static void @@ -3433,13 +3651,14 @@ _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) } #endif -static void -_jmpi(jit_state_t *_jit, jit_word_t i0) +static jit_word_t +_jmpi(jit_state_t *_jit, jit_word_t i0, jit_bool_t i1) { - jit_word_t d; jit_int32_t reg; - d = (i0 - _jit->pc.w) >> 1; - if (s16_p(d)) + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 1; + if (i1 && s16_p(d)) J(x16(d)); else if (s32_p(d)) BRL(d); @@ -3449,6 +3668,7 @@ _jmpi(jit_state_t *_jit, jit_word_t i0) jmpr(rn(reg)); jit_unget_reg_but_zero(reg); } + return (w); } static jit_word_t @@ -3463,13 +3683,16 @@ _jmpi_p(jit_state_t *_jit, jit_word_t i0) return (w); } -static void -_calli(jit_state_t *_jit, jit_word_t i0) +static jit_word_t +_calli(jit_state_t *_jit, jit_word_t i0, jit_bool_t i1) { - jit_word_t d; jit_int32_t reg; - d = (i0 - _jit->pc.w) >> 1; - if (s32_p(d)) + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 1; + if (i1 && s16_p(d)) + BRAS(_R14_REGNO, x16(d)); + else if (s32_p(d)) BRASL(_R14_REGNO, d); else { reg = jit_get_reg_but_zero(0); @@ -3477,6 +3700,7 @@ _calli(jit_state_t *_jit, jit_word_t i0) callr(rn(reg)); jit_unget_reg_but_zero(reg); } + return (w); } static jit_word_t @@ -3825,17 +4049,17 @@ _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) u.s[7] = i1.s; #endif } - /* BRC */ + /* BRC or BRL */ else if (i0.b.op == 0xA7) { - assert(i0.b.r3 == 0x4); + assert(i0.b.r3 == 0x4 || i0.b.r3 == 0x5); d = (label - instr) >> 1; assert(s16_p(d)); i1.b.i2 = d; u.s[1] = i1.s; } - /* BRCL */ + /* BRCL or BRASL */ else if (i0.b.op == 0xC0) { - assert(i0.b.r3 == 0x4); + assert(i0.b.r3 == 0x4 || i0.b.r3 == 0x5); d = (label - instr) >> 1; assert(s32_p(d)); i12.i = d; diff --git a/parallel-rsp/lightning/lib/jit_s390-fpu.c b/parallel-rsp/lightning/lib/jit_s390-fpu.c index 6d60513..6c3c4ac 100644 --- a/parallel-rsp/lightning/lib/jit_s390-fpu.c +++ b/parallel-rsp/lightning/lib/jit_s390-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * diff --git a/parallel-rsp/lightning/lib/jit_s390-sz.c b/parallel-rsp/lightning/lib/jit_s390-sz.c index bb8b2dc..ee30447 100644 --- a/parallel-rsp/lightning/lib/jit_s390-sz.c +++ b/parallel-rsp/lightning/lib/jit_s390-sz.c @@ -1,20 +1,23 @@ - #if __WORDSIZE == 32 -#define JIT_INSTR_MAX 104 +#define JIT_INSTR_MAX 164 0, /* data */ 0, /* live */ - 6, /* align */ + 4, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 2, /* label */ - 42, /* prolog */ + 38, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -22,184 +25,227 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 44, /* va_start */ - 104, /* va_arg */ - 100, /* va_arg_d */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 40, /* va_start */ + 82, /* va_arg */ + 78, /* va_arg_d */ 0, /* va_end */ - 8, /* addr */ - 24, /* addi */ - 8, /* addcr */ - 20, /* addci */ - 8, /* addxr */ - 12, /* addxi */ - 12, /* subr */ - 24, /* subi */ - 12, /* subcr */ - 20, /* subci */ - 12, /* subxr */ - 12, /* subxi */ - 28, /* rsbi */ - 8, /* mulr */ - 24, /* muli */ - 60, /* qmulr */ - 68, /* qmuli */ - 16, /* qmulr_u */ - 32, /* qmuli_u */ - 12, /* divr */ - 28, /* divi */ + 4, /* addr */ + 12, /* addi */ + 4, /* addcr */ + 10, /* addci */ + 6, /* addxr */ + 10, /* addxi */ + 6, /* subr */ + 12, /* subi */ + 6, /* subcr */ + 10, /* subci */ + 8, /* subxr */ + 10, /* subxi */ + 14, /* rsbi */ + 6, /* mulr */ + 14, /* muli */ + 38, /* qmulr */ + 42, /* qmuli */ + 10, /* qmulr_u */ + 18, /* qmuli_u */ + 10, /* divr */ + 18, /* divi */ 16, /* divr_u */ - 32, /* divi_u */ - 16, /* qdivr */ - 20, /* qdivi */ - 20, /* qdivr_u */ - 24, /* qdivi_u */ - 12, /* remr */ - 28, /* remi */ + 24, /* divi_u */ + 12, /* qdivr */ + 16, /* qdivi */ + 18, /* qdivr_u */ + 22, /* qdivi_u */ + 10, /* remr */ + 18, /* remi */ 16, /* remr_u */ - 32, /* remi_u */ - 8, /* andr */ - 20, /* andi */ - 8, /* orr */ - 20, /* ori */ - 8, /* xorr */ - 24, /* xori */ - 6, /* lshr */ - 10, /* lshi */ - 6, /* rshr */ - 10, /* rshi */ - 6, /* rshr_u */ - 10, /* rshi_u */ - 4, /* negr */ - 12, /* comr */ - 20, /* ltr */ - 24, /* lti */ - 20, /* ltr_u */ - 24, /* lti_u */ - 20, /* ler */ - 24, /* lei */ - 20, /* ler_u */ - 24, /* lei_u */ - 20, /* eqr */ - 24, /* eqi */ - 20, /* ger */ - 24, /* gei */ - 20, /* ger_u */ - 24, /* gei_u */ - 20, /* gtr */ - 24, /* gti */ - 20, /* gtr_u */ - 24, /* gti_u */ - 20, /* ner */ - 24, /* nei */ - 4, /* movr */ - 16, /* movi */ + 24, /* remi_u */ + 4, /* andr */ + 10, /* andi */ + 4, /* orr */ + 10, /* ori */ + 4, /* xorr */ + 12, /* xori */ + 8, /* lshr */ + 6, /* lshi */ + 8, /* rshr */ + 6, /* rshi */ + 8, /* rshr_u */ + 6, /* rshi_u */ + 2, /* negr */ + 8, /* comr */ + 16, /* ltr */ + 20, /* lti */ + 16, /* ltr_u */ + 20, /* lti_u */ + 16, /* ler */ + 20, /* lei */ + 16, /* ler_u */ + 20, /* lei_u */ + 16, /* eqr */ + 20, /* eqi */ + 16, /* ger */ + 20, /* gei */ + 16, /* ger_u */ + 20, /* gei_u */ + 16, /* gtr */ + 20, /* gti */ + 16, /* gtr_u */ + 20, /* gti_u */ + 16, /* ner */ + 20, /* nei */ + 2, /* movr */ + 8, /* movi */ + 14, /* movnr */ + 14, /* movzr */ + 22, /* casr */ + 28, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ - 4, /* extr_i */ - 4, /* extr_ui */ + 0, /* extr_i */ + 0, /* extr_ui */ + 12, /* bswapr_us */ + 4, /* bswapr_ui */ + 0, /* bswapr_ul */ 4, /* htonr_us */ - 4, /* htonr_ui */ - 4, /* htonr_ul */ + 2, /* htonr_ui */ + 0, /* htonr_ul */ 6, /* ldr_c */ - 18, /* ldi_c */ + 12, /* ldi_c */ 6, /* ldr_uc */ - 18, /* ldi_uc */ - 6, /* ldr_s */ - 18, /* ldi_s */ + 14, /* ldi_uc */ + 4, /* ldr_s */ + 10, /* ldi_s */ 6, /* ldr_us */ - 18, /* ldi_us */ + 12, /* ldi_us */ 6, /* ldr_i */ - 18, /* ldi_i */ - 6, /* ldr_ui */ - 18, /* ldi_ui */ - 6, /* ldr_l */ - 18, /* ldi_l */ - 14, /* ldxr_c */ - 26, /* ldxi_c */ - 14, /* ldxr_uc */ - 26, /* ldxi_uc */ - 14, /* ldxr_s */ - 26, /* ldxi_s */ - 14, /* ldxr_us */ - 26, /* ldxi_us */ - 14, /* ldxr_i */ - 26, /* ldxi_i */ - 14, /* ldxr_ui */ - 26, /* ldxi_ui */ - 14, /* ldxr_l */ - 26, /* ldxi_l */ + 12, /* ldi_i */ + 0, /* ldr_ui */ + 0, /* ldi_ui */ + 0, /* ldr_l */ + 0, /* ldi_l */ + 10, /* ldxr_c */ + 16, /* ldxi_c */ + 10, /* ldxr_uc */ + 16, /* ldxi_uc */ + 8, /* ldxr_s */ + 14, /* ldxi_s */ + 10, /* ldxr_us */ + 16, /* ldxi_us */ + 10, /* ldxr_i */ + 16, /* ldxi_i */ + 0, /* ldxr_ui */ + 0, /* ldxi_ui */ + 0, /* ldxr_l */ + 0, /* ldxi_l */ 4, /* str_c */ - 16, /* sti_c */ + 12, /* sti_c */ 4, /* str_s */ - 16, /* sti_s */ + 10, /* sti_s */ 4, /* str_i */ - 16, /* sti_i */ - 6, /* str_l */ - 18, /* sti_l */ - 12, /* stxr_c */ - 28, /* stxi_c */ - 12, /* stxr_s */ - 28, /* stxi_s */ - 12, /* stxr_i */ - 28, /* stxi_i */ - 14, /* stxr_l */ - 30, /* stxi_l */ - 10, /* bltr */ - 14, /* blti */ - 10, /* bltr_u */ - 14, /* blti_u */ - 10, /* bler */ - 14, /* blei */ - 10, /* bler_u */ - 14, /* blei_u */ - 10, /* beqr */ - 26, /* beqi */ - 10, /* bger */ - 14, /* bgei */ - 10, /* bger_u */ - 14, /* bgei_u */ - 10, /* bgtr */ - 14, /* bgti */ - 10, /* bgtr_u */ - 14, /* bgti_u */ - 10, /* bner */ - 26, /* bnei */ - 18, /* bmsr */ - 18, /* bmsi */ - 18, /* bmcr */ - 18, /* bmci */ - 10, /* boaddr */ - 14, /* boaddi */ - 10, /* boaddr_u */ - 14, /* boaddi_u */ - 10, /* bxaddr */ - 14, /* bxaddi */ - 10, /* bxaddr_u */ - 14, /* bxaddi_u */ - 10, /* bosubr */ - 14, /* bosubi */ - 10, /* bosubr_u */ - 14, /* bosubi_u */ - 10, /* bxsubr */ - 14, /* bxsubi */ - 10, /* bxsubr_u */ - 14, /* bxsubi_u */ + 10, /* sti_i */ + 0, /* str_l */ + 0, /* sti_l */ + 8, /* stxr_c */ + 16, /* stxi_c */ + 8, /* stxr_s */ + 16, /* stxi_s */ + 8, /* stxr_i */ + 16, /* stxi_i */ + 0, /* stxr_l */ + 0, /* stxi_l */ + 8, /* bltr */ + 12, /* blti */ + 8, /* bltr_u */ + 12, /* blti_u */ + 8, /* bler */ + 12, /* blei */ + 8, /* bler_u */ + 12, /* blei_u */ + 8, /* beqr */ + 16, /* beqi */ + 8, /* bger */ + 12, /* bgei */ + 8, /* bger_u */ + 12, /* bgei_u */ + 8, /* bgtr */ + 12, /* bgti */ + 8, /* bgtr_u */ + 12, /* bgti_u */ + 8, /* bner */ + 16, /* bnei */ + 12, /* bmsr */ + 14, /* bmsi */ + 12, /* bmcr */ + 14, /* bmci */ + 8, /* boaddr */ + 12, /* boaddi */ + 8, /* boaddr_u */ + 12, /* boaddi_u */ + 8, /* bxaddr */ + 12, /* bxaddi */ + 8, /* bxaddr_u */ + 12, /* bxaddi_u */ + 8, /* bosubr */ + 12, /* bosubi */ + 8, /* bosubr_u */ + 12, /* bosubi_u */ + 8, /* bxsubr */ + 12, /* bxsubi */ + 8, /* bxsubr_u */ + 12, /* bxsubi_u */ 2, /* jmpr */ - 18, /* jmpi */ + 6, /* jmpi */ 2, /* callr */ - 18, /* calli */ + 6, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -207,20 +253,20 @@ 0, /* retval_i */ 0, /* retval_ui */ 0, /* retval_l */ - 40, /* epilog */ + 36, /* epilog */ 0, /* arg_f */ 0, /* getarg_f */ 0, /* putargr_f */ 0, /* putargi_f */ 6, /* addr_f */ - 26, /* addi_f */ + 24, /* addi_f */ 8, /* subr_f */ - 26, /* subi_f */ + 24, /* subi_f */ 28, /* rsbi_f */ 6, /* mulr_f */ - 26, /* muli_f */ + 24, /* muli_f */ 8, /* divr_f */ - 26, /* divi_f */ + 24, /* divi_f */ 4, /* negr_f */ 4, /* absr_f */ 4, /* sqrtr_f */ @@ -253,21 +299,21 @@ 16, /* unordr_f */ 36, /* unordi_f */ 4, /* truncr_f_i */ - 4, /* truncr_f_l */ + 0, /* truncr_f_l */ 4, /* extr_f */ 4, /* extr_d_f */ 2, /* movr_f */ 20, /* movi_f */ 4, /* ldr_f */ - 16, /* ldi_f */ - 12, /* ldxr_f */ - 24, /* ldxi_f */ + 10, /* ldi_f */ + 8, /* ldxr_f */ + 14, /* ldxi_f */ 4, /* str_f */ - 16, /* sti_f */ - 12, /* stxr_f */ - 24, /* stxi_f */ + 10, /* sti_f */ + 8, /* stxr_f */ + 14, /* stxi_f */ 10, /* bltr_f */ - 30, /* blti_f */ + 28, /* blti_f */ 10, /* bler_f */ 30, /* blei_f */ 10, /* beqr_f */ @@ -279,11 +325,11 @@ 10, /* bner_f */ 30, /* bnei_f */ 10, /* bunltr_f */ - 30, /* bunlti_f */ + 28, /* bunlti_f */ 10, /* bunler_f */ - 30, /* bunlei_f */ + 28, /* bunlei_f */ 18, /* buneqr_f */ - 38, /* buneqi_f */ + 36, /* buneqi_f */ 10, /* bunger_f */ 30, /* bungei_f */ 10, /* bungtr_f */ @@ -293,7 +339,7 @@ 10, /* bordr_f */ 30, /* bordi_f */ 10, /* bunordr_f */ - 30, /* bunordi_f */ + 28, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -304,87 +350,87 @@ 0, /* putargr_d */ 0, /* putargi_d */ 6, /* addr_d */ - 26, /* addi_d */ + 34, /* addi_d */ 8, /* subr_d */ - 26, /* subi_d */ - 28, /* rsbi_d */ + 34, /* subi_d */ + 38, /* rsbi_d */ 6, /* mulr_d */ - 26, /* muli_d */ + 34, /* muli_d */ 8, /* divr_d */ - 26, /* divi_d */ + 34, /* divi_d */ 4, /* negr_d */ 4, /* absr_d */ 4, /* sqrtr_d */ 16, /* ltr_d */ - 36, /* lti_d */ + 46, /* lti_d */ 16, /* ler_d */ - 36, /* lei_d */ + 46, /* lei_d */ 16, /* eqr_d */ - 36, /* eqi_d */ + 46, /* eqi_d */ 16, /* ger_d */ - 36, /* gei_d */ + 46, /* gei_d */ 16, /* gtr_d */ - 36, /* gti_d */ + 46, /* gti_d */ 16, /* ner_d */ - 36, /* nei_d */ + 46, /* nei_d */ 16, /* unltr_d */ - 36, /* unlti_d */ + 46, /* unlti_d */ 16, /* unler_d */ - 36, /* unlei_d */ + 46, /* unlei_d */ 20, /* uneqr_d */ - 40, /* uneqi_d */ + 50, /* uneqi_d */ 16, /* unger_d */ - 36, /* ungei_d */ + 46, /* ungei_d */ 16, /* ungtr_d */ - 36, /* ungti_d */ + 46, /* ungti_d */ 20, /* ltgtr_d */ - 40, /* ltgti_d */ + 50, /* ltgti_d */ 16, /* ordr_d */ - 36, /* ordi_d */ + 46, /* ordi_d */ 16, /* unordr_d */ - 36, /* unordi_d */ + 46, /* unordi_d */ 4, /* truncr_d_i */ - 4, /* truncr_d_l */ + 0, /* truncr_d_l */ 4, /* extr_d */ 4, /* extr_f_d */ 2, /* movr_d */ - 24, /* movi_d */ + 30, /* movi_d */ 4, /* ldr_d */ - 16, /* ldi_d */ - 12, /* ldxr_d */ - 24, /* ldxi_d */ + 10, /* ldi_d */ + 8, /* ldxr_d */ + 14, /* ldxi_d */ 4, /* str_d */ - 16, /* sti_d */ - 12, /* stxr_d */ - 24, /* stxi_d */ + 10, /* sti_d */ + 8, /* stxr_d */ + 14, /* stxi_d */ 10, /* bltr_d */ - 30, /* blti_d */ + 38, /* blti_d */ 10, /* bler_d */ - 30, /* blei_d */ + 38, /* blei_d */ 10, /* beqr_d */ - 34, /* beqi_d */ + 40, /* beqi_d */ 10, /* bger_d */ - 30, /* bgei_d */ + 40, /* bgei_d */ 10, /* bgtr_d */ - 30, /* bgti_d */ + 40, /* bgti_d */ 10, /* bner_d */ - 30, /* bnei_d */ + 40, /* bnei_d */ 10, /* bunltr_d */ - 30, /* bunlti_d */ + 38, /* bunlti_d */ 10, /* bunler_d */ - 30, /* bunlei_d */ + 38, /* bunlei_d */ 18, /* buneqr_d */ - 38, /* buneqi_d */ + 46, /* buneqi_d */ 10, /* bunger_d */ - 30, /* bungei_d */ + 40, /* bungei_d */ 10, /* bungtr_d */ - 30, /* bungti_d */ + 40, /* bungti_d */ 18, /* bltgtr_d */ - 38, /* bltgti_d */ + 48, /* bltgti_d */ 10, /* bordr_d */ - 30, /* bordi_d */ + 40, /* bordi_d */ 10, /* bunordr_d */ - 30, /* bunordi_d */ + 38, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -399,15 +445,20 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 36, /* clo */ + 28, /* clz */ + 164, /* cto */ + 158, /* ctz */ #endif /* __WORDSIZE */ #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 104 +#define JIT_INSTR_MAX 280 0, /* data */ 0, /* live */ - 6, /* align */ + 20, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 2, /* label */ @@ -416,7 +467,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -424,11 +478,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 44, /* va_start */ - 104, /* va_arg */ - 100, /* va_arg_d */ + 100, /* va_arg */ + 96, /* va_arg_d */ 0, /* va_end */ 8, /* addr */ 24, /* addi */ @@ -445,8 +511,8 @@ 28, /* rsbi */ 8, /* mulr */ 24, /* muli */ - 60, /* qmulr */ - 68, /* qmuli */ + 52, /* qmulr */ + 60, /* qmuli */ 16, /* qmulr_u */ 32, /* qmuli_u */ 12, /* divr */ @@ -468,11 +534,11 @@ 8, /* xorr */ 24, /* xori */ 6, /* lshr */ - 10, /* lshi */ + 6, /* lshi */ 6, /* rshr */ - 10, /* rshi */ + 6, /* rshi */ 6, /* rshr_u */ - 10, /* rshi_u */ + 6, /* rshi_u */ 4, /* negr */ 12, /* comr */ 20, /* ltr */ @@ -497,19 +563,26 @@ 24, /* nei */ 4, /* movr */ 16, /* movi */ + 18, /* movnr */ + 18, /* movzr */ + 30, /* casr */ + 42, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 4, /* extr_i */ 4, /* extr_ui */ + 12, /* bswapr_us */ + 8, /* bswapr_ui */ + 4, /* bswapr_ul */ 4, /* htonr_us */ 4, /* htonr_ui */ 4, /* htonr_ul */ 6, /* ldr_c */ 18, /* ldi_c */ 6, /* ldr_uc */ - 18, /* ldi_uc */ + 22, /* ldi_uc */ 6, /* ldr_s */ 18, /* ldi_s */ 6, /* ldr_us */ @@ -535,7 +608,7 @@ 14, /* ldxr_l */ 26, /* ldxi_l */ 4, /* str_c */ - 16, /* sti_c */ + 20, /* sti_c */ 4, /* str_s */ 16, /* sti_s */ 4, /* str_i */ @@ -591,17 +664,41 @@ 10, /* bxsubr_u */ 14, /* bxsubi_u */ 2, /* jmpr */ - 18, /* jmpi */ + 6, /* jmpi */ 2, /* callr */ - 18, /* calli */ + 14, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -801,4 +898,8 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 24, /* clo */ + 12, /* clz */ + 280, /* cto */ + 272, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_s390.c b/parallel-rsp/lightning/lib/jit_s390.c index 7cd1d7f..6934b11 100644 --- a/parallel-rsp/lightning/lib/jit_s390.c +++ b/parallel-rsp/lightning/lib/jit_s390.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -16,6 +16,12 @@ * Authors: * Paulo Cesar Pereira de Andrade */ +#define CHECK_FLOGR 0 + +#if CHECK_FLOGR +#include +#include +#endif #include #include @@ -88,11 +94,15 @@ extern void __clear_cache(void *, void *); #define PROTO 1 # include "jit_s390-cpu.c" # include "jit_s390-fpu.c" +# if CHECK_FLOGR +# include "jit_fallback.c" +# endif #undef PROTO /* * Initialization */ +jit_cpu_t jit_cpu; jit_register_t _rvs[] = { { rc(gpr) | 0x0, "%r0" }, { rc(gpr) | 0x1, "%r1" }, @@ -129,13 +139,48 @@ jit_register_t _rvs[] = { { rc(fpr) | rc(arg) | 0x0, "%f0" }, { _NOREG, "" }, }; +#if CHECK_FLOGR +static sigjmp_buf jit_env; +#endif /* * Implementation */ +#if CHECK_FLOGR +static void +sigill_handler(int signum) +{ + jit_cpu.flogr = 0; + siglongjmp(jit_env, 1); +} +#endif + void jit_get_cpu(void) { +#if CHECK_FLOGR + int r12, r13; + struct sigaction new_action, old_action; + new_action.sa_handler = sigill_handler; + sigemptyset(&new_action.sa_mask); + new_action.sa_flags = 0; + sigaction(SIGILL, NULL, &old_action); + if (old_action.sa_handler != SIG_IGN) { + sigaction(SIGILL, &new_action, NULL); + if (!sigsetjmp(jit_env, 1)) { + jit_cpu.flogr = 1; + /* flogr %r12, %r12 */ + __asm__ volatile("lgr %%r12, %0; lgr %%r13, %1;" + "flogr %%r12, %%r12;" + "lgr %1, %%r13; lgr %0, %%r12;" + : "=r" (r12), "=r" (r13)); + sigaction(SIGILL, &old_action, NULL); + } + } +#else + /* By default, assume it is available */ + jit_cpu.flogr = 1; +#endif } void @@ -240,18 +285,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); + jit_code_inc_synth_w(code, u); jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -305,7 +350,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_f_reg_p(u->u.w)); @@ -352,18 +397,22 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { offset = _jitc->function->self.size; _jitc->function->self.size += sizeof(jit_word_t); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -408,7 +457,7 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, _R2 - v->u.w); @@ -421,7 +470,7 @@ _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, _R2 - v->u.w); @@ -434,7 +483,7 @@ _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, _R2 - v->u.w); @@ -447,7 +496,7 @@ _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, _R2 - v->u.w); @@ -460,7 +509,7 @@ _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) { #if __WORDSIZE == 32 @@ -479,7 +528,7 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, _R2 - v->u.w); @@ -492,7 +541,7 @@ _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, _R2 - v->u.w); @@ -503,10 +552,10 @@ _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) #endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(_R2 - v->u.w, u); else @@ -515,11 +564,11 @@ _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(_R2 - v->u.w, u); else { @@ -627,10 +676,10 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(_R2 - _jitc->function->call.argi, u); @@ -644,11 +693,11 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(_R2 - _jitc->function->call.argi, u); @@ -890,6 +939,7 @@ _emit_code(jit_state_t *_jit) struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1027,11 +1077,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 1) & ~1); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1083,6 +1135,10 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh, _u); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(and,); case_rrw(and,); case_rrr(or,); @@ -1151,6 +1207,11 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _ui); #if __WORDSIZE == 64 case_rr(hton, _ul); +#endif + case_rr(bswap, _us); + case_rr(bswap, _ui); +#if __WORDSIZE == 64 + case_rr(bswap, _ul); #endif case_rr(ext, _c); case_rr(ext, _uc); @@ -1160,6 +1221,16 @@ _emit_code(jit_state_t *_jit) case_rr(ext, _i); case_rr(ext, _ui); #endif + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1413,14 +1484,21 @@ _emit_code(jit_state_t *_jit) assert(temp->code == jit_code_label || temp->code == jit_code_epilog); if (temp->flag & jit_flag_patch) - jmpi(temp->u.w); + jmpi(temp->u.w, 1); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s32_p(word)) { + offset = s16_p(word); + word = jmpi(_jit->pc.w, offset); + } + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } else - jmpi(node->u.w); + jmpi(node->u.w, 1); break; case jit_code_callr: callr(rn(node->u.w)); @@ -1431,19 +1509,27 @@ _emit_code(jit_state_t *_jit) assert(temp->code == jit_code_label || temp->code == jit_code_epilog); if (temp->flag & jit_flag_patch) - calli(temp->u.w); + calli(temp->u.w, 1); else { - word = calli_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s32_p(word)) { + offset =s16_p(word); + word = calli(_jit->pc.w, offset); + } + else + word = calli_p(_jit->pc.w); patch(word, node); } } else - calli(node->u.w); + calli(node->u.w, 1); break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1464,6 +1550,16 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1490,11 +1586,23 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +# if __WORDSIZE == 64 + case jit_code_arg_l: +# endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1504,10 +1612,26 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_ui: case jit_code_getarg_l: #endif case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1551,6 +1675,9 @@ _emit_code(jit_state_t *_jit) #define CODE 1 # include "jit_s390-cpu.c" # include "jit_s390-fpu.c" +# if CHECK_FLOGR +# include "jit_fallback.c" +# endif #undef CODE void diff --git a/parallel-rsp/lightning/lib/jit_size.c b/parallel-rsp/lightning/lib/jit_size.c index 61f1aa4..143a5d9 100644 --- a/parallel-rsp/lightning/lib/jit_size.c +++ b/parallel-rsp/lightning/lib/jit_size.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -28,7 +28,7 @@ */ static jit_int16_t _szs[jit_code_last_code] = { #if GET_JIT_SIZE -# define JIT_INSTR_MAX 256 +# define JIT_INSTR_MAX 1024 #else # if defined(__i386__) || defined(__x86_64__) # include "jit_x86-sz.c" @@ -52,6 +52,8 @@ static jit_int16_t _szs[jit_code_last_code] = { # include "jit_alpha-sz.c" # elif defined(__riscv) # include "jit_riscv-sz.c" +# elif defined(__loongarch__) +# include "jit_loongarch-sz.c" # endif #endif }; @@ -101,11 +103,40 @@ _jit_get_size(jit_state_t *_jit) { jit_word_t size; jit_node_t *node; +# if __riscv && __WORDSIZE == 64 + jit_word_t extra = 0; +# endif - for (size = JIT_INSTR_MAX, node = _jitc->head; node; node = node->next) - size += _szs[node->code]; + for (size = JIT_INSTR_MAX, node = _jitc->head; node; node = node->next) { +# if __riscv && __WORDSIZE == 64 + /* Get estimative of extra memory for constants at end of code. */ + switch (node->code) { + case jit_code_movi: + case jit_code_movi_f: + case jit_code_movi_d: + case jit_code_jmpi: + case jit_code_calli: + extra += sizeof(jit_word_t); + default: + break; + } +# endif + switch (node->code) { + /* The instructions are special because they can be arbitrarily long. */ + case jit_code_align: + case jit_code_skip: + size += node->u.w; + break; + default: + size += _szs[node->code]; + } + } +# if __riscv && __WORDSIZE == 64 + /* Heuristically only 20% of constants are unique. */ + size += extra / 5; +# endif - return ((size + 4095) & -4096); + return size; } #endif @@ -120,7 +151,7 @@ jit_finish_size(void) { #if GET_JIT_SIZE FILE *fp; - jit_word_t offset; + int offset; /* Define a single path */ fp = fopen(JIT_SIZE_PATH, "a"); diff --git a/parallel-rsp/lightning/lib/jit_sparc-cpu.c b/parallel-rsp/lightning/lib/jit_sparc-cpu.c index 051647a..f4ce621 100644 --- a/parallel-rsp/lightning/lib/jit_sparc-cpu.c +++ b/parallel-rsp/lightning/lib/jit_sparc-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -100,6 +100,9 @@ _f2bp(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t, static void _f3r(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); # if __WORDSIZE == 64 +# define f3ri(op, rd, op3, rs1, rs2) _f3ri(_jit, op, rd, op3, rs1, rs2) +static void _f3ri(jit_state_t*, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); # define f3rx(op, rd, op3, rs1, rs2) _f3rx(_jit, op, rd, op3, rs1, rs2) static void _f3rx(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); @@ -113,10 +116,15 @@ static void _f3s(jit_state_t*, # define f3t(cond, rs1, i, ri) _f3t(_jit, cond, rs1, i, ri) static void _f3t(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t) maybe_unused; -# define f3a(op, rd, op3, rs1, rs2) _f3a(_jit, op, rd, op3, rs1, asi, rs2) +# define f3a(op,rd,op3,rs1,asi,rs2) _f3a(_jit, op, rd, op3, rs1, asi, rs2) static void _f3a(jit_state_t*,jit_int32_t, jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t) maybe_unused; +# define f2c1(op,rd,op3,rs1,opf,rs2) _f2c1(_jit,op,rd,op3,rs1,opf,rs2) +static void +_f2c1(jit_state_t*,jit_int32_t, jit_int32_t, + jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t) + maybe_unused; # define LDSB(rs1, rs2, rd) f3r(3, rd, 9, rs1, rs2) # define LDSBI(rs1, imm, rd) f3i(3, rd, 9, rs1, imm) # define LDSH(rs1, rs2, rd) f3r(3, rd, 10, rs1, rs2) @@ -194,6 +202,11 @@ static void _f3a(jit_state_t*,jit_int32_t, # define SWAP(rs1, rs2, rd) f3r(3, rd, 15, rs1, rs2) # define SWAPI(rs1, imm, rd) f3r(3, rd, 15, rs1, imm) # define SWAPA(rs1, rs2, asi, rd) f3a(3, rd, 23, rs1, asi, rs2) +/* Sparc v9 deprecates SWAP* in favor of CAS*A */ +# define CASA(rs1, rs2, rd) f3a(3, rd, 60, rs1, 128, rs2) +# if __WORDSIZE == 64 +# define CASXA(rs1, rs2, rd) f3a(3, rd, 62, rs1, 128, rs2) +# endif # define NOP() SETHI(0, 0) # define HI(im) ((im) >> 10) # define LO(im) ((im) & 0x3ff) @@ -537,6 +550,7 @@ static void _f3a(jit_state_t*,jit_int32_t, # define UNIMP(imm) f2r(0, 0, 0, imm) # define FLUSH(rs1, rs2) f3r(2, 0, 59, rs1, rs2) # define FLUSHI(rs1, im) f3i(2, 0, 59, rs1, imm) +# define LZCNT(rs2, rd) f2c1(2, rd, 54, 0, 23, rs2) # define nop(i0) _nop(_jit, i0) static void _nop(jit_state_t*, jit_int32_t); # define movr(r0, r1) _movr(_jit, r0, r1) @@ -545,8 +559,30 @@ static void _movr(jit_state_t*, jit_int32_t, jit_int32_t); static void _movi(jit_state_t*, jit_int32_t, jit_word_t); # define movi_p(r0, i0) _movi_p(_jit, r0, i0) static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t); +# define bswapr_us(r0, r1) generic_bswapr_us(_jit, r0, r1) +# define bswapr_ui(r0, r1) generic_bswapr_ui(_jit, r0, r1) +# define bswapr_ul(r0, r1) generic_bswapr_ul(_jit, r0, r1) +# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2) +static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2) +static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) # define comr(r0, r1) XNOR(r1, 0, r0) # define negr(r0, r1) NEG(r1, r0) +# define bitswap(r0, r1) _bitswap(_jit, r0, r1) +static void _bitswap(jit_state_t*, jit_int32_t, jit_int32_t); +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) _clzr(_jit, r0, r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # define addr(r0, r1, r2) ADD(r1, r2, r0) # define addi(r0, r1, i0) _addi(_jit, r0, r1, i0) static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); @@ -669,7 +705,6 @@ static void _xori(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); # define rshr_u(r0, r1, r2) SRLX(r1, r2, r0) # define rshi_u(r0, r1, i0) SRLXI(r1, i0, r0) # endif -# define htonr_us(r0,r1) extr_us(r0,r1) # define extr_c(r0,r1) _extr_c(_jit,r0,r1) static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_uc(r0,r1) andi(r0, r1, 0xff) @@ -677,11 +712,7 @@ static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); static void _extr_s(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_us(r0,r1) _extr_us(_jit,r0,r1) static void _extr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# if __WORDSIZE == 32 -# define htonr_ui(r0,r1) movr(r0,r1) -# else -# define htonr_ui(r0,r1) extr_ui(r0,r1) -# define htonr_ul(r0,r1) movr(r0,r1) +# if __WORDSIZE == 64 # define extr_i(r0,r1) _extr_i(_jit,r0,r1) static void _extr_i(jit_state_t*,jit_int32_t,jit_int32_t); # define extr_ui(r0,r1) _extr_ui(_jit,r0,r1) @@ -926,13 +957,13 @@ _bm_w(jit_state_t*,jit_bool_t,jit_word_t,jit_int32_t,jit_word_t); # define jmpr(r0) _jmpr(_jit, r0) static void _jmpr(jit_state_t*,jit_int32_t); # define jmpi(i0) _jmpi(_jit, i0) -static void _jmpi(jit_state_t*,jit_word_t); +static jit_word_t _jmpi(jit_state_t*,jit_word_t); # define jmpi_p(i0) _jmpi_p(_jit, i0) static jit_word_t _jmpi_p(jit_state_t*,jit_word_t); # define callr(r0) _callr(_jit, r0) static void _callr(jit_state_t*,jit_int32_t); # define calli(i0) _calli(_jit, i0) -static void _calli(jit_state_t*,jit_word_t); +static jit_word_t _calli(jit_state_t*,jit_word_t); # define calli_p(i0) _calli_p(_jit, i0) static jit_word_t _calli_p(jit_state_t*,jit_word_t); # define prolog(node) _prolog(_jit, node) @@ -1028,6 +1059,26 @@ _f3r(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd, } # if __WORDSIZE == 64 +static void +_f3ri(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd, + jit_int32_t op3, jit_int32_t rs1, jit_int32_t rs2) +{ + jit_instr_t v; + assert(!(op & 0xfffffffc)); + assert(!(rd & 0xffffffe0)); + assert(!(op3 & 0xffffffc0)); + assert(!(rs1 & 0xffffffe0)); + assert(!(rs2 & 0xffffffe0)); + v.op.b = op; + v.rd.b = rd; + v.op3.b = op3; + v.rs1.b = rs1; + v.i.b = 1; + v.asi.b = 0; + v.rs2.b = rs2; + ii(v.v); +} + static void _f3rx(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd, jit_int32_t op3, jit_int32_t rs1, jit_int32_t rs2) @@ -1147,6 +1198,26 @@ _f1(jit_state_t *_jit, jit_int32_t op, jit_int32_t disp30) ii(v.v); } +static void +_f2c1(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd, + jit_int32_t op3, jit_int32_t rs1, jit_int32_t opf, jit_int32_t rs2) +{ + jit_instr_t v; + assert(!(op & 0xfffffffc)); + assert(!(rd & 0xffffffe0)); + assert(!(res & 0xffffffc0)); + assert(!(rs1 & 0xffffffe0)); + assert(!(opf & 0xfffffe00)); + assert(!(rs2 & 0xfffffe00)); + v.op.b = op; + v.rd.b = rd; + v.op3.b = op3; + v.rs1.b = rs1; + v.opf.b = opf; + v.rs2.b = rs2; + ii(v.v); +} + static void _nop(jit_state_t *_jit, jit_int32_t i0) { @@ -1213,6 +1284,159 @@ _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) return (w); } +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = beqi(_jit->pc.w, r2, 0); + ORI(r1, 0, r0); + patch_at(w, _jit->pc.w); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + jit_word_t w; + w = bnei(_jit->pc.w, r2, 0); + ORI(r1, 0, r0); + patch_at(w, _jit->pc.w); +} + +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t iscasi, r1_reg; + if ((iscasi = (r1 == _NOREG))) { + r1_reg = jit_get_reg(jit_class_gpr); + r1 = rn(r1_reg); + movi(r1, i0); + } + /* Do not clobber r2 */ + movr(r0, r3); + /* The CASXA instruction compares the value in register r[rs2] with + * the doubleword in memory pointed to by the doubleword address in + * r[rs1]. If the values are equal, the value in r[rd] is swapped + * with the doubleword pointed to by the doubleword address in r[rs1]. + * If the values are not equal, the contents of the doubleword pointed + * to by r[rs1] replaces the value in r[rd], but the memory location + * remains unchanged. + */ +# if __WORDSIZE == 32 + CASA(r1, r2, r0); +# else + CASXA(r1, r2, r0); +# endif + eqr(r0, r0, r2); + if (iscasi) + jit_unget_reg(r1_reg); +} + +static void +_bitswap(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t t0, t1, t2, t3, t4; + movr(r0, r1); + t0 = jit_get_reg(jit_class_gpr); + t1 = jit_get_reg(jit_class_gpr); + t2 = jit_get_reg(jit_class_gpr); + movi(rn(t0), __WORDSIZE == 32 ? 0x55555555L : 0x5555555555555555L); + rshi_u(rn(t1), r0, 1); /* t1 = v >> 1 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 1); /* t2 <<= 1 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x33333333L : 0x3333333333333333L); + rshi_u(rn(t1), r0, 2); /* t1 = v >> 2 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 2); /* t2 <<= 2 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x0f0f0f0fL : 0x0f0f0f0f0f0f0f0fL); + rshi_u(rn(t1), r0, 4); /* t1 = v >> 4 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 4); /* t2 <<= 4 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + movi(rn(t0), __WORDSIZE == 32 ? 0x00ff00ffL : 0x00ff00ff00ff00ffL); + rshi_u(rn(t1), r0, 8); /* t1 = v >> 8 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 8); /* t2 <<= 8 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# if __WORDSIZE == 32 + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + lshi(rn(t2), r0, 16); /* t2 = v << 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# else + movi(rn(t0), 0x0000ffff0000ffffL); + rshi_u(rn(t1), r0, 16); /* t1 = v >> 16 */ + andr(rn(t1), rn(t1), rn(t0)); /* t1 &= t0 */ + andr(rn(t2), r0, rn(t0)); /* t2 = v & t0*/ + lshi(rn(t2), rn(t2), 16); /* t2 <<= 16 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ + rshi_u(rn(t1), r0, 32); /* t1 = v >> 32 */ + lshi(rn(t2), r0, 32); /* t2 = v << 32 */ + orr(r0, rn(t1), rn(t2)); /* v = t1 | t2 */ +# endif + jit_unget_reg(t2); + jit_unget_reg(t1); + jit_unget_reg(t0); +} + +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.lzcnt) { + comr(r0, r1); + clzr(r0, r0); + } + else + fallback_clo(r0, r1); +} + +static void +_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.lzcnt) { +#if __WORDSIZE == 32 + jit_word_t w; + SLLXI(r1, 32, r0); + LZCNT(r0, r0); +#if __WORDSIZE == 32 + w = blei(_jit->pc.w, r0, 31); + rshi(r0, r0, 1); /* r0 is 64 */ + patch_at(w, _jit->pc.w); +#endif +#else + LZCNT(r1, r0); + } + else + fallback_clz(r0, r1); +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.lzcnt) { + bitswap(r0, r1); + clor(r0, r0); + } + else + fallback_cto(r0, r1); +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + if (jit_cpu.lzcnt) { + bitswap(r0, r1); + clzr(r0, r0); + } + else + fallback_ctz(r0, r1); +} + static void _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { @@ -2227,7 +2451,7 @@ _bw(jit_state_t *_jit, jit_int32_t cc, # if __WORDSIZE == 32 B(cc, (i0 - w) >> 2); # else - B(cc, (i0 - w) >> 2); + BP(cc, (i0 - w) >> 2); # endif NOP(); } @@ -2347,14 +2571,15 @@ _jmpr(jit_state_t *_jit, jit_int32_t r0) NOP(); } -static void +static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { - jit_word_t w; jit_int32_t reg; - w = (i0 - _jit->pc.w) >> 2; - if (s22_p(w)) { - BA(w); + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 2; + if (s22_p(d)) { + BA(d); NOP(); } else { @@ -2363,6 +2588,7 @@ _jmpi(jit_state_t *_jit, jit_word_t i0) jmpr(rn(reg)); jit_unget_reg(reg); } + return (w); } static jit_word_t @@ -2384,13 +2610,19 @@ _callr(jit_state_t *_jit, jit_int32_t r0) NOP(); } -static void +static jit_word_t _calli(jit_state_t *_jit, jit_word_t i0) { - jit_word_t w; - w = (i0 - _jit->pc.w) >> 2; - CALLI(w); - NOP(); + jit_word_t d, w; + w = _jit->pc.w; + d = (i0 - w) >> 2; + if (s30_p(d)) { + CALLI(d); + NOP(); + } + else + w = calli_p(i0); + return (w); } static jit_word_t @@ -2464,24 +2696,24 @@ _epilog(jit_state_t *_jit, jit_node_t *node) { if (_jitc->function->assume_frame) return; - /* (most) other backends do not save incoming arguments, so, - * only save locals here */ + if (_jitc->function->allocar) + subi(_SP_REGNO, _FP_REGNO, _jitc->function->stack); if (jit_regset_tstbit(&_jitc->function->regset, _L0)) - ldxi(_L0_REGNO, _FP_REGNO, _jitc->function->stack + OFF(0)); + ldxi(_L0_REGNO, _SP_REGNO, _jitc->function->stack + OFF(0)); if (jit_regset_tstbit(&_jitc->function->regset, _L1)) - ldxi(_L1_REGNO, _FP_REGNO, _jitc->function->stack + OFF(1)); + ldxi(_L1_REGNO, _SP_REGNO, _jitc->function->stack + OFF(1)); if (jit_regset_tstbit(&_jitc->function->regset, _L2)) - ldxi(_L2_REGNO, _FP_REGNO, _jitc->function->stack + OFF(2)); + ldxi(_L2_REGNO, _SP_REGNO, _jitc->function->stack + OFF(2)); if (jit_regset_tstbit(&_jitc->function->regset, _L3)) - ldxi(_L3_REGNO, _FP_REGNO, _jitc->function->stack + OFF(3)); + ldxi(_L3_REGNO, _SP_REGNO, _jitc->function->stack + OFF(3)); if (jit_regset_tstbit(&_jitc->function->regset, _L4)) - ldxi(_L4_REGNO, _FP_REGNO, _jitc->function->stack + OFF(4)); + ldxi(_L4_REGNO, _SP_REGNO, _jitc->function->stack + OFF(4)); if (jit_regset_tstbit(&_jitc->function->regset, _L5)) - ldxi(_L5_REGNO, _FP_REGNO, _jitc->function->stack + OFF(5)); + ldxi(_L5_REGNO, _SP_REGNO, _jitc->function->stack + OFF(5)); if (jit_regset_tstbit(&_jitc->function->regset, _L6)) - ldxi(_L6_REGNO, _FP_REGNO, _jitc->function->stack + OFF(6)); + ldxi(_L6_REGNO, _SP_REGNO, _jitc->function->stack + OFF(6)); if (jit_regset_tstbit(&_jitc->function->regset, _L7)) - ldxi(_L7_REGNO, _FP_REGNO, _jitc->function->stack + OFF(7)); + ldxi(_L7_REGNO, _SP_REGNO, _jitc->function->stack + OFF(7)); RESTOREI(0, 0, 0); RETL(); NOP(); @@ -2562,6 +2794,11 @@ _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) else abort(); } + else if (i.op.b == 1) { + assert(s30_p((label - instr) >> 2)); + i.disp30.b = (label - instr) >> 2; + u.i[0] = i.v; + } else abort(); } diff --git a/parallel-rsp/lightning/lib/jit_sparc-fpu.c b/parallel-rsp/lightning/lib/jit_sparc-fpu.c index ae2cbab..d0e7e81 100644 --- a/parallel-rsp/lightning/lib/jit_sparc-fpu.c +++ b/parallel-rsp/lightning/lib/jit_sparc-fpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -498,6 +498,44 @@ _f3f(jit_state_t *_jit, jit_int32_t rd, } # if __WORDSIZE == 64 +/* Handle the special case of using all float registers, as exercised + * in check/carg.c. + * For example: + * putargr_f JIT_F0 $ARG + * where JIT_F0 is %f32 and $ARG is %f31 and if %f30 (the mapping for %f31) + * is live, the jit_get_reg() call might return %f30, but, because it is + * live, will spill/reload it, generating assembly: + * + * std %f30, [ %fp + OFFS ] + * fmovd %f32, %f30 + * fmovs %f30, %f31 + * ldd [ %fp + OFFS ], %f30 + * + * what basically becomes a noop as it restores the old value. + */ +#define get_sng_reg(u) _get_sng_reg(_jit, u) +static jit_int32_t +_get_sng_reg(jit_state_t *_jit, jit_int32_t r0) +{ + jit_int32_t reg, tmp; + /* Attempt to get a nospill register */ + reg = jit_get_reg(CLASS_SNG | jit_class_nospill | jit_class_chk); + if (reg == JIT_NOREG) { + /* Will need to spill, so allow spilling it. */ + reg = jit_get_reg(CLASS_SNG); + /* If the special condition happens, allocate another one. + * This will generate uglier machine code (code for floats + * is already ugly), but will work, but doing a double + * spill/reload; the first one being a noop. */ + if (rn(reg) == r0 - 1) { + tmp = reg; + reg = jit_get_reg(CLASS_SNG); + jit_unget_reg(tmp); + } + } + return (reg); +} + static void _movr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { @@ -507,7 +545,7 @@ _movr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) if (single_precision_p(r1)) FMOVS(r1, r0); else { - t1 = jit_get_reg(CLASS_SNG); + t1 = get_sng_reg(r0); movr_d(rn(t1), r1); FMOVS(rn(t1), r0); jit_unget_reg(t1); @@ -515,13 +553,13 @@ _movr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } else { if (single_precision_p(r1)) { - t0 = jit_get_reg(CLASS_SNG); + t0 = get_sng_reg(r0); FMOVS(r1, rn(t0)); movr_d(r0, rn(t0)); jit_unget_reg(t0); } else { - t1 = jit_get_reg(CLASS_SNG); + t1 = get_sng_reg(r0); movr_d(rn(t1), r1); FMOVS(rn(t1), rn(t1)); movr_d(r0, rn(t1)); @@ -1491,7 +1529,12 @@ _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) assert(_jitc->function->self.call & jit_call_varargs); /* Load argument. */ +#if __WORDSIZE == 64 ldr_d(r0, r1); +#else + ldr_f(r0, r1); + ldxi_f(r0 + 1, r1, 4); +#endif /* Update vararg stack pointer. */ addi(r1, r1, 8); diff --git a/parallel-rsp/lightning/lib/jit_sparc-sz.c b/parallel-rsp/lightning/lib/jit_sparc-sz.c index ac683b6..95954d9 100644 --- a/parallel-rsp/lightning/lib/jit_sparc-sz.c +++ b/parallel-rsp/lightning/lib/jit_sparc-sz.c @@ -1,10 +1,11 @@ #if __WORDSIZE == 32 -#define JIT_INSTR_MAX 44 +#define JIT_INSTR_MAX 180 0, /* data */ 0, /* live */ 0, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 0, /* label */ @@ -13,7 +14,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -21,11 +25,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ - 8, /* va_arg_d */ + 12, /* va_arg_d */ 0, /* va_end */ 4, /* addr */ 12, /* addi */ @@ -94,12 +110,19 @@ 16, /* nei */ 4, /* movr */ 8, /* movi */ + 16, /* movnr */ + 16, /* movzr */ + 24, /* casr */ + 32, /* casi */ 8, /* extr_c */ 4, /* extr_uc */ 8, /* extr_s */ 8, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 20, /* bswapr_us */ + 52, /* bswapr_ui */ + 0, /* bswapr_ul */ 8, /* htonr_us */ 4, /* htonr_ui */ 0, /* htonr_ul */ @@ -188,17 +211,41 @@ 12, /* bxsubr_u */ 12, /* bxsubi_u */ 8, /* jmpr */ - 16, /* jmpi */ + 8, /* jmpi */ 8, /* callr */ - 16, /* calli */ + 8, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -398,15 +445,20 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 176, /* clo */ + 148, /* clz */ + 180, /* cto */ + 152, /* ctz */ #endif /* __WORDSIZE */ #if __WORDSIZE == 64 -#define JIT_INSTR_MAX 64 +#define JIT_INSTR_MAX 216 0, /* data */ 0, /* live */ - 4, /* align */ + 24, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 4, /* label */ @@ -415,7 +467,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -423,8 +478,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 4, /* va_start */ 8, /* va_arg */ 8, /* va_arg_d */ @@ -496,17 +563,24 @@ 16, /* nei */ 4, /* movr */ 24, /* movi */ + 16, /* movnr */ + 16, /* movzr */ + 24, /* casr */ + 44, /* casi */ 8, /* extr_c */ 4, /* extr_uc */ 8, /* extr_s */ 8, /* extr_us */ 8, /* extr_i */ 8, /* extr_ui */ + 20, /* bswapr_us */ + 52, /* bswapr_ui */ + 116, /* bswapr_ul */ 8, /* htonr_us */ 8, /* htonr_ui */ 4, /* htonr_ul */ 4, /* ldr_c */ - 28, /* ldi_c */ + 24, /* ldi_c */ 4, /* ldr_uc */ 28, /* ldi_uc */ 4, /* ldr_s */ @@ -590,17 +664,41 @@ 12, /* bxsubr_u */ 12, /* bxsubi_u */ 8, /* jmpr */ - 32, /* jmpi */ + 8, /* jmpi */ 8, /* callr */ - 32, /* calli */ + 40, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -657,7 +755,7 @@ 16, /* truncr_f_l */ 20, /* extr_f */ 12, /* extr_d_f */ - 16, /* movr_f */ + 24, /* movr_f */ 32, /* movi_f */ 8, /* ldr_f */ 32, /* ldi_f */ @@ -672,13 +770,13 @@ 20, /* bler_f */ 44, /* blei_f */ 28, /* beqr_f */ - 60, /* beqi_f */ + 52, /* beqi_f */ 20, /* bger_f */ 44, /* bgei_f */ 20, /* bgtr_f */ 44, /* bgti_f */ 20, /* bner_f */ - 44, /* bnei_f */ + 60, /* bnei_f */ 20, /* bunltr_f */ 44, /* bunlti_f */ 20, /* bunler_f */ @@ -800,4 +898,8 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 216, /* clo */ + 188, /* clz */ + 204, /* cto */ + 176, /* ctz */ #endif /* __WORDSIZE */ diff --git a/parallel-rsp/lightning/lib/jit_sparc.c b/parallel-rsp/lightning/lib/jit_sparc.c index 158c09d..9e837d8 100644 --- a/parallel-rsp/lightning/lib/jit_sparc.c +++ b/parallel-rsp/lightning/lib/jit_sparc.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2019 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -17,6 +17,16 @@ * Paulo Cesar Pereira de Andrade */ +/* Handling SIGILL should not be done by Lightning, but can either use + * sample, or use another approach to set jit_cpu.lzcnt + */ +#define CHECK_LZCNT 0 + +#if CHECK_LZCNT +#include +#include +#endif + #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 6) #if __WORDSIZE == 32 # define jit_arg_d_reg_p(i) ((i) >= 0 && (i) < 5) @@ -40,11 +50,13 @@ static void _patch(jit_state_t*,jit_word_t,jit_node_t*); #define PROTO 1 # include "jit_sparc-cpu.c" # include "jit_sparc-fpu.c" +# include "jit_fallback.c" #undef PROTO /* * Initialization */ +jit_cpu_t jit_cpu; jit_register_t _rvs[] = { { 0x00, "%g0" }, { 0x01, "%g1" }, @@ -147,13 +159,45 @@ jit_register_t _rvs[] = { # endif { _NOREG, "" }, }; +#if CHECK_LZCNT +sigjmp_buf jit_env; +#endif /* * Implementation */ +#if CHECK_LZCNT +static void +sigill_handler(int signum) +{ + jit_cpu.lzcnt = 0; + siglongjmp(jit_env, 1); +} +#endif + void jit_get_cpu(void) { +#if CHECK_LZCNT + int g2; + struct sigaction new_action, old_action; + new_action.sa_handler = sigill_handler; + sigemptyset(&new_action.sa_mask); + new_action.sa_flags = 0; + sigaction(SIGILL, NULL, &old_action); + if (old_action.sa_handler != SIG_IGN) { + sigaction(SIGILL, &new_action, NULL); + if (!sigsetjmp(jit_env, 1)) { + jit_cpu.lzcnt = 1; + /* lzcnt %g2, %g2 */ + __asm__ volatile("mov %%g2, %0; .long 0xa3b0021; mov %0, %%g2" + : "=r" (g2)); + sigaction(SIGILL, &old_action, NULL); + } + } +#else + jit_cpu.lzcnt = 0; +#endif } void @@ -184,7 +228,7 @@ _jit_prolog(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++; _jitc->function->self.size = stack_framesize; _jitc->function->self.argi = _jitc->function->self.argf = - _jitc->function->self.aoff = _jitc->function->self.alen = 0; + _jitc->function->self.alen = 0; /* float conversion */ # if __WORDSIZE == 32 _jitc->function->self.aoff = -8; @@ -265,20 +309,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -339,12 +381,13 @@ jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { # if __WORDSIZE == 32 - if (u->code == jit_code_arg || u->code == jit_code_arg_f) + if ((u->code >= jit_code_arg_c && u->code <= jit_code_arg) || + u->code == jit_code_arg_f) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_d); return (jit_arg_d_reg_p(u->u.w)); # else - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_d || u->code == jit_code_arg_f); return (jit_arg_d_reg_p(u->u.w)); @@ -379,11 +422,15 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { @@ -394,7 +441,7 @@ _jit_arg(jit_state_t *_jit) offset = BIAS(_jitc->function->self.size); _jitc->function->self.size += sizeof(jit_word_t); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -471,7 +518,7 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, _I0 + v->u.w); @@ -484,7 +531,7 @@ _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, _I0 + v->u.w); @@ -497,7 +544,7 @@ _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, _I0 + v->u.w); @@ -510,7 +557,7 @@ _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, _I0 + v->u.w); @@ -523,7 +570,7 @@ _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) { # if __WORDSIZE == 64 @@ -542,7 +589,7 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, _I0 + v->u.w); @@ -555,7 +602,7 @@ _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_i, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, _I0 + v->u.w); @@ -566,10 +613,10 @@ _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) # endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(_I0 + v->u.w, u); else @@ -578,11 +625,11 @@ _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_arg_reg_p(v->u.w)) jit_movi(_I0 + v->u.w, u); else { @@ -795,9 +842,9 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(_O0 + _jitc->function->call.argi, u); @@ -816,10 +863,10 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(_O0 + _jitc->function->call.argi, u); @@ -1193,6 +1240,7 @@ _emit_code(jit_state_t *_jit) struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1321,11 +1369,13 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); - if (node->u.w == sizeof(jit_word_t) && - (word = _jit->pc.w & (sizeof(jit_word_t) - 1))) - nop(sizeof(jit_word_t) - word); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); + if ((word = _jit->pc.w & (node->u.w - 1))) + nop(node->u.w - word); + break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; @@ -1463,6 +1513,11 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _ui); #if __WORDSIZE == 64 case_rr(hton, _ul); +#endif + case_rr(bswap, _us); + case_rr(bswap, _ui); +#if __WORDSIZE == 64 + case_rr(bswap, _ul); #endif case_rr(ext, _c); case_rr(ext, _uc); @@ -1472,6 +1527,16 @@ _emit_code(jit_state_t *_jit) case_rr(ext, _i); case_rr(ext, _ui); #endif + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1492,6 +1557,10 @@ _emit_code(jit_state_t *_jit) break; case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_brr(blt,); case_brw(blt,); case_brr(blt, _u); @@ -1709,7 +1778,12 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s22_p(word >> 2)) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } @@ -1724,9 +1798,17 @@ _emit_code(jit_state_t *_jit) temp = node->u.n; assert(temp->code == jit_code_label || temp->code == jit_code_epilog); - word = calli_p(temp->u.w); - if (!(temp->flag & jit_flag_patch)) + if (temp->flag & jit_flag_patch) + calli(temp->u.w); + else { + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s30_p(word >> 2)) + word = calli(_jit->pc.w); + else + word = calli_p(_jit->pc.w); patch(word, node); + } } else calli(node->u.w); @@ -1735,6 +1817,7 @@ _emit_code(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1755,6 +1838,16 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1779,11 +1872,23 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +#if __WORDSIZE == 64 + case jit_code_arg_l: +#endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1793,10 +1898,26 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_ui: case jit_code_getarg_l: #endif case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1868,6 +1989,7 @@ _emit_code(jit_state_t *_jit) #define CODE 1 # include "jit_sparc-cpu.c" # include "jit_sparc-fpu.c" +# include "jit_fallback.c" #undef CODE void diff --git a/parallel-rsp/lightning/lib/jit_x86-cpu.c b/parallel-rsp/lightning/lib/jit_x86-cpu.c index 4627783..f0e4155 100644 --- a/parallel-rsp/lightning/lib/jit_x86-cpu.c +++ b/parallel-rsp/lightning/lib/jit_x86-cpu.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -136,12 +136,6 @@ # else # define il(l) ii(l) # endif -# define patch_abs(instr, label) \ - *(jit_word_t *)(instr - sizeof(jit_word_t)) = label -# define patch_rel(instr, label) \ - *(jit_int32_t *)(instr - 4) = label - instr -# define patch_rel_char(instr, label) \ - *(jit_int8_t *)(instr - 1) = label - instr # define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b) static void _rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t); @@ -186,7 +180,8 @@ static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); #define addci(r0, r1, i0) _addci(_jit, r0, r1, i0) static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t); -# define iaddxr(r0, r1) alur(X86_ADC, r0, r1) +# define iaddxr(r0, r1) _iaddxr(_jit, r0, r1) +static void _iaddxr(jit_state_t*, jit_int32_t, jit_int32_t); # define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2) static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); # define iaddxi(r0, i0) alui(X86_ADC, r0, i0) @@ -308,6 +303,14 @@ static void _incr(jit_state_t*, jit_int32_t, jit_int32_t); # define decr(r0, r1) _decr(_jit, r0, r1) static void _decr(jit_state_t*, jit_int32_t, jit_int32_t); # endif +# define clor(r0, r1) _clor(_jit, r0, r1) +static void _clor(jit_state_t*, jit_int32_t, jit_int32_t); +# define clzr(r0, r1) _clzr(_jit, r0, r1) +static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctor(r0, r1) _ctor(_jit, r0, r1) +static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t); +# define ctzr(r0, r1) _ctzr(_jit, r0, r1) +static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t); # define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2) static void _cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t); @@ -358,7 +361,13 @@ static void _movr(jit_state_t*, jit_int32_t, jit_int32_t); # define imovi(r0, i0) _imovi(_jit, r0, i0) static void _imovi(jit_state_t*, jit_int32_t, jit_word_t); # define movi(r0, i0) _movi(_jit, r0, i0) -static void _movi(jit_state_t*, jit_int32_t, jit_word_t); +static +# if CAN_RIP_ADDRESS +jit_word_t +# else +void +# endif +_movi(jit_state_t*, jit_int32_t, jit_word_t); # define movi_p(r0, i0) _movi_p(_jit, r0, i0) static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t); # define movcr(r0, r1) _movcr(_jit, r0, r1) @@ -369,19 +378,28 @@ static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t); static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t); # define movsr_u(r0, r1) _movsr_u(_jit, r0, r1) static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t); +# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0) +static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t, + jit_int32_t,jit_int32_t,jit_word_t); +#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0) +#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0) +#define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2) +static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); +#define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2) +static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t); # if __X64 && !__X64_32 # define movir(r0, r1) _movir(_jit, r0, r1) static void _movir(jit_state_t*,jit_int32_t,jit_int32_t); # define movir_u(r0, r1) _movir_u(_jit, r0, r1) static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t); # endif -# define htonr_us(r0, r1) _htonr_us(_jit, r0, r1) -static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t); -# define htonr_ui(r0, r1) _htonr_ui(_jit, r0, r1) -static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1) +static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t); +# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1) +static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t); # if __X64 && !__X64_32 -#define htonr_ul(r0, r1) _htonr_ul(_jit, r0, r1) -static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t); +#define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1) +static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t); #endif # define extr_c(r0, r1) _extr_c(_jit, r0, r1) static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t); @@ -538,7 +556,7 @@ static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); # define jng(i0) jcc(X86_CC_NG, i0) # define jg(i0) jcc(X86_CC_G, i0) # define jnle(i0) jcc(X86_CC_NLE, i0) -static void _jcc(jit_state_t*, jit_int32_t, jit_word_t); +static jit_word_t _jcc(jit_state_t*, jit_int32_t, jit_word_t); # define jccs(code, i0) _jccs(_jit, code, i0) # define jos(i0) jccs(X86_CC_O, i0) # define jnos(i0) jccs(X86_CC_NO, i0) @@ -570,13 +588,15 @@ static void _jcc(jit_state_t*, jit_int32_t, jit_word_t); # define jngs(i0) jccs(X86_CC_NG, i0) # define jgs(i0) jccs(X86_CC_G, i0) # define jnles(i0) jccs(X86_CC_NLE, i0) -static void _jccs(jit_state_t*, jit_int32_t, jit_word_t); +static jit_word_t _jccs(jit_state_t*, jit_int32_t, jit_word_t); # define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1) -static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t); +static jit_word_t _jcr(jit_state_t*, + jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t); # define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1) -static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t); +static jit_word_t _jci(jit_state_t*, + jit_int32_t,jit_word_t,jit_int32_t,jit_word_t); # define jci0(code, i0, r0) _jci0(_jit, code, i0, r0) -static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t); +static jit_word_t _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t); # define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1) static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t); # define blti(i0, r0, i1) _blti(_jit, i0, r0, i1) @@ -661,12 +681,24 @@ static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t); static void _callr(jit_state_t*, jit_int32_t); # define calli(i0) _calli(_jit, i0) static jit_word_t _calli(jit_state_t*, jit_word_t); +# if __X64 +# define calli_p(i0) _calli_p(_jit, i0) +static jit_word_t _calli_p(jit_state_t*, jit_word_t); +# else +# define calli_p(i0) calli(i0) +# endif # define jmpr(r0) _jmpr(_jit, r0) static void _jmpr(jit_state_t*, jit_int32_t); # define jmpi(i0) _jmpi(_jit, i0) static jit_word_t _jmpi(jit_state_t*, jit_word_t); +# if __X64 +# define jmpi_p(i0) _jmpi_p(_jit, i0) +static jit_word_t _jmpi_p(jit_state_t*, jit_word_t); +# else +# define jmpi_p(i0) jmpi(i0) +# endif # define jmpsi(i0) _jmpsi(_jit, i0) -static void _jmpsi(jit_state_t*, jit_uint8_t); +static jit_word_t _jmpsi(jit_state_t*, jit_uint8_t); # define prolog(node) _prolog(_jit, node) static void _prolog(jit_state_t*, jit_node_t*); # define epilog(node) _epilog(_jit, node) @@ -677,8 +709,8 @@ static void _vastart(jit_state_t*, jit_int32_t); static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t); # define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0) static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t); -# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label) -static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t); +# define patch_at(instr, label) _patch_at(_jit, instr, label) +static void _patch_at(jit_state_t*, jit_word_t, jit_word_t); # if !defined(HAVE_FFSL) # if __X32 # define ffsl(i) __builtin_ffs(i) @@ -686,6 +718,7 @@ static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t); # define ffsl(l) __builtin_ffsl(l) # endif # endif +# define jit_cmov_p() jit_cpu.cmov #endif #if CODE @@ -713,11 +746,16 @@ _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md, { if (ri == _NOREG) { if (rb == _NOREG) { -#if __X32 - mrm(0x00, r7(rd), 0x05); -#else - mrm(0x00, r7(rd), 0x04); - sib(_SCL1, 0x04, 0x05); + /* Use ms == _SCL8 to tell it is a %rip relative displacement */ +#if __X64 + if (ms == _SCL8) +#endif + mrm(0x00, r7(rd), 0x05); +#if __X64 + else { + mrm(0x00, r7(rd), 0x04); + sib(_SCL1, 0x04, 0x05); + } #endif ii(md); } @@ -780,44 +818,49 @@ _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md, static void _nop(jit_state_t *_jit, jit_int32_t count) { - switch (count) { - case 0: - break; - case 1: /* NOP */ - ic(0x90); break; - case 2: /* 66 NOP */ - ic(0x66); ic(0x90); - break; - case 3: /* NOP DWORD ptr [EAX] */ - ic(0x0f); ic(0x1f); ic(0x00); - break; - case 4: /* NOP DWORD ptr [EAX + 00H] */ - ic(0x0f); ic(0x1f); ic(0x40); ic(0x00); - break; - case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */ - ic(0x0f); ic(0x1f); ic(0x44); ic(0x00); - ic(0x00); - break; - case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */ - ic(0x66); ic(0x0f); ic(0x1f); ic(0x44); - ic(0x00); ic(0x00); - break; - case 7: /* NOP DWORD ptr [EAX + 00000000H] */ - ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000); - break; - case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */ - ic(0x0f); ic(0x1f); ic(0x84); ic(0x00); - ii(0x0000); - break; - case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */ - ic(0x66); ic(0x0f); ic(0x1f); ic(0x84); - ic(0x00); ii(0x0000); - break; - default: - abort(); + jit_int32_t i; + while (count) { + if (count > 9) + i = 9; + else + i = count; + switch (i) { + case 0: + break; + case 1: /* NOP */ + ic(0x90); break; + case 2: /* 66 NOP */ + ic(0x66); ic(0x90); + break; + case 3: /* NOP DWORD ptr [EAX] */ + ic(0x0f); ic(0x1f); ic(0x00); + break; + case 4: /* NOP DWORD ptr [EAX + 00H] */ + ic(0x0f); ic(0x1f); ic(0x40); ic(0x00); + break; + case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */ + ic(0x0f); ic(0x1f); ic(0x44); ic(0x00); + ic(0x00); + break; + case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */ + ic(0x66); ic(0x0f); ic(0x1f); ic(0x44); + ic(0x00); ic(0x00); + break; + case 7: /* NOP DWORD ptr [EAX + 00000000H] */ + ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000); + break; + case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */ + ic(0x0f); ic(0x1f); ic(0x84); ic(0x00); + ii(0x0000); + break; + case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */ + ic(0x66); ic(0x0f); ic(0x1f); ic(0x84); + ic(0x00); ii(0x0000); + break; + } + count -= i; } } - static void _lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb, jit_int32_t ri, jit_int32_t ms, jit_int32_t rd) @@ -1009,6 +1052,49 @@ _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) } } +static void +_iaddxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + /* FIXME: this is not doing what I did expect for the simple test case: + * mov $0xffffffffffffffff, %rax -- rax = 0xffffffffffffffff (-1) + * mov $0xffffffffffffffff, %r10 -- r10 = 0xffffffffffffffff (-1) + * mov $0x1, %r11d -- r11 = 1 + * xor %rbx, %rbx -- rbx = 0 + * (gdb) p $eflags + * $1 = [ PF ZF IF ] + * add %r11, %rax -- r11 = 0x10000000000000000 (0) + * does not fit in 64 bit ^ + * (gdb) p $eflags + * $2 = [ CF PF AF ZF IF ] + * adcx %r10, %rbx -- r10 = 0xffffffffffffffff (-1) + * (gdb) p $eflags + * $3 = [ CF PF AF ZF IF ] + * (gdb) p/x $r10 + * $4 = 0xffffffffffffffff + * but, r10 should be zero, as it is: + * -1 (%r10) + 0 (%rbx) + carry (!!eflags.CF) + * FIXME: maybe should only use ADCX in the third operation onward, that + * is, after the first ADC? In either case, the add -1+0+carry should + * have used and consumed the carry? At least this is what is expected + * in Lightning... + */ +#if 0 + /* Significantly longer instruction, but avoid cpu stalls as only + * the carry flag is used in a sequence. */ + if (jit_cpu.adx) { + /* ADCX */ + ic(0x66); + rex(0, WIDE, r1, _NOREG, r0); + ic(0x0f); + ic(0x38); + ic(0xf6); + mrm(0x03, r7(r1), r7(r0)); + } + else +#endif + alur(X86_ADC, r0, r1); +} + static void _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) { @@ -1024,7 +1110,12 @@ static void _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { + if ( +#if 0 + /* Do not mix ADC and ADCX */ + !jit_cpu.adx && +#endif + can_sign_extend_int_p(i0)) { movr(r0, r1); iaddxi(r0, i0); } @@ -1886,6 +1977,88 @@ _decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } #endif +static void +_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + comr(r0, r1); + clzr(r0, r0); +} + +static void +_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w, x; + /* LZCNT */ + if (jit_cpu.abm) + ic(0xf3); + /* else BSR */ + rex(0, WIDE, r0, _NOREG, r1); + ic(0x0f); + ic(0xbd); + mrm(0x3, r7(r0), r7(r1)); + if (!jit_cpu.abm) { + /* jump if undefined: r1 == 0 */ + w = jccs(X86_CC_E, _jit->pc.w); + /* count leading zeros */ + rsbi(r0, r0, __WORDSIZE - 1); + /* done */ + x = jmpsi(_jit->pc.w); + /* if r1 == 0 */ + patch_at(w, _jit->pc.w); + movi(r0, __WORDSIZE); + /* not undefined */ + patch_at(x, _jit->pc.w); + } + /* LZCNT has defined behavior for value zero and count leading zeros */ +} + +static void +_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + comr(r0, r1); + ctzr(r0, r0); +} + +static void +_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_word_t w; + jit_int32_t t0; + if (!jit_cpu.abm) { + if (jit_cmov_p()) + t0 = jit_get_reg(jit_class_gpr|jit_class_nospill|jit_class_chk); + else + t0 = _NOREG; + if (t0 != _NOREG) + movi(rn(t0), __WORDSIZE); + } + /* TZCNT */ + if (jit_cpu.abm) + ic(0xf3); + /* else BSF */ + rex(0, WIDE, r0, _NOREG, r1); + ic(0x0f); + ic(0xbc); + mrm(0x3, r7(r0), r7(r1)); + if (!jit_cpu.abm) { + /* No conditional move or need spill/reload a temporary */ + if (t0 == _NOREG) { + w = jccs(X86_CC_E, _jit->pc.w); + movi(r0, __WORDSIZE); + patch_at(w, _jit->pc.w); + } + else { + /* CMOVE */ + rex(0, WIDE, r0, _NOREG, rn(t0)); + ic(0x0f); + ic(0x44); + mrm(0x3, r7(r0), r7(rn(t0))); + jit_unget_reg(t0); + } + } + /* TZCNT has defined behavior for value zero */ +} + static void _cr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) @@ -2135,6 +2308,12 @@ _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) ii(i0); # if !__X64_32 } + else if (can_sign_extend_int_p(i0)) { + rex(0, 1, _NOREG, _NOREG, r0); + ic(0xc7); + ic(0xc0 | r7(r0)); + ii(i0); + } else { rex(0, 1, _NOREG, _NOREG, r0); ic(0xb8 | r7(r0)); @@ -2147,22 +2326,45 @@ _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) #endif } +#if CAN_RIP_ADDRESS +static jit_word_t +#else static void +#endif _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { +#if CAN_RIP_ADDRESS + jit_word_t w, rel; + w = _jit->pc.w; + rel = i0 - (w + 8); + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + /* lea rel(%rip), %r0 */ + rex(0, WIDE, r0, _NOREG, _NOREG); + w = _jit->pc.w; + ic(0x8d); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif if (i0) imovi(r0, i0); else ixorr(r0, r0); +#if CAN_RIP_ADDRESS + return (w); +#endif } static jit_word_t _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { + jit_word_t w; rex(0, WIDE, _NOREG, _NOREG, r0); + w = _jit->pc.w; ic(0xb8 | r7(r0)); il(i0); - return (_jit->pc.w); + return (w); } static void @@ -2201,6 +2403,92 @@ _movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) mrm(0x03, r7(r0), r7(r1)); } +static void +_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, + jit_int32_t r2, jit_int32_t r3, jit_word_t i0) +{ + jit_int32_t save_rax, restore_rax; + jit_int32_t ascasr_reg, ascasr_use; + if (r0 != _RAX_REGNO) { /* result not in %rax */ + if (r2 != _RAX_REGNO) { /* old value not in %rax */ + save_rax = jit_get_reg(jit_class_gpr); + movr(rn(save_rax), _RAX_REGNO); + restore_rax = 1; + } + else + restore_rax = 0; + } + else + restore_rax = 0; + if (r2 != _RAX_REGNO) + movr(_RAX_REGNO, r2); + if (r1 == _NOREG) { /* using immediate address */ + if (!can_sign_extend_int_p(i0)) { + ascasr_reg = jit_get_reg(jit_class_gpr); + if (ascasr_reg == _RAX) { + ascasr_reg = jit_get_reg(jit_class_gpr); + jit_unget_reg(_RAX); + } + ascasr_use = 1; + movi(rn(ascasr_reg), i0); + } + else + ascasr_use = 0; + } + else + ascasr_use = 0; + ic(0xf0); /* lock */ + if (ascasr_use) + rex(0, WIDE, r3, _NOREG, rn(ascasr_reg)); + else + rex(0, WIDE, r3, _NOREG, r1); + ic(0x0f); + ic(0xb1); + if (r1 != _NOREG) /* casr */ + rx(r3, 0, r1, _NOREG, _SCL1); + else { /* casi */ + if (ascasr_use) + rx(r3, 0, rn(ascasr_reg), _NOREG, _SCL1); /* address in reg */ + else + rx(r3, i0, _NOREG, _NOREG, _SCL1); /* address in offset */ + } + cc(X86_CC_E, r0); + if (r0 != _RAX_REGNO) + movr(r0, _RAX_REGNO); + if (restore_rax) { + movr(_RAX_REGNO, rn(save_rax)); + jit_unget_reg(save_rax); + } + if (ascasr_use) + jit_unget_reg(ascasr_reg); +} + +static void +_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + assert(jit_cmov_p()); + + testr(r2, r2); + + rex(0, WIDE, r0, _NOREG, r1); + ic(0x0f); + ic(0x45); + mrm(0x03, r7(r0), r7(r1)); +} + +static void +_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) +{ + assert(jit_cmov_p()); + + testr(r2, r2); + + rex(0, WIDE, r0, _NOREG, r1); + ic(0x0f); + ic(0x44); + mrm(0x03, r7(r0), r7(r1)); +} + #if __X64 static void _movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) @@ -2220,7 +2508,7 @@ _movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) #endif static void -_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { extr_us(r0, r1); ic(0x66); @@ -2231,7 +2519,7 @@ _htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) } static void -_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { movr(r0, r1); rex(0, 0, _NOREG, _NOREG, r0); @@ -2241,7 +2529,7 @@ _htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) #if __X64 && !__X64_32 static void -_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { movr(r0, r1); rex(0, 1, _NOREG, _NOREG, r0); @@ -2291,7 +2579,18 @@ static void _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xbe); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x0f); ic(0xbe); @@ -2318,7 +2617,18 @@ static void _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xb6); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x0f); ic(0xb6); @@ -2345,7 +2655,18 @@ static void _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xbf); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x0f); ic(0xbf); @@ -2372,7 +2693,18 @@ static void _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x0f); + ic(0xb7); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x0f); ic(0xb7); @@ -2403,7 +2735,17 @@ static void _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x63); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { #if __X64 rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x63); @@ -2434,7 +2776,17 @@ static void _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +# if !__X64_32 + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x63); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { rex(0, 0, r0, _NOREG, _NOREG); ic(0x63); rx(r0, i0, _NOREG, _NOREG, _SCL1); @@ -2442,7 +2794,11 @@ _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); +# if __X64_32 + ldr_i(r0, rn(reg)); +# else ldr_ui(r0, rn(reg)); +# endif jit_unget_reg(reg); } } @@ -2460,8 +2816,15 @@ static void _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { - rex(0, 1, r0, _NOREG, _NOREG); + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x8b); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else if (can_sign_extend_int_p(i0)) { + rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x8b); rx(r0, i0, _NOREG, _NOREG, _SCL1); } @@ -2665,7 +3028,11 @@ _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0) else { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), i0); +# if __X64_32 + ldxr_i(r0, r1, rn(reg)); +# else ldxr_ui(r0, r1, rn(reg)); +# endif jit_unget_reg(reg); } } @@ -2721,7 +3088,27 @@ static void _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 16 : rel + 16; + if (can_sign_extend_int_p(rel)) { + if (reg8_p(r0)) { + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x88); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else { + reg = jit_get_reg(jit_class_gpr|jit_class_rg8); + movr(rn(reg), r0); + rex(0, 0, rn(reg), _NOREG, _NOREG); + ic(0x88); + rx(rn(reg), i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + jit_unget_reg(reg); + } + } + else +#endif + if (address_p(i0)) { if (reg8_p(r0)) { rex(0, 0, r0, _NOREG, _NOREG); ic(0x88); @@ -2757,7 +3144,18 @@ static void _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + ic(0x66); + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x89); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { ic(0x66); rex(0, 0, r0, _NOREG, _NOREG); ic(0x89); @@ -2783,7 +3181,17 @@ static void _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; - if (can_sign_extend_int_p(i0)) { +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, 0, r0, _NOREG, _NOREG); + ic(0x89); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif + if (address_p(i0)) { rex(0, 0, r0, _NOREG, _NOREG); ic(0x89); rx(r0, i0, _NOREG, _NOREG, _SCL1); @@ -2809,8 +3217,18 @@ static void _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - _jit->pc.w; + rel = rel < 0 ? rel - 8 : rel + 8; + if (can_sign_extend_int_p(rel)) { + rex(0, WIDE, r0, _NOREG, _NOREG); + ic(0x89); + rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8); + } + else +#endif if (can_sign_extend_int_p(i0)) { - rex(0, 1, r0, _NOREG, _NOREG); + rex(0, WIDE, r0, _NOREG, _NOREG); ic(0x89); rx(r0, i0, _NOREG, _NOREG, _SCL1); } @@ -2971,208 +3389,221 @@ _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) } #endif -static void +static jit_word_t _jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0) { + jit_word_t d; jit_word_t w; + w = _jit->pc.w; + d = i0 - (w + 1); ic(0x70 | code); - w = i0 - (_jit->pc.w + 1); - ic(w); + ic(d); + return (w); } -static void +static jit_word_t _jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0) { + jit_word_t d; jit_word_t w; + w = _jit->pc.w; ic(0x0f); + d = i0 - (w + 6); ic(0x80 | code); - w = i0 - (_jit->pc.w + 4); - ii(w); + ii(d); + return (w); } -static void +static jit_word_t _jcr(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { alur(X86_CMP, r0, r1); - jcc(code, i0); + return (jcc(code, i0)); } -static void +static jit_word_t _jci(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { alui(X86_CMP, r0, i1); - jcc(code, i0); + return (jcc(code, i0)); } -static void +static jit_word_t _jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0) { testr(r0, r0); - jcc(code, i0); + return (jcc(code, i0)); } static jit_word_t _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - jcr(X86_CC_L, i0, r0, r1); - return (_jit->pc.w); + return (jcr(X86_CC_L, i0, r0, r1)); } static jit_word_t _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_L, i0, r0, i1); - else jci0(X86_CC_S, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_L, i0, r0, i1); + else w = jci0(X86_CC_S, i0, r0); + return (w); } static jit_word_t _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - jcr(X86_CC_B, i0, r0, r1); - return (_jit->pc.w); + return (jcr(X86_CC_B, i0, r0, r1)); } static jit_word_t _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_B, i0, r0, i1); - else jci0(X86_CC_B, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_B, i0, r0, i1); + else w = jci0(X86_CC_B, i0, r0); + return (w); } static jit_word_t _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - if (r0 == r1) jmpi(i0); - else jcr (X86_CC_LE, i0, r0, r1); - return (_jit->pc.w); + jit_word_t w; + if (r0 == r1) w = jmpi(i0); + else w = jcr (X86_CC_LE, i0, r0, r1); + return (w); } static jit_word_t _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_LE, i0, r0, i1); - else jci0(X86_CC_LE, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_LE, i0, r0, i1); + else w = jci0(X86_CC_LE, i0, r0); + return (w); } static jit_word_t _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - if (r0 == r1) jmpi(i0); - else jcr (X86_CC_BE, i0, r0, r1); - return (_jit->pc.w); + jit_word_t w; + if (r0 == r1) w = jmpi(i0); + else w = jcr (X86_CC_BE, i0, r0, r1); + return (w); } static jit_word_t _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_BE, i0, r0, i1); - else jci0(X86_CC_BE, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_BE, i0, r0, i1); + else w = jci0(X86_CC_BE, i0, r0); + return (w); } static jit_word_t _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - if (r0 == r1) jmpi(i0); - else jcr (X86_CC_E, i0, r0, r1); - return (_jit->pc.w); + jit_word_t w; + if (r0 == r1) w = jmpi(i0); + else w = jcr (X86_CC_E, i0, r0, r1); + return (w); } static jit_word_t _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_E, i0, r0, i1); - else jci0(X86_CC_E, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_E, i0, r0, i1); + else w = jci0(X86_CC_E, i0, r0); + return (w); } static jit_word_t _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - if (r0 == r1) jmpi(i0); - else jcr (X86_CC_GE, i0, r0, r1); - return (_jit->pc.w); + jit_word_t w; + if (r0 == r1) w = jmpi(i0); + else w = jcr (X86_CC_GE, i0, r0, r1); + return (w); } static jit_word_t _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_GE, i0, r0, i1); - else jci0(X86_CC_NS, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_GE, i0, r0, i1); + else w = jci0(X86_CC_NS, i0, r0); + return (w); } static jit_word_t _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - if (r0 == r1) jmpi(i0); - else jcr (X86_CC_AE, i0, r0, r1); - return (_jit->pc.w); + jit_word_t w; + if (r0 == r1) w = jmpi(i0); + else w = jcr (X86_CC_AE, i0, r0, r1); + return (w); } static jit_word_t _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_AE, i0, r0, i1); - else jmpi(i0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_AE, i0, r0, i1); + else w = jmpi(i0); + return (w); } static jit_word_t _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - jcr(X86_CC_G, i0, r0, r1); - return (_jit->pc.w); + return (jcr(X86_CC_G, i0, r0, r1)); } static jit_word_t _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - jci(X86_CC_G, i0, r0, i1); - return (_jit->pc.w); + return (jci(X86_CC_G, i0, r0, i1)); } static jit_word_t _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - jcr(X86_CC_A, i0, r0, r1); - return (_jit->pc.w); + return (jcr(X86_CC_A, i0, r0, r1)); } static jit_word_t _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_A, i0, r0, i1); - else jci0(X86_CC_NE, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_A, i0, r0, i1); + else w = jci0(X86_CC_NE, i0, r0); + return (w); } static jit_word_t _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { - jcr(X86_CC_NE, i0, r0, r1); - return (_jit->pc.w); + return (jcr(X86_CC_NE, i0, r0, r1)); } static jit_word_t _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) { - if (i1) jci (X86_CC_NE, i0, r0, i1); - else jci0(X86_CC_NE, i0, r0); - return (_jit->pc.w); + jit_word_t w; + if (i1) w = jci (X86_CC_NE, i0, r0, i1); + else w = jci0(X86_CC_NE, i0, r0); + return (w); } static jit_word_t _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { testr(r0, r1); - jnz(i0); - return (_jit->pc.w); + return (jnz(i0)); } static jit_word_t @@ -3187,16 +3618,14 @@ _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) testr(r0, rn(reg)); jit_unget_reg(reg); } - jnz(i0); - return (_jit->pc.w); + return (jnz(i0)); } static jit_word_t _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { testr(r0, r1); - jz(i0); - return (_jit->pc.w); + return (jz(i0)); } static jit_word_t @@ -3211,16 +3640,14 @@ _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) testr(r0, rn(reg)); jit_unget_reg(reg); } - jz(i0); - return (_jit->pc.w); + return (jz(i0)); } static jit_word_t _boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { iaddr(r0, r1); - jo(i0); - return (_jit->pc.w); + return (jo(i0)); } static jit_word_t @@ -3229,8 +3656,7 @@ _boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { iaddi(r0, i1); - jo(i0); - return (_jit->pc.w); + return (jo(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3242,8 +3668,7 @@ static jit_word_t _boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { iaddr(r0, r1); - jc(i0); - return (_jit->pc.w); + return (jc(i0)); } static jit_word_t @@ -3252,8 +3677,7 @@ _boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { iaddi(r0, i1); - jc(i0); - return (_jit->pc.w); + return (jc(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3265,8 +3689,7 @@ static jit_word_t _bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { iaddr(r0, r1); - jno(i0); - return (_jit->pc.w); + return (jno(i0)); } static jit_word_t @@ -3275,8 +3698,7 @@ _bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { iaddi(r0, i1); - jno(i0); - return (_jit->pc.w); + return (jno(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3288,8 +3710,7 @@ static jit_word_t _bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { iaddr(r0, r1); - jnc(i0); - return (_jit->pc.w); + return (jnc(i0)); } static jit_word_t @@ -3298,8 +3719,7 @@ _bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { iaddi(r0, i1); - jnc(i0); - return (_jit->pc.w); + return (jnc(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3311,8 +3731,7 @@ static jit_word_t _bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { isubr(r0, r1); - jo(i0); - return (_jit->pc.w); + return (jo(i0)); } static jit_word_t @@ -3321,8 +3740,7 @@ _bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { isubi(r0, i1); - jo(i0); - return (_jit->pc.w); + return (jo(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3334,8 +3752,7 @@ static jit_word_t _bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { isubr(r0, r1); - jc(i0); - return (_jit->pc.w); + return (jc(i0)); } static jit_word_t @@ -3344,8 +3761,7 @@ _bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { isubi(r0, i1); - jc(i0); - return (_jit->pc.w); + return (jc(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3357,8 +3773,7 @@ static jit_word_t _bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { isubr(r0, r1); - jno(i0); - return (_jit->pc.w); + return (jno(i0)); } static jit_word_t @@ -3367,8 +3782,7 @@ _bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { isubi(r0, i1); - jno(i0); - return (_jit->pc.w); + return (jno(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3380,8 +3794,7 @@ static jit_word_t _bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { isubr(r0, r1); - jnc(i0); - return (_jit->pc.w); + return (jnc(i0)); } static jit_word_t @@ -3390,8 +3803,7 @@ _bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1) jit_int32_t reg; if (can_sign_extend_int_p(i1)) { isubi(r0, i1); - jnc(i0); - return (_jit->pc.w); + return (jnc(i0)); } reg = jit_get_reg(jit_class_gpr|jit_class_nospill); movi(rn(reg), i1); @@ -3410,28 +3822,46 @@ _callr(jit_state_t *_jit, jit_int32_t r0) static jit_word_t _calli(jit_state_t *_jit, jit_word_t i0) { - jit_word_t word; + jit_word_t w; + jit_word_t d; + jit_word_t l = _jit->pc.w + 5; + d = i0 - l; #if __X64 - jit_int32_t reg; + if ( +# if __X64_32 + !((d < 0) ^ (l < 0)) && +# endif + (jit_int32_t)d == d) { +#endif + w = _jit->pc.w; + ic(0xe8); + ii(d); +#if __X64 + } + else + w = calli_p(i0); +#endif + return (w); +} +#if __X64 +static jit_word_t +_calli_p(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t w; + jit_int32_t reg; reg = jit_get_reg(jit_class_gpr); - word = movi_p(rn(reg), i0); + w = movi_p(rn(reg), i0); callr(rn(reg)); jit_unget_reg(reg); -#else - jit_word_t w; - ic(0xe8); - w = i0 - (_jit->pc.w + 4); - ii(w); - word = _jit->pc.w; -#endif - return (word); + return (w); } +#endif static void _jmpr(jit_state_t *_jit, jit_int32_t r0) { - rex(0, WIDE, _NOREG, _NOREG, r0); + rex(0, 0, _NOREG, _NOREG, r0); ic(0xff); mrm(0x03, 0x04, r7(r0)); } @@ -3440,25 +3870,57 @@ static jit_word_t _jmpi(jit_state_t *_jit, jit_word_t i0) { jit_word_t w; - ic(0xe9); - w = i0 - (_jit->pc.w + 4); - ii(w); - return (_jit->pc.w); + jit_word_t d; + jit_word_t l = _jit->pc.w + 5; + d = i0 - l; +#if __X64 + if ( +# if __X64_32 + !((d < 0) ^ (l < 0)) && +# endif + (jit_int32_t)d == d) { +#endif + w = _jit->pc.w; + ic(0xe9); + ii(d); +#if __X64 + } + else + w = jmpi_p(i0); +#endif + return (w); } -static void +#if __X64 +static jit_word_t +_jmpi_p(jit_state_t *_jit, jit_word_t i0) +{ + jit_word_t w; + jit_int32_t reg; + reg = jit_get_reg(jit_class_gpr|jit_class_nospill); + w = movi_p(rn(reg), i0); + jmpr(rn(reg)); + jit_unget_reg(reg); + return (w); +} +#endif + +static jit_word_t _jmpsi(jit_state_t *_jit, jit_uint8_t i0) { + jit_word_t w = _jit->pc.w; ic(0xeb); ic(i0); + return (w); } static void _prolog(jit_state_t *_jit, jit_node_t *node) { - jit_int32_t reg; + jit_int32_t reg, offs; if (_jitc->function->define_frame || _jitc->function->assume_frame) { jit_int32_t frame = -_jitc->function->frame; + jit_check_frame(); assert(_jitc->function->self.aoff >= frame); if (_jitc->function->assume_frame) return; @@ -3471,76 +3933,51 @@ _prolog(jit_state_t *_jit, jit_node_t *node) (_jitc->function->self.alen > 32 ? _jitc->function->self.alen : 32) - /* align stack at 16 bytes */ - _jitc->function->self.aoff) + 15) & -16) + - stack_adjust; + _jitc->function->self.aoff) + 15) & -16); #else _jitc->function->stack = (((_jitc->function->self.alen - - _jitc->function->self.aoff) + 15) & -16) + - stack_adjust; + _jitc->function->self.aoff) + 15) & -16); #endif - subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE); + + if (_jitc->function->stack) + _jitc->function->need_stack = 1; + + if (!_jitc->function->need_frame && !_jitc->function->need_stack) { + /* check if any callee save register needs to be saved */ + for (reg = 0; reg < _jitc->reglen; ++reg) + if (jit_regset_tstbit(&_jitc->function->regset, reg) && + (_rvs[reg].spec & jit_class_sav)) { + _jitc->function->need_stack = 1; + break; + } + } + + if (_jitc->function->need_frame || _jitc->function->need_stack) + subi(_RSP_REGNO, _RSP_REGNO, jit_framesize()); /* callee save registers */ -#if __X32 - if (jit_regset_tstbit(&_jitc->function->regset, _RDI)) - stxi(12, _RSP_REGNO, _RDI_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _RSI)) - stxi( 8, _RSP_REGNO, _RSI_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _RBX)) - stxi( 4, _RSP_REGNO, _RBX_REGNO); -#else -# if __CYGWIN__ || _WIN32 - if (jit_regset_tstbit(&_jitc->function->regset, _XMM15)) - sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM14)) - sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM13)) - sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM12)) - sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM11)) - sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM10)) - sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM9)) - sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM8)) - sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM7)) - sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM6)) - sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R15)) - stxi(56, _RSP_REGNO, _R15_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R14)) - stxi(48, _RSP_REGNO, _R14_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R13)) - stxi(40, _RSP_REGNO, _R13_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R12)) - stxi(32, _RSP_REGNO, _R12_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _RSI)) - stxi(24, _RSP_REGNO, _RSI_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _RDI)) - stxi(16, _RSP_REGNO, _RDI_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _RBX)) - stxi( 8, _RSP_REGNO, _RBX_REGNO); -# else - if (jit_regset_tstbit(&_jitc->function->regset, _RBX)) - stxi(40, _RSP_REGNO, _RBX_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R12)) - stxi(32, _RSP_REGNO, _R12_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R13)) - stxi(24, _RSP_REGNO, _R13_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R14)) - stxi(16, _RSP_REGNO, _R14_REGNO); - if (jit_regset_tstbit(&_jitc->function->regset, _R15)) - stxi( 8, _RSP_REGNO, _R15_REGNO); -# endif + for (reg = 0, offs = REAL_WORDSIZE; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + stxi(offs, _RSP_REGNO, rn(iregs[reg])); + offs += REAL_WORDSIZE; + } + } +#if __X64 && (__CYGWIN__ || _WIN32) + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + sse_stxi_d(offs, _RSP_REGNO, rn(fregs[reg])); + offs += sizeof(jit_float64_t); + } + } #endif - stxi(0, _RSP_REGNO, _RBP_REGNO); - movr(_RBP_REGNO, _RSP_REGNO); + + if (_jitc->function->need_frame) { + stxi(0, _RSP_REGNO, _RBP_REGNO); + movr(_RBP_REGNO, _RSP_REGNO); + } /* alloca */ - subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack); + if (_jitc->function->stack) + subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack); if (_jitc->function->allocar) { reg = jit_get_reg(jit_class_gpr); movi(rn(reg), _jitc->function->self.aoff); @@ -3564,8 +4001,7 @@ _prolog(jit_state_t *_jit, jit_node_t *node) /* test %al, %al */ ic(0x84); ic(0xc0); - jes(0); - nofp_code = _jit->pc.w; + nofp_code = jes(0); /* Save fp registers in the save area, if any is a vararg */ /* Note that the full 16 byte xmm is not saved, because @@ -3576,7 +4012,7 @@ _prolog(jit_state_t *_jit, jit_node_t *node) sse_stxi_d(_jitc->function->vaoff + first_fp_offset + reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg)); - patch_rel_char(nofp_code, _jit->pc.w); + patch_at(nofp_code, _jit->pc.w); } } #endif @@ -3585,68 +4021,38 @@ _prolog(jit_state_t *_jit, jit_node_t *node) static void _epilog(jit_state_t *_jit, jit_node_t *node) { + jit_int32_t reg, offs; if (_jitc->function->assume_frame) return; + if (_jitc->function->need_frame) + movr(_RSP_REGNO, _RBP_REGNO); + /* callee save registers */ - movr(_RSP_REGNO, _RBP_REGNO); -#if __X32 - if (jit_regset_tstbit(&_jitc->function->regset, _RDI)) - ldxi(_RDI_REGNO, _RSP_REGNO, 12); - if (jit_regset_tstbit(&_jitc->function->regset, _RSI)) - ldxi(_RSI_REGNO, _RSP_REGNO, 8); - if (jit_regset_tstbit(&_jitc->function->regset, _RBX)) - ldxi(_RBX_REGNO, _RSP_REGNO, 4); -#else -# if __CYGWIN__ || _WIN32 - if (jit_regset_tstbit(&_jitc->function->regset, _XMM15)) - sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM14)) - sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM13)) - sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM12)) - sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM11)) - sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM10)) - sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM9)) - sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM8)) - sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM7)) - sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72); - if (jit_regset_tstbit(&_jitc->function->regset, _XMM6)) - sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64); - if (jit_regset_tstbit(&_jitc->function->regset, _R15)) - ldxi(_R15_REGNO, _RSP_REGNO, 56); - if (jit_regset_tstbit(&_jitc->function->regset, _R14)) - ldxi(_R14_REGNO, _RSP_REGNO, 48); - if (jit_regset_tstbit(&_jitc->function->regset, _R13)) - ldxi(_R13_REGNO, _RSP_REGNO, 40); - if (jit_regset_tstbit(&_jitc->function->regset, _R12)) - ldxi(_R12_REGNO, _RSP_REGNO, 32); - if (jit_regset_tstbit(&_jitc->function->regset, _RSI)) - ldxi(_RSI_REGNO, _RSP_REGNO, 24); - if (jit_regset_tstbit(&_jitc->function->regset, _RDI)) - ldxi(_RDI_REGNO, _RSP_REGNO, 16); - if (jit_regset_tstbit(&_jitc->function->regset, _RBX)) - ldxi(_RBX_REGNO, _RSP_REGNO, 8); -# else - if (jit_regset_tstbit(&_jitc->function->regset, _RBX)) - ldxi(_RBX_REGNO, _RSP_REGNO, 40); - if (jit_regset_tstbit(&_jitc->function->regset, _R12)) - ldxi(_R12_REGNO, _RSP_REGNO, 32); - if (jit_regset_tstbit(&_jitc->function->regset, _R13)) - ldxi(_R13_REGNO, _RSP_REGNO, 24); - if (jit_regset_tstbit(&_jitc->function->regset, _R14)) - ldxi(_R14_REGNO, _RSP_REGNO, 16); - if (jit_regset_tstbit(&_jitc->function->regset, _R15)) - ldxi(_R15_REGNO, _RSP_REGNO, 8); -# endif + for (reg = 0, offs = REAL_WORDSIZE; reg < jit_size(iregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) { + ldxi(rn(iregs[reg]), _RSP_REGNO, offs); + offs += REAL_WORDSIZE; + } + } +#if __X64 && (__CYGWIN__ || _WIN32) + for (reg = 0; reg < jit_size(fregs); reg++) { + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) { + sse_ldxi_d(rn(fregs[reg]), _RSP_REGNO, offs); + offs += sizeof(jit_float64_t); + } + } #endif - ldxi(_RBP_REGNO, _RSP_REGNO, 0); - addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE); + + if (_jitc->function->need_frame) { + ldxi(_RBP_REGNO, _RSP_REGNO, 0); + addi(_RSP_REGNO, _RSP_REGNO, jit_framesize()); + } + /* This condition does not happen as much as expected because + * it is not safe to not create a frame pointer if any function + * is called, even jit functions, as those might call external + * functions. */ + else if (_jitc->function->need_stack) + addi(_RSP_REGNO, _RSP_REGNO, jit_framesize()); ic(0xc3); } @@ -3656,7 +4062,7 @@ _vastart(jit_state_t *_jit, jit_int32_t r0) { #if __X32 || __CYGWIN__ || _WIN32 assert(_jitc->function->self.call & jit_call_varargs); - addi(r0, _RBP_REGNO, _jitc->function->self.size); + addi(r0, _RBP_REGNO, jit_selfsize()); #else jit_int32_t reg; @@ -3675,7 +4081,7 @@ _vastart(jit_state_t *_jit, jit_int32_t r0) stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg)); /* Initialize overflow pointer to the first stack argument. */ - addi(rn(reg), _RBP_REGNO, _jitc->function->self.size); + addi(rn(reg), _RBP_REGNO, jit_selfsize()); stxi(offsetof(jit_va_list_t, over), r0, rn(reg)); /* Initialize register save area pointer. */ @@ -3709,8 +4115,7 @@ _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) /* Jump over if there are no remaining arguments in the save area. */ icmpi(rn(rg0), va_gp_max_offset); - jaes(0); - ge_code = _jit->pc.w; + ge_code = jaes(0); /* Load the save area pointer in the second temporary. */ ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save)); @@ -3726,11 +4131,10 @@ _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) jit_unget_reg(rg1); /* Jump over overflow code. */ - jmpsi(0); - lt_code = _jit->pc.w; + lt_code = jmpsi(0); /* Where to land if argument is in overflow area. */ - patch_rel_char(ge_code, _jit->pc.w); + patch_at(ge_code, _jit->pc.w); /* Load overflow pointer. */ ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over)); @@ -3743,7 +4147,7 @@ _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) stxi(offsetof(jit_va_list_t, over), r1, rn(rg0)); /* Where to land if argument is in save area. */ - patch_rel_char(lt_code, _jit->pc.w); + patch_at(lt_code, _jit->pc.w); jit_unget_reg(rg0); #endif @@ -3777,8 +4181,7 @@ _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87) /* Jump over if there are no remaining arguments in the save area. */ icmpi(rn(rg0), va_fp_max_offset); - jaes(0); - ge_code = _jit->pc.w; + ge_code = jaes(0); /* Load the save area pointer in the second temporary. */ ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save)); @@ -3797,11 +4200,10 @@ _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87) jit_unget_reg(rg1); /* Jump over overflow code. */ - jmpsi(0); - lt_code = _jit->pc.w; + lt_code = jmpsi(0); /* Where to land if argument is in overflow area. */ - patch_rel_char(ge_code, _jit->pc.w); + patch_at(ge_code, _jit->pc.w); /* Load overflow pointer. */ ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over)); @@ -3817,26 +4219,57 @@ _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87) stxi(offsetof(jit_va_list_t, over), r1, rn(rg0)); /* Where to land if argument is in save area. */ - patch_rel_char(lt_code, _jit->pc.w); + patch_at(lt_code, _jit->pc.w); jit_unget_reg(rg0); #endif } static void -_patch_at(jit_state_t *_jit, jit_node_t *node, - jit_word_t instr, jit_word_t label) +_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label) { - switch (node->code) { -# if __X64 - case jit_code_calli: -# endif - case jit_code_movi: - patch_abs(instr, label); + jit_word_t disp; + jit_uint8_t *code = (jit_uint8_t *)instr; + ++instr; + switch (code[0]) { + /* movi_p */ + case 0xb8 ... 0xbf: + *(jit_word_t *)instr = label; + break; + /* forward pc relative address known to be in range */ +#if CAN_RIP_ADDRESS + /* movi */ + case 0x8d: + ++instr; + goto apply; +#endif + /* jcc */ + case 0x0f: + ++instr; + if (code[1] < 0x80 || code[1] > 0x8f) + goto fail; + /* calli */ + case 0xe8: + /* jmpi */ + case 0xe9: +#if CAN_RIP_ADDRESS + apply: +#endif + disp = label - (instr + 4); + assert((jit_int32_t)disp == disp); + *(jit_int32_t *)instr = disp; + break; + /* jccs */ + case 0x70 ... 0x7f: + /* jmpsi */ + case 0xeb: + disp = label - (instr + 1); + assert((jit_int8_t)disp == disp); + *(jit_int8_t *)instr = disp; break; default: - patch_rel(instr, label); - break; + fail: + abort(); } } #endif diff --git a/parallel-rsp/lightning/lib/jit_x86-sse.c b/parallel-rsp/lightning/lib/jit_x86-sse.c index d09bda9..c3ac895 100644 --- a/parallel-rsp/lightning/lib/jit_x86-sse.c +++ b/parallel-rsp/lightning/lib/jit_x86-sse.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -18,15 +18,6 @@ */ #if PROTO -# if __X32 -# define sse_address_p(i0) 1 -# else -# if __X64_32 -# define sse_address_p(i0) ((jit_word_t)(i0) >= 0) -# else -# define sse_address_p(i0) can_sign_extend_int_p(i0) -# endif -# endif # define _XMM6_REGNO 6 # define _XMM7_REGNO 7 # define _XMM8_REGNO 8 @@ -470,14 +461,14 @@ _sse_b##name##i_##type(jit_state_t *_jit, \ jit_word_t i0, jit_int32_t r0, \ jit_float##size##_t *i1) \ { \ - jit_word_t word; \ + jit_word_t w; \ jit_int32_t reg = jit_get_reg(jit_class_fpr|jit_class_xpr| \ jit_class_nospill); \ assert(jit_sse_reg_p(reg)); \ sse_movi_##type(rn(reg), i1); \ - word = sse_b##name##r_##type(i0, r0, rn(reg)); \ + w = sse_b##name##r_##type(i0, r0, rn(reg)); \ jit_unget_reg(reg); \ - return (word); \ + return (w); \ } # define fopi(name) fpr_opi(name, f, 32) # define fbopi(name) fpr_bopi(name, f, 32) @@ -809,8 +800,17 @@ _sse_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t *i0) ldi = !_jitc->no_data; #if __X64 /* if will allocate a register for offset, just use immediate */ - if (ldi && !sse_address_p(i0)) +# if CAN_RIP_ADDRESS + if (ldi) { + jit_word_t rel = (jit_word_t)i0 - (_jit->pc.w + 8 + !!(r0 & 8)); + ldi = can_sign_extend_int_p(rel); + if (!ldi && address_p(i0)) + ldi = 1; + } +# else + if (ldi && !address_p(i0)) ldi = 0; +# endif #endif if (ldi) sse_ldi_f(r0, (jit_word_t)i0); @@ -840,10 +840,9 @@ _sse_eqr_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) } ixorr(reg, reg); ucomissr(r2, r1); - jpes(0); - jp_code = _jit->pc.w; + jp_code = jpes(0); cc(X86_CC_E, reg); - patch_rel_char(jp_code, _jit->pc.w); + patch_at(jp_code, _jit->pc.w); if (!rc) xchgr(r0, reg); } @@ -866,10 +865,9 @@ _sse_ner_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) } imovi(reg, 1); ucomissr(r2, r1); - jpes(0); - jp_code = _jit->pc.w; + jp_code = jpes(0); cc(X86_CC_NE, reg); - patch_rel_char(jp_code, _jit->pc.w); + patch_at(jp_code, _jit->pc.w); if (!rc) xchgr(r0, reg); } @@ -928,7 +926,13 @@ static void _sse_ldi_f(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (sse_address_p(i0)) +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - (_jit->pc.w + 8 + !!(r0 & 8)); + if (can_sign_extend_int_p(rel)) + movssmr(rel, _NOREG, _NOREG, _SCL8, r0); + else +#endif + if (address_p(i0)) movssmr(i0, _NOREG, _NOREG, _SCL1, r0); else { reg = jit_get_reg(jit_class_gpr); @@ -975,7 +979,13 @@ static void _sse_sti_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; - if (sse_address_p(i0)) +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - (_jit->pc.w + 8 + !!(r0 & 8)); + if (can_sign_extend_int_p(rel)) + movssrm(r0, rel, _NOREG, _NOREG, _SCL8); + else +#endif + if (address_p(i0)) movssrm(r0, i0, _NOREG, _NOREG, _SCL1); else { reg = jit_get_reg(jit_class_gpr); @@ -1022,8 +1032,7 @@ static jit_word_t _sse_bltr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r1, r0); - ja(i0); - return (_jit->pc.w); + return (ja(i0)); } fbopi(lt) @@ -1031,21 +1040,20 @@ static jit_word_t _sse_bler_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r1, r0); - jae(i0); - return (_jit->pc.w); + return (jae(i0)); } fbopi(le) static jit_word_t _sse_beqr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; jit_word_t jp_code; ucomissr(r0, r1); - jps(0); - jp_code = _jit->pc.w; - je(i0); - patch_rel_char(jp_code, _jit->pc.w); - return (_jit->pc.w); + jp_code = jps(0); + w = je(i0); + patch_at(jp_code, _jit->pc.w); + return (w); } fbopi(eq) @@ -1053,8 +1061,7 @@ static jit_word_t _sse_bger_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r0, r1); - jae(i0); - return (_jit->pc.w); + return (jae(i0)); } fbopi(ge) @@ -1062,25 +1069,23 @@ static jit_word_t _sse_bgtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r0, r1); - ja(i0); - return (_jit->pc.w); + return (ja(i0)); } fbopi(gt) static jit_word_t _sse_bner_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; jit_word_t jp_code; jit_word_t jz_code; ucomissr(r0, r1); - jps(0); - jp_code = _jit->pc.w; - jzs(0); - jz_code = _jit->pc.w; - patch_rel_char(jp_code, _jit->pc.w); - jmpi(i0); - patch_rel_char(jz_code, _jit->pc.w); - return (_jit->pc.w); + jp_code = jps(0); + jz_code = jzs(0); + patch_at(jp_code, _jit->pc.w); + w = jmpi(i0); + patch_at(jz_code, _jit->pc.w); + return (w); } fbopi(ne) @@ -1088,47 +1093,49 @@ static jit_word_t _sse_bunltr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r0, r1); - jnae(i0); - return (_jit->pc.w); + return (jnae(i0)); } fbopi(unlt) static jit_word_t _sse_bunler_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; if (r0 == r1) - jmpi(i0); + w = jmpi(i0); else { ucomissr(r0, r1); - jna(i0); + w = jna(i0); } - return (_jit->pc.w); + return (w); } fbopi(unle) static jit_word_t _sse_buneqr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; if (r0 == r1) - jmpi(i0); + w = jmpi(i0); else { ucomissr(r0, r1); - je(i0); + w = je(i0); } - return (_jit->pc.w); + return (w); } fbopi(uneq) static jit_word_t _sse_bunger_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; if (r0 == r1) - jmpi(i0); + w = jmpi(i0); else { ucomissr(r1, r0); - jna(i0); + w = jna(i0); } - return (_jit->pc.w); + return (w); } fbopi(unge) @@ -1136,8 +1143,7 @@ static jit_word_t _sse_bungtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r1, r0); - jnae(i0); - return (_jit->pc.w); + return (jnae(i0)); } fbopi(ungt) @@ -1145,8 +1151,7 @@ static jit_word_t _sse_bltgtr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r0, r1); - jne(i0); - return (_jit->pc.w); + return (jne(i0)); } fbopi(ltgt) @@ -1154,8 +1159,7 @@ static jit_word_t _sse_bordr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r0, r1); - jnp(i0); - return (_jit->pc.w); + return (jnp(i0)); } fbopi(ord) @@ -1163,8 +1167,7 @@ static jit_word_t _sse_bunordr_f(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomissr(r0, r1); - jp(i0); - return (_jit->pc.w); + return (jp(i0)); } fbopi(unord) @@ -1185,10 +1188,9 @@ _sse_eqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) } ixorr(reg, reg); ucomisdr(r2, r1); - jpes(0); - jp_code = _jit->pc.w; + jp_code = jpes(0); cc(X86_CC_E, reg); - patch_rel_char(jp_code, _jit->pc.w); + patch_at(jp_code, _jit->pc.w); if (!rc) xchgr(r0, reg); } @@ -1211,10 +1213,9 @@ _sse_ner_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) } imovi(reg, 1); ucomisdr(r2, r1); - jpes(0); - jp_code = _jit->pc.w; + jp_code = jpes(0); cc(X86_CC_NE, reg); - patch_rel_char(jp_code, _jit->pc.w); + patch_at(jp_code, _jit->pc.w); if (!rc) xchgr(r0, reg); } @@ -1294,8 +1295,17 @@ _sse_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t *i0) ldi = !_jitc->no_data; #if __X64 /* if will allocate a register for offset, just use immediate */ - if (ldi && !sse_address_p(i0)) +# if CAN_RIP_ADDRESS + if (ldi) { + jit_word_t rel = (jit_word_t)i0 - (_jit->pc.w + 8 + !!(r0 & 8)); + ldi = can_sign_extend_int_p(rel); + if (!ldi && address_p(i0)) + ldi = 1; + } +# else + if (ldi && !address_p(i0)) ldi = 0; +# endif #endif if (ldi) sse_ldi_d(r0, (jit_word_t)i0); @@ -1306,6 +1316,7 @@ _sse_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t *i0) movdqxr(r0, rn(reg)); jit_unget_reg(reg); #else + CHECK_CVT_OFFSET(); movi(rn(reg), data.ii[0]); stxi_i(CVT_OFFSET, _RBP_REGNO, rn(reg)); movi(rn(reg), data.ii[1]); @@ -1321,7 +1332,13 @@ static void _sse_ldi_d(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0) { jit_int32_t reg; - if (sse_address_p(i0)) +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - (_jit->pc.w + 8 + !!(r0 & 8)); + if (can_sign_extend_int_p(rel)) + movsdmr(rel, _NOREG, _NOREG, _SCL8, r0); + else +#endif + if (address_p(i0)) movsdmr(i0, _NOREG, _NOREG, _SCL1, r0); else { reg = jit_get_reg(jit_class_gpr); @@ -1368,7 +1385,13 @@ static void _sse_sti_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0) { jit_int32_t reg; - if (sse_address_p(i0)) +#if CAN_RIP_ADDRESS + jit_word_t rel = i0 - (_jit->pc.w + 8 + !!(r0 & 8)); + if (can_sign_extend_int_p(rel)) + movsdrm(r0, rel, _NOREG, _NOREG, _SCL8); + else +#endif + if (address_p(i0)) movsdrm(r0, i0, _NOREG, _NOREG, _SCL1); else { reg = jit_get_reg(jit_class_gpr); @@ -1415,8 +1438,7 @@ static jit_word_t _sse_bltr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r1, r0); - ja(i0); - return (_jit->pc.w); + return (ja(i0)); } dbopi(lt) @@ -1424,21 +1446,20 @@ static jit_word_t _sse_bler_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r1, r0); - jae(i0); - return (_jit->pc.w); + return (jae(i0)); } dbopi(le) static jit_word_t _sse_beqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; jit_word_t jp_code; ucomisdr(r0, r1); - jps(0); - jp_code = _jit->pc.w; - je(i0); - patch_rel_char(jp_code, _jit->pc.w); - return (_jit->pc.w); + jp_code = jps(0); + w = je(i0); + patch_at(jp_code, _jit->pc.w); + return (w); } dbopi(eq) @@ -1446,8 +1467,7 @@ static jit_word_t _sse_bger_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r0, r1); - jae(i0); - return (_jit->pc.w); + return (jae(i0)); } dbopi(ge) @@ -1455,25 +1475,23 @@ static jit_word_t _sse_bgtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r0, r1); - ja(i0); - return (_jit->pc.w); + return (ja(i0)); } dbopi(gt) static jit_word_t _sse_bner_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; jit_word_t jp_code; jit_word_t jz_code; ucomisdr(r0, r1); - jps(0); - jp_code = _jit->pc.w; - jzs(0); - jz_code = _jit->pc.w; - patch_rel_char(jp_code, _jit->pc.w); - jmpi(i0); - patch_rel_char(jz_code, _jit->pc.w); - return (_jit->pc.w); + jp_code = jps(0); + jz_code = jzs(0); + patch_at(jp_code, _jit->pc.w); + w = jmpi(i0); + patch_at(jz_code, _jit->pc.w); + return (w); } dbopi(ne) @@ -1481,47 +1499,49 @@ static jit_word_t _sse_bunltr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r0, r1); - jnae(i0); - return (_jit->pc.w); + return (jnae(i0)); } dbopi(unlt) static jit_word_t _sse_bunler_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; if (r0 == r1) - jmpi(i0); + w = jmpi(i0); else { ucomisdr(r0, r1); - jna(i0); + w = jna(i0); } - return (_jit->pc.w); + return (w); } dbopi(unle) static jit_word_t _sse_buneqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; if (r0 == r1) - jmpi(i0); + w = jmpi(i0); else { ucomisdr(r0, r1); - je(i0); + w = je(i0); } - return (_jit->pc.w); + return (w); } dbopi(uneq) static jit_word_t _sse_bunger_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; if (r0 == r1) - jmpi(i0); + w = jmpi(i0); else { ucomisdr(r1, r0); - jna(i0); + w = jna(i0); } - return (_jit->pc.w); + return (w); } dbopi(unge) @@ -1529,8 +1549,7 @@ static jit_word_t _sse_bungtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r1, r0); - jnae(i0); - return (_jit->pc.w); + return (jnae(i0)); } dbopi(ungt) @@ -1538,8 +1557,7 @@ static jit_word_t _sse_bltgtr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r0, r1); - jne(i0); - return (_jit->pc.w); + return (jne(i0)); } dbopi(ltgt) @@ -1547,8 +1565,7 @@ static jit_word_t _sse_bordr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r0, r1); - jnp(i0); - return (_jit->pc.w); + return (jnp(i0)); } dbopi(ord) @@ -1556,8 +1573,7 @@ static jit_word_t _sse_bunordr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { ucomisdr(r0, r1); - jp(i0); - return (_jit->pc.w); + return (jp(i0)); } dbopi(unord) # undef fopi diff --git a/parallel-rsp/lightning/lib/jit_x86-sz.c b/parallel-rsp/lightning/lib/jit_x86-sz.c index 663b840..5c4515a 100644 --- a/parallel-rsp/lightning/lib/jit_x86-sz.c +++ b/parallel-rsp/lightning/lib/jit_x86-sz.c @@ -3,9 +3,10 @@ #define JIT_INSTR_MAX 42 0, /* data */ 0, /* live */ - 3, /* align */ + 11, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 3, /* label */ @@ -14,7 +15,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -22,8 +26,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 3, /* va_start */ 5, /* va_arg */ 7, /* va_arg_d */ @@ -36,9 +52,9 @@ 5, /* addxi */ 4, /* subr */ 6, /* subi */ - 6, /* subcr */ + 12, /* subcr */ 6, /* subci */ - 6, /* subxr */ + 12, /* subxr */ 5, /* subxi */ 8, /* rsbi */ 5, /* mulr */ @@ -52,9 +68,9 @@ 22, /* divr_u */ 25, /* divi_u */ 23, /* qdivr */ - 26, /* qdivi */ + 28, /* qdivi */ 24, /* qdivr_u */ - 27, /* qdivi_u */ + 29, /* qdivi_u */ 21, /* remr */ 24, /* remi */ 22, /* remr_u */ @@ -95,12 +111,19 @@ 16, /* nei */ 2, /* movr */ 5, /* movi */ + 5, /* movnr */ + 5, /* movzr */ + 9, /* casr */ + 13, /* casi */ 11, /* extr_c */ 11, /* extr_uc */ 3, /* extr_s */ 3, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 7, /* bswapr_us */ + 4, /* bswapr_ui */ + 0, /* bswapr_ul */ 7, /* htonr_us */ 4, /* htonr_ui */ 0, /* htonr_ul */ @@ -193,13 +216,37 @@ 2, /* callr */ 5, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -207,93 +254,93 @@ 0, /* retval_i */ 0, /* retval_ui */ 0, /* retval_l */ - 24, /* epilog */ + 23, /* epilog */ 0, /* arg_f */ 0, /* getarg_f */ 0, /* putargr_f */ 0, /* putargi_f */ 8, /* addr_f */ - 19, /* addi_f */ + 17, /* addi_f */ 12, /* subr_f */ - 19, /* subi_f */ + 17, /* subi_f */ 21, /* rsbi_f */ 8, /* mulr_f */ - 19, /* muli_f */ + 17, /* muli_f */ 12, /* divr_f */ - 19, /* divi_f */ + 17, /* divi_f */ 12, /* negr_f */ 12, /* absr_f */ - 6, /* sqrtr_f */ - 13, /* ltr_f */ + 4, /* sqrtr_f */ + 12, /* ltr_f */ 27, /* lti_f */ - 13, /* ler_f */ + 12, /* ler_f */ 27, /* lei_f */ - 15, /* eqr_f */ + 14, /* eqr_f */ 29, /* eqi_f */ - 13, /* ger_f */ + 12, /* ger_f */ 27, /* gei_f */ - 13, /* gtr_f */ + 12, /* gtr_f */ 27, /* gti_f */ - 18, /* ner_f */ + 17, /* ner_f */ 32, /* nei_f */ - 13, /* unltr_f */ + 12, /* unltr_f */ 27, /* unlti_f */ - 13, /* unler_f */ + 12, /* unler_f */ 27, /* unlei_f */ - 13, /* uneqr_f */ + 12, /* uneqr_f */ 27, /* uneqi_f */ - 13, /* unger_f */ + 12, /* unger_f */ 27, /* ungei_f */ - 13, /* ungtr_f */ + 12, /* ungtr_f */ 27, /* ungti_f */ - 13, /* ltgtr_f */ + 12, /* ltgtr_f */ 27, /* ltgti_f */ - 13, /* ordr_f */ + 12, /* ordr_f */ 27, /* ordi_f */ - 13, /* unordr_f */ + 12, /* unordr_f */ 27, /* unordi_f */ - 8, /* truncr_f_i */ + 4, /* truncr_f_i */ 0, /* truncr_f_l */ - 8, /* extr_f */ + 4, /* extr_f */ 4, /* extr_d_f */ 10, /* movr_f */ - 19, /* movi_f */ + 15, /* movi_f */ 4, /* ldr_f */ 8, /* ldi_f */ 5, /* ldxr_f */ 8, /* ldxi_f */ - 6, /* str_f */ - 10, /* sti_f */ - 7, /* stxr_f */ + 4, /* str_f */ + 8, /* sti_f */ + 5, /* stxr_f */ 8, /* stxi_f */ - 10, /* bltr_f */ - 23, /* blti_f */ - 10, /* bler_f */ - 23, /* blei_f */ - 12, /* beqr_f */ - 25, /* beqi_f */ - 10, /* bger_f */ - 23, /* bgei_f */ - 10, /* bgtr_f */ - 23, /* bgti_f */ - 13, /* bner_f */ - 26, /* bnei_f */ - 10, /* bunltr_f */ - 23, /* bunlti_f */ - 10, /* bunler_f */ - 23, /* bunlei_f */ - 10, /* buneqr_f */ - 23, /* buneqi_f */ - 10, /* bunger_f */ - 23, /* bungei_f */ - 10, /* bungtr_f */ - 23, /* bungti_f */ - 10, /* bltgtr_f */ - 23, /* bltgti_f */ - 10, /* bordr_f */ - 23, /* bordi_f */ - 10, /* bunordr_f */ - 23, /* bunordi_f */ + 9, /* bltr_f */ + 18, /* blti_f */ + 9, /* bler_f */ + 18, /* blei_f */ + 11, /* beqr_f */ + 20, /* beqi_f */ + 9, /* bger_f */ + 18, /* bgei_f */ + 9, /* bgtr_f */ + 18, /* bgti_f */ + 12, /* bner_f */ + 21, /* bnei_f */ + 9, /* bunltr_f */ + 18, /* bunlti_f */ + 9, /* bunler_f */ + 18, /* bunlei_f */ + 9, /* buneqr_f */ + 18, /* buneqi_f */ + 9, /* bunger_f */ + 18, /* bungei_f */ + 9, /* bungtr_f */ + 18, /* bungti_f */ + 9, /* bltgtr_f */ + 18, /* bltgti_f */ + 9, /* bordr_f */ + 18, /* bordi_f */ + 9, /* bunordr_f */ + 18, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -314,7 +361,7 @@ 26, /* divi_d */ 18, /* negr_d */ 13, /* absr_d */ - 6, /* sqrtr_d */ + 4, /* sqrtr_d */ 13, /* ltr_d */ 37, /* lti_d */ 13, /* ler_d */ @@ -343,19 +390,19 @@ 37, /* ordi_d */ 13, /* unordr_d */ 37, /* unordi_d */ - 8, /* truncr_d_i */ + 4, /* truncr_d_i */ 0, /* truncr_d_l */ - 8, /* extr_d */ + 4, /* extr_d */ 4, /* extr_f_d */ 10, /* movr_d */ - 24, /* movi_d */ + 33, /* movi_d */ 4, /* ldr_d */ 8, /* ldi_d */ 5, /* ldxr_d */ 8, /* ldxi_d */ - 6, /* str_d */ - 10, /* sti_d */ - 7, /* stxr_d */ + 4, /* str_d */ + 8, /* sti_d */ + 5, /* stxr_d */ 8, /* stxi_d */ 10, /* bltr_d */ 28, /* blti_d */ @@ -399,16 +446,21 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ -#endif + 21, /* clo */ + 17, /* clz */ + 15, /* cto */ + 11, /* ctz */ +#endif /* __X32 */ #if __X64 #if __CYGWIN__ || _WIN32 #define JIT_INSTR_MAX 130 0, /* data */ 0, /* live */ - 6, /* align */ + 27, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 7, /* label */ @@ -417,7 +469,10 @@ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -425,8 +480,20 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ 7, /* va_start */ 7, /* va_arg */ 9, /* va_arg_d */ @@ -498,29 +565,36 @@ 14, /* nei */ 3, /* movr */ 10, /* movi */ + 7, /* movnr */ + 7, /* movzr */ + 11, /* casr */ + 21, /* casi */ 7, /* extr_c */ 7, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 3, /* extr_i */ 3, /* extr_ui */ + 9, /* bswapr_us */ + 6, /* bswapr_ui */ + 6, /* bswapr_ul */ 9, /* htonr_us */ 6, /* htonr_ui */ 6, /* htonr_ul */ 4, /* ldr_c */ - 15, /* ldi_c */ + 14, /* ldi_c */ 4, /* ldr_uc */ - 15, /* ldi_uc */ + 14, /* ldi_uc */ 4, /* ldr_s */ - 15, /* ldi_s */ + 14, /* ldi_s */ 4, /* ldr_us */ - 15, /* ldi_us */ + 14, /* ldi_us */ 3, /* ldr_i */ - 14, /* ldi_i */ + 13, /* ldi_i */ 3, /* ldr_ui */ - 14, /* ldi_ui */ + 13, /* ldi_ui */ 3, /* ldr_l */ - 14, /* ldi_l */ + 13, /* ldi_l */ 5, /* ldxr_c */ 8, /* ldxi_c */ 5, /* ldxr_uc */ @@ -536,13 +610,13 @@ 4, /* ldxr_l */ 7, /* ldxi_l */ 6, /* str_c */ - 17, /* sti_c */ + 16, /* sti_c */ 4, /* str_s */ - 15, /* sti_s */ + 14, /* sti_s */ 3, /* str_i */ - 14, /* sti_i */ + 13, /* sti_i */ 3, /* str_l */ - 14, /* sti_l */ + 13, /* sti_l */ 7, /* stxr_c */ 7, /* stxi_c */ 5, /* stxr_s */ @@ -591,18 +665,42 @@ 10, /* bxsubi */ 9, /* bxsubr_u */ 10, /* bxsubi_u */ - 3, /* jmpr */ + 2, /* jmpr */ 5, /* jmpi */ - 3, /* callr */ - 13, /* calli */ + 2, /* callr */ + 20, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -616,87 +714,87 @@ 0, /* putargr_f */ 0, /* putargi_f */ 10, /* addr_f */ - 21, /* addi_f */ + 19, /* addi_f */ 15, /* subr_f */ - 21, /* subi_f */ - 27, /* rsbi_f */ + 19, /* subi_f */ + 26, /* rsbi_f */ 10, /* mulr_f */ - 21, /* muli_f */ + 19, /* muli_f */ 15, /* divr_f */ - 21, /* divi_f */ - 15, /* negr_f */ + 19, /* divi_f */ + 14, /* negr_f */ 15, /* absr_f */ 5, /* sqrtr_f */ 16, /* ltr_f */ - 31, /* lti_f */ + 30, /* lti_f */ 16, /* ler_f */ - 31, /* lei_f */ + 30, /* lei_f */ 18, /* eqr_f */ - 33, /* eqi_f */ + 32, /* eqi_f */ 16, /* ger_f */ - 31, /* gei_f */ + 30, /* gei_f */ 16, /* gtr_f */ - 31, /* gti_f */ + 30, /* gti_f */ 20, /* ner_f */ - 35, /* nei_f */ + 34, /* nei_f */ 16, /* unltr_f */ - 31, /* unlti_f */ + 30, /* unlti_f */ 16, /* unler_f */ - 31, /* unlei_f */ + 30, /* unlei_f */ 16, /* uneqr_f */ - 31, /* uneqi_f */ + 30, /* uneqi_f */ 16, /* unger_f */ - 31, /* ungei_f */ + 30, /* ungei_f */ 16, /* ungtr_f */ - 31, /* ungti_f */ + 30, /* ungti_f */ 16, /* ltgtr_f */ - 31, /* ltgti_f */ + 30, /* ltgti_f */ 16, /* ordr_f */ - 31, /* ordi_f */ + 30, /* ordi_f */ 16, /* unordr_f */ - 31, /* unordi_f */ + 30, /* unordi_f */ 5, /* truncr_f_i */ 5, /* truncr_f_l */ 5, /* extr_f */ 5, /* extr_d_f */ 5, /* movr_f */ - 15, /* movi_f */ + 18, /* movi_f */ 5, /* ldr_f */ - 16, /* ldi_f */ + 15, /* ldi_f */ 6, /* ldxr_f */ 8, /* ldxi_f */ 5, /* str_f */ - 16, /* sti_f */ + 15, /* sti_f */ 6, /* stxr_f */ 9, /* stxi_f */ 10, /* bltr_f */ - 21, /* blti_f */ + 19, /* blti_f */ 10, /* bler_f */ - 24, /* blei_f */ + 23, /* blei_f */ 12, /* beqr_f */ 27, /* beqi_f */ 10, /* bger_f */ - 25, /* bgei_f */ + 24, /* bgei_f */ 10, /* bgtr_f */ - 25, /* bgti_f */ + 24, /* bgti_f */ 13, /* bner_f */ - 28, /* bnei_f */ + 27, /* bnei_f */ 10, /* bunltr_f */ - 25, /* bunlti_f */ + 24, /* bunlti_f */ 10, /* bunler_f */ - 25, /* bunlei_f */ + 24, /* bunlei_f */ 10, /* buneqr_f */ - 25, /* buneqi_f */ + 24, /* buneqi_f */ 10, /* bunger_f */ - 25, /* bungei_f */ + 24, /* bungei_f */ 10, /* bungtr_f */ - 25, /* bungti_f */ + 24, /* bungti_f */ 10, /* bltgtr_f */ - 25, /* bltgti_f */ + 24, /* bltgti_f */ 10, /* bordr_f */ - 25, /* bordi_f */ + 24, /* bordi_f */ 10, /* bunordr_f */ - 25, /* bunordi_f */ + 24, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -715,7 +813,7 @@ 25, /* muli_d */ 15, /* divr_d */ 25, /* divi_d */ - 22, /* negr_d */ + 21, /* negr_d */ 16, /* absr_d */ 5, /* sqrtr_d */ 17, /* ltr_d */ @@ -751,13 +849,13 @@ 5, /* extr_d */ 5, /* extr_f_d */ 5, /* movr_d */ - 15, /* movi_d */ + 29, /* movi_d */ 5, /* ldr_d */ - 16, /* ldi_d */ + 15, /* ldi_d */ 6, /* ldxr_d */ 8, /* ldxi_d */ 5, /* str_d */ - 16, /* sti_d */ + 15, /* sti_d */ 6, /* stxr_d */ 9, /* stxi_d */ 11, /* bltr_d */ @@ -802,24 +900,32 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 27, /* clo */ + 21, /* clz */ + 20, /* cto */ + 14, /* ctz */ #else # if __X64_32 -#define JIT_INSTR_MAX 108 +#define JIT_INSTR_MAX 105 0, /* data */ 0, /* live */ - 3, /* align */ + 7, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 3, /* label */ - 108, /* prolog */ + 105, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -827,11 +933,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 41, /* va_start */ - 45, /* va_arg */ - 54, /* va_arg_d */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 33, /* va_start */ + 43, /* va_arg */ + 45, /* va_arg_d */ 0, /* va_end */ 5, /* addr */ 7, /* addi */ @@ -900,12 +1018,19 @@ 14, /* nei */ 3, /* movr */ 6, /* movi */ + 7, /* movnr */ + 7, /* movzr */ + 11, /* casr */ + 16, /* casi */ 7, /* extr_c */ 7, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 0, /* extr_i */ 0, /* extr_ui */ + 9, /* bswapr_us */ + 6, /* bswapr_ui */ + 0, /* bswapr_ul */ 9, /* htonr_us */ 6, /* htonr_ui */ 0, /* htonr_ul */ @@ -945,11 +1070,11 @@ 8, /* sti_i */ 0, /* str_l */ 0, /* sti_l */ - 12, /* stxr_c */ + 11, /* stxr_c */ 7, /* stxi_c */ - 10, /* stxr_s */ + 9, /* stxr_s */ 7, /* stxi_s */ - 9, /* stxr_i */ + 8, /* stxr_i */ 6, /* stxi_i */ 0, /* stxr_l */ 0, /* stxi_l */ @@ -995,16 +1120,40 @@ 10, /* bxsubi_u */ 2, /* jmpr */ 5, /* jmpi */ - 3, /* callr */ + 2, /* callr */ 9, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -1018,14 +1167,14 @@ 0, /* putargr_f */ 0, /* putargi_f */ 10, /* addr_f */ - 21, /* addi_f */ + 20, /* addi_f */ 15, /* subr_f */ - 21, /* subi_f */ - 26, /* rsbi_f */ + 20, /* subi_f */ + 25, /* rsbi_f */ 10, /* mulr_f */ - 21, /* muli_f */ + 20, /* muli_f */ 15, /* divr_f */ - 21, /* divi_f */ + 20, /* divi_f */ 15, /* negr_f */ 15, /* absr_f */ 5, /* sqrtr_f */ @@ -1065,40 +1214,40 @@ 11, /* movi_f */ 6, /* ldr_f */ 10, /* ldi_f */ - 11, /* ldxr_f */ + 10, /* ldxr_f */ 9, /* ldxi_f */ 6, /* str_f */ 10, /* sti_f */ - 11, /* stxr_f */ + 10, /* stxr_f */ 9, /* stxi_f */ 10, /* bltr_f */ - 21, /* blti_f */ + 20, /* blti_f */ 10, /* bler_f */ - 21, /* blei_f */ + 20, /* blei_f */ 12, /* beqr_f */ 23, /* beqi_f */ 10, /* bger_f */ - 21, /* bgei_f */ + 20, /* bgei_f */ 10, /* bgtr_f */ - 21, /* bgti_f */ + 20, /* bgti_f */ 13, /* bner_f */ - 24, /* bnei_f */ + 23, /* bnei_f */ 10, /* bunltr_f */ - 21, /* bunlti_f */ + 20, /* bunlti_f */ 10, /* bunler_f */ - 21, /* bunlei_f */ + 20, /* bunlei_f */ 10, /* buneqr_f */ - 21, /* buneqi_f */ + 20, /* buneqi_f */ 10, /* bunger_f */ - 21, /* bungei_f */ + 20, /* bungei_f */ 10, /* bungtr_f */ - 21, /* bungti_f */ + 20, /* bungti_f */ 10, /* bltgtr_f */ - 21, /* bltgti_f */ + 20, /* bltgti_f */ 10, /* bordr_f */ - 21, /* bordi_f */ + 20, /* bordi_f */ 10, /* bunordr_f */ - 21, /* bunordi_f */ + 20, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -1109,14 +1258,14 @@ 0, /* putargr_d */ 0, /* putargi_d */ 10, /* addr_d */ - 33, /* addi_d */ + 29, /* addi_d */ 15, /* subr_d */ - 33, /* subi_d */ - 38, /* rsbi_d */ + 29, /* subi_d */ + 34, /* rsbi_d */ 10, /* mulr_d */ - 33, /* muli_d */ + 29, /* muli_d */ 15, /* divr_d */ - 33, /* divi_d */ + 29, /* divi_d */ 22, /* negr_d */ 16, /* absr_d */ 5, /* sqrtr_d */ @@ -1156,40 +1305,40 @@ 23, /* movi_d */ 6, /* ldr_d */ 10, /* ldi_d */ - 11, /* ldxr_d */ + 10, /* ldxr_d */ 9, /* ldxi_d */ 6, /* str_d */ 10, /* sti_d */ - 11, /* stxr_d */ + 10, /* stxr_d */ 9, /* stxi_d */ 11, /* bltr_d */ - 34, /* blti_d */ + 30, /* blti_d */ 11, /* bler_d */ - 34, /* blei_d */ + 30, /* blei_d */ 13, /* beqr_d */ 36, /* beqi_d */ 11, /* bger_d */ - 34, /* bgei_d */ + 30, /* bgei_d */ 11, /* bgtr_d */ - 34, /* bgti_d */ + 30, /* bgti_d */ 14, /* bner_d */ - 37, /* bnei_d */ + 33, /* bnei_d */ 11, /* bunltr_d */ - 34, /* bunlti_d */ + 30, /* bunlti_d */ 11, /* bunler_d */ - 34, /* bunlei_d */ + 30, /* bunlei_d */ 11, /* buneqr_d */ - 34, /* buneqi_d */ + 30, /* buneqi_d */ 11, /* bunger_d */ - 34, /* bungei_d */ + 30, /* bungei_d */ 11, /* bungtr_d */ - 34, /* bungti_d */ + 30, /* bungti_d */ 11, /* bltgtr_d */ - 34, /* bltgti_d */ + 30, /* bltgti_d */ 11, /* bordr_d */ - 34, /* bordi_d */ + 30, /* bordi_d */ 11, /* bunordr_d */ - 34, /* bunordi_d */ + 30, /* bunordi_d */ 0, /* pushargr_d */ 0, /* pushargi_d */ 0, /* retr_d */ @@ -1204,23 +1353,31 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 11, /* clo */ + 5, /* clz */ + 11, /* cto */ + 5, /* ctz */ +#else -# else -#define JIT_INSTR_MAX 115 +#define JIT_INSTR_MAX 112 0, /* data */ 0, /* live */ - 6, /* align */ + 27, /* align */ 0, /* save */ 0, /* load */ + 4, /* skip */ 0, /* #name */ 0, /* #note */ 7, /* label */ - 115, /* prolog */ + 112, /* prolog */ 0, /* ellipsis */ 0, /* va_push */ 0, /* allocai */ 0, /* allocar */ - 0, /* arg */ + 0, /* arg_c */ + 0, /* arg_s */ + 0, /* arg_i */ + 0, /* arg_l */ 0, /* getarg_c */ 0, /* getarg_uc */ 0, /* getarg_s */ @@ -1228,11 +1385,23 @@ 0, /* getarg_i */ 0, /* getarg_ui */ 0, /* getarg_l */ - 0, /* putargr */ - 0, /* putargi */ - 42, /* va_start */ + 0, /* putargr_c */ + 0, /* putargi_c */ + 0, /* putargr_uc */ + 0, /* putargi_uc */ + 0, /* putargr_s */ + 0, /* putargi_s */ + 0, /* putargr_us */ + 0, /* putargi_us */ + 0, /* putargr_i */ + 0, /* putargi_i */ + 0, /* putargr_ui */ + 0, /* putargi_ui */ + 0, /* putargr_l */ + 0, /* putargi_l */ + 38, /* va_start */ 41, /* va_arg */ - 50, /* va_arg_d */ + 48, /* va_arg_d */ 0, /* va_end */ 5, /* addr */ 13, /* addi */ @@ -1301,12 +1470,19 @@ 14, /* nei */ 3, /* movr */ 10, /* movi */ + 7, /* movnr */ + 7, /* movzr */ + 11, /* casr */ + 16, /* casi */ 4, /* extr_c */ 4, /* extr_uc */ 4, /* extr_s */ 4, /* extr_us */ 3, /* extr_i */ 3, /* extr_ui */ + 9, /* bswapr_us */ + 6, /* bswapr_ui */ + 6, /* bswapr_ul */ 9, /* htonr_us */ 6, /* htonr_ui */ 6, /* htonr_ul */ @@ -1394,18 +1570,42 @@ 10, /* bxsubi */ 9, /* bxsubr_u */ 10, /* bxsubi_u */ - 3, /* jmpr */ + 2, /* jmpr */ 5, /* jmpi */ - 3, /* callr */ + 2, /* callr */ 13, /* calli */ 0, /* prepare */ - 0, /* pushargr */ - 0, /* pushargi */ + 0, /* pushargr_c */ + 0, /* pushargi_c */ + 0, /* pushargr_uc */ + 0, /* pushargi_uc */ + 0, /* pushargr_s */ + 0, /* pushargi_s */ + 0, /* pushargr_us */ + 0, /* pushargi_us */ + 0, /* pushargr_i */ + 0, /* pushargi_i */ + 0, /* pushargr_ui */ + 0, /* pushargi_ui */ + 0, /* pushargr_l */ + 0, /* pushargi_l */ 0, /* finishr */ 0, /* finishi */ 0, /* ret */ - 0, /* retr */ - 0, /* reti */ + 0, /* retr_c */ + 0, /* reti_c */ + 0, /* retr_uc */ + 0, /* reti_uc */ + 0, /* retr_s */ + 0, /* reti_s */ + 0, /* retr_us */ + 0, /* reti_us */ + 0, /* retr_i */ + 0, /* reti_i */ + 0, /* retr_ui */ + 0, /* reti_ui */ + 0, /* retr_l */ + 0, /* reti_l */ 0, /* retval_c */ 0, /* retval_uc */ 0, /* retval_s */ @@ -1419,14 +1619,14 @@ 0, /* putargr_f */ 0, /* putargi_f */ 10, /* addr_f */ - 21, /* addi_f */ + 20, /* addi_f */ 15, /* subr_f */ - 21, /* subi_f */ + 20, /* subi_f */ 30, /* rsbi_f */ 10, /* mulr_f */ - 21, /* muli_f */ + 20, /* muli_f */ 15, /* divr_f */ - 21, /* divi_f */ + 20, /* divi_f */ 15, /* negr_f */ 15, /* absr_f */ 5, /* sqrtr_f */ @@ -1473,33 +1673,33 @@ 7, /* stxr_f */ 9, /* stxi_f */ 10, /* bltr_f */ - 21, /* blti_f */ + 20, /* blti_f */ 10, /* bler_f */ - 25, /* blei_f */ + 22, /* blei_f */ 12, /* beqr_f */ - 27, /* beqi_f */ + 22, /* beqi_f */ 10, /* bger_f */ - 25, /* bgei_f */ + 22, /* bgei_f */ 10, /* bgtr_f */ - 25, /* bgti_f */ + 22, /* bgti_f */ 13, /* bner_f */ - 28, /* bnei_f */ + 25, /* bnei_f */ 10, /* bunltr_f */ - 25, /* bunlti_f */ + 23, /* bunlti_f */ 10, /* bunler_f */ - 25, /* bunlei_f */ + 23, /* bunlei_f */ 10, /* buneqr_f */ - 25, /* buneqi_f */ + 23, /* buneqi_f */ 10, /* bunger_f */ - 25, /* bungei_f */ + 23, /* bungei_f */ 10, /* bungtr_f */ - 25, /* bungti_f */ + 22, /* bungti_f */ 10, /* bltgtr_f */ - 25, /* bltgti_f */ + 22, /* bltgti_f */ 10, /* bordr_f */ - 25, /* bordi_f */ + 22, /* bordi_f */ 10, /* bunordr_f */ - 25, /* bunordi_f */ + 22, /* bunordi_f */ 0, /* pushargr_f */ 0, /* pushargi_f */ 0, /* retr_f */ @@ -1605,6 +1805,10 @@ 0, /* movi_d_ww */ 0, /* movr_d_w */ 0, /* movi_d_w */ + 11, /* clo */ + 5, /* clz */ + 11, /* cto */ + 5, /* ctz */ #endif /* __CYGWIN__ || _WIN32 */ # endif /* __X64_32 */ #endif /* __X64 */ diff --git a/parallel-rsp/lightning/lib/jit_x86-x87.c b/parallel-rsp/lightning/lib/jit_x86-x87.c index 4453bf3..3de0214 100644 --- a/parallel-rsp/lightning/lib/jit_x86-x87.c +++ b/parallel-rsp/lightning/lib/jit_x86-x87.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -408,14 +408,14 @@ _x87_b##name##i_##type(jit_state_t *_jit, \ jit_word_t i0, jit_int32_t r0, \ jit_float##size##_t *i1) \ { \ - jit_word_t word; \ + jit_word_t w; \ jit_int32_t reg = jit_get_reg(jit_class_fpr| \ jit_class_nospill); \ assert(jit_x87_reg_p(reg)); \ x87_movi_##type(rn(reg), i1); \ - word = x87_b##name##r_##type(i0, r0, rn(reg)); \ + w = x87_b##name##r_##type(i0, r0, rn(reg)); \ jit_unget_reg(reg); \ - return (word); \ + return (w); \ } # define fopi(name) fpr_opi(name, f, 32) # define fbopi(name) fpr_bopi(name, f, 32) @@ -662,6 +662,7 @@ _x87_sqrtr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _x87_truncr_d_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); #if defined(sun) /* for the sake of passing test cases in x87 mode, otherwise only sse * is supported */ @@ -692,6 +693,7 @@ _x87_truncr_d_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _x87_truncr_d_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); fldr(r1); fisttpqm(CVT_OFFSET, _RBP_REGNO, _NOREG, _SCL1); ldxi(r0, _RBP_REGNO, CVT_OFFSET); @@ -701,6 +703,7 @@ _x87_truncr_d_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _x87_extr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); stxi(CVT_OFFSET, _RBP_REGNO, r1); # if __X32 fildlm(CVT_OFFSET, _RBP_REGNO, _NOREG, _SCL1); @@ -771,8 +774,7 @@ _x87jcc(jit_state_t *_jit, jit_int32_t code, fldr(r0); fucomipr(r1 + 1); } - jcc(code, i0); - return (_jit->pc.w); + return (jcc(code, i0)); } static jit_word_t @@ -788,8 +790,7 @@ _x87jcc2(jit_state_t *_jit, jit_int32_t code, fldr(f0); fucomipr(f1 + 1); } - jcc(code, i0); - return (_jit->pc.w); + return (jcc(code, i0)); } fopi(lt) @@ -847,6 +848,7 @@ _x87_movi_f(jit_state_t *_jit, jit_int32_t r0, jit_float32_t *i0) fldln2(); else { if (_jitc->no_data) { + CHECK_CVT_OFFSET(); reg = jit_get_reg(jit_class_gpr); movi(rn(reg), data.i); stxi_i(CVT_OFFSET, _RBP_REGNO, rn(reg)); @@ -1038,6 +1040,7 @@ _x87_movi_d(jit_state_t *_jit, jit_int32_t r0, jit_float64_t *i0) fldln2(); else { if (_jitc->no_data) { + CHECK_CVT_OFFSET(); reg = jit_get_reg(jit_class_gpr); #if __X32 || __X64_32 movi(rn(reg), data.ii[0]); @@ -1082,10 +1085,9 @@ _x87_eqr_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) fldr(f1); fucomipr(f2 + 1); } - jpes(0); - jp_code = _jit->pc.w; + jp_code = jpes(0); cc(X86_CC_E, reg); - patch_rel_char(jp_code, _jit->pc.w); + patch_at(jp_code, _jit->pc.w); if (!rc) xchgr(r0, reg); } @@ -1115,10 +1117,9 @@ _x87_ner_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2) fldr(f1); fucomipr(f2 + 1); } - jpes(0); - jp_code = _jit->pc.w; + jp_code = jpes(0); cc(X86_CC_NE, reg); - patch_rel_char(jp_code, _jit->pc.w); + patch_at(jp_code, _jit->pc.w); if (!rc) xchgr(r0, reg); } @@ -1283,6 +1284,7 @@ dbopi(le) static jit_word_t _x87_beqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; jit_int32_t f0, f1; jit_word_t jp_code; if (r1 == _ST0_REGNO) f0 = r1, f1 = r0; @@ -1293,11 +1295,10 @@ _x87_beqr_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) fldr(f0); fucomipr(f1 + 1); } - jpes(0); - jp_code = _jit->pc.w; - jcc(X86_CC_E, i0); - patch_rel_char(jp_code, _jit->pc.w); - return (_jit->pc.w); + jp_code = jpes(0); + w = jcc(X86_CC_E, i0); + patch_at(jp_code, _jit->pc.w); + return (w); } dbopi(eq) dbopi(ge) @@ -1306,6 +1307,7 @@ dbopi(gt) static jit_word_t _x87_bner_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) { + jit_word_t w; jit_int32_t f0, f1; jit_word_t jp_code; jit_word_t jz_code; @@ -1317,14 +1319,12 @@ _x87_bner_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) fldr(f0); fucomipr(f1 + 1); } - jpes(0); - jp_code = _jit->pc.w; - jzs(0); - jz_code = _jit->pc.w; - patch_rel_char(jp_code, _jit->pc.w); - jmpi(i0); - patch_rel_char(jz_code, _jit->pc.w); - return (_jit->pc.w); + jp_code = jpes(0); + jz_code = jzs(0); + patch_at(jp_code, _jit->pc.w); + w = jmpi(i0); + patch_at(jz_code, _jit->pc.w); + return (w); } dbopi(ne) dbopi(unlt) diff --git a/parallel-rsp/lightning/lib/jit_x86.c b/parallel-rsp/lightning/lib/jit_x86.c index c34a117..b409457 100644 --- a/parallel-rsp/lightning/lib/jit_x86.c +++ b/parallel-rsp/lightning/lib/jit_x86.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -21,23 +21,39 @@ #include #if __X32 +# define CAN_RIP_ADDRESS 0 +# define address_p(i0) 1 # define jit_arg_reg_p(i) 0 # define jit_arg_f_reg_p(i) 0 -# define stack_framesize 20 -# define stack_adjust 12 -# define CVT_OFFSET -12 +/* callee save + 16 byte align + * align16(%ebp + %rbx + %rsi + %rdi) + (16 - 4) */ +# define stack_framesize 28 # define REAL_WORDSIZE 4 # define va_gp_increment 4 # define va_fp_increment 8 #else +# if _WIN32 || __X64_32 +# define CAN_RIP_ADDRESS 0 +# else +# define CAN_RIP_ADDRESS 1 +# endif +# if __X64_32 +# define address_p(i0) ((jit_word_t)(i0) >= 0) +# else +# define address_p(i0) can_sign_extend_int_p(i0) +# endif # if __CYGWIN__ || _WIN32 # define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4) # define jit_arg_f_reg_p(i) jit_arg_reg_p(i) +/* callee save + 16 byte align + * align16(%rbp+%rbx+%rdi+%rsi+%r1[2-5]+%xmm[6-9]+%xmm1[0-5]) + (16 - 8) */ # define stack_framesize 152 # define va_fp_increment 8 # else # define jit_arg_reg_p(i) ((i) >= 0 && (i) < 6) # define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) +/* callee save + 16 byte align + * align16(%rbp + %r15 + %r14 + %r13 + %r12 + %rbx) + (16 - 8) */ # define stack_framesize 56 # define first_gp_argument rdi # define first_gp_offset offsetof(jit_va_list_t, rdi) @@ -54,10 +70,18 @@ # define first_fp_from_offset(fp) (((fp) - va_gp_max_offset) / 16) # endif # define va_gp_increment 8 -# define stack_adjust 8 -# define CVT_OFFSET -8 # define REAL_WORDSIZE 8 #endif +#define CVT_OFFSET _jitc->function->cvt_offset + +#define CHECK_CVT_OFFSET() \ + do { \ + if (!_jitc->function->cvt_offset) { \ + _jitc->again = 1; \ + _jitc->function->cvt_offset = \ + jit_allocai(sizeof(jit_float64_t)); \ + } \ + } while (0) /* * Types @@ -99,6 +123,8 @@ typedef struct jit_va_list { /* * Prototypes */ +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); #define patch(instr, node) _patch(_jit, instr, node) static void _patch(jit_state_t*,jit_word_t,jit_node_t*); #define sse_from_x87_f(r0, r1) _sse_from_x87_f(_jit, r0, r1) @@ -227,6 +253,22 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { +#if __X32 + _RBX, _RSI, _RDI, +#elif (__CYGWIN__ || _WIN32) + _RBX, _RDI, _RSI, _R12, _R13, _R14, _R15, +#else + _R15, _R14, _R13, _R12, _RBX, +#endif +}; + +#if __X64 && (__CYGWIN__ || _WIN32) +static jit_int32_t fregs[] = { + _XMM6, _XMM7, _XMM8, _XMM9, _XMM10, _XMM11, _XMM12, _XMM13, _XMM14, _XMM15, +}; +#endif + /* * Implementation */ @@ -234,6 +276,45 @@ void jit_get_cpu(void) { union { + /* eax=7 and ecx=0 */ + struct { + jit_uword_t fsgsbase : 1; + jit_uword_t IA32_TSC_ADJUST : 1; + jit_uword_t sgx : 1; + jit_uword_t bmi1 : 1; + jit_uword_t hle : 1; + jit_uword_t avx2 : 1; + jit_uword_t FDP_EXCPTN_ONLY : 1; + jit_uword_t smep : 1; + jit_uword_t bmi2 : 1; + jit_uword_t erms : 1; + jit_uword_t invpcid : 1; + jit_uword_t rtm : 1; + jit_uword_t rdt_m_pqm : 1; + jit_uword_t dep_FPU_CS_DS : 1; + jit_uword_t mpx : 1; + jit_uword_t rdt_a_pqe : 1; + jit_uword_t avx512_f : 1; + jit_uword_t avx512_dq : 1; + jit_uword_t rdseed : 1; + jit_uword_t adx : 1; + jit_uword_t smap : 1; + jit_uword_t avx512_ifma : 1; + jit_uword_t __reserved0 : 1; + jit_uword_t clflushopt : 1; + jit_uword_t clwb : 1; + jit_uword_t pt : 1; + jit_uword_t avx512_pf : 1; + jit_uword_t avx512_er : 1; + jit_uword_t avx512_cd : 1; + jit_uword_t sha : 1; + jit_uword_t avx512_bw : 1; + jit_uword_t avx512_vl : 1; + } bits; + jit_uword_t cpuid; + } ebx; + union { + /* eax=0 */ struct { jit_uint32_t sse3 : 1; jit_uint32_t pclmulqdq : 1; @@ -271,6 +352,7 @@ jit_get_cpu(void) jit_uword_t cpuid; } ecx; union { + /* eax=0 */ struct { jit_uint32_t fpu : 1; jit_uint32_t vme : 1; @@ -310,7 +392,7 @@ jit_get_cpu(void) #if __X32 int ac, flags; #endif - jit_uword_t eax, ebx; + jit_uword_t eax; #if __X32 /* adapted from glibc __sysconf */ @@ -339,7 +421,7 @@ jit_get_cpu(void) #else __asm__ volatile ("xchgq %%rbx, %1; cpuid; xchgq %%rbx, %1" #endif - : "=a" (eax), "=r" (ebx), + : "=a" (eax), "=r" (ebx.cpuid), "=c" (ecx.cpuid), "=d" (edx.cpuid) : "0" (1)); @@ -361,6 +443,15 @@ jit_get_cpu(void) jit_cpu.aes = ecx.bits.aes; jit_cpu.avx = ecx.bits.avx; + /* query %eax = 7 and ecx = 0 function */ +#if __X64 + __asm__ volatile ("cpuid" + : "=a" (eax), "=b" (ebx.cpuid), "=c" (ecx), "=d" (edx) + : "a" (7), "c" (0)); +#endif + jit_cpu.adx = ebx.bits.adx; + + /* query %eax = 0x80000001 function */ #if __X64 # if __X64_32 @@ -368,10 +459,11 @@ jit_get_cpu(void) # else __asm__ volatile ("xchgq %%rbx, %1; cpuid; xchgq %%rbx, %1" # endif - : "=a" (eax), "=r" (ebx), + : "=a" (eax), "=r" (ebx.cpuid), "=c" (ecx.cpuid), "=d" (edx.cpuid) : "0" (0x80000001)); - jit_cpu.lahf = ecx.cpuid & 1; + jit_cpu.lahf = !!(ecx.cpuid & 1); + jit_cpu.abm = !!(ecx.cpuid & 32); #endif } @@ -414,11 +506,15 @@ _jit_prolog(jit_state_t *_jit) _jitc->functions.length += 16; } _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++; - _jitc->function->self.size = stack_framesize; + /* One extra stack slot for implicit saved returned address */ + _jitc->function->self.size = stack_framesize + REAL_WORDSIZE; _jitc->function->self.argi = _jitc->function->self.argf = _jitc->function->self.aoff = _jitc->function->self.alen = 0; - /* sse/x87 conversion */ - _jitc->function->self.aoff = CVT_OFFSET; + _jitc->function->cvt_offset = 0; +#if __X64 && (__CYGWIN__ || _WIN32) + /* force framepointer */ + jit_check_frame(); +#endif _jitc->function->self.call = jit_call_default; jit_alloc((jit_pointer_t *)&_jitc->function->regoff, _jitc->reglen * sizeof(jit_int32_t)); @@ -444,6 +540,13 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + jit_check_frame(); +#if __X32 + /* Stack is 4 bytes aligned but jit functions keep it 8 bytes aligned. + * Called functions have 16 byte aligned stack. */ + if (!_jitc->function->self.aoff) + _jitc->function->self.aoff = -4; +#endif switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -500,22 +603,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - /* movr(%ret, %ret) would be optimized out */ - if (JIT_RET != u) - jit_movr(JIT_RET, u); - /* explicitly tell it is live */ - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -575,7 +674,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_f_reg_p(u->u.w)); @@ -585,6 +684,7 @@ void _jit_ellipsis(jit_state_t *_jit) { jit_inc_synth(ellipsis); + jit_check_frame(); if (_jitc->prepare) { jit_link_prepare(); /* Remember that a varargs function call is being constructed. */ @@ -629,12 +729,15 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; assert(_jitc->function); assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif #if __X64 if (jit_arg_reg_p(_jitc->function->self.argi)) { offset = _jitc->function->self.argi++; @@ -647,8 +750,9 @@ _jit_arg(jit_state_t *_jit) { offset = _jitc->function->self.size; _jitc->function->self.size += REAL_WORDSIZE; + jit_check_frame(); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -676,6 +780,7 @@ _jit_arg_f(jit_state_t *_jit) { offset = _jitc->function->self.size; _jitc->function->self.size += REAL_WORDSIZE; + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_f, offset, ++_jitc->function->self.argn); @@ -705,6 +810,7 @@ _jit_arg_d(jit_state_t *_jit) { offset = _jitc->function->self.size; _jitc->function->self.size += sizeof(jit_float64_t); + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_d, offset, ++_jitc->function->self.argn); @@ -715,63 +821,75 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, JIT_RA0 - v->u.w); else #endif - jit_ldxi_c(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_c(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, JIT_RA0 - v->u.w); else #endif - jit_ldxi_uc(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_uc(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, JIT_RA0 - v->u.w); else #endif - jit_ldxi_s(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_s(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, JIT_RA0 - v->u.w); else #endif - jit_ldxi_us(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_us(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) { @@ -783,7 +901,10 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) } else #endif - jit_ldxi_i(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_i(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -791,57 +912,66 @@ _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); if (jit_arg_reg_p(v->u.w)) jit_extr_ui(u, JIT_RA0 - v->u.w); - else - jit_ldxi_ui(u, _RBP, v->u.w); + else { + jit_node_t *node = jit_ldxi_ui(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); - else - jit_ldxi_l(u, _RBP, v->u.w); + else { + jit_node_t *node = jit_ldxi_l(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } #endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) jit_movr(JIT_RA0 - v->u.w, u); else #endif - jit_stxi(v->u.w, _RBP, u); + { + jit_node_t *node = jit_stxi(v->u.w, _RBP, u); + jit_link_alist(node); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); #if __X64 if (jit_arg_reg_p(v->u.w)) jit_movi(JIT_RA0 - v->u.w, u); else #endif { + jit_node_t *node; regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w, _RBP, regno); + node = jit_stxi(v->u.w, _RBP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -857,7 +987,10 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr_f(u, _XMM0 - v->u.w); else #endif - jit_ldxi_f(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_f(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -867,11 +1000,14 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) assert(v->code == jit_code_arg_f); jit_inc_synth_wp(putargr_f, u, v); #if __X64 - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(_XMM0 - v->u.w, u); else #endif - jit_stxi_f(v->u.w, _RBP, u); + { + jit_node_t *node = jit_stxi_f(v->u.w, _RBP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -882,14 +1018,16 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) assert(v->code == jit_code_arg_f); jit_inc_synth_fp(putargi_f, u, v); #if __X64 - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_f_reg_p(v->u.w)) jit_movi_f(_XMM0 - v->u.w, u); else #endif { - regno = jit_get_reg(jit_class_gpr); + jit_node_t *node; + regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, _RBP, regno); + node = jit_stxi_f(v->u.w, _RBP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -905,7 +1043,10 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_movr_d(u, _XMM0 - v->u.w); else #endif - jit_ldxi_d(u, _RBP, v->u.w); + { + jit_node_t *node = jit_ldxi_d(u, _RBP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -915,11 +1056,14 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) assert(v->code == jit_code_arg_d); jit_inc_synth_wp(putargr_d, u, v); #if __X64 - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(_XMM0 - v->u.w, u); else #endif - jit_stxi_d(v->u.w, _RBP, u); + { + jit_node_t *node = jit_stxi_d(v->u.w, _RBP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -930,24 +1074,26 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) assert(v->code == jit_code_arg_d); jit_inc_synth_dp(putargi_d, u, v); #if __X64 - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_f_reg_p(v->u.w)) jit_movi_d(_XMM0 - v->u.w, u); else #endif { - regno = jit_get_reg(jit_class_gpr); + jit_node_t *node; + regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, _RBP, regno); + node = jit_stxi_d(v->u.w, _RBP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); #if __X64 if (jit_arg_reg_p(_jitc->function->call.argi)) { @@ -964,16 +1110,17 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) { jit_stxi(_jitc->function->call.size, _RSP, u); _jitc->function->call.size += REAL_WORDSIZE; + jit_check_frame(); } jit_dec_synth(); } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); #if __X64 if (jit_arg_reg_p(_jitc->function->call.argi)) { @@ -994,6 +1141,7 @@ _jit_pushargi(jit_state_t *_jit, jit_word_t u) jit_stxi(_jitc->function->call.size, _RSP, regno); _jitc->function->call.size += REAL_WORDSIZE; jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -1028,6 +1176,7 @@ _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) { jit_stxi_f(_jitc->function->call.size, _RSP, u); _jitc->function->call.size += REAL_WORDSIZE; + jit_check_frame(); } jit_dec_synth(); } @@ -1066,6 +1215,7 @@ _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) jit_stxi_f(_jitc->function->call.size, _RSP, regno); _jitc->function->call.size += REAL_WORDSIZE; jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -1100,6 +1250,7 @@ _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) { jit_stxi_d(_jitc->function->call.size, _RSP, u); _jitc->function->call.size += sizeof(jit_float64_t); + jit_check_frame(); } jit_dec_synth(); } @@ -1138,6 +1289,7 @@ _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) jit_stxi_d(_jitc->function->call.size, _RSP, regno); _jitc->function->call.size += sizeof(jit_float64_t); jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -1171,6 +1323,7 @@ _jit_finishr(jit_state_t *_jit, jit_int32_t r0) jit_int32_t reg; jit_node_t *call; assert(_jitc->function); + jit_check_frame(); reg = r0; jit_inc_synth_w(finishr, r0); if (_jitc->function->self.alen < _jitc->function->call.size) @@ -1203,32 +1356,26 @@ _jit_finishr(jit_state_t *_jit, jit_int32_t r0) jit_node_t * _jit_finishi(jit_state_t *_jit, jit_pointer_t i0) { -#if __X64 - jit_int32_t reg; -#endif jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishi, (jit_word_t)i0); if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; #if __X64 - /* FIXME preventing %rax allocation is good enough, but for consistency - * it should automatically detect %rax is dead, in case it has run out - * registers, and not save/restore it, what would be wrong if using the - * the return value, otherwise, just a needless noop */ - /* >> prevent %rax from being allocated as the function pointer */ - jit_regset_setbit(&_jitc->regarg, _RAX); - reg = jit_get_reg(jit_class_gpr); - node = jit_movi(reg, (jit_word_t)i0); - jit_finishr(reg); - jit_unget_reg(reg); - /* << prevent %rax from being allocated as the function pointer */ - jit_regset_clrbit(&_jitc->regarg, _RAX); -#else +# if !(__CYGWIN__ || _WIN32) + if (_jitc->function->call.call & jit_call_varargs) { + if (_jitc->function->call.argf) + jit_movi(_RAX, _jitc->function->call.argf); + else + jit_movi(_RAX, 0); + jit_live(_RAX); + } +# endif +#endif node = jit_calli(i0); node->v.w = _jitc->function->call.argi; node->w.w = _jitc->function->call.argf; -#endif _jitc->function->call.argi = _jitc->function->call.argf = _jitc->function->call.size = 0; _jitc->prepare = 0; @@ -1333,6 +1480,7 @@ _emit_code(jit_state_t *_jit) struct { jit_node_t *node; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1593,12 +1741,15 @@ _emit_code(jit_state_t *_jit) jit_regarg_set(node, value); switch (node->code) { case jit_code_align: - assert(!(node->u.w & (node->u.w - 1)) && - node->u.w <= sizeof(jit_word_t)); + /* Must align to a power of two */ + assert(!(node->u.w & (node->u.w - 1))); if ((word = _jit->pc.w & (node->u.w - 1))) nop(node->u.w - word); break; - case jit_code_note: case jit_code_name: + case jit_code_skip: + nop(node->u.w); + break; + case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; break; case jit_code_label: @@ -1654,6 +1805,10 @@ _emit_code(jit_state_t *_jit) case_rrw(rsh, _u); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); case_rrr(lt,); case_rrw(lt,); case_rrr(lt, _u); @@ -1674,6 +1829,16 @@ _emit_code(jit_state_t *_jit) case_rrw(gt, _u); case_rrr(ne,); case_rrw(ne,); + case jit_code_casr: + casr(rn(node->u.w), rn(node->v.w), + rn(node->w.q.l), rn(node->w.q.h)); + break; + case jit_code_casi: + casi(rn(node->u.w), node->v.w, + rn(node->w.q.l), rn(node->w.q.h)); + break; + case_rrr(movn,); + case_rrr(movz,); case_rr(mov,); case jit_code_movi: if (node->flag & jit_flag_node) { @@ -1685,7 +1850,14 @@ _emit_code(jit_state_t *_jit) else { assert(temp->code == jit_code_label || temp->code == jit_code_epilog); - word = movi_p(rn(node->u.w), node->v.w); +#if CAN_RIP_ADDRESS + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if ((jit_int32_t)word == word) + word = movi(rn(node->u.w), _jit->pc.w); + else +#endif + word = movi_p(rn(node->u.w), node->v.w); patch(word, node); } } @@ -1696,6 +1868,11 @@ _emit_code(jit_state_t *_jit) case_rr(hton, _ui); #if __X64 && !__X64_32 case_rr(hton, _ul); +#endif + case_rr(bswap, _us); + case_rr(bswap, _ui); +#if __X64 && !__X64_32 + case_rr(bswap, _ul); #endif case_rr(ext, _c); case_rr(ext, _uc); @@ -2002,6 +2179,7 @@ _emit_code(jit_state_t *_jit) case_bff(unord, _d); case_bfw(unord, _d, 64); case jit_code_jmpr: + jit_check_frame(); jmpr(rn(node->u.w)); break; case jit_code_jmpi: @@ -2012,14 +2190,24 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi(_jit->pc.w); +#if __X64 + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if ((jit_int32_t)word == word) + word = jmpi(_jit->pc.w); + else +#endif + word = jmpi_p(_jit->pc.w); patch(word, node); } } - else + else { + jit_check_frame(); jmpi(node->u.w); + } break; case jit_code_callr: + jit_check_frame(); callr(rn(node->u.w)); break; case jit_code_calli: @@ -2027,22 +2215,37 @@ _emit_code(jit_state_t *_jit) temp = node->u.n; assert(temp->code == jit_code_label || temp->code == jit_code_epilog); - word = calli(temp->u.w); - if (!(temp->flag & jit_flag_patch)) + if (temp->flag & jit_flag_patch) + calli(temp->u.w); + else { +#if __X64 + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if ((jit_int32_t)word == word) + word = calli(_jit->pc.w); + else +#endif + word = calli_p(_jit->pc.w); patch(word, node); + } } - else + else { + jit_check_frame(); calli(node->u.w); + } break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif undo.patch_offset = _jitc->patches.offset; restart_function: + compute_framesize(); + patch_alist(0); _jitc->again = 0; prolog(node); break; @@ -2058,10 +2261,29 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + /* real stack framesize is not in the jit_function_t, + * if it were, would need to not be undone */ + /* cvt_offset must also not be undone */ + undo.func.cvt_offset = _jitc->function->cvt_offset; + /* this will be recomputed but undo anyway to have it + * better self documented.*/ + undo.func.need_stack = _jitc->function->need_stack; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif _jitc->patches.offset = undo.patch_offset; + patch_alist(1); goto restart_function; } if (node->link && @@ -2085,11 +2307,23 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +# if __WORDSIZE == 64 + case jit_code_arg_l: +# endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -2099,10 +2333,26 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_ui: case jit_code_getarg_l: #endif case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -2164,7 +2414,7 @@ _emit_code(jit_state_t *_jit) for (offset = 0; offset < _jitc->patches.offset; offset++) { node = _jitc->patches.ptr[offset].node; word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w; - patch_at(node, _jitc->patches.ptr[offset].inst, word); + patch_at(_jitc->patches.ptr[offset].inst, word); } jit_flush(_jit->code.ptr, _jit->pc.uc); @@ -2213,6 +2463,26 @@ _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1) sse_stxi_d(i0, rn(r0), rn(r1)); } +static void +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + /* Save stack pointer in first slot */ + _jitc->framesize = REAL_WORDSIZE; + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += REAL_WORDSIZE; + +#if __X64 && (__CYGWIN__ || _WIN32) + for (reg = 0; reg < jit_size(fregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) + _jitc->framesize += sizeof(jit_float64_t); +#endif + /* Make sure functions called have a 16 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 15) & -16; + _jitc->framesize += 16 - REAL_WORDSIZE; +} + static void _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) { @@ -2238,6 +2508,7 @@ _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) static void _sse_from_x87_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); x87_stxi_f(CVT_OFFSET, _RBP_REGNO, r1); sse_ldxi_f(r0, _RBP_REGNO, CVT_OFFSET); } @@ -2245,6 +2516,7 @@ _sse_from_x87_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _sse_from_x87_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); x87_stxi_d(CVT_OFFSET, _RBP_REGNO, r1); sse_ldxi_d(r0, _RBP_REGNO, CVT_OFFSET); } @@ -2252,6 +2524,7 @@ _sse_from_x87_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _x87_from_sse_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); sse_stxi_f(CVT_OFFSET, _RBP_REGNO, r1); x87_ldxi_f(r0, _RBP_REGNO, CVT_OFFSET); } @@ -2259,6 +2532,7 @@ _x87_from_sse_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) static void _x87_from_sse_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) { + CHECK_CVT_OFFSET(); sse_stxi_d(CVT_OFFSET, _RBP_REGNO, r1); x87_ldxi_d(r0, _RBP_REGNO, CVT_OFFSET); } diff --git a/parallel-rsp/lightning/lib/lightning.c b/parallel-rsp/lightning/lib/lightning.c index 538556d..97cc358 100644 --- a/parallel-rsp/lightning/lib/lightning.c +++ b/parallel-rsp/lightning/lib/lightning.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2019 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -19,7 +19,9 @@ #include #include -#include +#if HAVE_MMAP +# include +#endif #if defined(__sgi) # include #endif @@ -62,13 +64,28 @@ static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*); static void _jit_dataset(jit_state_t *_jit); +#define block_update_set(block, target) _block_update_set(_jit, block, target) +static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*); + +#define propagate_backward(block) _propagate_backward(_jit, block) +static void _propagate_backward(jit_state_t*, jit_block_t*); + +#define check_block_again() _check_block_again(_jit) +static jit_bool_t _check_block_again(jit_state_t*); + +#define do_setup() _do_setup(_jit) +static void _do_setup(jit_state_t*); + #define jit_setup(block) _jit_setup(_jit, block) static void _jit_setup(jit_state_t *_jit, jit_block_t *block); -#define jit_follow(block, todo) _jit_follow(_jit, block, todo) +#define do_follow(always) _do_follow(_jit, always) +static void _do_follow(jit_state_t*, jit_bool_t); + +#define jit_follow(block) _jit_follow(_jit, block) static void -_jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo); +_jit_follow(jit_state_t *_jit, jit_block_t *block); #define jit_update(node, live, mask) _jit_update(_jit, node, live, mask) static void @@ -103,7 +120,7 @@ static jit_bool_t _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node); #define redundant_store(node, jump) _redundant_store(_jit, node, jump) -static void +static jit_bool_t _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump); #define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s) @@ -129,7 +146,7 @@ static void _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno); #define simplify() _simplify(_jit) -static void +static jit_bool_t _simplify(jit_state_t *_jit); #define jit_reg_undef -1 @@ -210,8 +227,25 @@ _jit_get_reg(jit_state_t *_jit, jit_int32_t regspec) for (regno = 0; regno < _jitc->reglen; regno++) { if ((jit_class(_rvs[regno].spec) & spec) == spec && !jit_regset_tstbit(&_jitc->regarg, regno) && - !jit_regset_tstbit(&_jitc->reglive, regno)) + !jit_regset_tstbit(&_jitc->reglive, regno)) { + if (jit_regset_tstbit(&_jitc->regmask, regno)) { + /* search further, attempting to find a truly known + * free register, not just one in unknown state. */ + jit_int32_t regfree; + + for (regfree = regno + 1; + regfree < _jitc->reglen; regfree++) { + if ((jit_class(_rvs[regfree].spec) & spec) == spec && + !jit_regset_tstbit(&_jitc->regarg, regfree) && + !jit_regset_tstbit(&_jitc->reglive, regfree) && + !jit_regset_tstbit(&_jitc->regmask, regfree)) { + regno = regfree; + break; + } + } + } goto regarg; + } } /* search for a register matching spec that is not an argument @@ -857,6 +891,7 @@ jit_new_state(void) jit_regset_new(&_jitc->regsav); jit_regset_new(&_jitc->reglive); jit_regset_new(&_jitc->regmask); + jit_regset_new(&_jitc->explive); jit_init(); @@ -956,10 +991,12 @@ _jit_destroy_state(jit_state_t *_jit) #if DEVEL_DISASSEMBLER jit_really_clear_state(); #endif +#if HAVE_MMAP if (!_jit->user_code) munmap(_jit->code.ptr, _jit->code.length); if (!_jit->user_data) munmap(_jit->data.ptr, _jit->data.length); +#endif jit_free((jit_pointer_t *)&_jit); } @@ -1134,6 +1171,20 @@ _jit_new_node_qww(jit_state_t *_jit, jit_code_t code, return (link_node(node)); } +jit_node_t * +_jit_new_node_wwq(jit_state_t *_jit, jit_code_t code, + jit_word_t u, jit_word_t v, + jit_int32_t l, jit_int32_t h) +{ + jit_node_t *node = new_node(code); + assert(!_jitc->realize); + node->u.w = u; + node->v.w = v; + node->w.q.l = l; + node->w.q.h = h; + return (link_node(node)); +} + jit_node_t * _jit_new_node_wwf(jit_state_t *_jit, jit_code_t code, jit_word_t u, jit_word_t v, jit_float32_t w) @@ -1302,14 +1353,36 @@ _jit_classify(jit_state_t *_jit, jit_code_t code) mask = 0; break; case jit_code_live: case jit_code_va_end: - case jit_code_retr: case jit_code_retr_f: case jit_code_retr_d: - case jit_code_pushargr: case jit_code_pushargr_f: + case jit_code_retr_c: case jit_code_retr_uc: + case jit_code_retr_s: case jit_code_retr_us: + case jit_code_retr_i: case jit_code_retr_ui: + case jit_code_retr_l: + case jit_code_retr_f: case jit_code_retr_d: + case jit_code_pushargr_c: + case jit_code_pushargr_uc: + case jit_code_pushargr_s: + case jit_code_pushargr_us: + case jit_code_pushargr_i: + case jit_code_pushargr_ui: + case jit_code_pushargr_l: + case jit_code_pushargr_f: case jit_code_pushargr_d: case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */ mask = jit_cc_a0_reg; break; - case jit_code_align: case jit_code_reti: case jit_code_pushargi: - case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */ + case jit_code_align: case jit_code_skip: + case jit_code_reti_c: case jit_code_reti_uc: + case jit_code_reti_s: case jit_code_reti_us: + case jit_code_reti_i: case jit_code_reti_ui: + case jit_code_reti_l: + case jit_code_pushargi_c: + case jit_code_pushargi_uc: + case jit_code_pushargi_s: + case jit_code_pushargi_us: + case jit_code_pushargi_i: + case jit_code_pushargi_ui: + case jit_code_pushargi_l: + case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */ mask = jit_cc_a0_int; break; case jit_code_reti_f: case jit_code_pushargi_f: @@ -1321,7 +1394,9 @@ _jit_classify(jit_state_t *_jit, jit_code_t code) case jit_code_allocai: mask = jit_cc_a0_int|jit_cc_a1_int; break; - case jit_code_arg: case jit_code_arg_f: case jit_code_arg_d: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_l: + case jit_code_arg_f: case jit_code_arg_d: mask = jit_cc_a0_int|jit_cc_a0_arg; break; case jit_code_calli: case jit_code_jmpi: @@ -1345,11 +1420,17 @@ _jit_classify(jit_state_t *_jit, jit_code_t code) case jit_code_getarg_f: case jit_code_getarg_d: mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg; break; - case jit_code_putargr: case jit_code_putargr_f: - case jit_code_putargr_d: + case jit_code_putargr_c:case jit_code_putargr_uc: + case jit_code_putargr_s:case jit_code_putargr_us: + case jit_code_putargr_i:case jit_code_putargr_ui: + case jit_code_putargr_l: + case jit_code_putargr_f:case jit_code_putargr_d: mask = jit_cc_a0_reg|jit_cc_a1_arg; break; - case jit_code_putargi: + case jit_code_putargi_c:case jit_code_putargi_uc: + case jit_code_putargi_s:case jit_code_putargi_us: + case jit_code_putargi_i:case jit_code_putargi_ui: + case jit_code_putargi_l: mask = jit_cc_a0_int|jit_cc_a1_arg; break; case jit_code_putargi_f: @@ -1380,6 +1461,7 @@ _jit_classify(jit_state_t *_jit, jit_code_t code) case jit_code_truncr_f_i: case jit_code_truncr_f_l: case jit_code_truncr_d_i: case jit_code_truncr_d_l: case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul: + case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul: case jit_code_ldr_c: case jit_code_ldr_uc: case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i: case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f: @@ -1388,6 +1470,8 @@ _jit_classify(jit_state_t *_jit, jit_code_t code) case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d: case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d: case jit_code_ldr_d: + case jit_code_clor: case jit_code_clzr: + case jit_code_ctor: case jit_code_ctzr: case jit_code_movr_w_f: case jit_code_movr_f_w: case jit_code_movr_w_d: case jit_code_movr_d_w: case jit_code_va_arg: case jit_code_va_arg_d: @@ -1531,6 +1615,17 @@ _jit_classify(jit_state_t *_jit, jit_code_t code) case jit_code_bxsubr: case jit_code_bxsubr_u: mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg; break; + case jit_code_movnr: case jit_code_movzr: + mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg; + break; + case jit_code_casr: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg| + jit_cc_a2_reg|jit_cc_a2_rlh; + break; + case jit_code_casi: + mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int| + jit_cc_a2_reg|jit_cc_a2_rlh; + break; default: abort(); } @@ -1593,6 +1688,146 @@ _jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label) label->link = instr; } +static void +_do_setup(jit_state_t *_jit) +{ + jit_block_t *block; + jit_word_t offset; + + /* create initial mapping of live register values + * at the start of a basic block */ + for (offset = 0; offset < _jitc->blocks.offset; offset++) { + block = _jitc->blocks.ptr + offset; + if (!block->label) + continue; + if (block->label->code == jit_code_epilog) { + jit_regset_setbit(&block->reglive, JIT_RET); + jit_regset_setbit(&block->reglive, JIT_FRET); + jit_regset_com(&block->regmask, &block->reglive); + continue; + } + jit_setup(block); + } +} + +static jit_bool_t +_block_update_set(jit_state_t *_jit, + jit_block_t *block, jit_block_t *target) +{ + jit_regset_t regmask; + + jit_regset_ior(®mask, &block->reglive, &target->reglive); + jit_regset_and(®mask, ®mask, &block->regmask); + if (jit_regset_set_p(®mask)) { + jit_regset_ior(&block->reglive, &block->reglive, ®mask); + jit_regset_and(®mask, &block->reglive, &block->regmask); + jit_regset_com(®mask, ®mask); + jit_regset_and(&block->regmask, &block->regmask, ®mask); + block->again = 1; + return (1); + } + return (0); +} + +static void +_propagate_backward(jit_state_t *_jit, jit_block_t *block) +{ + jit_block_t *prev; + jit_word_t offset; + + for (offset = block->label->v.w - 1; + offset >= 0; --offset) { + prev = _jitc->blocks.ptr + offset; + if (!block_update_set(prev, block) || + !(prev->label->flag & jit_flag_head)) + break; + } +} + +static jit_bool_t +_check_block_again(jit_state_t *_jit) +{ + jit_int32_t todo; + jit_word_t offset; + jit_node_t *node, *label; + jit_block_t *block, *target; + + todo = 0; + for (offset = 0; offset < _jitc->blocks.offset; offset++) { + block = _jitc->blocks.ptr + offset; + if (block->again) { + todo = 1; + break; + } + } + /* If no block changed state */ + if (!todo) + return (0); + + do { + todo = 0; + block = NULL; + for (node = _jitc->head; node; node = node->next) { + /* Special jumps that match jit_cc_a0_jmp */ + if (node->code == jit_code_calli || node->code == jit_code_callr) + continue; + + /* Remember current label */ + if (node->code == jit_code_label || + node->code == jit_code_prolog || + node->code == jit_code_epilog) { + + /* If previous block does not pass through */ + if (!(node->flag & jit_flag_head)) + block = NULL; + + target = _jitc->blocks.ptr + node->v.w; + if (block && target->again && block_update_set(block, target)) { + propagate_backward(block); + todo = 1; + } + block = target; + } + /* If not the first jmpi */ + else if (block) { + /* If a jump to dynamic address or if a jump to raw address */ + if (!(jit_classify(node->code) & jit_cc_a0_jmp) || + !(node->flag & jit_flag_node)) + continue; + label = node->u.n; + /* Mark predecessor needs updating due to target change */ + target = _jitc->blocks.ptr + label->v.w; + if (target->again && block_update_set(block, target)) { + propagate_backward(block); + todo = 1; + } + } + } + } + while (todo); + + return (todo); +} + +static void +_do_follow(jit_state_t *_jit, jit_bool_t always) +{ + jit_block_t *block; + jit_word_t offset; + + /* set live state of registers not referenced in a block, but + * referenced in a jump target or normal flow */ + for (offset = 0; offset < _jitc->blocks.offset; offset++) { + block = _jitc->blocks.ptr + offset; + if (!block->label || block->label->code == jit_code_epilog) + continue; + if (always || block->again) { + block->again = 0; + jit_follow(block); + } + } +} + void _jit_optimize(jit_state_t *_jit) { @@ -1602,48 +1837,46 @@ _jit_optimize(jit_state_t *_jit) jit_node_t *node; jit_block_t *block; jit_word_t offset; + jit_regset_t regmask; + todo = 0; _jitc->function = NULL; thread_jumps(); sequential_labels(); split_branches(); - - /* create initial mapping of live register values - * at the start of a basic block */ - for (offset = 0; offset < _jitc->blocks.offset; offset++) { - block = _jitc->blocks.ptr + offset; - if (!block->label) - continue; - if (block->label->code != jit_code_epilog) - jit_setup(block); - } - - /* set live state of registers not referenced in a block, but - * referenced in a jump target or normal flow */ - do { - todo = 0; - for (offset = 0; offset < _jitc->blocks.offset; offset++) { - block = _jitc->blocks.ptr + offset; - if (!block->label) - continue; - if (block->label->code != jit_code_epilog) - jit_follow(block, &todo); - } - } while (todo); + do_setup(); + do_follow(1); patch_registers(); - simplify(); + if (simplify()) + todo = 1; - /* figure out labels that are only reached with a jump - * and is required to do a simple redundant_store removal - * on jit_beqi below */ + jit_regset_set_ui(®mask, 0); + for (offset = 0; offset < _jitc->reglen; offset++) { + if ((jit_class(_rvs[offset].spec) & (jit_class_gpr|jit_class_fpr)) && + (jit_class(_rvs[offset].spec) & jit_class_sav) == jit_class_sav) + jit_regset_setbit(®mask, offset); + } + + /* Figure out labels that are only reached with a jump */ jump = 1; for (node = _jitc->head; node; node = node->next) { switch (node->code) { case jit_code_label: - if (!jump) + if (!jump) { node->flag |= jit_flag_head; + if (!node->link) { + /* Block is dead code or only reachable with an + * indirect jumps. In such condition, must assume + * all callee save registers are live. */ + block = _jitc->blocks.ptr + node->v.w; + jit_regset_ior(&block->reglive, + &block->reglive, ®mask); + /* Cleanup regmask */ + block_update_set(block, block); + } + } break; case jit_code_jmpi: case jit_code_jmpr: case jit_code_epilog: @@ -1665,69 +1898,98 @@ _jit_optimize(jit_state_t *_jit) node->v.w &= ~jit_regno_patch; if (mask & jit_cc_a2_reg) node->w.w &= ~jit_regno_patch; - switch (node->code) { - case jit_code_prolog: - _jitc->function = _jitc->functions.ptr + node->w.w; - break; - case jit_code_epilog: - _jitc->function = NULL; - break; - case jit_code_beqi: - redundant_store(node, 1); - break; - case jit_code_bnei: - redundant_store(node, 0); - break; - default: + if (node->code == jit_code_beqi) { + if (redundant_store(node, 1)) { + block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w; + block->again = 1; + todo = 1; + } + } + else if (node->code == jit_code_bnei) { + if (redundant_store(node, 0)) { + block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w; + block->again = 1; + todo = 1; + } + } + } + + if (!todo) + todo = check_block_again(); + + /* If instructions were removed or first pass did modify the entry + * state of any block */ + if (todo) { + do_setup(); + todo = 0; + do { + do_follow(0); + /* If any block again has the entry state modified. */ + todo = check_block_again(); + } while (todo); + } + + for (node = _jitc->head; node; node = node->next) { + mask = jit_classify(node->code); + if (mask & jit_cc_a0_reg) + node->u.w &= ~jit_regno_patch; + if (mask & jit_cc_a1_reg) + node->v.w &= ~jit_regno_patch; + if (mask & jit_cc_a2_reg) + node->w.w &= ~jit_regno_patch; + if (node->code == jit_code_prolog) + _jitc->function = _jitc->functions.ptr + node->w.w; + else if(node->code == jit_code_epilog) + _jitc->function = NULL; + else { #if JIT_HASH_CONSTS - if (mask & jit_cc_a0_flt) { - node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a0_dbl) { - node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a1_flt) { - node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a1_dbl) { - node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a2_flt) { - node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4); - node->flag |= jit_flag_node | jit_flag_data; - } - else if (mask & jit_cc_a2_dbl) { - node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8); - node->flag |= jit_flag_node | jit_flag_data; - } + if (mask & jit_cc_a0_flt) { + node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a0_dbl) { + node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a1_flt) { + node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a1_dbl) { + node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a2_flt) { + node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4); + node->flag |= jit_flag_node | jit_flag_data; + } + else if (mask & jit_cc_a2_dbl) { + node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8); + node->flag |= jit_flag_node | jit_flag_data; + } #endif - if (_jitc->function) { - if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) == - (jit_cc_a0_reg|jit_cc_a0_chg)) { - if (mask & jit_cc_a0_rlh) { - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->u.q.l)); - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->u.q.h)); - } - else - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->u.w)); + if (_jitc->function) { + if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) == + (jit_cc_a0_reg|jit_cc_a0_chg)) { + if (mask & jit_cc_a0_rlh) { + jit_regset_setbit(&_jitc->function->regset, + jit_regno(node->u.q.l)); + jit_regset_setbit(&_jitc->function->regset, + jit_regno(node->u.q.h)); } - if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) == - (jit_cc_a1_reg|jit_cc_a1_chg)) + else jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->v.w)); - if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) == - (jit_cc_a2_reg|jit_cc_a2_chg)) - jit_regset_setbit(&_jitc->function->regset, - jit_regno(node->w.w)); + jit_regno(node->u.w)); } - break; + if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) == + (jit_cc_a1_reg|jit_cc_a1_chg)) + jit_regset_setbit(&_jitc->function->regset, + jit_regno(node->v.w)); + if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) == + (jit_cc_a2_reg|jit_cc_a2_chg)) + jit_regset_setbit(&_jitc->function->regset, + jit_regno(node->w.w)); + } } } } @@ -1743,6 +2005,10 @@ _jit_reglive(jit_state_t *_jit, jit_node_t *node) case jit_code_label: case jit_code_prolog: case jit_code_epilog: block = _jitc->blocks.ptr + node->v.w; jit_regset_set(&_jitc->reglive, &block->reglive); + jit_regset_set_ui(&_jitc->explive, 0); + break; + case jit_code_live: + jit_regset_setbit(&_jitc->explive, node->u.w); break; case jit_code_callr: value = jit_regno(node->u.w); @@ -1798,13 +2064,24 @@ _jit_reglive(jit_state_t *_jit, jit_node_t *node) else jit_regset_setbit(&_jitc->reglive, node->v.w); } - if ((value & jit_cc_a2_reg) && !(node->w.w & jit_regno_patch)) { - if (value & jit_cc_a2_chg) { - jit_regset_clrbit(&_jitc->reglive, node->w.w); - jit_regset_setbit(&_jitc->regmask, node->w.w); + if (value & jit_cc_a2_reg) { + if (value & jit_cc_a2_rlh) { + /* Assume registers are not changed */ + if (!(node->w.q.l & jit_regno_patch)) + jit_regset_setbit(&_jitc->reglive, node->w.q.l); + if (!(node->w.q.h & jit_regno_patch)) + jit_regset_setbit(&_jitc->reglive, node->w.q.h); + } + else { + if (!(node->w.w & jit_regno_patch)) { + if (value & jit_cc_a2_chg) { + jit_regset_clrbit(&_jitc->reglive, node->w.w); + jit_regset_setbit(&_jitc->regmask, node->w.w); + } + else + jit_regset_setbit(&_jitc->reglive, node->w.w); + } } - else - jit_regset_setbit(&_jitc->reglive, node->w.w); } if (jit_regset_set_p(&_jitc->regmask)) { jit_update(node->next, &_jitc->reglive, &_jitc->regmask); @@ -1835,8 +2112,27 @@ _jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value) } if (value & jit_cc_a1_reg) jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w)); - if (value & jit_cc_a2_reg) - jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w)); + if (value & jit_cc_a2_reg) { + if (value & jit_cc_a2_rlh) { + jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l)); + jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h)); + } + else + jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w)); + } + /* Prevent incorrect detection of running out of registers + * if will need to patch jump, and all registers have been + * used in the current block. */ + if (node->code == jit_code_jmpi && (node->flag & jit_flag_node)) { + jit_node_t *label = node->u.n; + jit_block_t *block = _jitc->blocks.ptr + label->v.w; + jit_regset_set(&_jitc->reglive, &block->reglive); + jit_regset_set(&_jitc->regmask, &block->regmask); + if (jit_regset_set_p(&_jitc->explive)) { + jit_regset_ior(&_jitc->reglive, &block->reglive, &_jitc->explive); + jit_regset_xor(&_jitc->regmask, &_jitc->regmask, &_jitc->explive); + } + } } void @@ -1855,8 +2151,14 @@ _jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value) } if (value & jit_cc_a1_reg) jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w)); - if (value & jit_cc_a2_reg) - jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w)); + if (value & jit_cc_a2_reg) { + if (value & jit_cc_a2_rlh) { + jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l)); + jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h)); + } + else + jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w)); + } } void @@ -1891,6 +2193,9 @@ _jit_dataset(jit_state_t *_jit) #endif assert(!_jitc->dataset); +#if !HAVE_MMAP + assert(_jit->user_data); +#else if (!_jit->user_data) { /* create read only data buffer */ @@ -1908,6 +2213,7 @@ _jit_dataset(jit_state_t *_jit) close(mmap_fd); #endif } +#endif /* !HAVE_MMAP */ if (!_jitc->no_data) jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset); @@ -2013,6 +2319,7 @@ _jit_emit(jit_state_t *_jit) #if defined(__sgi) int mmap_fd; #endif + int mmap_prot, mmap_flags; if (!_jitc->realize) jit_realize(); @@ -2022,20 +2329,39 @@ _jit_emit(jit_state_t *_jit) _jitc->emit = 1; +#if !HAVE_MMAP + assert(_jit->user_code); +#else if (!_jit->user_code) { + mmap_prot = PROT_READ | PROT_WRITE; +#if !(__OpenBSD__ || __APPLE__) + mmap_prot |= PROT_EXEC; +#endif +#if __NetBSD__ + mmap_prot = PROT_MPROTECT(mmap_prot); + mmap_flags = 0; +#else + mmap_flags = MAP_PRIVATE; +#endif + mmap_flags |= MAP_ANON; #if defined(__sgi) mmap_fd = open("/dev/zero", O_RDWR); #endif _jit->code.ptr = mmap(NULL, _jit->code.length, - PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, mmap_fd, 0); + mmap_prot, mmap_flags, mmap_fd, 0); assert(_jit->code.ptr != MAP_FAILED); } +#endif /* !HAVE_MMAP */ _jitc->code.end = _jit->code.ptr + _jit->code.length - jit_get_max_instr(); _jit->pc.uc = _jit->code.ptr; for (;;) { +#if __NetBSD__ + result = mprotect(_jit->code.ptr, _jit->code.length, + PROT_READ | PROT_WRITE); + assert(result == 0); +#endif if ((code = emit_code()) == NULL) { _jitc->patches.offset = 0; for (node = _jitc->head; node; node = node->next) { @@ -2044,6 +2370,9 @@ _jit_emit(jit_state_t *_jit) node->code == jit_code_epilog)) node->flag &= ~jit_flag_patch; } +#if !HAVE_MMAP + assert(_jit->user_code); +#else if (_jit->user_code) goto fail; #if GET_JIT_SIZE @@ -2068,8 +2397,7 @@ _jit_emit(jit_state_t *_jit) # endif #else _jit->code.ptr = mmap(NULL, length, - PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, mmap_fd, 0); + mmap_prot, mmap_flags, mmap_fd, 0); #endif assert(_jit->code.ptr != MAP_FAILED); @@ -2077,6 +2405,7 @@ _jit_emit(jit_state_t *_jit) _jitc->code.end = _jit->code.ptr + _jit->code.length - jit_get_max_instr(); _jit->pc.uc = _jit->code.ptr; +#endif /* !HAVE_MMAP */ } else break; @@ -2093,30 +2422,63 @@ _jit_emit(jit_state_t *_jit) if (_jit->user_data) jit_free((jit_pointer_t *)&_jitc->data.ptr); +#if HAVE_MMAP else { #ifdef _WIN32 - result = _mprotect(_jit->data.ptr, _jit->data.length, PROT_READ); + result = _mprotect(_jit->data.ptr, + _jit->data.length, PROT_READ); #else - result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ); + result = mprotect(_jit->data.ptr, + _jit->data.length, PROT_READ); #endif assert(result == 0); } if (!_jit->user_code) { + _jit->code.protected = _jit->pc.uc - _jit->code.ptr; +# if __riscv && __WORDSIZE == 64 + /* FIXME should start adding consts at a page boundary */ + _jit->code.protected -= _jitc->consts.hash.count * sizeof(jit_word_t); +# endif #ifdef _WIN32 - result = _mprotect(_jit->code.ptr, _jit->code.length, - PROT_READ | PROT_EXEC); + result = _mprotect(_jit->code.ptr, _jit->code.protected, PROT_READ | PROT_EXEC); #else - result = mprotect(_jit->code.ptr, _jit->code.length, - PROT_READ | PROT_EXEC); + result = mprotect(_jit->code.ptr, _jit->code.protected, PROT_READ | PROT_EXEC); #endif assert(result == 0); } +#endif /* HAVE_MMAP */ return (_jit->code.ptr); fail: return (NULL); } +void +_jit_protect(jit_state_t *_jit) +{ +#if !HAVE_MMAP + assert (_jit->user_code); +#else + int result; + if (_jit->user_code) return; + result = mprotect (_jit->code.ptr, _jit->code.protected, PROT_READ | PROT_EXEC); + assert (result == 0); +#endif +} + +void +_jit_unprotect(jit_state_t *_jit) +{ +#if !HAVE_MMAP + assert (_jit->user_code); +#else + int result; + if (_jit->user_code) return; + result = mprotect (_jit->code.ptr, _jit->code.protected, PROT_READ | PROT_WRITE); + assert (result == 0); +#endif +} + void _jit_frame(jit_state_t *_jit, jit_int32_t frame) { @@ -2232,7 +2594,7 @@ _jit_setup(jit_state_t *_jit, jit_block_t *block) * or normal flow that have a live register not used in this block. */ static void -_jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo) +_jit_follow(jit_state_t *_jit, jit_block_t *block) { jit_node_t *node; jit_block_t *next; @@ -2261,7 +2623,7 @@ _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo) /* Remove from unknown state bitmask. */ jit_regset_com(®temp, ®temp); jit_regset_and(&block->regmask, &block->regmask, ®temp); - *todo = 1; + block->again = 1; } case jit_code_prolog: case jit_code_epilog: @@ -2289,11 +2651,26 @@ _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo) default: value = jit_classify(node->code); if (value & jit_cc_a2_reg) { - if (!(node->w.w & jit_regno_patch)) { - if (jit_regset_tstbit(®mask, node->w.w)) { - jit_regset_clrbit(®mask, node->w.w); - if (!(value & jit_cc_a2_chg)) - jit_regset_setbit(®live, node->w.w); + if (value & jit_cc_a2_rlh) { + if (!(node->w.q.l & jit_regno_patch)) { + /* Assume register is not changed */ + if (jit_regset_tstbit(®mask, node->w.q.l)) + jit_regset_clrbit(®mask, node->w.q.l); + } + if (!(node->w.q.h & jit_regno_patch)) { + if (jit_regset_tstbit(®mask, node->w.q.h)) + jit_regset_clrbit(®mask, node->w.q.h); + } + } + else { + if (value & jit_cc_a2_reg) { + if (!(node->w.w & jit_regno_patch)) { + if (jit_regset_tstbit(®mask, node->w.w)) { + jit_regset_clrbit(®mask, node->w.w); + if (!(value & jit_cc_a2_chg)) + jit_regset_setbit(®live, node->w.w); + } + } } } } @@ -2348,7 +2725,7 @@ _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo) jit_regset_com(®temp, ®temp); jit_regset_and(&block->regmask, &block->regmask, ®temp); - *todo = 1; + block->again = 1; } } else { @@ -2361,19 +2738,19 @@ _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo) * means that only JIT_Vn registers can be trusted on * arrival of jmpr. */ + jit_regset_set_ui(®mask, 0); for (regno = 0; regno < _jitc->reglen; regno++) { spec = jit_class(_rvs[regno].spec); - if (jit_regset_tstbit(®mask, regno) && - (spec & (jit_class_gpr|jit_class_fpr)) && - !(spec & jit_class_sav)) - jit_regset_clrbit(®mask, regno); + if ((spec & (jit_class_gpr|jit_class_fpr)) && + (spec & jit_class_sav)) + jit_regset_setbit(®mask, regno); } /* Assume non callee save registers are live due * to jump to unknown location. */ /* Treat all callee save as live. */ - jit_regset_ior(®live, ®live, ®mask); + jit_regset_ior(&block->reglive, ®live, ®mask); /* Treat anything else as dead. */ - jit_regset_set_ui(®mask, 0); + return; } } break; @@ -2440,11 +2817,24 @@ _jit_update(jit_state_t *_jit, jit_node_t *node, default: value = jit_classify(node->code); if (value & jit_cc_a2_reg) { - if (!(node->w.w & jit_regno_patch)) { - if (jit_regset_tstbit(mask, node->w.w)) { - jit_regset_clrbit(mask, node->w.w); - if (!(value & jit_cc_a2_chg)) - jit_regset_setbit(live, node->w.w); + if (value & jit_cc_a2_rlh) { + if (!(node->w.q.l & jit_regno_patch)) { + /* Assume register is not changed */ + if (jit_regset_tstbit(mask, node->w.q.l)) + jit_regset_clrbit(mask, node->w.q.l); + } + if (!(node->w.q.h & jit_regno_patch)) { + if (jit_regset_tstbit(mask, node->w.q.h)) + jit_regset_clrbit(mask, node->w.q.h); + } + } + else { + if (!(node->w.w & jit_regno_patch)) { + if (jit_regset_tstbit(mask, node->w.w)) { + jit_regset_clrbit(mask, node->w.w); + if (!(value & jit_cc_a2_chg)) + jit_regset_setbit(live, node->w.w); + } } } } @@ -2509,19 +2899,22 @@ _jit_update(jit_state_t *_jit, jit_node_t *node, * means that only JIT_Vn registers can be trusted on * arrival of jmpr. */ + jit_regset_set_ui(mask, 0); for (regno = 0; regno < _jitc->reglen; regno++) { spec = jit_class(_rvs[regno].spec); - if (jit_regset_tstbit(mask, regno) && - (spec & (jit_class_gpr|jit_class_fpr)) && - !(spec & jit_class_sav)) - jit_regset_clrbit(mask, regno); + if ((spec & (jit_class_gpr|jit_class_fpr)) && + (spec & jit_class_sav)) + jit_regset_setbit(mask, regno); } /* Assume non callee save registers are live due * to jump to unknown location. */ /* Treat all callee save as live. */ jit_regset_ior(live, live, mask); + /* Prevent explicitly set as live registers to + * be used as a temporary for the jmpi. */ + jit_regset_ior(live, live, &_jitc->explive); /* Treat anything else as dead. */ - jit_regset_set_ui(mask, 0); + return; } } break; @@ -2587,7 +2980,10 @@ _sequential_labels(jit_state_t *_jit) if ((jump = node->link)) { for (; jump; jump = link) { link = jump->link; - jump->u.n = prev; + if (jump->code == jit_code_movi) + jump->v.n = prev; + else + jump->u.n = prev; jump->link = prev->link; prev->link = jump; } @@ -2601,7 +2997,10 @@ _sequential_labels(jit_state_t *_jit) if ((jump = next->link)) { for (; jump; jump = link) { link = jump->link; - jump->u.n = node; + if (jump->code == jit_code_movi) + jump->v.n = node; + else + jump->u.n = node; jump->link = node->link; node->link = jump; } @@ -2623,36 +3022,59 @@ _split_branches(jit_state_t *_jit) jit_node_t *next; jit_node_t *label; jit_block_t *block; + jit_block_t *blocks; + jit_word_t offset; + jit_word_t length; + length = _jitc->blocks.length; + jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t)); + if ((node = _jitc->head) && + (node->code == jit_code_label || node->code == jit_code_prolog)) { + block = _jitc->blocks.ptr + node->v.w; + memcpy(blocks, block, sizeof(jit_block_t)); + node->v.w = 0; + offset = 1; + } + else + offset = 0; for (node = _jitc->head; node; node = next) { if ((next = node->next)) { if (next->code == jit_code_label || next->code == jit_code_prolog || - next->code == jit_code_epilog) - continue; + next->code == jit_code_epilog) { + if (offset >= length) { + jit_realloc((jit_pointer_t *)&blocks, + length * sizeof(jit_block_t), + (length + 16) * sizeof(jit_block_t)); + length += 16; + } + block = _jitc->blocks.ptr + next->v.w; + memcpy(blocks + offset, block, sizeof(jit_block_t)); + next->v.w = offset++; + } /* split block on branches */ - if (jit_classify(node->code) & jit_cc_a0_jmp) { + else if (jit_classify(node->code) & jit_cc_a0_jmp) { label = new_node(jit_code_label); label->next = next; node->next = label; - if (_jitc->blocks.offset >= _jitc->blocks.length) { - jit_word_t length; - - length = _jitc->blocks.length + 16; - jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr, - _jitc->blocks.length * sizeof(jit_block_t), - length * sizeof(jit_block_t)); - _jitc->blocks.length = length; + if (offset >= length) { + jit_realloc((jit_pointer_t *)&blocks, + length * sizeof(jit_block_t), + (length + 16) * sizeof(jit_block_t)); + length += 16; } - block = _jitc->blocks.ptr + _jitc->blocks.offset; + block = blocks + offset; block->label = label; - label->v.w = _jitc->blocks.offset; + label->v.w = offset++; jit_regset_new(&block->reglive); jit_regset_new(&block->regmask); - ++_jitc->blocks.offset; } } } + jit_free((jit_pointer_t *)&_jitc->blocks.ptr); + _jitc->blocks.ptr = blocks; + _jitc->blocks.offset = offset; + _jitc->blocks.length = length; } static jit_bool_t @@ -2733,7 +3155,6 @@ _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) } break; case jit_code_name: case jit_code_note: - case jit_code_align: break; default: return (0); @@ -2784,7 +3205,7 @@ reverse_jump_code(jit_code_t code) case jit_code_bgti_f: return (jit_code_bunlei_f); case jit_code_bner_f: return (jit_code_beqr_f); - case jit_code_bnei_f: return (jit_code_beqr_f); + case jit_code_bnei_f: return (jit_code_beqi_f); case jit_code_bunltr_f: return (jit_code_bger_f); case jit_code_bunlti_f: return (jit_code_bgei_f); @@ -2917,7 +3338,7 @@ _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) return (0); } -static void +static jit_bool_t _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump) { jit_node_t *iter; @@ -2925,30 +3346,33 @@ _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump) jit_word_t word; jit_int32_t spec; jit_int32_t regno; + jit_bool_t result; if (jump) { prev = node->u.n; if (prev->code == jit_code_epilog) - return; + return (0); assert(prev->code == jit_code_label); if ((prev->flag & jit_flag_head) || node->link || prev->link != node) /* multiple sources */ - return; + return (0); /* if there are sequential labels it will return below */ } else prev = node; + result = 0; word = node->w.w; regno = jit_regno(node->v.w); for (iter = prev->next; iter; prev = iter, iter = iter->next) { switch (iter->code) { case jit_code_label: case jit_code_prolog: case jit_code_epilog: - return; + return (result); case jit_code_movi: if (regno == jit_regno(iter->u.w)) { if (iter->flag || iter->v.w != word) - return; + return (result); + result = 1; del_node(prev, iter); iter = prev; } @@ -2956,32 +3380,34 @@ _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump) default: spec = jit_classify(iter->code); if (spec & jit_cc_a0_jmp) - return; + return (result); if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) == (jit_cc_a0_reg|jit_cc_a0_chg)) { if (spec & jit_cc_a0_rlh) { if (regno == jit_regno(iter->u.q.l) || regno == jit_regno(iter->u.q.h)) - return; + return (result); } else { if (regno == jit_regno(iter->u.w)) - return; + return (result); } } if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) == (jit_cc_a1_reg|jit_cc_a1_chg)) { if (regno == jit_regno(iter->v.w)) - return; + return (result); } if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) == (jit_cc_a2_reg|jit_cc_a2_chg)) { if (regno == jit_regno(iter->w.w)) - return; + return (result); } break; } } + + return (result); } static jit_bool_t @@ -3128,7 +3554,6 @@ _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node) /* no multiple information, so, if set to a constant, * prefer to keep that information */ if (value->kind == 0) { - value->kind = jit_kind_code; switch (node->code) { /* no information about signed/unsigned either */ case jit_code_stxi_c: value->code = jit_code_ldxi_c; break; @@ -3169,7 +3594,7 @@ _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno) * once to the same value, and is a common pattern of calls * to jit_pushargi and jit_pushargr */ -static void +static jit_bool_t _simplify(jit_state_t *_jit) { jit_node_t *prev; @@ -3177,7 +3602,9 @@ _simplify(jit_state_t *_jit) jit_node_t *next; jit_int32_t info; jit_int32_t regno; + jit_bool_t result; + result = 0; for (prev = NULL, node = _jitc->head; node; prev = node, node = next) { next = node->next; switch (node->code) { @@ -3200,6 +3627,7 @@ _simplify(jit_state_t *_jit) * already holding */ patch_register(node->link->next, node, jit_regno_patch|regno, regno); + result = 1; del_node(_jitc->spill[regno], node->link); del_node(prev, node); node = prev; @@ -3209,58 +3637,74 @@ _simplify(jit_state_t *_jit) case jit_code_movr: regno = jit_regno(node->u.w); if (simplify_movr(prev, node, - jit_kind_word, sizeof(jit_word_t))) + jit_kind_word, sizeof(jit_word_t))) { + result = 1; simplify_spill(node = prev, regno); + } break; case jit_code_movi: regno = jit_regno(node->u.w); if (simplify_movi(prev, node, - jit_kind_word, sizeof(jit_word_t))) + jit_kind_word, sizeof(jit_word_t))) { + result = 1; simplify_spill(node = prev, regno); + } break; case jit_code_movr_f: regno = jit_regno(node->u.w); if (simplify_movr(prev, node, - jit_kind_float32, sizeof(jit_float32_t))) + jit_kind_float32, sizeof(jit_float32_t))) { + result = 1; simplify_spill(node = prev, regno); + } break; case jit_code_movi_f: regno = jit_regno(node->u.w); if (simplify_movi(prev, node, - jit_kind_float32, sizeof(jit_float32_t))) + jit_kind_float32, sizeof(jit_float32_t))) { + result = 1; simplify_spill(node = prev, regno); + } break; case jit_code_movr_d: regno = jit_regno(node->u.w); if (simplify_movr(prev, node, - jit_kind_float64, sizeof(jit_float64_t))) + jit_kind_float64, sizeof(jit_float64_t))) { + result = 1; simplify_spill(node = prev, regno); + } break; case jit_code_movi_d: regno = jit_regno(node->u.w); if (simplify_movi(prev, node, - jit_kind_float64, sizeof(jit_float64_t))) + jit_kind_float64, sizeof(jit_float64_t))) { + result = 1; simplify_spill(node = prev, regno); + } break; case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s: case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui: case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d: - regno = jit_regno(node->u.w); #if __WORDSIZE == 32 /* XXX (maister): This is buggy with the sequence of sll, movd-to-reg, movslq-to-reg. * The sign extension may be elided on 64-bit. */ - if (simplify_ldxi(prev, node)) + regno = jit_regno(node->u.w); + if (simplify_ldxi(prev, node)) { + result = 1; simplify_spill(node = prev, regno); + } #endif break; case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i: case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d: regno = jit_regno(node->u.w); - if (simplify_stxi(prev, node)) + if (simplify_stxi(prev, node)) { + result = 1; simplify_spill(node = prev, regno); + } break; default: info = jit_classify(node->code); @@ -3289,13 +3733,29 @@ _simplify(jit_state_t *_jit) ++_jitc->gen[regno]; } if (info & jit_cc_a2_chg) { - regno = jit_regno(node->w.w); - _jitc->values[regno].kind = 0; - ++_jitc->gen[regno]; +#if 0 + /* Assume registers are not changed */ + if (info & jit_cc_a2_rlh) { + regno = jit_regno(node->w.q.l); + _jitc->values[regno].kind = 0; + ++_jitc->gen[regno]; + regno = jit_regno(node->w.q.h); + _jitc->values[regno].kind = 0; + ++_jitc->gen[regno]; + } + else { +#endif + regno = jit_regno(node->w.w); + _jitc->values[regno].kind = 0; + ++_jitc->gen[regno]; +#if 0 + } +#endif } break; } } + return (result); } static jit_int32_t @@ -3316,7 +3776,7 @@ _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link, default: value = jit_classify(node->code); /* lack of extra information */ - if (value & jit_cc_a0_jmp) + if (value & (jit_cc_a0_jmp|jit_cc_a0_cnd)) return (jit_reg_change); else if ((value & (jit_cc_a0_reg|jit_cc_a0_chg)) == (jit_cc_a0_reg|jit_cc_a0_chg) && @@ -3496,11 +3956,49 @@ _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link, } if ((value & jit_cc_a1_reg) && node->v.w == regno) node->v.w = patch; - if ((value & jit_cc_a2_reg) && node->w.w == regno) - node->w.w = patch; + if (value & jit_cc_a2_reg) { + if (value & jit_cc_a2_rlh) { + if (node->w.q.l == regno) + node->w.q.l = patch; + if (node->w.q.h == regno) + node->w.q.h = patch; + } + else { + if (node->w.w == regno) + node->w.w = patch; + } + } } } +#if __BYTE_ORDER == __LITTLE_ENDIAN +# define htonr_us(r0,r1) bswapr_us(r0,r1) +# define htonr_ui(r0,r1) bswapr_ui(r0,r1) +# if __WORDSIZE == 64 +# define htonr_ul(r0,r1) bswapr_ul(r0,r1) +# endif +#else +# define htonr_us(r0,r1) extr_us(r0,r1) +# if __WORDSIZE == 32 +# define htonr_ui(r0,r1) movr(r0,r1) +# else +# define htonr_ui(r0,r1) extr_ui(r0,r1) +# define htonr_ul(r0,r1) movr(r0,r1) +# endif +#endif + +static maybe_unused void +generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1); +static maybe_unused void +generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1); +#if __WORDSIZE == 64 +static maybe_unused void +generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1); +#endif + +#define patch_alist(revert) _patch_alist(_jit, revert) +static maybe_unused void _patch_alist(jit_state_t *_jit, jit_bool_t revert); + #if defined(__i386__) || defined(__x86_64__) # include "jit_x86.c" #elif defined(__mips__) @@ -3523,4 +4021,87 @@ _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link, # include "jit_alpha.c" #elif defined(__riscv) # include "jit_riscv.c" +#elif defined(__loongarch__) +# include "jit_loongarch.c" +#endif + +static maybe_unused void +generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_gpr); + + rshi(rn(reg), r1, 8); + andi(r0, r1, 0xff); + andi(rn(reg), rn(reg), 0xff); + lshi(r0, r0, 8); + orr(r0, r0, rn(reg)); + + jit_unget_reg(reg); +} + +static maybe_unused void +generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_gpr); + + rshi(rn(reg), r1, 16); + bswapr_us(r0, r1); + bswapr_us(rn(reg), rn(reg)); + lshi(r0, r0, 16); + orr(r0, r0, rn(reg)); + + jit_unget_reg(reg); +} + +#if __WORDSIZE == 64 +static maybe_unused void +generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1) +{ + jit_int32_t reg = jit_get_reg(jit_class_gpr); + + rshi_u(rn(reg), r1, 32); + bswapr_ui(r0, r1); + bswapr_ui(rn(reg), rn(reg)); + lshi(r0, r0, 32); + orr(r0, r0, rn(reg)); + + jit_unget_reg(reg); +} +#endif + +#if defined(stack_framesize) +static maybe_unused void +_patch_alist(jit_state_t *_jit, jit_bool_t revert) +{ + jit_int32_t diff; + jit_node_t *node; + diff = jit_diffsize(); + if (diff) { + if (revert) + diff = -diff; + for (node = _jitc->function->alist; node; node = node->link) { + switch (node->code) { + case jit_code_ldxi_c: case jit_code_ldxi_uc: + case jit_code_ldxi_s: case jit_code_ldxi_us: + case jit_code_ldxi_i: +#if __WORDSIZE == 64 + case jit_code_ldxi_ui: case jit_code_ldxi_l: +#endif + case jit_code_ldxi_f: case jit_code_ldxi_d: + node->w.w -= diff; + break; + case jit_code_stxi_c: case jit_code_stxi_s: + case jit_code_stxi_i: +#if __WORDSIZE == 64 + case jit_code_stxi_l: +#endif + case jit_code_stxi_f: case jit_code_stxi_d: + node->u.w -= diff; + break; + default: + abort(); + } + } + } +} #endif diff --git a/parallel-rsp/lightning/m4/.gitignore b/parallel-rsp/lightning/m4/.gitignore new file mode 100644 index 0000000..24e2f3f --- /dev/null +++ b/parallel-rsp/lightning/m4/.gitignore @@ -0,0 +1,10 @@ +/lt~obsolete.m4 +/ltversion.m4 +/ltsugar.m4 +/ltoptions.m4 +/libtool.m4 +/00gnulib.m4 +/gnulib-common.m4 +/gnulib-comp.m4 +/gnulib-tool.m4 +/zzgnulib.m4 diff --git a/parallel-rsp/lightning/m4/gnulib-cache.m4 b/parallel-rsp/lightning/m4/gnulib-cache.m4 new file mode 100644 index 0000000..45be7ba --- /dev/null +++ b/parallel-rsp/lightning/m4/gnulib-cache.m4 @@ -0,0 +1,57 @@ +# Copyright (C) 2002-2021 Free Software Foundation, Inc. +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This file is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this file. If not, see . +# +# As a special exception to the GNU General Public License, +# this file may be distributed as part of a program that +# contains a configuration script generated by Autoconf, under +# the same distribution terms as the rest of that program. +# +# Generated by gnulib-tool. +# +# This file represents the specification of how gnulib-tool is used. +# It acts as a cache: It is written and read by gnulib-tool. +# In projects that use version control, this file is meant to be put under +# version control, like the configure.ac and various Makefile.am files. + + +# Specification in the form of a command-line invocation: +# gnulib-tool --import --local-dir=gl \ +# --lib=libgnu \ +# --source-base=gnulib-lib \ +# --m4-base=m4 \ +# --doc-base=gnulib-doc \ +# --tests-base=tests \ +# --aux-dir=build-aux \ +# --no-conditional-dependencies \ +# --libtool \ +# --macro-prefix=gl + +# Specification in the form of a few gnulib-tool.m4 macro invocations: +gl_LOCAL_DIR([gl]) +gl_MODULES([ + +]) +gl_AVOID([]) +gl_SOURCE_BASE([gnulib-lib]) +gl_M4_BASE([m4]) +gl_PO_BASE([]) +gl_DOC_BASE([gnulib-doc]) +gl_TESTS_BASE([tests]) +gl_LIB([libgnu]) +gl_MAKEFILE_NAME([]) +gl_LIBTOOL +gl_MACRO_PREFIX([gl]) +gl_PO_DOMAIN([]) +gl_WITNESS_C_MACRO([]) diff --git a/parallel-rsp/lightning/size.c b/parallel-rsp/lightning/size.c index 4e93370..1f31ed6 100644 --- a/parallel-rsp/lightning/size.c +++ b/parallel-rsp/lightning/size.c @@ -68,14 +68,6 @@ main(int argc, char *argv[]) # else fprintf(fp, "#if !defined(__ARM_PCS_VFP)\n"); # endif -#elif defined(__mips__) -# if __WORDSIZE == 32 -# if NEW_ABI - fprintf(fp, "#if NEW_ABI\n"); -# else - fprintf(fp, "#if !NEW_ABI\n"); -# endif -# endif #elif defined(__powerpc__) fprintf(fp, "#if defined(__powerpc__)\n"); fprintf(fp, "#if __BYTE_ORDER == %s\n", @@ -94,17 +86,15 @@ main(int argc, char *argv[]) fprintf(fp, " %d, /* %s */\n", _szs[offset], code_name[offset]); #if defined(__arm__) fprintf(fp, "#endif /* __ARM_PCS_VFP */\n"); -#elif defined(__mips__) -# if __WORDSIZE == 32 - fprintf(fp, "#endif /* NEW_ABI */\n"); -# endif #elif defined(__powerpc__) +# if __WORDSIZE == 32 fprintf(fp, "#endif /* " -# if !_CALL_SYSV +# if !_CALL_SYSV "!" -# endif +# endif "_CALL_SYSV" " */\n"); +# endif fprintf(fp, "#endif /* __BYTE_ORDER */\n"); fprintf(fp, "#endif /* __powerpc__ */\n"); #endif diff --git a/parallel-rsp/rsp/cp2.cpp b/parallel-rsp/rsp/cp2.cpp index 2d2780a..0c70250 100644 --- a/parallel-rsp/rsp/cp2.cpp +++ b/parallel-rsp/rsp/cp2.cpp @@ -29,23 +29,21 @@ extern "C" void RSP_MTC2(RSP::CPUState *rsp, unsigned rt, unsigned rd, unsigned element) { - uint16_t *e = rsp->cp2.regs[rd].e; - #ifdef INTENSE_DEBUG fprintf(stderr, "MTC2, rt = %u, [rt] = 0x%x, rd = %u, e = %u\n", rt, rsp->sr[rt], rd, element); #endif - unsigned lo = element >> 1; - rt = rsp->sr[rt]; - + uint16_t *e = rsp->cp2.regs[rd].e; + const uint16_t v = rsp->sr[rt]; if (element & 1) { - unsigned hi = (element + 1) >> 1; - e[lo] = (e[lo] & 0xff00) | ((rt >> 8) & 0xff); - e[hi] = (e[lo] & 0x00ff) | ((rt & 0xff) << 8); + const auto i = element >> 1; + e[i] = (e[i] & 0xff00) | (v >> 8); + if (element != 0xf) + e[i+1] = (e[i+1] & 0xff) | (v << 8); } else - e[lo] = rt; + e[element >> 1] = v; } void RSP_MFC2(RSP::CPUState *rsp, unsigned rt, unsigned rd, unsigned element) diff --git a/parallel-rsp/rsp/ls.cpp b/parallel-rsp/rsp/ls.cpp index e1acb21..0544dc3 100644 --- a/parallel-rsp/rsp/ls.cpp +++ b/parallel-rsp/rsp/ls.cpp @@ -9,10 +9,24 @@ extern "C" { - // Using mostly CXD4 implementation as a base here since it's easier to follow. - // CEN64's implementation seems much better, but takes more effort to port for now. - // Reading wide words together with SSE4 blend, SSSE3 pshufb, etc should make this much faster. + // Using mostly Ares' implementation as a base here + static inline uint8_t byteFromHalfWords(const uint16_t *arr, unsigned i) + { + return (i & 1) ? + (uint8_t)(arr[i >> 1] & 0xff) : + (uint8_t)(arr[i >> 1] >> 8); + } + + static inline void writeByteToHalfWords(uint16_t *arr, unsigned i, uint8_t b) + { + const unsigned n = i >> 1; + if (i & 1) + arr[n] = (arr[n] & 0xff00) | (uint16_t)b; + else + arr[n] = (arr[n] & 0xff) | ((uint16_t)b << 8); + } + // Load 8-bit void RSP_LBV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { @@ -39,21 +53,10 @@ extern "C" void RSP_LSV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LSV); - if (e & 1) - return; - - unsigned addr = (rsp->sr[base] + offset * 2) & 0xfff; - unsigned correction = addr & 3; - if (correction == 3) - return; - - uint16_t result; - if (correction == 1) - result = (READ_MEM_U8(rsp->dmem, addr + 0) << 8) | (READ_MEM_U8(rsp->dmem, addr + 1) << 0); - else - result = READ_MEM_U16(rsp->dmem, addr); - - rsp->cp2.regs[rt].e[e >> 1] = result; + unsigned addr = rsp->sr[base] + offset * 2; + const unsigned end = (e > 14) ? 16 : (e + 2); + for (unsigned i = e; i < end; i++) + writeByteToHalfWords(rsp->cp2.regs[rt].e, i & 0xf, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); } // Store 16-bit @@ -76,63 +79,34 @@ extern "C" void RSP_LLV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LLV); - unsigned addr = (rsp->sr[base] + offset * 4) & 0xfff; - if (e & 1) - return; - if (addr & 1) - return; - e >>= 1; - - rsp->cp2.regs[rt].e[e] = READ_MEM_U16(rsp->dmem, addr); - rsp->cp2.regs[rt].e[(e + 1) & 7] = READ_MEM_U16(rsp->dmem, (addr + 2) & 0xfff); + unsigned addr = rsp->sr[base] + offset * 4; + const unsigned end = (e > 12) ? 16 : (e + 4); + for (unsigned i = e; i < end; i++) + writeByteToHalfWords(rsp->cp2.regs[rt].e, i & 0xf, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); } // Store 32-bit void RSP_SLV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SLV); - if ((e & 1) || (e > 0xc)) - return; unsigned addr = (rsp->sr[base] + offset * 4) & 0xfff; #ifdef INTENSE_DEBUG fprintf(stderr, "SLV 0x%x, e = %u\n", addr, e); #endif - if (addr & 1) - return; - e >>= 1; - - uint16_t v0 = rsp->cp2.regs[rt].e[e]; - uint16_t v1 = rsp->cp2.regs[rt].e[e + 1]; - WRITE_MEM_U16(rsp->dmem, addr, v0); - WRITE_MEM_U16(rsp->dmem, (addr + 2) & 0xfff, v1); + for (unsigned i = e; i < e + 4; i++) + WRITE_MEM_U8(rsp->dmem, addr++, byteFromHalfWords(rsp->cp2.regs[rt].e, i & 0xf)); } // Load 64-bit void RSP_LDV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LDV); - if (e & 1) - return; - unsigned addr = (rsp->sr[base] + offset * 8) & 0xfff; - auto *reg = rsp->cp2.regs[rt].e; - e >>= 1; - - if (addr & 1) - { - reg[e + 0] = (READ_MEM_U8(rsp->dmem, addr + 0) << 8) | READ_MEM_U8(rsp->dmem, addr + 1); - reg[e + 1] = (READ_MEM_U8(rsp->dmem, addr + 2) << 8) | READ_MEM_U8(rsp->dmem, addr + 3); - reg[e + 2] = (READ_MEM_U8(rsp->dmem, addr + 4) << 8) | READ_MEM_U8(rsp->dmem, addr + 5); - reg[e + 3] = (READ_MEM_U8(rsp->dmem, addr + 6) << 8) | READ_MEM_U8(rsp->dmem, addr + 7); - } - else - { - reg[e + 0] = READ_MEM_U16(rsp->dmem, addr); - reg[e + 1] = READ_MEM_U16(rsp->dmem, (addr + 2) & 0xfff); - reg[e + 2] = READ_MEM_U16(rsp->dmem, (addr + 4) & 0xfff); - reg[e + 3] = READ_MEM_U16(rsp->dmem, (addr + 6) & 0xfff); - } + unsigned addr = rsp->sr[base] + offset * 8; + const unsigned end = (e > 8) ? 16 : (e + 8); + for (unsigned i = e; i < end; i++) + writeByteToHalfWords(rsp->cp2.regs[rt].e, i & 0xf, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); } // Store 64-bit @@ -168,24 +142,25 @@ extern "C" void RSP_LPV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LPV); - if (e != 0) - return; - unsigned addr = (rsp->sr[base] + offset * 8) & 0xfff; + const unsigned index = (addr & 7) - e; + addr &= ~7; + auto *reg = rsp->cp2.regs[rt].e; for (unsigned i = 0; i < 8; i++) - reg[i] = READ_MEM_U8(rsp->dmem, (addr + i) & 0xfff) << 8; + reg[i] = READ_MEM_U8(rsp->dmem, (addr + (i + index & 0xf)) & 0xfff) << 8; } void RSP_SPV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SPV); - if (e != 0) - return; unsigned addr = (rsp->sr[base] + offset * 8) & 0xfff; auto *reg = rsp->cp2.regs[rt].e; - for (unsigned i = 0; i < 8; i++) - WRITE_MEM_U8(rsp->dmem, (addr + i) & 0xfff, int16_t(reg[i]) >> 8); + + for (unsigned i = e; i < e + 8; i++) { + const unsigned shift = ((i & 0xf) < 8) ? 8 : 7; + WRITE_MEM_U8(rsp->dmem, addr++ & 0xfff, int16_t(reg[i & 7]) >> shift); + } } // Load 8x8-bit into high bits, but shift by 7 instead of 8. @@ -195,36 +170,24 @@ extern "C" { TRACE_LS(LUV); unsigned addr = (rsp->sr[base] + offset * 8) & 0xfff; - auto *reg = rsp->cp2.regs[rt].e; + const unsigned index = (addr & 7) - e; + addr &= ~7; - if (e != 0) - { - // Special path for Mia Hamm soccer. - addr += -e & 0xf; - for (unsigned b = 0; b < 8; b++) - { - reg[b] = READ_MEM_U8(rsp->dmem, addr) << 7; - --e; - addr -= e ? 0 : 16; - ++addr; - } - } - else - { - for (unsigned i = 0; i < 8; i++) - reg[i] = READ_MEM_U8(rsp->dmem, (addr + i) & 0xfff) << 7; - } + auto *reg = rsp->cp2.regs[rt].e; + for (unsigned i = 0; i < 8; i++) + reg[i] = READ_MEM_U8(rsp->dmem, (addr + (i + index & 0xf)) & 0xfff) << 7; } void RSP_SUV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SUV); - if (e != 0) - return; unsigned addr = (rsp->sr[base] + offset * 8) & 0xfff; auto *reg = rsp->cp2.regs[rt].e; - for (unsigned i = 0; i < 8; i++) - WRITE_MEM_U8(rsp->dmem, (addr + i) & 0xfff, int16_t(reg[i]) >> 7); + + for (unsigned i = e; i < e + 8; i++) { + const unsigned shift = ((i & 0xf) < 8) ? 7 : 8; + WRITE_MEM_U8(rsp->dmem, addr++ & 0xfff, int16_t(reg[i & 7]) >> shift); + } } // Load 8x8-bits into high bits, but shift by 7 instead of 8. @@ -232,171 +195,199 @@ extern "C" void RSP_LHV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LHV); - if (e != 0) - return; - unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - if (addr & 0xe) - return; + unsigned addr = rsp->sr[base] + offset * 16; + const unsigned index = (addr & 7) - e; + addr &= ~7; auto *reg = rsp->cp2.regs[rt].e; for (unsigned i = 0; i < 8; i++) - reg[i] = READ_MEM_U8(rsp->dmem, addr + 2 * i) << 7; + reg[i] = (uint16_t)READ_MEM_U8(rsp->dmem, (addr + (index + i * 2 & 0xf)) & 0xfff) << 7; } void RSP_SHV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SHV); - if (e != 0) - return; unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - auto *reg = rsp->cp2.regs[rt].e; + const unsigned index = addr & 7; + addr &= ~7; + + const auto *reg = rsp->cp2.regs[rt].e; for (unsigned i = 0; i < 8; i++) - WRITE_MEM_U8(rsp->dmem, (addr + 2 * i) & 0xfff, int16_t(reg[i]) >> 7); + { + const unsigned b = e + (i << 1); + const uint8_t byte = byteFromHalfWords(reg, b & 0xf) << 1 | byteFromHalfWords(reg, b + 1 & 0xf) >> 7; + WRITE_MEM_U8(rsp->dmem, addr + (index + i * 2 & 0xf), byte); + } } - // No idea what the purpose of this is. + void RSP_LFV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) + { + TRACE_LS(LFV); + uint16_t temp[8]; + + unsigned addr = rsp->sr[base] + offset * 16; + const unsigned index = (addr & 7) - e; + const unsigned end = (e > 8) ? 16 : (e + 8); + addr &= ~7; + + for (unsigned i = 0; i < 4; i++) + { + temp[i] = (uint16_t)READ_MEM_U8(rsp->dmem, (addr + (index + i * 4 & 0xf)) & 0xfff) << 7; + temp[i+4] = (uint16_t)READ_MEM_U8(rsp->dmem, (addr + (index + i * 4 + 8 & 0xf)) & 0xfff) << 7; + } + + for (unsigned i = e; i < end; i++) + writeByteToHalfWords(rsp->cp2.regs[rt].e, i, byteFromHalfWords(temp, i)); + } + +#define RSP_SFV_CASE(a,b,c,d) \ + WRITE_MEM_U8(rsp->dmem, addr + base, int16_t(reg[a]) >> 7); \ + WRITE_MEM_U8(rsp->dmem, addr + 4 + base, int16_t(reg[b]) >> 7); \ + WRITE_MEM_U8(rsp->dmem, addr + (8 + base & 0xf), int16_t(reg[c]) >> 7); \ + WRITE_MEM_U8(rsp->dmem, addr + (12 + base & 0xf), int16_t(reg[d]) >> 7); + void RSP_SFV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SFV); - unsigned addr = (rsp->sr[base] + offset * 16) & 0xff3; + unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; + base = addr & 7; + addr &= ~7; + auto *reg = rsp->cp2.regs[rt].e; switch (e) { case 0: - WRITE_MEM_U8(rsp->dmem, (addr + 0) & 0xfff, int16_t(reg[0]) >> 7); - WRITE_MEM_U8(rsp->dmem, (addr + 4) & 0xfff, int16_t(reg[1]) >> 7); - WRITE_MEM_U8(rsp->dmem, (addr + 8) & 0xfff, int16_t(reg[2]) >> 7); - WRITE_MEM_U8(rsp->dmem, (addr + 12) & 0xfff, int16_t(reg[3]) >> 7); + case 15: + RSP_SFV_CASE(0,1,2,3) + break; + case 1: + RSP_SFV_CASE(6,7,4,5) + break; + case 4: + RSP_SFV_CASE(1,2,3,0) + break; + case 5: + RSP_SFV_CASE(7,4,5,6) break; - case 8: - WRITE_MEM_U8(rsp->dmem, (addr + 0) & 0xfff, int16_t(reg[4]) >> 7); - WRITE_MEM_U8(rsp->dmem, (addr + 4) & 0xfff, int16_t(reg[5]) >> 7); - WRITE_MEM_U8(rsp->dmem, (addr + 8) & 0xfff, int16_t(reg[6]) >> 7); - WRITE_MEM_U8(rsp->dmem, (addr + 12) & 0xfff, int16_t(reg[7]) >> 7); + RSP_SFV_CASE(4,5,6,7) + break; + case 11: + RSP_SFV_CASE(3,0,1,2) + break; + case 12: + RSP_SFV_CASE(5,6,7,4) break; - default: + WRITE_MEM_U8(rsp->dmem, addr + base, 0); + WRITE_MEM_U8(rsp->dmem, addr + 4 + base, 0); + WRITE_MEM_U8(rsp->dmem, addr + (8 + base & 0xf), 0); + WRITE_MEM_U8(rsp->dmem, addr + (12 + base & 0xf), 0); break; } } - // Loads full 128-bit register, however, it seems to handle unaligned addresses in a very - // strange way. + void RSP_LWV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) + { + TRACE_LS(LWV); + unsigned addr = rsp->sr[base] + offset * 16; + for (unsigned i = 16 - e; i < 16 + e; i++) + { + writeByteToHalfWords(rsp->cp2.regs[rt].e, i & 0xf, READ_MEM_U8(rsp->dmem, addr & 0xfff)); + addr += 4; + } + } + + void RSP_SWV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) + { + TRACE_LS(SWV); + + unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; + base = addr & 7; + addr &= ~7; + + for (unsigned i = e; i < e + 16; i++) + WRITE_MEM_U8(rsp->dmem, addr + (base++ & 0xf), byteFromHalfWords(rsp->cp2.regs[rt].e, i & 0xf)); + } + void RSP_LQV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LQV); - if (e & 1) - return; - unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; + unsigned addr = rsp->sr[base] + offset * 16; + unsigned end = 16 + e - (addr & 0xf); + if (end > 16) end = 16; -#ifdef INTENSE_DEBUG - fprintf(stderr, "LQV: 0x%x, e = %u, vt = %u, base = %u\n", addr, e, rt, base); -#endif - - if (addr & 1) - return; - - unsigned b = (addr & 0xf) >> 1; - e >>= 1; - - auto *reg = rsp->cp2.regs[rt].e; - for (unsigned i = b; i < 8; i++, e++, addr += 2) - reg[e] = READ_MEM_U16(rsp->dmem, addr & 0xfff); + for (unsigned i = e; i < end; i++) + writeByteToHalfWords(rsp->cp2.regs[rt].e, i & 0xf, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); } void RSP_SQV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SQV); unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - if (addr & 1) - return; - - unsigned b = addr & 0xf; - - auto *reg = rsp->cp2.regs[rt].e; - - if (e != 0) - { - // Mia Hamm Soccer - for (unsigned i = 0; i < 16 - b; i++, addr++) - { - WRITE_MEM_U8(rsp->dmem, addr & 0xfff, reinterpret_cast(reg)[MES((e + i) & 0xf)]); - } - } - else - { - b >>= 1; - for (unsigned i = b; i < 8; i++, e++, addr += 2) - WRITE_MEM_U16(rsp->dmem, addr & 0xfff, reg[e]); - } + + const unsigned end = e + (16 - (addr & 15)); + for (unsigned i = e; i < end; i++) + WRITE_MEM_U8(rsp->dmem, addr++, byteFromHalfWords(rsp->cp2.regs[rt].e, i & 15)); } - // Complements LQV? void RSP_LRV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LRV); - if (e != 0) - return; - unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - if (addr & 1) - return; - - unsigned b = (addr & 0xf) >> 1; + unsigned addr = rsp->sr[base] + offset * 16; + const unsigned start = 16 - ((addr & 0xf) - e); addr &= ~0xf; - auto *reg = rsp->cp2.regs[rt].e; - for (e = 8 - b; e < 8; e++, addr += 2) - reg[e] = READ_MEM_U16(rsp->dmem, addr & 0xfff); + for (unsigned i = start; i < 16; i++) + writeByteToHalfWords(rsp->cp2.regs[rt].e, i & 0xf, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); } void RSP_SRV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(SRV); - if (e != 0) - return; unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - if (addr & 1) - return; - - unsigned b = (addr & 0xf) >> 1; + const unsigned end = e + (addr & 0xf); + base = 16 - (addr & 0xf); addr &= ~0xf; - auto *reg = rsp->cp2.regs[rt].e; - for (e = 8 - b; e < 8; e++, addr += 2) - WRITE_MEM_U16(rsp->dmem, addr & 0xfff, reg[e]); + for (unsigned i = e; i < end; i++) + WRITE_MEM_U8(rsp->dmem, addr++, byteFromHalfWords(rsp->cp2.regs[rt].e, i + base & 0xf)); } - // Transposed stuff? void RSP_LTV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(LTV); - if (e & 1) - return; - if (rt & 7) - return; - unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - if (addr & 0xf) - return; + unsigned addr = rsp->sr[base] + offset * 16; + const unsigned start = addr & ~7; + const unsigned vt0 = rt & ~7; + addr = start + ((e + (addr & 8)) & 0xf); + unsigned j = e >> 1; - for (unsigned i = 0; i < 8; i++) - rsp->cp2.regs[rt + i].e[(-e / 2 + i) & 7] = READ_MEM_U16(rsp->dmem, addr + 2 * i); + for (unsigned i = 0; i < 16; j++) + { + j &= 7; + writeByteToHalfWords(rsp->cp2.regs[vt0+j].e, i++, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); + if (addr == start + 16) addr = start; + writeByteToHalfWords(rsp->cp2.regs[vt0+j].e, i++, READ_MEM_U8(rsp->dmem, addr++ & 0xfff)); + if (addr == start + 16) addr = start; + } } void RSP_STV(RSP::CPUState *rsp, unsigned rt, unsigned e, int offset, unsigned base) { TRACE_LS(STV); - if (e & 1) - return; - if (rt & 7) - return; - unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; - if (addr & 0xf) - return; + e &= ~1; + rt &= ~7; - for (unsigned i = 0; i < 8; i++) + unsigned addr = (rsp->sr[base] + offset * 16) & 0xfff; + unsigned element = 16 - e; + base = (addr & 7) - e; + addr &= ~7; + + for (unsigned i = rt; i < rt + 8; i++ ) { - WRITE_MEM_U16(rsp->dmem, addr + 2 * i, rsp->cp2.regs[rt + ((e / 2 + i) & 7)].e[i]); + WRITE_MEM_U8(rsp->dmem, addr + (base++ & 0xf), byteFromHalfWords(rsp->cp2.regs[i].e, element++ & 0xf)); + WRITE_MEM_U8(rsp->dmem, addr + (base++ & 0xf), byteFromHalfWords(rsp->cp2.regs[i].e, element++ & 0xf)); } } } diff --git a/parallel-rsp/rsp_jit.cpp b/parallel-rsp/rsp_jit.cpp index 8758d32..c677743 100644 --- a/parallel-rsp/rsp_jit.cpp +++ b/parallel-rsp/rsp_jit.cpp @@ -1471,7 +1471,7 @@ void CPU::jit_instruction(jit_state_t *_jit, uint32_t pc, uint32_t instr, unsigned rt = (instr >> 16) & 31; NOP_IF_RT_ZERO(); int16_t imm = int16_t(instr); - regs.immediate_mips_register(_jit, rt, uint16_t(imm) << 16); + regs.immediate_mips_register(_jit, rt, imm << 16); regs.unlock_mips_register(rt); break; } @@ -1695,7 +1695,7 @@ void CPU::jit_instruction(jit_state_t *_jit, uint32_t pc, uint32_t instr, using LWC2Op = void (*)(RSP::CPUState *, unsigned rt, unsigned imm, int simm, unsigned rs); static const LWC2Op ops[32] = { - RSP_LBV, RSP_LSV, RSP_LLV, RSP_LDV, RSP_LQV, RSP_LRV, RSP_LPV, RSP_LUV, RSP_LHV, nullptr, nullptr, RSP_LTV, + RSP_LBV, RSP_LSV, RSP_LLV, RSP_LDV, RSP_LQV, RSP_LRV, RSP_LPV, RSP_LUV, RSP_LHV, RSP_LFV, nullptr, RSP_LTV, }; auto *op = ops[rd]; @@ -1727,7 +1727,7 @@ void CPU::jit_instruction(jit_state_t *_jit, uint32_t pc, uint32_t instr, using SWC2Op = void (*)(RSP::CPUState *, unsigned rt, unsigned imm, int simm, unsigned rs); static const SWC2Op ops[32] = { - RSP_SBV, RSP_SSV, RSP_SLV, RSP_SDV, RSP_SQV, RSP_SRV, RSP_SPV, RSP_SUV, RSP_SHV, RSP_SFV, nullptr, RSP_STV, + RSP_SBV, RSP_SSV, RSP_SLV, RSP_SDV, RSP_SQV, RSP_SRV, RSP_SPV, RSP_SUV, RSP_SHV, RSP_SFV, RSP_SWV, RSP_STV, }; auto *op = ops[rd]; diff --git a/parallel-rsp/rsp_op.hpp b/parallel-rsp/rsp_op.hpp index 8c8bd28..76722aa 100644 --- a/parallel-rsp/rsp_op.hpp +++ b/parallel-rsp/rsp_op.hpp @@ -30,6 +30,7 @@ extern "C" DECL_LS(LUV); DECL_LS(LHV); DECL_LS(LFV); + DECL_LS(LWV); DECL_LS(LTV); DECL_LS(SBV); @@ -42,6 +43,7 @@ extern "C" DECL_LS(SUV); DECL_LS(SHV); DECL_LS(SFV); + DECL_LS(SWV); DECL_LS(STV); #define DECL_COP2(op) void RSP_##op(RSP::CPUState *rsp, unsigned vd, unsigned vs, unsigned vt, unsigned e)