Lightning+Lightrec updates and other optimizations

Makefile

@@ -468,6 +468,8 @@ else
 
    ifeq ($(HAVE_LIGHTREC), 1)
       FLAGS += -DHAVE_WIN_SHM
+      #For lightrec to get number of cpus, mingw should provide pthread_num_processors_np like pthread-win32
+      FLAGS += -DPTW32_VERSION
    endif
 endif
 

Makefile.common

@@ -183,11 +183,7 @@ ifeq ($(HAVE_LIGHTREC), 1)
    ifeq ($(THREADED_RECOMPILER), 0)
        FLAGS += -DENABLE_THREADED_COMPILER=0
    else
-      ifeq ($(DEBUG), 0)
-          FLAGS += -DENABLE_THREADED_COMPILER=1
-      else
-          FLAGS += -DENABLE_THREADED_COMPILER=0
-      endif
+       FLAGS += -DENABLE_THREADED_COMPILER=1
    endif
 
    ifneq (,$(findstring win,$(platform)))
@@ -197,8 +193,8 @@ ifeq ($(HAVE_LIGHTREC), 1)
 
    INCFLAGS += -I$(DEPS_DIR)/lightning/include \
                -I$(DEPS_DIR)/lightrec \
-               -I$(CORE_DIR)/lightning-lightrec-include \
-               -include $(CORE_DIR)/lightning-lightrec-include/debug.h
+               -I$(CORE_DIR)/include \
+               -include $(CORE_DIR)/include/debug.h
 endif
 
 ifneq ($(HAVE_GRIFFIN), 1)
@@ -375,12 +371,11 @@ ifeq ($(HAVE_LIGHTREC), 1)
                   $(DEPS_DIR)/lightrec/memmanager.c \
                   $(DEPS_DIR)/lightrec/optimizer.c \
                   $(DEPS_DIR)/lightrec/reaper.c \
-                  $(DEPS_DIR)/lightrec/regcache.c
+                  $(DEPS_DIR)/lightrec/regcache.c \
+                  $(DEPS_DIR)/lightrec/tlsf/tlsf.c
 
    ifeq ($(THREADED_RECOMPILER), 1)
-      ifeq ($(DEBUG), 0)
-          SOURCES_C +=   $(DEPS_DIR)/lightrec/recompiler.c
-      endif
+      SOURCES_C +=   $(DEPS_DIR)/lightrec/recompiler.c
    endif
 
    ifeq ($(LIGHTREC_LOG_LEVEL), 4)

deps/lightning/.gitignore

@@ -1,4 +1,15 @@
+/build-aux
 +*
+
+*.o
+*.lo
+*.la
+
+.libs/
+.deps/
+*/.libs/
+*/.deps/
+
 autom4te.cache
 aclocal.m4
 depcomp
@@ -20,14 +31,14 @@ missing
 size
 stamp-h1
 test-driver
-check/.deps
-doc/.deps
-lib/.deps
+
 m4/libtool.m4
 m4/lt~obsolete.m4
 m4/ltoptions.m4
 m4/ltsugar.m4
 m4/ltversion.m4
-doc/mdate-sh
-doc/texinfo.tex
+
 lightning.pc
+include/lightning.h
+
+build-aux/

deps/lightning/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://git.savannah.gnu.org/git/lightning.git
 	branch = master
-	commit = 876c1043bec5bfd594482b40700c84693e40d0eb
+	commit = ccb8bd77a46c2123c634fb3115b0023165793975
 	parent = 9f797430963d9cf0fcef7d963466f9cac7026de2
 	method = merge
 	cmdver = 0.4.3

deps/lightning/ChangeLog

@@ -1,3 +1,211 @@
+2022-11-09 Paulo Andrade <pcpa@gnu.org>
+
+	* configure.ac: Add new --enable-devel-strong-type-checking
+	option.
+	* include/lightning.h.in: Rework to not need to know if
+	PACKED_STACK is defined, and add a new argument to _jit_arg,
+	_jit_putarg{r,i}, _jit_pusharg{r,i} and _jit_ret{r,i} to have
+	the same code path if PACKED_STACK is defined or not, and also
+	to implement STRONG_TYPE_CHECK enabled with the new
+	--enable-devel-strong-type-checking.
+	* include/lightning/jit_private.h: Add new macros to add assertions
+	for STRONG_TYPE_CHECK and avoid pasting tokens in jit_inc_synth*
+	when the token is not a static known value.
+	* lib/jit_aarch64.c: The first implementation of the new code,
+	working correctly in Apple M1 and with and without STRONG_TYPE_CHECK
+	in Linux.
+
+2022-11-08 Paulo Andrade <pcpa@gnu.org>
+
+	Add support for packed stack arguments as used by Apple M1
+	aarch64 cpus. This requires a major redesign in how Lightning
+	works, because contrary to all other supported ports, in this
+	case arguments must be truncated and sign/zero extended if
+	passed in registers, but when receiving the argument, there
+	is no need to truncate and sign/zero extend.
+	Return values are also treated this way. The callee must
+	truncate sign/zero extend, not the caller.
+	check/Makefile.am: Add LIGHTNING_CFLAGS to AM_CFLAGS.
+	check/all.tst: Implement paired arg/getarg/pusharg/putarg/ret
+	codes to validate they do not generate assertions.
+	* check/allocar.tst, check/call.tst, check/fib.tst, check/put.tst,
+	check/stack.tst: Update to pass in all build types.
+	check/lightning.c: Add new codes for extra codes to handle
+	packed stack.
+	* configure.ac: Add a preprocessor define to know if packed stack
+	need is required. This is not really used, as it was moved to
+	jit_aarch64.h.
+	* doc/Makefile.am: Add LIGHTNING_CFLAGS to AM_CFLAGS.
+	* doc/rpn.c: Update to pass in all build types.
+	include/lightning.h.in: Add new codes and reorder enum.
+	* include/lightning/jit_aarch64.h: Detect condition of needing
+	a packed stack.
+	* lib/jit_aarch64-sz.c: Regenerate.
+	* lib/jit_aarch64.c: Major updates for packed stack.
+	* lib/jit_names.c: Updates for debug output.
+	* lib/lightning.c: Update for new codes.
+
+2022-10-31  Marc Nieper-Wißkirchen  <marc@nieper-wisskirchen.de>
+
+	Add new skip instruction.
+	* .gitignore: Update from Gnulib.
+	* check/Makefile.am: Add tests.
+	* check/lightning.c: Handle skip instructions.
+	* check/protect.c: Rewrite with skip.
+	* check/skip.ok: New test.
+	* check/skip.tst: New test.
+	* doc/body.texi: Document the skip instruction.
+	* include/lightning.h.in: Add the skip instruction.
+	* lib/jit_aarch64-sz.c: Update for skip instruction.
+	* lib/jit_aarch64.c: Implement skip instruction.
+	* lib/jit_alpha-sz.c: Update for skip instruction.
+	* lib/jit_alpha.c: Implement skip instruction.
+	* lib/jit_arm-sz.c: Update for skip instruction.
+	* lib/jit_arm.c: Implement skip instruction.
+	* lib/jit_hppa-sz.c: Update for skip instruction.
+	* lib/jit_hppa.c: Implement skip instruction.
+	* lib/jit_ia64-sz.c: Update for skip instruction.
+	* lib/jit_ia64.c: Implement skip instruction.
+	* lib/jit_loongarch-sz.c: Update for skip instruction.
+	* lib/jit_loongarch.c: Implement skip instruction.
+	* lib/jit_mips-sz.c: Update for skip instruction.
+	* lib/jit_mips.c: Implement skip instruction.
+	* lib/jit_names.c: Update for skip instruction.
+	* lib/jit_ppc-sz.c: Update for skip instruction.
+	* lib/jit_ppc.c: Implement skip instruction.
+	* lib/jit_riscv-sz.c: Update for skip instruction.
+	* lib/jit_riscv.c: Implement skip instruction.
+	* lib/jit_s390-sz.c: Update for skip instruction.
+	* lib/jit_s390.c: Implement skip instruction.
+	* lib/jit_size.c: Treat align and skip in a special way.
+	* lib/jit_sparc-sz.c: Update for skip instruction.
+	* lib/jit_sparc.c: Implement skip instruction.
+	* lib/jit_x86-sz.c: Update for skip instruction.
+	* lib/jit_x86.c: Implement skip instruction.
+	* lib/lightning.c: Classify skip instruction.
+
+2022-10-30  Marc Nieper-Wißkirchen  <marc@nieper-wisskirchen.de>
+
+	Add user-visible functions jit_protect and jit_unprotect.
+	* check/Makefile.am: Add test for jit_protect and jit_unprotect.
+	* check/protect.c: New test.
+	* doc/body.texi: Add documentation for jit_protect and
+	jit_unprotect.
+	* include/lightning.h.in: Add prototypes for jit_protect and
+	jit_unprotect.
+	* include/lightning/jit_private.h: Add a field to store the size
+	of the protected memory.
+	* lib/lightning.c: Remember the size of the protected memory and
+	implement the two new functions.
+
+2022-10-12 Paulo Andrade <pcpa@gnu.org>
+
+	* include/lightning/jit_loongarch.h, lib/jit_loongarch-cpu.c,
+	lib/jit_loongarch-fpu.c, lib/jit_loongarch-sz.c, lib/jit_loongarch.c:
+	New files implementing the first version of the new loongarch port.
+	* check/float.tst: Add preprocessor checks for NaN and +-Inf
+	values converted to integers for loongarch.
+	* configure.ac: Add check and conditionals for new architecture.
+	* include/lightning.h.in, check/lightning.c,
+	include/lightning/Makefile.am, include/lightning/jit_private.h,
+	lib/Makefile.am, lib/jit_size.c, lib/lightning.c: Update for new
+	port.
+
+2022-10-05 Paulo Andrade <pcpa@gnu.org>
+
+	* check/lightning.c: Remove -Dmacro=value from usage and attempt
+	to parse it. It was buggy and not properly implemented. Now
+	it pass any extra options to the generated jit. To pass any
+	option starting with '-' need to also use '--'.
+	* check/collatz.e: New sample file showing an example of jit
+	generation.
+
+2022-10-04 Paulo Andrade <pcpa@gnu.org>
+
+	* include/lightning/jit_private.h: Add new flag to jit_block_t.
+	* lib/lightning.c: Rewrite register liveness and state at block
+	entry code to avoid a very expensive and non scaling code path.
+	Now it attempts to do as few as possible recomputations when
+	merging state of adjacent blocks, still doing one extra nop pass
+	(in the sense that it will not find any changes) to make sure the
+	logic is correct.
+
+2022-09-30 Paulo Andrade <pcpa@gnu.org>
+
+	* include/lightning/jit_private.h: Implement new data structures
+	specific to riscv.
+	* lib/jit_disasm.c: Extra disassemble code for riscv constant pool.
+	* lib/jit_riscv-cpu.c: Modify movi to use constant pool if 3 or
+	more instructions are required to construct constant and modify
+	movi_p to use a pc relative load from a constant pool.
+	lib/jit_riscv-sz.c: Update for new constant pool code. Most
+	instructions that need 64 bit constants are significantly reduced.
+	* lib/jit_riscv.c: Implement most of the constant pool code.
+	* lib/jit_size.c: Update for estimate of code generation size.
+	* lib/lightning.c: Update for riscv specific code, and also make
+	sure to mprotect the constant pool as executable.
+
+2022-09-08 Paulo Andrade <pcpa@gnu.org>
+
+	* lib/jit_fallback.c: Implement fallback compare and swap with
+	pthreads.
+	* check/Makefile.am: Update for new cas{r,i} simple test.
+	* check/catomic.c, check/catomic.ok: New test case for
+	simple compare and swap atomic operation.
+	* check/lightning.c: Add entries to be able to use
+	the new compare and swap atomic operation. Still missing
+	a general test, only the basic C version.
+	* include/lightning.h.in: Include pthread.h, even if not
+	needing a fallback compare and swap.
+	* include/lightning/jit_private.h: Add support for a register pair
+	in second argument. Required by the new casr and casi operations.
+	* lib/jit_aarch64-cpu.c, lib/jit_aarch64-sz.c, lib/jit_aarch64.c,
+	lib/jit_ppc-cpu.c, lib/jit_ppc-sz.c, lib/jit_ppc.c, lib/jit_x86-cpu.c,
+	lib/jit_x86-sz.c, lib/jit_x86.c: Implement inline code for compare
+	and swap.
+	* lib/jit_arm-cpu.c, lib/jit_arm-sz.c, lib/jit_arm.c: Implement
+	inline code for compare and swap if cpu is armv7, otherwise, use
+	a fallback with pthreads.
+	* lib/jit_alpha-cpu.c, lib/jit_alpha-sz.c, lib/jit_alpha.c,
+	lib/jit_hppa-cpu.c, lib/jit_hppa-sz.c, lib/jit_hppa.c,
+	lib/jit_ia64-cpu.c, lib/jit_ia64-sz.c, lib/jit_ia64.c,
+	lib/jit_mips-cpu.c, lib/jit_mips-sz.c, lib/jit_mips.c,
+	lib/jit_riscv-cpu.c, lib/jit_riscv-sz.c, lib/jit_riscv.c,
+	lib/jit_s390-cpu.c, lib/jit_s390-sz.c, lib/jit_s390.c,
+	lib/jit_sparc-cpu.c, lib/jit_sparc-sz.c, lib/jit_sparc.c: Implement
+	fallback compare and swap with pthreads. At least some of these
+	should be updated for inline code generation.
+	* lib/jit_names.c, lib/jit_print.c: lib/lightning.c: Update for the
+	new compare and	swap operation.
+	* doc/body.texi: Add simple documentation of the compare and swap
+	new operation.
+
+2022-08-12  Marc Nieper-Wißkirchen  <marc@nieper-wisskirchen.de>
+
+	Document jit_align.
+	* doc/body.texi: Add documentation for jit_align.
+
+2022-05-14 Paulo Andrade <pcpa@gnu.org>
+
+	* include/lightning.h.in: Reorder jit_mov{n,z}r in instruction list.
+	* lib/jit_alpha.c, lib/jit_alpha-cpu.c, lib/jit_hppa.c,
+	lib/jit_hppa-cpu.c, lib/jit_ia64.c, lib/jit_ia64-cpu.c,
+	lib/jit_riscv.c, lib/jit_riscv-cpu.c, lib/jit_s390.c,
+	lib/jit_s390-cpu.c, lib/jit_sparc.c, lib/jit_sparc-cpu.c:
+	Implement fallback jit_mov{n,z}r. These are a somewhat cheap
+	implementation, but should be reviewed for the arches that already
+	have a proper conditional move.
+	* lib/jit_arm-sz.c, lib/jit_mips-sz.c: Add missing maximum size
+	estimative and reorder.
+	* lib/jit_aarch64-sz.c, lib/jit_x86-sz.c, lib/jit_ppc-sz.c:
+	Reorder entry to match definition order.
+	* lib/jit_aarch64-sz.c, lib/jit_alpha-sz.c, lib/jit_hppa-sz.c,
+	lib/jit_ia64-sz.c, lib/jit_riscv-sz.c, lib/jit_s390-sz.c,
+	lib/jit_sparc-sz.c: Add heuristic value, basically the sum of
+	the cost of a movr + beqr.
+	* lib/jit_names.c: Add entries for debug output of mov{n,z}r.
+	* lib/lightning.c: Use proper bitmask in jit_classify.
+
 2021-04-03  Marc Nieper-Wißkirchen  <marc@nieper-wisskirchen.de>
 
 	* check/Makefile.am: Add test for the live instruction.

deps/lightning/README

@@ -1,3 +1,5 @@
 GNU lightning is a library to aid in making portable programs
 that compile assembly code at run time.  For more information,
 look at the info documentation.
+
+For help building lightning, see README-hacking.

deps/lightning/README-hacking

@@ -22,6 +22,12 @@ for Debian-based systems such as Ubuntu:
 
 ** Building
 
+If you intend to do development work with lightning, it's useful to build
+lightning with its disassembler feature enabled.  This optional feature
+requires additional dependencies.  On Ubuntu, this command should work:
+
+    $ sudo apt-get install binutils-dev libiberty-dev zlib1g-dev
+
 After getting the git sources, and installing the tools above, you can run
 
     $ ./bootstrap
@@ -38,6 +44,10 @@ should output no difference.
 
 After that first time, running make should suffice.
 
+To install lightning:
+
+    $ sudo make install
+
 ** Gnulib
 
 This distribution also uses Gnulib (https://www.gnu.org/software/gnulib) to

deps/lightning/THANKS

@@ -19,3 +19,4 @@ Holger Hans Peter Freyther      <holger@moiji-mobile.com>
 Jon Arintok                     <jon.arintok@gmail.com>
 Bruno Haible                    <bruno@clisp.org>
 Marc Nieper-Wißkirchen		<marc@nieper-wisskirchen.de>
+Paul Cercueil                   <paul@crapouillou.net>

deps/lightning/TODO

@@ -1,28 +1 @@
-	* Validate that divrem in jit_x86-cpu.c is not modifying
-	the non result arguments. This is not verified by clobber.tst,
-	as it only checks registers not involved in the operation
-	(because it does not know about values being set as input
-	for the the operation).
 
-	* Write a simple higher level language implementation generating
-	jit with lightning, that could be some lisp or C like language.
-
-	* rerun ./configure --enable-devel-get-jit-size and regenerate
-	the related jit_$arch-sz.c for the ports where nodata is
-	meaningful:
-	hppa		(done)
-	i586		(done)
-	ia64
-	mips o32	(done)
-	mips n32
-	mips n64
-	powerpc 32	(done)
-	powerpc 64	(done)
-	ppc
-	s390x		(done)
-	sparc		(done)
-	x86_64		(done)
-	Missing ones are due to no longer (remote) access to such hosts
-	and may be broken with jit_set_data(..., JIT_DISABLE_DATA).
-	(ia64 hp-ux or linx), (irix mips for 32 or 64 abi), and
-	(darwin ppc).

deps/lightning/include/lightning/jit_aarch64.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013-2019  Free Software Foundation, Inc.
+ * Copyright (C) 2013-2022  Free Software Foundation, Inc.
  *
  * This file is part of GNU lightning.
  *
@@ -23,6 +23,10 @@
 #define JIT_HASH_CONSTS		0
 #define JIT_NUM_OPERANDS	3
 
+#if __APPLE__
+#  define PACKED_STACK		1
+#endif
+
 /*
  * Types
  */

deps/lightning/include/lightning/jit_alpha.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2014-2019  Free Software Foundation, Inc.
+ * Copyright (C) 2014-2022  Free Software Foundation, Inc.
  *
  * This file is part of GNU lightning.
  *