Merge pull request #141 from TheBlueMatt/main

LDK 0.0.118

.github/workflows/build.yml

@@ -40,11 +40,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://github.com/lightningdevkit/rust-lightning
           cd rust-lightning
-          git checkout origin/0.0.117-bindings
+          git checkout origin/0.0.118-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.117
+          git checkout 0.0.118
           cd lightning-c-bindings
           cargo update -p memchr --precise "2.5.0" --verbose
       - name: Rebuild C bindings without STD
@@ -108,15 +108,15 @@ jobs:
     runs-on: ubuntu-latest
     # Ubuntu's version of rustc uses its own LLVM instead of being a real native package.
     # This leaves us with an incompatible LLVM version when linking. Instead, use a real OS.
-    container: debian:bullseye
+    container: debian:bookworm
     strategy:
       fail-fast: false
     steps:
       - name: Install native Rust toolchain, Valgrind, and build utilitis
         run: |
           apt-get update
           apt-get -y dist-upgrade
-          apt-get -y install cargo valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl
+          apt-get -y install cargo valgrind lld git g++ clang openjdk-17-jdk maven faketime zip unzip llvm curl
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
@@ -136,11 +136,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://github.com/lightningdevkit/rust-lightning
           cd rust-lightning
-          git checkout origin/0.0.117-bindings
+          git checkout origin/0.0.118-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.117
+          git checkout 0.0.118
           cd lightning-c-bindings
           cargo update -p memchr --precise "2.5.0" --verbose
       - name: Pin proc-macro and quote to meet MSRV
@@ -153,15 +153,15 @@ jobs:
           cd ldk-c-bindings
           ./genbindings.sh ../rust-lightning true
       - name: Build Java Debug Bindings
-        run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false
+        run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-17-openjdk-amd64/include/ -I/usr/lib/jvm/java-17-openjdk-amd64/include/linux/" true false
       - name: Run Java Tests against Debug Bindings
         run: |
           mv liblightningjni_debug_Linux-amd64.so liblightningjni.so
           export ASAN_OPTIONS=detect_leaks=0
-          LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so LD_LIBRARY_PATH=. mvn test
+          LD_PRELOAD=/usr/lib/llvm-14/lib/clang/14.0.6/lib/linux/libclang_rt.asan-x86_64.so LD_LIBRARY_PATH=. mvn test
       - name: Build Java Release Bindings
         run: |
-          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
+          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-17-openjdk-amd64/include/ -I/usr/lib/jvm/java-17-openjdk-amd64/include/linux/" false false
       - name: Check latest headers are in git
         run: |
           git checkout pom.xml
@@ -171,15 +171,15 @@ jobs:
     runs-on: ubuntu-latest
     # Ubuntu's version of rustc uses its own LLVM instead of being a real native package.
     # This leaves us with an incompatible LLVM version when linking. Instead, use a real OS.
-    container: debian:bullseye
+    container: debian:bookworm
     strategy:
       fail-fast: false
     steps:
       - name: Install native Rust toolchain, Valgrind, and build utilitis
         run: |
           apt-get update
           apt-get -y dist-upgrade
-          apt-get -y install cargo valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl
+          apt-get -y install cargo valgrind lld git g++ clang openjdk-17-jdk maven faketime zip unzip llvm curl
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
@@ -199,11 +199,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://github.com/lightningdevkit/rust-lightning
           cd rust-lightning
-          git checkout origin/0.0.117-bindings
+          git checkout origin/0.0.118-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.117
+          git checkout 0.0.118
           cd lightning-c-bindings
           cargo update -p memchr --precise "2.5.0" --verbose
       - name: Pin proc-macro and quote to meet MSRV
@@ -238,7 +238,7 @@ jobs:
       - name: Build Leaktracking Java Release Bindings
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" leaks false
+          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-17-openjdk-amd64/include/ -I/usr/lib/jvm/java-17-openjdk-amd64/include/linux/" leaks false
       - name: Build deterministic release jar
         run: |
           ./build-release-jar.sh
@@ -248,30 +248,33 @@ jobs:
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
           cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_MacOSX-"{aarch64,x86_64}.nativelib src/main/resources/
+      - name: Build Java leaktracking Bindings
+        run: |
+          export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
+          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-17-openjdk-amd64/include/ -I/usr/lib/jvm/java-17-openjdk-amd64/include/linux/" leaks false
+          mkdir ldk-java-bins/new/
+          cp src/main/resources/liblightningjni_Linux-amd64.nativelib ldk-java-bins/new/liblightningjni_Linux-amd64-leaktracking.nativelib
       - name: Build Java Release Bindings
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
+          ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-17-openjdk-amd64/include/ -I/usr/lib/jvm/java-17-openjdk-amd64/include/linux/" false false
       - name: Build deterministic release jar
         run: ./build-release-jar.sh
       - name: Check latest library and jars are in bins repo
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          mkdir ldk-java-bins/new/
           cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_Mac*.nativelib ldk-java-bins/new/
           cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"LDK-release.aar ldk-java-bins/new/
           cp src/main/resources/liblightningjni_Linux-amd64.nativelib ldk-java-bins/new/
           cp ldk-java-sources.jar ldk-java-bins/new/
           cp ldk-java.jar ldk-java-bins/new/
           cp ldk-java-leaktracking.jar ldk-java-bins/new/
           cp ldk-java-classes.jar ldk-java-bins/new/
-          cp ldk-java-javadoc.jar ldk-java-bins/new/
           cd ldk-java-bins
           rm "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjs.wasm" # TODO: Test this
           if ! diff -r "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" new/; then
             apt-get -y install diffoscope
             diffoscope new/ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar" || echo
-            diffoscope new/ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar" || echo
             diffoscope new/ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" || echo
             diffoscope new/ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar" || echo
             diffoscope new/ldk-java-leaktracking.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-leaktracking.jar" || echo
@@ -316,11 +319,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://github.com/lightningdevkit/rust-lightning
           cd rust-lightning
-          git checkout origin/0.0.117-bindings
+          git checkout origin/0.0.118-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.117
+          git checkout 0.0.118
       - name: Fetch MacOS SDK
         run: |
           curl -o Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz
@@ -409,15 +412,15 @@ jobs:
   android:
     runs-on: ubuntu-latest
     # Frankly, I'm not really sure why debian and ubuntu differ in the results here, they really shouldn't
-    container: debian:bullseye
+    container: debian:bookworm
     strategy:
       fail-fast: false
     steps:
       - name: Install rust targets
         run: |
           apt-get update
           apt-get -y dist-upgrade
-          apt-get -y install git g++ clang faketime zip unzip curl openjdk-11-jdk
+          apt-get -y install git g++ clang faketime zip unzip curl openjdk-17-jdk
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh
           chmod +x ./rustup.sh
           ./rustup.sh -y
@@ -453,11 +456,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://github.com/lightningdevkit/rust-lightning
           cd rust-lightning
-          git checkout origin/0.0.117-bindings
+          git checkout origin/0.0.118-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.117
+          git checkout 0.0.118
           cd lightning-c-bindings
           . $HOME/.cargo/env
           cargo update -p memchr --precise "2.5.0" --verbose
@@ -529,11 +532,11 @@ jobs:
           # Note this is a different endpoint, as we need one non-upstream commit!
           git clone https://github.com/rghtningdevkit/ust-lightning
           cd rust-lightning
-          git checkout origin/0.0.117-bindings
+          git checkout origin/0.0.118-bindings
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.117
+          git checkout 0.0.118
           cd lightning-c-bindings
           cargo update -p memchr --precise "2.5.0" --verbose
       - name: Rebuild C bindings and check the sample app builds + links

build-release-jar.sh

@@ -12,9 +12,9 @@ ls src/main/resources/liblightningjni_MacOSX-aarch64.nativelib
 export LC_ALL=C
 
 # We need to fetch dependencies first as faketime will break PKI cert checks!
-mvn -DskipTests=true -Dorg.lightningdevkit.skipdocs=false package
+mvn -DskipTests=true package
 mvn clean
-faketime 2021-01-01 mvn -DskipTests=true -Dorg.lightningdevkit.skipdocs=false package
+faketime 2021-01-01 mvn -DskipTests=true package
 
 mkdir ziptmp
 cd ziptmp
@@ -26,26 +26,6 @@ rm ../target/ldk-java-*-sources.jar
 cd ..
 rm -r ziptmp
 
-mkdir ziptmp
-cd ziptmp
-unzip ../target/ldk-java-*-javadoc.jar | grep inflating | awk '{ print $2 }' | sort > ../javadoc-zip-files.txt
-find . | grep '.html$' | xargs -L1 sed -i 's/Generated by javadoc \(.*\) on Fri Jan 01 .* UTC 2021/Generated by javadoc \1 on Fri Jan 01 00:00:00 UTC 2021/g'
-for ZIP in *.zip; do
-	mkdir ziptmptmp
-	cd ziptmptmp
-	unzip ../$ZIP | grep inflating | awk '{ print $2 }' | sort > ../../javadoc-zip-zip-files.txt
-	rm ../$ZIP
-	touch -d "2021-01-01 00:00 UTC" $(cat ../../javadoc-zip-zip-files.txt)
-	cat ../../javadoc-zip-zip-files.txt | zip -X@ ../$ZIP
-	cd ..
-	rm -r ziptmptmp
-done
-touch -d "2021-01-01 00:00 UTC" $(cat ../javadoc-zip-files.txt)
-cat ../javadoc-zip-files.txt | grep -v nativelib | zip -X@ ../ldk-java-javadoc.jar
-rm ../target/ldk-java-*-javadoc.jar
-cd ..
-rm -r ziptmp
-
 mkdir ziptmp
 cd ziptmp
 unzip ../target/ldk-java-*.jar | grep inflating | awk '{ print $2 }' | sort > ../classes-zip-files.txt

csharp_strings.py

@@ -440,7 +440,7 @@ def __init__(self, DEBUG: bool, target: Target, **kwargs):
         self.usize_c_ty = "int64_t"
         self.usize_native_ty = "long"
         self.native_zero_ptr = "0"
-        self.result_c_ty = "jclass"
+        self.unitary_enum_c_ty = "int32_t"
         self.ptr_arr = "jobjectArray"
         self.is_arr_some_check = ("", " != NULL")
         self.get_native_arr_len_call = ("(*env)->GetArrayLength(env, ", ")")
@@ -1033,7 +1033,7 @@ def map_complex_enum(self, struct_name, variant_list, camel_to_snake, enum_doc_c
                 out_c += "\t" + struct_name + " *obj = (" + struct_name + "*)untag_ptr(ptr);\n"
                 out_c += f"\tassert(obj->tag == {struct_name}_{var.var_name});\n"
                 if field_map.ret_conv is not None:
-                    out_c += ("\t\t\t" + field_map.ret_conv[0].replace("\n", "\n\t\t\t"))
+                    out_c += ("\t" + field_map.ret_conv[0].replace("\n", "\n\t"))
                     if var.tuple_variant:
                         out_c += "obj->" + camel_to_snake(var.var_name)
                     else:

genbindings.py

@@ -384,7 +384,7 @@ def java_c_types(fn_arg, ret_arr_len):
             assert ma.group(1).strip().startswith("LDK")
             java_ty = ma.group(1).strip()[3:]
             java_hu_ty = java_ty
-            c_ty = consts.result_c_ty
+            c_ty = consts.unitary_enum_c_ty
             fn_ty_arg = "Lorg/ldk/enums/" + java_ty + ";"
             fn_arg = ma.group(2).strip()
             rust_obj = ma.group(1).strip()

genbindings.sh

@@ -280,62 +280,10 @@ else
 		LDK_LIB="$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
 		if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then
 			LINK="$LINK -Wl,--version-script=libcode.version -fuse-ld=lld"
-			# __cxa_thread_atexit_impl is used to more effeciently cleanup per-thread local storage by rust libstd.
-			# However, it is not available on glibc versions 2.17 or earlier, and rust libstd has a null-check and
-			# fallback in case it is missing.
-			# Because it is weak-linked on the rust side, we should be able to simply define it
-			# explicitly, forcing rust to use the fallback. However, for some reason involving ancient
-			# dark magic and haunted code segments, overriding the weak symbol only impacts sites which
-			# *call* the symbol in question, not sites which *compare with* the symbol in question.
-			# This means that the NULL check in rust's libstd will always think the function is
-			# callable while the function which is called ends up being NULL (leading to a jmp to the
-			# zero page and a quick SEGFAULT).
-			# This issue persists not only with directly providing a symbol, but also ld.lld's -wrap
-			# and --defsym arguments.
-			# In smaller programs, it appears to be possible to work around this with -Bsymbolic and
-			# -nostdlib, however when applied the full-sized JNI library here it no longer works.
-			# After exhausting nearly every flag documented in lld, the only reliable method appears
-			# to be editing the LDK binary. Luckily, LLVM's tooling makes this rather easy as we can
-			# disassemble it into very readable code, edit it, and then reassemble it.
-			# Note that if we do so we don't have to bother overriding the actual call, LLVM should
-			# optimize it away, which also provides a good check that there isn't anything actually
-			# relying on it elsewhere.
-			[ ! -f "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a ] && exit 1
-			if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep -v "\.o$" || echo)" != "" ]; then
-				echo "Archive contained non-object files!"
-				exit 1
-			fi
-			if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep ldk.*-cgu.*.rcgu.o | wc -l)" != "1" ]; then
-				echo "Archive contained more than one LDK object file"
-				exit 1
-			fi
-			mkdir -p tmp
-			rm -f tmp/*
-			ar x --output=tmp "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
-			pushd tmp
-			llvm-dis ldk*-cgu.*.rcgu.o
-			sed -i 's/br i1 icmp eq (i8\* @__cxa_thread_atexit_impl, i8\* null)/br i1 icmp eq (i8* null, i8* null)/g' ldk*-cgu.*.rcgu.o.ll
-			llvm-as ldk*-cgu.*.rcgu.o.ll -o ./libldk.bc
-			ar q libldk.a *.o
-			popd
-			LDK_LIB="tmp/libldk.bc tmp/libldk.a"
 		fi
 		$COMPILE -o bindings.o -c -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c
 		$COMPILE $LINK -o liblightningjni_release$LDK_TARGET_SUFFIX.so -O3 -I"$1"/lightning-c-bindings/include/ $2 bindings.o $LDK_LIB -lm
 		[ "$IS_APPLE_CLANG" != "true" ] && llvm-strip liblightningjni_release$LDK_TARGET_SUFFIX.so
-		if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then
-			GLIBC_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep GLIBC_ | grep -v "GLIBC_2\.2\." | grep -v "GLIBC_2\.3\(\.\| \)" | grep -v "GLIBC_2.\(14\|17\) " || echo)"
-			if [ "$GLIBC_SYMBS" != "" ]; then
-				echo "Unexpected glibc version dependency! Some users need glibc 2.17 support, symbols for newer glibcs cannot be included."
-				echo "$GLIBC_SYMBS"
-				exit 1
-			fi
-			REALLOC_ARRAY_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep reallocarray || echo)"
-			if [ "$REALLOC_ARRAY_SYMBS" != "" ]; then
-				echo "Unexpected reallocarray dependency!"
-				exit 1
-			fi
-		fi
 		if [ "$LDK_JAR_TARGET" = "true" ]; then
 			# Copy to JNI native directory for inclusion in JARs
 			mkdir -p src/main/resources/

java_strings.py

@@ -627,7 +627,7 @@ class CommonBase {
         self.usize_c_ty = "int64_t"
         self.usize_native_ty = "long"
         self.native_zero_ptr = "0"
-        self.result_c_ty = "jclass"
+        self.unitary_enum_c_ty = "jclass"
         self.ptr_arr = "jobjectArray"
         self.is_arr_some_check = ("", " != NULL")
         self.get_native_arr_len_call = ("(*env)->GetArrayLength(env, ", ")")

python_strings.py

@@ -679,7 +679,7 @@ def __init__(self, DEBUG: bool, target: Target, outdir: str, **kwargs):
         self.usize_c_ty = "uint32_t"
         self.usize_native_ty = "number"
         self.native_zero_ptr = "0n"
-        self.result_c_ty = "uint32_t"
+        self.unitary_enum_c_ty = "uint32_t"
         self.ptr_arr = "ptrArray"
         self.is_arr_some_check = ("", " != 0")
         self.get_native_arr_len_call = ("", "->arr_len")

src/main/java/org/ldk/batteries/ChannelManagerConstructor.java

@@ -54,8 +54,8 @@ public static class InvalidSerializedDataException extends Exception {
     public PeerManager peer_manager = null;
     /**
      * A NioPeerHandler which manages a background thread to handle socket events and pass them to the peer_manager.
-	 *
-	 * This is `null` until `chain_sync_completed` is called.
+     *
+     * This is `null` until `chain_sync_completed` is called.
      */
     public NioPeerHandler nio_peer_handler = null;
 
@@ -288,8 +288,9 @@ public void chain_sync_completed(EventHandler event_handler, boolean use_p2p_gra
             routing_msg_handler = graph_msg_handler.as_RoutingMessageHandler();
         else
             routing_msg_handler = ignoring_handler.as_RoutingMessageHandler();
+        OnionMessenger messenger = OnionMessenger.of(this.entropy_source, this.node_signer, this.logger, DefaultMessageRouter.of().as_MessageRouter(), channel_manager.as_OffersMessageHandler(), IgnoringMessageHandler.of().as_CustomOnionMessageHandler());
         this.peer_manager = PeerManager.of(channel_manager.as_ChannelMessageHandler(),
-                routing_msg_handler, ignoring_handler.as_OnionMessageHandler(),
+                routing_msg_handler, messenger.as_OnionMessageHandler(),
                 ignoring_handler.as_CustomMessageHandler(), (int)(System.currentTimeMillis() / 1000),
                 this.entropy_source.get_secure_random_bytes(), logger, this.node_signer);