Merge pull request #42 from joeltucci/jira/COMMON-5148

COMMON-5148 Added support for AARCH64 in Linux

compress/README.md

@@ -1,23 +1,24 @@
 # util-compress
 
 ## About
-util-compress is a library for compressing and uncompressing data. Includes snappy and gzip.
+util-compress is a library for compressing and uncompressing data. Includes snappy and gzip and a pluggable framework to support other codecs.
 
-A binary is included for Linux-amd64. It will not work on 32-bit platforms. Other platforms will need to compile from source and may not compile or work as-is.
+A binary is included for X86_64 and AARCH64 on both Linux and MacOS. Other platforms, including 32 bit platforms, will need to compile from source and may not compile or work as-is.
 
 ## Compiling native code on 64-bit linux
 
-**Requires libsnappy-dev**
+**Requires libsnappy-dev and libz-dev**
 
 You can build the native code for compress yourself by executing the `updateNative` task. It compiles the native code
-using the provided Makefile, which depends on GCC. On Linux, a 64-bit environment with `gcc-multilib` installed is
-required so both 64-bit and 32-bit versions will be installed. On macOS, a universal dylib for both `x86_64` and `arm64`
-is built automatically. Binaries are installed to `compress/src/main/resources/`. for inclusion in the packaged jar.
+using the provided Makefile, which depends on GCC. 
+For Linux if you are running on an x86 you will need to install "aarch64-linux-gnu-gcc" compiler, if you are running on an AARCH64 you will need to install "x86_64-linux-gnu-gcc" compiler. 
+
+On macOS, a universal dylib for both `x86_64` and `arm64` is built automatically. Binaries are installed to `compress/src/main/resources/`. for inclusion in the packaged jar.
 
 The makefile can also be invoked directly with:
 
 ```
-$ ./gradlew :mmap:compileJava
+$ ./gradlew :compress:compileJava
 $ cd compress/src/main/c/
 $ make clean install
 ```

compress/src/main/c/Makefile

@@ -6,7 +6,9 @@ all: build
 
 OS := $(shell uname -s | tr A-Z a-z)
 ARCH ?= $(shell uname -m)
-CC := gcc
+REALARCH := $(shell uname -m)
+
+CC ?= gcc
 
 #CFLAGS += -g -fPIC -O3 -Wall
 CFLAGS += -fPIC -O3 -std=c99
@@ -23,16 +25,23 @@ ifeq ($(OS), darwin)
 	LD_FLAGS += -dynamiclib -arch arm64 -arch x86_64 -Wl,-rpath,/opt/homebrew/lib
 	DEST_DIR := "../resources/native/Mac OS X"
 else
-	ifneq ($(ARCH), x86_64)
-		CFLAGS += -m32
-		LD_FLAGS += -m32
-		DEST_DIR := "../resources/native/Linux-i386"
-	else
-		DEST_DIR := "../resources/native/Linux-amd64"
-	endif
+	ifeq ($(ARCH), x86_64)
+                DEST_DIR := "../resources/native/Linux-amd64/"
+        else
+                DEST_DIR := "../resources/native/Linux-$(ARCH)/"
+        endif
 	LIBTYPE := so
 	LD_FLAGS += -shared
-	#LD_FLAGS += -shared -Wl,-soname,$(LIBSONAME)
+
+	ifeq ($(ARCH),$(REALARCH))
+            ifeq ($(ARCH),x86_64)
+                ADDITIONAL_ARCH := aarch64
+                ADDITIONAL_CC := aarch64-linux-gnu-gcc
+            else
+                ADDITIONAL_ARCH := x86_64
+                ADDITIONAL_CC := x86_64-linux-gnu-gcc
+            endif
+	endif
 endif
 
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
@@ -56,22 +65,20 @@ OBJS = \
 $(LIBNAME): $(OBJS)
 	$(CC) $(LD_FLAGS) -o $@ $(OBJS)
 
-build: $(LIBNAME) $(if $(OS:linux=),,$(if $(ARCH:x86_64=),,build32))
-
-build32:
-	$(MAKE) ARCH=i386 build
+buildAdditional:
+	$(MAKE) CC=$(ADDITIONAL_CC) ARCH=$(ADDITIONAL_ARCH)
+build: $(LIBNAME) $(if $(ADDITIONAL_ARCH),buildAdditional)
 
 $(DEST_DIR):
 	mkdir -p $(DEST_DIR)
 
-install: build $(DEST_DIR) $(if $(OS:linux=),,$(if $(ARCH:x86_64=),,install32))
+install: build $(DEST_DIR) $(if $(ADDITIONAL_ARCH),installAdditional)
 	cp $(LIBNAME) $(DEST_DIR)/$(subst -$(ARCH),,$(LIBNAME))
 
-install32:
-	$(MAKE) ARCH=i386 install
+installAdditional: 
+	$(MAKE) ARCH=$(ADDITIONAL_ARCH) install
 
-clean: $(if $(OS:linux=),,$(if $(ARCH:x86_64=),,clean32))
+clean: $(if $(ADDITIONAL_ARCH),cleanAdditional)
 	rm -rf $(OBJS) $(TARGET)
-
-clean32:
-	$(MAKE) ARCH=i386 clean
+cleanAdditional:
+	$(MAKE) ARCH=$(ADDITIONAL_ARCH) clean

mmap/README.md

@@ -22,14 +22,14 @@ The latest util-mmap JAR file can be downloaded via Maven ([link](http://search.
 
 The library depends on [Unix mmap](http://www.gnu.org/software/libc/manual/html_node/Memory_002dmapped-I_002fO.html)
 for its underlying functionality, so it uses JNI. The published JAR file contains 
-native builds for Linux `i386`, Linux `amd64`, macOS `x86_64` and macOS `arm64`. The Java code in the library loads the 
+native builds for Linux `aarch64`, Linux `amd64`, macOS `x86_64` and macOS `arm64`. The Java code in the library loads the 
 correct Linux native shared object (.so, .dylib) file from the JAR file at runtime based
 on `os.name` and `os.arch` system properties. If you need to run on an unsupported OS, you'll
 need to rebuild the native code. See the instructions in [Building](#building).
 
 ## Usage
 
-A good starting point for using util-mmap is the [MMapBuffer class](https://eng-git.ausoff.indeed.net/opensource/util/blob/master/mmap/src/main/java/com/indeed/util/mmap/MMapBuffer.java).
+A good starting point for using util-mmap is the [MMapBuffer class](https://github.com/indeedeng/util/blob/main/mmap/src/main/java/com/indeed/util/mmap/MMapBuffer.java).
 The following example maps in a large file containing an array of longs, written in little-endian order.
 
 ```
@@ -45,10 +45,11 @@ final long firstValue = longArray.get(0);
 ## Building
 
 You can build the native code for util-mmap yourself by executing the `updateNative` task. It compiles the native code
-using the provided Makefile, which depends on GCC. On Linux, a 64-bit environment with `gcc-multilib` installed is
-required so both 64-bit and 32-bit versions will be installed. On macOS, a universal dylib for both `x86_64` and `arm64`
+using the provided Makefile, which depends on GCC. On Linux you will need a 64 bit x86 or aarch64 environment, other architectures are not currently supported. On macOS, a universal dylib for both `x86_64` and `arm64`
 is built automatically. Binaries are installed to `mmap/src/main/resources/`. for inclusion in the packaged jar.
 
+For Linux if you are running on an x86 you will need to install "aarch64-linux-gnu-gcc" to build the AARCH64 library, if you are running on an AARCH64 you will need to install "x86_64-linux-gnu-gcc" to build the x86 compatible library 
+
 The makefile can also be invoked directly with:
 
 ```

mmap/src/main/c/Makefile

@@ -6,7 +6,9 @@ all: build
 
 OS := $(shell uname -s | tr A-Z a-z)
 ARCH ?= $(shell uname -m)
-CC := gcc
+REALARCH := $(shell uname -m)
+
+CC ?= gcc
 
 CFLAGS += -g -fPIC -O3 -Wall
 
@@ -23,13 +25,21 @@ ifeq ($(OS), darwin)
 	LD_FLAGS += -dynamiclib -arch arm64 -arch x86_64
 	DEST_DIR := "../resources/native/Mac OS X/"
 else
-	ifneq ($(ARCH), x86_64)
-		CFLAGS += -m32
-		LD_FLAGS += -m32
-		DEST_DIR := "../resources/native/Linux-i386/"
-	else
+	ifeq ($(ARCH), x86_64)
 		DEST_DIR := "../resources/native/Linux-amd64/"
+	else
+		DEST_DIR := "../resources/native/Linux-$(ARCH)/"
 	endif
+	ifeq ($(ARCH), $(REALARCH))
+		ifeq ($(ARCH), x86_64)
+			ADDITIONAL_ARCH := aarch64
+			ADDITIONAL_CC := aarch64-linux-gnu-gcc
+		else
+			ADDITIONAL_ARCH := x86_64
+			ADDITIONAL_CC := x86_64-linux-gnu-gcc
+		endif
+	endif
+
 	LIBTYPE := so
 	LD_FLAGS += -shared -Wl,-soname,$(LIBSONAME)
 endif
@@ -54,19 +64,18 @@ OBJS = \
 $(REALLIBNAME): $(OBJS)
 	$(CC) $(LD_FLAGS) -o $@ $(OBJS)
 
-build: $(REALLIBNAME) $(if $(OS:linux=),,$(if $(ARCH:x86_64=),,build32))
+build: $(REALLIBNAME) $(if $(ADDITIONAL_ARCH),buildAdditional)
 
-build32:
-	$(MAKE) ARCH=i386 build
+buildAdditional:
+	$(MAKE) CC=$(ADDITIONAL_CC) ARCH=$(ADDITIONAL_ARCH)
 
-install: build $(if $(OS:linux=),,$(if $(ARCH:x86_64=),,install32))
+install: build $(if $(ADDITIONAL_ARCH),installAdditional)
 	cp $(REALLIBNAME) $(DEST_DIR)/$(subst -$(ARCH),,$(REALLIBNAME))
 
-install32:
-	$(MAKE) ARCH=i386 install
+installAdditional:
+	$(MAKE) ARCH=$(ADDITIONAL_ARCH) install
 
-clean: $(if $(OS:linux=),,$(if $(ARCH:x86_64=),,clean32))
+clean: $(if $(ADDITIONAL_ARCH),cleanAdditional)
 	rm -rf $(OBJS) $(TARGET)
-
-clean32:
-	$(MAKE) ARCH=i386 clean
+cleanAdditional:
+	$(MAKE) ARCH=$(ADDITIONAL_ARCH) clean