.make.defaults

#######################################################################################
# Include this to get access to a common set of rules for reuse in Makefile-based projects.
# include ../../.make.defaults
#
# Before including, the following must be defined:
# REPOROOT points to the top of the git repository. 
# 	For example REPOROOT=../../..
# DOCKER_NAME - defines the name of the docker image
#	For example DOCKER_NAME=noop
# EXTRA_INDEX_URL - can define an additional pipy index for use 
#	in creating venv and images.  This will be used on pip installs
#	when building the venv and will be made available as
#	a docker build argument when building the docker image.
#	This is useful when the repo is used in conjunction with an 
#	inhouse or private pypi.
#
# Targets defined here use double colon so can be overriden
#
# Reusable rules begin with '.'. To reuse without modification, for example,
# define your local Makefile rule as follows:
# clean: .clean
# To augment the the clean rule
# clean: .clean
# 	rm -rf other-stuff
#
#######################################################################################
SHELL=/bin/bash
# Command to run python 
PYTHON?=python
PIP=$(PYTHON) -m pip
# Command to run pytest
PYTEST=pytest -s
PYTHON_VERSION=$(shell $(PYTHON) --version)
ABS_REPOROOT=$(shell (cd $(REPOROOT); pwd))
DOCKER_FILE?=Dockerfile
#DOCKER_NAME?=xyzzy	# Must be defined by the includeing makefile
DOCKER?=docker
DOCKER_HOSTNAME?=quay.io
DOCKER_NAMESPACE ?= dataprep1/data-prep-kit
DOCKER_REGISTRY_USER?=$(DPK_DOCKER_REGISTRY_USER)
DOCKER_REGISTRY_KEY?=$(DPK_DOCKER_REGISTRY_KEY)
DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE)
DOCKER_IMAGE?=${DOCKER_REGISTRY_ENDPOINT}/$(DOCKER_NAME):$(DOCKER_IMAGE_VERSION)
include $(REPOROOT)/.make.versions

KIND_CLUSTER_NAME=dataprep
DPK_RAY_LIB_DIR=$(REPOROOT)/data-processing-lib/ray
#######################################################################################
# Lists all targets and optional help text found in the target.
# Adapted from https://stackoverflow.com/a/65243296/45375
# and modified to support help at the top or middle of the recursive tree and at 
# the leaves (the head and tail piece).
.PHONY: help
.newhelp::
	# Expects a line of the following in the rule
	@printf "%-15s %s\n" "Target" "Description"
	@printf "%-15s %s\n" "--------------" "------------------------"
	@make -pqR : 2>/dev/null  | grep '@# Help:' | grep -v make | grep -v IGNORE | sed -e 's/.*Help://' -e 's/[ 	]*//' | \
		awk -F, '{printf("%-15s%s\n",$$1,$$2)}'

# Gnu awk requires running awk in 2 steps, below.
help::
	@printf "%-20s %s\n" "Target" "Description"
	@printf "%-20s %s\n" "------" "-----------"
	@export submakes=$$(find ./* -mindepth 1 -maxdepth 1 -name Makefile);	\
	make -pqR : 2>/dev/null \
		| awk  '/^# File/,/^# Finished Make data base/ {print $$0}' \
		| awk  -v RS= -F:  '{if ($$1 !~ "^[#.]") {print $$1}}' \
		| sort \
		| egrep -v -e '^[^[:alnum:]]' -e '^$@$$' \
		| xargs -I _ sh -c 'printf "%-20s " _; make _ -nB 2>/dev/null | (grep -i "^# Help:" || echo "") \
		| if [ -z "$$submakes" ]; then	\
			tail -1;		\
		else				\
			 head -1;		\
		fi				\
		| sed "s/^# Help: //g"' | uniq


#################################################################################################################
#
# This provides common support for all Makefiles in the project. 
# It enables the processing of  a common set of rules on all sub-projects underneath this directory.  
# Currently, the common/standardized set of rules are as follows and supported by makefile.include
#
# clean:
# setup: 
# build:
# test:
#
# When finally getting to a makefile that requires a rule implementation, for example to test the build,
# that makefile should override/implement the rule to meet its needs.  Such a rule may continue to recurse
# using "$(MAKE) <rule>-recurse", for example "$(MAKE) test-recurse". 
#
# Each rule is called recursively on sub-directories and if a similar inclusion is done in the sub-Makefiles,
# the rules will be applied/executed recursively in their sub-directories. By default, sub-directories
# are traversed in lexigraphic order.  If you need to control the ordering, create a .make.subdirs
# listing the sub-directories to process and their order.

# Rule to recurse into subdirectories.  
# Usage: $(MAKE) RULE=yourrule .recurse
.PHONY: .recurse
.recurse:: 
	@if [ -z "$(RULE)" ]; then		\
		echo RULE must be specified to use the .recurse target;	\
		exit 1;				\
	fi
	@if [ -e .make.subdirs ]; then		\
	    SUB_MAKE_DIRS=$$(cat .make.subdirs | sed -e 's/^#.*//' -e 's/ //g') ;	\
	    if [ -z "$$SUB_MAKE_DIRS" ]; then	\
		SUB_MAKE_DIRS=$$(echo */ | sort);	\
	    fi;					\
	else					\
	    SUB_MAKE_DIRS=$$(echo */ | sort);	\
	fi;					\
	echo SUB_MAKE_DIRS=$$SUB_MAKE_DIRS;	\
	if [ ! -z "$$SUB_MAKE_DIRS" ]; then	\
	    for i in $$SUB_MAKE_DIRS; do	\
		if [ -e $$i/Makefile ]; then	\
		    echo Using recursive $(RULE) rule in $$i;	\
		    (cd $$i; $(MAKE) $(RULE));	\
		    status=$$?;			\
		    if [ $$status -ne 0 ]; then	\
			exit $$status;		\
		    fi;				\
		else				\
		    echo No Makefile found in $$i. Skipping.;	\
		fi;				\
	    done;				\
	fi

PWD=$(shell pwd)
# This rule allows sub-directories to override/define rule X for each X-recurse rule above.
# See https://stackoverflow.com/questions/11958626/make-file-warning-overriding-commands-for-target
%:: .%-recurse
	@echo Finished executing rule $@ recursively in $(PWD) > /dev/null	

#######################################################################################

check_defined = \
	$(strip $(foreach 1,$1, \
	$(call __check_defined,$1,$(strip $(value 2)))))
__check_defined = \
	$(if $(value $1),, \
	$(error Undefined $1$(if $2, ($2))))

.PHONY: .check_python_version
.check_python_version::
	@version=$$(echo $(PYTHON_VERSION) | sed -e 's/Python[ ]*//');  \
	major=$$(echo $$version | awk -F. '{print $$1}');	       \
	minor=$$(echo $$version | awk -F. '{print $$2}');	       \
	if [ $$major -lt 3 -o $$minor -lt 10 -o $$minor -gt 11 ]; then  \
		echo Python 3.10 or 3.11 is required, but got $$version;	\
		echo Try overriding PYTHON=$(PYTHON).  For example, "'"make PYTHON=python3.10" ...'";   \
		exit 1;						 \
	fi


.PHONY: .defaults.clean
.defaults.clean:: 
	@# Help: Clean up the virtual environment.
	rm -rf venv

.PHONY: .defaults.image
.defaults.image:: # Must be called with a DOCKER_IMAGE= settings.
	@# Help: Create the docker image $(DOCKER_IMAGE)
	$(DOCKER) build -t $(DOCKER_IMAGE) 					\
		--build-arg EXTRA_INDEX_URL=$(EXTRA_INDEX_URL)	\
		--build-arg BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ')  	\
		--build-arg GIT_COMMIT=$(shell git log -1 --format=%h)  . 

# Copy a source tree in LIB_PATH, including src, pyproject.toml to LIB_NAME
# Generally used to copy source from within the repo into a local directory for use by a Dockerfile
.PHONY: .defaults.copy-lib
.defaults.copy-lib:
	rm -rf ${LIB_NAME}
	mkdir ${LIB_NAME}
	cp -p -R ${LIB_PATH}/src ${LIB_NAME}
	cp -p -R ${LIB_PATH}/pyproject.toml ${LIB_NAME}
	cp -p -R ${LIB_PATH}/README.md ${LIB_NAME}

# Build and image using the local Dockerfile and make the data-processing-lib 
# available in the current directory for use by the Dockerfile (i.e. to install the library). 
.PHONY: .defaults.ray-lib-src-image
.defaults.ray-lib-src-image:: # Must be called with a DOCKER_IMAGE= settings.
	@# Help: Build the $(DOCKER_IMAGE) using the $(DOCKER_FILE), requirements.txt and install data-processing-lib source
	$(MAKE) LIB_PATH=$(DPK_RAY_LIB_DIR) LIB_NAME=data-processing-lib .defaults.copy-lib
	$(MAKE) DOCKER_IMAGE=$(DOCKER_IMAGE) .defaults.image
	rm -rf data-processing-lib


.PHONY: .defaults.ray-lib-src-venv
.defaults.ray-lib-src-venv:: .defaults.venv
	@# Help: Install the source from the data processing library for $(PYTHON)
	@echo Installing source from data processing library for venv
	@source venv/bin/activate;				      	\
	pip install pytest;						\
	pip uninstall -y data-prep-toolkit;		 		\
	if [ ! -z "$(EXTRA_INDEX_URL)" ]; then				\
		extra_url='--extra-index-url $(EXTRA_INDEX_URL)';	\
	fi;								\
	pip install $${extra_url}  -e $(DPK_RAY_LIB_DIR);			\
	if [ $$? -eq 0 ]; then  					\
		echo Installed source from data processing library for `which $(PYTHON)`;       \
	else 								\
		echo ERROR installing source into `which $(PYTHON)`;    \
	fi  

.PHONY: .defaults.test-src
.defaults.test-src:: 
	@# Help: Run pytest on the test directory inside the venv 
	source venv/bin/activate;       \
	export PYTHONPATH=../src;       \
	cd test; $(PYTEST) .

# This is small convenience and the image itself must already be created.
.PHONY: .defaults.test-image-pytest
.defaults.test-image-pytest:: 
	# Put this 2nd so its help showss up instead of .defaults.image help
	@# Help: Test an $(DOCKER_IMAGE) use test source inside the image. 
	$(DOCKER) run -t --rm $(DOCKER_IMAGE) pytest -s test 
	
.PHONY: .defaults.test-locals
.defaults.test-locals::
	@# Help: Run the *local*.py files in the src directory
	@source venv/bin/activate;              \
	export PYTHONPATH=src;                  \
	files=$$(find src -name '*local*.py' );         \
	for i in $$files; do                    \
	    echo Executing: python $$i;         \
	    python $$i;                         \
	    if [ $$? -ne 0 ];   then            \
		exit 1;                         \
	    fi;                                 \
	done   

# Expects RUN_FILE RUN_ARGS
.PHONY: .defaults.run-src-file
.defaults.run-src-file:
	@# Help: Run $(RUN_FILE) (if it exists).
	@if [ ! -e "src/$(RUN_FILE)" ];	then	\
	    echo "";					\
	    echo src/$(RUN_FILE) does not exist.;	\
	    echo "";					\
	    exit 1;					\
	fi
	@if [ ! -d "venv" ];	then			\
	    echo "";					\
	    echo "Please 'make venv' before running $(RUN_FILE).";		\
	    echo "";					\
	    exit 1;					\
	fi
	source venv/bin/activate;	\
	cd src;				\
	python $(RUN_FILE) $(RUN_ARGS)

# This expects the image to already be built and so does not depending on .defaults.publish.
# This allows others to define their own image building prior to publishing.
.PHONY: .defaults.publish
.defaults.publish:: 
	@# Help: Publish the $(DOCKER_IMAGE) to $(DOCKER_HOSTNAME) container registry
	-$(DOCKER) logout $(DOCKER_HOSTNAME)
	$(DOCKER) login $(DOCKER_HOSTNAME) -u '$(DOCKER_REGISTRY_USER)' -p '$(DOCKER_REGISTRY_KEY)'
	$(DOCKER) push  $(DOCKER_IMAGE)

# Create the local virtual environment, assuming python is already installed and available
# We upgrade pip as that seems to be required by watson_nlp
# We install wheel, because it seems to be required for fasttext install on redhat.
# We use "pip" instead of "$(PIP)" below because otherwise if the user has overriddent PYTHON
# they will end up installing into that PYTHON and NOT the venv.
.PHONY: .defaults.venv
.defaults.venv::	requirements.txt .check_python_version
	@# Help: Create the virtual environment using requirements.txt
	$(PYTHON) -m venv venv
	@source venv/bin/activate;		\
	pip install --upgrade pip;		\
	pip install wheel;			\
	if [ ! -z "$(EXTRA_INDEX_URL)" ]; then				\
		extra_url='--extra-index-url $(EXTRA_INDEX_URL)';	\
	fi;								\
	pip install $$extra_url -r requirements.txt;	

.PHONY: .defaults.check.installed
.defaults.check.installed::
	@if [ ! command -v $(CHECK_RUNNABLE) &>/dev/null ]; then \
	    echo $(CHECK_RUNNABLE) must be installed;		\
	    exit 1;						\
	fi

.PHONY: .defaults.minio.check
.defaults.minio.check::
	@# Help: Check that minio and the mc CLI is installed. 
	$(MAKE) CHECK_RUNNABLE=minio .defaults.check.installed
	$(MAKE) CHECK_RUNNABLE=mc .defaults.check.installed

MINIO_DIR=/tmp/data-prep-kit
MINIO_ALIAS=local
# These are the credentials used by samples.
MINIO_ADMIN_USER=localminioaccesskey
MINIO_ADMIN_PWD= localminiosecretkey

.PHONY: .defaults.minio.verify-running
.defaults.minio.verify-running:
	@z=$$(ps -elf | grep minio | grep -v grep);	\
	if [ -z "$$z" ]; then				\
	    echo "";					\
	    echo "Please start minio (see make help)";	\
	    echo "";					\
	    exit 1;					\
	fi

.PHONY: .defaults.minio.stop
.defaults.minio.stop::
	$(MAKE) .defaults.minio.check
	@# Help: Stop the minio server used for S3-based samples
	-mc admin service stop $(MINIO_ALIAS) > /dev/null 2>&1
	-mc alias rm $(MINIO_ALIAS)  > /dev/null 2>&1

.PHONY: .defaults.minio.start
.defaults.minio.start::
	$(MAKE) .defaults.minio.stop
	@# Help: Stop, then start, the minio server used for S3-based samples
	rm -rf $(MINIO_DIR) 
	mkdir  $(MINIO_DIR) 
	echo Starting minio server			
	minio server $(MINIO_DIR) &
	sleep 1
	-mc alias rm $(MINIO_ALIAS)  > /dev/null 2>&1
	mc alias set $(MINIO_ALIAS) http://127.0.0.1:9000 minioadmin minioadmin
	mc admin user svcacct add --access-key "$(MINIO_ADMIN_USER)" --secret-key "$(MINIO_ADMIN_PWD)" $(MINIO_ALIAS) minioadmin
	mc alias ls $(MINIO_ALIAS)
	@echo "Test minio server started" 

.PHONY: .defaults.minio.load-test-data
.defaults.minio.load-test-data::
	@# Help: Load the minio server with S3 sample data from $(MINIO_SRC) to $(MINIO_DEST) 
	@if [ -z "$(MINIO_SRC)" -o -z "$(MINIO_DEST)" ]; then	\
		echo MINIO_SRC and MINIO_DEST must be defined when use $? target;\
		exit 1;						\
	fi
	$(MAKE) .defaults.minio.check
	@# Help: Load $(MINIO_SRC) into minio at local/test/$(MINIO_DEST) 
	-mc mb  local/test > /dev/null 2>&1	# Igore if it already exists
	mc cp --recursive $(MINIO_SRC) local/test/$(MINIO_DEST)